def create_weather_dataset(city, since, until):
dataframes = (retrieve_city_weather(city=obj, since=since, until=until)
for obj in city)
dataset = pd.concat(dataframes, axis=0, ignore_index=False)
util.notify('Saving global dataframe...', 'cyan')
dataset.to_csv('test_weather.csv')
def retrieve_city_weather(city, since, until):
# Make sure the city exists
try:
util.notify('Querying {city} in database...'.format(city=city), 'cyan')
CITY = next(srv.get_cities(slug=city, serialized=False))
except CityNotFound as exc:
util.notify(exc, 'red')
sys.exit()
df = CITY.get_weather_updates(since, until)
df['city'] = [CITY.slug] * len(df.index)
return df
def get_city_stations(city):
click.secho('city: {}'.format(city), fg='yellow', bold=True)
util.notify('Querying {city} in database...'.format(city=city), 'cyan')
# Make sure the city exists
try:
CITY = next(srv.get_cities(name=city, serialized=False))
except CityNotFound as exc:
util.notify(exc, 'red')
sys.exit()
# Make sure the city is predictable
if not CITY.predictable:
util.notify(CityUnpredicable, 'red')
sys.exit()
util.notify('Querying {city} stations in database...'.format(city=city),
'cyan')
# Search city stations
return srv.get_stations(city_slug=CITY.slug, serialized=False)
def create_dataset_challenge(place, moment, blank):
dataframes = (init_station_dataframe(city=obj[0],
station=obj[1],
moment=moment,
blank=blank) for obj in place)
dataset = pd.concat(dataframes, axis=0, ignore_index=False)
print(dataset)
try:
util.notify('Saving global dataframe...', 'cyan')
dataset.to_csv('test-blank.csv' if blank else 'test-filled.csv')
except Exception as exc:
print(exc)
util.notify(exc, 'red')
sys.exit()
util.notify('Done!', 'green')
def init_station_dataframe(city, station, moment, blank):
click.secho('city: {}'.format(city), fg='yellow', bold=True)
click.secho('station: {}'.format(station), fg='yellow', bold=True)
click.secho('moment: {}'.format(moment), fg='yellow')
click.secho('blank: {}'.format(blank), fg='blue', bold=True)
SINCE = moment
FILTERED_DATES = [SINCE + dt.timedelta(**moment) for moment in MOMENTS]
UNTIL = FILTERED_DATES[-1]
util.notify('Querying {city} in database...'.format(city=city), 'cyan')
# Make sure the city exists
try:
CITY = next(srv.get_cities(slug=city, serialized=False))
except CityNotFound as exc:
util.notify(exc, 'red')
sys.exit()
# Make sure the city is predictable
if not CITY.predictable:
util.notify(CityUnpredicable, 'red')
sys.exit()
util.notify('Querying {city} stations in database...'.format(city=city),
'cyan')
# Search city station
STATION = next(
srv.get_stations(slug=station, city_slug=CITY.slug, serialized=False))
# Get bike updates
if not blank:
util.notify('Getting station updates...', 'cyan')
try:
df = STATION.get_updates(SINCE, UNTIL)
except Exception as exc:
util.notify(exc, 'red')
sys.exit()
util.notify('Generating dataframe...', 'cyan')
# Create dataframe
columns = dict(bikes=[
find_tuple_according_moment(df, moment).bikes
for moment in FILTERED_DATES
] if not blank else [''] * len(FILTERED_DATES),
station=[station] * len(FILTERED_DATES),
city=[city] * len(FILTERED_DATES))
index = FILTERED_DATES
column_order = ['city', 'station', 'bikes']
return pd.DataFrame(columns, index)[column_order]
from app import services as srv
from app.exceptions import CityNotFound, CityUnpredicable
from scripts import util
if __name__ == '__main__':
PARSER = argparse.ArgumentParser()
PARSER.add_argument('city', type=str, help='City for which to import data')
PARSER.add_argument('since', type=str, help='Get the data from this date')
PARSER.add_argument('until', type=str, help='Get the data until this date')
PARAMS = PARSER.parse_args()
# Make sure the city exists
try:
CITY = next(srv.get_cities(name=PARAMS.city, serialized=False))
except CityNotFound as exc:
util.notify(exc, 'red')
sys.exit()
# Make sure the city is predictable
if not CITY.predictable:
util.notify(CityUnpredicable, 'red')
sys.exit()
# Parse the dates
SINCE = dt.datetime.strptime(PARAMS.since, '%Y/%m/%d-%H:%M:%S')
UNTIL = dt.datetime.strptime(PARAMS.until, '%Y/%m/%d-%H:%M:%S')
# Create the necessary folders if they don't exist
if not os.path.exists(CITY.slug):
os.makedirs(CITY.slug)
if not os.path.exists(os.path.join(CITY.slug, 'stations/')):