def main():
captor = AqiCaptor(logger)
file_manager = FileManager()
try:
try:
locations = file_manager.get_locations(source)
except IOError:
raise InitError("File %s is missing"% file_manager.get_locations_path(source))
except ValueError:
raise InitError("The %s file does not contain any correct JSON object"% file_manager.get_locations_path(source))
for city, location in locations.iteritems():
logger.warning("- Capting for %s"% city)
my_date = datetime.now().strftime('%y-%m-%d-%H')
loaded_json = captor.get_data(location)
try:
file_manager.write_json(loaded_json, source, city, my_date)
except IOError:
raise InitError("Folder %s is missing"% file_manager.get_folder_path(source, city))
return 0
except InitError as e:
logger.critical("%s: %s"% (type(e).__name__, e))
return 1
except requests.exceptions.RequestException as e:
logger.critical("%s: %s"% (type(e).__name__, e))
return 2
except Exception as e:
logger.critical(e, exc_info=True)
return 3
def main():
captor = YahooWeatherCaptor()
file_manager = FileManager()
try:
try:
woe_ids = file_manager.get_locations(source)
except IOError:
raise InitError("File %s is missing"% file_manager.get_locations_path(source))
except ValueError:
raise InitError("The %s file does not contain any correct JSON object"% file_manager.get_locations_path(source))
for city, woe_id in woe_ids.iteritems():
logger.warning("Capting for %s" % city)
loaded_json = captor.get_data(woe_id)
last_build_date = dateutil.parser.parse(loaded_json['lastBuildDate'])
date_for_path = last_build_date.strftime('%y-%m-%d-%H-%M')
if os.path.isfile(file_manager.get_path(source, city, date_for_path)):
logger.warning("Same build date for %s (%s)" % (city, last_build_date))
else:
try:
file_manager.write_json(loaded_json, source, city, date_for_path)
except IOError:
raise InitError("Folder %s is missing"% file_manager.get_folder_path(source, city))
logger.warning("New JSON written for %s %s" % (city, last_build_date))
return 0
except InitError as e:
logger.critical("%s: %s"% (type(e).__name__, e))
return 1
except requests.exceptions.RequestException as e:
logger.critical("%s: %s"% (type(e).__name__, e))
return 2
except Exception as e:
logger.critical(e, exc_info=True)
return 3
def main():
return_value = 0
captor = YelpCaptor(logger)
file_manager = FileManager()
db_conn = None
try:
db_conn = sqlite3.connect(file_manager.get_path(source, None, yelp_db, ".db"))
db_cursor = db_conn.cursor()
db_manager = YelpDatabaseManager(logger)
try:
cities = file_manager.get_locations(source)
except IOError:
raise InitError("File %s is missing"% file_manager.get_locations_path(source))
except ValueError:
raise InitError("The %s file does not contain any correct JSON object"% file_manager.get_locations_path(source))
my_cities = exit_logger.read_back_file()
if len(my_cities) == 0:
my_cities = cities
remaining_locations = my_cities.copy()
now = datetime.now()
for city, locations in my_cities.iteritems():
nb_total_locations = len(cities[city])
logger.warning("Capting for %s" % city)
for location in locations:
loaded_json = captor.get_data(location)
if len(loaded_json) > 0:
db_manager.insert_venues(db_cursor, loaded_json)
db_conn.commit()
remaining_locations[city].remove(location)
nb_locations = nb_total_locations - len(remaining_locations[city])
logger.warning("%d location(s) done over %d locations for %s"% (nb_locations, nb_total_locations, city))
venues = db_manager.get_venues(db_cursor)
nb_venues = len(venues)
if nb_venues > 0:
try:
file_manager.write_json(venues, source, city, now.strftime('%y-%m-%d'))
except IOError:
raise InitError("Folder %s is missing"% file_manager.get_folder_path(source, city))
logger.warning("New JSON written for %s: %d venues"% (city, nb_venues))
db_manager.delete_all_venues(db_cursor)
db_conn.commit()
remaining_locations.pop(city)
exit_logger.write_back_file(None)
except InitError as e:
logger.critical("%s: %s"% (type(e).__name__, e))
return_value = 1
except requests.exceptions.RequestException as e:
logger.critical("%s: %s"% (type(e).__name__, e))
exit_logger.write_back_file(remaining_locations)
return_value = 2
except YelpCaptor.YelpApiError as e:
logger.critical("%s: %s"% (type(e).__name__, e))
exit_logger.write_back_file(remaining_locations)
return_value = 3
except sqlite3.OperationalError as e:
logger.critical("SQLite3 %s: %s"% (type(e).__name__, e))
return_value = 4
except Exception as e:
logger.critical(e, exc_info=True)
exit_logger.write_back_file(remaining_locations)
return_value = 5
finally:
if db_conn<> None:
db_conn.close()
return return_value
def main():
last_date = datetime.now() - timedelta(days=2)
captor = FlickrCaptor(logger)
file_manager = FileManager()
try:
try:
woe_ids = file_manager.get_locations(source)
except IOError:
raise InitError("File %s is missing"% file_manager.get_locations_path(source))
except ValueError:
raise InitError("The %s file does not contain any correct JSON object"% file_manager.get_locations_path(source))
min_date_file_path = file_manager.get_path(source, None, min_date_file)
try:
min_date_json = file_manager.read_json(source, min_date_file)
min_date = datetime.strptime(min_date_json['min_date'], min_date_format)
except IOError:
raise InitError("File %s is missing. You should create this file and set {'min_date':YYYY-MM-DD} in it."% min_date_file_path)
except (ValueError, KeyError):
raise InitError("You need to set {\"min_date\":\"YYYY-MM-DD\"} in file %s"% min_date_file_path)
zero_day = timedelta(days=0)
one_day = timedelta(days=1)
if (last_date - min_date) < zero_day:
raise InitError("The date set as min_date in %s is after 2 days ago."% min_date_file_path)
while (last_date - min_date) >= zero_day:
logger.warning("---- Capting for %s" % min_date.strftime('%y-%m-%d'))
for city, woe_id in woe_ids.iteritems():
max_date = min_date + one_day
logger.warning("Capting for %s" % city)
loaded_json = captor.get_data(min_date, max_date, woe_id)
try:
file_manager.write_json(loaded_json, source, city, min_date.strftime('%y-%m-%d'))
except IOError:
raise InitError("Folder %s is missing"% file_manager.get_folder_path(source, city))
logger.warning("New JSON written for %s " % city)
min_date = max_date
f = open(min_date_file_path, 'w+')
json.dump({'min_date':min_date.strftime(min_date_format)}, f)
f.close()
logger.warning("... Sleeping for 5 s")
sleep(5)
return 0
except InitError as e:
logger.critical("%s: %s"% (type(e).__name__, e))
return 1
except requests.exceptions.RequestException as e:
logger.critical("%s: %s"% (type(e).__name__, e))
return 2
except FlickrCaptor.FlickrApiError as e:
logger.critical("%s: %s"% (type(e).__name__, e))
return 3
except Exception as e:
logger.critical(e, exc_info=True)
return 4
