def renew_all_data(self):
"""
Fetch all new data from website and cover the database.
:return:
"""
try:
my_scraper = WeatherScraper()
my_scraper.scrape_now_to_earliest_month_weather()
self.my_db.purge_data()
self.my_db.save_data(my_scraper.weather)
except Exception as e:
self.logger.error(e)
with DBOperations(self.db_name) as cursor:
sql_purge_data_1 = """DELETE FROM samples;"""
sql_purge_data_2 = """DELETE FROM sqlite_sequence WHERE name = 'samples';"""
cursor.execute(sql_purge_data_1)
cursor.execute(sql_purge_data_2)
except Exception as e:
self.logger.error(e)
if __name__ == '__main__':
mydb = DBOperations('weather.sqlite')
mydb.initialize_db()
my_scraper = WeatherScraper()
my_scraper.scrape_month_weather(2020, 12)
my_scraper.scrape_now_to_earliest_month_weather(1998, 5)
mydb.purge_data()
mydb.save_data(my_scraper.weather)
for key, value in my_scraper.weather.items():
print(key + ': ' + str(value))
print('years data')
for item in mydb.fetch_data(1996):
print(item)
print('month data')
for item in mydb.fetch_data(2020, 12):
print(item)
print('the last one in the database')
mkdir_p(output_dir)
file_path = '{0}/{1}'.format(output_dir, file_name)
self.line_plot_path_saving_dict[str(specific_year) + '-' +
str(specific_month)] = file_path
plt.savefig(file_path)
plt.show()
return self.line_plot_path_saving_dict
except Exception as e:
self.logger.error(e)
if __name__ == '__main__':
mydb = DBOperations('weather.sqlite')
mydb.initialize_db()
my_scraper = WeatherScraper()
my_scraper.scrape_now_to_earliest_month_weather(
1998, 5) # For testing, range is 1996-1997
my_scraper.scrape_month_weather(2018, 5)
my_scraper.scrape_month_weather(2020, 12)
mydb.purge_data()
mydb.save_data(my_scraper.weather)
my_plot = PlotOperations()
my_plot.generate_box_plot(1996, 1997)
my_plot.generate_line_plot(2018, 5)
my_plot.generate_line_plot(2020, 12)