def fill_missing_data(self):
"""
Fetch the gap data from now to the last one in the database and just insert these data.
:return:
"""
try:
last_one_date = self.my_db.fetch_last_one()[0][0]
last_one_year = int(last_one_date[:4])
last_one_month = int(last_one_date[5:7])
year = date.today().year
month = date.today().month
my_scraper = WeatherScraper()
if last_one_year == year and last_one_month == month:
my_scraper.scrape_month_weather(year, month)
while last_one_year != year and last_one_month != month:
my_scraper.scrape_month_weather(year, month)
month -= 1
if month == 0:
year -= 1
month = 12
self.my_db.save_data(my_scraper.weather)
except Exception as e:
self.logger.error(e)
print('Purging all the data from the database... ')
with DBOperations(self.db_name) as cursor:
sql_purge_data_1 = """DELETE FROM samples;"""
sql_purge_data_2 = """DELETE FROM sqlite_sequence WHERE name = 'samples';"""
cursor.execute(sql_purge_data_1)
cursor.execute(sql_purge_data_2)
except Exception as e:
self.logger.error(e)
if __name__ == '__main__':
mydb = DBOperations('weather.sqlite')
mydb.initialize_db()
my_scraper = WeatherScraper()
my_scraper.scrape_month_weather(2020, 12)
my_scraper.scrape_now_to_earliest_month_weather(1998, 5)
mydb.purge_data()
mydb.save_data(my_scraper.weather)
for key, value in my_scraper.weather.items():
print(key + ': ' + str(value))
print('years data')
for item in mydb.fetch_data(1996):
print(item)
print('month data')
for item in mydb.fetch_data(2020, 12):
print(item)