def plot_results_histogram(self, results_data_category):
self._add_default_histogram_graph_settings()
results_data = self._determine_results_data(results_data_category)
labels = self._determine_plot_labels(results_data_category)
self._queue_histogram_plots(results_data, labels)
PlotOperations.show_plots()
def generate_lineplot(self):
"""Fetches data from the db, formats it, and then displays that data
in a lineplot"""
month_index = self.get_month_index(self.line_month.get())
formatted_month = self.get_formatted_month(month_index)
year = self.year_entry.get()
try:
year = int(year)
if year <= 0:
raise ValueError()
else:
self.line_plot_error['text'] = ' '
data = self.db_ops.fetch_data(
start_date=(str(year) + "-" + formatted_month + "-01"),
end_date=(str(year) + "-" + formatted_month + "-31"))
boxplot_data = self.format_data_for_lineplot(data)
# passing empty data as this data is not used in generating line-plot
plot_ops = PlotOperations(data={})
print(boxplot_data)
plot_ops.show_lineplot(boxplot_data, month_index, year)
except Exception as e:
if self.line_plot_error['text'] != ' ':
self.line_plot_error[
'text'] = 'Please enter valid Month and Year values!'
print("ERROR :", str(e))
def lineplot(self, event):
" Generate and save lineplot "
year = self.year_text_ctrl.GetValue()
month = self.month_text_ctrl.GetValue()
db_name = 'weather.sqlite'
table_name = 'weather'
my_plot_operations = PlotOperations(db_name, table_name)
my_plot_operations.generate_lineplot(int(year), int(month))
def boxplot(self, event):
" Generate and save boxplot "
start_year = self.start_year_text_ctrl.GetValue()
end_year = self.end_year_text_ctrl.GetValue()
db_name = 'weather.sqlite'
table_name = 'weather'
my_plot_operations = PlotOperations(db_name, table_name)
my_plot_operations.generate_boxplot(int(start_year), int(end_year))
def _queue_barplots(self, results_data, labels):
if (len(results_data) > 1):
bar_width = 1.0 / len(results_data)
else:
bar_width = 0.8
for (index, result_data) in enumerate(results_data):
offset = bar_width * index
PlotOperations.queue_plot(
Barplot(x=np.arange(offset, len(result_data), step=1),
height=result_data,
width=bar_width,
label=labels[index]))
def __init__(self):
""" Initialize classes and variables for db operations, web scraping, plot operations and UI configuration """
try:
self.db = DBOperations("weather.sqlite")
self.ws = WeatherScraper()
self.pl = PlotOperations()
self.last_updated = self.db.fetch_last(
)[0]["sample_date"] if self.db.is_table_exist() else ""
self.first_updated = self.db.fetch_first(
)[0]["sample_date"] if self.db.is_table_exist() else ""
except Exception as e:
logging.error(f"weatherprocessor:__init__, {e}")
def save_results_boxplot(self, results_data_category, filename):
PlotOperations.initialize_figure()
self._add_default_boxplot_graph_settings()
results_data = self._determine_results_data(results_data_category)
labels = self._determine_plot_labels(results_data_category)
filename_with_extension = FileOperations.apply_extension_to_filename(
original_filename=filename, file_extension='.png')
self._queue_boxplots(results_data, labels)
FileOperations.save_file_with_fallback(
save_method_or_function=PlotOperations.save_plots,
filename=filename_with_extension,
fallback_filename=FileOperations.apply_extension_to_filename(
original_filename=DefaultFilenames.PLOT_FALLBACK,
file_extension='.png'))
def generate_line_plot(self, specific_year: int,
specific_month: int) -> None:
"""
Generate a line plot for a month.
:param specific_year:
:param specific_month:
:return:
"""
try:
month_data = self.my_db.fetch_data(specific_year, specific_month)
if not month_data:
print(
'Warning: there is no data of [{0}-{1}] in the database. Please update first.'
.format(specific_year, specific_month))
else:
my_plot = PlotOperations()
my_plot.generate_line_plot(specific_year, specific_month)
except Exception as e:
self.logger.error(e)
def create_plot(self, event):
"""This method create a plot with range of year"""
start_year = eval(self.m_textCtrl2.GetValue())
end_year = eval(self.m_textCtrl1.GetValue())
plot = PlotOperations(start_year, end_year)
plot.create_weather_data()
plot.create_plot(plot.weather_data)
def generate_box_plot(self, start_year: int, end_year: int) -> None:
"""
Generate a box plot for a year range.
:param start_year:
:param end_year:
:return:
"""
try:
start_year_data = self.my_db.fetch_data(start_year)
end_year_data = self.my_db.fetch_data(end_year)
if not start_year_data:
print(
'Warning: there is no data of year[{0}] in the database. Please update first.'
.format(start_year))
elif not end_year_data:
print(
'Warning: there is no data of year[{0}] in the database. Please update first.'
.format(end_year_data))
else:
my_plot = PlotOperations()
my_plot.generate_box_plot(start_year, end_year)
except Exception as e:
self.logger.error(e)
def plot_selection():
"""This method is to create weather plot"""
print(
'Please enter year range of interest '
'(from year, to year example: 2000,2017)')
x = input().split(",")
test = PlotOperations(int(x[0]), int(x[1]))
test.create_weather_data()
test.create_plot(test.weather_data)
def generate_boxplot(self):
"""Uses the data in the db to generate the requested boxplot"""
start_year = self.start_year_entry.get()
end_year = self.end_year_entry.get()
try:
start_year = int(start_year)
end_year = int(end_year)
if (start_year <= 0 or end_year <= 0):
raise ValueError()
elif start_year > end_year:
self.box_plot_error[
'text'] = 'ERROR: Start Year can not be greater than end Year!'
else:
self.box_plot_error['text'] = ' '
data = self.db_ops.fetch_data(
start_date=(str(start_year) + "-01-01"),
end_date=(str(end_year) + "-12-31"))
boxplot_data = self.format_data_for_boxplot(data)
plot_ops = PlotOperations(data=boxplot_data)
plot_ops.show_boxplot()
except Exception as e:
if self.box_plot_error['text'] != ' ':
self.box_plot_error['text'] = 'Please enter valid Year values!'
print("ERROR :", str(e))
def OnClickedLineGraph(self, event):
plot = PlotOperations()
plot.linegraph(self.txtYear.GetValue(), self.txtMonth.GetValue())
def OnClickedBoxPlot(self, event):
plot = PlotOperations()
plot.boxplot(self.txtYearOne.GetValue(), self.txtYearTwo.GetValue())
def main(self):
"""
When the program starts, prompt the user to download a full set of
weather data, or to update it (optional).
• Then prompt the user for a year range of interest (from year, to year).
• Use this class to launch and manage all the other tasks.
"""
user_selection = ''
while user_selection != '4':
try:
print("1. Update a set of weather data up to today")
print("2. Download a full set of weather data")
print("3. A year range of interest (from year, to year)")
print("4. Exit")
user_selection = input("Please make your choice...")
if user_selection == '1':
try:
my_scraper = WeatherScraper()
now_date = datetime.datetime.now()
is_loop = False
for i in range(now_date.year, now_date.year - 1, -1):
my_scraper.url_year = i
if is_loop:
break
for j in range(now_date.month - 2,
now_date.month + 1):
my_scraper.url_month = j
my_url = f"https://climate.weather.gc.ca/climate_data/daily_data_e.html?%20StationID=27174&timeframe=2&StartYear=1840&EndYear=2018&Day=%201&Year={my_scraper.url_year}&Month={my_scraper.url_month}#"
with urllib.request.urlopen(
my_url) as response:
html = str(response.read())
my_scraper.feed(html)
if my_scraper.is_equal is False:
is_loop = True
break
# print(f"inner{my_scraper.dict_Inner}")
# print(f"outer{my_scraper.dict_outer}")
my_database = DBOperations()
my_database.create_table(my_scraper.dict_outer)
except Exception as e:
print(
"Error in Updating a set of weather data up to today: ",
e)
elif user_selection == '2':
try:
my_scraper = WeatherScraper()
now_date = datetime.datetime.now()
is_loop = False
for i in reversed(range(now_date.year)):
my_scraper.url_year = i
if is_loop:
break
for j in range(0, 13):
my_scraper.url_month = j
my_url = f"https://climate.weather.gc.ca/climate_data/daily_data_e.html?%20StationID=27174&timeframe=2&StartYear=1840&EndYear=2018&Day=%201&Year={my_scraper.url_year}&Month={my_scraper.url_month}#"
with urllib.request.urlopen(
my_url) as response:
html = str(response.read())
my_scraper.feed(html)
if my_scraper.is_equal is False:
is_loop = True
break
# print(f"inner{my_scraper.dict_Inner}")
# print(f"outer{my_scraper.dict_outer}")
my_database = DBOperations()
my_database.create_table(my_scraper.dict_outer)
except Exception as e:
print(
"Error in downloading a full set of weather data: ",
e)
elif user_selection == '3':
try:
range_value = input(
"Please select a RANGE of your interest(e.g 2017 2019): "
)
range_value = range_value.split()
my_database = DBOperations()
dict_value = my_database.query_infos(
range_value[0], range_value[1])
my_plot_operation = PlotOperations()
my_plot_operation.diplay_box_plot(
dict_value, range_value[0], range_value[1])
except Exception as e:
print(
"Error in A year range of interest (from year, to year): ",
e)
elif user_selection == '4':
break
else:
print("Invalid choice")
except Exception as e:
print("Error plot_operations.py: ", e)
def _queue_histogram_plots(self, results_data, labels):
PlotOperations.queue_plot(
Histogram(x=results_data,
bins=self._default_histogram_bins(),
label=labels,
rwidth=0.8))
def monthly_plot(self, event):
"""This method create a monthly plot"""
year = eval(self.m_textCtrl3.GetValue())
month = eval(self.m_textCtrl4.GetValue())
plot = PlotOperations()
plot.create_day_plot(year, month)
def _add_default_histogram_graph_settings(self):
PlotOperations.set_title(self.title_prefix +
' - Histograma de acurácia')
PlotOperations.set_xlabel('Acurácia')
PlotOperations.set_ylabel('Número de particionamentos')
PlotOperations.set_xticks(self._default_histogram_bins())
def _add_default_boxplot_graph_settings(self):
PlotOperations.set_title(self.title_prefix +
' - Gráfico de caixa de acurácia')
PlotOperations.set_xlabel('Dados')
PlotOperations.set_ylabel('Acurácia')
def _add_default_barplot_graph_settings(self):
PlotOperations.set_title(self.title_prefix +
' - Acurácia por particionamento')
PlotOperations.set_xlabel('Número do particionamento')
PlotOperations.set_ylabel('Acurácia')
PlotOperations.set_xticks(np.arange(0, 101, step=5))
class WeatherProcessor:
"""
This class manages the user interaction to generate plots and update the data.
"""
def __init__(self):
""" Initialize classes and variables for db operations, web scraping, plot operations and UI configuration """
try:
self.db = DBOperations("weather.sqlite")
self.ws = WeatherScraper()
self.pl = PlotOperations()
self.last_updated = self.db.fetch_last(
)[0]["sample_date"] if self.db.is_table_exist() else ""
self.first_updated = self.db.fetch_first(
)[0]["sample_date"] if self.db.is_table_exist() else ""
except Exception as e:
logging.error(f"weatherprocessor:__init__, {e}")
def download_data(self):
""" Clears the database, reinitializes it, then downloads all the data to it. """
try:
self.db.purge_data()
self.db.initialize_db()
self.collect_data()
except Exception as e:
logging.error(f"weatherprocessor:download_data, {e}")
def update_data(self):
""" Ensures the database exists then downloads all
the data up to the most recent date in the database. """
try:
self.db.initialize_db()
self.collect_data()
self.last_updated = self.db.fetch_last()[0]["sample_date"]
except Exception as e:
logging.error(f"weatherprocessor:update_data, {e}")
def get_box_plot(self, start_year, end_year):
""" Fetches data within the users inputted range then
generates a box plot for the mean temperatures of each month. """
try:
weather = self.db.fetch_data(start_year, int(end_year) + 1, False)
self.pl.generate_box_plot(weather, start_year, end_year)
except Exception as e:
logging.error(f"weatherprocessor:get_box_plot, {e}")
def get_line_plot(self, year, month):
""" User inputs the month and year of the data to be fetched
then generates a line plot for the daily mean temperatures of that month. """
try:
weather = self.db.fetch_data(year, month, True)
self.pl.generate_line_plot(weather, year, month)
except Exception as e:
logging.error(f"weatherprocessor:get_line_plot, {e}")
def collect_data(self):
""" This method collects the data by looping through and prepping for save,
Get the current date and break it down into variables,
Query db for the latest recorded data by date,
Call the scraper class to collect necessary data,
Stop collecting after duplicates are found. """
try:
today = date.today()
year = int(today.strftime("%Y"))
month = int(today.strftime("%m"))
duplicate_month, duplicate_day = False, False
recent_date = ""
dates = self.db.fetch_last()
if len(dates) > 0:
recent_date = dates[0]["sample_date"]
while not duplicate_month and not duplicate_day:
""" Iterates through each year starting with the
latest and working backwards until duplicate data is found. """
try:
month_dict = dict()
while not duplicate_day and month > 0:
""" Iterate through each month starting with the latest
and working backwards until duplicate data is found. """
try:
url = self.ws.get_url(year, month)
with urllib.request.urlopen(url) as response:
html = str(response.read())
self.ws.feed(html)
month_dict[month] = self.ws.return_dict()
if month + 1 in month_dict.keys(
) and month_dict[month] == month_dict[month + 1]:
"""Checks if month is the same as the prior month. Used for download_data """
month_dict.popitem()
duplicate_month = True
break
if recent_date != "":
temp_dict = {}
for key, value in reversed(
month_dict[month].items()):
"""Iterates through each months data enusring there is not a duplicate in the database."""
try:
check_date = f"{year}-{month:02d}-{key}"
if check_date == recent_date:
duplicate_day = True
break
temp_dict[key] = value
except Exception as e:
logging.error(
f"weatherprocessor:collect_data:loop:loop2:loop3, {e}"
)
month_dict[month] = temp_dict
self.db.save_data(month_dict[month], month, year)
month -= 1
except Exception as e:
logging.error(
f"weatherprocessor:collect_data:loop:loop2, {e}"
)
pub.sendMessage('update_latest_download', year=str(year))
month = 12
year -= 1
except Exception as e:
logging.error(f"weatherprocessor:collect_data:loop, {e}")
except Exception as e:
logging.error(f"weatherprocessor:collect_data, {e}")
def get_years_for_dropdown(self, min_year):
"""Retrieves the years for the combo boxes based on a given min_year."""
try:
years = []
if self.db.is_table_exist():
self.last_updated = self.db.fetch_last(
)[0]["sample_date"] if self.db.is_table_exist() else ""
self.first_updated = self.db.fetch_first(
)[0]["sample_date"] if self.db.is_table_exist() else ""
if min_year == "":
firstyear = int(self.first_updated[:4])
else:
firstyear = int(min_year)
lastyear = int(self.last_updated[:4])
while firstyear <= lastyear:
"""Starting from the first year add each year to the years list."""
try:
years.append(str(firstyear))
firstyear += 1
except Exception as e:
logging.error(
f"weatherprocessor:get_years_for_dropdown:loop, {e}"
)
return years
except Exception as e:
logging.error(f"weatherprocessor:get_years_for_dropdown, {e}")
def get_months_for_dropdown(self, year):
"""Retrieves the months for the month combo box based on the selected year."""
try:
months = []
if self.db.is_table_exist():
self.first_updated = self.db.fetch_first(
)[0]["sample_date"] if self.db.is_table_exist() else ""
if year == "":
year = int(self.first_updated[:4])
data = self.db.fetch_months(year)
for item in data:
"""Goes through the list of returned data"""
try:
for value in item.values():
"""Adds each month to a list of months."""
try:
months.append(str(value[-2:]))
except Exception as e:
logging.error(
f"weatherprocessor:get_months_for_dropdown:loop:loop2, {e}"
)
except Exception as e:
logging.error(
f"weatherprocessor:get_months_for_dropdown:loop, {e}"
)
return months[::-1]
except Exception as e:
logging.error(f"weatherprocessor:get_months_for_dropdown, {e}")
def setUp(self):
self.myplot = PlotOperations()
def _queue_boxplots(self, results_data, labels):
PlotOperations.queue_plot(
Boxplot(x=results_data,
positions=range(len(results_data)),
labels=labels))