def _assemble_location(wd_location: wd.Location, properties: bool, histories: bool, weather_data: wd.WeatherData) -> Location:
dto = Location.from_domain(wd_location)
if properties:
dto.properties = LocationProperties.from_domain(weather_data.history_properties(wd_location))
if histories:
histories = weather_data.history_date_ranges(wd_location)
if histories:
dto.histories = [DateRange.from_domain(dr) for dr in weather_data.history_date_ranges(wd_location)]
return dto
def execute(self, options: argparse.Namespace, weather_data: WeatherData):
location = None
if options.al_cmd == "df":
try:
location = self.location_from_fields(options, weather_data)
except ValueError as err:
log.error(str(err), exc_info=True)
else:
location = self.location_from_city(options, weather_data)
if location:
weather_data.add_location(location)
async def _get_location_hourly_history(
name: str = Path(None, description="Location full name or alias."),
starting: date = Query(None, description='Starting date (YYYY-MM-DD).', alias="from"),
ending: date = Query(None, description='Ending date (YYYY-MM-DD).', alias="thru"),
weather_data: wd.WeatherData = Security(_get_weather_data, scopes=[Permissions.ReadWeatherData])
) -> WeatherHistory:
location = _get_weather_data_location(name, weather_data)
date_range = wd.DateRange(starting, ending)
history_dates = weather_data.history_dates(location, date_range.low, date_range.high)
histories = [HourlyHistory.from_domain(hourly_history)
for hourly_history in weather_data.get_history(location, history_dates, hourly_history=True)]
return WeatherHistory(location=Location.from_domain(location),
date_range=DateRange.from_domain(date_range),
hourly_histories=histories)
def execute(self, args=None):
options = self._parser.parse_args(args)
weather_data = WeatherData(
options.data) if options.data else WeatherData()
if options.debug:
log.getLogger().setLevel(log.DEBUG)
log.debug("%s", options)
if options.cmd:
try:
self._dispatcher[options.cmd](options, weather_data)
except Exception as error:
log.exception("Error executing '%s': %s...",
options.cmd,
str(error),
exc_info=error)
else:
self._parser.print_usage()
def execute(self, options: argparse.Namespace, weather_data: WeatherData):
locations = sorted(get_locations(weather_data, *options.locations), key=lambda l: l.name)
histories: Dict[str, List[DateRange]] = {
location.name: weather_data.history_date_ranges(location) for location in locations
}
if not (options.csv or options.json):
field_widths = {
"lw": max_width(locations, get=lambda l: l.name),
"dw": len("YYYY-MM-DD to YYYY-MM-DD")
}
print("{:-^{lw}} {:-^{dw}}".format("Location", "History Dates", **field_widths))
details = "{:<{lw}} {:<{dw}}"
for location in locations:
weather_ranges = histories.get(location.name)
if not weather_ranges:
print(details.format(location.name, "None.", **field_widths))
else:
for i, (start, end) in enumerate(weather_ranges):
print(details.format("" if i else location.name,
"{} to {}".format(start, end) if start and end else str(start),
**field_widths))
return
DictKeys = namedtuple('DictKeys', ["name", "dates", "start", "end"])
def to_dict(_dict_keys: DictKeys) -> List[Dict]:
return [
{
_dict_keys.name: _location.name,
_dict_keys.dates: [
{
_dict_keys.start: str(_start),
_dict_keys.end: str(_end)
}
for _start, _end in histories.get(_location.name)
]
}
for _location in locations
]
if options.json:
print(json.dumps({"history": to_dict(DictKeys("location", "dates", "start", "end"))}, indent=2))
elif options.csv:
fields = DictKeys("location", "dates", "start_date", "end_date")
with CsvDictWriter([fields.name, fields.start, fields.end]).stdout() as dict_write:
for history_dict in to_dict(fields):
location_dict = {fields.name: history_dict[fields.name]}
dates = history_dict[fields.dates]
if not dates:
dict_write(location_dict)
else:
for history_date in dates:
dict_write({**location_dict, **history_date})
def __init__(self, master, **kwargs):
if os.name == 'nt' and get_bool_setting("gui", "windows_native_theme"):
style = Style()
print("theme names: {}, current theme: {}".format(
style.theme_names(), style.theme_use()))
theme = "winnative"
Style().theme_use(theme)
print("theme set to: {}".format(style.theme_use()))
master.title("Weather Data")
master.columnconfigure(0, weight=1)
master.rowconfigure(0, weight=1)
frame = Frame(master, **kwargs)
frame.grid(sticky=NSEW)
frame.columnconfigure(0, weight=1)
frame.rowconfigure(0, weight=1)
notebook = NotebookWidget(frame)
notebook.grid(row=0, column=0, sticky=NSEW)
status = StatusWidget(frame)
status.grid(row=1, column=0, sticky=(S, E, W))
locations_view = LocationsWidget(frame)
notebook.add_tab(locations_view, "Locations")
domain = WeatherDomain(master, notebook, status, locations_view)
controller = WeatherController(domain)
# domain.root.configure(menu=controller.locations_menu)
notebook.bind_notebook_tab_changed(controller.tab_change_event)
locations_view.on_event("<Button-3>",
lambda e: controller.right_click_action())
locations_view.on_event("<<TreeviewSelect>>",
controller.location_view_select)
data_path = Path(WeatherData.WEATHER_DATA_DIR)
# todo: change so WeatherData is always instantiated
domain.load_locations(
WeatherData(data_path)
if data_path.exists() and data_path.is_dir() else None)
window_width = frame.winfo_reqwidth()
window_height = frame.winfo_reqheight()
x_position = int(master.winfo_screenwidth() / 3 - window_width / 2)
y_position = int(master.winfo_screenheight() / 3 - window_height / 2)
frame.master.geometry("+{}+{}".format(x_position, y_position))
frame.update()
frame.master.wm_minsize(frame.winfo_width(), frame.winfo_height())
master.protocol("WM_DELETE_WINDOW", master.quit)
self._frame = frame
async def _get_location_history_dates(
name: str = Path(..., description="Location full name or alias."),
starting: date = Query(None, description='Starting date (YYYY-MM-DD).', alias="from"),
ending: date = Query(None, description='Ending date (YYYY-MM-DD).', alias="thru"),
weather_data: wd.WeatherData = Security(_get_weather_data, scopes=[Permissions.ReadWeatherData])
) -> List[date]:
location = _get_weather_data_location(name, weather_data)
if not ending:
ending = starting
elif ending < starting:
raise HTTPException(status.HTTP_400_BAD_REQUEST, "From date after thru date.")
return [hd for hd in weather_data.history_dates(location, starting, ending)]
def execute(self, options: argparse.Namespace, weather_data: WeatherData):
if not options.terse:
log.setLevel(INFO)
location = weather_data.get_location(options.location)
if not location:
log.error("Location '{}' was not found...".format(options.location))
return
one_day = timedelta(days=1)
starting = to_date(options.starting)
if not starting:
return
if not options.ending:
ending = starting
else:
ending = to_date(options.ending)
if not ending:
return
history_dates = weather_data.history_dates(location)
existing_history = set(history_dates) if history_dates else set()
history_dates = []
history_dates_exist = []
while starting <= ending:
history_dates_exist.append(starting) if starting in existing_history else history_dates.append(starting)
starting += one_day
if history_dates_exist:
with StringIO() as text:
columns, _ = get_terminal_size(fallback=(80, 24))
dates_per_line = int(columns / len(", YYYY-MM-DD"))
for i, history_date in enumerate(history_dates_exist):
if i % dates_per_line:
text.write(", {}".format(history_date))
else:
text.write("\n{}".format(history_date))
log.warning("History dates exists for:%s", text.getvalue())
weather_data.add_history(location, history_dates, lambda hd: log.info(f"getting {hd}"))
def change_weather_data_dir(self):
if not self._weather_data:
weather_data_dir = Path('.').absolute()
elif self._weather_data.data_path().parent:
weather_data_dir = self._weather_data.data_path().parent.absolute()
else:
weather_data_dir = self._weather_data.data_path().absolute()
data_dir = filedialog.askdirectory(master=self._root,
initialdir=weather_data_dir,
title="Open Weather Data",
mustexist=True)
if data_dir:
data_path = Path(data_dir)
if not self._weather_data or data_path.absolute(
) != self._weather_data.data_path().absolute():
if data_path.parent.absolute() == Path(".").absolute():
data_path = data_path.name
if self._weather_data:
self._weather_data.close()
log.debug("Creating weather data for {}".format(data_path))
self._weather_data = WeatherData(data_path)
self.load_locations()
class WeatherDomain:
def __init__(self, root: Tk, notebook: NotebookWidget,
status: StatusWidget, locations_view: LocationsWidget):
self._weather_data: Optional[WeatherData] = None
self._view_ids: Dict[str, Union[Location, DateRange]] = {}
self._history_id_to_location_id: Dict[str, str] = {}
self._root = root
self._locations_view = locations_view
self._notebook = notebook
self._status = status
@property
def root(self) -> Tk:
return self._root
@property
def weather_data(self) -> WeatherData:
return self._weather_data
def exit(self, prompt_to_exit=False):
if prompt_to_exit:
if messagebox.askyesno("Exit", "Exit Weather Data?") == NO:
return
self._root.quit()
def get_id_value(self, view_id: str) -> Union[Location, DateRange]:
return self._view_ids.get(view_id)
def get_history_location(self, history_id: str) -> Location:
location_id = self._history_id_to_location_id.get(history_id)
if location_id:
return self._view_ids.get(location_id)
def set_menu(self, menu):
self._root.configure(menu=menu)
def get_selections(self) -> List[LocationViewSelection]:
return [(selection, self.get_id_value(selection))
for selection in self._locations_view.get_selection()]
def get_cursor_position(self) -> Coord:
root = self._root
return Coord(root.winfo_pointerx() - root.winfo_vrootx(),
root.winfo_pointery() - root.winfo_vrooty())
def change_weather_data_dir(self):
if not self._weather_data:
weather_data_dir = Path('.').absolute()
elif self._weather_data.data_path().parent:
weather_data_dir = self._weather_data.data_path().parent.absolute()
else:
weather_data_dir = self._weather_data.data_path().absolute()
data_dir = filedialog.askdirectory(master=self._root,
initialdir=weather_data_dir,
title="Open Weather Data",
mustexist=True)
if data_dir:
data_path = Path(data_dir)
if not self._weather_data or data_path.absolute(
) != self._weather_data.data_path().absolute():
if data_path.parent.absolute() == Path(".").absolute():
data_path = data_path.name
if self._weather_data:
self._weather_data.close()
log.debug("Creating weather data for {}".format(data_path))
self._weather_data = WeatherData(data_path)
self.load_locations()
def load_locations(self, weather_data: Optional[WeatherData] = None):
self._locations_view.clear()
self._history_id_to_location_id.clear()
self._view_ids.clear()
if weather_data:
self._weather_data = weather_data
elif self._weather_data:
weather_data = self._weather_data
else:
# this may happen if the app is started and there is no weather_data dir
return
self._status.set_weather_data_dir(weather_data)
if weather_data:
locations = sorted([loc for loc in weather_data.locations()],
key=lambda l: l.name)
log.debug("locations: {}".format([loc.name for loc in locations]))
# noinspection PyTupleAssignmentBalance
name, alias, longitude, latitude, tz, history = LocationsWidget.columns
name_chars = name.min_chars
alias_chars = alias.min_chars
tz_chars = tz.min_chars
for location in locations:
name_chars = max(len(location.name), name_chars)
alias_chars = max(len(location.alias), alias_chars)
tz_chars = max(len(location.tz), tz_chars)
self._locations_view.set_column_chars(name.id, name_chars)
self._locations_view.set_column_chars(alias.id, alias_chars)
self._locations_view.set_column_chars(tz.id, tz_chars)
self._locations_view.set_column_chars(
history.id, len("0000-00-00 to 0000-00-00"))
for location in locations:
location_id = self._locations_view.add_location(location)
self._view_ids[location_id] = location
for history_range in weather_data.history_date_ranges(
location):
history_id = self._locations_view.add_history(
location_id, history_range)
self._view_ids[history_id] = history_range
self._history_id_to_location_id[history_id] = location_id
def add_location(self):
if not self._weather_data:
return
city_db = CityDB()
dialog = NewLocationDialog(
self._root, city_db,
[loc for loc in self._weather_data.locations()])
if not dialog.canceled:
self._weather_data.add_location(dialog.location)
self.load_locations()
def add_location_history(self):
selections = self.get_selections()
selection_count = len(selections)
if 0 == selection_count:
messagebox.showwarning(title="Add History",
message="Select a location.")
return
if 1 == selection_count:
# item_value = self.get_id_value(items[0])
iid, item_value = selections[0]
location = item_value if isinstance(
item_value, Location) else self.get_history_location(iid)
else:
# make sure all the histories are from the same location
location = self.get_history_location(selections[0][0])
for item in selections[1:]:
if self.get_history_location(item[0]) != location:
messagebox.showwarning(title="Add History",
message="Select a single location.")
return
add_history = AddWeatherHistoryDialog(self._root,
title="Add History - {}".format(
location.name))
if add_history.canceled:
return
date_range = add_history.date_range
history_dates = self._weather_data.history_dates(location)
existing_history = set(history_dates) if history_dates else set()
add_dates = []
for add_date in date_range.get_dates():
if add_date not in existing_history:
add_dates.append(add_date)
if 0 == len(add_dates):
messagebox.showwarning(
title="Add History",
message="History already exists\nfor the selected dates...")
else:
progress = self._status.create_progress_widget(
"add history", len(add_dates))
try:
self._weather_data.add_history(location, add_dates,
lambda a: progress.step())
finally:
progress.end()
self.load_locations()
def delete_location(self):
selections = self.get_selections()
selection_count = len(selections)
if 0 == selection_count:
messagebox.showwarning(title="Delete",
message="Select a location.")
return
iid, item_value = selections[0]
if not isinstance(item_value, Location):
return
if 1 < selection_count:
messagebox.showwarning(
title="Delete",
message="Only 1 Location can be deleted at a time.")
return
if messagebox.askyesno(title="Delete",
message="Delete {}?".format(
item_value.name)) == YES:
self._weather_data.remove_location(item_value)
self.load_locations()
def weather_data_properties(self):
if not self._weather_data:
return
history_properties = self._weather_data.all_history_properties()
if not history_properties:
messagebox.showinfo(title="Weather Data properties",
message="Weather Data is empty...")
else:
make_content = WeatherDataPropertiesDialog.Property.make
contents: List[WeatherDataPropertiesDialog.Property] = []
for location_properties in history_properties:
location, properties = location_properties
if not properties:
content = make_content(location.name)
else:
content = make_content(
location.name,
size=properties.size,
entries=properties.entries,
entries_size=properties.entries_size,
compressed_size=properties.compressed_size)
contents.append(content)
WeatherDataPropertiesDialog(self._root, contents)
DataConverters = Dict[DailyWeatherContent, Callable[[Any], Any]]
DailyWeatherContentGenerator = Generator[Dict[DailyWeatherContent, Any],
None, None]
def report_history(self):
selections = self.get_selections()
if 1 != len(selections):
raise RuntimeError(
"Yikes... The length of selections for report history is {}!".
format(len(selections)))
# get the history selection metrics
iid, date_range = selections[0]
history_selection = WeatherHistoryDialog(
self._root,
date_range,
selected_content=[
DailyWeatherContent.TIME, DailyWeatherContent.TEMPERATURE_HIGH,
DailyWeatherContent.TEMPERATURE_HIGH_TIME,
DailyWeatherContent.TEMPERATURE_LOW,
DailyWeatherContent.TEMPERATURE_LOW_TIME
])
if history_selection.canceled:
return
location = self.get_history_location(iid)
date_range = history_selection.get_date_range()
selected_content = history_selection.get_content_selection()
# add the daily weather widget to the notebook
widget = DailyWeatherWidget(self._notebook,
history_selection.get_content_selection())
start, end = date_range
if end.year > start.year:
tab_dates = "{} to {}".format(start.strftime("%b %Y"),
end.strftime("%b %Y"))
elif end.month > start.month:
tab_dates = "{} to {}".format(start.strftime("%b"),
end.strftime("%b %Y"))
else:
tab_dates = "{} to {}".format(start.strftime("%b %d"),
end.strftime("%d %Y"))
self._notebook.add_tab(widget,
tab_name="{}\n{}".format(
location.name, tab_dates))
tz = pytz.timezone(location.tz)
data_converters = {
DailyWeatherContent.TIME:
lambda v: DataConverter.to_date(v, tz, fmt="%b-%d-%Y"),
DailyWeatherContent.TEMPERATURE_HIGH:
lambda v: DataConverter.to_fahrenheit(v),
DailyWeatherContent.TEMPERATURE_HIGH_TIME:
lambda v: DataConverter.to_time(v, tz, fmt="%H:%M"),
DailyWeatherContent.TEMPERATURE_LOW:
lambda v: DataConverter.to_fahrenheit(v),
DailyWeatherContent.TEMPERATURE_LOW_TIME:
lambda v: DataConverter.to_time(v, tz, fmt="%H:%M"),
DailyWeatherContent.TEMPERATURE_MAX:
lambda v: DataConverter.to_fahrenheit(v),
DailyWeatherContent.TEMPERATURE_MAX_TIME:
lambda v: DataConverter.to_time(v, tz, fmt="%H:%M"),
DailyWeatherContent.TEMPERATURE_MIN:
lambda v: DataConverter.to_fahrenheit(v),
DailyWeatherContent.TEMPERATURE_MIN_TIME:
lambda v: DataConverter.to_time(v, tz, fmt="%H:%M"),
DailyWeatherContent.WIND_SPEED:
lambda v: DataConverter.to_str(v),
DailyWeatherContent.WIND_GUST:
lambda v: DataConverter.to_str(v),
DailyWeatherContent.WIND_GUST_TIME:
lambda v: DataConverter.to_time(v, tz, fmt="%H:%M"),
DailyWeatherContent.WIND_BEARING:
lambda v: DataConverter.wind_bearing(v),
DailyWeatherContent.CLOUD_COVER:
lambda v: DataConverter.to_str(v)
}
report_rows = []
content_order = widget.content_order
for history in self._get_location_histories(location, date_range,
selected_content,
data_converters):
row = []
for content in content_order:
row.append(history[content])
report_rows.append(row)
widget.load(report_rows)
def graph_history(self):
selections = self.get_selections()
selections_len = len(selections)
if 0 == selections_len:
return
if 5 < selections_len:
messagebox.showinfo(
title="Graph",
message="Current graph history only supports 5 selections.")
return
# to get here the controller should only have allowed a multiple selection of date ranges
location_histories: List[LocationDateRange] = []
for iid, selection in selections:
assert isinstance(
selection,
DateRange), "Yikes... Selection for graph is not a DateRange!"
location_histories.append(
(self.get_history_location(iid), selection))
history_date_mappings = self._get_history_date_mapping(
location_histories)
if not history_date_mappings:
return
date_selection = WeatherHistoryGraphDatesDialog(
self._root, history_date_mappings)
if date_selection.canceled:
return
def start_end_months(_month_span: List[bool]) -> Tuple[int, int]:
_start_month = _month_span.index(True)
try:
_end_month = _month_span.index(False, _start_month) - 1
except ValueError:
_end_month = len(_month_span) - 1
return _start_month, _end_month
graph_start_month, graph_end_month = start_end_months(
date_selection.month_selections)
def graph_date(_year: int, _month: int, _day: Callable[[int, int],
int]) -> date:
_year = (_year + 1) if 12 < _month else _year
_month = (_month - 12) if 12 < _month else _month
return date(_year, _month, _day(_year, _month))
graph_date_range = DateRange(
graph_date(MINYEAR, graph_start_month, lambda y, m: 1),
graph_date(MINYEAR, graph_end_month,
lambda y, m: monthrange(y, m)[1]))
locations = [hdm[0] for hdm in history_date_mappings]
multiple_locations = locations.count(locations[0]) != len(locations)
if multiple_locations:
title = "Daily Temperatures for Multiple Locations"
tab_label = "History Graph for\nMultiple Locations"
else:
title = "Daily Temperatures for {}".format(
location_histories[0][0].name)
tab_label = "{}\nHistory Graph".format(
location_histories[0][0].name)
graph = DailyTemperatureGraph(self._notebook,
graph_date_range,
title=title)
self._notebook.add_tab(graph, tab_label)
# sanitize the colors jic...
colors = get_setting("gui", "graph_colors")
accepted_color_names = {c.name.casefold() for c in get_colors()}
accepted_hex_color_names = {c.to_hex() for c in get_colors()}
for idx, color in enumerate(colors):
if color.casefold(
) not in accepted_color_names and color not in accepted_hex_color_names:
log.warning(
"'%s' is not an accepted color. Review UI graph colors in File->Settings.",
color)
colors[idx] = "black"
selected_content = [
DailyWeatherContent.TIME, DailyWeatherContent.TEMPERATURE_LOW,
DailyWeatherContent.TEMPERATURE_HIGH
]
for location, location_date_range, location_month_slots in history_date_mappings:
tz = pytz.timezone(location.tz)
data_converters = {
DailyWeatherContent.TIME:
lambda v: DataConverter.to_binary_date(v, tz),
DailyWeatherContent.TEMPERATURE_HIGH:
lambda v: DataConverter.to_binary_float(v),
DailyWeatherContent.TEMPERATURE_LOW:
lambda v: DataConverter.to_binary_float(v)
}
location_start_month, location_end_month = start_end_months(
location_month_slots)
location_year = location_date_range.low.year
if 12 < location_start_month and not location_date_range.spans_years(
):
# adjust the location date range due to history being moved for date intersection
location_year -= 1
if location_start_month < graph_start_month:
location_start_month = graph_start_month
if location_end_month > graph_end_month:
location_end_month = graph_end_month
date_range = DateRange(
graph_date(location_year, location_start_month,
lambda y, m: 1),
graph_date(location_year, location_end_month,
lambda y, m: monthrange(y, m)[1]))
daily_temperatures: List[DailyTemperature] = []
for history in self._get_location_histories(
location, date_range, selected_content, data_converters):
daily_temperatures.append(
DailyTemperature(
history.get(DailyWeatherContent.TIME),
history.get(DailyWeatherContent.TEMPERATURE_LOW),
history.get(DailyWeatherContent.TEMPERATURE_HIGH)))
sorted(daily_temperatures, key=lambda dt: dt.ts)
color = colors[graph.plot_count % len(colors)]
starting = daily_temperatures[0].ts
ending = daily_temperatures[-1].ts
if starting.year < ending.year:
label = "{} to {}".format(starting.year, ending.year)
else:
label = str(starting.year)
if multiple_locations:
label = "{}\n{}".format(location.name, label)
graph.plot(daily_temperatures, color=color, label=label)
def settings(self):
dialog = SettingsDialog(self._notebook)
if not dialog.canceled:
# assume the logging levels have changed
set_module_logging_levels()
def _get_location_histories(
self, location: Location, date_range: DateRange,
selected_content: List[DailyWeatherContent],
data_converters: DataConverters) -> DailyWeatherContentGenerator:
low_t, high_t = date_range
history_dates = self._weather_data.history_dates(
location, low_t, high_t)
data_converter = GenericDataConverter[DailyWeatherContent](
data_converters)
progress = self._status.create_progress_widget(
"graph creation", maximum=len(history_dates))
try:
for history in self._weather_data.get_history(
location, history_dates):
progress.step()
yield data_converter.convert_contents(history,
selected_content)
finally:
progress.end()
@staticmethod
def _get_history_date_mapping(
location_date_ranges: List[LocationDateRange]
) -> List[HistoryDateMapping]:
location_date_ranges_len = len(location_date_ranges)
assert 0 < location_date_ranges_len, "Yikes... Location date ranges are emtpy!"
# move the date range to a year neutral format
neutral_date_ranges = [
ldr[1].as_neutral_date_range() for ldr in location_date_ranges
]
# create the map location date months will go into to see where they might intersect
month_slots: List[List[LocationDateRange]] = [[] for _ in range(25)]
for idx, location_date_range in enumerate(location_date_ranges):
date_range = neutral_date_ranges[idx]
high_month = date_range.high.month
if location_date_range[1].spans_years():
high_month += 12
for month in range(date_range.low.month, high_month + 1):
month_slots[month].append(location_date_range)
assert 0 == len(month_slots[0]), "Yikes... Month slot 0 has a value!"
# make another pass across the months checking if 1-12 month dates should be moved to the next year
for idx, month_slot in enumerate(month_slots):
# only look at the first year of month slots
if 12 < idx:
break
# check slots that have location date ranges and are not full
if month_slot and len(month_slot) != location_date_ranges_len:
month_slot_next_year = month_slots[idx + 12]
if month_slot_next_year:
month_slot_next_year += [
ldr for ldr in month_slot if not ldr[1].spans_years()
]
month_slot[:] = [
ldr for ldr in month_slot if ldr[1].spans_years()
]
# now search through the slots looking for intersecting month groups
intersecting_months: List[Tuple[int, int]] = []
start = end = 0
for idx, month_slot in enumerate(month_slots):
if month_slot:
if start:
end = idx
else:
start = end = idx
elif start:
intersecting_months.append((start, end))
start = end = 0
if start:
intersecting_months.append((start, end))
history_date_mappings: List[HistoryDateMapping] = []
if 1 < len(intersecting_months):
# todo: show the date groupings
messagebox.showerror(
title="Graph",
message="History mapping selection is too complex...")
else:
# walk back through the intersecting months and get the location month slots for each location date range
def eq(lhs: LocationDateRange, rhs: LocationDateRange) -> bool:
return lhs[0].name == rhs[0].name and lhs[1].low == rhs[
1].low and lhs[1].high == rhs[1].high
start_month, end_month = intersecting_months[0]
for location_date_range in location_date_ranges:
history_month_slots = [False for _ in range(len(month_slots))]
for slot in range(start_month, end_month + 1):
for slot_ldr in month_slots[slot]:
if eq(location_date_range, slot_ldr):
history_month_slots[slot] = True
break
history_date_mappings.append(
(location_date_range[0], location_date_range[1],
history_month_slots))
return history_date_mappings
def get_location(weather_data: WeatherData, name: str, alias: str = None) -> Optional[Location]:
location = weather_data.get_location(name)
if not location and alias:
location = weather_data.get_location(alias)
return location
def _get_weather_data_location(name: str, weather_data: wd.WeatherData) -> wd.Location:
location = weather_data.get_location(name)
if not location:
raise HTTPException(status.HTTP_404_NOT_FOUND, f'"{name}" was not found.')
return location
def get_locations(weather_data: WeatherData, *location_names: str) -> List[Location]:
if location_names:
return [location for location in (weather_data.get_location(name) for name in location_names) if location]
return [location for location in weather_data.locations()]
class HistoryPerformance:
def __init__(self):
self.weather_data = WeatherData()
def run(self, single_threaded, skip_load):
load_control_blocks: List[Tuple[Location, WeatherHistory]] = []
for location in self.weather_data.locations():
# noinspection PyProtectedMember
weather_history = self.weather_data._get_weather_history(location)
load_control_blocks.append((location, weather_history))
for p in range(2):
print("{} pass {}threaded {}loading json".format(
"2nd" if p else "1st",
"single " if single_threaded else "multi-",
"skip " if skip_load else ""))
read_stats: List[Tuple[Location, Tuple]] = []
overall_stop_watch = StopWatch()
if single_threaded:
for location, weather_history in load_control_blocks:
read_stats.append(
(location, self.load_history(weather_history,
skip_load)))
else:
with ThreadPoolExecutor(max_workers=20) as executor:
future_to_location = {
executor.submit(self.load_history, wh, skip_load): loc
for loc, wh in load_control_blocks
}
for future in as_completed(future_to_location):
location = future_to_location[future]
if location:
try:
read_stats.append((location, future.result()))
except Exception as err:
print(f'"{location.name}" error: {str(err)}')
overall_stop_watch.stop()
print(f'History read took: {str(overall_stop_watch)}')
for location, stats in read_stats:
if stats:
stop_watch, histories = stats
loads_per_second = histories / stop_watch.elapsed(
) if stop_watch.elapsed() else float(histories)
print('"{}":(elapsed={},histories={},loads/sec={:,.1f})'.
format(location.name, stop_watch, histories,
loads_per_second))
@staticmethod
def load_history(weather_history: WeatherHistory, skip_load: bool):
if not weather_history:
return
history_count = 0
stop_watch = StopWatch()
with weather_history.reader() as history_reader:
for history_date in weather_history.dates():
history_count += 1
data = history_reader(
weather_history.make_pathname(history_date))
if skip_load:
continue
# full_history = json.loads(data)
full_history = orjson.loads(data)
# full_history = ujson.loads(data)
hourly_history = full_history.get("hourly")
assert hourly_history, "Did not find 'hourly' in weather history"
daily_history = full_history.get("daily")
assert daily_history, "Did not find 'daily' in weather history"
stop_watch.stop()
return stop_watch, history_count
def execute(self, options: argparse.Namespace, weather_data: WeatherData):
history_properties = weather_data.all_history_properties()
if not history_properties:
return
location_field_name = "location"
property_field_names = WeatherHistoryProperties._fields
def to_dict(_location: Location, _properties: WeatherHistoryProperties) -> dict:
_dict = {location_field_name: _location.name}
if _properties:
for _field in property_field_names:
_dict[_field] = getattr(_properties, _field)
return _dict
if options.csv:
with CsvDictWriter([location_field_name] + list(property_field_names)).stdout() as write:
for location, properties in history_properties:
write(to_dict(location, properties))
return
if options.json:
location_list = [to_dict(location, properties) for location, properties in history_properties]
print(json.dumps({"locations": location_list}, indent=2))
return
def fmt(value: int, to_kib=True) -> str:
if not value:
return ""
return "{: >,d}".format(value) if not to_kib or value < 1024 else "{: >,d} kiB".format(round(value / 1024))
Field = namedtuple('Field', ['label', 'key', 'header', 'detail', 'width'])
fields = (
Field("Location", "lnw", "{:^{lnw}}", "{:<{lnw}}",
lambda: max_width(history_properties, 10, lambda hp: hp[0].name)),
Field("Overall Size", "osw", "{:^{osw}}", "{:>{osw}}", lambda: len(fields[1].label)),
Field("History Count", "hcw", "{:^{hcw}}", "{:>{hcw}}", lambda: len(fields[2].label)),
Field("Raw History Size", "hsw", "{:^{hsw}}", "{:>{hsw}}", lambda: len(fields[3].label)),
Field("Compressed Size", "csw", "{:^{csw}}", "{:>{csw}}", lambda: len(fields[4].label)),
)
header = " ".join([field.header for field in fields])
field_sizes = {field.key: field.width() for field in fields}
print(header.format(*(field.label for field in fields), **field_sizes))
details = " ".join([field.detail for field in fields])
print(details.format(*("-" * field.width() for field in fields), **field_sizes))
for location, properties in history_properties:
print(details.format(
location.name,
fmt(properties.size),
fmt(properties.entries, to_kib=False),
fmt(properties.entries_size),
fmt(properties.compressed_size),
**field_sizes)
)
print(details.format(*("=" * field.width() for field in fields), **field_sizes))
print(details.format(
"Totals",
fmt(sum(p.size for _, p in history_properties)),
fmt(sum(p.entries for _, p in history_properties), to_kib=False),
fmt(sum(p.entries_size for _, p in history_properties)),
fmt(sum(p.compressed_size for _, p in history_properties)),
**field_sizes
))
def __init__(self):
self.weather_data = WeatherData()
def execute(self, options: argparse.Namespace, weather_data: WeatherData):
location = weather_data.get_location(options.location)
if not location:
log.warning("Location '{}' was not found...".format(options.location))
return
if not weather_data.history_exists(location):
log.warning("'{}' does not have any weather history.".format(location.name))
return
output_type = "json" if options.json else "csv" if options.csv else "txt"
output_path = Path(options.file) if options.file else None
if output_path:
if output_path.suffix != output_type:
output_path = output_path.with_suffix("." + output_type)
if output_path.exists() and not output_path.is_file():
raise ValueError("The {} destination, '{}', is not a file.".format(output_type, output_path))
starting = to_date(options.starting)
if not starting:
return
if not options.ending:
ending = starting
else:
ending = to_date(options.ending)
if not ending:
return
history_dates = weather_data.history_dates(location, starting, ending)
# suck up all the histories
histories = weather_data.get_history(location, history_dates, hourly_history=options.hourly)
def to_json(_histories: Iterable) -> str:
return json.dumps({
"location": location.name,
"type": "hourly_history" if options.hourly else "daily_history",
"history": [_h for _h in _histories]
}, indent=2)
if options.raw:
if output_path:
output_path.write_text(to_json(histories))
else:
print(to_json(histories))
return
report_builder = self.ReportBuilder(location, options)
if options.csv:
csv_writer = CsvDictWriter(report_builder.data_converter.field_names())
with csv_writer.file_writer(output_path) if output_path else csv_writer.stdout() as csv_write:
for data in report_builder.data_generator(histories):
csv_write(data)
return
if options.json:
history = to_json(report_builder.data_generator(histories))
if output_path:
output_path.write_text(history)
else:
print(history)
return
Field = namedtuple('Field', ['key', 'header', 'detail'])
if report_builder.hourly:
fields: Dict[Enum, Field] = {
HourlyWeatherContent.TIME: Field("tw", "{:^{tw}}", "{:<{tw}}"),
HourlyWeatherContent.TEMPERATURE: Field("tm", "{:^{tm}}", "{:<{tm}}"),
HourlyWeatherContent.APPARENT_TEMPERATURE: Field("at", "{:^{at}}", "{:<{at}}"),
HourlyWeatherContent.WIND_SPEED: Field("ws", "{:^{ws}}", "{:<{ws}}"),
HourlyWeatherContent.WIND_GUST: Field("wg", "{:^{wg}}", "{:<{wg}}"),
HourlyWeatherContent.WIND_BEARING: Field("wb", "{:^{wb}}", "{:<{wb}}"),
HourlyWeatherContent.CLOUD_COVER: Field("cc", "{:^{cc}}", "{:<{cc}}"),
HourlyWeatherContent.UV_INDEX: Field("uv", "{:^{uv}}", "{:<{uv}}"),
HourlyWeatherContent.HUMIDITY: Field("hw", "{:^{hw}}", "{:<{hw}}"),
HourlyWeatherContent.DEW_POINT: Field("dp", "{:^{dp}}", "{:<{dp}}"),
HourlyWeatherContent.SUMMARY: Field("sw", "{:^{sw}}", "{:<{sw}})"),
}
else:
fields: Dict[Enum, Field] = {
DailyWeatherContent.TIME: Field("tw", "{:^{tw}}", "{:<{tw}}"),
DailyWeatherContent.TEMPERATURE_HIGH: Field("th", "{:^{th}}", "{:^{th}}"),
DailyWeatherContent.TEMPERATURE_HIGH_TIME: Field("tht", "{:^{tht}}", "{:^{tht}}"),
DailyWeatherContent.TEMPERATURE_LOW: Field("tl", "{:^{tl}}", "{:^{tl}}"),
DailyWeatherContent.TEMPERATURE_LOW_TIME: Field("tlt", "{:^{tlt}}", "{:^{tlt}}"),
DailyWeatherContent.TEMPERATURE_MAX: Field("mx", "{:^{mx}}", "{:^{mx}}"),
DailyWeatherContent.TEMPERATURE_MAX_TIME: Field("mxt", "{:^{mxt}}", "{:^{mxt}}"),
DailyWeatherContent.TEMPERATURE_MIN: Field("mn", "{:^{mn}}", "{:^{mn}}"),
DailyWeatherContent.TEMPERATURE_MIN_TIME: Field("mnt", "{:^{mnt}}", "{:^{mnt}}"),
DailyWeatherContent.WIND_SPEED: Field("ws", "{:^{ws}}", "{:>{ws}}"),
DailyWeatherContent.WIND_GUST: Field("wg", "{:^{wg}}", "{:>{wg}}"),
DailyWeatherContent.WIND_GUST_TIME: Field("wgt", "{:^{wgt}}", "{:^{wgt}}"),
DailyWeatherContent.WIND_BEARING: Field("wb", "{:^{wb}}", "{:^{wb}}"),
DailyWeatherContent.CLOUD_COVER: Field("cc", "{:^{cc}}", "{:<{cc}}"),
DailyWeatherContent.UV_INDEX: Field("uv", "{:^{uv}}", "{:>{uv}}"),
DailyWeatherContent.UV_INDEX_TIME: Field("uvt", "{:^{uvt}}", "{:^{uvt}}"),
DailyWeatherContent.SUMMARY: Field("sw", "{:<{sw}}", "{:<{sw}}"),
DailyWeatherContent.HUMIDITY: Field("hw", "{:^{hw}}", "{:>{hw}}"),
DailyWeatherContent.DEW_POINT: Field("dp", "{:^{dp}}", "{:<{dp}}"),
DailyWeatherContent.SUNRISE_TIME: Field("sr", "{:^{sr}}", "{:^{sr}}"),
DailyWeatherContent.SUNSET_TIME: Field("ss", "{:^{ss}}", "{:^{ss}}"),
DailyWeatherContent.MOON_PHASE: Field("mp", "{:^{mp}}", "{:>{mp}}"),
}
selected_fields = report_builder.data_converter.keys()
header = " ".join([fields[f].header for f in selected_fields])
detail = " ".join([fields[f].detail for f in selected_fields])
field_widths = {fields[f].key: report_builder.label_widths[f] for f in selected_fields}
print(header.format(*(report_builder.labels[f][0] for f in selected_fields), **field_widths))
print(header.format(*(report_builder.labels[f][1] for f in selected_fields), **field_widths))
print(header.format(*("-" * field_widths[fields[f].key] for f in selected_fields), **field_widths))
def safe_format(value):
return "" if value is None else value
for history in report_builder.data_generator(histories):
print(detail.format(*(safe_format(history[f]) for f in selected_fields), **field_widths))
def execute(self, options: argparse.Namespace, weather_data: WeatherData):
location = get_location(weather_data, options.name)
if not location:
log.error("Location '%s' does not exist...", options.name)
elif not weather_data.remove_location(location):
log.error("Weather data for '%s' was not completely removed...", options.name)
async def _get_locations(
properties: bool = Query(False, description="Optionally include location history statistics."),
history: bool = Query(False, description="Optionally include a summary of location history."),
weather_data: wd.WeatherData = Security(_get_weather_data, scopes=[Permissions.ReadWeatherData])
) -> List[Location]:
return [_assemble_location(location, properties, history, weather_data) for location in weather_data.locations()]