def on_enter(self):
ab = self.manager.nav_bar
ab.set_title('Data Manager: Hourly Residential Load Profiles')
LoadTypeRVEntry.host_screen = self
LocationRVEntry.host_screen = self
StateRVEntry.host_screen = self
ssl_verify, proxy_settings = check_connection_settings()
self._get_load_types(ssl_verify, proxy_settings)
def on_enter(self):
ab = self.manager.nav_bar
ab.set_title('Data Manager: Hourly Commercial Load Profiles')
StateRVEntry.host_screen = self
LocationRVEntry.host_screen = self
BuildingRVEntry.host_screen = self
if self.df_locations.empty:
logging.info(
'LoadProfileDM: Retrieving locations from database...')
ssl_verify, proxy_settings = check_connection_settings()
self._get_locations(ssl_verify, proxy_settings)
def on_load_type_selected(self, instance, value):
try:
logging.info(
'LoadProfileDM: Load type selection changed to {0}.'.format(
value['name']))
except KeyError:
logging.info('LoadProfileDM: Load type selection reset.')
else:
load_type_root_link = value['link']
ssl_verify, proxy_settings = check_connection_settings()
self._get_locations(load_type_root_link, ssl_verify,
proxy_settings)
# thread_building_types = threading.Thread(target=self._get_locations, args=[load_type_root_link, ssl_verify, proxy_settings])
# thread_building_types.start()
finally:
self.state_rv.deselect_all_nodes()
self.state_rv_filter.text = ''
self.state_selected = ''
def _query_api(self, api_query):
"""Uses PVWatts API to query for a PV profile."""
ssl_verify, proxy_settings = check_connection_settings()
try:
with requests.Session() as req:
http_request = req.get(api_query,
proxies=proxy_settings,
timeout=10,
verify=ssl_verify,
stream=True)
if http_request.status_code != requests.codes.ok:
http_request.raise_for_status()
except requests.HTTPError as e:
logging.error('PVProfileDM: {0}'.format(repr(e)))
raise requests.ConnectionError
except requests.exceptions.ProxyError:
logging.error('PVProfileDM: Could not connect to proxy.')
raise requests.ConnectionError
except requests.ConnectionError as e:
logging.error(
'PVProfileDM: Failed to establish a connection to the host server.'
)
raise requests.ConnectionError
except requests.Timeout as e:
logging.error('PVProfileDM: The connection timed out.')
raise requests.ConnectionError
except requests.RequestException as e:
logging.error('PVProfileDM: {0}'.format(repr(e)))
raise requests.ConnectionError
except Exception as e:
# Something else went wrong.
logging.error(
'PVProfileDM: An unexpected error has occurred. ({0})'.format(
repr(e)))
raise requests.ConnectionError
else:
request_content = http_request.json()
if not self.save_name_field.text:
popup = WarningPopup()
popup.popup_text.text = 'Please specify a name to save the PV profile as.'
popup.open()
return
else:
outname = self.save_name_field.text
# Strip non-alphanumeric chars from given name for filename.
delchars = ''.join(c for c in map(chr, range(256))
if not c.isalnum())
outname = outname.translate({ord(i): None for i in delchars})
# Save.
destination_dir = os.path.join(DATA_HOME, 'pv')
os.makedirs(destination_dir, exist_ok=True)
destination_file = os.path.join(destination_dir, outname + '.json')
if not os.path.exists(destination_file):
with open(destination_file, 'w') as outfile:
json.dump(request_content, outfile)
logging.info('PVProfileDM: Profile successfully saved.')
popup = WarningPopup()
popup.title = 'Success!'
popup.popup_text.text = 'PV profile successfully saved.'
popup.open()
else:
# File already exists with same name.
popup = WarningPopup()
popup.popup_text.text = 'A PV profile with the provided name already exists. Please specify a new name.'
popup.open()
finally:
self.save_button.disabled = False
def save_load_data(self):
"""Saves the data for the building type selected."""
try:
csv_link = self._validate_selections()
except Exception as e:
print(e)
else:
ssl_verify, proxy_settings = check_connection_settings()
attempt_download = True
n_tries = 0
self.connection_error_occurred = False
while attempt_download:
n_tries += 1
if n_tries >= MAX_WHILE_ATTEMPTS:
logging.warning('LoadProfileDM: Hit download retry limit.')
attempt_download = False
self.connection_error_occurred = True
break
if App.get_running_app().root.stop.is_set():
return
try:
with requests.Session() as req:
page = req.get(csv_link,
timeout=10,
verify=ssl_verify,
proxies=proxy_settings)
if page.status_code != requests.codes.ok:
page.raise_for_status()
else:
attempt_download = False
except requests.HTTPError as e:
logging.error('LoadProfileDM: {0}'.format(repr(e)))
except requests.exceptions.ProxyError:
logging.error('LoadProfileDM: Could not connect to proxy.')
except requests.ConnectionError as e:
logging.error(
'LoadProfileDM: Failed to establish a connection to the host server.'
)
except requests.Timeout as e:
logging.error('LoadProfileDM: The connection timed out.')
except requests.RequestException as e:
logging.error('LoadProfileDM: {0}'.format(repr(e)))
except Exception as e:
# Something else went wrong.
logging.error(
'LoadProfileDM: An unexpected error has occurred. ({0})'
.format(repr(e)))
else:
data_down = page.content.decode(page.encoding)
csv_data = pd.read_csv(io.StringIO(data_down))
electricity_data = csv_data[[
'Date/Time', 'Electricity:Facility [kW](Hourly)'
]]
# Save to persistent object on disk.
url_split = csv_link.split('/')
destination_dir = os.path.join(DATA_HOME, 'load',
'residential',
url_split[-2])
os.makedirs(destination_dir, exist_ok=True)
destination_file = os.path.join(destination_dir,
url_split[-1])
electricity_data.to_csv(destination_file,
sep=',',
index=False)
popup = WarningPopup()
popup.title = 'Success!'
popup.popup_text.text = 'Load data successfully saved.'
popup.open()
logging.info(
'LoadProfileDM: Load data successfully saved.')
def _query_api_for_rate_structures(self, api_query):
"""Uses OpenEI API to query the rate structures for given EIA ID and populates rate structure RecycleView."""
ssl_verify, proxy_settings = check_connection_settings()
attempt_download = True
n_tries = 0
while attempt_download:
n_tries += 1
if n_tries >= MAX_WHILE_ATTEMPTS:
logging.warning('RateStructureDM: Hit download retry limit.')
attempt_download = False
break
if App.get_running_app().root.stop.is_set():
return
try:
with requests.Session() as req:
http_request = req.get(api_query,
proxies=proxy_settings,
timeout=10,
verify=ssl_verify,
stream=True)
if http_request.status_code != requests.codes.ok:
http_request.raise_for_status()
else:
attempt_download = False
except requests.HTTPError as e:
logging.error('RateStructureDM: {0}'.format(repr(e)))
raise requests.ConnectionError
except requests.exceptions.ProxyError:
logging.error('RateStructureDM: Could not connect to proxy.')
raise requests.ConnectionError
except requests.ConnectionError as e:
logging.error(
'RateStructureDM: Failed to establish a connection to the host server.'
)
raise requests.ConnectionError
except requests.Timeout as e:
logging.error('RateStructureDM: The connection timed out.')
raise requests.ConnectionError
except requests.RequestException as e:
logging.error('RateStructureDM: {0}'.format(repr(e)))
raise requests.ConnectionError
except Exception as e:
# Something else went wrong.
logging.error(
'RateStructureDM: An unexpected error has occurred. ({0})'.
format(repr(e)))
raise requests.ConnectionError
else:
structure_list = http_request.json()['items']
structure_df = pd.DataFrame.from_records(structure_list)
structure_df.dropna(subset=['energyratestructure'],
inplace=True)
# Filter out entries whose energyratestructure array does not contain "rate" terms.
mask = structure_df['energyratestructure'].apply(lambda x: all(
['rate' in hr.keys() for row in x for hr in row]))
structure_df = structure_df[mask]
structure_list = structure_df.to_dict(orient='records')
# First, sort by effective date.
# structure_list = sorted(structure_list, key=lambda x: (x['name'], x.get('startdate', np.nan)))
structure_list = sorted(
structure_list,
key=lambda x: x.get('startdate', np.nan),
reverse=True)
# Then, sort by name.
structure_list = sorted(structure_list,
key=lambda x: x['name'])
# Display name: Name (record['startdate']).
effective_dates = [
'(Effective Date : {0})'.format(
dt.datetime.fromtimestamp(
record['startdate']).strftime('%m/%d/%Y'))
if not np.isnan(record['startdate']) else ''
for record in structure_list
]
records = [{
'name': record['name'] + ' ' + effective_dates[ix],
'record': record
} for ix, record in enumerate(structure_list, start=0)]
# records = sorted(records, key=lambda t: t['name'])
self.rate_structure_rv.data = records
self.rate_structure_rv.unfiltered_data = records
logging.info(
'RateStructureDM: Retrieved utility rate structures.')
fade_in_animation(self.rate_structure_select_bx)
finally:
self.loading_screen.dismiss()
def _download_utility_ref_table(self):
"""Downloads and builds the utility reference table from OpenEI."""
ssl_verify, proxy_settings = check_connection_settings()
# Invester-owned utilities.
attempt_download = True
n_tries = 0
while attempt_download:
n_tries += 1
if n_tries >= MAX_WHILE_ATTEMPTS:
logging.warning('RateStructureDM: Hit download retry limit.')
attempt_download = False
break
if App.get_running_app().root.stop.is_set():
return
try:
with requests.Session() as req:
http_request = req.get(URL_OPENEI_IOU,
proxies=proxy_settings,
timeout=6,
verify=ssl_verify,
stream=True)
if http_request.status_code != requests.codes.ok:
http_request.raise_for_status()
else:
attempt_download = False
except requests.HTTPError as e:
logging.error('RateStructureDM: {0}'.format(repr(e)))
except requests.exceptions.ProxyError:
logging.error('RateStructureDM: Could not connect to proxy.')
except requests.ConnectionError as e:
logging.error(
'RateStructureDM: Failed to establish a connection to the host server.'
)
except requests.Timeout as e:
logging.error('RateStructureDM: The connection timed out.')
except requests.RequestException as e:
logging.error('RateStructureDM: {0}'.format(repr(e)))
except Exception as e:
# Something else went wrong.
logging.error(
'RateStructureDM: An unexpected error has occurred. ({0})'.
format(repr(e)))
else:
data_down = http_request.content.decode(
http_request.apparent_encoding)
data_iou = pd.read_csv(io.StringIO(data_down))
# Non-investor-owned utilities.
attempt_download = True
n_tries = 0
while attempt_download:
n_tries += 1
if n_tries >= MAX_WHILE_ATTEMPTS:
logging.warning('RateStructureDM: Hit download retry limit.')
attempt_download = False
break
if App.get_running_app().root.stop.is_set():
return
try:
with requests.Session() as req:
http_request = req.get(URL_OPENEI_NONIOU,
proxies=proxy_settings,
timeout=6,
verify=ssl_verify,
stream=True)
if http_request.status_code != requests.codes.ok:
http_request.raise_for_status()
else:
attempt_download = False
except requests.HTTPError as e:
logging.error('RateStructureDM: {0}'.format(repr(e)))
except requests.exceptions.ProxyError:
logging.error('RateStructureDM: Could not connect to proxy.')
except requests.ConnectionError as e:
logging.error(
'RateStructureDM: Failed to establish a connection to the host server.'
)
except requests.Timeout as e:
logging.error('RateStructureDM: The connection timed out.')
except requests.RequestException as e:
logging.error('RateStructureDM: {0}'.format(repr(e)))
except Exception as e:
# Something else went wrong.
logging.error(
'RateStructureDM: An unexpected error has occurred. ({0})'.
format(repr(e)))
else:
data_down = http_request.content.decode(
http_request.apparent_encoding)
data_noniou = pd.read_csv(io.StringIO(data_down))
try:
df_combined = pd.concat([data_iou, data_noniou],
ignore_index=True)
except NameError:
# Connection error prevented downloads.
raise requests.ConnectionError
else:
# Add column for state/district name.
df_combined['state name'] = df_combined['state'].apply(
lambda state_abbr: STATE_ABBR_TO_NAME.get(state_abbr, ''))
self.utility_ref_table = df_combined
logging.info(
'RateStructureDM: Retrieved list of all utilities.')
