def on_enter(self): ab = self.manager.nav_bar ab.set_title('Data Manager: Hourly Residential Load Profiles') LoadTypeRVEntry.host_screen = self LocationRVEntry.host_screen = self StateRVEntry.host_screen = self ssl_verify, proxy_settings = check_connection_settings() self._get_load_types(ssl_verify, proxy_settings)
def on_enter(self): ab = self.manager.nav_bar ab.set_title('Data Manager: Hourly Commercial Load Profiles') StateRVEntry.host_screen = self LocationRVEntry.host_screen = self BuildingRVEntry.host_screen = self if self.df_locations.empty: logging.info( 'LoadProfileDM: Retrieving locations from database...') ssl_verify, proxy_settings = check_connection_settings() self._get_locations(ssl_verify, proxy_settings)
def on_load_type_selected(self, instance, value): try: logging.info( 'LoadProfileDM: Load type selection changed to {0}.'.format( value['name'])) except KeyError: logging.info('LoadProfileDM: Load type selection reset.') else: load_type_root_link = value['link'] ssl_verify, proxy_settings = check_connection_settings() self._get_locations(load_type_root_link, ssl_verify, proxy_settings) # thread_building_types = threading.Thread(target=self._get_locations, args=[load_type_root_link, ssl_verify, proxy_settings]) # thread_building_types.start() finally: self.state_rv.deselect_all_nodes() self.state_rv_filter.text = '' self.state_selected = ''
def _query_api(self, api_query): """Uses PVWatts API to query for a PV profile.""" ssl_verify, proxy_settings = check_connection_settings() try: with requests.Session() as req: http_request = req.get(api_query, proxies=proxy_settings, timeout=10, verify=ssl_verify, stream=True) if http_request.status_code != requests.codes.ok: http_request.raise_for_status() except requests.HTTPError as e: logging.error('PVProfileDM: {0}'.format(repr(e))) raise requests.ConnectionError except requests.exceptions.ProxyError: logging.error('PVProfileDM: Could not connect to proxy.') raise requests.ConnectionError except requests.ConnectionError as e: logging.error( 'PVProfileDM: Failed to establish a connection to the host server.' ) raise requests.ConnectionError except requests.Timeout as e: logging.error('PVProfileDM: The connection timed out.') raise requests.ConnectionError except requests.RequestException as e: logging.error('PVProfileDM: {0}'.format(repr(e))) raise requests.ConnectionError except Exception as e: # Something else went wrong. logging.error( 'PVProfileDM: An unexpected error has occurred. ({0})'.format( repr(e))) raise requests.ConnectionError else: request_content = http_request.json() if not self.save_name_field.text: popup = WarningPopup() popup.popup_text.text = 'Please specify a name to save the PV profile as.' popup.open() return else: outname = self.save_name_field.text # Strip non-alphanumeric chars from given name for filename. delchars = ''.join(c for c in map(chr, range(256)) if not c.isalnum()) outname = outname.translate({ord(i): None for i in delchars}) # Save. destination_dir = os.path.join(DATA_HOME, 'pv') os.makedirs(destination_dir, exist_ok=True) destination_file = os.path.join(destination_dir, outname + '.json') if not os.path.exists(destination_file): with open(destination_file, 'w') as outfile: json.dump(request_content, outfile) logging.info('PVProfileDM: Profile successfully saved.') popup = WarningPopup() popup.title = 'Success!' popup.popup_text.text = 'PV profile successfully saved.' popup.open() else: # File already exists with same name. popup = WarningPopup() popup.popup_text.text = 'A PV profile with the provided name already exists. Please specify a new name.' popup.open() finally: self.save_button.disabled = False
def save_load_data(self): """Saves the data for the building type selected.""" try: csv_link = self._validate_selections() except Exception as e: print(e) else: ssl_verify, proxy_settings = check_connection_settings() attempt_download = True n_tries = 0 self.connection_error_occurred = False while attempt_download: n_tries += 1 if n_tries >= MAX_WHILE_ATTEMPTS: logging.warning('LoadProfileDM: Hit download retry limit.') attempt_download = False self.connection_error_occurred = True break if App.get_running_app().root.stop.is_set(): return try: with requests.Session() as req: page = req.get(csv_link, timeout=10, verify=ssl_verify, proxies=proxy_settings) if page.status_code != requests.codes.ok: page.raise_for_status() else: attempt_download = False except requests.HTTPError as e: logging.error('LoadProfileDM: {0}'.format(repr(e))) except requests.exceptions.ProxyError: logging.error('LoadProfileDM: Could not connect to proxy.') except requests.ConnectionError as e: logging.error( 'LoadProfileDM: Failed to establish a connection to the host server.' ) except requests.Timeout as e: logging.error('LoadProfileDM: The connection timed out.') except requests.RequestException as e: logging.error('LoadProfileDM: {0}'.format(repr(e))) except Exception as e: # Something else went wrong. logging.error( 'LoadProfileDM: An unexpected error has occurred. ({0})' .format(repr(e))) else: data_down = page.content.decode(page.encoding) csv_data = pd.read_csv(io.StringIO(data_down)) electricity_data = csv_data[[ 'Date/Time', 'Electricity:Facility [kW](Hourly)' ]] # Save to persistent object on disk. url_split = csv_link.split('/') destination_dir = os.path.join(DATA_HOME, 'load', 'residential', url_split[-2]) os.makedirs(destination_dir, exist_ok=True) destination_file = os.path.join(destination_dir, url_split[-1]) electricity_data.to_csv(destination_file, sep=',', index=False) popup = WarningPopup() popup.title = 'Success!' popup.popup_text.text = 'Load data successfully saved.' popup.open() logging.info( 'LoadProfileDM: Load data successfully saved.')
def _query_api_for_rate_structures(self, api_query): """Uses OpenEI API to query the rate structures for given EIA ID and populates rate structure RecycleView.""" ssl_verify, proxy_settings = check_connection_settings() attempt_download = True n_tries = 0 while attempt_download: n_tries += 1 if n_tries >= MAX_WHILE_ATTEMPTS: logging.warning('RateStructureDM: Hit download retry limit.') attempt_download = False break if App.get_running_app().root.stop.is_set(): return try: with requests.Session() as req: http_request = req.get(api_query, proxies=proxy_settings, timeout=10, verify=ssl_verify, stream=True) if http_request.status_code != requests.codes.ok: http_request.raise_for_status() else: attempt_download = False except requests.HTTPError as e: logging.error('RateStructureDM: {0}'.format(repr(e))) raise requests.ConnectionError except requests.exceptions.ProxyError: logging.error('RateStructureDM: Could not connect to proxy.') raise requests.ConnectionError except requests.ConnectionError as e: logging.error( 'RateStructureDM: Failed to establish a connection to the host server.' ) raise requests.ConnectionError except requests.Timeout as e: logging.error('RateStructureDM: The connection timed out.') raise requests.ConnectionError except requests.RequestException as e: logging.error('RateStructureDM: {0}'.format(repr(e))) raise requests.ConnectionError except Exception as e: # Something else went wrong. logging.error( 'RateStructureDM: An unexpected error has occurred. ({0})'. format(repr(e))) raise requests.ConnectionError else: structure_list = http_request.json()['items'] structure_df = pd.DataFrame.from_records(structure_list) structure_df.dropna(subset=['energyratestructure'], inplace=True) # Filter out entries whose energyratestructure array does not contain "rate" terms. mask = structure_df['energyratestructure'].apply(lambda x: all( ['rate' in hr.keys() for row in x for hr in row])) structure_df = structure_df[mask] structure_list = structure_df.to_dict(orient='records') # First, sort by effective date. # structure_list = sorted(structure_list, key=lambda x: (x['name'], x.get('startdate', np.nan))) structure_list = sorted( structure_list, key=lambda x: x.get('startdate', np.nan), reverse=True) # Then, sort by name. structure_list = sorted(structure_list, key=lambda x: x['name']) # Display name: Name (record['startdate']). effective_dates = [ '(Effective Date : {0})'.format( dt.datetime.fromtimestamp( record['startdate']).strftime('%m/%d/%Y')) if not np.isnan(record['startdate']) else '' for record in structure_list ] records = [{ 'name': record['name'] + ' ' + effective_dates[ix], 'record': record } for ix, record in enumerate(structure_list, start=0)] # records = sorted(records, key=lambda t: t['name']) self.rate_structure_rv.data = records self.rate_structure_rv.unfiltered_data = records logging.info( 'RateStructureDM: Retrieved utility rate structures.') fade_in_animation(self.rate_structure_select_bx) finally: self.loading_screen.dismiss()
def _download_utility_ref_table(self): """Downloads and builds the utility reference table from OpenEI.""" ssl_verify, proxy_settings = check_connection_settings() # Invester-owned utilities. attempt_download = True n_tries = 0 while attempt_download: n_tries += 1 if n_tries >= MAX_WHILE_ATTEMPTS: logging.warning('RateStructureDM: Hit download retry limit.') attempt_download = False break if App.get_running_app().root.stop.is_set(): return try: with requests.Session() as req: http_request = req.get(URL_OPENEI_IOU, proxies=proxy_settings, timeout=6, verify=ssl_verify, stream=True) if http_request.status_code != requests.codes.ok: http_request.raise_for_status() else: attempt_download = False except requests.HTTPError as e: logging.error('RateStructureDM: {0}'.format(repr(e))) except requests.exceptions.ProxyError: logging.error('RateStructureDM: Could not connect to proxy.') except requests.ConnectionError as e: logging.error( 'RateStructureDM: Failed to establish a connection to the host server.' ) except requests.Timeout as e: logging.error('RateStructureDM: The connection timed out.') except requests.RequestException as e: logging.error('RateStructureDM: {0}'.format(repr(e))) except Exception as e: # Something else went wrong. logging.error( 'RateStructureDM: An unexpected error has occurred. ({0})'. format(repr(e))) else: data_down = http_request.content.decode( http_request.apparent_encoding) data_iou = pd.read_csv(io.StringIO(data_down)) # Non-investor-owned utilities. attempt_download = True n_tries = 0 while attempt_download: n_tries += 1 if n_tries >= MAX_WHILE_ATTEMPTS: logging.warning('RateStructureDM: Hit download retry limit.') attempt_download = False break if App.get_running_app().root.stop.is_set(): return try: with requests.Session() as req: http_request = req.get(URL_OPENEI_NONIOU, proxies=proxy_settings, timeout=6, verify=ssl_verify, stream=True) if http_request.status_code != requests.codes.ok: http_request.raise_for_status() else: attempt_download = False except requests.HTTPError as e: logging.error('RateStructureDM: {0}'.format(repr(e))) except requests.exceptions.ProxyError: logging.error('RateStructureDM: Could not connect to proxy.') except requests.ConnectionError as e: logging.error( 'RateStructureDM: Failed to establish a connection to the host server.' ) except requests.Timeout as e: logging.error('RateStructureDM: The connection timed out.') except requests.RequestException as e: logging.error('RateStructureDM: {0}'.format(repr(e))) except Exception as e: # Something else went wrong. logging.error( 'RateStructureDM: An unexpected error has occurred. ({0})'. format(repr(e))) else: data_down = http_request.content.decode( http_request.apparent_encoding) data_noniou = pd.read_csv(io.StringIO(data_down)) try: df_combined = pd.concat([data_iou, data_noniou], ignore_index=True) except NameError: # Connection error prevented downloads. raise requests.ConnectionError else: # Add column for state/district name. df_combined['state name'] = df_combined['state'].apply( lambda state_abbr: STATE_ABBR_TO_NAME.get(state_abbr, '')) self.utility_ref_table = df_combined logging.info( 'RateStructureDM: Retrieved list of all utilities.')