def get_power_unit_hydro(mastr_unit_hydro): """Get Wassereinheit (unit-hydro) from API using GetEinheitWasser. Parameters ---------- mastr_unit_hydro : object Wasser from EinheitMastrNummerId. Returns ------- unit_hydro : DataFrame Wassereinheit. """ data_version = get_data_version() try: c = client_bind.GetEinheitWasser(apiKey=api_key, marktakteurMastrNummer=my_mastr, einheitMastrNummer=mastr_unit_hydro) s = serialize_object(c) df = pd.DataFrame(list(s.items()), ) unit_hydro = df.set_index(list(df.columns.values)[0]).transpose() unit_hydro.reset_index() unit_hydro.index.names = ['lid'] unit_hydro['version'] = data_version unit_hydro['timestamp'] = str(datetime.datetime.now()) return unit_hydro except Exception as e: # log.info('Download failed for %s', mastr_unit_hydro) pass
def get_unit_hydro_eeg(mastr_hydro_eeg): """Get EEG-Anlage-Wasser from API using GetAnlageEegWasser. Parameters ---------- mastr_hydro_eeg : str MaStR EEG Nr. Returns ------- unit_hydro_eeg : DataFrame EEG-Anlage-Wasser. """ data_version = get_data_version() try: c = client_bind.GetAnlageEegWasser(apiKey=api_key, marktakteurMastrNummer=my_mastr, eegMastrNummer=mastr_hydro_eeg) # c['VerknuepfteEinheit'] = c['MaStR']['VerknuepfteEinheit'] # del c['MaStR'] s = serialize_object(c) df = pd.DataFrame(list(s.items()), ) unit_hydro_eeg = df.set_index(list(df.columns.values)[0]).transpose() unit_hydro_eeg.reset_index() unit_hydro_eeg.index.names = ['lid'] unit_hydro_eeg["version"] = data_version unit_hydro_eeg["timestamp"] = str(datetime.datetime.now()) return unit_hydro_eeg except Exception as e: # log.info('Download failed for %s', mastr_hydro_eeg) pass
def get_unit_gsgk_eeg(mastr_gsgk_eeg): """Get EEG-Anlage-GeoSolarthermieGrubenKlaerschlamm from API using GetAnlageEegGeoSolarthermieGrubenKlaerschlamm. Parameters ---------- mastr_gsgk_eeg : str MaStR EEG Nr. Returns ------- unit_gsgk_eeg : DataFrame EEG-Anlage-GeoSolarthermieGrubenKlaerschlamm. """ data_version = get_data_version() try: c = client_bind.GetAnlageEegGeoSolarthermieGrubenKlaerschlamm( apiKey=api_key, marktakteurMastrNummer=my_mastr, eegMastrNummer=mastr_gsgk_eeg ) s = serialize_object(c) df = pd.DataFrame(list(s.items()), ) unit_gsgk_eeg = df.set_index(list(df.columns.values)[0]).transpose() unit_gsgk_eeg.reset_index() unit_gsgk_eeg.index.names = ['lid'] unit_gsgk_eeg["version"] = data_version unit_gsgk_eeg["timestamp"] = str(datetime.datetime.now()) return unit_gsgk_eeg except Exception as e: # log.info('Download failed for %s', mastr_gsgk_eeg) pass
def get_daily_contingent(): """Return current contingent status. Returns ------- daily_contingent_limit : DataFrame AktuellesLimitTageskontingent. daily_contingent_status : DataFrame AktuellerStandTageskontingent. """ data_version = get_data_version() try: c = client.service.GetAktuellerStandTageskontingent( apiKey=api_key, marktakteurMastrNummer=my_mastr) s = serialize_object(c) df = pd.DataFrame(list(s.items()), ) daily_contingent = df.set_index(list(df.columns.values)[0]).transpose() daily_contingent['version'] = data_version daily_contingent['timestamp'] = str(datetime.datetime.now()) daily_contingent_limit = daily_contingent[ 'AktuellesLimitTageskontingent'].values[0] daily_contingent_status = daily_contingent[ 'AktuellerStandTageskontingent'].values[0] log.info( f'Daily contigent: {daily_contingent_status} from daily Limit: {daily_contingent_limit}' ) return daily_contingent, daily_contingent_limit, daily_contingent_status except Exception as e: log.info(f'Failed to get daily contingent.')
def get_power_unit_wind(mastr_unit_wind): """Get Windeinheit (unit-wind) from API using GetEinheitWind. Parameters ---------- mastr_unit_wind : object Wind from EinheitMastrNummerId. Returns ------- unit_wind : DataFrame Windeinheit. """ data_version = get_data_version() try: c = client_bind.GetEinheitWind(apiKey=api_key, marktakteurMastrNummer=my_mastr, einheitMastrNummer=mastr_unit_wind) c = disentangle_manufacturer(c) del c['Hersteller'] s = serialize_object(c) df = pd.DataFrame(list(s.items()), ) unit_wind = df.set_index(list(df.columns.values)[0]).transpose() unit_wind.reset_index() unit_wind.index.names = ['lid'] unit_wind['version'] = data_version unit_wind['timestamp'] = str(datetime.datetime.now()) return unit_wind except Exception as e: # log.info('Download failed for %s', mastr_unit_wind) pass
def get_power_unit_solar(mastr_unit_solar): """Get Solareinheit from API using GetEinheitSolar. Parameters ---------- mastr_unit_solar : object Solar from EinheitMastrNummerId. Returns ------- unit_solar : DataFrame Solareinheit. """ # with mp.Lock(): # log.info('downloading data unit... %s', mastr_unit_solar) data_version = get_data_version() unit_solar = pd.DataFrame() try: c = client_bind.GetEinheitSolar(apiKey=api_key, marktakteurMastrNummer=my_mastr, einheitMastrNummer=mastr_unit_solar) s = serialize_object(c) df = pd.DataFrame(list(s.items()), ) unit_solar = df.set_index(list(df.columns.values)[0]).transpose() unit_solar.reset_index() unit_solar.index.names = ['lid'] unit_solar['version'] = data_version unit_solar['timestamp'] = str(datetime.datetime.now()) return unit_solar except Exception as e: log.info('Download failed for {}: {}'.format(mastr_unit_solar, e))
def setup_storage_units(overwrite=True): data_version = get_data_version() if overwrite: if os.path.isfile(fname_storage): remove_csv(fname_storage) if os.path.isfile(fname_power_unit): power_unit = read_power_units(fname_power_unit) if not power_unit.empty: power_unit = power_unit.drop_duplicates() power_unit_storage = power_unit[power_unit.Einheittyp == 'Stromspeichereinheit'] power_unit_storage.index.names = ['see_id'] power_unit_storage.reset_index() power_unit_storage.index.names = ['id'] if not power_unit_storage.empty: write_to_csv(fname_storage_unit, power_unit_storage) else: log.info('No storage units in this dataset. Storage units can be found starting at index: approx. 1 220 000') power_unit.iloc[0:0] return power_unit_storage else: log.info('no storageunits found') return pd.DataFrame() else: power_unit_solar = read_power_units(name_storage) return power_unit_solar
def get_power_unit_nuclear(mastr_unit_nuclear): """Get Kernenergieeinheit from API using GetEinheitKernenergie. Parameters ---------- mastr_unit_nuclear : object Kernenergie from EinheitMastrNummerId. Returns ------- unit_nuclear : DataFrame Kernenergieeinheit. """ data_version = get_data_version() try: c = client_bind.GetEinheitKernkraft( apiKey=api_key, marktakteurMastrNummer=my_mastr, einheitMastrNummer=mastr_unit_nuclear) s = serialize_object(c) df = pd.DataFrame(list(s.items()), ) unit_nuclear = df.set_index(list(df.columns.values)[0]).transpose() unit_nuclear.reset_index() unit_nuclear.index.names = ['lid'] unit_nuclear['version'] = data_version unit_nuclear['timestamp'] = str(datetime.datetime.now()) return unit_nuclear except Exception as e: # log.info('Download failed for %s', mastr_unit_nuclear) pass
def get_unit_solar_eeg(mastr_solar_eeg): """Get EEG-Anlage-Solar from API using GetAnlageEegSolar. Parameters ---------- mastr_solar_eeg : str MaStR EEG Nr. Returns ------- unit_solar_eeg : DataFrame EEG-Anlage-Solar. """ data_version = get_data_version() try: c = client_bind.GetAnlageEegSolar(apiKey=api_key, marktakteurMastrNummer=my_mastr, eegMastrNummer=mastr_solar_eeg) s = serialize_object(c) df = pd.DataFrame(list(s.items()), ) unit_solar_eeg = df.set_index(list(df.columns.values)[0]).transpose() unit_solar_eeg.reset_index() unit_solar_eeg.index.names = ['lid'] unit_solar_eeg["version"] = data_version unit_solar_eeg["timestamp"] = str(datetime.datetime.now()) except Exception as e: return None return unit_solar_eeg
def get_power_unit_gsgk(mastr_unit_gsgk): """Get GeoSolarthermieGrubenKlaerschlammeinheit from API using GetEinheitGeoSolarthermieGrubenKlaerschlamm. Parameters ---------- mastr_unit_gsgk : object Biomass from EinheitMastrNummerId. Returns ------- unit_gsgk : DataFrame GeoSolarthermieGrubenKlaerschlammeinheit. """ data_version = get_data_version() try: c = client_bind.GetEinheitGeoSolarthermieGrubenKlaerschlamm(apiKey=api_key, marktakteurMastrNummer=my_mastr, einheitMastrNummer=mastr_unit_gsgk) s = serialize_object(c) df = pd.DataFrame(list(s.items()), ) unit_gsgk = df.set_index(list(df.columns.values)[0]).transpose() unit_gsgk.reset_index() unit_gsgk.index.names = ['lid'] unit_gsgk['version'] = data_version unit_gsgk['timestamp'] = str(datetime.datetime.now()) return unit_gsgk except Exception as e: # log.info('Download failed for %s', mastr_unit_gsgk) pass
def get_unit_wind_permit(mastr_wind_permit): """Get Genehmigung-Wind-Wind from API using GetEinheitGenehmigung. Parameters ---------- mastr_wind_permit : str Genehmigungnummer MaStR Returns ------- unit_wind_permit : DataFrame Genehmigung-Einheit-Wind. """ data_version = get_data_version() c = client_bind.GetEinheitGenehmigung(apiKey=api_key, marktakteurMastrNummer=my_mastr, genMastrNummer=mastr_wind_permit) s = serialize_object(c) df = pd.DataFrame(list(s.items()), ) unit_wind_permit = df.set_index(list(df.columns.values)[0]).transpose() unit_wind_permit.reset_index() unit_wind_permit.index.names = ['lid'] unit_wind_permit["version"] = data_version unit_wind_permit["timestamp"] = str(datetime.datetime.now()) unit_wind_permit = unit_wind_permit.replace('\r', '', regex=True) return unit_wind_permit
def get_power_unit_wind(mastr_unit_wind, mastr_unit_eeg, eeg=False): """Get Windeinheit from API using GetEinheitWind. Parameters ---------- mastr_unit_wind : object Wind from EinheitMastrNummerId. Returns ------- unit_wind : DataFrame Windeinheit. """ data_version = get_data_version() unit_wind = pd.DataFrame() unit_wind_eeg = pd.DataFrame() if eeg== True: try: c = client_bind.GetAnlageEegWind(apiKey=api_key, marktakteurMastrNummer=my_mastr, eegMastrNummer=mastr_unit_eeg) s = serialize_object(c) dicts = list((s['VerknuepfteEinheit'][0]).items())[0] s['VerknuepfteEinheit']= dicts[1] s['EinheitMastrNummer'] = s.pop('VerknuepfteEinheit') """for i in s.items(): if isinstance(i[1], OrderedDict): title = re.findall('[A-Z][^A-Z]*', str(i[0])) if len(i[1].keys()) == len(title):""" #s[i[0]] = df = pd.DataFrame(list(s.items()), ) unit_wind_eeg = df.set_index(list(df.columns.values)[0]).transpose() unit_wind_eeg.reset_index() unit_wind_eeg.index.names = ['lid'] unit_wind_eeg["version"] = data_version unit_wind_eeg["timestamp"] = str(datetime.datetime.now()) except Exception as e: log.info('Download eeg failed for %s', mastr_unit_wind) try: c = client_bind.GetEinheitWind(apiKey=api_key, marktakteurMastrNummer=my_mastr, einheitMastrNummer=mastr_unit_wind) c = disentangle_manufacturer(c) del c['Hersteller'] s = serialize_object(c) df = pd.DataFrame(list(s.items()), ) unit_wind = df.set_index(list(df.columns.values)[0]).transpose() unit_wind.reset_index() unit_wind.index.names = ['lid'] unit_wind['version'] = data_version unit_wind['timestamp'] = str(datetime.datetime.now()) except Exception as e: log.info('Download failed for %s', mastr_unit_wind) return unit_wind, unit_wind_eeg
def get_unit_storage(mastr_unit_storage): """Get Solareinheit from API using GetEinheitStromspeicher.""" data_version = get_data_version() try: c = client_bind.GetEinheitStromSpeicher(apiKey=api_key, marktakteurMastrNummer=my_mastr, einheitMastrNummer=mastr_unit_storage) s = serialize_object(c) df = pd.DataFrame(list(s.items()), ) unit_storage = df.set_index(list(df.columns.values)[0]).transpose() unit_storage.reset_index() unit_storage.index.names = ['lid'] unit_storage['version'] = data_version unit_storage['timestamp'] = str(datetime.datetime.now()) write_to_csv(fname_storage, unit_storage) except Exception as e: return return unit_storage
def get_mastr_time(): """Return current time from MaStR. Returns ------- mastr_time : DataFrame LokaleUhrzeit. """ data_version = get_data_version() try: c = client.service.GetLokaleUhrzeit() s = serialize_object(c) df = pd.DataFrame(list(s.items()), ) mastr_time = df.set_index(list(df.columns.values)[0]).transpose() mastr_time['version'] = data_version mastr_time['timestamp'] = str(datetime.datetime.now()) mastr_time_str = mastr_time['LokaleUhrzeit'].values[0] log.info(f'MaStR-Time: {mastr_time_str}') return mastr_time, mastr_time_str except Exception as e: log.info(f'Failed to get time.')
def get_power_unit(start_from, energy_carrier, datum='1900-01-01 00:00:00.00000', limit=API_MAX_DEMANDS): """Get Stromerzeugungseinheit from API using GetGefilterteListeStromErzeuger. Parameters ---------- start_from : int Skip first entries. datum: String the starting datestring to retrieve data, can be used for updating a data set limit : int Number of power unit to get (default: 2000) """ power_unit = pd.DataFrame() # status = 'InBetrieb' # power = 30 try: c = client_bind.GetGefilterteListeStromErzeuger( apiKey=api_key, marktakteurMastrNummer=my_mastr, # einheitBetriebsstatus=status, startAb=start_from, energietraeger=energy_carrier, limit=limit #bruttoleistungGroesser=power #datumAb = datum ) s = serialize_object(c) power_unit = pd.DataFrame(s['Einheiten']) power_unit.index.names = ['lid'] # power_unit['db_offset'] = [i for i in range(start_from, start_from+len(power_unit))] power_unit['version'] = get_data_version() power_unit['timestamp'] = str(datetime.now()) return [start_from, power_unit] except Exception as e: log.info(e)
def get_mastr_time_auth(): """Return current time with authentication from MaStR. Returns ------- mastr_time_auth_str : Str LokaleUhrzeitMitAuthentifizierung. """ data_version = get_data_version() try: c = client.service.GetLokaleUhrzeitMitAuthentifizierung(apiKey=api_key) s = serialize_object(c) df = pd.DataFrame(list(s.items()), ) mastr_time_auth = df.set_index(list(df.columns.values)[0]).transpose() mastr_time_auth['version'] = data_version mastr_time_auth['timestamp'] = str(datetime.datetime.now()) mastr_time_auth_str = mastr_time_auth[ 'LokaleUhrzeitMitAuthentifizierung'].values[0] log.info(f'MaStR-Time-Auth: {mastr_time_auth_str}') return mastr_time_auth except Exception as e: log.info(f'Failed to get time with authentication.')
def download_unit_solar_eeg(): """Download unit_solar_eeg using GetAnlageEegSolar request. Parameters ---------- sublist : list list to process in parallel parallelism : int number of threads """ data_version = get_data_version() unit_solar = setup_power_unit_solar() unit_solar_list = unit_solar['EegMastrNummer'].values.tolist() unit_solar_list_len = len(unit_solar_list) for i in range(0, unit_solar_list_len, 1): try: unit_solar_eeg = get_unit_solar_eeg(unit_solar_list[i]) write_to_csv(fname_solar_eeg, unit_solar_eeg) except: log.exception( f'Download failed unit_solar_eeg ({i}): {unit_solar_list[i]}')