def get_power_unit_hydro(mastr_unit_hydro):
"""Get Wassereinheit (unit-hydro) from API using GetEinheitWasser.
Parameters
----------
mastr_unit_hydro : object
Wasser from EinheitMastrNummerId.
Returns
-------
unit_hydro : DataFrame
Wassereinheit.
"""
data_version = get_data_version()
try:
c = client_bind.GetEinheitWasser(apiKey=api_key,
marktakteurMastrNummer=my_mastr,
einheitMastrNummer=mastr_unit_hydro)
s = serialize_object(c)
df = pd.DataFrame(list(s.items()), )
unit_hydro = df.set_index(list(df.columns.values)[0]).transpose()
unit_hydro.reset_index()
unit_hydro.index.names = ['lid']
unit_hydro['version'] = data_version
unit_hydro['timestamp'] = str(datetime.datetime.now())
return unit_hydro
except Exception as e:
# log.info('Download failed for %s', mastr_unit_hydro)
pass
def get_unit_hydro_eeg(mastr_hydro_eeg):
"""Get EEG-Anlage-Wasser from API using GetAnlageEegWasser.
Parameters
----------
mastr_hydro_eeg : str
MaStR EEG Nr.
Returns
-------
unit_hydro_eeg : DataFrame
EEG-Anlage-Wasser.
"""
data_version = get_data_version()
try:
c = client_bind.GetAnlageEegWasser(apiKey=api_key,
marktakteurMastrNummer=my_mastr,
eegMastrNummer=mastr_hydro_eeg)
# c['VerknuepfteEinheit'] = c['MaStR']['VerknuepfteEinheit']
# del c['MaStR']
s = serialize_object(c)
df = pd.DataFrame(list(s.items()), )
unit_hydro_eeg = df.set_index(list(df.columns.values)[0]).transpose()
unit_hydro_eeg.reset_index()
unit_hydro_eeg.index.names = ['lid']
unit_hydro_eeg["version"] = data_version
unit_hydro_eeg["timestamp"] = str(datetime.datetime.now())
return unit_hydro_eeg
except Exception as e:
# log.info('Download failed for %s', mastr_hydro_eeg)
pass
def get_unit_gsgk_eeg(mastr_gsgk_eeg):
"""Get EEG-Anlage-GeoSolarthermieGrubenKlaerschlamm from API using GetAnlageEegGeoSolarthermieGrubenKlaerschlamm.
Parameters
----------
mastr_gsgk_eeg : str
MaStR EEG Nr.
Returns
-------
unit_gsgk_eeg : DataFrame
EEG-Anlage-GeoSolarthermieGrubenKlaerschlamm.
"""
data_version = get_data_version()
try:
c = client_bind.GetAnlageEegGeoSolarthermieGrubenKlaerschlamm(
apiKey=api_key,
marktakteurMastrNummer=my_mastr,
eegMastrNummer=mastr_gsgk_eeg
)
s = serialize_object(c)
df = pd.DataFrame(list(s.items()), )
unit_gsgk_eeg = df.set_index(list(df.columns.values)[0]).transpose()
unit_gsgk_eeg.reset_index()
unit_gsgk_eeg.index.names = ['lid']
unit_gsgk_eeg["version"] = data_version
unit_gsgk_eeg["timestamp"] = str(datetime.datetime.now())
return unit_gsgk_eeg
except Exception as e:
# log.info('Download failed for %s', mastr_gsgk_eeg)
pass
def get_daily_contingent():
"""Return current contingent status.
Returns
-------
daily_contingent_limit : DataFrame
AktuellesLimitTageskontingent.
daily_contingent_status : DataFrame
AktuellerStandTageskontingent.
"""
data_version = get_data_version()
try:
c = client.service.GetAktuellerStandTageskontingent(
apiKey=api_key, marktakteurMastrNummer=my_mastr)
s = serialize_object(c)
df = pd.DataFrame(list(s.items()), )
daily_contingent = df.set_index(list(df.columns.values)[0]).transpose()
daily_contingent['version'] = data_version
daily_contingent['timestamp'] = str(datetime.datetime.now())
daily_contingent_limit = daily_contingent[
'AktuellesLimitTageskontingent'].values[0]
daily_contingent_status = daily_contingent[
'AktuellerStandTageskontingent'].values[0]
log.info(
f'Daily contigent: {daily_contingent_status} from daily Limit: {daily_contingent_limit}'
)
return daily_contingent, daily_contingent_limit, daily_contingent_status
except Exception as e:
log.info(f'Failed to get daily contingent.')
def get_power_unit_wind(mastr_unit_wind):
"""Get Windeinheit (unit-wind) from API using GetEinheitWind.
Parameters
----------
mastr_unit_wind : object
Wind from EinheitMastrNummerId.
Returns
-------
unit_wind : DataFrame
Windeinheit.
"""
data_version = get_data_version()
try:
c = client_bind.GetEinheitWind(apiKey=api_key,
marktakteurMastrNummer=my_mastr,
einheitMastrNummer=mastr_unit_wind)
c = disentangle_manufacturer(c)
del c['Hersteller']
s = serialize_object(c)
df = pd.DataFrame(list(s.items()), )
unit_wind = df.set_index(list(df.columns.values)[0]).transpose()
unit_wind.reset_index()
unit_wind.index.names = ['lid']
unit_wind['version'] = data_version
unit_wind['timestamp'] = str(datetime.datetime.now())
return unit_wind
except Exception as e:
# log.info('Download failed for %s', mastr_unit_wind)
pass
def get_power_unit_solar(mastr_unit_solar):
"""Get Solareinheit from API using GetEinheitSolar.
Parameters
----------
mastr_unit_solar : object
Solar from EinheitMastrNummerId.
Returns
-------
unit_solar : DataFrame
Solareinheit.
"""
# with mp.Lock():
# log.info('downloading data unit... %s', mastr_unit_solar)
data_version = get_data_version()
unit_solar = pd.DataFrame()
try:
c = client_bind.GetEinheitSolar(apiKey=api_key,
marktakteurMastrNummer=my_mastr,
einheitMastrNummer=mastr_unit_solar)
s = serialize_object(c)
df = pd.DataFrame(list(s.items()), )
unit_solar = df.set_index(list(df.columns.values)[0]).transpose()
unit_solar.reset_index()
unit_solar.index.names = ['lid']
unit_solar['version'] = data_version
unit_solar['timestamp'] = str(datetime.datetime.now())
return unit_solar
except Exception as e:
log.info('Download failed for {}: {}'.format(mastr_unit_solar, e))
def setup_storage_units(overwrite=True):
data_version = get_data_version()
if overwrite:
if os.path.isfile(fname_storage):
remove_csv(fname_storage)
if os.path.isfile(fname_power_unit):
power_unit = read_power_units(fname_power_unit)
if not power_unit.empty:
power_unit = power_unit.drop_duplicates()
power_unit_storage = power_unit[power_unit.Einheittyp == 'Stromspeichereinheit']
power_unit_storage.index.names = ['see_id']
power_unit_storage.reset_index()
power_unit_storage.index.names = ['id']
if not power_unit_storage.empty:
write_to_csv(fname_storage_unit, power_unit_storage)
else:
log.info('No storage units in this dataset. Storage units can be found starting at index: approx. 1 220 000')
power_unit.iloc[0:0]
return power_unit_storage
else:
log.info('no storageunits found')
return pd.DataFrame()
else:
power_unit_solar = read_power_units(name_storage)
return power_unit_solar
def get_power_unit_nuclear(mastr_unit_nuclear):
"""Get Kernenergieeinheit from API using GetEinheitKernenergie.
Parameters
----------
mastr_unit_nuclear : object
Kernenergie from EinheitMastrNummerId.
Returns
-------
unit_nuclear : DataFrame
Kernenergieeinheit.
"""
data_version = get_data_version()
try:
c = client_bind.GetEinheitKernkraft(
apiKey=api_key,
marktakteurMastrNummer=my_mastr,
einheitMastrNummer=mastr_unit_nuclear)
s = serialize_object(c)
df = pd.DataFrame(list(s.items()), )
unit_nuclear = df.set_index(list(df.columns.values)[0]).transpose()
unit_nuclear.reset_index()
unit_nuclear.index.names = ['lid']
unit_nuclear['version'] = data_version
unit_nuclear['timestamp'] = str(datetime.datetime.now())
return unit_nuclear
except Exception as e:
# log.info('Download failed for %s', mastr_unit_nuclear)
pass
def get_unit_solar_eeg(mastr_solar_eeg):
"""Get EEG-Anlage-Solar from API using GetAnlageEegSolar.
Parameters
----------
mastr_solar_eeg : str
MaStR EEG Nr.
Returns
-------
unit_solar_eeg : DataFrame
EEG-Anlage-Solar.
"""
data_version = get_data_version()
try:
c = client_bind.GetAnlageEegSolar(apiKey=api_key,
marktakteurMastrNummer=my_mastr,
eegMastrNummer=mastr_solar_eeg)
s = serialize_object(c)
df = pd.DataFrame(list(s.items()), )
unit_solar_eeg = df.set_index(list(df.columns.values)[0]).transpose()
unit_solar_eeg.reset_index()
unit_solar_eeg.index.names = ['lid']
unit_solar_eeg["version"] = data_version
unit_solar_eeg["timestamp"] = str(datetime.datetime.now())
except Exception as e:
return None
return unit_solar_eeg
def get_power_unit_gsgk(mastr_unit_gsgk):
"""Get GeoSolarthermieGrubenKlaerschlammeinheit from API using GetEinheitGeoSolarthermieGrubenKlaerschlamm.
Parameters
----------
mastr_unit_gsgk : object
Biomass from EinheitMastrNummerId.
Returns
-------
unit_gsgk : DataFrame
GeoSolarthermieGrubenKlaerschlammeinheit.
"""
data_version = get_data_version()
try:
c = client_bind.GetEinheitGeoSolarthermieGrubenKlaerschlamm(apiKey=api_key,
marktakteurMastrNummer=my_mastr,
einheitMastrNummer=mastr_unit_gsgk)
s = serialize_object(c)
df = pd.DataFrame(list(s.items()), )
unit_gsgk = df.set_index(list(df.columns.values)[0]).transpose()
unit_gsgk.reset_index()
unit_gsgk.index.names = ['lid']
unit_gsgk['version'] = data_version
unit_gsgk['timestamp'] = str(datetime.datetime.now())
return unit_gsgk
except Exception as e:
# log.info('Download failed for %s', mastr_unit_gsgk)
pass
def get_unit_wind_permit(mastr_wind_permit):
"""Get Genehmigung-Wind-Wind from API using GetEinheitGenehmigung.
Parameters
----------
mastr_wind_permit : str
Genehmigungnummer MaStR
Returns
-------
unit_wind_permit : DataFrame
Genehmigung-Einheit-Wind.
"""
data_version = get_data_version()
c = client_bind.GetEinheitGenehmigung(apiKey=api_key,
marktakteurMastrNummer=my_mastr,
genMastrNummer=mastr_wind_permit)
s = serialize_object(c)
df = pd.DataFrame(list(s.items()), )
unit_wind_permit = df.set_index(list(df.columns.values)[0]).transpose()
unit_wind_permit.reset_index()
unit_wind_permit.index.names = ['lid']
unit_wind_permit["version"] = data_version
unit_wind_permit["timestamp"] = str(datetime.datetime.now())
unit_wind_permit = unit_wind_permit.replace('\r', '', regex=True)
return unit_wind_permit
def get_power_unit_wind(mastr_unit_wind, mastr_unit_eeg, eeg=False):
"""Get Windeinheit from API using GetEinheitWind.
Parameters
----------
mastr_unit_wind : object
Wind from EinheitMastrNummerId.
Returns
-------
unit_wind : DataFrame
Windeinheit.
"""
data_version = get_data_version()
unit_wind = pd.DataFrame()
unit_wind_eeg = pd.DataFrame()
if eeg== True:
try:
c = client_bind.GetAnlageEegWind(apiKey=api_key,
marktakteurMastrNummer=my_mastr,
eegMastrNummer=mastr_unit_eeg)
s = serialize_object(c)
dicts = list((s['VerknuepfteEinheit'][0]).items())[0]
s['VerknuepfteEinheit']= dicts[1]
s['EinheitMastrNummer'] = s.pop('VerknuepfteEinheit')
"""for i in s.items():
if isinstance(i[1], OrderedDict):
title = re.findall('[A-Z][^A-Z]*', str(i[0]))
if len(i[1].keys()) == len(title):"""
#s[i[0]] =
df = pd.DataFrame(list(s.items()), )
unit_wind_eeg = df.set_index(list(df.columns.values)[0]).transpose()
unit_wind_eeg.reset_index()
unit_wind_eeg.index.names = ['lid']
unit_wind_eeg["version"] = data_version
unit_wind_eeg["timestamp"] = str(datetime.datetime.now())
except Exception as e:
log.info('Download eeg failed for %s', mastr_unit_wind)
try:
c = client_bind.GetEinheitWind(apiKey=api_key,
marktakteurMastrNummer=my_mastr,
einheitMastrNummer=mastr_unit_wind)
c = disentangle_manufacturer(c)
del c['Hersteller']
s = serialize_object(c)
df = pd.DataFrame(list(s.items()), )
unit_wind = df.set_index(list(df.columns.values)[0]).transpose()
unit_wind.reset_index()
unit_wind.index.names = ['lid']
unit_wind['version'] = data_version
unit_wind['timestamp'] = str(datetime.datetime.now())
except Exception as e:
log.info('Download failed for %s', mastr_unit_wind)
return unit_wind, unit_wind_eeg
def get_unit_storage(mastr_unit_storage):
"""Get Solareinheit from API using GetEinheitStromspeicher."""
data_version = get_data_version()
try:
c = client_bind.GetEinheitStromSpeicher(apiKey=api_key,
marktakteurMastrNummer=my_mastr,
einheitMastrNummer=mastr_unit_storage)
s = serialize_object(c)
df = pd.DataFrame(list(s.items()), )
unit_storage = df.set_index(list(df.columns.values)[0]).transpose()
unit_storage.reset_index()
unit_storage.index.names = ['lid']
unit_storage['version'] = data_version
unit_storage['timestamp'] = str(datetime.datetime.now())
write_to_csv(fname_storage, unit_storage)
except Exception as e:
return
return unit_storage
def get_mastr_time():
"""Return current time from MaStR.
Returns
-------
mastr_time : DataFrame
LokaleUhrzeit.
"""
data_version = get_data_version()
try:
c = client.service.GetLokaleUhrzeit()
s = serialize_object(c)
df = pd.DataFrame(list(s.items()), )
mastr_time = df.set_index(list(df.columns.values)[0]).transpose()
mastr_time['version'] = data_version
mastr_time['timestamp'] = str(datetime.datetime.now())
mastr_time_str = mastr_time['LokaleUhrzeit'].values[0]
log.info(f'MaStR-Time: {mastr_time_str}')
return mastr_time, mastr_time_str
except Exception as e:
log.info(f'Failed to get time.')
def get_power_unit(start_from,
energy_carrier,
datum='1900-01-01 00:00:00.00000',
limit=API_MAX_DEMANDS):
"""Get Stromerzeugungseinheit from API using GetGefilterteListeStromErzeuger.
Parameters
----------
start_from : int
Skip first entries.
datum: String
the starting datestring to retrieve data, can be used for updating a data set
limit : int
Number of power unit to get (default: 2000)
"""
power_unit = pd.DataFrame()
# status = 'InBetrieb'
# power = 30
try:
c = client_bind.GetGefilterteListeStromErzeuger(
apiKey=api_key,
marktakteurMastrNummer=my_mastr,
# einheitBetriebsstatus=status,
startAb=start_from,
energietraeger=energy_carrier,
limit=limit
#bruttoleistungGroesser=power
#datumAb = datum
)
s = serialize_object(c)
power_unit = pd.DataFrame(s['Einheiten'])
power_unit.index.names = ['lid']
# power_unit['db_offset'] = [i for i in range(start_from, start_from+len(power_unit))]
power_unit['version'] = get_data_version()
power_unit['timestamp'] = str(datetime.now())
return [start_from, power_unit]
except Exception as e:
log.info(e)
def get_mastr_time_auth():
"""Return current time with authentication from MaStR.
Returns
-------
mastr_time_auth_str : Str
LokaleUhrzeitMitAuthentifizierung.
"""
data_version = get_data_version()
try:
c = client.service.GetLokaleUhrzeitMitAuthentifizierung(apiKey=api_key)
s = serialize_object(c)
df = pd.DataFrame(list(s.items()), )
mastr_time_auth = df.set_index(list(df.columns.values)[0]).transpose()
mastr_time_auth['version'] = data_version
mastr_time_auth['timestamp'] = str(datetime.datetime.now())
mastr_time_auth_str = mastr_time_auth[
'LokaleUhrzeitMitAuthentifizierung'].values[0]
log.info(f'MaStR-Time-Auth: {mastr_time_auth_str}')
return mastr_time_auth
except Exception as e:
log.info(f'Failed to get time with authentication.')
def download_unit_solar_eeg():
"""Download unit_solar_eeg using GetAnlageEegSolar request.
Parameters
----------
sublist : list
list to process in parallel
parallelism : int
number of threads
"""
data_version = get_data_version()
unit_solar = setup_power_unit_solar()
unit_solar_list = unit_solar['EegMastrNummer'].values.tolist()
unit_solar_list_len = len(unit_solar_list)
for i in range(0, unit_solar_list_len, 1):
try:
unit_solar_eeg = get_unit_solar_eeg(unit_solar_list[i])
write_to_csv(fname_solar_eeg, unit_solar_eeg)
except:
log.exception(
f'Download failed unit_solar_eeg ({i}): {unit_solar_list[i]}')
