def test_query_with_meta_false(conn, test_pd_df, kwargs): # test reading timeseries data (including subannual data) exp = IamDataFrame(test_pd_df, subannual='Year')\ .append(MODEL_B_DF, model='model_b', scenario='scen_a', region='World') # test method via Connection df = conn.query(meta=False, **kwargs) assert_iamframe_equal(df, exp.filter(**kwargs)) # test top-level method df = read_iiasa(TEST_API, meta=False, **kwargs) assert_iamframe_equal(df, exp.filter(**kwargs))
def get_production_capacities(conf, source='offline', verbose=True): """Read production capacities, either offline (locally) or online (from scenario explorer) """ # if offline source, read local data if source == 'offline': hourly_capacities = pd.read_csv( 'Input/ProductionCapacities.csv', index_col=0).iloc[conf['t_start']:conf['t_end'], :] # if online source, read data from openENTRANCE scenario explorer if source == 'online': openentrance_capacities = pyam.read_iiasa( 'openentrance', model=conf['openEntrance']['capacities']['model'], variable=conf['openEntrance']['capacities']['variable'], region=conf['openEntrance']['capacities']['region'], scenario=conf['openEntrance']['capacities']['scenario']) openentrance_capacities = openentrance_capacities.filter( year=conf['openEntrance']['capacities']['year']) if verbose: print('Production capacities (openENTRANCE):') print(openentrance_capacities.timeseries()) # try to match downloaded data to technologies specified in .yaml file. If that fails, use local data try: installed_capacities = { source: openentrance_capacities.filter( variable=conf['openEntrance']['capacities']['variable'] + source).timeseries()[int( conf['openEntrance']['capacities']['year'])][-1] for source in conf['openEntrance']['sources'] } except (IndexError, ValueError, AttributeError): warnings.warn( 'Capacities data from scenario explorer does not fit sources supplied in Settings.yaml - using mock-up data.' ) installed_capacities = { source: 1 for source in conf['openEntrance']['sources'] } # translate installed capacities to hourly capacities # for dispatchable sources, this is trivial; for non-dispatchable sources, use profiles supplied locally hourly_capacities = { source: np.repeat(installed_capacities[source], len(conf['T'])) if source in conf['openEntrance']['dispatchable_sources'] else pd.read_csv('input/' + source + '.csv', header=None).iloc[:, 0].values[conf['T']] * installed_capacities[source] for source in conf['openEntrance']['sources'] } return hourly_capacities
def test_query_year(conn, test_df_year, kwargs): # test reading timeseries data (`model_a` has only yearly data) exp = test_df_year.copy() for i in ['version'] + META_COLS: exp.set_meta(META_DF.iloc[[0, 1]][i]) # test method via Connection df = conn.query(model='model_a', **kwargs) assert_iamframe_equal(df, exp.filter(**kwargs)) # test top-level method df = read_iiasa(TEST_API, model='model_a', **kwargs) assert_iamframe_equal(df, exp.filter(**kwargs))
def download(): pyam.iiasa.Connection('openentrance') # read from database # select model, scenario, regions #! Adjust as soon as GENeSYS-MOD data available data_IAMC = pyam.read_iiasa('openentrance', model='GENeSYS-MOD 2.9.0-oe', scenario='Directed Transition 1.0', region="Norway") # covert data to pandas dataframe data = data_IAMC.as_pandas() return data
def test_query_with_meta_arg(conn, test_pd_df, kwargs): # test reading timeseries data (including subannual data) exp = IamDataFrame(test_pd_df, subannual='Year')\ .append(MODEL_B_DF, model='model_b', scenario='scen_a', region='World') for i in ['version', 'string']: exp.set_meta(META_DF.iloc[[0, 1, 3]][i]) # test method via Connection df = conn.query(meta=['string'], **kwargs) assert_iamframe_equal(df, exp.filter(**kwargs)) # test top-level method df = read_iiasa(TEST_API, meta=['string'], **kwargs) assert_iamframe_equal(df, exp.filter(**kwargs))
def test_query_with_meta_arg(conn, test_pd_df, meta, kwargs): # test reading timeseries data (including subannual data) exp = IamDataFrame(test_pd_df, subannual="Year").append( MODEL_B_DF, model="model_b", scenario="scen_a", region="World" ) for i in ["version", "string"]: exp.set_meta(META_DF.iloc[[0, 1, 3]][i]) # test method via Connection df = conn.query(meta=meta, **kwargs) assert_iamframe_equal(df, exp.filter(**kwargs)) # test top-level method df = read_iiasa(TEST_API, meta=meta, **kwargs) assert_iamframe_equal(df, exp.filter(**kwargs))
def test_query_non_default(conn, test_pd_df): # test reading timeseries data with non-default versions & index test_pd_df["subannual"] = "Year" test_pd_df["version"] = 1 df = pd.concat([test_pd_df[NON_DEFAULT_DF.columns], NON_DEFAULT_DF]) meta = META_DF.set_index("version", append=True) index = ["model", "scenario", "version"] exp = IamDataFrame(df, meta=meta, index=index, region="World") # test method via Connection df = conn.query(default=False) assert_iamframe_equal(df, exp) # test top-level method df = read_iiasa(TEST_API, default=False) assert_iamframe_equal(df, exp)
def get_initial_demand(conf, source='offline', verbose=True): """Read initial demand, either offline (locally) or online (from scenario explorer) """ # if offline source, read local data if source == 'offline': initial_demand = pd.read_csv( 'Input/InitialDemand.csv', index_col=0).iloc[conf['t_start']:conf['t_end'], :] # if online source, read data from openENTRANCE scenario explorer if source == 'online': initial_demand = pyam.read_iiasa( 'openentrance', model=conf['openEntrance']['initial_demand']['model'], variable=conf['openEntrance']['initial_demand']['variable'], region=conf['openEntrance']['initial_demand']['region'], scenario=conf['openEntrance']['initial_demand']['scenario']) initial_demand = initial_demand.filter( year=conf['openEntrance']['initial_demand']['year']) if verbose: print('Initial demand:') print(initial_demand.timeseries()) # set yearly aggregate yearly_aggregate = initial_demand.as_pandas()['value'].sum() # create normalised hourly pattern from offline data hourly_basis = pd.read_csv( 'Input/InitialDemand.csv', index_col=0).iloc[conf['t_start']:conf['t_end'], :] hourly_basis_normalised = hourly_basis['2050'] / hourly_basis[ '2050'].max() # create non-normalised hourly demand initial_demand = yearly_aggregate * hourly_basis_normalised initial_demand = initial_demand.to_numpy() return initial_demand
Import of Data for the SCOPE-SD Model. """ import pandas as pd import pyam import nomenclature as nc import xlwings # In settings it is stored where to store which variable of GENeSYS-MOD # in the Excel sheet of the SCOPE Model settings = pd.read_excel("settings.xlsx") # set up the connection to openENTRANCE with pyam # and read the GENeSYS-MOD conn = pyam.iiasa.Connection() conn.connect("openentrance") df = pyam.read_iiasa("openentrance", model="GENeSYS-MOD 2.9.0-oe") # Convert EJ/yr to TWh/yr df = df.convert_unit("EJ/yr", to="TWh/yr") # filter for the scenario specified in the settings sheet. # and filter for the year 2050 df = df.filter(scenario=settings["scenario"][0]) df = df.filter(year=[2050]) # read the excel workbook that is used by the SCOPE-Model scope = pd.read_excel("input_SCOPE_SD.xlsx", sheet_name="2050") # open the excel workbook that is used by the SCOPE-Model with xlwings, to # store later the new values in the sheets. wb = xlwings.Book("input_SCOPE_SD.xlsx")
import pyam # import nomenclature # 1) Installation process is already accomplished. # 2) Push results. This step is already completed. Data exists on the scenario explorer. # 3) Pull results from the scenario explorer pyam.iiasa.Connection('openentrance') # read from database # select model, scenario, regions, variable df = pyam.read_iiasa('openentrance', model='HEROSCARS v1.0') # show data print(df.head()) # save data as pandas dataframe data = df.as_pandas() # 4) Unit conversion using pyam #%% a. Using units already defined in pyam # filter Final Energy|Electricity|Profile from dataset obtained in step 3 Filter_A = df.filter(model='HEROSCARS v1.0', variable='Final Energy|Electricity|Profile', level='1-')
import pyam # read from the open platform (scenario explorer) _dataframe = pyam.read_iiasa( 'openentrance', model='GUSTO v1.0', scenario='CS3_2030oE_Storyline', region='Norway|Finmark', variable='LoadFactor|Electricity|Solar|Profile') _dataframe.as_pandas().to_excel('Solar.xlsx')
# -*- coding: utf-8 -*- """ Created on Fri Nov 13 21:39:05 2020 @author: pisciell """ import pyam # set username to get access to the scenario explorer user_name = 'myusername' user_password = '******' pyam.iiasa.set_config(user_name, user_password) pyam.iiasa.Connection('openentrance') df = pyam.read_iiasa('openentrance', model='GENeSYS-MOD 2.9.0-oe', scenario='Directed Transition 1.0', variable=[ 'SecondaryEnergy|Electricity|Oil', 'SecondaryEnergy|Electricity|Solar' ]) data = df.as_pandas() data.to_excel(r'C:\Users\pisciell.WIN-NTNU-NO\Desktop\oePathway.xlsx', index=False)
paramZone=paramZone[ ~paramZone.Name.isin(listdelete)] paramZone=paramZone.dropna() NbSkipRows=len(pd.read_excel(dataxls,'Parameter', index_col=False, skiprows=1, skip_footer = 10, header=None))-6 paramDate=pd.read_excel(dataxls,'Parameter', index_col=False, skiprows=NbSkipRows, skipfooter=5, header=None) years=pd.to_datetime(paramDate[2],format='%Y-%m-%d %H:%M:%S').dt.strftime('%Y').tolist() # upload of relevant data from platform # creation of a csv file and a pandas dataframe containing all data #########################################################" i=0 if cfg['mode']=='platform': for datagroup in cfg['datagroups'].keys(): groupdf=pyam.read_iiasa('openentrance',model=cfg['datagroups'][datagroup]['model'], variable=cfg['datagroups'][datagroup]['listvariables'], region=cfg['datagroups'][datagroup]['listregionsGET'],year=years[0], scenario=cfg['datagroups'][datagroup]['scenario'] ) if i==0: df=groupdf i=1 else: df=df.append(groupdf) else: file=cfg['inputfilepath']+cfg['inputdata'] df=pyam.IamDataFrame(data=file) #regional aggregations for datagroup in cfg['datagroups'].keys(): if isinstance(cfg['datagroups'][datagroup]['aggregateregions'],dict) :
pyam.iiasa.set_config(cfg['user'], cfg['password']) pyam.iiasa.Connection('openentrance') # upload of relevant data from openentrance scenarii # for variables : # -needed to compute technologies MaxPower: 'Capacity|Electricity|all technologies (available) # -needed to compute electricity demand: 'Final Energy|Electricity', 'Final Energy|Electricity|Heat', # 'Final Energy|Electricity|Transportation', (not yet available) (unit: PJ) # 'Final Energy|Electricity|Cooling (will not be available, value from plan4res DB will be used) # -for computing investment costs : 'CapitalCost|Electricity|<all technos> (not yet available) (unit: M€2015/MW) # -for computing emissions per technologies : CO2 Emmissions|Electricity|<allgenerationtechnologies> (not yet available) (unit: tons) # -for computing variable costs : VariableCost|Electricity||<all generation technologies> (not yet available) # -for pumping: Pumping Efficiency|Electricity|Hydro|Pumped Storage (not yet available - unit:%) df = pyam.read_iiasa('openentrance', model=cfg['model'], variable=cfg['listvariables'], region=cfg['listregionsScen'], year=cfg['years'], scenario=cfg['scenarios']) #conversion final energy from PJ to MWh df.convert_unit('PJ', to='MWh') #check wether this conversion already exist.... #regional aggregations for reg in cfg['aggregateregions'].keys(): df.aggregate_region(df.variable, region=reg, subregions=cfg['aggregateregions'][reg], append=True) #remove aggregated subregions