def find_markets(self, grid_locations=None, voltage='high'):
elec_filter_base = [
{'filter': 'equals', 'args': ['unit', 'kilowatt hour']},
{'filter': 'startswith', 'args': ['name', 'market for electricity, {} voltage'.format(voltage)]},
{'filter': 'doesnt_contain_any', 'args': ['name', ['aluminium industry',
'internal use in coal mining',
'Swiss Federal Railways',
'label-certified',
'electricity, from municipal waste incineration'
]]},
]
if grid_locations:
elec_filter_base += [{'filter': 'either', 'args':
[{'filter': 'equals', 'args': ['location', x]} for x in grid_locations]
}]
elec_filter = create_filter_from_description(elec_filter_base)
elec_list = list(w.get_many(self.loader.database.db, *elec_filter))
grid_dict = {
e['location']: ecoinvent_to_image_locations(e['location'])[0]
for e in elec_list if 'IAI' not in e['location']
and e['location'] not in ['GLO']
and len(ecoinvent_to_image_locations(e['location'])) == 1
}
return elec_list, grid_dict
def confirm_setup(self):
print("This is the last page")
this_filter = create_filter_from_description(parse_filter_widget(self.filter_widget))
db = findMainWindow().loader.database.db
this_item_set = [WurstProcess(x) for x in w.get_many(db, *this_filter)]
#this_item = w.get_one(db, *this_filter)
print(this_item_set)
item_string = ""
for n, this_item in enumerate(this_item_set):
item_string += "{} ({}) [{}]".format(this_item['name'], this_item['unit'], this_item['location'])
if n != len(this_item_set):
item_string += "\n"
self.processLabel.setText(item_string)
if len(this_item_set) > 1:
self.processDescriptionLabel.setText('Base processes: ')
else:
self.processDescriptionLabel.setText('Base process: ')
location_list = ", ".join([x['display'] for x in self.location_widget.checked_items])
self.locationLabel.setText(location_list)
def classify_technologies(self, elec_list=None):
if not elec_list:
elec_list = self.markets
assert elec_list
exchange_filter = [{'filter': 'equals', 'args': ['unit', 'kilowatt hour']},
{'filter': 'exclude', 'args': [
{'filter': 'startswith', 'args': ['name', 'market ']},
]},
]
wurst_exchange_filter = create_filter_from_description(exchange_filter)
exchange_dict = defaultdict(list)
for x in elec_list:
exc_list = list(w.get_many(x['exchanges'], *wurst_exchange_filter))
for e in exc_list:
e['image_location'] = get_loc(x['location'])[0]
e['image_classification'] = classify_by_name(e['name'])
assert not exchange_dict[x['location']]
exchange_dict[x['location']] = exc_list
return exchange_dict
def find_markets(self, grid_locations=None):
elec_filter_base = [
{'filter': 'equals', 'args': ['unit', 'kilowatt hour']},
{'filter': 'startswith', 'args': ['name', 'market for electricity, high voltage']},
{'filter': 'doesnt_contain_any', 'args': ['name', ['Swiss Federal Railways', 'label-certified']]},
]
if grid_locations:
elec_filter_base += [{'filter': 'either', 'args':
[{'filter': 'equals', 'args': ['location', x]} for x in grid_locations]
}]
elec_filter = create_filter_from_description(elec_filter_base)
elec_list = list(w.get_many(self.loader.database.db, *elec_filter))
grid_dict = {
e['location']: get_loc(e['location'])[0]
for e in elec_list if 'IAI' not in e['location']
and e['location'] not in ['GLO']
and len(get_loc(e['location'])) == 1
}
return elec_list, grid_dict
def format(self, recipe_item):
function = recipe_item['function']
original_kwargs = recipe_item.get('kwargs', {})
copy_kwargs = copy(original_kwargs)
for k, v in copy_kwargs.items():
if "_filter" in k:
print(k)
this_filter = create_filter_from_description(v)
these_items = list(
w.get_many(self.loader.database.db, *this_filter))
string = '\n'.join([
'{name} ({unit}) [{location}]'.format(**x)
for x in these_items
])
copy_kwargs[k] = string
elif isinstance(v, list):
print("{} is a list".format(v))
if k not in ['database', 'db']:
copy_kwargs[k] = ', '.join(v)
if function == 'transfer_pv':
if 'factor' in copy_kwargs.keys():
copy_kwargs['amount'] = copy_kwargs['factor'] * 100
copy_kwargs['unit'] = "%"
else:
copy_kwargs['unit'] = ""
mask = self.masks[function]
return mask.format(**copy_kwargs)
def find_possible_additional_electricity_exchanges(process, database, include_transmission=False, include_glo=False):
assert isinstance(database, FuturaDatabase), "database needs to be a futura FuturaDatabase object"
assert process['name'].startswith('market') and not process['name'].startswith('market group'), \
'This function only works with market processes (not market groups or production processes)'
technosphere_exchanges = [e for e in w.technosphere(process)]
result = []
# result_dict = {}
for g, v in groupby(technosphere_exchanges, lambda x: (x['product'],
x['location'],
x['unit'])):
# print(g)
v_list = list(v)
names = [e['name'] for e in v_list if e['name'] != process['name']]
possible_additions_filter = []
possible_additions_filter += [w.equals('unit', g[2])]
possible_additions_filter += [w.equals('reference product', g[0])]
possible_additions_filter += [exclude(w.startswith('name', 'market'))]
possible_additions_filter += [exclude(w.equals('location', 'RoW'))]
possible_additions_filter += [w.doesnt_contain_any('name', ['production mix',
'pulp',
'ethanol',
'petroleum refinery',
'blast furnace',
'coal gas',
'bagasse',
'aluminium industry'
])]
possible_additions_filter += [w.doesnt_contain_any('name', names)]
if not include_transmission:
possible_additions_filter += [w.doesnt_contain_any('name', ['transport', 'transmission'])]
if include_glo:
possible_additions_filter += [w.either(*[w.equals('location', g[1]), w.equals('location', 'GLO')])]
else:
possible_additions_filter += [w.equals('location', g[1])]
possibles = list(w.get_many(database.db, *possible_additions_filter))
if len(possibles) > 0:
for p in possibles:
e = check_exchange_in_activity(process, p)
if e:
result.append(e)
# result.extend(possibles)
# result_dict[g] = possibles
return result # , result_dict
def update_electricity_markets(self, grid_locations=None):
image_electricity_market_df = self.image_loader.get_mixes(self.year)
#print(image_electricity_market_df)
changes = {}
# update high voltage markets:
for ds in w.get_many(self.loader.database.db, *self.electricity_filter(grid_locations, voltage='high')):
print(ds['name'])
changes[ds['code']] = {}
changes[ds['code']].update({('meta data', x): ds[x] for x in ['name', 'location']})
changes[ds['code']].update({('original exchanges', k): v for k, v in get_exchange_amounts(ds).items()})
changes[ds['code']]['pretend'] = pretend_to_add_new_datasets_to_electricity_market(ds, self.loader.database.db, image_electricity_market_df)
#changes[ds['code']].update({('updated exchanges', k): v for k, v in get_exchange_amounts(ds).items()})
return changes
def test_filter(self):
test_filter = self.create_filter()
db = findMainWindow().loader.database.db
result = list(w.get_many(db, *test_filter))
if len(result) == 0:
message = "No results found!"
elif len(result) == 1:
message = "{} result found - {} ({}) [{}]".format(
len(result), result[0]['name'], result[0]['unit'],
result[0]['location'])
else:
message = "{} results found, including {} ({}) [{}]".format(
len(result), result[0]['name'], result[0]['unit'],
result[0]['location'])
self.test_result.setText(message)
def restrict_locations(self):
base_filter = parse_filter_widget(self.filter_widget)
no_location_filter = [x for x in base_filter if x['args'][0] != 'location']
this_filter = create_filter_from_description(base_filter)
no_location = create_filter_from_description(no_location_filter)
db = findMainWindow().loader.database.db
this_item = w.get_one(db, *this_filter)
item_location = this_item['location']
other_items = w.get_many(db, *no_location)
other_locations = [x['location'] for x in other_items]
other_locations = [x for x in other_locations if x != 'RoW']
locations = list(set(other_locations + [item_location]))
print(locations)
self.location_widget.find_and_disable(locations)
def pretend_to_add_new_datasets_to_electricity_market(ds, db, df):
# This function adds new electricity datasets to a market based on image results. We pass not only a dataset to modify, but also a pandas dataframe containing the new electricity mix information, and the db from which we should find the datasets
# find out which image regions correspond to our dataset:
image_locations = ecoinvent_to_image_locations(ds['location'])
# here we find the mix of technologies in the new market and how much they contribute:
mix = find_average_mix(df.loc[image_locations]) # could be several image locations - we just take the average
# here we find the datasets that will make up the mix for each technology
mock_ds = {'name':ds.get('name'),
'location':ds.get('location'),
'code': ds.get('code'),
'exchanges':[]
}
datasets = {}
for i in mix.index:
if mix[i] != 0:
# print('Next Technology: ',i)
# First try to find a dataset that is from that location (or image region for new datasets):
datasets[i] = find_ecoinvent_electricity_datasets_in_same_ecoinvent_location(i, ds['location'], db)
# print('First round: ',i, [(ds['name'], ds['location']) for ds in datasets[i]])
# If this doesn't work, we try to take a dataset from another ecoinvent region within the same image region
if len(datasets[i]) == 0:
datasets[i] = find_ecoinvent_electricity_datasets_in_image_location(i, ds['location'], db)
# print('Second round: ',i, [(ds['name'], ds['location']) for ds in datasets[i]])
# If even this doesn't work, try taking a global datasets
if len(datasets[i]) == 0:
datasets[i] = find_ecoinvent_electricity_datasets_in_same_ecoinvent_location(i, 'GLO', db)
# print('Third round: ',i, [(ds['name'], ds['location']) for ds in datasets[i]])
# if no global dataset available, we just take the average of all datasets we have:
if len(datasets[i]) == 0:
datasets[i] = find_ecoinvent_electricity_datasets_in_all_locations(i, db)
# print('Fourth round: ',i, [(ds['name'], ds['location']) for ds in datasets[i]])
# If we still can't find a dataset, we just take the global market group
if len(datasets[i]) == 0:
print('No match found for location: ', ds['location'], ' Technology: ', i,
'. Taking global market group for electricity')
datasets[i] = [x for x in w.get_many(db, *[w.equals('name', 'market group for electricity, high voltage'),
w.equals('location', 'GLO')])]
# Now we add the new exchanges:
for i in mix.index:
if mix[i] != 0:
total_amount = mix[i]
amount = total_amount / len(datasets[i])
for dataset in datasets[i]:
mock_ds['exchanges'].append({
'amount': amount,
'unit': dataset['unit'],
'input': (dataset['database'], dataset['code']),
'type': 'technosphere',
'name': dataset['name'],
'location': dataset['location']
})
# confirm that exchanges sum to 1!
sum = np.sum([exc['amount'] for exc in w.technosphere(mock_ds, *[w.equals('unit', 'kilowatt hour'),
w.doesnt_contain_any('name', [
'market for electricity, high voltage'])])])
if round(sum, 4) != 1.00: print(ds['location'], " New exchanges don't add to one! something is wrong!", sum)
return mock_ds
def regionalisation_wizard(self):
print('Starting the wizard')
rw = RegionalisationWizard()
if rw.exec_():
print('Wizard Complete')
filter_description = parse_filter_widget(rw.filter_widget)
print(filter_description)
this_filter = create_filter_from_description(filter_description)
db = findMainWindow().loader.database.db
this_item_set = [
WurstProcess(x) for x in w.get_many(db, *this_filter)
]
#this_item = w.get_one(db, *this_filter)
#print(this_item)
location_code_list = [
x['code'] for x in rw.location_widget.checked_items
]
if len(this_item_set) == 1:
recipe_entry = {
'action':
'regionalisation',
'tasks': [{
'function': 'create_regional_activities_from_filter',
'kwargs': {
'new_regions': location_code_list,
'base_activity_filter': filter_description,
}
}]
}
else:
recipe_entry = {
'action':
'regionalisation',
'tasks': [{
'function': 'regionalise_multiple_processes',
'kwargs': {
'locations': location_code_list,
'base_activity_filter': filter_description,
}
}]
}
loader = findMainWindow().loader
executor = FuturaRecipeExecutor(findMainWindow().loader)
executor.execute_recipe_action(recipe_entry)
loader.recipe['actions'].append(recipe_entry)
signals.update_recipe.emit()
# create_regional_activities(this_item, location_code_list, db)
no_location_filter_description = [
x for x in filter_description if x['args'][0] != 'location'
]
new_locations_filter_description = [{
'filter': 'equals',
'args': ['location', l]
} for l in location_code_list]
new_locations_filter_description = [{
'filter':
'either',
'args':
new_locations_filter_description
}]
new_locations_filter_description = no_location_filter_description + new_locations_filter_description
new_locations_filter = create_filter_from_description(
new_locations_filter_description)
result = list(w.get_many(db, *new_locations_filter))
print([
"{} ({}) [{}]".format(x['name'], x['unit'], x['location'])
for x in result
])
message = QtWidgets.QMessageBox()
message.setText("{} new processes created".format(len(result)))
message.setInformativeText("\n".join([
"{} ({}) [{}]".format(x['name'], x['unit'], x['location'])
for x in result
]))
message.exec_()
signals.reset_status_message.emit()
else:
print('Wizard Cancelled')
def test_regionalisation(loader):
database = loader.database
original_database_size = len(database.db)
locations = ['GB']
base_activity_filter = [{
'filter': 'equals',
'args': ['database', 'Carma CCS']
}, {
'filter': 'equals',
'args': ['unit', 'kilowatt hour']
}, {
'filter':
'either',
'args': [{
'filter': 'contains',
'args': ['name', 'hard coal']
}, {
'filter': 'contains',
'args': ['name', 'Hard coal']
}]
}, {
'filter': 'equals',
'args': ['location', 'GLO']
}]
progress_message = 'coal CCS for GB'
regionalise_multiple_processes(database, locations, base_activity_filter,
progress_message)
intermediate_database_size = len(database.db)
assert intermediate_database_size > original_database_size
gb_activity_filter = [{
'filter': 'equals',
'args': ['database', 'Carma CCS']
}, {
'filter': 'equals',
'args': ['unit', 'kilowatt hour']
}, {
'filter':
'either',
'args': [{
'filter': 'contains',
'args': ['name', 'hard coal']
}, {
'filter': 'contains',
'args': ['name', 'Hard coal']
}]
}, {
'filter': 'equals',
'args': ['location', 'GB']
}]
gb = create_filter_from_description(gb_activity_filter)
gb_list = list(w.get_many(database.db, *gb))
assert gb_list
locations = ['DE']
base_activity_filter = [{
'filter': 'equals',
'args': ['database', 'Carma CCS']
}, {
'filter': 'equals',
'args': ['unit', 'kilowatt hour']
}, {
'filter':
'either',
'args': [{
'filter': 'contains',
'args': ['name', 'lignite']
}, {
'filter': 'contains',
'args': ['name', 'Lignite']
}]
}, {
'filter': 'equals',
'args': ['location', 'GLO']
}]
progress_message = 'lignite CCS for DE'
regionalise_multiple_processes(database, locations, base_activity_filter,
progress_message)
final_database_size = len(database.db)
assert final_database_size > intermediate_database_size
de_activity_filter = [{
'filter': 'equals',
'args': ['database', 'Carma CCS']
}, {
'filter': 'equals',
'args': ['unit', 'kilowatt hour']
}, {
'filter':
'either',
'args': [{
'filter': 'contains',
'args': ['name', 'lignite']
}, {
'filter': 'contains',
'args': ['name', 'Lignite']
}]
}, {
'filter': 'equals',
'args': ['location', 'DE']
}]
de = create_filter_from_description(de_activity_filter)
de_list = list(w.get_many(database.db, *de))
assert de_list
def classify_exchanges(self, elec_list=None):
if not elec_list:
elec_list = self.markets
assert elec_list
exchange_filter = [{'filter': 'equals', 'args': ['unit', 'kilowatt hour']},
{'filter': 'exclude', 'args': [
{'filter': 'startswith', 'args': ['name', 'market ']},
]},
]
wurst_exchange_filter = create_filter_from_description(exchange_filter)
full_exchange_list = []
for x in elec_list:
full_exchange_list.extend(list(w.get_many(x['exchanges'], *wurst_exchange_filter)))
exchange_tuples = {(x['name'], x['location']) for x in full_exchange_list}
# return exchange_names
# assert 0
convert_dict = OrderedDict()
convert_dict["import"] = "Imports"
convert_dict.update(
{
'coal, post': 'Coal CCS',
'coal, oxy': 'Coal CCS',
'coal, pre': 'Coal CCS',
'lignite, post': 'Coal CCS',
'lignite, pre': 'Coal CCS',
'lignite, oxy': 'Coal CCS',
'gas, post': 'Natural gas CCS',
'gas, pre': 'Natural gas CCS',
'gas, oxy': 'Natural gas CCS',
'wood burning power plant 20 MW, truck 25km, post': 'Biomass CCS',
'co-generation, biogas': 'Biomass CHP',
'co-generation, wood': 'Biomass CHP',
'biogas': 'Biomass ST',
'biogas': 'Biomass ST',
'wood': 'Biomass ST',
'co-generation, hard coal': 'Coal CHP',
'co-generation, lignite': 'Coal CHP',
'hard coal': 'Coal ST',
'lignite': 'Coal ST',
'hydro': 'Hydro',
'co-generation, natural gas': 'Natural gas CHP',
'natural gas, combined cycle': 'Natural gas CC',
'natural gas': 'Natural gas OC',
'nuclear': 'Nuclear',
'co-generation, oil': 'Oil CHP',
'oil': 'Oil ST',
'solar': 'Solar PV',
'offshore': 'Wind offshore',
'onshore': 'Wind onshore',
'peat': 'Other',
'blast furnace gas': 'Other',
'geothermal': 'Other',
'coal': 'Coal ST',
'diesel': 'Other',
'digester sludge': 'Other',
'CC plant': 'Other',
'BIGCC': 'IGCC',
}
)
# translate all exchanges to their IMAGE equivalents
exchange_dict = {}
for e in exchange_tuples:
found = False
for n in convert_dict.keys():
if n in e[0]:
exchange_dict[e] = convert_dict[n]
found = True
break
if not found:
exchange_dict[e] = 'Non-Specified'
return exchange_dict
def update_grid(self, grid_locations=None):
if not grid_locations:
grid_locations = self.all_locations
if isinstance(grid_locations, str):
grid_locations = [grid_locations]
assert isinstance(grid_locations, list)
#get electricity market(s)
elec_filter_base = [{
'filter': 'equals',
'args': ['unit', 'kilowatt hour']
}, {
'filter':
'startswith',
'args': ['name', 'market for electricity, high voltage']
}, {
'filter':
'doesnt_contain_any',
'args': ['name', ['Swiss Federal Railways', 'label-certified']]
}, {
'filter':
'either',
'args': [{
'filter': 'equals',
'args': ['location', x]
} for x in grid_locations]
}]
elec_filter = create_filter_from_description(elec_filter_base)
elec_list = list(w.get_many(self.loader.database.db, *elec_filter))
# get exchanges to categorise
exchange_filter = [
{
'filter': 'equals',
'args': ['unit', 'kilowatt hour']
},
{
'filter': 'exclude',
'args': [
{
'filter': 'startswith',
'args': ['name', 'market ']
},
]
},
]
wurst_exchange_filter = create_filter_from_description(exchange_filter)
full_exchange_list = []
for x in elec_list:
full_exchange_list.extend(
list(w.get_many(x['exchanges'], *wurst_exchange_filter)))
exchange_names = set([x['name'] for x in full_exchange_list])
convert_dict = OrderedDict()
convert_dict["import"] = "Imports"
convert_dict.update({
"coal": "Coal",
"lignite": "Coal",
"oil": "Oil",
"natural gas": "Natural Gas",
"wood": "Combustible Renewables",
"biogas": "Combustible Renewables",
"peat": "Other Combustibles",
"blast furnace gas": "Other Combustibles",
"nuclear": "Nuclear",
"hydro": "Hydro",
"wind": "Wind",
"solar": "Solar",
"geothermal": "Geothermal",
})
# translate all exchanges to their IEA equivalents
exchange_dict = {}
for e in exchange_names:
found = False
for n in convert_dict.keys():
if n in e:
exchange_dict[e] = convert_dict[n]
found = True
break
if not found:
exchange_dict[e] = 'Non-Specified'
last_fm = None
for market in elec_list:
stratification_dict = self.aggregated_data.xs(
market['location'][:2]).to_dict()['Latest Year Total']
fm = FuturaMarket(market, self.loader.database)
# create a dataframe of market production volumes by exchange
pv_df = pd.DataFrame([{
'input': k,
'production volume': v['production volume']
} for k, v in fm.process_dict.items()])
# add a Group column to classify each exchange to an IEA type
pv_df['Group'] = pv_df['input'].apply(
lambda x: exchange_dict.get(x, None))
grand_total = pv_df['production volume'].sum()
# figure out how to stratify the data based on the proportion of production within IEA groups
stratification_data = {}
for g, v in pv_df.groupby('Group'):
this_total = v['production volume'].sum()
stratification_data[g] = {}
# print(v)
for row_index, row in v.iterrows():
if this_total != 0:
stratification_data[g][row[
'input']] = row['production volume'] / this_total
else:
stratification_data[g][row['input']] = 0
# multiply these proportions by the actual new grid mix sections
actual_stratification = {
k: v * grand_total
for k, v in stratification_dict.items()
}
final_dict = {}
for k, v in stratification_data.items():
this_pv = actual_stratification[k]
for x, n in v.items():
final_dict[x] = n * this_pv
# apply the new numbers to the FuturaMarket
for k, v in final_dict.items():
fm.set_pv(k, v)
fm.relink()
print('Updated grid mix for {}'.format(
cc.convert(market['location'], to="short_name")))