def wrap_calc_func(*args, **kwargs): pc = PerfCounter('%s.%s' % (func.__module__, func.__name__)) pc.display('enter') hash_data = _get_func_hash_data(func) cache_key = _calculate_cache_key(hash_data) assert 'variables' not in kwargs assert 'datasets' not in kwargs if not args and not kwargs: should_cache_func = True else: should_cache_func = False print('not caching func %s.%s' % (func.__module__, func.__name__)) if should_cache_func: ret = cache.get(cache_key) if ret is not None: # calcfuncs must not return None pc.display('cache hit') return ret if variables is not None: kwargs['variables'] = {x: get_variable(y) for x, y in variables.items()} if datasets is not None: datasets_to_load = set(list(datasets.values())) - set(_dataset_cache.keys()) if datasets_to_load: loaded_datasets = [] for dataset_name in datasets_to_load: ds_pc = PerfCounter('dataset %s' % dataset_name) df = load_datasets(dataset_name) ds_pc.display('loaded') loaded_datasets.append(df) del ds_pc for dataset_name, dataset in zip(datasets_to_load, loaded_datasets): _dataset_cache[dataset_name] = dataset kwargs['datasets'] = {ds_name: _dataset_cache[ds_url] for ds_name, ds_url in datasets.items()} ret = func(*args, **kwargs) pc.display('func ret') if should_cache_func: assert ret is not None cache.set(cache_key, ret, timeout=600) return ret
'jyrjola/aluesarjat/hginseutu_va_ve01_vaestoennuste_pks', ] import pandas as pd import scipy import importlib from utils import dict_merge from utils.quilt import load_datasets import aplans_graphs import plotly import plotly.graph_objs as go import cufflinks as cf kaup_rakennus_tyypeittain, pks_khk_paastot, vaestoennuste_pks = load_datasets(INPUT_DATASETS) plotly.offline.init_notebook_mode(connected=True) cf.set_config_file(offline=True) # %% df = kaup_rakennus_tyypeittain.copy() df.Vuosi = df.Vuosi.astype(int) buildings_by_type = df.pivot_table(index=['Vuosi', 'Rakennustyyppi'], columns='Muuttuja', values='value').reset_index() display(buildings_by_type) kaup_rakennus_tyypeittain = df[df.Rakennustyyppi == 'Kaikki yhteensä'].pivot(index='Vuosi', columns='Muuttuja', values='value') # %% df = vaestoennuste_pks df = df[(df.Laadintavuosi == 'Laadittu 2018') & (df.Sukupuoli == 'Molemmat sukupuolet') & (df.Ikä == 'Väestö yhteensä')] df = df[df.Alue == 'Helsinki']
def prepare_input_datasets(): global ghg_emissions ghg_in = load_datasets(INPUT_DATASETS) ghg_emissions = prepare_ghg_emissions_dataset(ghg_in)
def wrap_calc_func(*args, **kwargs): should_profile = os.environ.get('PROFILE_CALC', '').lower() in ('1', 'true', 'yes') only_if_in_cache = kwargs.pop('only_if_in_cache', False) skip_cache = kwargs.pop('skip_cache', False) var_store = kwargs.pop('variable_store', None) if should_profile: pc = PerfCounter('%s.%s' % (func.__module__, func.__name__)) pc.display('enter') hash_data = _get_func_hash_data(func, None) cache_key = _calculate_cache_key(func, hash_data, var_store=var_store) assert 'variables' not in kwargs assert 'datasets' not in kwargs unknown_kwargs = set(kwargs.keys()) - set(['step_callback']) if not args and not unknown_kwargs and not skip_cache: should_cache_func = True else: should_cache_func = False if should_cache_func: ret = cache.get(cache_key) if ret is not None: # calcfuncs must not return None if should_profile: pc.display('cache hit (%s)' % cache_key) return ret if only_if_in_cache: if should_profile: pc.display('cache miss so leaving as requested (%s)' % cache_key) return None if variables is not None: kwargs['variables'] = {x: get_variable(y, var_store=var_store) for x, y in variables.items()} if datasets is not None: datasets_to_load = set(list(datasets.values())) - set(_dataset_cache.keys()) if datasets_to_load: loaded_datasets = [] for dataset_name in datasets_to_load: if should_profile: ds_pc = PerfCounter('dataset %s' % dataset_name) df = load_datasets(dataset_name) if should_profile: ds_pc.display('loaded') del ds_pc loaded_datasets.append(df) for dataset_name, dataset in zip(datasets_to_load, loaded_datasets): _dataset_cache[dataset_name] = dataset kwargs['datasets'] = {ds_name: _dataset_cache[ds_url] for ds_name, ds_url in datasets.items()} ret = func(*args, **kwargs) if should_profile: pc.display('func ret') if should_cache_func: assert ret is not None cache.set(cache_key, ret, timeout=3600) return ret
def prepare_input_datasets(): global buildings_by_heating_method buildings_in = load_datasets(INPUT_DATASETS) buildings_by_heating_method = prepare_buildings_dataset(buildings_in)
def process_input_datasets(): global population_forecast pop_in = load_datasets(INPUT_DATASETS) population_forecast = prepare_population_forecast_dataset(pop_in)
'jyrjola/fingrid_hourly/power', 'jyrjola/energiateollisuus/electricity_production_hourly', 'jyrjola/energiateollisuus/electricity_production_fuels' ] from datetime import timedelta import pandas as pd import numpy as np import altair as alt from utils.quilt import load_datasets from data_import import statfi, energiateollisuus fuel_emission_factors = statfi.get_fuel_classification(include_units=True) energy_production = statfi.get_energy_production_stats() fg_hourly, et_hourly, et_fuels = load_datasets(INPUT_DATASETS, include_units=False) import plotly from plotly.offline import iplot import plotly.graph_objs as go import cufflinks as cf import aplans_graphs plotly.offline.init_notebook_mode(connected=True) cf.set_config_file(offline=True) # %% [markdown] # ## Fuel emission coefficients # # An emission coefficient is a value that relates the quantity of pollutants released to the atmosphere with an associated activity. Each of the fuel has a unit emissions coefficient that is based on the amount of gases with [global warming potential](https://en.wikipedia.org/wiki/Global_warming_potential) that are emitted. The emission coefficient is converted to match the mass of $\ce{CO2}$ that would produce the equivalent amount of GWP.
] import pandas as pd import pintpandas from utils.quilt import load_datasets import plotly import plotly.graph_objs as go import cufflinks as cf import aplans_graphs plotly.offline.init_notebook_mode(connected=True) cf.set_config_file(offline=True) dh_fuel_df, dh_production_df, fuel_classification, dh_demand_df, city_demand_df = load_datasets( INPUT_DATASETS) # %% # %% fuel_co2 = fuel_classification[['code', 'co2e_emission_factor', 'is_bio']].set_index('code') df = dh_fuel_df[dh_fuel_df.Operator == DISTRICT_HEATING_OPERATOR] df = df.merge(fuel_co2, how='left', left_on='StatfiFuelCode', right_index=True) df.co2e_emission_factor = df.co2e_emission_factor.astype('pint[t/TJ]') df.Value = df.Value.astype('pint[GWh]') operator_fuel_with_co2 = df.copy() df['Emissions'] = (df.Value * df.co2e_emission_factor).pint.to('tonne').pint.m df.loc[df.is_bio == True, 'Emissions'] = 0 emissions = df[df.Operator == DISTRICT_HEATING_OPERATOR].groupby(
from calc import calcfunc from calc.electricity import calculate_electricity_supply_emission_factor from components.cards import GraphCard from components.graphs import make_layout from utils.quilt import load_datasets from .base import Page DEFAULT_PRICE_PER_KWH = 12 INITIAL_INSTALL_PRICE = 5000 PRICE_PER_PEAK_KW = 1000 PEAK_WATTS_PER_M2 = 150 hel_buildings, hsy_buildings, fingrid_price = load_datasets([ 'jyrjola/karttahel/buildings', 'jyrjola/hsy/buildings', 'jyrjola/fingrid_hourly/price' ]) def read_nuuka_data(): DATA_DIR = 'data/nuuka' buildings = pd.read_parquet(DATA_DIR + '/buildings.parquet') sensors = pd.read_parquet(DATA_DIR + '/sensors.parquet') def is_building_needed(b_id): if os.path.exists('data/nuuka/%s.parquet' % b_id): return True else: return False
import math import pandas as pd import scipy.stats from variables import get_variable from utils.quilt import load_datasets import pintpandas # noqa INPUT_DATASETS = [ 'jyrjola/energiateollisuus/district_heating_fuel', 'jyrjola/energiateollisuus/district_heating_production', 'jyrjola/statfi/fuel_classification' ] dh_fuel_df, dh_production_df, fuel_classification = load_datasets( INPUT_DATASETS) def _prepare_fuel_emissions_dataset(fuel_use_df): return df HEAT_PUMP_COL = 'Lämmön talteenotto tai lämpöpumpun tuotanto' HEAT_DEMAND_COL = 'Käyttö' FUEL_NET_PRODUCTION_COL = 'Nettotuotanto polttoaineilla' PRODUCTION_LOSS_COL = 'Verkkohäviöt ja mittauserot' TOTAL_PRODUCTION_COL = 'Yhteensä' CHP_ELECTRICITY_PRODUCTION_COL = 'Kaukolämmön tuotantoon liittyvä sähkön nettotuotanto' ALL_FUEL_PRODUCTION_TOTAL_COL = 'Kaukolämmön ja yhteistuotantosähkön tuotantoon käytetyt polttoaineet yhteensä'