def setUp(self):
super().setUp()
self.ureg_bak = get_application_registry()
self.ureg = UnitRegistry(None)
self.ureg.define("foo = []")
self.ureg.define("bar = foo / 2")
set_application_registry(self.ureg)
assert get_application_registry() is self.ureg
def test_update_saved_registries():
ureg1 = get_application_registry()
ureg2 = get_application_registry()
new = UnitRegistry()
new.define("foo = []")
set_application_registry(new)
assert ureg1.Unit("foo") == ureg2.Unit("foo")
def class_tiny_app_registry():
ureg_bak = pint.get_application_registry()
ureg = pint.UnitRegistry(None)
ureg.define("foo = []")
ureg.define("bar = foo / 2")
pint.set_application_registry(ureg)
assert pint.get_application_registry() is ureg
yield ureg
pint.set_application_registry(ureg_bak)
def bldg_model_to_idf_no_exception(bldg_model, idf_file_path, profiles_files_handler, custom_config):
"""
For each building defined it creates building geometry, construction and internal conditions and writes the
assembled information to and idf file.
:param args: list with [bldg_model: BuildingModel, idf_file_path, profiles_file_handler, custom_config], for last three see arguments of cesarp.eplus_adapter.CesarIDFWriter()
:param idf_file_path: full filepath to idf file to be created
:return: tuple(fid, True, weather_file) when idf writing was successful, tuple(fid, False, None) otherwise
"""
unit_reg = pint.get_application_registry()
logger = logging.getLogger(__name__)
try:
logger.info(f"create idf and get weather file for building fid {bldg_model.fid}, {idf_file_path}")
my_idf_writer = CesarIDFWriter(idf_file_path, unit_reg, profiles_files_handler, custom_config=custom_config)
my_idf_writer.write_bldg_model(bldg_model)
return (True, idf_file_path, bldg_model.site.weather_file_path)
except Exception as ex:
fid = bldg_model.fid if bldg_model else None
if os.path.isfile(idf_file_path):
os.remove(idf_file_path)
logger.error(f"Could not write IDF for {fid}")
logger.exception(ex)
return (False, None, None)
def add_test_data(scen: Scenario):
# New sets
t_foo = ["foo{}".format(i) for i in (1, 2, 3)]
t_bar = ["bar{}".format(i) for i in (4, 5, 6)]
t = t_foo + t_bar
y = list(map(str, range(2000, 2051, 10)))
# Add to scenario
scen.init_set("t")
scen.add_set("t", t)
scen.init_set("y")
scen.add_set("y", y)
# Data
ureg = pint.get_application_registry()
x = Quantity(
xr.DataArray(np.random.rand(len(t), len(y)), coords=[("t", t), ("y", y)]),
units=ureg.kg,
)
# As a pd.DataFrame with units
x_df = x.to_series().rename("value").reset_index()
x_df["unit"] = "kg"
scen.init_par("x", ["t", "y"])
scen.add_par("x", x_df)
return t, t_foo, t_bar, x
def test_issue1175(self):
import pickle
foo1 = get_application_registry().Quantity(1, "s")
foo2 = pickle.loads(pickle.dumps(foo1))
self.assertIsInstance(foo1, foo2.__class__)
self.assertIsInstance(foo2, foo1.__class__)
def test_apply_units(data, caplog):
# Unpack
*_, x = data
registry = pint.get_application_registry()
# Brute-force replacement with incompatible units
with assert_logs(caplog, "Replace 'kilogram' with incompatible 'liter'"):
result = computations.apply_units(x, 'litres')
assert result.attrs['_unit'] == registry.Unit('litre')
# No change in values
assert_series_equal(result.to_series(), x.to_series())
# Compatible units: magnitudes are also converted
with assert_logs(caplog, "Convert 'kilogram' to 'metric_ton'",
at_level=logging.DEBUG):
result = computations.apply_units(x, 'tonne')
assert result.attrs['_unit'] == registry.Unit('tonne')
assert_series_equal(result.to_series(), x.to_series() * 0.001)
# Remove unit
x.attrs['_unit'] = registry.Unit('dimensionless')
caplog.clear()
result = computations.apply_units(x, 'kg')
# Nothing logged when _unit attr is missing
assert len(caplog.messages) == 0
assert result.attrs['_unit'] == registry.Unit('kg')
assert_series_equal(result.to_series(), x.to_series())
def add_test_data(scen):
# New sets
t_foo = ['foo{}'.format(i) for i in (1, 2, 3)]
t_bar = ['bar{}'.format(i) for i in (4, 5, 6)]
t = t_foo + t_bar
y = list(map(str, range(2000, 2051, 10)))
# Add to scenario
scen.init_set('t')
scen.add_set('t', t)
scen.init_set('y')
scen.add_set('y', y)
# Data
ureg = pint.get_application_registry()
x = xr.DataArray(np.random.rand(len(t), len(y)),
coords=[t, y],
dims=['t', 'y'],
attrs={'_unit': ureg.Unit('kg')})
x = Quantity(x)
# As a pd.DataFrame with units
x_df = x.to_series().rename('value').reset_index()
x_df['unit'] = 'kg'
scen.init_par('x', ['t', 'y'])
scen.add_par('x', x_df)
return t, t_foo, t_bar, x
def setUp(self):
super().setUp()
self.ureg_bak = get_application_registry()
self.ureg1 = UnitRegistry(None)
self.ureg1.define("foo = [dim1]")
self.ureg1.define("bar = foo / 2")
self.ureg2 = UnitRegistry(None)
self.ureg2.define("foo = [dim2]")
self.ureg2.define("bar = foo / 3")
def ureg():
"""Application-wide units registry."""
registry = pint.get_application_registry()
# Used by .compat.ixmp, .compat.pyam
registry.define("USD = [USD]")
registry.define("case = [case]")
yield registry
def parse_units(units_series):
"""Return a :class:`pint.Unit` for a :class:`pd.Series` of strings."""
unit = pd.unique(units_series)
if len(unit) > 1:
raise ValueError(f'mixed units {list(unit)}')
registry = pint.get_application_registry()
# Helper method to return an intelligible exception
def invalid(unit):
chars = ''.join(c for c in '-?$' if c in unit)
msg = (f'unit {repr(unit)} cannot be parsed; contains invalid '
f'character(s) {repr(chars)}')
return ValueError(msg)
# Helper method to add unit definitions
def define_unit_parts(expr):
# Split possible compound units
for part in expr.split('/'):
try:
registry.parse_units(part)
except pint.UndefinedUnitError:
# Part was unparseable; define it
definition = f'{part} = [{part}]'
log.info(f'Add unit definition: {definition}')
# This line will fail silently for parts like 'G$' containing
# characters like '$' that are discarded by pint
registry.define(definition)
# Parse units
try:
unit = clean_units(unit[0])
unit = registry.parse_units(unit)
except IndexError:
# Quantity has no unit
unit = registry.parse_units('')
except pint.UndefinedUnitError:
try:
# Unit(s) do not exist; define them in the UnitRegistry
define_unit_parts(unit)
# Try to parse again
unit = registry.parse_units(unit)
except (pint.UndefinedUnitError, pint.RedefinitionError):
# Handle the silent failure of define(), above; or
# define_unit_parts didn't work
raise invalid(unit)
except AttributeError:
# Unit contains a character like '-' that throws off pint
# NB this 'except' clause must be *after* UndefinedUnitError, since
# that is a subclass of AttributeError.
raise invalid(unit)
return unit
def ureg():
"""Session-scoped :class:`pint.UnitRegistry` with units needed by tests."""
registry = pint.get_application_registry()
for unit in "USD", "case":
try:
registry.define(f"{unit} = [{unit}]")
except pint.DefinitionSyntaxError:
# Already defined
pass
yield registry
def collect_units(*args):
"""Return an list of '_unit' attributes for *args*."""
registry = pint.get_application_registry()
for arg in args:
if '_unit' in arg.attrs:
# Convert units if necessary
if isinstance(arg.attrs['_unit'], str):
arg.attrs['_unit'] = registry.parse_units(arg.attrs['_unit'])
else:
log.debug('assuming {} is unitless'.format(arg))
arg.attrs['_unit'] = registry.parse_units('')
return [arg.attrs['_unit'] for arg in args]
def protect_pint_app_registry():
"""Protect pint's application registry.
Use this fixture on tests which invoke code that calls
:meth:`pint.set_application_registry`. It ensures that the environment for
other tests is not altered.
"""
import pint
# Use deepcopy() in case the wrapped code modifies the application
# registry without swapping out the UnitRegistry instance for a different
# one
saved = deepcopy(pint.get_application_registry())
yield
pint.set_application_registry(saved)
def parse_units(units_series):
"""Return a :class:`pint.Unit` for a :class:`pd.Series` of strings."""
ureg = pint.get_application_registry()
unit = pd.unique(units_series)
if len(unit) > 1:
raise ValueError(f'mixed units {list(unit)!r}')
# Helper method to return an intelligible exception
def invalid(unit):
chars = ''.join(c for c in '-?$' if c in unit)
return ValueError(("unit {!r} cannot be parsed; contains invalid "
"character(s) {!r}").format(unit, chars))
# Parse units
try:
unit = clean_units(unit[0])
unit = ureg.parse_units(unit)
except IndexError:
# Quantity has no unit
unit = ureg.parse_units('')
except pint.UndefinedUnitError:
# Unit(s) do not exist; define them in the UnitRegistry
# Split possible compound units
for u in unit.split('/'):
if u in dir(ureg):
# Unit already defined
continue
definition = f'{u} = [{u}]'
log.info(f'Add unit definition: {definition}')
# This line will fail silently for units like 'G$'
ureg.define(definition)
# Try to parse again
try:
unit = ureg.parse_units(unit)
except pint.UndefinedUnitError:
# Handle the silent failure of define(), above
raise invalid(unit) from None
except AttributeError:
# Unit contains a character like '-' that throws off pint
# NB this 'except' clause must be *after* UndefinedUnitError, since
# that is a subclass of AttributeError.
raise invalid(unit)
return unit
def custom_ureg_global():
import pint
ureg = pint.UnitRegistry()
ureg.define("test_unit = 123 kg")
prev = pint.get_application_registry()
try:
# pint.__version__ > 0.18
prev = prev.get()
except AttributeError:
pass
pint.set_application_registry(ureg)
yield ureg
pint.set_application_registry(prev)
def configure(path=None, **config):
"""Configure reporting globally.
Modifies global variables that affect the behaviour of *all* Reporters and
computations, namely :obj:`.RENAME_DIMS` and :obj:`.REPLACE_UNITS`.
Valid configuration keys—passed as *config* keyword arguments—include:
Other Parameters
----------------
units : mapping
Configuration for handling of units. Valid sub-keys include:
- **replace** (mapping of str -> str): replace units before they are
parsed by :doc:`pint <pint:index>`. Added to :obj:`.REPLACE_UNITS`.
- **define** (:class:`str`): block of unit definitions, added to the
:mod:`pint` application registry so that units are recognized. See
the pint :ref:`documentation on defining units <pint:defining>`.
rename_dims : mapping of str -> str
Update :obj:`.RENAME_DIMS`.
Warns
-----
UserWarning
If *config* contains unrecognized keys.
"""
config = _config_args(path, config)
# Units
units = config.get('units', {})
# Define units
ureg = pint.get_application_registry()
try:
ureg.define(units['define'].strip())
except KeyError:
pass
except pint.DefinitionSyntaxError as e:
log.warning(e)
# Add replacements
for old, new in units.get('replace', {}).items():
REPLACE_UNITS[old] = new
# Dimensions to be renamed
RENAME_DIMS.update(config.get('rename_dims', {}))
def create_bldg_models_batch_no_exception(bldg_fids_to_create_model_for, config, sia_params_gen_lock) -> Tuple[Dict[int, BuildingModel], pd.DataFrame, List[int]]:
"""
Method used to create building models for a batch of fids.
Module-Level method to be able to parallelize to mutliple processes.
For building model creation we need to load all site vertices which is memory and time intensive, thus
it makes sense to create one BldgModelFactory and reuse the instance for several buildings.
:param args:
:return: tuple with three entries (sucessfully created building models,
infos per building used during model creation,
fids for which building model could not be created)
"""
bldg_models_factory = BldgModelFactory(pint.get_application_registry(), config, sia_params_gen_lock)
bldg_models = {bldg_fid: _create_bldg_model_no_exception(bldg_fid, bldg_models_factory) for bldg_fid in bldg_fids_to_create_model_for} # type: ignore
failed_fids = [fid for fid, model in bldg_models.items() if not model]
bldg_models_successful = {fid: model for fid, model in bldg_models.items() if model}
per_bldg_infos = bldg_models_factory.per_bldg_infos_used.loc[bldg_fids_to_create_model_for]
return (bldg_models_successful, per_bldg_infos, failed_fids)
def ureg(self) -> UnitRegistry:
"""Return:
1. the registry that was defined in ``__init__``
2. if it was omitted, the global registry defined with
:func:`pint.set_application_registry`
3. if the global registry was never set, a standard registry built with
:file:`defaults_en.txt`
"""
if self._ureg:
return self._ureg
import pint
application_registry = pint.get_application_registry()
try:
return application_registry.get()
except AttributeError:
return application_registry
def _collect_result_summary_batch(
input_tuples_per_fid, do_calc_op_emissions_and_costs, custom_config
) -> Tuple[Dict[int, EplusErrorLevel], Dict[int, Optional[EnergyDemandSimulationResults]], Dict[int, Optional[OperationalEmissionsAndCostsResult]]]:
"""
:param input_tuples_per_fid: {fid: (eplus_output_folder, heating_energy_carrier, dhw_energy_carrier, simulation_year)}}
:param custom_config: custom configuration entries
:return: (simulation res table, emission res table, fuel cost res table)
"""
unit_reg = pint.get_application_registry()
res_processor = ResultProcessor(unit_reg, do_calc_op_emissions_and_costs, custom_config)
for (
fid,
(eplus_output_folder, heating_energy_carrier, dhw_energy_carrier, sim_year),
) in input_tuples_per_fid.items():
res_processor.process_results_for(fid, eplus_output_folder, heating_energy_carrier, dhw_energy_carrier, sim_year)
return (
res_processor.eplus_err_level_per_bldg,
res_processor.simulation_result_per_bldg,
res_processor.emission_and_cost_per_bldg,
)
def apply_units(qty, units, quiet=False):
"""Simply apply *units* to *qty*.
Logs on level ``WARNING`` if *qty* already has existing units.
Parameters
----------
qty : .Quantity
units : str or pint.Unit
Units to apply to *qty*
quiet : bool, optional
If :obj:`True` log on level ``DEBUG``.
"""
registry = pint.get_application_registry()
existing = qty.attrs.get('_unit', None)
existing_dims = getattr(existing, 'dimensionality', {})
new_units = registry.parse_units(units)
if len(existing_dims):
# Some existing dimensions: log a message either way
if existing_dims == new_units.dimensionality:
log.debug(f"Convert '{existing}' to '{new_units}'")
# NB use a factor because pint.Quantity cannot wrap AttrSeries
factor = registry.Quantity(1.0, existing).to(new_units).magnitude
result = qty * factor
else:
msg = f"Replace '{existing}' with incompatible '{new_units}'"
log.warning(msg)
result = qty.copy()
else:
# No units, or dimensionless
result = qty.copy()
result.attrs['_unit'] = new_units
return result
def ureg():
yield pint.get_application_registry()
class PintType(ExtensionDtype):
"""
A Pint duck-typed class, suitable for holding a quantity (with unit specified) dtype.
"""
type = _Quantity
# kind = 'O'
# str = '|O08'
# base = np.dtype('O')
# num = 102
_metadata = ("units",)
_match = re.compile(r"(P|p)int\[(?P<units>.+)\]")
_cache = {}
ureg = pint.get_application_registry()
@property
def _is_numeric(self):
# type: () -> bool
return True
def __new__(cls, units=None):
"""
Parameters
----------
units : Pint units or string
"""
if isinstance(units, PintType):
return units
elif units is None:
# empty constructor for pickle compat
# import pdb
# pdb.set_trace()
return object.__new__(cls)
if not isinstance(units, _Unit):
units = cls._parse_dtype_strict(units)
# ureg.unit returns a quantity with a magnitude of 1
# eg 1 mm. Initialising a quantity and taking it's unit
# TODO: Seperate units from quantities in pint
# to simplify this bit
units = cls.ureg.Quantity(1, units).units
try:
return cls._cache["{:P}".format(units)]
except KeyError:
u = object.__new__(cls)
u.units = units
cls._cache["{:P}".format(units)] = u
return u
@classmethod
def _parse_dtype_strict(cls, units):
if isinstance(units, str):
if units.startswith("pint[") or units.startswith("Pint["):
if not units[-1] == "]":
raise ValueError("could not construct PintType")
m = cls._match.search(units)
if m is not None:
units = m.group("units")
if units is not None:
return units
raise ValueError("could not construct PintType")
@classmethod
def construct_from_string(cls, string):
"""
Strict construction from a string, raise a TypeError if not
possible
"""
if isinstance(string, str) and (
string.startswith("pint[") or string.startswith("Pint[")
):
# do not parse string like U as pint[U]
# avoid tuple to be regarded as unit
try:
return cls(units=string)
except ValueError:
pass
# This else block may allow pd.Series([1,2],dtype="m")
# else:
# try:
# return cls(units=string)
# except ValueError:
# pass
raise TypeError("Cannot construct a 'PintType' from '{}'".format(string))
# def __unicode__(self):
# return compat.text_type(self.name)
@property
def name(self):
return str("pint[{units}]".format(units=self.units))
@property
def na_value(self):
return np.nan * self.units
def __hash__(self):
# make myself hashable
return hash(str(self))
def __eq__(self, other):
try:
other = PintType(other)
except (ValueError, errors.UndefinedUnitError):
return False
return self.units == other.units
@classmethod
def is_dtype(cls, dtype):
"""
Return a boolean if we if the passed type is an actual dtype that we
can match (via string or type)
"""
if isinstance(dtype, str):
if dtype.startswith("pint[") or dtype.startswith("Pint["):
try:
if cls._parse_dtype_strict(dtype) is not None:
return True
else:
return False
except ValueError:
return False
else:
return False
return super(PintType, cls).is_dtype(dtype)
@classmethod
def construct_array_type(cls):
return PintArray
def __repr__(self):
"""
Return a string representation for this object.
Invoked by unicode(df) in py2 only. Yields a Unicode String in both
py2/py3.
"""
return self.name
import logging
from pandas.testing import assert_series_equal
import pint
import pytest
import ixmp
from ixmp.reporting import Reporter, as_quantity, computations
from ixmp.testing import assert_logs
from . import add_test_data
REGISTRY = pint.get_application_registry()
@pytest.fixture(scope='function')
def data(test_mp, request):
scen = ixmp.Scenario(test_mp, request.node.name, request.node.name, 'new')
rep = Reporter.from_scenario(scen)
yield [scen, rep] + list(add_test_data(scen))
def test_apply_units(data, caplog):
# Unpack
*_, x = data
# Brute-force replacement with incompatible units
with assert_logs(caplog, "Replace 'kilogram' with incompatible 'liter'"):
result = computations.apply_units(x, 'litres')
assert result.attrs['_unit'] == REGISTRY.Unit('litre')
def data_for_quantity(ix_type, name, column, scenario, config):
"""Retrieve data from *scenario*.
Parameters
----------
ix_type : 'equ' or 'par' or 'var'
Type of the ixmp object.
name : str
Name of the ixmp object.
column : 'mrg' or 'lvl' or 'value'
Data to retrieve. 'mrg' and 'lvl' are valid only for ``ix_type='equ'``,and
'level' otherwise.
scenario : ixmp.Scenario
Scenario containing data to be retrieved.
config : dict of (str -> dict)
The key 'filters' may contain a mapping from dimensions to iterables of allowed
values along each dimension. The key 'units'/'apply' may contain units to apply
to the quantity; any such units overwrite existing units, without conversion.
Returns
-------
:class:`Quantity`
Data for *name*.
"""
log.debug(f"{name}: retrieve data")
registry = pint.get_application_registry()
# Only use the relevant filters
idx_names = scenario.idx_names(name)
filters = config.get("filters", None)
if filters:
# Dimensions of the object
dims = dims_for_qty(idx_names)
# Mapping from renamed dimensions to Scenario dimension names
MAP = get_reversed_rename_dims()
filters_to_use = {}
for dim, values in filters.items():
if dim in dims:
# *dim* is in this ixmp object, so the filter can be used
filters_to_use[MAP.get(dim, dim)] = values
filters = filters_to_use
# Retrieve quantity data
data = getattr(scenario, ix_type)(name, filters)
# ixmp/GAMS scalar is not returned as pd.DataFrame
if isinstance(data, dict): # pragma: no cover
data = pd.DataFrame.from_records([data])
# Warn if no values are returned.
# TODO construct an empty Quantity with the correct dimensions *even if* no values
# are returned.
if len(data) == 0:
log.debug(
f"0 values for {ix_type} {repr(name)} using filters: {repr(filters)}"
)
log.debug("May be the cause of subsequent errors.")
# Convert columns with categorical dtype to str
data = data.astype({
dt[0]: str
for dt in data.dtypes.items() if isinstance(dt[1], pd.CategoricalDtype)
})
# List of the dimensions
dims = dims_for_qty(data)
# Remove the unit from the DataFrame
try:
attrs = {"_unit": parse_units(data.pop("unit"))}
except KeyError: # pragma: no cover
# 'equ' are returned without units
attrs = {}
except ValueError as e:
if "mixed units" in e.args[0]:
# Discard mixed units
log.warning(f"{name}: {e.args[0]} discarded")
attrs = {"_unit": registry.Unit("")}
else:
# Raise all other ValueErrors
raise
# Apply units
try:
new_unit = config["units"]["apply"][name]
except KeyError:
pass
else:
log.info(
f"{name}: replace units {attrs.get('_unit', '(none)')} with {new_unit}"
)
attrs["_unit"] = registry.Unit(new_unit)
# Set index if 1 or more dimensions
if len(dims):
# First rename, then set index
data = data.rename(columns=RENAME_DIMS).set_index(dims)
# Convert to a Quantity, assign attrbutes and name
qty = Quantity(data[column],
name=name + ("-margin" if column == "mrg" else ""),
attrs=attrs)
try:
# Remove length-1 dimensions for scalars
qty = qty.squeeze("index", drop=True)
except (KeyError, ValueError):
# KeyError if "index" does not exist; ValueError if its length is > 1
pass
return qty
def unit_registry(self):
"""The :meth:`pint.UnitRegistry` used by the Reporter."""
return pint.get_application_registry()
import numpy as np
import pint
import pytest
from numpy.testing import assert_almost_equal, assert_array_almost_equal
from pint.util import UnitsContainer
from iam_units import convert_gwp, emissions, format_mass, registry
DEFAULTS = pint.get_application_registry()
# Parameters for test_units(), tuple of:
# 1. A literal string to be parsed as a unit.
# 2. Expected dimensionality of the parsed unit.
# 3. True if the units are not in pint's default_en.txt, but are defined in
# definitions.txt.
energy = UnitsContainer({"[energy]": 1})
PARAMS = [
("GW a", energy, False),
("kWa", energy, True),
("Lge", energy, True),
("tce", energy, True),
("toe", energy, False),
("EUR_2005", UnitsContainer({"[currency]": 1}), True),
]
@pytest.mark.parametrize("unit_str, dim, new_def",
PARAMS,
ids=lambda v: v if isinstance(v, str) else "")
def test_units(unit_str, dim, new_def):
if new_def:
def test_get_application_registry(self):
ureg = get_application_registry()
u = ureg.Unit("kg")
assert str(u) == "kilogram"
def test_modify_application_registry():
ar = get_application_registry()
u = ar.get()
ar.force_ndarray_like = True
assert ar.force_ndarray_like == u.force_ndarray_like
_warnings.warn(f"{_library_path}.\nREFPROP not configured.")
__version__ = "0.1.19"
__version__full = (
f"ccp: {__version__} | " +
f'CP : {_CP.get_global_param_string("version")} | ' +
f'REFPROP : {_CP.get_global_param_string("REFPROP_version")}')
###############################################################################
# pint
###############################################################################
import pint as _pint
_new_units = _Path(__file__).parent / "config/new_units.txt"
ureg = _pint.get_application_registry()
if isinstance(ureg.get(), _pint.registry.LazyRegistry):
ureg = _pint.UnitRegistry()
ureg.load_definitions(str(_new_units))
# set ureg to make pickle possible
_pint.set_application_registry(ureg)
Q_ = ureg.Quantity
_warnings.filterwarnings("ignore",
message="The unit of the quantity is stripped")
###############################################################################
# plotly theme
###############################################################################
from plotly import io as _pio