def forwards(self, orm):
"""Create a column definition for each ESPM field we know about."""
bedes = orm['seed.schema'].objects.create(name='BEDES')
espm_schema = espm.schema['flat_schema']
espm_float_fields = espm.schema['types'].keys()
for col in espm_schema:
unit = None
if espm_schema[col]:
# If there's a non-empty string, then that's the unit name.
unit_type = STRING
if col in espm_float_fields:
# If we have an entry for a column in the types field,
# then it's a float.
unit_type = FLOAT
unit, _ = orm['seed.unit'].objects.get_or_create(
unit_name=espm_schema[col],
unit_type=unit_type
)
# We determine which columns aren't listed in the model itself.
is_extra_data = col not in m_utils.get_mappable_columns()
column = orm['seed.column'].objects.create(
column_name=col,
unit=unit,
is_extra_data=is_extra_data
)
bedes.columns.add(column)
def forwards(self, orm):
"Write your forwards methods here."
# Note: Don't use "from appname.models import ModelName".
# Use orm.ModelName to refer to models in this application,
# and orm['appname.ModelName'] for models in other applications.
schema, _ = orm['seed.schema'].objects.get_or_create(
name='BEDES',
)
mappable_columns = m_utils.get_mappable_columns()
orig_cols = schema.columns.all()
for raw_name, unit_type_str in bedes.schema.iteritems():
unit = None
col_name, unit_desc = name_and_unit_desc(raw_name)
unit_type = unit_enum(unit_type_str)
unit, _ = orm['seed.unit'].objects.get_or_create(
unit_name=unit_desc,
unit_type=unit_type,
)
is_extra_data = col_name not in mappable_columns
new_col = orm['seed.column'].objects.create(
column_name=col_name,
unit=unit,
is_extra_data=is_extra_data
)
if new_col not in orig_cols:
schema.columns.add(new_col)
def map_row_chunk(chunk, file_pk, source_type, prog_key, increment, *args,
**kwargs):
"""Does the work of matching a mapping to a source type and saving
:param chunk: list of dict of str. One row's worth of parse data.
:param file_pk: int, the PK for an ImportFile obj.
:param source_type: int, represented by either ASSESSED_RAW, or
PORTFOLIO_RAW.
:param cleaner: (optional), the cleaner class you want to send
to mapper.map_row. (e.g. turn numbers into floats.).
:param raw_ids: (optional kwarg), the list of ids in chunk order.
"""
import_file = ImportFile.objects.get(pk=file_pk)
save_type = PORTFOLIO_BS
if source_type == ASSESSED_RAW:
save_type = ASSESSED_BS
concats = []
org = Organization.objects.get(
pk=import_file.import_record.super_organization.pk)
mapping, concats = get_column_mappings(org)
map_cleaner = _build_cleaner(org)
# For those column mapping which are not db columns, we
# need to let MCM know that we apply our mapping function to those.
apply_columns = []
mappable_columns = get_mappable_columns()
for item in mapping:
if mapping[item] not in mappable_columns:
apply_columns.append(item)
apply_func = apply_data_func(mappable_columns)
for row in chunk:
model = mapper.map_row(row,
mapping,
BuildingSnapshot,
cleaner=map_cleaner,
concat=concats,
apply_columns=apply_columns,
apply_func=apply_func,
*args,
**kwargs)
model.import_file = import_file
model.source_type = save_type
model.clean()
model.super_organization = import_file.import_record.super_organization
model.save()
if model:
# Make sure that we've saved all of the extra_data column names
save_column_names(model, mapping=mapping)
increment_cache(prog_key, increment)
def save_column_names(bs, mapping=None):
"""Save unique column names for extra_data in this organization.
Basically this is a record of all the extra_data keys we've ever seen
for a particular organization.
:param bs: BuildingSnapshot instance.
"""
from seed.utils import mapping as mapping_utils
for key in bs.extra_data:
# Assertain if our key is ``extra_data`` or not.
is_extra_data = key not in mapping_utils.get_mappable_columns()
Column.objects.get_or_create(
organization=bs.super_organization,
column_name=key[:511],
is_extra_data=is_extra_data
)
PortfolioRaw_to_BuildingSnapshot = (
(u'property_id', u'pm_property_id'),
(u'custom_property_id_1_-_id', u'custom_id_1'),
(u'property_notes', u'property_notes'),
(u'year_ending', u'year_ending'),
(u'property_name', u'property_name'),
(u'property_floor_area_bldg_park', u'gross_floor_area'),
(u'address_line_1', u'address_line_1'),
(u'address_line_2', u'address_line_2'),
(u'city', u'city'),
(u'postal_code', u'postal_code'),
(u'county', u'district'),
(u'year_built', u'year_built'),
(u'energy_score', u'energy_score'),
(u'generation_date', u'generation_date'),
(u'release_date', u'release_date'),
(u'state_province', u'state_province'),
(u'site_eui', u'site_eui'),
(u'propertys_portfolio_manager_account_holder', u'owner'),
(u'propertys_portfolio_manager_account_holder_email', u'owner_email'),
(u'weather_normalized_site_eui', u'site_eui_weather_normalized'),
(u'weather_normalized_source_eui', u'source_eui_weather_normalized'),
(u'energy_alerts', u'energy_alerts'),
(u'third_party_certification', u'building_certification'),
)
BuildingSnapshot_to_BuildingSnapshot = (
#(u'import_records', map_import_records),
) + tuple([(k, k) for k in get_mappable_columns()])
:author Dan Gunter <*****@*****.**>
"""
import logging
from collections import defaultdict
from seed.models import PropertyState
from seed.models import TaxLotState
LINEAR_UNITS = set([u'ft', u'm', u'in']) # ??more??
from seed.utils.mapping import get_mappable_columns
from seed.lib.mappings.mapping_data import MappingData
from seed.models.deprecate import SYSTEM_MATCH
# TODO: Fix name of this method / remove if possible.
BuildingSnapshot_to_BuildingSnapshot = tuple([(k, k) for k in get_mappable_columns()])
md = MappingData()
property_state_fields = [x['name'] for x in md.property_data]
tax_lot_state_fields = [x['name'] for x in md.tax_lot_data]
PropertyState_to_PropertyState = tuple([(k, k) for k in property_state_fields])
TaxLotState_to_TaxLotState = tuple([(k, k) for k in tax_lot_state_fields])
_log = logging.getLogger(__name__)
def get_attrs_with_mapping(data_set_buildings, mapping):
"""Returns a dictionary of attributes from each data_set_building.
:param buildings: list, group of BS instances to merge.
:copyright (c) 2014 - 2016, The Regents of the University of California, through Lawrence Berkeley National Laboratory (subject to receipt of any required approvals from the U.S. Department of Energy) and contributors. All rights reserved. # NOQA
:author Dan Gunter <*****@*****.**>
"""
import logging
from collections import defaultdict
from seed.lib.mappings.mapping_data import MappingData
from seed.models import PropertyState
from seed.models import TaxLotState
from seed.utils.mapping import get_mappable_columns
LINEAR_UNITS = {u'ft', u'm', u'in'}
# TODO: Fix name of this method / remove if possible.
BuildingSnapshot_to_BuildingSnapshot = tuple([(k, k)
for k in get_mappable_columns()])
md = MappingData()
property_state_fields = [x['name'] for x in md.property_data]
tax_lot_state_fields = [x['name'] for x in md.tax_lot_data]
PropertyState_to_PropertyState = tuple([(k, k) for k in property_state_fields])
TaxLotState_to_TaxLotState = tuple([(k, k) for k in tax_lot_state_fields])
_log = logging.getLogger(__name__)
def get_attrs_with_mapping(data_set_buildings, mapping):
"""
Returns a dictionary of attributes from each data_set_building.
def map_row_chunk(chunk, file_pk, source_type, prog_key, increment, *args, **kwargs):
"""Does the work of matching a mapping to a source type and saving
:param chunk: list of dict of str. One row's worth of parse data.
:param file_pk: int, the PK for an ImportFile obj.
:param source_type: int, represented by either ASSESSED_RAW, or
PORTFOLIO_RAW.
:param prog_key: string, key of the progress key
:param increment: double, value by which to increment progress key
:param cleaner: (optional), the cleaner class you want to send
to mapper.map_row. (e.g. turn numbers into floats.).
:param raw_ids: (optional kwarg), the list of ids in chunk order.
"""
import_file = ImportFile.objects.get(pk=file_pk)
save_type = PORTFOLIO_BS
if source_type == ASSESSED_RAW:
save_type = ASSESSED_BS
concats = []
org = Organization.objects.get(
pk=import_file.import_record.super_organization.pk
)
mapping, concats = get_column_mappings(org)
map_cleaner = _build_cleaner(org)
# For those column mapping which are not db columns, we
# need to let MCM know that we apply our mapping function to those.
apply_columns = []
mappable_columns = get_mappable_columns()
for item in mapping:
if mapping[item] not in mappable_columns:
apply_columns.append(item)
apply_func = apply_data_func(mappable_columns)
for row in chunk:
model = mapper.map_row(
row,
mapping,
BuildingSnapshot,
cleaner=map_cleaner,
concat=concats,
apply_columns=apply_columns,
apply_func=apply_func,
*args,
**kwargs
)
model.import_file = import_file
model.source_type = save_type
model.clean()
model.super_organization = import_file.import_record.super_organization
model.save()
if model:
# Make sure that we've saved all of the extra_data column names
save_column_names(model, mapping=mapping)
increment_cache(prog_key, increment)
PortfolioRaw_to_BuildingSnapshot = (
(u'property_id', u'pm_property_id'),
(u'custom_property_id_1_-_id', u'custom_id_1'),
(u'property_notes', u'property_notes'),
(u'year_ending', u'year_ending'),
(u'property_name', u'property_name'),
(u'property_floor_area_bldg_park', u'gross_floor_area'),
(u'address_line_1', u'address_line_1'),
(u'address_line_2', u'address_line_2'),
(u'city', u'city'),
(u'postal_code', u'postal_code'),
(u'county', u'district'),
(u'year_built', u'year_built'),
(u'energy_score', u'energy_score'),
(u'generation_date', u'generation_date'),
(u'release_date', u'release_date'),
(u'state_province', u'state_province'),
(u'site_eui', u'site_eui'),
(u'propertys_portfolio_manager_account_holder', u'owner'),
(u'propertys_portfolio_manager_account_holder_email', u'owner_email'),
(u'weather_normalized_site_eui', u'site_eui_weather_normalized'),
(u'weather_normalized_source_eui', u'source_eui_weather_normalized'),
(u'energy_alerts', u'energy_alerts'),
(u'third_party_certification', u'building_certification'),
)
BuildingSnapshot_to_BuildingSnapshot = (
#(u'import_records', map_import_records),
) + tuple([(k, k) for k in get_mappable_columns()])
# Values are expected data types so we can present reasonable approximations
# of validation in the frontend.
PortfolioRaw_to_BuildingSnapshot = (
(u'property_id', u'pm_property_id'),
(u'custom_property_id_1_-_id', u'custom_id_1'),
(u'property_notes', u'property_notes'),
(u'year_ending', u'year_ending'),
(u'property_name', u'property_name'),
(u'property_floor_area_bldg_park', u'gross_floor_area'),
(u'address_line_1', u'address_line_1'),
(u'address_line_2', u'address_line_2'),
(u'city', u'city'),
(u'postal_code', u'postal_code'),
(u'county', u'district'),
(u'year_built', u'year_built'),
(u'energy_score', u'energy_score'),
(u'generation_date', u'generation_date'),
(u'release_date', u'release_date'),
(u'state_province', u'state_province'),
(u'site_eui', u'site_eui'),
(u'propertys_portfolio_manager_account_holder', u'owner'),
(u'propertys_portfolio_manager_account_holder_email', u'owner_email'),
(u'weather_normalized_site_eui', u'site_eui_weather_normalized'),
(u'weather_normalized_source_eui', u'source_eui_weather_normalized'),
(u'energy_alerts', u'energy_alerts'),
(u'third_party_certification', u'building_certification'),
)
BuildingSnapshot_to_BuildingSnapshot = tuple([(k, k) for k in get_mappable_columns()])
