def handle(self, *args, **options):
logging_info("RUN create_m2m_relatinships_organization with args={},kwds={}".format(args, options))
if options['organization']:
core_organization = map(int, options['organization'].split(","))
else:
core_organization = get_core_organizations()
logging_info("Processing organization list: {}".format(core_organization))
if options['stats']:
for org_id in core_organization:
self.display_stats(org_id)
return
org_taxlot_splitdata_rules = collections.defaultdict(lambda : (UNIQUE_LIST, ("jurisdiction_tax_lot_id",)))
org_taxlot_splitdata_rules[20] = (UNIQUE_LIST, ("jurisdiction_tax_lot_id", "extra_data/Philadelphia Building ID",))
for org_id in core_organization:
self.split_taxlots_into_m2m_relationships(org_id, org_taxlot_splitdata_rules)
# At the end run two checks:
# Go through the tax Lots, collect any that are left, make
# sure they aren't a part of any m2m entities.
for view in TaxLotView.objects.filter(taxlot__organization_id=org_id).all():
try:
# pdb.set_trace()
# aggregate_value_from_state(view.state, org_taxlot_split_extra[org_id])
taxlot_field_list = get_id_fields(view.state.jurisdiction_tax_lot_id)
if len(taxlot_field_list) > 1:
logging_warn("Danger - tax lot '{}' still exists.".format(view.state.jurisdiction_tax_lot_id))
except TaxLotIDValueError, e:
continue
def handle(self, *args, **options):
logging_info(
"RUN migrate_extradata_columns with args={},kwds={}".format(
args, options))
if options['organization']:
organization_ids = map(int, options['organization'].split(","))
else:
organization_ids = get_core_organizations()
update_columns = options['update_columns']
add_unmapped_columns = options['add_unmapped_columns']
create_missing_columns = options['create_missing_columns']
for org_id in organization_ids:
org = Organization.objects.get(pk=org_id)
# Update
if update_columns:
self.update_columns_based_on_mappings_file(org)
if create_missing_columns:
self.find_missing_columns_based_on_extra_data(org)
logging_info("END migrate_extradata_columns")
return
def handle(self, *args, **options):
logging_info(
"RUN migrate_extradata_columns with args={},kwds={}".format(args, options))
if options['organization']:
organization_ids = map(int, options['organization'].split(","))
else:
organization_ids = get_core_organizations()
update_columns = options['update_columns']
add_unmapped_columns = options['add_unmapped_columns']
create_missing_columns = options['create_missing_columns']
for org_id in organization_ids:
org = Organization.objects.get(pk=org_id)
# Update
if update_columns:
self.update_columns_based_on_mappings_file(org)
if create_missing_columns:
self.find_missing_columns_based_on_extra_data(org)
logging_info("END migrate_extradata_columns")
return
def getOrganizationsToDelete():
"""Get all organizations that are not in the global white list."""
all_organizations = seed.models.Organization.objects.all()
bad_organizations = [
org for org in all_organizations
if org.id not in get_core_organizations()
]
return bad_organizations
def handle(self, *args, **options):
logging_info(
"RUN create_m2m_relatinships_organization with args={},kwds={}".
format(args, options))
if options['organization']:
core_organization = map(int, options['organization'].split(","))
else:
core_organization = get_core_organizations()
logging_info(
"Processing organization list: {}".format(core_organization))
if options['stats']:
for org_id in core_organization:
self.display_stats(org_id)
return
org_taxlot_splitdata_rules = collections.defaultdict(
lambda: (UNIQUE_LIST, ("jurisdiction_tax_lot_id", )))
org_taxlot_splitdata_rules[20] = (UNIQUE_LIST, (
"jurisdiction_tax_lot_id",
"extra_data/Philadelphia Building ID",
))
for org_id in core_organization:
self.split_taxlots_into_m2m_relationships(
org_id, org_taxlot_splitdata_rules)
# At the end run two checks:
# Go through the tax Lots, collect any that are left, make
# sure they are not a part of any m2m entities.
for view in TaxLotView.objects.filter(
taxlot__organization_id=org_id).all():
try:
# pdb.set_trace()
# aggregate_value_from_state(view.state, org_taxlot_split_extra[org_id])
taxlot_field_list = get_id_fields(
view.state.jurisdiction_tax_lot_id)
if len(taxlot_field_list) > 1:
logging_warn("Danger - tax lot '{}' still exists.".format(
view.state.jurisdiction_tax_lot_id))
except TaxLotIDValueError, e:
continue
def handle(self, *args, **options):
"""Go through organization by organization and look for m2m."""
logging_info("RUN create_primarysecondary_taxlots with args={},kwds={}".format(args, options))
if options['organization']:
core_organization = map(int, options['organization'].split(","))
else:
core_organization = get_core_organizations()
for org_id in core_organization:
# Writing loop over organizations
org = Organization.objects.filter(pk=org_id).first()
logging_info("Processing organization {}".format(org))
assert org, "Organization {} not found".format(org_id)
self.assign_primarysecondary_tax_lots(org)
logging_info("END create_primarysecondary_taxlots")
return
def handle(self, *args, **options):
"""Go through organization by organization and look for m2m."""
logging_info(
"RUN create_primarysecondary_taxlots with args={},kwds={}".format(
args, options))
if options['organization']:
core_organization = map(int, options['organization'].split(","))
else:
core_organization = get_core_organizations()
for org_id in core_organization:
# Writing loop over organizations
org = Organization.objects.filter(pk=org_id).first()
logging_info("Processing organization {}".format(org))
assert org, "Organization {} not found".format(org_id)
self.assign_primarysecondary_tax_lots(org)
logging_info("END create_primarysecondary_taxlots")
return
def handle(self, *args, **options):
logging_info(
"RUN migrate_extradata_columns with args={},kwds={}".format(
args, options))
# Process Arguments
if options['organization']:
organization_ids = map(int, options['organization'].split(","))
else:
organization_ids = get_core_organizations()
clear_bluesky_labels = options["clear_bluesky_labels"]
add_property_labels = options["add_property_labels"]
add_taxlot_labels = options["add_taxlot_labels"]
for org_id in organization_ids:
##############################
# Handle the clear case. This is a bit inelegant the
# way the loop on org_ids is setup.
if clear_bluesky_labels:
print "Org={}: Clearing all labels on Property and TaxLot objects.".format(
org_id)
for prop in Property.objects.filter(organization=org_id).all():
prop.labels.clear()
for tax_lot in TaxLot.objects.filter(
organization=org_id).all():
tax_lot.labels.clear()
continue
# End Clear Case
##############################
print(
"Org={}: Migrating Labels with settings add_property_labels={}"
", add_taxlot_labels={}").format(org_id, add_property_labels,
add_taxlot_labels)
##############################
# Copy Property
if add_property_labels:
# This is inefficient, in that it does each
# property/tax lot multiple times for each of it's
# views - but it's clear and should not be
# prohibitively wasteful.
# Alternatively you could grab the first
# propertyview/taxlotview for each property/taxlot
# and then use the state on that to populate the
# property/taxlot.
for pv in PropertyView.objects.filter(
property__organization=org_id).select_related(
'state').all():
if not "prop_cb_id" in pv.state.extra_data:
print "Warning: key 'prop_cb_id' was not found for PropertyView={}".format(
pv)
continue
cb_id = pv.state.extra_data['prop_cb_id']
try:
cb = CanonicalBuilding.objects.get(pk=cb_id)
except ObjectDoesNotExist, xcpt:
print "Warning: Canonical Building={} was not found in the DB".format(
cb_id)
continue
cb_labels = cb.labels.all()
preexisting_pv_labels = set(
map(lambda l: l.pk, pv.property.labels.all()))
for label in cb_labels:
if label.pk not in preexisting_pv_labels:
pv.property.labels.add(label)
else:
pv.property.save()
#
##############################
##############################
# Copy Tax Lot labels
if add_taxlot_labels:
for tlv in TaxLotView.objects.filter(
taxlot__organization=org_id).select_related(
'state').all():
if not "taxlot_cb_id" in tlv.state.extra_data:
print "Warning: key 'prop_cb_id' was not found for TaxLotView={}".format(
tlv)
continue
cb_id = tlv.state.extra_data['taxlot_cb_id']
try:
cb = CanonicalBuilding.objects.get(pk=cb_id)
except ObjectDoesNotExist, xcpt:
print "Warning: Canonical Building={} was not found in the DB".format(
cb_id)
continue
cb_labels = cb.labels.all()
preexisting_tlv_labels = set(
map(lambda l: l.pk, tlv.taxlot.labels.all()))
for label in cb_labels:
if label.pk not in preexisting_tlv_labels:
tlv.taxlot.labels.add(label)
else:
tlv.taxlot.save()
def handle(self, *args, **options):
"""Migrate the CanonicalBuildings for one or more Organizations into
the new 'BlueSky' data structures."""
logging_info("RUN migrate_organization with args={},kwds={}".format(
args, options))
# Process Arguments
if options['organization']:
core_organization = map(int, options['organization'].split(","))
else:
core_organization = get_core_organizations()
limit = options['limit'] if "limit" in options else 0
starting_from_canonical = False if not options[
'starting_on_canonical'] else options['starting_on_canonical']
starting_on_canonical_following = False if not options['starting_following_canonical'] else \
options['starting_following_canonical']
ADD_METADATA = options['add_metadata']
assert not (
starting_on_canonical_following and starting_from_canonical
), "Can only specify one of --starting_on_canonical and --starting_following_canonical"
canonical_buildings_whitelist = map(int, options['cb_whitelist_string'].split(",")) if \
options['cb_whitelist_string'] else False
# End Processing
tree_file = get_static_building_snapshot_tree_file()
m2m = read_building_snapshot_tree_structure(tree_file)
all_nodes = set(map(projection_onto_index(0),
m2m)).union(set(map(projection_onto_index(1),
m2m)))
child_dictionary = collections.defaultdict(lambda: set())
parent_dictionary = collections.defaultdict(lambda: set())
adj_dim = max(all_nodes) + 1
adj_matrix = dok_matrix((adj_dim, adj_dim), dtype=np.bool)
for (from_node, to_node) in m2m:
adj_matrix[from_node, to_node] = 1
child_dictionary[from_node].add(to_node)
parent_dictionary[to_node].add(from_node)
# Core data struct, possible refactor point.
all_nodes, child_dictionary, parent_dictionary, adj_matrix
# We don't care about the total number because
_, labelarray = connected_components(adj_matrix)
counts = collections.Counter()
for label in labelarray:
counts[label] += 1
logging_info("Migration organization: {}".format(",".join(
map(str, core_organization))))
for org_id in core_organization:
# Writing loop over organizations
org = Organization.objects.get(pk=org_id)
logging_info("Processing organization {}".format(org))
assert org, "Organization {} not found".format(org_id)
org_canonical_buildings = seed.models.CanonicalBuilding.objects.filter(
canonical_snapshot__super_organization=org_id,
active=True).all()
org_canonical_snapshots = [
cb.canonical_snapshot for cb in org_canonical_buildings
]
# FIXME: Turns out the ids are on Building
# Snapshot. Leaving it this way because the other code
# should be displaying canonical building indexes.
if starting_from_canonical or starting_on_canonical_following:
# org_canonical_ids = map(lambda x: x.pk, org_canonical_buildings)
org_canonical_ids = map(lambda x: x.pk,
org_canonical_snapshots)
try:
canonical_index = max(starting_from_canonical,
starting_on_canonical_following)
canonical_index = org_canonical_ids.index(canonical_index)
if starting_on_canonical_following: canonical_index += 1
org_canonical_buildings = list(
org_canonical_buildings)[canonical_index:]
org_canonical_snapshots = list(
org_canonical_snapshots)[canonical_index:]
ref_str = "starting" if starting_from_canonical else "following"
logging_info(
"Restricting to canonical starting ndx={} (id={}), was {} now {}."
.format(canonical_index, len(org_canonical_ids),
len(org_canonical_ids),
len(org_canonical_buildings)))
except ValueError:
raise RuntimeError(
"Requested to start referencing canonical building id={} which was not found."
.format(starting_from_canonical))
if canonical_buildings_whitelist:
good_canonical_building_indexes = [
ndx for (ndx, cb) in enumerate(org_canonical_buildings)
if cb.pk in canonical_buildings_whitelist
]
org_canonical_buildings = [
org_canonical_buildings[ndx]
for ndx in good_canonical_building_indexes
]
org_canonical_snapshots = [
org_canonical_snapshots[ndx]
for ndx in good_canonical_building_indexes
]
logging_info("Restricted to {} elements in whitelist.".format(
len(org_canonical_buildings)))
logging_info("IDS: {}".format(", ".join(
map(lambda cs: str(cs.pk), org_canonical_buildings))))
if canonical_buildings_whitelist:
good_canonical_building_indexes = [
ndx for (ndx, cb) in enumerate(org_canonical_buildings)
if cb.pk in canonical_buildings_whitelist
]
org_canonical_buildings = [
org_canonical_buildings[ndx]
for ndx in good_canonical_building_indexes
]
org_canonical_snapshots = [
org_canonical_snapshots[ndx]
for ndx in good_canonical_building_indexes
]
logging_info("Restricted to {} elements in whitelist.".format(
len(org_canonical_buildings)))
logging_info("IDS: {}".format(", ".join(
map(lambda cs: str(cs.pk), org_canonical_buildings))))
if len(org_canonical_buildings) == 0:
logging_info("Organization {} has no buildings".format(org_id))
continue
last_date = max([cs.modified for cs in org_canonical_snapshots])
# create_bluesky_cycles_for_org(org, last_date)
tree_sizes = [
counts[labelarray[bs.id]] for bs in org_canonical_snapshots
]
## For each of those trees find the tip
## For each of those trees find the import records
## For each of those trees find the cycles associated with it
for ndx, (cb, bs) in enumerate(
zip(org_canonical_buildings, org_canonical_snapshots)):
if limit and (ndx + 1) > limit:
logging_info("Migrated limit={} buildings.".format(limit))
logging_info(
"Skipping remainder of buildings for organization.")
break
logging_info("Processing Building {}/{}".format(
ndx + 1, len(org_canonical_snapshots)))
bs_label = labelarray[bs.id]
import_nodes, leaf_nodes, other_nodes = get_node_sinks(
bs_label, labelarray, parent_dictionary, child_dictionary)
# Load all those building snapshots
tree_nodes = itertools.chain(import_nodes, leaf_nodes,
other_nodes)
building_dict = {
bs.id: bs
for bs in seed.models.BuildingSnapshot.objects.filter(
pk__in=tree_nodes).all()
}
missing_buildings = [
node for node in itertools.chain(import_nodes, leaf_nodes,
other_nodes)
if node not in building_dict
]
import_buildingsnapshots = [
building_dict[bs_id] for bs_id in import_nodes
]
leaf_buildingsnapshots = [
building_dict[bs_id] for bs_id in leaf_nodes
]
assert len(leaf_buildingsnapshots), (
"Expected only one leaf of the canonical building family of "
"buildings. Found {}").format(len(leaf_buildingsnapshots))
leaf = leaf_buildingsnapshots[0]
other_buildingsnapshots = [
building_dict[bs_id] for bs_id in other_nodes
]
logging_info(
"Creating Blue Sky Data for for CanonicalBuilding={}".
format(cb.pk))
# embed()
create_associated_bluesky_taxlots_properties(
org, import_buildingsnapshots, leaf,
other_buildingsnapshots, child_dictionary,
parent_dictionary, adj_matrix, cb)
logging_info("END migrate_organization")
return
def handle(self, *args, **options):
"""Do something."""
logging_info("RUN create_campus_relationships_organization with args={},kwds={}".format(args, options))
if options['organization']:
core_organization = map(int, options['organization'].split(","))
else:
core_organization = get_core_organizations()
for org_id in core_organization:
# Writing loop over organizations
org = Organization.objects.filter(pk=org_id).first()
logging.info("Processing organization {}".format(org))
assert org, "Organization {} not found".format(org_id)
property_views = PropertyView.objects.filter(cycle__organization=org) \
.exclude(state__pm_parent_property_id=None) \
.exclude(state__pm_parent_property_id="Not Applicable: Standalone Property") \
.all()
property_views = list(property_views)
property_views.sort(key=lambda pv: pv.cycle.start)
states = map(lambda pv: pv.state, list(property_views))
# All property views where the state has a parent property that isn't "Not Applicable."
for (pv, state) in zip(property_views, states):
pm_parent_property_id = state.pm_parent_property_id
# What is the difference between these two fields?
if pm_parent_property_id == state.pm_property_id:
logging_info("Auto reference: site id={}/pm_property_id={} is it's own campus (pm_parent_property_id={})".format(pv.property.id, state.pm_property_id, state.pm_parent_property_id))
prop = pv.property
prop.campus = True
prop.save()
continue
parent_property = find_property_associated_with_portfolio_manager_id(pm_parent_property_id)
if not parent_property:
logging_info("Could not find parent property with pm_property_id={}. Creating new Property/View/State for it.".format(pm_parent_property_id))
parent_property = Property(organization_id=org_id)
parent_property.campus = True
parent_property.save()
# Create a view and a state for the active cycle.
parent_property_state = PropertyState(organization=org,
pm_property_id=pm_parent_property_id,
pm_parent_property_id=pm_parent_property_id,
property_notes="Created by campus relations migration on {}".format(
datetime.datetime.now().strftime(
"%Y-%m-%d %H:%M")))
parent_property_state.save()
parent_property_view = PropertyView(property=parent_property, cycle=pv.cycle,
state=parent_property_state)
parent_property_view.save()
child_property = pv.property
child_property = child_property.parent_property = parent_property
child_property.save()
else:
logging_info("Found property matching pm_parent_property_id={}".format(pm_parent_property_id))
parent_property.campus = True
parent_property.save()
child_property = pv.property
child_property.parent_property = parent_property
child_property.save()
# Make sure the parent has a view for the same
# cycle as the pv in question.
if not PropertyView.objects.filter(property=parent_property,
cycle=pv.cycle).count():
parent_views = list(PropertyView.objects.filter(property=parent_property).all())
parent_views.sort(key=lambda pv: pv.cycle.start)
# parent_views = [ppv for ppv in parent_views if ppv.cycle.start <= pv.cycle.start]
assert len(parent_views), "This should always be true."
ps = parent_views[-1].state
ps.pk = None
ps.save()
parent_view = PropertyView(property=parent_property, cycle=pv.cycle,
state=ps)
parent_view.save()
logging_info("END create_campus_relationships_organization")
return
def handle(self, *args, **options):
logging_info("RUN migrate_extradata_columns with args={},kwds={}".format(args, options))
# Process Arguments
if options['organization']:
organization_ids = map(int, options['organization'].split(","))
else:
organization_ids = get_core_organizations()
clear_bluesky_labels = options["clear_bluesky_labels"]
add_property_labels = options["add_property_labels"]
add_taxlot_labels = options["add_taxlot_labels"]
for org_id in organization_ids:
##############################
# Handle the clear case. This is a bit inelegant the
# way the loop on org_ids is setup.
if clear_bluesky_labels:
print "Org={}: Clearing all labels on Property and TaxLot objects.".format(org_id)
for prop in Property.objects.filter(organization = org_id).all():
prop.labels.clear()
for tax_lot in TaxLot.objects.filter(organization = org_id).all():
tax_lot.labels.clear()
continue
# End Clear Case
##############################
print ("Org={}: Migrating Labels with settings add_property_labels={}"
", add_taxlot_labels={}").format(org_id, add_property_labels, add_taxlot_labels)
##############################
# Copy Property
if add_property_labels:
# This is inefficient, in that it does each
# property/tax lot multiple times for each of it's
# views - but it's clear and shouldn't be
# prohibitively wasteful.
# Alternatively you could grab the first
# propertyview/taxlotview for each property/taxlot
# and then use the state on that to populate the
# property/taxlot.
for pv in PropertyView.objects.filter(property__organization = org_id).select_related('state').all():
if not "prop_cb_id" in pv.state.extra_data:
print "Warning: key 'prop_cb_id' was not found for PropertyView={}".format(pv)
continue
cb_id = pv.state.extra_data['prop_cb_id']
try:
cb = CanonicalBuilding.objects.get(pk=cb_id)
except ObjectDoesNotExist, xcpt:
print "Warning: Canonical Building={} was not found in the DB".format(cb_id)
continue
cb_labels = cb.labels.all()
preexisting_pv_labels = set(map(lambda l: l.pk, pv.property.labels.all()))
for label in cb_labels:
if label.pk not in preexisting_pv_labels:
pv.property.labels.add(label)
else:
pv.property.save()
#
##############################
##############################
# Copy Tax Lot labels
if add_taxlot_labels:
for tlv in TaxLotView.objects.filter(taxlot__organization = org_id).select_related('state').all():
if not "taxlot_cb_id" in tlv.state.extra_data:
print "Warning: key 'prop_cb_id' was not found for TaxLotView={}".format(tlv)
continue
cb_id = tlv.state.extra_data['taxlot_cb_id']
try:
cb = CanonicalBuilding.objects.get(pk=cb_id)
except ObjectDoesNotExist, xcpt:
print "Warning: Canonical Building={} was not found in the DB".format(cb_id)
continue
cb_labels = cb.labels.all()
preexisting_tlv_labels = set(map(lambda l: l.pk, tlv.taxlot.labels.all()))
for label in cb_labels:
if label.pk not in preexisting_tlv_labels:
tlv.taxlot.labels.add(label)
else:
tlv.taxlot.save()
def handle(self, *args, **options):
"""Migrate the CanonicalBuildings for one or more Organizations into
the new 'BlueSky' data structures."""
logging_info("RUN migrate_organization with args={},kwds={}".format(args, options))
# Process Arguments
if options['organization']:
core_organization = map(int, options['organization'].split(","))
else:
core_organization = get_core_organizations()
limit = options['limit'] if "limit" in options else 0
starting_from_canonical = False if not options['starting_on_canonical'] else options[
'starting_on_canonical']
starting_on_canonical_following = False if not options['starting_following_canonical'] else \
options['starting_following_canonical']
ADD_METADATA = options['add_metadata']
assert not (
starting_on_canonical_following and starting_from_canonical), "Can only specify one of --starting_on_canonical and --starting_following_canonical"
canonical_buildings_whitelist = map(int, options['cb_whitelist_string'].split(",")) if \
options['cb_whitelist_string'] else False
# End Processing
tree_file = get_static_building_snapshot_tree_file()
m2m = read_building_snapshot_tree_structure(tree_file)
all_nodes = set(map(projection_onto_index(0), m2m)).union(
set(map(projection_onto_index(1), m2m)))
child_dictionary = collections.defaultdict(lambda: set())
parent_dictionary = collections.defaultdict(lambda: set())
adj_dim = max(all_nodes) + 1
adj_matrix = dok_matrix((adj_dim, adj_dim), dtype=np.bool)
for (from_node, to_node) in m2m:
adj_matrix[from_node, to_node] = 1
child_dictionary[from_node].add(to_node)
parent_dictionary[to_node].add(from_node)
# Core data struct, possible refactor point.
all_nodes, child_dictionary, parent_dictionary, adj_matrix
# We don't care about the total number because
_, labelarray = connected_components(adj_matrix)
counts = collections.Counter()
for label in labelarray: counts[label] += 1
logging_info("Migration organization: {}".format(",".join(map(str, core_organization))))
for org_id in core_organization:
# Writing loop over organizations
org = Organization.objects.get(pk=org_id)
logging_info("Processing organization {}".format(org))
assert org, "Organization {} not found".format(org_id)
org_canonical_buildings = seed.models.CanonicalBuilding.objects.filter(
canonical_snapshot__super_organization=org_id, active=True).all()
org_canonical_snapshots = [cb.canonical_snapshot for cb in org_canonical_buildings]
# FIXME: Turns out the ids are on Building
# Snapshot. Leaving it this way because the other code
# should be displaying canonical building indexes.
if starting_from_canonical or starting_on_canonical_following:
# org_canonical_ids = map(lambda x: x.pk, org_canonical_buildings)
org_canonical_ids = map(lambda x: x.pk, org_canonical_snapshots)
try:
canonical_index = max(starting_from_canonical, starting_on_canonical_following)
canonical_index = org_canonical_ids.index(canonical_index)
if starting_on_canonical_following: canonical_index += 1
org_canonical_buildings = list(org_canonical_buildings)[canonical_index:]
org_canonical_snapshots = list(org_canonical_snapshots)[canonical_index:]
ref_str = "starting" if starting_from_canonical else "following"
logging_info(
"Restricting to canonical starting ndx={} (id={}), was {} now {}.".format(
canonical_index, len(org_canonical_ids), len(org_canonical_ids),
len(org_canonical_buildings)))
except ValueError:
raise RuntimeError(
"Requested to start referencing canonical building id={} which was not found.".format(
starting_from_canonical))
if canonical_buildings_whitelist:
good_canonical_building_indexes = [ndx for (ndx, cb) in
enumerate(org_canonical_buildings) if
cb.pk in canonical_buildings_whitelist]
org_canonical_buildings = [org_canonical_buildings[ndx] for ndx in
good_canonical_building_indexes]
org_canonical_snapshots = [org_canonical_snapshots[ndx] for ndx in
good_canonical_building_indexes]
logging_info(
"Restricted to {} elements in whitelist.".format(len(org_canonical_buildings)))
logging_info("IDS: {}".format(
", ".join(map(lambda cs: str(cs.pk), org_canonical_buildings))))
if canonical_buildings_whitelist:
good_canonical_building_indexes = [ndx for (ndx, cb) in
enumerate(org_canonical_buildings) if
cb.pk in canonical_buildings_whitelist]
org_canonical_buildings = [org_canonical_buildings[ndx] for ndx in
good_canonical_building_indexes]
org_canonical_snapshots = [org_canonical_snapshots[ndx] for ndx in
good_canonical_building_indexes]
logging_info(
"Restricted to {} elements in whitelist.".format(len(org_canonical_buildings)))
logging_info("IDS: {}".format(
", ".join(map(lambda cs: str(cs.pk), org_canonical_buildings))))
if len(org_canonical_buildings) == 0:
logging_info("Organization {} has no buildings".format(org_id))
continue
last_date = max([cs.modified for cs in org_canonical_snapshots])
# create_bluesky_cycles_for_org(org, last_date)
tree_sizes = [counts[labelarray[bs.id]] for bs in org_canonical_snapshots]
## For each of those trees find the tip
## For each of those trees find the import records
## For each of those trees find the cycles associated with it
for ndx, (cb, bs) in enumerate(zip(org_canonical_buildings, org_canonical_snapshots)):
if limit and (ndx + 1) > limit:
logging_info("Migrated limit={} buildings.".format(limit))
logging_info("Skipping remainder of buildings for organization.")
break
logging_info(
"Processing Building {}/{}".format(ndx + 1, len(org_canonical_snapshots)))
bs_label = labelarray[bs.id]
import_nodes, leaf_nodes, other_nodes = get_node_sinks(bs_label, labelarray,
parent_dictionary,
child_dictionary)
# Load all those building snapshots
tree_nodes = itertools.chain(import_nodes, leaf_nodes, other_nodes)
building_dict = {bs.id: bs for bs in seed.models.BuildingSnapshot.objects.filter(
pk__in=tree_nodes).all()}
missing_buildings = [node for node in
itertools.chain(import_nodes, leaf_nodes, other_nodes) if
node not in building_dict]
import_buildingsnapshots = [building_dict[bs_id] for bs_id in import_nodes]
leaf_buildingsnapshots = [building_dict[bs_id] for bs_id in leaf_nodes]
assert len(leaf_buildingsnapshots), (
"Expected only one leaf of the canonical building family of "
"buildings. Found {}").format(len(leaf_buildingsnapshots))
leaf = leaf_buildingsnapshots[0]
other_buildingsnapshots = [building_dict[bs_id] for bs_id in other_nodes]
logging_info("Creating Blue Sky Data for for CanonicalBuilding={}".format(cb.pk))
# embed()
create_associated_bluesky_taxlots_properties(org, import_buildingsnapshots, leaf,
other_buildingsnapshots,
child_dictionary, parent_dictionary,
adj_matrix, cb)
logging_info("END migrate_organization")
return
def handle(self, *args, **options):
"""Do something."""
logging_info("RUN create_campus_relationships_organization with args={},kwds={}".format(args, options))
if options['organization']:
core_organization = map(int, options['organization'].split(","))
else:
core_organization = get_core_organizations()
for org_id in core_organization:
# Writing loop over organizations
org = Organization.objects.filter(pk=org_id).first()
logging.info("Processing organization {}".format(org))
assert org, "Organization {} not found".format(org_id)
property_views = PropertyView.objects.filter(cycle__organization=org) \
.exclude(state__pm_parent_property_id=None) \
.exclude(state__pm_parent_property_id="Not Applicable: Standalone Property") \
.all()
property_views = list(property_views)
property_views.sort(key=lambda pv: pv.cycle.start)
states = map(lambda pv: pv.state, list(property_views))
# All property views where the state has a parent property that isn't "Not Applicable."
for (pv, state) in zip(property_views, states):
pm_parent_property_id = state.pm_parent_property_id
# What is the difference between these two fields?
if pm_parent_property_id == state.pm_property_id:
logging_info("Auto reference: site id={}/pm_property_id={} is it's own campus (pm_parent_property_id={})".format(pv.property.id, state.pm_property_id, state.pm_parent_property_id))
prop = pv.property
prop.campus = True
prop.save()
continue
parent_property = find_property_associated_with_portfolio_manager_id(pm_parent_property_id)
if not parent_property:
logging_info("Could not find parent property with pm_property_id={}. Creating new Property/View/State for it.".format(pm_parent_property_id))
parent_property = Property(organization_id=org_id)
parent_property.campus = True
parent_property.save()
# Create a view and a state for the active cycle.
parent_property_state = PropertyState(organization=org,
pm_property_id=pm_parent_property_id,
pm_parent_property_id=pm_parent_property_id,
property_notes="Created by campus relations migration on {}".format(
datetime.datetime.now().strftime(
"%Y-%m-%d %H:%M")))
parent_property_state.save()
parent_property_view = PropertyView(property=parent_property, cycle=pv.cycle,
state=parent_property_state)
parent_property_view.save()
child_property = pv.property
child_property = child_property.parent_property = parent_property
child_property.save()
else:
logging_info("Found property matching pm_parent_property_id={}".format(pm_parent_property_id))
parent_property.campus = True
parent_property.save()
child_property = pv.property
child_property.parent_property = parent_property
child_property.save()
# Make sure the parent has a view for the same
# cycle as the pv in question.
if not PropertyView.objects.filter(property=parent_property,
cycle=pv.cycle).count():
parent_views = list(PropertyView.objects.filter(property=parent_property).all())
parent_views.sort(key=lambda pv: pv.cycle.start)
# parent_views = [ppv for ppv in parent_views if ppv.cycle.start <= pv.cycle.start]
assert len(parent_views), "This should always be true."
ps = parent_views[-1].state
ps.pk = None
ps.save()
parent_view = PropertyView(property=parent_property, cycle=pv.cycle,
state=ps)
parent_view.save()
logging_info("END create_campus_relationships_organization")
return
def getOrganizationsToDelete():
"""Get all organizations that aren't in the global white list."""
all_organizations = seed.models.Organization.objects.all()
bad_organizations = [org for org in all_organizations if org.id not in get_core_organizations()]
return bad_organizations
