def handle(self, *args, **options): logging_info("RUN create_m2m_relatinships_organization with args={},kwds={}".format(args, options)) if options['organization']: core_organization = map(int, options['organization'].split(",")) else: core_organization = get_core_organizations() logging_info("Processing organization list: {}".format(core_organization)) if options['stats']: for org_id in core_organization: self.display_stats(org_id) return org_taxlot_splitdata_rules = collections.defaultdict(lambda : (UNIQUE_LIST, ("jurisdiction_tax_lot_id",))) org_taxlot_splitdata_rules[20] = (UNIQUE_LIST, ("jurisdiction_tax_lot_id", "extra_data/Philadelphia Building ID",)) for org_id in core_organization: self.split_taxlots_into_m2m_relationships(org_id, org_taxlot_splitdata_rules) # At the end run two checks: # Go through the tax Lots, collect any that are left, make # sure they aren't a part of any m2m entities. for view in TaxLotView.objects.filter(taxlot__organization_id=org_id).all(): try: # pdb.set_trace() # aggregate_value_from_state(view.state, org_taxlot_split_extra[org_id]) taxlot_field_list = get_id_fields(view.state.jurisdiction_tax_lot_id) if len(taxlot_field_list) > 1: logging_warn("Danger - tax lot '{}' still exists.".format(view.state.jurisdiction_tax_lot_id)) except TaxLotIDValueError, e: continue
def handle(self, *args, **options): logging_info( "RUN migrate_extradata_columns with args={},kwds={}".format( args, options)) if options['organization']: organization_ids = map(int, options['organization'].split(",")) else: organization_ids = get_core_organizations() update_columns = options['update_columns'] add_unmapped_columns = options['add_unmapped_columns'] create_missing_columns = options['create_missing_columns'] for org_id in organization_ids: org = Organization.objects.get(pk=org_id) # Update if update_columns: self.update_columns_based_on_mappings_file(org) if create_missing_columns: self.find_missing_columns_based_on_extra_data(org) logging_info("END migrate_extradata_columns") return
def handle(self, *args, **options): logging_info( "RUN migrate_extradata_columns with args={},kwds={}".format(args, options)) if options['organization']: organization_ids = map(int, options['organization'].split(",")) else: organization_ids = get_core_organizations() update_columns = options['update_columns'] add_unmapped_columns = options['add_unmapped_columns'] create_missing_columns = options['create_missing_columns'] for org_id in organization_ids: org = Organization.objects.get(pk=org_id) # Update if update_columns: self.update_columns_based_on_mappings_file(org) if create_missing_columns: self.find_missing_columns_based_on_extra_data(org) logging_info("END migrate_extradata_columns") return
def getOrganizationsToDelete(): """Get all organizations that are not in the global white list.""" all_organizations = seed.models.Organization.objects.all() bad_organizations = [ org for org in all_organizations if org.id not in get_core_organizations() ] return bad_organizations
def handle(self, *args, **options): logging_info( "RUN create_m2m_relatinships_organization with args={},kwds={}". format(args, options)) if options['organization']: core_organization = map(int, options['organization'].split(",")) else: core_organization = get_core_organizations() logging_info( "Processing organization list: {}".format(core_organization)) if options['stats']: for org_id in core_organization: self.display_stats(org_id) return org_taxlot_splitdata_rules = collections.defaultdict( lambda: (UNIQUE_LIST, ("jurisdiction_tax_lot_id", ))) org_taxlot_splitdata_rules[20] = (UNIQUE_LIST, ( "jurisdiction_tax_lot_id", "extra_data/Philadelphia Building ID", )) for org_id in core_organization: self.split_taxlots_into_m2m_relationships( org_id, org_taxlot_splitdata_rules) # At the end run two checks: # Go through the tax Lots, collect any that are left, make # sure they are not a part of any m2m entities. for view in TaxLotView.objects.filter( taxlot__organization_id=org_id).all(): try: # pdb.set_trace() # aggregate_value_from_state(view.state, org_taxlot_split_extra[org_id]) taxlot_field_list = get_id_fields( view.state.jurisdiction_tax_lot_id) if len(taxlot_field_list) > 1: logging_warn("Danger - tax lot '{}' still exists.".format( view.state.jurisdiction_tax_lot_id)) except TaxLotIDValueError, e: continue
def handle(self, *args, **options): """Go through organization by organization and look for m2m.""" logging_info("RUN create_primarysecondary_taxlots with args={},kwds={}".format(args, options)) if options['organization']: core_organization = map(int, options['organization'].split(",")) else: core_organization = get_core_organizations() for org_id in core_organization: # Writing loop over organizations org = Organization.objects.filter(pk=org_id).first() logging_info("Processing organization {}".format(org)) assert org, "Organization {} not found".format(org_id) self.assign_primarysecondary_tax_lots(org) logging_info("END create_primarysecondary_taxlots") return
def handle(self, *args, **options): """Go through organization by organization and look for m2m.""" logging_info( "RUN create_primarysecondary_taxlots with args={},kwds={}".format( args, options)) if options['organization']: core_organization = map(int, options['organization'].split(",")) else: core_organization = get_core_organizations() for org_id in core_organization: # Writing loop over organizations org = Organization.objects.filter(pk=org_id).first() logging_info("Processing organization {}".format(org)) assert org, "Organization {} not found".format(org_id) self.assign_primarysecondary_tax_lots(org) logging_info("END create_primarysecondary_taxlots") return
def handle(self, *args, **options): logging_info( "RUN migrate_extradata_columns with args={},kwds={}".format( args, options)) # Process Arguments if options['organization']: organization_ids = map(int, options['organization'].split(",")) else: organization_ids = get_core_organizations() clear_bluesky_labels = options["clear_bluesky_labels"] add_property_labels = options["add_property_labels"] add_taxlot_labels = options["add_taxlot_labels"] for org_id in organization_ids: ############################## # Handle the clear case. This is a bit inelegant the # way the loop on org_ids is setup. if clear_bluesky_labels: print "Org={}: Clearing all labels on Property and TaxLot objects.".format( org_id) for prop in Property.objects.filter(organization=org_id).all(): prop.labels.clear() for tax_lot in TaxLot.objects.filter( organization=org_id).all(): tax_lot.labels.clear() continue # End Clear Case ############################## print( "Org={}: Migrating Labels with settings add_property_labels={}" ", add_taxlot_labels={}").format(org_id, add_property_labels, add_taxlot_labels) ############################## # Copy Property if add_property_labels: # This is inefficient, in that it does each # property/tax lot multiple times for each of it's # views - but it's clear and should not be # prohibitively wasteful. # Alternatively you could grab the first # propertyview/taxlotview for each property/taxlot # and then use the state on that to populate the # property/taxlot. for pv in PropertyView.objects.filter( property__organization=org_id).select_related( 'state').all(): if not "prop_cb_id" in pv.state.extra_data: print "Warning: key 'prop_cb_id' was not found for PropertyView={}".format( pv) continue cb_id = pv.state.extra_data['prop_cb_id'] try: cb = CanonicalBuilding.objects.get(pk=cb_id) except ObjectDoesNotExist, xcpt: print "Warning: Canonical Building={} was not found in the DB".format( cb_id) continue cb_labels = cb.labels.all() preexisting_pv_labels = set( map(lambda l: l.pk, pv.property.labels.all())) for label in cb_labels: if label.pk not in preexisting_pv_labels: pv.property.labels.add(label) else: pv.property.save() # ############################## ############################## # Copy Tax Lot labels if add_taxlot_labels: for tlv in TaxLotView.objects.filter( taxlot__organization=org_id).select_related( 'state').all(): if not "taxlot_cb_id" in tlv.state.extra_data: print "Warning: key 'prop_cb_id' was not found for TaxLotView={}".format( tlv) continue cb_id = tlv.state.extra_data['taxlot_cb_id'] try: cb = CanonicalBuilding.objects.get(pk=cb_id) except ObjectDoesNotExist, xcpt: print "Warning: Canonical Building={} was not found in the DB".format( cb_id) continue cb_labels = cb.labels.all() preexisting_tlv_labels = set( map(lambda l: l.pk, tlv.taxlot.labels.all())) for label in cb_labels: if label.pk not in preexisting_tlv_labels: tlv.taxlot.labels.add(label) else: tlv.taxlot.save()
def handle(self, *args, **options): """Migrate the CanonicalBuildings for one or more Organizations into the new 'BlueSky' data structures.""" logging_info("RUN migrate_organization with args={},kwds={}".format( args, options)) # Process Arguments if options['organization']: core_organization = map(int, options['organization'].split(",")) else: core_organization = get_core_organizations() limit = options['limit'] if "limit" in options else 0 starting_from_canonical = False if not options[ 'starting_on_canonical'] else options['starting_on_canonical'] starting_on_canonical_following = False if not options['starting_following_canonical'] else \ options['starting_following_canonical'] ADD_METADATA = options['add_metadata'] assert not ( starting_on_canonical_following and starting_from_canonical ), "Can only specify one of --starting_on_canonical and --starting_following_canonical" canonical_buildings_whitelist = map(int, options['cb_whitelist_string'].split(",")) if \ options['cb_whitelist_string'] else False # End Processing tree_file = get_static_building_snapshot_tree_file() m2m = read_building_snapshot_tree_structure(tree_file) all_nodes = set(map(projection_onto_index(0), m2m)).union(set(map(projection_onto_index(1), m2m))) child_dictionary = collections.defaultdict(lambda: set()) parent_dictionary = collections.defaultdict(lambda: set()) adj_dim = max(all_nodes) + 1 adj_matrix = dok_matrix((adj_dim, adj_dim), dtype=np.bool) for (from_node, to_node) in m2m: adj_matrix[from_node, to_node] = 1 child_dictionary[from_node].add(to_node) parent_dictionary[to_node].add(from_node) # Core data struct, possible refactor point. all_nodes, child_dictionary, parent_dictionary, adj_matrix # We don't care about the total number because _, labelarray = connected_components(adj_matrix) counts = collections.Counter() for label in labelarray: counts[label] += 1 logging_info("Migration organization: {}".format(",".join( map(str, core_organization)))) for org_id in core_organization: # Writing loop over organizations org = Organization.objects.get(pk=org_id) logging_info("Processing organization {}".format(org)) assert org, "Organization {} not found".format(org_id) org_canonical_buildings = seed.models.CanonicalBuilding.objects.filter( canonical_snapshot__super_organization=org_id, active=True).all() org_canonical_snapshots = [ cb.canonical_snapshot for cb in org_canonical_buildings ] # FIXME: Turns out the ids are on Building # Snapshot. Leaving it this way because the other code # should be displaying canonical building indexes. if starting_from_canonical or starting_on_canonical_following: # org_canonical_ids = map(lambda x: x.pk, org_canonical_buildings) org_canonical_ids = map(lambda x: x.pk, org_canonical_snapshots) try: canonical_index = max(starting_from_canonical, starting_on_canonical_following) canonical_index = org_canonical_ids.index(canonical_index) if starting_on_canonical_following: canonical_index += 1 org_canonical_buildings = list( org_canonical_buildings)[canonical_index:] org_canonical_snapshots = list( org_canonical_snapshots)[canonical_index:] ref_str = "starting" if starting_from_canonical else "following" logging_info( "Restricting to canonical starting ndx={} (id={}), was {} now {}." .format(canonical_index, len(org_canonical_ids), len(org_canonical_ids), len(org_canonical_buildings))) except ValueError: raise RuntimeError( "Requested to start referencing canonical building id={} which was not found." .format(starting_from_canonical)) if canonical_buildings_whitelist: good_canonical_building_indexes = [ ndx for (ndx, cb) in enumerate(org_canonical_buildings) if cb.pk in canonical_buildings_whitelist ] org_canonical_buildings = [ org_canonical_buildings[ndx] for ndx in good_canonical_building_indexes ] org_canonical_snapshots = [ org_canonical_snapshots[ndx] for ndx in good_canonical_building_indexes ] logging_info("Restricted to {} elements in whitelist.".format( len(org_canonical_buildings))) logging_info("IDS: {}".format(", ".join( map(lambda cs: str(cs.pk), org_canonical_buildings)))) if canonical_buildings_whitelist: good_canonical_building_indexes = [ ndx for (ndx, cb) in enumerate(org_canonical_buildings) if cb.pk in canonical_buildings_whitelist ] org_canonical_buildings = [ org_canonical_buildings[ndx] for ndx in good_canonical_building_indexes ] org_canonical_snapshots = [ org_canonical_snapshots[ndx] for ndx in good_canonical_building_indexes ] logging_info("Restricted to {} elements in whitelist.".format( len(org_canonical_buildings))) logging_info("IDS: {}".format(", ".join( map(lambda cs: str(cs.pk), org_canonical_buildings)))) if len(org_canonical_buildings) == 0: logging_info("Organization {} has no buildings".format(org_id)) continue last_date = max([cs.modified for cs in org_canonical_snapshots]) # create_bluesky_cycles_for_org(org, last_date) tree_sizes = [ counts[labelarray[bs.id]] for bs in org_canonical_snapshots ] ## For each of those trees find the tip ## For each of those trees find the import records ## For each of those trees find the cycles associated with it for ndx, (cb, bs) in enumerate( zip(org_canonical_buildings, org_canonical_snapshots)): if limit and (ndx + 1) > limit: logging_info("Migrated limit={} buildings.".format(limit)) logging_info( "Skipping remainder of buildings for organization.") break logging_info("Processing Building {}/{}".format( ndx + 1, len(org_canonical_snapshots))) bs_label = labelarray[bs.id] import_nodes, leaf_nodes, other_nodes = get_node_sinks( bs_label, labelarray, parent_dictionary, child_dictionary) # Load all those building snapshots tree_nodes = itertools.chain(import_nodes, leaf_nodes, other_nodes) building_dict = { bs.id: bs for bs in seed.models.BuildingSnapshot.objects.filter( pk__in=tree_nodes).all() } missing_buildings = [ node for node in itertools.chain(import_nodes, leaf_nodes, other_nodes) if node not in building_dict ] import_buildingsnapshots = [ building_dict[bs_id] for bs_id in import_nodes ] leaf_buildingsnapshots = [ building_dict[bs_id] for bs_id in leaf_nodes ] assert len(leaf_buildingsnapshots), ( "Expected only one leaf of the canonical building family of " "buildings. Found {}").format(len(leaf_buildingsnapshots)) leaf = leaf_buildingsnapshots[0] other_buildingsnapshots = [ building_dict[bs_id] for bs_id in other_nodes ] logging_info( "Creating Blue Sky Data for for CanonicalBuilding={}". format(cb.pk)) # embed() create_associated_bluesky_taxlots_properties( org, import_buildingsnapshots, leaf, other_buildingsnapshots, child_dictionary, parent_dictionary, adj_matrix, cb) logging_info("END migrate_organization") return
def handle(self, *args, **options): """Do something.""" logging_info("RUN create_campus_relationships_organization with args={},kwds={}".format(args, options)) if options['organization']: core_organization = map(int, options['organization'].split(",")) else: core_organization = get_core_organizations() for org_id in core_organization: # Writing loop over organizations org = Organization.objects.filter(pk=org_id).first() logging.info("Processing organization {}".format(org)) assert org, "Organization {} not found".format(org_id) property_views = PropertyView.objects.filter(cycle__organization=org) \ .exclude(state__pm_parent_property_id=None) \ .exclude(state__pm_parent_property_id="Not Applicable: Standalone Property") \ .all() property_views = list(property_views) property_views.sort(key=lambda pv: pv.cycle.start) states = map(lambda pv: pv.state, list(property_views)) # All property views where the state has a parent property that isn't "Not Applicable." for (pv, state) in zip(property_views, states): pm_parent_property_id = state.pm_parent_property_id # What is the difference between these two fields? if pm_parent_property_id == state.pm_property_id: logging_info("Auto reference: site id={}/pm_property_id={} is it's own campus (pm_parent_property_id={})".format(pv.property.id, state.pm_property_id, state.pm_parent_property_id)) prop = pv.property prop.campus = True prop.save() continue parent_property = find_property_associated_with_portfolio_manager_id(pm_parent_property_id) if not parent_property: logging_info("Could not find parent property with pm_property_id={}. Creating new Property/View/State for it.".format(pm_parent_property_id)) parent_property = Property(organization_id=org_id) parent_property.campus = True parent_property.save() # Create a view and a state for the active cycle. parent_property_state = PropertyState(organization=org, pm_property_id=pm_parent_property_id, pm_parent_property_id=pm_parent_property_id, property_notes="Created by campus relations migration on {}".format( datetime.datetime.now().strftime( "%Y-%m-%d %H:%M"))) parent_property_state.save() parent_property_view = PropertyView(property=parent_property, cycle=pv.cycle, state=parent_property_state) parent_property_view.save() child_property = pv.property child_property = child_property.parent_property = parent_property child_property.save() else: logging_info("Found property matching pm_parent_property_id={}".format(pm_parent_property_id)) parent_property.campus = True parent_property.save() child_property = pv.property child_property.parent_property = parent_property child_property.save() # Make sure the parent has a view for the same # cycle as the pv in question. if not PropertyView.objects.filter(property=parent_property, cycle=pv.cycle).count(): parent_views = list(PropertyView.objects.filter(property=parent_property).all()) parent_views.sort(key=lambda pv: pv.cycle.start) # parent_views = [ppv for ppv in parent_views if ppv.cycle.start <= pv.cycle.start] assert len(parent_views), "This should always be true." ps = parent_views[-1].state ps.pk = None ps.save() parent_view = PropertyView(property=parent_property, cycle=pv.cycle, state=ps) parent_view.save() logging_info("END create_campus_relationships_organization") return
def handle(self, *args, **options): logging_info("RUN migrate_extradata_columns with args={},kwds={}".format(args, options)) # Process Arguments if options['organization']: organization_ids = map(int, options['organization'].split(",")) else: organization_ids = get_core_organizations() clear_bluesky_labels = options["clear_bluesky_labels"] add_property_labels = options["add_property_labels"] add_taxlot_labels = options["add_taxlot_labels"] for org_id in organization_ids: ############################## # Handle the clear case. This is a bit inelegant the # way the loop on org_ids is setup. if clear_bluesky_labels: print "Org={}: Clearing all labels on Property and TaxLot objects.".format(org_id) for prop in Property.objects.filter(organization = org_id).all(): prop.labels.clear() for tax_lot in TaxLot.objects.filter(organization = org_id).all(): tax_lot.labels.clear() continue # End Clear Case ############################## print ("Org={}: Migrating Labels with settings add_property_labels={}" ", add_taxlot_labels={}").format(org_id, add_property_labels, add_taxlot_labels) ############################## # Copy Property if add_property_labels: # This is inefficient, in that it does each # property/tax lot multiple times for each of it's # views - but it's clear and shouldn't be # prohibitively wasteful. # Alternatively you could grab the first # propertyview/taxlotview for each property/taxlot # and then use the state on that to populate the # property/taxlot. for pv in PropertyView.objects.filter(property__organization = org_id).select_related('state').all(): if not "prop_cb_id" in pv.state.extra_data: print "Warning: key 'prop_cb_id' was not found for PropertyView={}".format(pv) continue cb_id = pv.state.extra_data['prop_cb_id'] try: cb = CanonicalBuilding.objects.get(pk=cb_id) except ObjectDoesNotExist, xcpt: print "Warning: Canonical Building={} was not found in the DB".format(cb_id) continue cb_labels = cb.labels.all() preexisting_pv_labels = set(map(lambda l: l.pk, pv.property.labels.all())) for label in cb_labels: if label.pk not in preexisting_pv_labels: pv.property.labels.add(label) else: pv.property.save() # ############################## ############################## # Copy Tax Lot labels if add_taxlot_labels: for tlv in TaxLotView.objects.filter(taxlot__organization = org_id).select_related('state').all(): if not "taxlot_cb_id" in tlv.state.extra_data: print "Warning: key 'prop_cb_id' was not found for TaxLotView={}".format(tlv) continue cb_id = tlv.state.extra_data['taxlot_cb_id'] try: cb = CanonicalBuilding.objects.get(pk=cb_id) except ObjectDoesNotExist, xcpt: print "Warning: Canonical Building={} was not found in the DB".format(cb_id) continue cb_labels = cb.labels.all() preexisting_tlv_labels = set(map(lambda l: l.pk, tlv.taxlot.labels.all())) for label in cb_labels: if label.pk not in preexisting_tlv_labels: tlv.taxlot.labels.add(label) else: tlv.taxlot.save()
def handle(self, *args, **options): """Migrate the CanonicalBuildings for one or more Organizations into the new 'BlueSky' data structures.""" logging_info("RUN migrate_organization with args={},kwds={}".format(args, options)) # Process Arguments if options['organization']: core_organization = map(int, options['organization'].split(",")) else: core_organization = get_core_organizations() limit = options['limit'] if "limit" in options else 0 starting_from_canonical = False if not options['starting_on_canonical'] else options[ 'starting_on_canonical'] starting_on_canonical_following = False if not options['starting_following_canonical'] else \ options['starting_following_canonical'] ADD_METADATA = options['add_metadata'] assert not ( starting_on_canonical_following and starting_from_canonical), "Can only specify one of --starting_on_canonical and --starting_following_canonical" canonical_buildings_whitelist = map(int, options['cb_whitelist_string'].split(",")) if \ options['cb_whitelist_string'] else False # End Processing tree_file = get_static_building_snapshot_tree_file() m2m = read_building_snapshot_tree_structure(tree_file) all_nodes = set(map(projection_onto_index(0), m2m)).union( set(map(projection_onto_index(1), m2m))) child_dictionary = collections.defaultdict(lambda: set()) parent_dictionary = collections.defaultdict(lambda: set()) adj_dim = max(all_nodes) + 1 adj_matrix = dok_matrix((adj_dim, adj_dim), dtype=np.bool) for (from_node, to_node) in m2m: adj_matrix[from_node, to_node] = 1 child_dictionary[from_node].add(to_node) parent_dictionary[to_node].add(from_node) # Core data struct, possible refactor point. all_nodes, child_dictionary, parent_dictionary, adj_matrix # We don't care about the total number because _, labelarray = connected_components(adj_matrix) counts = collections.Counter() for label in labelarray: counts[label] += 1 logging_info("Migration organization: {}".format(",".join(map(str, core_organization)))) for org_id in core_organization: # Writing loop over organizations org = Organization.objects.get(pk=org_id) logging_info("Processing organization {}".format(org)) assert org, "Organization {} not found".format(org_id) org_canonical_buildings = seed.models.CanonicalBuilding.objects.filter( canonical_snapshot__super_organization=org_id, active=True).all() org_canonical_snapshots = [cb.canonical_snapshot for cb in org_canonical_buildings] # FIXME: Turns out the ids are on Building # Snapshot. Leaving it this way because the other code # should be displaying canonical building indexes. if starting_from_canonical or starting_on_canonical_following: # org_canonical_ids = map(lambda x: x.pk, org_canonical_buildings) org_canonical_ids = map(lambda x: x.pk, org_canonical_snapshots) try: canonical_index = max(starting_from_canonical, starting_on_canonical_following) canonical_index = org_canonical_ids.index(canonical_index) if starting_on_canonical_following: canonical_index += 1 org_canonical_buildings = list(org_canonical_buildings)[canonical_index:] org_canonical_snapshots = list(org_canonical_snapshots)[canonical_index:] ref_str = "starting" if starting_from_canonical else "following" logging_info( "Restricting to canonical starting ndx={} (id={}), was {} now {}.".format( canonical_index, len(org_canonical_ids), len(org_canonical_ids), len(org_canonical_buildings))) except ValueError: raise RuntimeError( "Requested to start referencing canonical building id={} which was not found.".format( starting_from_canonical)) if canonical_buildings_whitelist: good_canonical_building_indexes = [ndx for (ndx, cb) in enumerate(org_canonical_buildings) if cb.pk in canonical_buildings_whitelist] org_canonical_buildings = [org_canonical_buildings[ndx] for ndx in good_canonical_building_indexes] org_canonical_snapshots = [org_canonical_snapshots[ndx] for ndx in good_canonical_building_indexes] logging_info( "Restricted to {} elements in whitelist.".format(len(org_canonical_buildings))) logging_info("IDS: {}".format( ", ".join(map(lambda cs: str(cs.pk), org_canonical_buildings)))) if canonical_buildings_whitelist: good_canonical_building_indexes = [ndx for (ndx, cb) in enumerate(org_canonical_buildings) if cb.pk in canonical_buildings_whitelist] org_canonical_buildings = [org_canonical_buildings[ndx] for ndx in good_canonical_building_indexes] org_canonical_snapshots = [org_canonical_snapshots[ndx] for ndx in good_canonical_building_indexes] logging_info( "Restricted to {} elements in whitelist.".format(len(org_canonical_buildings))) logging_info("IDS: {}".format( ", ".join(map(lambda cs: str(cs.pk), org_canonical_buildings)))) if len(org_canonical_buildings) == 0: logging_info("Organization {} has no buildings".format(org_id)) continue last_date = max([cs.modified for cs in org_canonical_snapshots]) # create_bluesky_cycles_for_org(org, last_date) tree_sizes = [counts[labelarray[bs.id]] for bs in org_canonical_snapshots] ## For each of those trees find the tip ## For each of those trees find the import records ## For each of those trees find the cycles associated with it for ndx, (cb, bs) in enumerate(zip(org_canonical_buildings, org_canonical_snapshots)): if limit and (ndx + 1) > limit: logging_info("Migrated limit={} buildings.".format(limit)) logging_info("Skipping remainder of buildings for organization.") break logging_info( "Processing Building {}/{}".format(ndx + 1, len(org_canonical_snapshots))) bs_label = labelarray[bs.id] import_nodes, leaf_nodes, other_nodes = get_node_sinks(bs_label, labelarray, parent_dictionary, child_dictionary) # Load all those building snapshots tree_nodes = itertools.chain(import_nodes, leaf_nodes, other_nodes) building_dict = {bs.id: bs for bs in seed.models.BuildingSnapshot.objects.filter( pk__in=tree_nodes).all()} missing_buildings = [node for node in itertools.chain(import_nodes, leaf_nodes, other_nodes) if node not in building_dict] import_buildingsnapshots = [building_dict[bs_id] for bs_id in import_nodes] leaf_buildingsnapshots = [building_dict[bs_id] for bs_id in leaf_nodes] assert len(leaf_buildingsnapshots), ( "Expected only one leaf of the canonical building family of " "buildings. Found {}").format(len(leaf_buildingsnapshots)) leaf = leaf_buildingsnapshots[0] other_buildingsnapshots = [building_dict[bs_id] for bs_id in other_nodes] logging_info("Creating Blue Sky Data for for CanonicalBuilding={}".format(cb.pk)) # embed() create_associated_bluesky_taxlots_properties(org, import_buildingsnapshots, leaf, other_buildingsnapshots, child_dictionary, parent_dictionary, adj_matrix, cb) logging_info("END migrate_organization") return
def getOrganizationsToDelete(): """Get all organizations that aren't in the global white list.""" all_organizations = seed.models.Organization.objects.all() bad_organizations = [org for org in all_organizations if org.id not in get_core_organizations()] return bad_organizations