def create_layer_selection(config_entity, layer, attribute_id): db_entity = layer.db_entity_interest.db_entity connection = connection_dict(layer.config_entity.db) tilestache_layers = [] users = set(get_users_with_perms(config_entity)) | set( get_users_with_perms(layer.db_entity_interest.db_entity)) # Make sure layer_selection instances exist for the users from footprint.main.publishing.layer_publishing import update_or_create_layer_selections_for_layer update_or_create_layer_selections_for_layer(layer, users=users) logger.info("Get/Create layer_selection for config_entity %s, layer %s, users %s" %\ (config_entity.key, layer.db_entity_key, ','.join(map(lambda user: user.username, users)))) # Each layer has a dynamic class representing its SelectedFeature table get_or_create_layer_selection_class_for_layer(layer) if not users: return tilestache_layers config_entity.db_entity_feature_class(key=layer.db_entity_key) layer_selection_class = get_or_create_layer_selection_class_for_layer( layer, config_entity) # Take the first user to create a template query user = list(users)[0] # Each LayerSelection instance is per user layer_selection = layer_selection_class.objects.get_or_create(user=user)[0] # Extract the query from the QuerySet query = re.sub( r'"layer_selection_id" = \d+', r'"layer_selection_id" = {user_id}', str( layer_selection.selected_features.values('wkb_geometry', 'id').query)) logger.info( "Creating tilestache layer_selection for layer %s, user %s, query: %s" % (layer.full_name, user.username, query)) user_id_lookup = map_to_dict( lambda layer_selection: [layer_selection.user.id, layer_selection.id], layer_selection_class.objects.all()) # Embed the id in the Geojson for each feature. # Nothing else is needed, since all other attributes can be looked up based on the id id_field = map(lambda field: field.name + '_id', layer_selection.feature_class._meta.parents.values())[0] vector_selection_layer = build_vector_layer_config( parameters=merge( connection, dict(query=query, column="wkb_geometry", user_id_lookup=user_id_lookup)), provider_id_property=id_field, client_id_property=db_entity._meta.pk.name) layer_key = "layer:{layer},attr_id:{attribute},type:{type}".format( layer=layer.id, attribute=attribute_id, type='selection') logger.info("Creating layer %s" % layer_key) tilestache_layers.append( TSLayer(key=layer_key, value=vector_selection_layer)) return tilestache_layers
def create_layer_selections(layers): """ Create LayerSelection classes and instances for the given Scenario subclasses among the classes in limit_to_classes. Also filters by db_entity_key if they are specified :return: """ for config_entity in unique(map(lambda layer: layer.config_entity, layers)): FeatureClassCreator(config_entity).ensure_dynamic_models() for selection_layer in layers: # Recreate get_or_create_layer_selection_class_for_layer(selection_layer) update_or_create_layer_selections(config_entity=None)
def create_layer_selections(layers): """ Create LayerSelection classes and instances for the given Scenario subclasses among the classes in limit_to_classes. Also filters by db_entity_key if they are specified :return: """ for config_entity in unique(map(lambda layer: layer.config_entity, layers)): FeatureClassCreator(config_entity).ensure_dynamic_models() for selection_layer in layers: # Recreate get_or_create_layer_selection_class_for_layer(selection_layer) update_or_create_layer_selections(config_entity=None)
def create_subclass(self, params, **kwargs): """ Subclasses the LayerSelectionResource instance's class for the given config_entity and layer. :param params Must contain a 'config_entity__id' and 'layer__id' :return: """ layer = self.resolve_layer(params) config_entity = self.resolve_config_entity(params) logger.debug( "Resolving LayerSelection for config_entity %s, layer %s" % (config_entity.key, layer.db_entity.key)) layer_selection_class = get_or_create_layer_selection_class_for_layer( layer, config_entity) # Have the LayerPublisher create the LayerSelection instance for the user if needed update_or_create_layer_selections_for_layer( layer, users=[self.resolve_user(params)]) if not layer_selection_class: raise Exception( "Layer with db_entity_key %s has no feature_class. Its LayerSelections should not be requested" % layer.db_entity_key) return get_dynamic_resource_class(self.__class__, layer_selection_class)
def update_or_create_layer_selections_for_layer(layer, users=None): """ Create LayerSelections for all users for this layer. :param layer: The Layer instance :param users: Optional users for which to create LayerSelection instances. These users must have permission to read the DbEntity and ConfigEntity of the Layer, or an exception will be thrown """ # Do nothing if the layer doesn't have features, such as background imagery if not layer.db_entity_interest.db_entity.feature_class_configuration: return layer_selection_class = get_or_create_layer_selection_class_for_layer(layer) if layer_selection_class and users: logger.info("Getting/Creating LayerSelection instances for Layer of DbEntity Key: %s" % layer.full_name) for authorizable_instance in [layer.config_entity.subclassed, layer.db_entity_interest.db_entity]: # This will raise an Exception if there any unauthorized users authorizable_instance.validate_permissions(users, permission_key_class=DbEntityPermissionKey) # Create an instance for each user in the list for user in users: if not layer_selection_class.objects.filter(user=user).count(): layer_selection_class.objects.create(user=user)
def update_or_create_layer_selections_for_layer(layer, users=None): """ Create LayerSelections for all users for this layer. :param layer: The Layer instance :param users: Optional users for which to create LayerSelection instances. These users must have permission to read the DbEntity and ConfigEntity of the Layer, or an exception will be thrown """ # Do nothing if the layer doesn't have features, such as background imagery if not layer.db_entity_interest.db_entity.feature_class_configuration: return layer_selection_class = get_or_create_layer_selection_class_for_layer( layer) if layer_selection_class and users: logger.info( "Getting/Creating LayerSelection instances for Layer of DbEntity Key: %s" % layer.full_name) for authorizable_instance in [ layer.config_entity.subclassed, layer.db_entity_interest.db_entity ]: # This will raise an Exception if there any unauthorized users authorizable_instance.validate_permissions( users, permission_key_class=DbEntityPermissionKey) # Create an instance for each user in the list for user in users: if not layer_selection_class.objects.filter(user=user).count(): layer_selection_class.objects.create(user=user)
def importer(self, config_entity, db_entity, **kwargs): """ Replaces the normal ImportProcessor importer with one to import a shapefile from disk """ user = db_entity.creator if InformationSchema.objects.table_exists(db_entity.schema, db_entity.table): # The table already exists. Skip the import an log a warning logger.warn("The target table for the layer selection import already exists. Skipping table import.") else: feature_class_creator = FeatureClassCreator(config_entity, db_entity) origin_feature_class_configuration = db_entity.origin_instance.feature_class_configuration # Create the new DbEntity FeatureClassConfiguration from the origin's. Pass in what has already been # created for the new feature_class_configuration. This should have things like generated=True feature_class_configuration = feature_class_creator.complete_or_create_feature_class_configuration( origin_feature_class_configuration, **merge(db_entity.feature_class_configuration.__dict__, dict(generated=True))) # Update the DbEntity feature_class_creator.update_db_entity(feature_class_configuration) if feature_class_configuration.source_from_origin_layer_selection and \ feature_class_configuration.origin_layer_id: # If desired, limit the layer clone to that of the source layer's current LayerSelection for the # User doing the update layer_selection_class = get_or_create_layer_selection_class_for_layer( Layer.objects.get(id=feature_class_configuration.origin_layer_id), True) layer_selection = layer_selection_class.objects.get(user=user) features = layer_selection.selected_features else: # Leave blank to copy all features by default features = None DefaultImportProcessor().peer_importer(config_entity, db_entity, import_from_origin=True, source_queryset=features)
def create_layer_from_layer_selection(self, params): """ Used to create a new Layer from the current LayerSelection features :param params: :return: """ # Resolve the source layer from the layer_selection__id source_layer = self.resolve_layer(params) config_entity = source_layer.config_entity db_entity = source_layer.db_entity_interest.db_enitty feature_class = FeatureClassCreator(config_entity, db_entity).dynamic_model_class() layer = Layer.objects.get(presentation__config_entity=config_entity, db_entity_key=db_entity.key) layer_selection = get_or_create_layer_selection_class_for_layer( layer, config_entity, False).objects.all()[0] # TODO no need to do geojson here feature_dict = dict(type="Feature") feature_dicts = map( lambda feature: deep_merge(feature_dict, {"geometry": geojson.loads(feature.wkb_geometry.json)}), layer_selection.selected_features or feature_class.objects.all()) json = dict({"type": "FeatureCollection", "features": feature_dicts}) db_entity_configuration = update_or_create_db_entity( config_entity, **dict(class_scope=FutureScenario, name='Import From Selection Test', key='import_selection_test', url='file://notusingthis')) self.make_geojson_db_entity(config_entity, db_entity_configuration, data=json)
def create_layer_from_layer_selection(self, params): """ Used to create a new Layer from the current LayerSelection features :param params: :return: """ # Resolve the source layer from the layer_selection__id source_layer = self.resolve_layer(params) config_entity = source_layer.config_entity db_entity = source_layer.db_entity_interest.db_enitty feature_class = FeatureClassCreator(config_entity, db_entity).dynamic_model_class() layer = Layer.objects.get(presentation__config_entity=config_entity, db_entity_key=db_entity.key) layer_selection = get_or_create_layer_selection_class_for_layer(layer, config_entity, False).objects.all()[0] # TODO no need to do geojson here feature_dict = dict( type="Feature" ) feature_dicts = map(lambda feature: deep_merge(feature_dict, {"geometry":geojson.loads(feature.wkb_geometry.json)}), layer_selection.selected_features or feature_class.objects.all()) json = dict({"type": "FeatureCollection", "features": feature_dicts}) db_entity_configuration = update_or_create_db_entity(config_entity, **dict( class_scope=FutureScenario, name='Import From Selection Test', key='import_selection_test', url='file://notusingthis' )) self.make_geojson_db_entity(config_entity, db_entity_configuration, data=json)
def create_layer_selection(config_entity, layer, attribute_id): db_entity = layer.db_entity_interest.db_entity connection = connection_dict(layer.config_entity.db) tilestache_layers = [] users = set(get_users_with_perms(config_entity)) | set(get_users_with_perms(layer.db_entity_interest.db_entity)) # Make sure layer_selection instances exist for the users from footprint.main.publishing.layer_publishing import update_or_create_layer_selections_for_layer update_or_create_layer_selections_for_layer(layer, users=users) logger.info("Get/Create layer_selection for config_entity %s, layer %s, users %s" %\ (config_entity.key, layer.db_entity_key, ','.join(map(lambda user: user.username, users)))) # Each layer has a dynamic class representing its SelectedFeature table get_or_create_layer_selection_class_for_layer(layer) if not users: return tilestache_layers config_entity.db_entity_feature_class(key=layer.db_entity_key) layer_selection_class = get_or_create_layer_selection_class_for_layer(layer, config_entity) # Take the first user to create a template query user = list(users)[0] # Each LayerSelection instance is per user layer_selection = layer_selection_class.objects.get_or_create(user=user)[0] # Extract the query from the QuerySet query = re.sub( r'"layer_selection_id" = \d+', r'"layer_selection_id" = {user_id}', str(layer_selection.selected_features.values('wkb_geometry', 'id').query)) logger.info("Creating tilestache layer_selection for layer %s, user %s, query: %s" % (layer.full_name, user.username, query)) user_id_lookup = map_to_dict(lambda layer_selection: [layer_selection.user.id, layer_selection.id], layer_selection_class.objects.all()) # Embed the id in the Geojson for each feature. # Nothing else is needed, since all other attributes can be looked up based on the id id_field = map(lambda field: field.name + '_id', layer_selection.feature_class._meta.parents.values())[0] vector_selection_layer = build_vector_layer_config( parameters=merge(connection, dict(query=query, column="wkb_geometry", user_id_lookup=user_id_lookup)), provider_id_property=id_field, client_id_property=db_entity._meta.pk.name ) layer_key = "layer:{layer},attr_id:{attribute},type:{type}".format(layer=layer.id, attribute=attribute_id, type='selection') logger.info("Creating layer %s" % layer_key) tilestache_layers.append(TSLayer(key=layer_key, value=vector_selection_layer)) return tilestache_layers
def resolve_layer_selection(self, params): """ Used to get that actual selected features, which is a short cut querying, so we don't have to query for potentially thousands of ids. If No layer exists then there is also no LayerSelection, in which case we return None """ layer = self.resolve_layer(params) config_entity = self.resolve_config_entity(params) if not layer: return None update_or_create_layer_selections_for_layer(layer, users=[self.resolve_user(params)]) layer_selection_class = get_or_create_layer_selection_class_for_layer(layer, config_entity, False) return layer_selection_class.objects.get(user=self.resolve_user(params))
def handle(self, *args, **options): if not options['skip']: application_initialization() update_or_create_config_entities() user = update_or_create_users()['user'] scenarios = update_or_create_scenarios() for scenario in scenarios: layer_library = scenario.presentation_set.filter(key=Keys.LAYER_LIBRARY_DEFAULT)[0] presentation_medium = layer_library.presentationmedium_set.get(db_entity_key=Keys.DB_ABSTRACT_PRIMARY_PARCEL_SOURCE) layer = Layer.objects.get(id=presentation_medium.id) # Cast layer_selection_class = get_or_create_layer_selection_class_for_layer(layer, scenario) layer_selection = layer_selection_class.objects.get(user=user, layer=layer) layer_selection.save()
def resolve_layer_selection(self, params): """ Used to get that actual selected features, which is a short cut querying, so we don't have to query for potentially thousands of ids. If No layer exists then there is also no LayerSelection, in which case we return None """ layer = self.resolve_layer(params) config_entity = self.resolve_config_entity(params) if not layer: return None update_or_create_layer_selections_for_layer( layer, users=[self.resolve_user(params)]) layer_selection_class = get_or_create_layer_selection_class_for_layer( layer, config_entity, False) return layer_selection_class.objects.get( user=self.resolve_user(params))
def delete_layer_selections(layers): for config_entity in unique(map(lambda layer: layer.config_entity, layers)): FeatureClassCreator(config_entity).ensure_dynamic_models() for selection_layer in layers: try: # Drop the table layer_selection_class = get_or_create_layer_selection_class_for_layer(selection_layer, no_table_creation=True) if layer_selection_class: if hasattr(layer_selection_class.features, 'through'): layer_selection_features_class = layer_selection_class.features.through drop_layer_selection_table(layer_selection_features_class) drop_layer_selection_table(layer_selection_class) except DatabaseError, e: logger.warning( "Couldn't destroy LayerSelection tables. Maybe the public.layer table no longer exists: %s" % e.message)
def handle(self, *args, **options): if not options['skip']: application_initialization() update_or_create_config_entities() user = update_or_create_users()['user'] scenarios = update_or_create_scenarios() for scenario in scenarios: layer_library = scenario.presentation_set.filter( key=Keys.LAYER_LIBRARY_DEFAULT)[0] presentation_medium = layer_library.presentationmedium_set.get( db_entity_key=Keys.DB_ABSTRACT_PRIMARY_PARCEL_SOURCE) layer = Layer.objects.get(id=presentation_medium.id) # Cast layer_selection_class = get_or_create_layer_selection_class_for_layer( layer, scenario) layer_selection = layer_selection_class.objects.get(user=user, layer=layer) layer_selection.save()
def delete_layer_selections(layers): for config_entity in unique(map(lambda layer: layer.config_entity, layers)): FeatureClassCreator(config_entity).ensure_dynamic_models() for selection_layer in layers: try: # Drop the table layer_selection_class = get_or_create_layer_selection_class_for_layer( selection_layer, no_table_creation=True) if layer_selection_class: if hasattr(layer_selection_class.features, 'through'): layer_selection_features_class = layer_selection_class.features.through drop_layer_selection_table(layer_selection_features_class) drop_layer_selection_table(layer_selection_class) except DatabaseError, e: logger.warning( "Couldn't destroy LayerSelection tables. Maybe the public.layer table no longer exists: %s" % e.message)
def create_subclass(self, params, **kwargs): """ Subclasses the LayerSelectionResource instance's class for the given config_entity and layer. :param params Must contain a 'config_entity__id' and 'layer__id' :return: """ layer = self.resolve_layer(params) config_entity = self.resolve_config_entity(params) logger.debug("Resolving LayerSelection for config_entity %s, layer %s" % (config_entity.key, layer.db_entity.key)) layer_selection_class = get_or_create_layer_selection_class_for_layer(layer, config_entity) # Have the LayerPublisher create the LayerSelection instance for the user if needed update_or_create_layer_selections_for_layer(layer, users=[self.resolve_user(params)]) if not layer_selection_class: raise Exception("Layer with db_entity_key %s has no feature_class. Its LayerSelections should not be requested" % layer.db_entity_key) return get_dynamic_resource_class( self.__class__, layer_selection_class )
def run_footprint_init(self, *args, **options): if not settings.CELERY_ALWAYS_EAGER: raise Exception('This command must run with settings.CELERY_ALWAYS_EQUAL = True. ' 'Add --settings=footprint.settings_init to the command line.') db_entity_keys = options.get('db_entity_keys').split(',') if options.get('db_entity_keys') else None # Replace so we can use options as kwargs options['db_entity_keys'] = db_entity_keys config_entity_keys = options.get('config_entity_keys').split(',') if options.get('config_entity_keys') else None # Replace so we can use options as kwargs options['config_entity_keys'] = config_entity_keys if not options.get('run_analysis'): AnalysisModule._no_post_save_task_run_global = True limit_to_classes = map( lambda cls: resolve_model('main.%s' % cls), (options.get('class').split(',') if options.get('class') else []) ) options['limit_to_classes'] = limit_to_classes # Perforance testing if options.get('memory'): ConfigEntity.init_heapy() ConfigEntity.start_heapy_diagnosis() # Delete all ConfigEntity intances so they can be recreated. # This will cascade delete related models, but it doesn't delete # BuiltForms and other independent models if options.get('recreate'): for cls in filter_classes(limit_to_classes): cls.objects.all().delete() # Delete deleted config_entities if options.get('recycle'): for cls in filter_classes(limit_to_classes): cls.objects.filter(deleted=True).delete() if options.get('delete_clones') or options.get('delete_scenario_clones'): # Delete clones and uploads for cls in filter_classes(limit_to_classes): all_config_entities = cls.objects.all() for config_entity in all_config_entities: if options.get('delete_clones'): db_entities = map( lambda db_entity_interest: db_entity_interest.db_entity, DbEntityInterest.objects.filter( config_entity=config_entity, db_entity__origin_instance__isnull=False) ) +\ filter( lambda db_entity: db_entity.feature_class_configuration and \ (isinstance(db_entity.feature_class_configuration, dict) or db_entity.feature_class_configuration.generated), config_entity.computed_db_entities()) layers_to_remove = Layer.objects.filter(layer_libraries__config_entity__in=[config_entity], db_entity_interest__db_entity__key__in=map(lambda db_entity: db_entity.key, db_entities)) for layer in layers_to_remove: # Drop the layer_selection classes layer_selection_class = get_or_create_layer_selection_class_for_layer(layer) drop_tables_for_dynamic_classes( layer_selection_class, layer_selection_class.features.field.rel.through ) layers_to_remove.delete() for layer in Layer.objects.all(): try: layer.db_entity_interest.db_entity except: # orphan try: # Drop the layer_selection classes layer_selection_class = get_or_create_layer_selection_class_for_layer(layer) drop_tables_for_dynamic_classes( layer_selection_class, layer_selection_class.features.field.rel.through ) layer.delete() except: pass # DbEntities for db_entity in db_entities: feature_class = None try: feature_class = FeatureClassCreator(config_entity, db_entity).dynamic_model_class() except Exception, e: logger.warn("No feature class for db_entity %s could be created. Exception: %s" % (db_entity.name, e.message)) DeleteImportProcessor().drop_data(config_entity, db_entity) db_entity.delete() if issubclass(cls, Scenario): cloned_config_entities = cls.objects.filter(origin_instance__isnull=False) # ConfigEntities and their schemas if options.get('delete_clones') or options.get('delete_scenario_clones'): for config_entity in cloned_config_entities: PGNamespace.objects.drop_schema(config_entity.schema()) for db_entity in config_entity.owned_db_entities(): db_entity.delete() cloned_config_entities.delete() if options.get('delete_clones') and False: for built_form_set in BuiltFormSet.objects.all(): built_form_set.built_forms.remove(*built_form_set.built_forms.filter(origin_instance__isnull=False)) # BuiltForms BuiltForm.objects.filter(origin_instance__isnull=False).delete() # Orphaned BuiltForm assets (only an issue when corrupt saves have happened) BuildingAttributeSet.objects.annotate( num_buildings=Count('building'), num_buildingtypes=Count('buildingtype'), num_placetypes=Count('building')).filter( num_buildings=0, num_buildingtypes=0, num_placetypes=0).delete() Medium.objects.annotate(num_built_form_sets=Count('builtform')).filter(num_built_form_sets=0, key__startswith='built_form').delete() BuiltFormExample.objects.annotate(num_built_form_sets=Count('builtform')).filter(num_built_form_sets=0).delete()