def create_layer_selection(config_entity, layer, attribute_id):
db_entity = layer.db_entity_interest.db_entity
connection = connection_dict(layer.config_entity.db)
tilestache_layers = []
users = set(get_users_with_perms(config_entity)) | set(
get_users_with_perms(layer.db_entity_interest.db_entity))
# Make sure layer_selection instances exist for the users
from footprint.main.publishing.layer_publishing import update_or_create_layer_selections_for_layer
update_or_create_layer_selections_for_layer(layer, users=users)
logger.info("Get/Create layer_selection for config_entity %s, layer %s, users %s" %\
(config_entity.key, layer.db_entity_key, ','.join(map(lambda user: user.username, users))))
# Each layer has a dynamic class representing its SelectedFeature table
get_or_create_layer_selection_class_for_layer(layer)
if not users:
return tilestache_layers
config_entity.db_entity_feature_class(key=layer.db_entity_key)
layer_selection_class = get_or_create_layer_selection_class_for_layer(
layer, config_entity)
# Take the first user to create a template query
user = list(users)[0]
# Each LayerSelection instance is per user
layer_selection = layer_selection_class.objects.get_or_create(user=user)[0]
# Extract the query from the QuerySet
query = re.sub(
r'"layer_selection_id" = \d+', r'"layer_selection_id" = {user_id}',
str(
layer_selection.selected_features.values('wkb_geometry',
'id').query))
logger.info(
"Creating tilestache layer_selection for layer %s, user %s, query: %s"
% (layer.full_name, user.username, query))
user_id_lookup = map_to_dict(
lambda layer_selection: [layer_selection.user.id, layer_selection.id],
layer_selection_class.objects.all())
# Embed the id in the Geojson for each feature.
# Nothing else is needed, since all other attributes can be looked up based on the id
id_field = map(lambda field: field.name + '_id',
layer_selection.feature_class._meta.parents.values())[0]
vector_selection_layer = build_vector_layer_config(
parameters=merge(
connection,
dict(query=query,
column="wkb_geometry",
user_id_lookup=user_id_lookup)),
provider_id_property=id_field,
client_id_property=db_entity._meta.pk.name)
layer_key = "layer:{layer},attr_id:{attribute},type:{type}".format(
layer=layer.id, attribute=attribute_id, type='selection')
logger.info("Creating layer %s" % layer_key)
tilestache_layers.append(
TSLayer(key=layer_key, value=vector_selection_layer))
return tilestache_layers
def create_layer_selections(layers):
"""
Create LayerSelection classes and instances for the given Scenario subclasses among the
classes in limit_to_classes. Also filters by db_entity_key if they are specified
:return:
"""
for config_entity in unique(map(lambda layer: layer.config_entity, layers)):
FeatureClassCreator(config_entity).ensure_dynamic_models()
for selection_layer in layers:
# Recreate
get_or_create_layer_selection_class_for_layer(selection_layer)
update_or_create_layer_selections(config_entity=None)
def create_layer_selections(layers):
"""
Create LayerSelection classes and instances for the given Scenario subclasses among the
classes in limit_to_classes. Also filters by db_entity_key if they are specified
:return:
"""
for config_entity in unique(map(lambda layer: layer.config_entity,
layers)):
FeatureClassCreator(config_entity).ensure_dynamic_models()
for selection_layer in layers:
# Recreate
get_or_create_layer_selection_class_for_layer(selection_layer)
update_or_create_layer_selections(config_entity=None)
def create_subclass(self, params, **kwargs):
"""
Subclasses the LayerSelectionResource instance's class for the given config_entity and layer.
:param params Must contain a 'config_entity__id' and 'layer__id'
:return:
"""
layer = self.resolve_layer(params)
config_entity = self.resolve_config_entity(params)
logger.debug(
"Resolving LayerSelection for config_entity %s, layer %s" %
(config_entity.key, layer.db_entity.key))
layer_selection_class = get_or_create_layer_selection_class_for_layer(
layer, config_entity)
# Have the LayerPublisher create the LayerSelection instance for the user if needed
update_or_create_layer_selections_for_layer(
layer, users=[self.resolve_user(params)])
if not layer_selection_class:
raise Exception(
"Layer with db_entity_key %s has no feature_class. Its LayerSelections should not be requested"
% layer.db_entity_key)
return get_dynamic_resource_class(self.__class__,
layer_selection_class)
def update_or_create_layer_selections_for_layer(layer, users=None):
"""
Create LayerSelections for all users for this layer.
:param layer: The Layer instance
:param users: Optional users for which to create LayerSelection instances. These
users must have permission to read the DbEntity and ConfigEntity of the Layer, or
an exception will be thrown
"""
# Do nothing if the layer doesn't have features, such as background imagery
if not layer.db_entity_interest.db_entity.feature_class_configuration:
return
layer_selection_class = get_or_create_layer_selection_class_for_layer(layer)
if layer_selection_class and users:
logger.info("Getting/Creating LayerSelection instances for Layer of DbEntity Key: %s" % layer.full_name)
for authorizable_instance in [layer.config_entity.subclassed,
layer.db_entity_interest.db_entity]:
# This will raise an Exception if there any unauthorized users
authorizable_instance.validate_permissions(users, permission_key_class=DbEntityPermissionKey)
# Create an instance for each user in the list
for user in users:
if not layer_selection_class.objects.filter(user=user).count():
layer_selection_class.objects.create(user=user)
def update_or_create_layer_selections_for_layer(layer, users=None):
"""
Create LayerSelections for all users for this layer.
:param layer: The Layer instance
:param users: Optional users for which to create LayerSelection instances. These
users must have permission to read the DbEntity and ConfigEntity of the Layer, or
an exception will be thrown
"""
# Do nothing if the layer doesn't have features, such as background imagery
if not layer.db_entity_interest.db_entity.feature_class_configuration:
return
layer_selection_class = get_or_create_layer_selection_class_for_layer(
layer)
if layer_selection_class and users:
logger.info(
"Getting/Creating LayerSelection instances for Layer of DbEntity Key: %s"
% layer.full_name)
for authorizable_instance in [
layer.config_entity.subclassed,
layer.db_entity_interest.db_entity
]:
# This will raise an Exception if there any unauthorized users
authorizable_instance.validate_permissions(
users, permission_key_class=DbEntityPermissionKey)
# Create an instance for each user in the list
for user in users:
if not layer_selection_class.objects.filter(user=user).count():
layer_selection_class.objects.create(user=user)
def importer(self, config_entity, db_entity, **kwargs):
"""
Replaces the normal ImportProcessor importer with one to import a shapefile from disk
"""
user = db_entity.creator
if InformationSchema.objects.table_exists(db_entity.schema, db_entity.table):
# The table already exists. Skip the import an log a warning
logger.warn("The target table for the layer selection import already exists. Skipping table import.")
else:
feature_class_creator = FeatureClassCreator(config_entity, db_entity)
origin_feature_class_configuration = db_entity.origin_instance.feature_class_configuration
# Create the new DbEntity FeatureClassConfiguration from the origin's. Pass in what has already been
# created for the new feature_class_configuration. This should have things like generated=True
feature_class_configuration = feature_class_creator.complete_or_create_feature_class_configuration(
origin_feature_class_configuration,
**merge(db_entity.feature_class_configuration.__dict__, dict(generated=True)))
# Update the DbEntity
feature_class_creator.update_db_entity(feature_class_configuration)
if feature_class_configuration.source_from_origin_layer_selection and \
feature_class_configuration.origin_layer_id:
# If desired, limit the layer clone to that of the source layer's current LayerSelection for the
# User doing the update
layer_selection_class = get_or_create_layer_selection_class_for_layer(
Layer.objects.get(id=feature_class_configuration.origin_layer_id), True)
layer_selection = layer_selection_class.objects.get(user=user)
features = layer_selection.selected_features
else:
# Leave blank to copy all features by default
features = None
DefaultImportProcessor().peer_importer(config_entity, db_entity, import_from_origin=True, source_queryset=features)
def create_layer_from_layer_selection(self, params):
"""
Used to create a new Layer from the current LayerSelection features
:param params:
:return:
"""
# Resolve the source layer from the layer_selection__id
source_layer = self.resolve_layer(params)
config_entity = source_layer.config_entity
db_entity = source_layer.db_entity_interest.db_enitty
feature_class = FeatureClassCreator(config_entity,
db_entity).dynamic_model_class()
layer = Layer.objects.get(presentation__config_entity=config_entity,
db_entity_key=db_entity.key)
layer_selection = get_or_create_layer_selection_class_for_layer(
layer, config_entity, False).objects.all()[0]
# TODO no need to do geojson here
feature_dict = dict(type="Feature")
feature_dicts = map(
lambda feature:
deep_merge(feature_dict,
{"geometry": geojson.loads(feature.wkb_geometry.json)}),
layer_selection.selected_features or feature_class.objects.all())
json = dict({"type": "FeatureCollection", "features": feature_dicts})
db_entity_configuration = update_or_create_db_entity(
config_entity,
**dict(class_scope=FutureScenario,
name='Import From Selection Test',
key='import_selection_test',
url='file://notusingthis'))
self.make_geojson_db_entity(config_entity,
db_entity_configuration,
data=json)
def create_layer_from_layer_selection(self, params):
"""
Used to create a new Layer from the current LayerSelection features
:param params:
:return:
"""
# Resolve the source layer from the layer_selection__id
source_layer = self.resolve_layer(params)
config_entity = source_layer.config_entity
db_entity = source_layer.db_entity_interest.db_enitty
feature_class = FeatureClassCreator(config_entity, db_entity).dynamic_model_class()
layer = Layer.objects.get(presentation__config_entity=config_entity, db_entity_key=db_entity.key)
layer_selection = get_or_create_layer_selection_class_for_layer(layer, config_entity, False).objects.all()[0]
# TODO no need to do geojson here
feature_dict = dict(
type="Feature"
)
feature_dicts = map(lambda feature:
deep_merge(feature_dict, {"geometry":geojson.loads(feature.wkb_geometry.json)}),
layer_selection.selected_features or feature_class.objects.all())
json = dict({"type": "FeatureCollection", "features": feature_dicts})
db_entity_configuration = update_or_create_db_entity(config_entity, **dict(
class_scope=FutureScenario,
name='Import From Selection Test',
key='import_selection_test',
url='file://notusingthis'
))
self.make_geojson_db_entity(config_entity, db_entity_configuration, data=json)
def create_layer_selection(config_entity, layer, attribute_id):
db_entity = layer.db_entity_interest.db_entity
connection = connection_dict(layer.config_entity.db)
tilestache_layers = []
users = set(get_users_with_perms(config_entity)) | set(get_users_with_perms(layer.db_entity_interest.db_entity))
# Make sure layer_selection instances exist for the users
from footprint.main.publishing.layer_publishing import update_or_create_layer_selections_for_layer
update_or_create_layer_selections_for_layer(layer, users=users)
logger.info("Get/Create layer_selection for config_entity %s, layer %s, users %s" %\
(config_entity.key, layer.db_entity_key, ','.join(map(lambda user: user.username, users))))
# Each layer has a dynamic class representing its SelectedFeature table
get_or_create_layer_selection_class_for_layer(layer)
if not users:
return tilestache_layers
config_entity.db_entity_feature_class(key=layer.db_entity_key)
layer_selection_class = get_or_create_layer_selection_class_for_layer(layer, config_entity)
# Take the first user to create a template query
user = list(users)[0]
# Each LayerSelection instance is per user
layer_selection = layer_selection_class.objects.get_or_create(user=user)[0]
# Extract the query from the QuerySet
query = re.sub(
r'"layer_selection_id" = \d+',
r'"layer_selection_id" = {user_id}',
str(layer_selection.selected_features.values('wkb_geometry', 'id').query))
logger.info("Creating tilestache layer_selection for layer %s, user %s, query: %s" % (layer.full_name, user.username, query))
user_id_lookup = map_to_dict(lambda layer_selection: [layer_selection.user.id, layer_selection.id], layer_selection_class.objects.all())
# Embed the id in the Geojson for each feature.
# Nothing else is needed, since all other attributes can be looked up based on the id
id_field = map(lambda field: field.name + '_id', layer_selection.feature_class._meta.parents.values())[0]
vector_selection_layer = build_vector_layer_config(
parameters=merge(connection, dict(query=query, column="wkb_geometry", user_id_lookup=user_id_lookup)),
provider_id_property=id_field,
client_id_property=db_entity._meta.pk.name
)
layer_key = "layer:{layer},attr_id:{attribute},type:{type}".format(layer=layer.id, attribute=attribute_id, type='selection')
logger.info("Creating layer %s" % layer_key)
tilestache_layers.append(TSLayer(key=layer_key, value=vector_selection_layer))
return tilestache_layers
def resolve_layer_selection(self, params):
"""
Used to get that actual selected features, which is a short cut querying, so we don't have to query
for potentially thousands of ids. If No layer exists then there is also no LayerSelection, in which case
we return None
"""
layer = self.resolve_layer(params)
config_entity = self.resolve_config_entity(params)
if not layer:
return None
update_or_create_layer_selections_for_layer(layer, users=[self.resolve_user(params)])
layer_selection_class = get_or_create_layer_selection_class_for_layer(layer, config_entity, False)
return layer_selection_class.objects.get(user=self.resolve_user(params))
def handle(self, *args, **options):
if not options['skip']:
application_initialization()
update_or_create_config_entities()
user = update_or_create_users()['user']
scenarios = update_or_create_scenarios()
for scenario in scenarios:
layer_library = scenario.presentation_set.filter(key=Keys.LAYER_LIBRARY_DEFAULT)[0]
presentation_medium = layer_library.presentationmedium_set.get(db_entity_key=Keys.DB_ABSTRACT_PRIMARY_PARCEL_SOURCE)
layer = Layer.objects.get(id=presentation_medium.id) # Cast
layer_selection_class = get_or_create_layer_selection_class_for_layer(layer, scenario)
layer_selection = layer_selection_class.objects.get(user=user, layer=layer)
layer_selection.save()
def resolve_layer_selection(self, params):
"""
Used to get that actual selected features, which is a short cut querying, so we don't have to query
for potentially thousands of ids. If No layer exists then there is also no LayerSelection, in which case
we return None
"""
layer = self.resolve_layer(params)
config_entity = self.resolve_config_entity(params)
if not layer:
return None
update_or_create_layer_selections_for_layer(
layer, users=[self.resolve_user(params)])
layer_selection_class = get_or_create_layer_selection_class_for_layer(
layer, config_entity, False)
return layer_selection_class.objects.get(
user=self.resolve_user(params))
def delete_layer_selections(layers):
for config_entity in unique(map(lambda layer: layer.config_entity, layers)):
FeatureClassCreator(config_entity).ensure_dynamic_models()
for selection_layer in layers:
try:
# Drop the table
layer_selection_class = get_or_create_layer_selection_class_for_layer(selection_layer, no_table_creation=True)
if layer_selection_class:
if hasattr(layer_selection_class.features, 'through'):
layer_selection_features_class = layer_selection_class.features.through
drop_layer_selection_table(layer_selection_features_class)
drop_layer_selection_table(layer_selection_class)
except DatabaseError, e:
logger.warning(
"Couldn't destroy LayerSelection tables. Maybe the public.layer table no longer exists: %s" % e.message)
def handle(self, *args, **options):
if not options['skip']:
application_initialization()
update_or_create_config_entities()
user = update_or_create_users()['user']
scenarios = update_or_create_scenarios()
for scenario in scenarios:
layer_library = scenario.presentation_set.filter(
key=Keys.LAYER_LIBRARY_DEFAULT)[0]
presentation_medium = layer_library.presentationmedium_set.get(
db_entity_key=Keys.DB_ABSTRACT_PRIMARY_PARCEL_SOURCE)
layer = Layer.objects.get(id=presentation_medium.id) # Cast
layer_selection_class = get_or_create_layer_selection_class_for_layer(
layer, scenario)
layer_selection = layer_selection_class.objects.get(user=user,
layer=layer)
layer_selection.save()
def delete_layer_selections(layers):
for config_entity in unique(map(lambda layer: layer.config_entity,
layers)):
FeatureClassCreator(config_entity).ensure_dynamic_models()
for selection_layer in layers:
try:
# Drop the table
layer_selection_class = get_or_create_layer_selection_class_for_layer(
selection_layer, no_table_creation=True)
if layer_selection_class:
if hasattr(layer_selection_class.features, 'through'):
layer_selection_features_class = layer_selection_class.features.through
drop_layer_selection_table(layer_selection_features_class)
drop_layer_selection_table(layer_selection_class)
except DatabaseError, e:
logger.warning(
"Couldn't destroy LayerSelection tables. Maybe the public.layer table no longer exists: %s"
% e.message)
def create_subclass(self, params, **kwargs):
"""
Subclasses the LayerSelectionResource instance's class for the given config_entity and layer.
:param params Must contain a 'config_entity__id' and 'layer__id'
:return:
"""
layer = self.resolve_layer(params)
config_entity = self.resolve_config_entity(params)
logger.debug("Resolving LayerSelection for config_entity %s, layer %s" % (config_entity.key, layer.db_entity.key))
layer_selection_class = get_or_create_layer_selection_class_for_layer(layer, config_entity)
# Have the LayerPublisher create the LayerSelection instance for the user if needed
update_or_create_layer_selections_for_layer(layer, users=[self.resolve_user(params)])
if not layer_selection_class:
raise Exception("Layer with db_entity_key %s has no feature_class. Its LayerSelections should not be requested" % layer.db_entity_key)
return get_dynamic_resource_class(
self.__class__,
layer_selection_class
)
def run_footprint_init(self, *args, **options):
if not settings.CELERY_ALWAYS_EAGER:
raise Exception('This command must run with settings.CELERY_ALWAYS_EQUAL = True. '
'Add --settings=footprint.settings_init to the command line.')
db_entity_keys = options.get('db_entity_keys').split(',') if options.get('db_entity_keys') else None
# Replace so we can use options as kwargs
options['db_entity_keys'] = db_entity_keys
config_entity_keys = options.get('config_entity_keys').split(',') if options.get('config_entity_keys') else None
# Replace so we can use options as kwargs
options['config_entity_keys'] = config_entity_keys
if not options.get('run_analysis'):
AnalysisModule._no_post_save_task_run_global = True
limit_to_classes = map(
lambda cls: resolve_model('main.%s' % cls), (options.get('class').split(',') if options.get('class') else [])
)
options['limit_to_classes'] = limit_to_classes
# Perforance testing
if options.get('memory'):
ConfigEntity.init_heapy()
ConfigEntity.start_heapy_diagnosis()
# Delete all ConfigEntity intances so they can be recreated.
# This will cascade delete related models, but it doesn't delete
# BuiltForms and other independent models
if options.get('recreate'):
for cls in filter_classes(limit_to_classes):
cls.objects.all().delete()
# Delete deleted config_entities
if options.get('recycle'):
for cls in filter_classes(limit_to_classes):
cls.objects.filter(deleted=True).delete()
if options.get('delete_clones') or options.get('delete_scenario_clones'):
# Delete clones and uploads
for cls in filter_classes(limit_to_classes):
all_config_entities = cls.objects.all()
for config_entity in all_config_entities:
if options.get('delete_clones'):
db_entities = map(
lambda db_entity_interest: db_entity_interest.db_entity,
DbEntityInterest.objects.filter(
config_entity=config_entity,
db_entity__origin_instance__isnull=False)
) +\
filter(
lambda db_entity: db_entity.feature_class_configuration and \
(isinstance(db_entity.feature_class_configuration, dict) or
db_entity.feature_class_configuration.generated),
config_entity.computed_db_entities())
layers_to_remove = Layer.objects.filter(layer_libraries__config_entity__in=[config_entity], db_entity_interest__db_entity__key__in=map(lambda db_entity: db_entity.key, db_entities))
for layer in layers_to_remove:
# Drop the layer_selection classes
layer_selection_class = get_or_create_layer_selection_class_for_layer(layer)
drop_tables_for_dynamic_classes(
layer_selection_class,
layer_selection_class.features.field.rel.through
)
layers_to_remove.delete()
for layer in Layer.objects.all():
try:
layer.db_entity_interest.db_entity
except:
# orphan
try:
# Drop the layer_selection classes
layer_selection_class = get_or_create_layer_selection_class_for_layer(layer)
drop_tables_for_dynamic_classes(
layer_selection_class,
layer_selection_class.features.field.rel.through
)
layer.delete()
except:
pass
# DbEntities
for db_entity in db_entities:
feature_class = None
try:
feature_class = FeatureClassCreator(config_entity, db_entity).dynamic_model_class()
except Exception, e:
logger.warn("No feature class for db_entity %s could be created. Exception: %s" % (db_entity.name, e.message))
DeleteImportProcessor().drop_data(config_entity, db_entity)
db_entity.delete()
if issubclass(cls, Scenario):
cloned_config_entities = cls.objects.filter(origin_instance__isnull=False)
# ConfigEntities and their schemas
if options.get('delete_clones') or options.get('delete_scenario_clones'):
for config_entity in cloned_config_entities:
PGNamespace.objects.drop_schema(config_entity.schema())
for db_entity in config_entity.owned_db_entities():
db_entity.delete()
cloned_config_entities.delete()
if options.get('delete_clones') and False:
for built_form_set in BuiltFormSet.objects.all():
built_form_set.built_forms.remove(*built_form_set.built_forms.filter(origin_instance__isnull=False))
# BuiltForms
BuiltForm.objects.filter(origin_instance__isnull=False).delete()
# Orphaned BuiltForm assets (only an issue when corrupt saves have happened)
BuildingAttributeSet.objects.annotate(
num_buildings=Count('building'), num_buildingtypes=Count('buildingtype'), num_placetypes=Count('building')).filter(
num_buildings=0, num_buildingtypes=0, num_placetypes=0).delete()
Medium.objects.annotate(num_built_form_sets=Count('builtform')).filter(num_built_form_sets=0, key__startswith='built_form').delete()
BuiltFormExample.objects.annotate(num_built_form_sets=Count('builtform')).filter(num_built_form_sets=0).delete()
