def importer(self, config_entity, db_entity, **kwargs):
"""
Replaces the normal ImportProcessor importer with one to import a shapefile from disk
"""
user = db_entity.creator
if InformationSchema.objects.table_exists(db_entity.schema, db_entity.table):
# The table already exists. Skip the import an log a warning
logger.warn("The target table for the layer selection import already exists. Skipping table import.")
else:
feature_class_creator = FeatureClassCreator(config_entity, db_entity)
origin_feature_class_configuration = db_entity.origin_instance.feature_class_configuration
# Create the new DbEntity FeatureClassConfiguration from the origin's. Pass in what has already been
# created for the new feature_class_configuration. This should have things like generated=True
feature_class_configuration = feature_class_creator.complete_or_create_feature_class_configuration(
origin_feature_class_configuration,
**merge(db_entity.feature_class_configuration.__dict__, dict(generated=True)))
# Update the DbEntity
feature_class_creator.update_db_entity(feature_class_configuration)
if feature_class_configuration.source_from_origin_layer_selection and \
feature_class_configuration.origin_layer_id:
# If desired, limit the layer clone to that of the source layer's current LayerSelection for the
# User doing the update
layer_selection_class = get_or_create_layer_selection_class_for_layer(
Layer.objects.get(id=feature_class_configuration.origin_layer_id), True)
layer_selection = layer_selection_class.objects.get(user=user)
features = layer_selection.selected_features
else:
# Leave blank to copy all features by default
features = None
DefaultImportProcessor().peer_importer(config_entity, db_entity, import_from_origin=True, source_queryset=features)
def db_entity_defaults(db_entity, config_entity=None):
# Instantiate a FeatureClassCreator to make the FeatureClassConfiguration
feature_class_creator = FeatureClassCreator(config_entity,
db_entity,
no_ensure=True)
if config_entity:
# Find the database of the configured client
connection = resolve_fixture(None, "init", InitFixture,
config_entity.schema()).import_database()
# Remove the _test suffix when running unit tests. Until we can auto-population the _test version
# of the database, we want to simply rely on the manually configured source database
if connection:
connection['database'] = connection['database'].replace(
'test_', '')
else:
# No config_entity abstract DbEntity case
connection = None
return dict(
# The name is passed in or the titleized version of key
name=db_entity.name or titleize(db_entity.key),
# Postgres URL for local sources, or possibly a remote url (e.g. for background layer sources)
# Unless overridden, create the url according to this postgres url scheme
url=db_entity.url or \
('postgres://{user}:{password}/{host}:{port}/{database}'.format(
**merge(dict(port=5432), connection)) if connection else None),
# Normally Equals the key, except for views of the table, like a Result DbEntity
# Views leave this null and rely on query
table=db_entity.table or (db_entity.key if not db_entity.query else None),
# Query to create a "view" of the underlying data. Used by Result DbEntity instances
query=db_entity.query,
# How to group the features or query results. Not yet well hashed out
group_by=db_entity.group_by,
# The source DbEntity key if this DbEntity resulted from cloning a peer DbEntity
source_db_entity_key=db_entity.source_db_entity_key,
# Array used by remote data sources whose URLs have different host names
# If so then the url will have a string variable for the host
hosts=db_entity.hosts,
# The User who created the DbEntity. TODO. Default should be an admin
creator=db_entity.creator if hasattr(db_entity, 'creator') else get_user_model().objects.filter()[0],
# The User who updated the DbEntity. TODO. Default should be an admin
updater=db_entity.creator if hasattr(db_entity, 'creator') else get_user_model().objects.filter()[0],
# The SRID of the Feature table
srid=db_entity.srid,
# This is a non-model object. So it is saved as a PickledObjectField
# Whether the same instance is returned or not does not matter
# If db_entity.feature_class_configuration is None, it will return None
feature_class_configuration=feature_class_creator.complete_or_create_feature_class_configuration(
db_entity.feature_class_configuration
),
no_feature_class_configuration=db_entity.no_feature_class_configuration
# feature_behavior is handled internally by DbEntity
)
def db_entity_defaults(db_entity, config_entity=None):
# Instantiate a FeatureClassCreator to make the FeatureClassConfiguration
feature_class_creator = FeatureClassCreator(config_entity, db_entity, no_ensure=True)
if config_entity:
# Find the database of the configured client
connection = resolve_fixture(None, "init", InitFixture, config_entity.schema()).import_database()
# Remove the _test suffix when running unit tests. Until we can auto-population the _test version
# of the database, we want to simply rely on the manually configured source database
if connection:
connection['database'] = connection['database'].replace('test_', '')
else:
# No config_entity abstract DbEntity case
connection = None
return dict(
# The name is passed in or the titleized version of key
name=db_entity.name or titleize(db_entity.key),
# Postgres URL for local sources, or possibly a remote url (e.g. for background layer sources)
# Unless overridden, create the url according to this postgres url scheme
url=db_entity.url or \
('postgres://{user}:{password}/{host}:{port}/{database}'.format(
**merge(dict(port=5432), connection)) if connection else None),
# Normally Equals the key, except for views of the table, like a Result DbEntity
# Views leave this null and rely on query
table=db_entity.table or (db_entity.key if not db_entity.query else None),
# Query to create a "view" of the underlying data. Used by Result DbEntity instances
query=db_entity.query,
# How to group the features or query results. Not yet well hashed out
group_by=db_entity.group_by,
# The source DbEntity key if this DbEntity resulted from cloning a peer DbEntity
source_db_entity_key=db_entity.source_db_entity_key,
# Array used by remote data sources whose URLs have different host names
# If so then the url will have a string variable for the host
hosts=db_entity.hosts,
# The User who created the DbEntity. TODO. Default should be an admin
creator=db_entity.creator if hasattr(db_entity, 'creator') else get_user_model().objects.filter()[0],
# The User who updated the DbEntity. TODO. Default should be an admin
updater=db_entity.creator if hasattr(db_entity, 'creator') else get_user_model().objects.filter()[0],
# The SRID of the Feature table
srid=db_entity.srid,
# This is a non-model object. So it is saved as a PickledObjectField
# Whether the same instance is returned or not does not matter
# If db_entity.feature_class_configuration is None, it will return None
feature_class_configuration=feature_class_creator.complete_or_create_feature_class_configuration(
db_entity.feature_class_configuration
),
no_feature_class_configuration=db_entity.no_feature_class_configuration
# feature_behavior is handled internally by DbEntity
)
def annotated_related_feature_class_pk_via_geographies(manager, config_entity, db_entity_keys):
"""
To join a related model by geographic join
"""
from footprint.main.models.feature.feature_class_creator import FeatureClassCreator
feature_class_creator = FeatureClassCreator.from_dynamic_model_class(manager.model)
def resolve_related_model_pk(db_entity_key):
related_model = config_entity.db_entity_feature_class(db_entity_key)
# The common Geography class
geography_class = feature_class_creator.common_geography_class(related_model)
geography_scope = feature_class_creator.common_geography_scope(related_model)
logger.warn("Resolved geography scope %s", geography_scope)
# Find the geographies ManyToMany fields that relates this model to the related_model
# via a Geography class. Which geography class depends on their common geography scope
geographies_field = feature_class_creator.geographies_field(geography_scope)
try:
# Find the queryable field name from the geography class to the related model
related_model_geographies_field_name = resolve_queryable_name_of_type(geography_class, related_model)
except:
# Sometimes the geography class hasn't had its fields cached properly. Fix here
clear_many_cache(geography_class)
related_model_geographies_field_name = resolve_queryable_name_of_type(geography_class, related_model)
return '%s__%s__pk' % (geographies_field.name, related_model_geographies_field_name)
pk_paths = map_to_dict(lambda db_entity_key:
[db_entity_key, Min(resolve_related_model_pk(db_entity_key))],
db_entity_keys)
return manager.annotate(**pk_paths)
def create_result_map(self, values_query_set):
related_models = self.resolve_join_models()
logger.debug(
"Creating result map for related models %s feature class %s" %
(', '.join(map(lambda r: str(r),
related_models)), self.feature_class))
feature_class_creator = FeatureClassCreator.from_dynamic_model_class(
self.feature_class)
geography_scopes = feature_class_creator.geography_scopes()
# Get the related model paths final segment. We want to map these to the db_entity_key names
related_model_path_to_name = map_to_dict(
lambda related_model: [
resolve_related_model_path_via_geographies(
self.feature_class.objects, related_model).split('__')[1],
related_model.db_entity_key
], related_models)
return values_query_set.create_result_map(
related_models=related_models,
# map_path_segments maps related object paths to their model name,
# and removes the geographies segment of the path
map_path_segments=merge(
# Map each geography scope to its corresponding field on the feature class
map_to_dict(
lambda geography_scope: [
feature_class_creator.geographies_field(geography_scope
).name, None
], geography_scopes),
related_model_path_to_name))
def create_layer_from_layer_selection(self, params):
"""
Used to create a new Layer from the current LayerSelection features
:param params:
:return:
"""
# Resolve the source layer from the layer_selection__id
source_layer = self.resolve_layer(params)
config_entity = source_layer.config_entity
db_entity = source_layer.db_entity_interest.db_enitty
feature_class = FeatureClassCreator(config_entity,
db_entity).dynamic_model_class()
layer = Layer.objects.get(presentation__config_entity=config_entity,
db_entity_key=db_entity.key)
layer_selection = get_or_create_layer_selection_class_for_layer(
layer, config_entity, False).objects.all()[0]
# TODO no need to do geojson here
feature_dict = dict(type="Feature")
feature_dicts = map(
lambda feature:
deep_merge(feature_dict,
{"geometry": geojson.loads(feature.wkb_geometry.json)}),
layer_selection.selected_features or feature_class.objects.all())
json = dict({"type": "FeatureCollection", "features": feature_dicts})
db_entity_configuration = update_or_create_db_entity(
config_entity,
**dict(class_scope=FutureScenario,
name='Import From Selection Test',
key='import_selection_test',
url='file://notusingthis'))
self.make_geojson_db_entity(config_entity,
db_entity_configuration,
data=json)
def get_sample_feature(config_entity_key, db_entity_key):
config_entity = ConfigEntity.objects.get(key=config_entity_key).subclassed
db_entity = config_entity.computed_db_entity(key=db_entity_key)
feature_class = FeatureClassCreator(config_entity,
db_entity).dynamic_model_class()
feature = feature_class.objects.filter()[0]
return feature
def feature_class(self):
"""
Resolves the concrete feature class by finding the db_entity owner
:return:
"""
return FeatureClassCreator(self.db_entity_owner,
self).dynamic_model_class()
def create_result_map(self, values_query_set):
related_models = self.resolve_join_models()
logger.debug("Creating result map for related models %s feature class %s" % (', '.join(map(lambda r: str(r), related_models)), self.feature_class))
feature_class_creator = FeatureClassCreator.from_dynamic_model_class(self.feature_class)
geography_scopes = feature_class_creator.geography_scopes()
# Get the related model paths final segment. We want to map these to the db_entity_key names
related_model_path_to_name = map_to_dict(
lambda related_model:
[resolve_related_model_path_via_geographies(
self.feature_class.objects,
related_model).split('__')[1],
related_model.db_entity_key],
related_models
)
return values_query_set.create_result_map(
related_models=related_models,
# map_path_segments maps related object paths to their model name,
# and removes the geographies segment of the path
map_path_segments=merge(
# Map each geography scope to its corresponding field on the feature class
map_to_dict(
lambda geography_scope: [
feature_class_creator.geographies_field(geography_scope).name,
None
],
geography_scopes),
related_model_path_to_name)
)
def annotated_related_feature_class_pk_via_geographies(manager, config_entity, db_entity_keys):
"""
To join a related model by geographic join
"""
from footprint.main.models.feature.feature_class_creator import FeatureClassCreator
feature_class_creator = FeatureClassCreator.from_dynamic_model_class(manager.model)
def resolve_related_model_pk(db_entity_key):
related_model = config_entity.db_entity_feature_class(db_entity_key)
# The common Geography class
geography_class = feature_class_creator.common_geography_class(related_model)
geography_scope = feature_class_creator.common_geography_scope(related_model)
logger.warn("Resolved geography scope %s", geography_scope)
# Find the geographies ManyToMany fields that relates this model to the related_model
# via a Geography class. Which geography class depends on their common geography scope
geographies_field = feature_class_creator.geographies_field(geography_scope)
try:
# Find the queryable field name from the geography class to the related model
related_model_geographies_field_name = resolve_queryable_name_of_type(geography_class, related_model)
except:
# Sometimes the geography class hasn't had its fields cached properly. Fix here
clear_many_cache(geography_class)
related_model_geographies_field_name = resolve_queryable_name_of_type(geography_class, related_model)
return "%s__%s__pk" % (geographies_field.name, related_model_geographies_field_name)
pk_paths = map_to_dict(
lambda db_entity_key: [db_entity_key, Min(resolve_related_model_pk(db_entity_key))], db_entity_keys
)
return manager.annotate(**pk_paths)
def feature_class_lookup(self):
# Get the client region fixture (or the default region if the former doesn't exist)
client_region = resolve_fixture("config_entity", "region", RegionFixture)
region_class_lookup = client_region.feature_class_lookup()
return merge(
region_class_lookup,
FeatureClassCreator(self.config_entity).key_to_dynamic_model_class_lookup(self.default_db_entities())
)
def resolve_field_path_via_geographies(field_path, manager, related_models):
"""
Resolve the given field path in case its not absolute.
For instance, if it is 'block' and one of our related models accessible via geographies__relatedmodel has that property,
return 'geographies_[scope_id]__relatedmodel__block'
It will also be tested against the main manager after all related models fail,
e.g. manager.values(field_path) if successful would simply return field_path
:param field_path: django field path. e.g. du or built_form__name
:param manager: The main manager by which the related models are resolved and by which the full path is computed
:param related_models: models joined to the manager. For instance. manager.model is CanvasFeature, a related_model could be
CensusBlock, which might be related to the former via 'geographies_[scope_id]__censusblock9rel'. The relationship is computed
by assuming that the related model is related by geographies and looking for a field matching its type
:return:
"""
from footprint.main.models.feature.feature_class_creator import FeatureClassCreator
feature_class_creator = FeatureClassCreator.from_dynamic_model_class(manager.model)
for related_model in related_models:
try:
# See if the field_name resolves
# There's probably a more efficient way to do this
related_model.objects.values(field_path)
resolved_field_path = field_path
except:
# See if the first segment matches the related_model db_entity_key
first_segment = field_path.split("__")[0]
if first_segment != related_model.db_entity_key:
# If not, move on
continue
# Take all but the first segment
resolved_field_path = "__".join(field_path.split("__")[1:])
# Success, find the path to this model from geographies
geography_class = feature_class_creator.common_geography_class(related_model)
geographies_field = feature_class_creator.geographies_field(
feature_class_creator.common_geography_scope(related_model)
)
geography_related_field_name = resolve_queryable_name_of_type(geography_class, related_model)
return "%s__%s__%s" % (geographies_field.name, geography_related_field_name, resolved_field_path)
# See if it matches the main model
try:
if field_path.split("__")[0] == manager.model.db_entity_key:
# If the manager model db_entity_key was used in the path, just strip it out
updated_field_path = "__".join(field_path.split("__")[1:])
manager.values(updated_field_path)
else:
# Otherwise test query with the full path
updated_field_path = field_path
manager.values(updated_field_path)
# Success, return the field_path
return updated_field_path
except:
logger.exception(
"Cannot resolve field path %s to the main model %s or any joined models %s",
field_path,
manager.model,
related_models,
)
raise
def resolve_field_path_via_geographies(field_path, manager, related_models):
"""
Resolve the given field path in case its not absolute.
For instance, if it is 'block' and one of our related models accessible via geographies__relatedmodel has that property,
return 'geographies_[scope_id]__relatedmodel__block'
It will also be tested against the main manager after all related models fail,
e.g. manager.values(field_path) if successful would simply return field_path
:param field_path: django field path. e.g. du or built_form__name
:param manager: The main manager by which the related models are resolved and by which the full path is computed
:param related_models: models joined to the manager. For instance. manager.model is CanvasFeature, a related_model could be
CensusBlock, which might be related to the former via 'geographies_[scope_id]__censusblock9rel'. The relationship is computed
by assuming that the related model is related by geographies and looking for a field matching its type
:return:
"""
from footprint.main.models.feature.feature_class_creator import FeatureClassCreator
feature_class_creator = FeatureClassCreator.from_dynamic_model_class(
manager.model)
for related_model in related_models:
try:
# See if the field_name resolves
# There's probably a more efficient way to do this
related_model.objects.values(field_path)
resolved_field_path = field_path
except:
# See if the first segment matches the related_model db_entity_key
first_segment = field_path.split('__')[0]
if first_segment != related_model.db_entity_key:
# If not, move on
continue
# Take all but the first segment
resolved_field_path = '__'.join(field_path.split('__')[1:])
# Success, find the path to this model from geographies
geography_class = feature_class_creator.common_geography_class(
related_model)
geographies_field = feature_class_creator.geographies_field(
feature_class_creator.common_geography_scope(related_model))
geography_related_field_name = resolve_queryable_name_of_type(
geography_class, related_model)
return '%s__%s__%s' % (geographies_field.name,
geography_related_field_name,
resolved_field_path)
# See if it matches the main model
try:
if field_path.split('__')[0] == manager.model.db_entity_key:
# If the manager model db_entity_key was used in the path, just strip it out
updated_field_path = '__'.join(field_path.split('__')[1:])
manager.values(updated_field_path)
else:
# Otherwise test query with the full path
updated_field_path = field_path
manager.values(updated_field_path)
# Success, return the field_path
return updated_field_path
except:
logger.exception(
'Cannot resolve field path %s to the main model %s or any joined models %s',
field_path, manager.model, related_models)
raise
def importer(self, config_entity, db_entity, **kwargs):
"""
Creates various GeojsonFeature classes by importing geojson and saving it to the database via a dynamic subclass of GeojsonFeature
:schema: The optional schema to use for the dynamic subclass's meta db_table attribute, which will allow the class's table to be saved in the specified schema. Defaults to public
:data: Optional python dict data to use instead of loading from the db_entity.url
:return: a list of lists. Each list is a list of features of distinct subclass of GeoJsonFeature that is created dynamically. To persist these features, you must first create the subclass's table in the database using create_table_for_dynamic_class(). You should also register the table as a DbEntity.
"""
if self.seed_data:
data = geojson.loads(jsonify(self.seed_data), object_hook=geojson.GeoJSON.to_instance)
else:
fp = open(db_entity.url.replace('file://', ''))
data = geojson.load(fp, object_hook=geojson.GeoJSON.to_instance)
feature_class_creator = FeatureClassCreator(config_entity, db_entity)
# find all unique properties
feature_class_configuration = feature_class_creator.feature_class_configuration_from_geojson_introspection(data)
feature_class_creator.update_db_entity(feature_class_configuration)
feature_class = feature_class_creator.dynamic_model_class(base_only=True)
# Create our base table. Normally this is done by the import, but we're just importing into memory
create_tables_for_dynamic_classes(feature_class)
# Now write each feature to our newly created table
for feature in map(lambda feature: self.instantiate_sub_class(feature_class, feature), data.features):
feature.save()
# Create the rel table too
rel_feature_class = feature_class_creator.dynamic_model_class()
create_tables_for_dynamic_classes(rel_feature_class)
# PostGIS 2 handles this for us now
# if InformationSchema.objects.table_exists(db_entity.schema, db_entity.table):
# # Tell PostGIS about the new geometry column or the table
# sync_geometry_columns(db_entity.schema, db_entity.table)
# Create association classes and tables and populate them with data
create_and_populate_relations(config_entity, db_entity)
def create_join_feature_class(self):
"""
Make an unmanaged Django model based on the joined fields
"""
return FeatureClassCreator(
self.config_entity,
self.layer.db_entity_interest.db_entity).dynamic_join_model_class(
self.resolve_join_models(),
self.result_map.related_model_lookup.keys())
def feature_class_lookup(self):
# Get the client project fixture (or the default region if the former doesn't exist)
project = merge(*map(
lambda project_fixture: project_fixture.feature_class_lookup(),
project_specific_project_fixtures(
config_entity=self.config_entity)))
return merge(
project,
FeatureClassCreator(
self.config_entity).key_to_dynamic_model_class_lookup(
self.default_db_entities()))
def feature_class_lookup(self):
"""
Adds mappings of custom Feature classes
:return:
"""
parent_fixture = self.parent_fixture
feature_class_lookup = parent_fixture.feature_class_lookup()
return merge(
feature_class_lookup,
FeatureClassCreator(
self.config_entity).key_to_dynamic_model_class_lookup(
self.default_db_entities()))
def create_layer_selections(layers):
"""
Create LayerSelection classes and instances for the given Scenario subclasses among the
classes in limit_to_classes. Also filters by db_entity_key if they are specified
:return:
"""
for config_entity in unique(map(lambda layer: layer.config_entity,
layers)):
FeatureClassCreator(config_entity).ensure_dynamic_models()
for selection_layer in layers:
# Recreate
get_or_create_layer_selection_class_for_layer(selection_layer)
update_or_create_layer_selections(config_entity=None)
def resolve_related_model_path_via_geographies(manager, related_model):
"""
Returns the query string path 'geographies_[scope_id]__[field name of the related model form the main model]'
The scope_id is the id of the ConfigEntity that both models share in common by ascending the ConfigEntity
hierarchy starting at each models' geography_scope
"""
from footprint.main.models.feature.feature_class_creator import FeatureClassCreator
feature_class_creator = FeatureClassCreator.from_dynamic_model_class(manager.model)
geography_scope = feature_class_creator.common_geography_scope(related_model)
geographies_field = feature_class_creator.geographies_field(geography_scope)
geography_class = feature_class_creator.common_geography_class(related_model)
geography_related_field_name = resolve_queryable_name_of_type(geography_class, related_model)
return '%s__%s' % (geographies_field.name, geography_related_field_name)
def resolve_related_model_path_via_geographies(manager, related_model):
"""
Returns the query string path 'geographies_[scope_id]__[field name of the related model form the main model]'
The scope_id is the id of the ConfigEntity that both models share in common by ascending the ConfigEntity
hierarchy starting at each models' geography_scope
"""
from footprint.main.models.feature.feature_class_creator import FeatureClassCreator
feature_class_creator = FeatureClassCreator.from_dynamic_model_class(manager.model)
geography_scope = feature_class_creator.common_geography_scope(related_model)
geographies_field = feature_class_creator.geographies_field(geography_scope)
geography_class = feature_class_creator.common_geography_class(related_model)
geography_related_field_name = resolve_queryable_name_of_type(geography_class, related_model)
return "%s__%s" % (geographies_field.name, geography_related_field_name)
def result_map(cls):
"""
Creates an caches a result map for the Feature class. The result_map has useful meta data about
the class
:param cls:
:return:
"""
if cls._result_map:
return cls._result_map
from footprint.main.models.feature.feature_class_creator import FeatureClassCreator
feature_class_creator = FeatureClassCreator.from_dynamic_model_class(cls)
if not feature_class_creator.dynamic_model_class_is_ready:
return None
cls._result_map = feature_class_creator.dynamic_model_class().objects.all().create_result_map()
return cls._result_map
def result_map(cls):
"""
Creates an caches a result map for the Feature class. The result_map has useful meta data about
the class
:param cls:
:return:
"""
if cls._result_map:
return cls._result_map
from footprint.main.models.feature.feature_class_creator import FeatureClassCreator
feature_class_creator = FeatureClassCreator.from_dynamic_model_class(
cls)
if not feature_class_creator.dynamic_model_class_is_ready:
return None
cls._result_map = feature_class_creator.dynamic_model_class(
).objects.all().create_result_map()
return cls._result_map
def delete_layer_selections(layers):
for config_entity in unique(map(lambda layer: layer.config_entity,
layers)):
FeatureClassCreator(config_entity).ensure_dynamic_models()
for selection_layer in layers:
try:
# Drop the table
layer_selection_class = get_or_create_layer_selection_class_for_layer(
selection_layer, no_table_creation=True)
if layer_selection_class:
if hasattr(layer_selection_class.features, 'through'):
layer_selection_features_class = layer_selection_class.features.through
drop_layer_selection_table(layer_selection_features_class)
drop_layer_selection_table(layer_selection_class)
except DatabaseError, e:
logger.warning(
"Couldn't destroy LayerSelection tables. Maybe the public.layer table no longer exists: %s"
% e.message)
def update_or_create_db_entity_and_interest(config_entity, config_db_entity):
"""
Sync a single db_entity_configuration or db_entity and its db_entity_interest
:return A tuple of the DbEntityInterest and the created flag
"""
unique_key_combo = ['key', 'schema']
db_entity, created, updated = DbEntity.objects.update_or_create(
# key and schema uniquely identify the DbEntity
key=config_db_entity.key,
schema=config_db_entity.schema,
defaults=remove_keys(model_dict(config_db_entity), unique_key_combo))
db_entity.feature_behavior = config_db_entity.feature_behavior
db_entity.save()
logger.info("ConfigEntity/DbEntity Publishing. DbEntity: %s" %
db_entity.full_name)
# Create the DbEntityInterest through class instance which associates the ConfigEntity instance
# to the DbEntity instance. For now the interest attribute is hard-coded to OWNER. This might
# be used in the future to indicate other levels of interest
interest = Interest.objects.get(key=Keys.INTEREST_OWNER)
db_entity_interest, created, updated = DbEntityInterest.objects.update_or_create(
config_entity=config_entity, db_entity=db_entity, interest=interest)
#update the geography scope after the db_entity_interest saves as this is required to find 'owned' db_entites in a config entity
if not db_entity.no_feature_class_configuration:
feature_class_creator = FeatureClassCreator(config_entity,
db_entity,
no_ensure=True)
db_entity.feature_class_configuration.geography_scope = config_entity.id if db_entity.feature_class_configuration.primary_geography \
else feature_class_creator.resolved_geography_scope.id
db_entity.save()
return db_entity_interest, created
def _post_save_publishing(job, config_entity, user, **kwargs):
"""
Runs all configured publishers via the Django signals they depend upon.
This is done in Celery in order to support long running tasks launched from a client.
Peer tasks are run in parallel. Dependent tasks are called after the tasks they depend on complete
:param hash_id: Job hash id
:param config_entity:
:param user: The current user or None if no user is in scope
:return:
"""
if not settings.FOOTPRINT_INIT:
# There is a poorly-understood issue related to uploading .gdb
# files with mutliple layers (possibly caused by some race
# condition) that, without sleeping here, causes the celery
# task to run without an `instance` (the object that triggered
# post save processing). Testing had shown 10 seconds to be the
# shortest amount of time to wait here that permits the post-save
# processing to complete successfully.
time.sleep(10)
bundle = kwargs['bundle']
# Make sure all the Feature subclasses that the celery worker might need are created
if config_entity:
config_entity._feature_classes_created = False
FeatureClassCreator(config_entity)
# Get the publisher_name, proportion, and signal_path
publishing_info = get_publishing_info(**kwargs)
try:
# Make sure no transactions are outstanding
# This shoudln't be needed once Django is upgraded
transaction.commit()
except Exception, e:
pass
def create_empty_source_table(self,
clazz,
source_table_name,
source_db_connection,
extra_fields={}):
project = Project(key='sutter_county', id=0)
db_entity = DbEntity(key=Keys.DB_ABSTRACT_BASE_AGRICULTURE_FEATURE,
feature_class_configuration=dict(
abstract_class=full_module_path(clazz)))
SourceClass = FeatureClassCreator(project, db_entity,
no_ensure=True).dynamic_model_class(
base_only=True,
schema='public',
table=source_table_name)
# class SourceClass(clazz):
# class Meta(clazz.Meta):
# db_table = source_table_name
create_tables_for_dynamic_classes(SourceClass)
for field in SourceClass._meta.fields[1:]:
setattr(field, 'null', True)
drop_table = "DROP TABLE IF EXISTS {final_table} CASCADE;".format(
final_table=source_table_name)
sql, refs = source_db_connection.creation.sql_create_model(
SourceClass, color_style())
add_geometry_fields = '''
ALTER TABLE {final_table} ADD COLUMN geography_id VARCHAR;
ALTER TABLE {final_table} ADD COLUMN wkb_geometry GEOMETRY;'''.format(
final_table=source_table_name)
sql = drop_table + sql[0] + add_geometry_fields
for dbfield, fieldtype in extra_fields.items():
sql += 'ALTER TABLE {final_table} ADD COLUMN {field} {type}'.format(
final_table=source_table_name, field=dbfield, type=fieldtype)
source_db_connection.cursor().execute(sql)
def parse_query(config_entity, manager, filters=None, joins=None, aggregates=None, group_bys=None):
queryset = manager
group_by_values = None
annotation_tuples = None
# Make sure all related models have been created before querying
from footprint.main.models.feature.feature_class_creator import FeatureClassCreator
FeatureClassCreator(config_entity).ensure_dynamic_models()
# Any joins are db_entity_keys and resolve to feature classes of the config_entity
related_models = map(lambda join: config_entity.db_entity_feature_class(join), joins or [])
# Use group_by_values to group by and then attach the aggregates to each unique result via annotation
# If aggregates are specified but group by is not, we use aggregate to just get a single result
# For now we assume any filtering should be applied BEFORE aggregation
if filters:
queryset = queryset.filter(parse_token(filters, manager, related_models))
# We only need to join explicitly if the join is not included in one of the group by fields
manual_joins = joins or [] if not group_bys else \
set(joins or [])-\
set(map(lambda group_by: resolve_db_entity_key_of_field_path(parse_group_by(group_by, manager, related_models), manager, related_models), group_bys))
if manual_joins:
# If there are joins, filter the queryset by inner join on the related_model pk through geography
for related_model in related_models:
related_field_pk_path = resolve_field_path_via_geographies('pk', manager, [related_model])
queryset = queryset.filter(**{'{0}__isnull'.format(related_field_pk_path):False})
# If there are aggregates, they are either part of the main table or join table
if aggregates:
# Resolve the field path using available joins via geographies or on the main model
# Then send the resolved field
annotation_tuples = map(
lambda aggregate: parse_annotation(aggregate, manager, related_models),
aggregates)
if group_bys:
group_by_values = map(
lambda group_by: parse_group_by(group_by, manager, related_models),
to_list(group_bys))
annotation_tuples = annotation_tuples or []
# Add a Count to the selection if one isn't present
if not first(lambda annotation_tuple: Count==annotation_tuple[0], annotation_tuples):
annotation_tuples.insert(0, (Count, group_by_values[0], 'count'))
# TODO. We might have to do rounding of floats here using an extra clause:
# extra(select={'custom_field': 'round(field, 2)'})
queryset = queryset.values(*group_by_values).order_by(*group_by_values)
elif annotation_tuples:
# If there are annotations but no group_bys, we need to fake a group by annotating
# the count of the pk to each row and then grouping by it. Since every row
# has one pk all the rows group together
# Otherwise we'd have to use the aggregate function which doesn't give us
# a query back
queryset = queryset.annotate(count=Count('pk')).values('count')
if annotation_tuples:
for annotation_tuple in annotation_tuples:
# Annotation built-in functions or custom functions of the queryset (like geo stuff)
annotate_method = getattr(queryset, annotation_tuple[0]) if\
isinstance(annotation_tuple[0], basestring) else\
annotation_tuple[0]
if len(annotation_tuple)==3:
# Apply alias if specified
queryset = queryset.annotate(**{annotation_tuple[2]:annotate_method(annotation_tuple[1])})
else:
# Otherwise default the name to the field
queryset = queryset.annotate(annotate_method(annotation_tuple[1]))
elif group_by_values:
# If no annotations are specified, add in a count annotation to make the group by take effect
# As long as we annotate a count of one we'll get the correct group_by, since django receives values(group_by1, group_by2, etc).annotate(count(group_by1))
queryset = queryset.annotate(count=Count(group_by_values[0]))
return queryset
def dynamic_resource_subclass(self,
layer_selection=None,
db_entity=None,
feature_class=None,
config_entity=None,
metadata=None,
params=None,
**kwargs):
"""
Creates the dynamic Feature Resource class by passing in a layer_selection, db_entity, or feature_class
:param layer_selection: Required if db_entity or metadata aren't present
:param db_entity: Required if layer_selection or metadata aren't present
:param metadata: Required along with config_entity if layer_selection or db_entity aren't present
:param kwargs:
:return:
"""
feature_class_configuration = None
if layer_selection:
# Coming in relative to a LayerSelection, which puts us in the context of the LayerSelection's
# feature query for this Feature subclass
layer = layer_selection.layer
# If we pass in a ConfigEntity it means we want to scope the Feature class to its scope.
# The ConfigEntity defaults to that of the Layer, but we can override it to be a lower
# scope to make sure that we have access to lower DbEntities of performing joins
config_entity = config_entity.subclassed if config_entity else layer.config_entity.subclassed
logger.debug(
"Resolving FeatureResource subclass for layer_selection: {0}, config_entity: {1}"
.format(layer_selection.unique_id, config_entity.id))
# Resolve the dynamic Feature class with the given config_entity so that we can access all DbEntities
# of the ConfigEntity for joins
feature_class = config_entity.db_entity_feature_class(
layer.db_entity.key)
elif db_entity:
# Coming in relative to a DbEntity, meaning we don't care about a particular LayerSelection's
# feature query for this Feature subclass
config_entity = db_entity.config_entity
logger.debug(
"Resolving FeatureResource subclass for db_entity: {0}, config_entity: {1}"
.format(db_entity.id, config_entity.id))
# Resolve the dynamic Feature class with the given config_entity so that we can access all DbEntities
# of the ConfigEntity for joins
feature_class = config_entity.db_entity_feature_class(
db_entity.key)
elif metadata:
# Coming in with metadata, meaning this is and uploaded or ArcGis table with no DbEntity yet
# We need to construct a FeatureClass from the metadata
logger.debug(
"Resolving FeatureResource subclass for metadata: {0}, config_entity: {1}"
.format(metadata, config_entity.id))
feature_class_creator = FeatureClassCreator(config_entity)
feature_class_configuration = feature_class_creator.feature_class_configuration_from_metadata(
metadata['schema'])
feature_class = FeatureClassCreator(
config_entity,
feature_class_configuration).dynamic_model_class()
if not feature_class_configuration:
# If we didn't already ensure all dynamic model classes have been created
# This only need to run once to get all dynamic feature subclasses into memory,
# in case they are needed by an association, join, or something similar
feature_class_creator = FeatureClassCreator.from_dynamic_model_class(
feature_class)
feature_class_creator.ensure_dynamic_models()
logger.debug("Resolving resource for Feature subclass: {0}".format(
feature_class))
# Resolve the FeatureResource subclass based on the given Feature subclass
# If self is already a subclass, just return self
# Else, return a preconfigured subclass or one dynamically created. The latter will probably be the only way in the future.
# If not already subclassed
is_singleton_feature = issubclass(self.__class__,
SingletonFeatureResourceMixin)
is_template_feature = self.__class__ == TemplateFeatureResource
if self.__class__ in [
FeatureResource, TemplateFeatureResource,
FeatureCategoryAttributeResource,
FeatureQuantitativeAttributeResource
]:
if is_singleton_feature or params.get('is_feature_attribute'):
queryset = feature_class.objects.none()
elif kwargs.get('method', None) == 'PATCH':
# It's possible to PATCH with an active join query.
# But we don't want to use a join query when patching
queryset = feature_class.objects.all()
else:
# Get the queryset stored by the layer_selection or an empty query if we don't have a layer_selection
queryset = layer_selection.selected_features_or_values if\
layer_selection else \
feature_class.objects.none()
if layer_selection and not (is_singleton_feature or kwargs.get(
'query_may_be_empty')) and queryset.count() == 0:
raise Exception(
"Unexpected empty queryset for layer_selection features: %s"
% queryset.query)
is_values_queryset = isinstance(queryset, ValuesQuerySet)
#returns queryset ordered by the table id
queryset = queryset.order_by('id')
if is_values_queryset:
join_feature_class = layer_selection.create_join_feature_class(
) if is_values_queryset else feature_class
logger.info("Created join_feature_class: %s" %
join_feature_class)
# Force the queryset to our new class so that Tastypie can map the dict results to it
queryset.model = join_feature_class
return self.__class__.resolve_resource_class(
join_feature_class,
queryset=queryset,
base_resource_class=self.join_feature_resource_class(
join_feature_class),
additional_fields_dict=dict(
# Pass these to the feature resource to help it resolve
# field mappings and add related fields (just need for join_feature_class)
# Use the layer_selection if it exists since it might have filtered or extra query fields
result_field_lookup=(layer_selection
or db_entity).result_field_lookup
if not metadata else {},
related_field_lookup=(
layer_selection or db_entity).related_field_lookup
if not metadata else {},
# We use these in the FeatureResource to create a unique id for each join Feature
join_model_attributes=layer_selection
and layer_selection.result_map.join_model_attributes),
is_join_query=True,
limit_fields=layer_selection.result_map['result_fields'])
else:
abstract_feature_resource_class = self.__class__
resource_class = abstract_feature_resource_class.resolve_resource_class(
feature_class,
queryset=queryset,
# Give FeatureResource a reference to the layer_selection
additional_fields_dict=merge(
dict(
# Pass this to the feature resource to help it resolve field mappings
result_field_lookup=(layer_selection or db_entity).
result_field_lookup if not metadata else {}),
dict(
# Not sure why it doesn't work to just stick this on the TemplateFeatureResource
feature_fields=ListField(
attribute='feature_fields',
null=True,
blank=True,
readonly=True),
feature_field_title_lookup=PickledDictField(
attribute='feature_field_title_lookup',
null=True,
blank=True,
readonly=True),
) if is_template_feature else dict()),
for_template=is_template_feature)
return resource_class
return self
def dynamic_resource_subclass(self, layer_selection=None, db_entity=None, feature_class=None, config_entity=None, metadata=None, params=None, **kwargs):
"""
Creates the dynamic Feature Resource class by passing in a layer_selection, db_entity, or feature_class
:param layer_selection: Required if db_entity or metadata aren't present
:param db_entity: Required if layer_selection or metadata aren't present
:param metadata: Required along with config_entity if layer_selection or db_entity aren't present
:param kwargs:
:return:
"""
feature_class_configuration = None
if layer_selection:
# Coming in relative to a LayerSelection, which puts us in the context of the LayerSelection's
# feature query for this Feature subclass
layer = layer_selection.layer
# If we pass in a ConfigEntity it means we want to scope the Feature class to its scope.
# The ConfigEntity defaults to that of the Layer, but we can override it to be a lower
# scope to make sure that we have access to lower DbEntities of performing joins
config_entity = config_entity.subclassed if config_entity else layer.config_entity.subclassed
logger.debug("Resolving FeatureResource subclass for layer_selection: {0}, config_entity: {1}".format(layer_selection.unique_id, config_entity.id))
# Resolve the dynamic Feature class with the given config_entity so that we can access all DbEntities
# of the ConfigEntity for joins
feature_class = config_entity.db_entity_feature_class(layer.db_entity.key)
elif db_entity:
# Coming in relative to a DbEntity, meaning we don't care about a particular LayerSelection's
# feature query for this Feature subclass
config_entity = db_entity.config_entity
logger.debug("Resolving FeatureResource subclass for db_entity: {0}, config_entity: {1}".format(db_entity.id, config_entity.id))
# Resolve the dynamic Feature class with the given config_entity so that we can access all DbEntities
# of the ConfigEntity for joins
feature_class = config_entity.db_entity_feature_class(db_entity.key)
elif metadata:
# Coming in with metadata, meaning this is and uploaded or ArcGis table with no DbEntity yet
# We need to construct a FeatureClass from the metadata
logger.debug("Resolving FeatureResource subclass for metadata: {0}, config_entity: {1}".format(metadata, config_entity.id))
feature_class_creator = FeatureClassCreator(
config_entity
)
feature_class_configuration = feature_class_creator.feature_class_configuration_from_metadata(metadata['schema'])
feature_class = FeatureClassCreator(
config_entity,
feature_class_configuration
).dynamic_model_class()
if not feature_class_configuration:
# If we didn't already ensure all dynamic model classes have been created
# This only need to run once to get all dynamic feature subclasses into memory,
# in case they are needed by an association, join, or something similar
feature_class_creator = FeatureClassCreator.from_dynamic_model_class(feature_class)
feature_class_creator.ensure_dynamic_models()
logger.debug("Resolving resource for Feature subclass: {0}".format(feature_class))
# Resolve the FeatureResource subclass based on the given Feature subclass
# If self is already a subclass, just return self
# Else, return a preconfigured subclass or one dynamically created. The latter will probably be the only way in the future.
# If not already subclassed
is_singleton_feature = issubclass(self.__class__, SingletonFeatureResourceMixin)
is_template_feature = self.__class__ == TemplateFeatureResource
if self.__class__ in [FeatureResource, TemplateFeatureResource, FeatureCategoryAttributeResource,
FeatureQuantitativeAttributeResource]:
if is_singleton_feature or params.get('is_feature_attribute'):
queryset = feature_class.objects.none()
elif kwargs.get('method', None) == 'PATCH':
# It's possible to PATCH with an active join query.
# But we don't want to use a join query when patching
queryset = feature_class.objects.all()
else:
# Get the queryset stored by the layer_selection or an empty query if we don't have a layer_selection
queryset = layer_selection.selected_features_or_values if\
layer_selection else \
feature_class.objects.none()
if layer_selection and not (is_singleton_feature or kwargs.get('query_may_be_empty')) and queryset.count()==0:
raise Exception(
"Unexpected empty queryset for layer_selection features: %s" %
queryset.query)
is_values_queryset = isinstance(queryset, ValuesQuerySet)
#returns queryset ordered by the table id
queryset = queryset.order_by('id')
if is_values_queryset:
join_feature_class = layer_selection.create_join_feature_class() if is_values_queryset else feature_class
logger.info("Created join_feature_class: %s" % join_feature_class)
# Force the queryset to our new class so that Tastypie can map the dict results to it
queryset.model = join_feature_class
return self.__class__.resolve_resource_class(
join_feature_class,
queryset=queryset,
base_resource_class=self.join_feature_resource_class(join_feature_class),
additional_fields_dict=dict(
# Pass these to the feature resource to help it resolve
# field mappings and add related fields (just need for join_feature_class)
# Use the layer_selection if it exists since it might have filtered or extra query fields
result_field_lookup=(layer_selection or db_entity).result_field_lookup if not metadata else {},
related_field_lookup=(layer_selection or db_entity).related_field_lookup if not metadata else {},
# We use these in the FeatureResource to create a unique id for each join Feature
join_model_attributes=layer_selection and layer_selection.result_map.join_model_attributes
),
is_join_query=True,
limit_fields=layer_selection.result_map['result_fields']
)
else:
abstract_feature_resource_class = self.__class__
resource_class = abstract_feature_resource_class.resolve_resource_class(
feature_class,
queryset=queryset,
# Give FeatureResource a reference to the layer_selection
additional_fields_dict=merge(
dict(
# Pass this to the feature resource to help it resolve field mappings
result_field_lookup=(layer_selection or db_entity).result_field_lookup if not metadata else {}
),
dict(
# Not sure why it doesn't work to just stick this on the TemplateFeatureResource
feature_fields=ListField(attribute='feature_fields', null=True, blank=True, readonly=True),
feature_field_title_lookup=PickledDictField(attribute='feature_field_title_lookup', null=True, blank=True, readonly=True),
) if is_template_feature else dict()
),
for_template=is_template_feature
)
return resource_class
return self
def importer(self, config_entity, db_entity, **kwargs):
"""
Replaces the normal ImportProcessor importer with one to import a sql from disk
"""
if InformationSchema.objects.table_exists(db_entity.schema, db_entity.table):
# The table already exists. Skip the import an log a warning
logger.warn("The target table for the feature table import already exists. Skipping table import.")
else:
# We don't store the upload_id alone, so pull it off the url
upload_id = db_entity.url.replace('file:///tmp/', '').replace('.sql.zip', '')
# Unpack the zipfile and return the path the sql file was placed at
if db_entity.url.startswith('file://'):
file_path = db_entity.url[len('file://'):]
logger.warn(file_path)
path = unpack_zipfile(file_path, upload_id)
# The file is always the name of the table defined therein
table_name = path.split('/')[-1].split('.')[0].lower()
db_entity.url = 'file://%s' % path
# Update the db_entity.url from the zip file url to the file_path
# This lets ImportData find it.
logger.info("Url of DbEntity is %s" % db_entity.url)
db_entity.save()
# Perform some sed updates to get the sql file ready for import
regex_substitutions = []
sql_file_path = file_url_to_path(db_entity.url)
# Add IF EXISTS to the drop table to prevent an error if IF EXISTS doesn't exist yet
regex_substitutions.append((r'DROP TABLE (?!IF EXISTS)', r'DROP TABLE IF EXISTS'))
# TODO temp, fix an AC bug. It seems that using a capitalized column is problematic (?)
# The suggested solution is to double quote it, but quotes cause other problems, so we simply lowercase
regex_substitutions.append((r' OGC_FID ', ' ogc_fid ', (4, 4))) # only line 4
regex_substitutions.append((r'PRIMARY KEY \(ogc_fid\)', 'PRIMARY KEY (ogc_fid)', (4, 4))) # only line 4
# TODO end temp fix
# Update the index name to include the schema. This format matches that created for preconfigured feature
# tables (see import_data.py)
spatial_index_name = '{schema}_{key}_geom_idx'.format(schema=db_entity.schema, key=db_entity.key)
regex_substitutions.append((r'CREATE INDEX ".*" ON', 'CREATE INDEX "%s" ON' % spatial_index_name, (6, 6))) # only line 6 6
# Remove the reference to the geometry_columns, since we use a materialized view
regex_substitutions.append((r'^DELETE FROM geometry_columns', '--DELETE FROM geometry_columns', (2, 2)))
# Update the sql to have a unique table name which matches the DbEntity key
# Also change public to our import schema to keep it from causing trouble in the public schema
# Otherwise we run into all kinds of trouble trying to get the SQL into the system
regex_substitutions.append((r'"public"."%s"' % table_name, '"import"."%s"' % db_entity.key))
regex_substitutions.append((r"'%s'" % table_name, "'%s'" % db_entity.key, (2, 5)))
regex_substitutions.append((r'"%s_pk"' % table_name, '"%s_pk"' % db_entity.key, (4, 4)))
# Update public to the import schema
regex_substitutions.append((r"AddGeometryColumn\('public'", "AddGeometryColumn('%s'" % settings.IMPORT_SCHEMA, (5, 5)))
regex_substitutions.append((r'"%s_wkb_geometry_geom_idx"' % table_name, '"%s_wkb_geometry_geom_idx"' % db_entity.key, (6, 6)))
for command in regex_substitutions:
logger.info("Applying the following substitution %s" % ', '.join(command[0:2]))
apply_regexes_to_file(sql_file_path, regex_substitutions)
ImportData(config_entity=config_entity, db_entity_key=db_entity.key).run()
# Add our normal primary key in the id column if negit eded
add_primary_key_if_needed(db_entity)
feature_class_creator = FeatureClassCreator(config_entity, db_entity)
# Inspect the imported table to create the feature_class_configuration
feature_class_configuration = feature_class_creator.feature_class_configuration_from_introspection()
# Merge the created feature_class_configuration with the on already defined for the db_entity
feature_class_creator.update_db_entity(feature_class_configuration)
logger.info("Finished import for DbEntity: %s, feature_class_configuration: %s" % (db_entity, db_entity.feature_class_configuration))
# Create association classes and tables and populate them with data
create_and_populate_relations(config_entity, feature_class_creator.db_entity)
class DbEntities(models.Model):
db_entities = models.ManyToManyField('DbEntity',
through='DbEntityInterest')
class Meta:
abstract = True
def add_db_entity_interests(self, *db_entity_interests):
"""
Adds one or more unsaved DbEntityInterests to the instance's collection.
If the instance has not yet overridden its parents' db_entities by adding at least one DbEntityInterest,
the parents DbEntityInterests will be adopted and then the db_entities give here will be added
:return:
"""
# This check exists to avoid infinite recursion, since db_entities are sometimes added post_config_entity_save handler
if len(db_entity_interests) > 0:
# Even though the field name is db_entities, we need to pass the DbEntityInterests
self._add('db_entities', *db_entity_interests)
def remove_db_entity_interests(self, *db_entity_interests):
self._remove('db_entities', *db_entity_interests)
def feature_class_of_base_class(self, base_feature_class):
"""
Finds the feature_class of this config_entity that is derived from the given base class
:param base_feature_class:
:return:
"""
db_entities = filter(
lambda db_entity:
# Get the configured abstract class
# The source_db_entity_key is a hack to prevent Result db_entities from being chosen
issubclass(
resolve_module_attr(
db_entity.feature_class_configuration.abstract_class_name,
object), base_feature_class) and not db_entity.
source_db_entity_key,
self.computed_db_entities())
if len(db_entities) != 1:
raise Exception(
"Expected exactly one db_entity matching the base_class {0} but got {1}"
.format(base_feature_class,
db_entities if len(db_entities) > 0 else 'none'))
return self.db_entity_feature_class(db_entities[0].key)
def db_entity_feature_class(self,
key,
abstract_class=False,
base_feature_class=False):
"""
Resolves the Feature class subclass of this config_entity for the given key, or the base class version
if base_class-True. Note that this is slightly different than asking a DbEntity for its corresponding
Feature class. A DbEntity will always use its owning ConfigEntity for the config_entity property
of the Feature class. This method uses self as the ConfigEntity, which is useful when self
is a lower ConfigEntity in the hierarchy (such as a Scenario) and the Feature class needs
access to related Feature classes that are also at a low level. This works because a lower ConfigEntity
has access to DbEntities that the higher one does not, which allows queries between from a higher
DbEntity to join a lower one.
:param key: The DbEntity key
:param abstract_class, Default False, if True returns the abstract base class instead of the subclass
:param base_feature_class, Default False, if True returns the base class instead of the default rel class
:return:
"""
try:
original_db_entity = self.computed_db_entities().get(key=key)
except Exception, e:
raise Exception(
"The DbEntity key %s could not be found in the computed DbEntities of the ConfigEntity %s, which contains %s"
% (key, self.name, ', '.join(
map(lambda db_entity: db_entity.key,
self.computed_db_entities()))))
source_db_entity_key = original_db_entity.source_db_entity_key if original_db_entity.source_db_entity_key else key
db_entity = self.computed_db_entities().get(key=source_db_entity_key)
# Resolve the source_db_entity_key of the DbEntity or just use key
from footprint.main.models.feature.feature_class_creator import FeatureClassCreator
if abstract_class:
subclass = resolve_module_attr(
db_entity.feature_class_configuration.abstract_class_name)
elif base_feature_class or db_entity.feature_class_configuration.no_table_associations:
subclass = FeatureClassCreator(
self, db_entity).dynamic_model_class(base_only=True)
else:
subclass = FeatureClassCreator(self,
db_entity).dynamic_model_class()
if not subclass:
raise Exception(
"For config_entity {0} no class associated with db_entity_key {1}{2}. "
"Register a table in default_db_entities() of the owning config_entity."
.format(
unicode(self), key,
", even after resolving source_db_entity_key to {0}".
format(source_db_entity_key)
if source_db_entity_key != key else ''))
return subclass
def crud_db_entities(config_entity, crud, db_entity_keys=None):
"""
Creates or updates the db_entities of the ConfigEntity
:param config_entity
:param crud CrudKey.CREATE, CrudType.CLONE, CrudType.UPDATE, CrudType.SYNC, CrudType.DELETE (unimplemented)
:return:
"""
from footprint.client.configuration.fixture import ConfigEntityFixture
# If not present, create the database schema for this ConfigEntity's feature table data
PGNamespace.objects.create_schema(config_entity.schema())
client_fixture = ConfigEntityFixture.resolve_config_entity_fixture(
config_entity)
db_entity_filter = dict(key__in=db_entity_keys) if db_entity_keys else {}
# Process the DbEntities from the origin_instance or the db_entity_configuration from the fixtures,
# but only the first time this scenario is saved
# We only get those scoped (owned) by the class of our config_entity. The scoped above will be adopted automatically
# and need not be created. This means a Scenario creates DbEntities scoped to Scenario and adopts those scoped
# to Project or Region. It does not clone the latter.
if CrudKey.CLONE == crud:
# CRUD the DbEntities to match the origin instance
origin_instance = config_entity.origin_instance
# Clone the DbEntities from the origin ConfigEntity.
db_entities = map(
lambda source_db_entity: clone_or_update_db_entity_and_interest(
config_entity, source_db_entity,
DbEntity(schema=config_entity.schema(),
feature_class_configuration=FeatureClassConfiguration(
geography_scope=FeatureClassCreator(
config_entity).resolved_geography_scope.id,
class_attrs={
'config_entity__id': config_entity.id,
'override_db': config_entity.db,
'db_entity_key': source_db_entity.key
}))).db_entity,
origin_instance.owned_db_entities(**db_entity_filter))
elif crud in [CrudKey.SYNC, CrudKey.CREATE]:
#TODO examine the two conditions below more carefully. We want syncing to be the same for clones and non-clones
if config_entity.origin_instance:
# Syncing previously cloned instance
db_entities = config_entity.owned_db_entities(**db_entity_filter)
update_or_create_db_entities_and_interests(config_entity,
*db_entities)
else:
# Create or Sync new instance
# Get the default DbEntity configurations from the fixture
default_db_entities = filter(
lambda db_entity: db_entity.key in db_entity_keys
if db_entity_keys else True,
client_fixture.default_db_entities())
# Find additional owned (not adopted) db_entities that aren't defaults, namely those that were created by the user
additional_db_entities = filter(
lambda db_entity: db_entity.key in db_entity_keys
if db_entity_keys else True,
client_fixture.non_default_owned_db_entities())
# Combine the defaults with the additions
db_entities = default_db_entities + list(additional_db_entities)
update_or_create_db_entities_and_interests(config_entity,
*db_entities)
elif CrudKey.UPDATE == crud:
# No complex updates are enabled for scenarios, so no post-save processing is needed
return
elif CrudKey.DELETE == crud:
raise NotImplementedError("DELETE is not implemented")
# Disable the post_post_save signal while saving to prevent an infinite loop
previous = config_entity._no_post_save_publishing
config_entity._no_post_save_publishing = True
# Save post_create changes. This is just to store selected DbEntities
config_entity.save()
config_entity._no_post_save_publishing = previous
reset_queries()