def create_result_map(self, values_query_set):
related_models = self.resolve_join_models()
logger.debug("Creating result map for related models %s feature class %s" % (', '.join(map(lambda r: str(r), related_models)), self.feature_class))
feature_class_creator = FeatureClassCreator.from_dynamic_model_class(self.feature_class)
geography_scopes = feature_class_creator.geography_scopes()
# Get the related model paths final segment. We want to map these to the db_entity_key names
related_model_path_to_name = map_to_dict(
lambda related_model:
[resolve_related_model_path_via_geographies(
self.feature_class.objects,
related_model).split('__')[1],
related_model.db_entity_key],
related_models
)
return values_query_set.create_result_map(
related_models=related_models,
# map_path_segments maps related object paths to their model name,
# and removes the geographies segment of the path
map_path_segments=merge(
# Map each geography scope to its corresponding field on the feature class
map_to_dict(
lambda geography_scope: [
feature_class_creator.geographies_field(geography_scope).name,
None
],
geography_scopes),
related_model_path_to_name)
)
def annotated_related_feature_class_pk_via_geographies(manager, config_entity, db_entity_keys):
"""
To join a related model by geographic join
"""
from footprint.main.models.feature.feature_class_creator import FeatureClassCreator
feature_class_creator = FeatureClassCreator.from_dynamic_model_class(manager.model)
def resolve_related_model_pk(db_entity_key):
related_model = config_entity.db_entity_feature_class(db_entity_key)
# The common Geography class
geography_class = feature_class_creator.common_geography_class(related_model)
geography_scope = feature_class_creator.common_geography_scope(related_model)
logger.warn("Resolved geography scope %s", geography_scope)
# Find the geographies ManyToMany fields that relates this model to the related_model
# via a Geography class. Which geography class depends on their common geography scope
geographies_field = feature_class_creator.geographies_field(geography_scope)
try:
# Find the queryable field name from the geography class to the related model
related_model_geographies_field_name = resolve_queryable_name_of_type(geography_class, related_model)
except:
# Sometimes the geography class hasn't had its fields cached properly. Fix here
clear_many_cache(geography_class)
related_model_geographies_field_name = resolve_queryable_name_of_type(geography_class, related_model)
return "%s__%s__pk" % (geographies_field.name, related_model_geographies_field_name)
pk_paths = map_to_dict(
lambda db_entity_key: [db_entity_key, Min(resolve_related_model_pk(db_entity_key))], db_entity_keys
)
return manager.annotate(**pk_paths)
def annotated_related_feature_class_pk_via_geographies(manager, config_entity, db_entity_keys):
"""
To join a related model by geographic join
"""
from footprint.main.models.feature.feature_class_creator import FeatureClassCreator
feature_class_creator = FeatureClassCreator.from_dynamic_model_class(manager.model)
def resolve_related_model_pk(db_entity_key):
related_model = config_entity.db_entity_feature_class(db_entity_key)
# The common Geography class
geography_class = feature_class_creator.common_geography_class(related_model)
geography_scope = feature_class_creator.common_geography_scope(related_model)
logger.warn("Resolved geography scope %s", geography_scope)
# Find the geographies ManyToMany fields that relates this model to the related_model
# via a Geography class. Which geography class depends on their common geography scope
geographies_field = feature_class_creator.geographies_field(geography_scope)
try:
# Find the queryable field name from the geography class to the related model
related_model_geographies_field_name = resolve_queryable_name_of_type(geography_class, related_model)
except:
# Sometimes the geography class hasn't had its fields cached properly. Fix here
clear_many_cache(geography_class)
related_model_geographies_field_name = resolve_queryable_name_of_type(geography_class, related_model)
return '%s__%s__pk' % (geographies_field.name, related_model_geographies_field_name)
pk_paths = map_to_dict(lambda db_entity_key:
[db_entity_key, Min(resolve_related_model_pk(db_entity_key))],
db_entity_keys)
return manager.annotate(**pk_paths)
def create_result_map(self, values_query_set):
related_models = self.resolve_join_models()
logger.debug(
"Creating result map for related models %s feature class %s" %
(', '.join(map(lambda r: str(r),
related_models)), self.feature_class))
feature_class_creator = FeatureClassCreator.from_dynamic_model_class(
self.feature_class)
geography_scopes = feature_class_creator.geography_scopes()
# Get the related model paths final segment. We want to map these to the db_entity_key names
related_model_path_to_name = map_to_dict(
lambda related_model: [
resolve_related_model_path_via_geographies(
self.feature_class.objects, related_model).split('__')[1],
related_model.db_entity_key
], related_models)
return values_query_set.create_result_map(
related_models=related_models,
# map_path_segments maps related object paths to their model name,
# and removes the geographies segment of the path
map_path_segments=merge(
# Map each geography scope to its corresponding field on the feature class
map_to_dict(
lambda geography_scope: [
feature_class_creator.geographies_field(geography_scope
).name, None
], geography_scopes),
related_model_path_to_name))
def resolve_field_path_via_geographies(field_path, manager, related_models):
"""
Resolve the given field path in case its not absolute.
For instance, if it is 'block' and one of our related models accessible via geographies__relatedmodel has that property,
return 'geographies_[scope_id]__relatedmodel__block'
It will also be tested against the main manager after all related models fail,
e.g. manager.values(field_path) if successful would simply return field_path
:param field_path: django field path. e.g. du or built_form__name
:param manager: The main manager by which the related models are resolved and by which the full path is computed
:param related_models: models joined to the manager. For instance. manager.model is CanvasFeature, a related_model could be
CensusBlock, which might be related to the former via 'geographies_[scope_id]__censusblock9rel'. The relationship is computed
by assuming that the related model is related by geographies and looking for a field matching its type
:return:
"""
from footprint.main.models.feature.feature_class_creator import FeatureClassCreator
feature_class_creator = FeatureClassCreator.from_dynamic_model_class(manager.model)
for related_model in related_models:
try:
# See if the field_name resolves
# There's probably a more efficient way to do this
related_model.objects.values(field_path)
resolved_field_path = field_path
except:
# See if the first segment matches the related_model db_entity_key
first_segment = field_path.split("__")[0]
if first_segment != related_model.db_entity_key:
# If not, move on
continue
# Take all but the first segment
resolved_field_path = "__".join(field_path.split("__")[1:])
# Success, find the path to this model from geographies
geography_class = feature_class_creator.common_geography_class(related_model)
geographies_field = feature_class_creator.geographies_field(
feature_class_creator.common_geography_scope(related_model)
)
geography_related_field_name = resolve_queryable_name_of_type(geography_class, related_model)
return "%s__%s__%s" % (geographies_field.name, geography_related_field_name, resolved_field_path)
# See if it matches the main model
try:
if field_path.split("__")[0] == manager.model.db_entity_key:
# If the manager model db_entity_key was used in the path, just strip it out
updated_field_path = "__".join(field_path.split("__")[1:])
manager.values(updated_field_path)
else:
# Otherwise test query with the full path
updated_field_path = field_path
manager.values(updated_field_path)
# Success, return the field_path
return updated_field_path
except:
logger.exception(
"Cannot resolve field path %s to the main model %s or any joined models %s",
field_path,
manager.model,
related_models,
)
raise
def resolve_field_path_via_geographies(field_path, manager, related_models):
"""
Resolve the given field path in case its not absolute.
For instance, if it is 'block' and one of our related models accessible via geographies__relatedmodel has that property,
return 'geographies_[scope_id]__relatedmodel__block'
It will also be tested against the main manager after all related models fail,
e.g. manager.values(field_path) if successful would simply return field_path
:param field_path: django field path. e.g. du or built_form__name
:param manager: The main manager by which the related models are resolved and by which the full path is computed
:param related_models: models joined to the manager. For instance. manager.model is CanvasFeature, a related_model could be
CensusBlock, which might be related to the former via 'geographies_[scope_id]__censusblock9rel'. The relationship is computed
by assuming that the related model is related by geographies and looking for a field matching its type
:return:
"""
from footprint.main.models.feature.feature_class_creator import FeatureClassCreator
feature_class_creator = FeatureClassCreator.from_dynamic_model_class(
manager.model)
for related_model in related_models:
try:
# See if the field_name resolves
# There's probably a more efficient way to do this
related_model.objects.values(field_path)
resolved_field_path = field_path
except:
# See if the first segment matches the related_model db_entity_key
first_segment = field_path.split('__')[0]
if first_segment != related_model.db_entity_key:
# If not, move on
continue
# Take all but the first segment
resolved_field_path = '__'.join(field_path.split('__')[1:])
# Success, find the path to this model from geographies
geography_class = feature_class_creator.common_geography_class(
related_model)
geographies_field = feature_class_creator.geographies_field(
feature_class_creator.common_geography_scope(related_model))
geography_related_field_name = resolve_queryable_name_of_type(
geography_class, related_model)
return '%s__%s__%s' % (geographies_field.name,
geography_related_field_name,
resolved_field_path)
# See if it matches the main model
try:
if field_path.split('__')[0] == manager.model.db_entity_key:
# If the manager model db_entity_key was used in the path, just strip it out
updated_field_path = '__'.join(field_path.split('__')[1:])
manager.values(updated_field_path)
else:
# Otherwise test query with the full path
updated_field_path = field_path
manager.values(updated_field_path)
# Success, return the field_path
return updated_field_path
except:
logger.exception(
'Cannot resolve field path %s to the main model %s or any joined models %s',
field_path, manager.model, related_models)
raise
def resolve_related_model_path_via_geographies(manager, related_model):
"""
Returns the query string path 'geographies_[scope_id]__[field name of the related model form the main model]'
The scope_id is the id of the ConfigEntity that both models share in common by ascending the ConfigEntity
hierarchy starting at each models' geography_scope
"""
from footprint.main.models.feature.feature_class_creator import FeatureClassCreator
feature_class_creator = FeatureClassCreator.from_dynamic_model_class(manager.model)
geography_scope = feature_class_creator.common_geography_scope(related_model)
geographies_field = feature_class_creator.geographies_field(geography_scope)
geography_class = feature_class_creator.common_geography_class(related_model)
geography_related_field_name = resolve_queryable_name_of_type(geography_class, related_model)
return '%s__%s' % (geographies_field.name, geography_related_field_name)
def resolve_related_model_path_via_geographies(manager, related_model):
"""
Returns the query string path 'geographies_[scope_id]__[field name of the related model form the main model]'
The scope_id is the id of the ConfigEntity that both models share in common by ascending the ConfigEntity
hierarchy starting at each models' geography_scope
"""
from footprint.main.models.feature.feature_class_creator import FeatureClassCreator
feature_class_creator = FeatureClassCreator.from_dynamic_model_class(manager.model)
geography_scope = feature_class_creator.common_geography_scope(related_model)
geographies_field = feature_class_creator.geographies_field(geography_scope)
geography_class = feature_class_creator.common_geography_class(related_model)
geography_related_field_name = resolve_queryable_name_of_type(geography_class, related_model)
return "%s__%s" % (geographies_field.name, geography_related_field_name)
def result_map(cls):
"""
Creates an caches a result map for the Feature class. The result_map has useful meta data about
the class
:param cls:
:return:
"""
if cls._result_map:
return cls._result_map
from footprint.main.models.feature.feature_class_creator import FeatureClassCreator
feature_class_creator = FeatureClassCreator.from_dynamic_model_class(cls)
if not feature_class_creator.dynamic_model_class_is_ready:
return None
cls._result_map = feature_class_creator.dynamic_model_class().objects.all().create_result_map()
return cls._result_map
def result_map(cls):
"""
Creates an caches a result map for the Feature class. The result_map has useful meta data about
the class
:param cls:
:return:
"""
if cls._result_map:
return cls._result_map
from footprint.main.models.feature.feature_class_creator import FeatureClassCreator
feature_class_creator = FeatureClassCreator.from_dynamic_model_class(
cls)
if not feature_class_creator.dynamic_model_class_is_ready:
return None
cls._result_map = feature_class_creator.dynamic_model_class(
).objects.all().create_result_map()
return cls._result_map
def dynamic_resource_subclass(self, layer_selection=None, db_entity=None, feature_class=None, config_entity=None, metadata=None, params=None, **kwargs):
"""
Creates the dynamic Feature Resource class by passing in a layer_selection, db_entity, or feature_class
:param layer_selection: Required if db_entity or metadata aren't present
:param db_entity: Required if layer_selection or metadata aren't present
:param metadata: Required along with config_entity if layer_selection or db_entity aren't present
:param kwargs:
:return:
"""
feature_class_configuration = None
if layer_selection:
# Coming in relative to a LayerSelection, which puts us in the context of the LayerSelection's
# feature query for this Feature subclass
layer = layer_selection.layer
# If we pass in a ConfigEntity it means we want to scope the Feature class to its scope.
# The ConfigEntity defaults to that of the Layer, but we can override it to be a lower
# scope to make sure that we have access to lower DbEntities of performing joins
config_entity = config_entity.subclassed if config_entity else layer.config_entity.subclassed
logger.debug("Resolving FeatureResource subclass for layer_selection: {0}, config_entity: {1}".format(layer_selection.unique_id, config_entity.id))
# Resolve the dynamic Feature class with the given config_entity so that we can access all DbEntities
# of the ConfigEntity for joins
feature_class = config_entity.db_entity_feature_class(layer.db_entity.key)
elif db_entity:
# Coming in relative to a DbEntity, meaning we don't care about a particular LayerSelection's
# feature query for this Feature subclass
config_entity = db_entity.config_entity
logger.debug("Resolving FeatureResource subclass for db_entity: {0}, config_entity: {1}".format(db_entity.id, config_entity.id))
# Resolve the dynamic Feature class with the given config_entity so that we can access all DbEntities
# of the ConfigEntity for joins
feature_class = config_entity.db_entity_feature_class(db_entity.key)
elif metadata:
# Coming in with metadata, meaning this is and uploaded or ArcGis table with no DbEntity yet
# We need to construct a FeatureClass from the metadata
logger.debug("Resolving FeatureResource subclass for metadata: {0}, config_entity: {1}".format(metadata, config_entity.id))
feature_class_creator = FeatureClassCreator(
config_entity
)
feature_class_configuration = feature_class_creator.feature_class_configuration_from_metadata(metadata['schema'])
feature_class = FeatureClassCreator(
config_entity,
feature_class_configuration
).dynamic_model_class()
if not feature_class_configuration:
# If we didn't already ensure all dynamic model classes have been created
# This only need to run once to get all dynamic feature subclasses into memory,
# in case they are needed by an association, join, or something similar
feature_class_creator = FeatureClassCreator.from_dynamic_model_class(feature_class)
feature_class_creator.ensure_dynamic_models()
logger.debug("Resolving resource for Feature subclass: {0}".format(feature_class))
# Resolve the FeatureResource subclass based on the given Feature subclass
# If self is already a subclass, just return self
# Else, return a preconfigured subclass or one dynamically created. The latter will probably be the only way in the future.
# If not already subclassed
is_singleton_feature = issubclass(self.__class__, SingletonFeatureResourceMixin)
is_template_feature = self.__class__ == TemplateFeatureResource
if self.__class__ in [FeatureResource, TemplateFeatureResource, FeatureCategoryAttributeResource,
FeatureQuantitativeAttributeResource]:
if is_singleton_feature or params.get('is_feature_attribute'):
queryset = feature_class.objects.none()
elif kwargs.get('method', None) == 'PATCH':
# It's possible to PATCH with an active join query.
# But we don't want to use a join query when patching
queryset = feature_class.objects.all()
else:
# Get the queryset stored by the layer_selection or an empty query if we don't have a layer_selection
queryset = layer_selection.selected_features_or_values if\
layer_selection else \
feature_class.objects.none()
if layer_selection and not (is_singleton_feature or kwargs.get('query_may_be_empty')) and queryset.count()==0:
raise Exception(
"Unexpected empty queryset for layer_selection features: %s" %
queryset.query)
is_values_queryset = isinstance(queryset, ValuesQuerySet)
#returns queryset ordered by the table id
queryset = queryset.order_by('id')
if is_values_queryset:
join_feature_class = layer_selection.create_join_feature_class() if is_values_queryset else feature_class
logger.info("Created join_feature_class: %s" % join_feature_class)
# Force the queryset to our new class so that Tastypie can map the dict results to it
queryset.model = join_feature_class
return self.__class__.resolve_resource_class(
join_feature_class,
queryset=queryset,
base_resource_class=self.join_feature_resource_class(join_feature_class),
additional_fields_dict=dict(
# Pass these to the feature resource to help it resolve
# field mappings and add related fields (just need for join_feature_class)
# Use the layer_selection if it exists since it might have filtered or extra query fields
result_field_lookup=(layer_selection or db_entity).result_field_lookup if not metadata else {},
related_field_lookup=(layer_selection or db_entity).related_field_lookup if not metadata else {},
# We use these in the FeatureResource to create a unique id for each join Feature
join_model_attributes=layer_selection and layer_selection.result_map.join_model_attributes
),
is_join_query=True,
limit_fields=layer_selection.result_map['result_fields']
)
else:
abstract_feature_resource_class = self.__class__
resource_class = abstract_feature_resource_class.resolve_resource_class(
feature_class,
queryset=queryset,
# Give FeatureResource a reference to the layer_selection
additional_fields_dict=merge(
dict(
# Pass this to the feature resource to help it resolve field mappings
result_field_lookup=(layer_selection or db_entity).result_field_lookup if not metadata else {}
),
dict(
# Not sure why it doesn't work to just stick this on the TemplateFeatureResource
feature_fields=ListField(attribute='feature_fields', null=True, blank=True, readonly=True),
feature_field_title_lookup=PickledDictField(attribute='feature_field_title_lookup', null=True, blank=True, readonly=True),
) if is_template_feature else dict()
),
for_template=is_template_feature
)
return resource_class
return self
def dynamic_resource_subclass(self,
layer_selection=None,
db_entity=None,
feature_class=None,
config_entity=None,
metadata=None,
params=None,
**kwargs):
"""
Creates the dynamic Feature Resource class by passing in a layer_selection, db_entity, or feature_class
:param layer_selection: Required if db_entity or metadata aren't present
:param db_entity: Required if layer_selection or metadata aren't present
:param metadata: Required along with config_entity if layer_selection or db_entity aren't present
:param kwargs:
:return:
"""
feature_class_configuration = None
if layer_selection:
# Coming in relative to a LayerSelection, which puts us in the context of the LayerSelection's
# feature query for this Feature subclass
layer = layer_selection.layer
# If we pass in a ConfigEntity it means we want to scope the Feature class to its scope.
# The ConfigEntity defaults to that of the Layer, but we can override it to be a lower
# scope to make sure that we have access to lower DbEntities of performing joins
config_entity = config_entity.subclassed if config_entity else layer.config_entity.subclassed
logger.debug(
"Resolving FeatureResource subclass for layer_selection: {0}, config_entity: {1}"
.format(layer_selection.unique_id, config_entity.id))
# Resolve the dynamic Feature class with the given config_entity so that we can access all DbEntities
# of the ConfigEntity for joins
feature_class = config_entity.db_entity_feature_class(
layer.db_entity.key)
elif db_entity:
# Coming in relative to a DbEntity, meaning we don't care about a particular LayerSelection's
# feature query for this Feature subclass
config_entity = db_entity.config_entity
logger.debug(
"Resolving FeatureResource subclass for db_entity: {0}, config_entity: {1}"
.format(db_entity.id, config_entity.id))
# Resolve the dynamic Feature class with the given config_entity so that we can access all DbEntities
# of the ConfigEntity for joins
feature_class = config_entity.db_entity_feature_class(
db_entity.key)
elif metadata:
# Coming in with metadata, meaning this is and uploaded or ArcGis table with no DbEntity yet
# We need to construct a FeatureClass from the metadata
logger.debug(
"Resolving FeatureResource subclass for metadata: {0}, config_entity: {1}"
.format(metadata, config_entity.id))
feature_class_creator = FeatureClassCreator(config_entity)
feature_class_configuration = feature_class_creator.feature_class_configuration_from_metadata(
metadata['schema'])
feature_class = FeatureClassCreator(
config_entity,
feature_class_configuration).dynamic_model_class()
if not feature_class_configuration:
# If we didn't already ensure all dynamic model classes have been created
# This only need to run once to get all dynamic feature subclasses into memory,
# in case they are needed by an association, join, or something similar
feature_class_creator = FeatureClassCreator.from_dynamic_model_class(
feature_class)
feature_class_creator.ensure_dynamic_models()
logger.debug("Resolving resource for Feature subclass: {0}".format(
feature_class))
# Resolve the FeatureResource subclass based on the given Feature subclass
# If self is already a subclass, just return self
# Else, return a preconfigured subclass or one dynamically created. The latter will probably be the only way in the future.
# If not already subclassed
is_singleton_feature = issubclass(self.__class__,
SingletonFeatureResourceMixin)
is_template_feature = self.__class__ == TemplateFeatureResource
if self.__class__ in [
FeatureResource, TemplateFeatureResource,
FeatureCategoryAttributeResource,
FeatureQuantitativeAttributeResource
]:
if is_singleton_feature or params.get('is_feature_attribute'):
queryset = feature_class.objects.none()
elif kwargs.get('method', None) == 'PATCH':
# It's possible to PATCH with an active join query.
# But we don't want to use a join query when patching
queryset = feature_class.objects.all()
else:
# Get the queryset stored by the layer_selection or an empty query if we don't have a layer_selection
queryset = layer_selection.selected_features_or_values if\
layer_selection else \
feature_class.objects.none()
if layer_selection and not (is_singleton_feature or kwargs.get(
'query_may_be_empty')) and queryset.count() == 0:
raise Exception(
"Unexpected empty queryset for layer_selection features: %s"
% queryset.query)
is_values_queryset = isinstance(queryset, ValuesQuerySet)
#returns queryset ordered by the table id
queryset = queryset.order_by('id')
if is_values_queryset:
join_feature_class = layer_selection.create_join_feature_class(
) if is_values_queryset else feature_class
logger.info("Created join_feature_class: %s" %
join_feature_class)
# Force the queryset to our new class so that Tastypie can map the dict results to it
queryset.model = join_feature_class
return self.__class__.resolve_resource_class(
join_feature_class,
queryset=queryset,
base_resource_class=self.join_feature_resource_class(
join_feature_class),
additional_fields_dict=dict(
# Pass these to the feature resource to help it resolve
# field mappings and add related fields (just need for join_feature_class)
# Use the layer_selection if it exists since it might have filtered or extra query fields
result_field_lookup=(layer_selection
or db_entity).result_field_lookup
if not metadata else {},
related_field_lookup=(
layer_selection or db_entity).related_field_lookup
if not metadata else {},
# We use these in the FeatureResource to create a unique id for each join Feature
join_model_attributes=layer_selection
and layer_selection.result_map.join_model_attributes),
is_join_query=True,
limit_fields=layer_selection.result_map['result_fields'])
else:
abstract_feature_resource_class = self.__class__
resource_class = abstract_feature_resource_class.resolve_resource_class(
feature_class,
queryset=queryset,
# Give FeatureResource a reference to the layer_selection
additional_fields_dict=merge(
dict(
# Pass this to the feature resource to help it resolve field mappings
result_field_lookup=(layer_selection or db_entity).
result_field_lookup if not metadata else {}),
dict(
# Not sure why it doesn't work to just stick this on the TemplateFeatureResource
feature_fields=ListField(
attribute='feature_fields',
null=True,
blank=True,
readonly=True),
feature_field_title_lookup=PickledDictField(
attribute='feature_field_title_lookup',
null=True,
blank=True,
readonly=True),
) if is_template_feature else dict()),
for_template=is_template_feature)
return resource_class
return self
