def create_result_map(self, values_query_set): related_models = self.resolve_join_models() logger.debug("Creating result map for related models %s feature class %s" % (', '.join(map(lambda r: str(r), related_models)), self.feature_class)) feature_class_creator = FeatureClassCreator.from_dynamic_model_class(self.feature_class) geography_scopes = feature_class_creator.geography_scopes() # Get the related model paths final segment. We want to map these to the db_entity_key names related_model_path_to_name = map_to_dict( lambda related_model: [resolve_related_model_path_via_geographies( self.feature_class.objects, related_model).split('__')[1], related_model.db_entity_key], related_models ) return values_query_set.create_result_map( related_models=related_models, # map_path_segments maps related object paths to their model name, # and removes the geographies segment of the path map_path_segments=merge( # Map each geography scope to its corresponding field on the feature class map_to_dict( lambda geography_scope: [ feature_class_creator.geographies_field(geography_scope).name, None ], geography_scopes), related_model_path_to_name) )
def annotated_related_feature_class_pk_via_geographies(manager, config_entity, db_entity_keys): """ To join a related model by geographic join """ from footprint.main.models.feature.feature_class_creator import FeatureClassCreator feature_class_creator = FeatureClassCreator.from_dynamic_model_class(manager.model) def resolve_related_model_pk(db_entity_key): related_model = config_entity.db_entity_feature_class(db_entity_key) # The common Geography class geography_class = feature_class_creator.common_geography_class(related_model) geography_scope = feature_class_creator.common_geography_scope(related_model) logger.warn("Resolved geography scope %s", geography_scope) # Find the geographies ManyToMany fields that relates this model to the related_model # via a Geography class. Which geography class depends on their common geography scope geographies_field = feature_class_creator.geographies_field(geography_scope) try: # Find the queryable field name from the geography class to the related model related_model_geographies_field_name = resolve_queryable_name_of_type(geography_class, related_model) except: # Sometimes the geography class hasn't had its fields cached properly. Fix here clear_many_cache(geography_class) related_model_geographies_field_name = resolve_queryable_name_of_type(geography_class, related_model) return "%s__%s__pk" % (geographies_field.name, related_model_geographies_field_name) pk_paths = map_to_dict( lambda db_entity_key: [db_entity_key, Min(resolve_related_model_pk(db_entity_key))], db_entity_keys ) return manager.annotate(**pk_paths)
def annotated_related_feature_class_pk_via_geographies(manager, config_entity, db_entity_keys): """ To join a related model by geographic join """ from footprint.main.models.feature.feature_class_creator import FeatureClassCreator feature_class_creator = FeatureClassCreator.from_dynamic_model_class(manager.model) def resolve_related_model_pk(db_entity_key): related_model = config_entity.db_entity_feature_class(db_entity_key) # The common Geography class geography_class = feature_class_creator.common_geography_class(related_model) geography_scope = feature_class_creator.common_geography_scope(related_model) logger.warn("Resolved geography scope %s", geography_scope) # Find the geographies ManyToMany fields that relates this model to the related_model # via a Geography class. Which geography class depends on their common geography scope geographies_field = feature_class_creator.geographies_field(geography_scope) try: # Find the queryable field name from the geography class to the related model related_model_geographies_field_name = resolve_queryable_name_of_type(geography_class, related_model) except: # Sometimes the geography class hasn't had its fields cached properly. Fix here clear_many_cache(geography_class) related_model_geographies_field_name = resolve_queryable_name_of_type(geography_class, related_model) return '%s__%s__pk' % (geographies_field.name, related_model_geographies_field_name) pk_paths = map_to_dict(lambda db_entity_key: [db_entity_key, Min(resolve_related_model_pk(db_entity_key))], db_entity_keys) return manager.annotate(**pk_paths)
def create_result_map(self, values_query_set): related_models = self.resolve_join_models() logger.debug( "Creating result map for related models %s feature class %s" % (', '.join(map(lambda r: str(r), related_models)), self.feature_class)) feature_class_creator = FeatureClassCreator.from_dynamic_model_class( self.feature_class) geography_scopes = feature_class_creator.geography_scopes() # Get the related model paths final segment. We want to map these to the db_entity_key names related_model_path_to_name = map_to_dict( lambda related_model: [ resolve_related_model_path_via_geographies( self.feature_class.objects, related_model).split('__')[1], related_model.db_entity_key ], related_models) return values_query_set.create_result_map( related_models=related_models, # map_path_segments maps related object paths to their model name, # and removes the geographies segment of the path map_path_segments=merge( # Map each geography scope to its corresponding field on the feature class map_to_dict( lambda geography_scope: [ feature_class_creator.geographies_field(geography_scope ).name, None ], geography_scopes), related_model_path_to_name))
def resolve_field_path_via_geographies(field_path, manager, related_models): """ Resolve the given field path in case its not absolute. For instance, if it is 'block' and one of our related models accessible via geographies__relatedmodel has that property, return 'geographies_[scope_id]__relatedmodel__block' It will also be tested against the main manager after all related models fail, e.g. manager.values(field_path) if successful would simply return field_path :param field_path: django field path. e.g. du or built_form__name :param manager: The main manager by which the related models are resolved and by which the full path is computed :param related_models: models joined to the manager. For instance. manager.model is CanvasFeature, a related_model could be CensusBlock, which might be related to the former via 'geographies_[scope_id]__censusblock9rel'. The relationship is computed by assuming that the related model is related by geographies and looking for a field matching its type :return: """ from footprint.main.models.feature.feature_class_creator import FeatureClassCreator feature_class_creator = FeatureClassCreator.from_dynamic_model_class(manager.model) for related_model in related_models: try: # See if the field_name resolves # There's probably a more efficient way to do this related_model.objects.values(field_path) resolved_field_path = field_path except: # See if the first segment matches the related_model db_entity_key first_segment = field_path.split("__")[0] if first_segment != related_model.db_entity_key: # If not, move on continue # Take all but the first segment resolved_field_path = "__".join(field_path.split("__")[1:]) # Success, find the path to this model from geographies geography_class = feature_class_creator.common_geography_class(related_model) geographies_field = feature_class_creator.geographies_field( feature_class_creator.common_geography_scope(related_model) ) geography_related_field_name = resolve_queryable_name_of_type(geography_class, related_model) return "%s__%s__%s" % (geographies_field.name, geography_related_field_name, resolved_field_path) # See if it matches the main model try: if field_path.split("__")[0] == manager.model.db_entity_key: # If the manager model db_entity_key was used in the path, just strip it out updated_field_path = "__".join(field_path.split("__")[1:]) manager.values(updated_field_path) else: # Otherwise test query with the full path updated_field_path = field_path manager.values(updated_field_path) # Success, return the field_path return updated_field_path except: logger.exception( "Cannot resolve field path %s to the main model %s or any joined models %s", field_path, manager.model, related_models, ) raise
def resolve_field_path_via_geographies(field_path, manager, related_models): """ Resolve the given field path in case its not absolute. For instance, if it is 'block' and one of our related models accessible via geographies__relatedmodel has that property, return 'geographies_[scope_id]__relatedmodel__block' It will also be tested against the main manager after all related models fail, e.g. manager.values(field_path) if successful would simply return field_path :param field_path: django field path. e.g. du or built_form__name :param manager: The main manager by which the related models are resolved and by which the full path is computed :param related_models: models joined to the manager. For instance. manager.model is CanvasFeature, a related_model could be CensusBlock, which might be related to the former via 'geographies_[scope_id]__censusblock9rel'. The relationship is computed by assuming that the related model is related by geographies and looking for a field matching its type :return: """ from footprint.main.models.feature.feature_class_creator import FeatureClassCreator feature_class_creator = FeatureClassCreator.from_dynamic_model_class( manager.model) for related_model in related_models: try: # See if the field_name resolves # There's probably a more efficient way to do this related_model.objects.values(field_path) resolved_field_path = field_path except: # See if the first segment matches the related_model db_entity_key first_segment = field_path.split('__')[0] if first_segment != related_model.db_entity_key: # If not, move on continue # Take all but the first segment resolved_field_path = '__'.join(field_path.split('__')[1:]) # Success, find the path to this model from geographies geography_class = feature_class_creator.common_geography_class( related_model) geographies_field = feature_class_creator.geographies_field( feature_class_creator.common_geography_scope(related_model)) geography_related_field_name = resolve_queryable_name_of_type( geography_class, related_model) return '%s__%s__%s' % (geographies_field.name, geography_related_field_name, resolved_field_path) # See if it matches the main model try: if field_path.split('__')[0] == manager.model.db_entity_key: # If the manager model db_entity_key was used in the path, just strip it out updated_field_path = '__'.join(field_path.split('__')[1:]) manager.values(updated_field_path) else: # Otherwise test query with the full path updated_field_path = field_path manager.values(updated_field_path) # Success, return the field_path return updated_field_path except: logger.exception( 'Cannot resolve field path %s to the main model %s or any joined models %s', field_path, manager.model, related_models) raise
def resolve_related_model_path_via_geographies(manager, related_model): """ Returns the query string path 'geographies_[scope_id]__[field name of the related model form the main model]' The scope_id is the id of the ConfigEntity that both models share in common by ascending the ConfigEntity hierarchy starting at each models' geography_scope """ from footprint.main.models.feature.feature_class_creator import FeatureClassCreator feature_class_creator = FeatureClassCreator.from_dynamic_model_class(manager.model) geography_scope = feature_class_creator.common_geography_scope(related_model) geographies_field = feature_class_creator.geographies_field(geography_scope) geography_class = feature_class_creator.common_geography_class(related_model) geography_related_field_name = resolve_queryable_name_of_type(geography_class, related_model) return '%s__%s' % (geographies_field.name, geography_related_field_name)
def resolve_related_model_path_via_geographies(manager, related_model): """ Returns the query string path 'geographies_[scope_id]__[field name of the related model form the main model]' The scope_id is the id of the ConfigEntity that both models share in common by ascending the ConfigEntity hierarchy starting at each models' geography_scope """ from footprint.main.models.feature.feature_class_creator import FeatureClassCreator feature_class_creator = FeatureClassCreator.from_dynamic_model_class(manager.model) geography_scope = feature_class_creator.common_geography_scope(related_model) geographies_field = feature_class_creator.geographies_field(geography_scope) geography_class = feature_class_creator.common_geography_class(related_model) geography_related_field_name = resolve_queryable_name_of_type(geography_class, related_model) return "%s__%s" % (geographies_field.name, geography_related_field_name)
def result_map(cls): """ Creates an caches a result map for the Feature class. The result_map has useful meta data about the class :param cls: :return: """ if cls._result_map: return cls._result_map from footprint.main.models.feature.feature_class_creator import FeatureClassCreator feature_class_creator = FeatureClassCreator.from_dynamic_model_class(cls) if not feature_class_creator.dynamic_model_class_is_ready: return None cls._result_map = feature_class_creator.dynamic_model_class().objects.all().create_result_map() return cls._result_map
def result_map(cls): """ Creates an caches a result map for the Feature class. The result_map has useful meta data about the class :param cls: :return: """ if cls._result_map: return cls._result_map from footprint.main.models.feature.feature_class_creator import FeatureClassCreator feature_class_creator = FeatureClassCreator.from_dynamic_model_class( cls) if not feature_class_creator.dynamic_model_class_is_ready: return None cls._result_map = feature_class_creator.dynamic_model_class( ).objects.all().create_result_map() return cls._result_map
def dynamic_resource_subclass(self, layer_selection=None, db_entity=None, feature_class=None, config_entity=None, metadata=None, params=None, **kwargs): """ Creates the dynamic Feature Resource class by passing in a layer_selection, db_entity, or feature_class :param layer_selection: Required if db_entity or metadata aren't present :param db_entity: Required if layer_selection or metadata aren't present :param metadata: Required along with config_entity if layer_selection or db_entity aren't present :param kwargs: :return: """ feature_class_configuration = None if layer_selection: # Coming in relative to a LayerSelection, which puts us in the context of the LayerSelection's # feature query for this Feature subclass layer = layer_selection.layer # If we pass in a ConfigEntity it means we want to scope the Feature class to its scope. # The ConfigEntity defaults to that of the Layer, but we can override it to be a lower # scope to make sure that we have access to lower DbEntities of performing joins config_entity = config_entity.subclassed if config_entity else layer.config_entity.subclassed logger.debug("Resolving FeatureResource subclass for layer_selection: {0}, config_entity: {1}".format(layer_selection.unique_id, config_entity.id)) # Resolve the dynamic Feature class with the given config_entity so that we can access all DbEntities # of the ConfigEntity for joins feature_class = config_entity.db_entity_feature_class(layer.db_entity.key) elif db_entity: # Coming in relative to a DbEntity, meaning we don't care about a particular LayerSelection's # feature query for this Feature subclass config_entity = db_entity.config_entity logger.debug("Resolving FeatureResource subclass for db_entity: {0}, config_entity: {1}".format(db_entity.id, config_entity.id)) # Resolve the dynamic Feature class with the given config_entity so that we can access all DbEntities # of the ConfigEntity for joins feature_class = config_entity.db_entity_feature_class(db_entity.key) elif metadata: # Coming in with metadata, meaning this is and uploaded or ArcGis table with no DbEntity yet # We need to construct a FeatureClass from the metadata logger.debug("Resolving FeatureResource subclass for metadata: {0}, config_entity: {1}".format(metadata, config_entity.id)) feature_class_creator = FeatureClassCreator( config_entity ) feature_class_configuration = feature_class_creator.feature_class_configuration_from_metadata(metadata['schema']) feature_class = FeatureClassCreator( config_entity, feature_class_configuration ).dynamic_model_class() if not feature_class_configuration: # If we didn't already ensure all dynamic model classes have been created # This only need to run once to get all dynamic feature subclasses into memory, # in case they are needed by an association, join, or something similar feature_class_creator = FeatureClassCreator.from_dynamic_model_class(feature_class) feature_class_creator.ensure_dynamic_models() logger.debug("Resolving resource for Feature subclass: {0}".format(feature_class)) # Resolve the FeatureResource subclass based on the given Feature subclass # If self is already a subclass, just return self # Else, return a preconfigured subclass or one dynamically created. The latter will probably be the only way in the future. # If not already subclassed is_singleton_feature = issubclass(self.__class__, SingletonFeatureResourceMixin) is_template_feature = self.__class__ == TemplateFeatureResource if self.__class__ in [FeatureResource, TemplateFeatureResource, FeatureCategoryAttributeResource, FeatureQuantitativeAttributeResource]: if is_singleton_feature or params.get('is_feature_attribute'): queryset = feature_class.objects.none() elif kwargs.get('method', None) == 'PATCH': # It's possible to PATCH with an active join query. # But we don't want to use a join query when patching queryset = feature_class.objects.all() else: # Get the queryset stored by the layer_selection or an empty query if we don't have a layer_selection queryset = layer_selection.selected_features_or_values if\ layer_selection else \ feature_class.objects.none() if layer_selection and not (is_singleton_feature or kwargs.get('query_may_be_empty')) and queryset.count()==0: raise Exception( "Unexpected empty queryset for layer_selection features: %s" % queryset.query) is_values_queryset = isinstance(queryset, ValuesQuerySet) #returns queryset ordered by the table id queryset = queryset.order_by('id') if is_values_queryset: join_feature_class = layer_selection.create_join_feature_class() if is_values_queryset else feature_class logger.info("Created join_feature_class: %s" % join_feature_class) # Force the queryset to our new class so that Tastypie can map the dict results to it queryset.model = join_feature_class return self.__class__.resolve_resource_class( join_feature_class, queryset=queryset, base_resource_class=self.join_feature_resource_class(join_feature_class), additional_fields_dict=dict( # Pass these to the feature resource to help it resolve # field mappings and add related fields (just need for join_feature_class) # Use the layer_selection if it exists since it might have filtered or extra query fields result_field_lookup=(layer_selection or db_entity).result_field_lookup if not metadata else {}, related_field_lookup=(layer_selection or db_entity).related_field_lookup if not metadata else {}, # We use these in the FeatureResource to create a unique id for each join Feature join_model_attributes=layer_selection and layer_selection.result_map.join_model_attributes ), is_join_query=True, limit_fields=layer_selection.result_map['result_fields'] ) else: abstract_feature_resource_class = self.__class__ resource_class = abstract_feature_resource_class.resolve_resource_class( feature_class, queryset=queryset, # Give FeatureResource a reference to the layer_selection additional_fields_dict=merge( dict( # Pass this to the feature resource to help it resolve field mappings result_field_lookup=(layer_selection or db_entity).result_field_lookup if not metadata else {} ), dict( # Not sure why it doesn't work to just stick this on the TemplateFeatureResource feature_fields=ListField(attribute='feature_fields', null=True, blank=True, readonly=True), feature_field_title_lookup=PickledDictField(attribute='feature_field_title_lookup', null=True, blank=True, readonly=True), ) if is_template_feature else dict() ), for_template=is_template_feature ) return resource_class return self
def dynamic_resource_subclass(self, layer_selection=None, db_entity=None, feature_class=None, config_entity=None, metadata=None, params=None, **kwargs): """ Creates the dynamic Feature Resource class by passing in a layer_selection, db_entity, or feature_class :param layer_selection: Required if db_entity or metadata aren't present :param db_entity: Required if layer_selection or metadata aren't present :param metadata: Required along with config_entity if layer_selection or db_entity aren't present :param kwargs: :return: """ feature_class_configuration = None if layer_selection: # Coming in relative to a LayerSelection, which puts us in the context of the LayerSelection's # feature query for this Feature subclass layer = layer_selection.layer # If we pass in a ConfigEntity it means we want to scope the Feature class to its scope. # The ConfigEntity defaults to that of the Layer, but we can override it to be a lower # scope to make sure that we have access to lower DbEntities of performing joins config_entity = config_entity.subclassed if config_entity else layer.config_entity.subclassed logger.debug( "Resolving FeatureResource subclass for layer_selection: {0}, config_entity: {1}" .format(layer_selection.unique_id, config_entity.id)) # Resolve the dynamic Feature class with the given config_entity so that we can access all DbEntities # of the ConfigEntity for joins feature_class = config_entity.db_entity_feature_class( layer.db_entity.key) elif db_entity: # Coming in relative to a DbEntity, meaning we don't care about a particular LayerSelection's # feature query for this Feature subclass config_entity = db_entity.config_entity logger.debug( "Resolving FeatureResource subclass for db_entity: {0}, config_entity: {1}" .format(db_entity.id, config_entity.id)) # Resolve the dynamic Feature class with the given config_entity so that we can access all DbEntities # of the ConfigEntity for joins feature_class = config_entity.db_entity_feature_class( db_entity.key) elif metadata: # Coming in with metadata, meaning this is and uploaded or ArcGis table with no DbEntity yet # We need to construct a FeatureClass from the metadata logger.debug( "Resolving FeatureResource subclass for metadata: {0}, config_entity: {1}" .format(metadata, config_entity.id)) feature_class_creator = FeatureClassCreator(config_entity) feature_class_configuration = feature_class_creator.feature_class_configuration_from_metadata( metadata['schema']) feature_class = FeatureClassCreator( config_entity, feature_class_configuration).dynamic_model_class() if not feature_class_configuration: # If we didn't already ensure all dynamic model classes have been created # This only need to run once to get all dynamic feature subclasses into memory, # in case they are needed by an association, join, or something similar feature_class_creator = FeatureClassCreator.from_dynamic_model_class( feature_class) feature_class_creator.ensure_dynamic_models() logger.debug("Resolving resource for Feature subclass: {0}".format( feature_class)) # Resolve the FeatureResource subclass based on the given Feature subclass # If self is already a subclass, just return self # Else, return a preconfigured subclass or one dynamically created. The latter will probably be the only way in the future. # If not already subclassed is_singleton_feature = issubclass(self.__class__, SingletonFeatureResourceMixin) is_template_feature = self.__class__ == TemplateFeatureResource if self.__class__ in [ FeatureResource, TemplateFeatureResource, FeatureCategoryAttributeResource, FeatureQuantitativeAttributeResource ]: if is_singleton_feature or params.get('is_feature_attribute'): queryset = feature_class.objects.none() elif kwargs.get('method', None) == 'PATCH': # It's possible to PATCH with an active join query. # But we don't want to use a join query when patching queryset = feature_class.objects.all() else: # Get the queryset stored by the layer_selection or an empty query if we don't have a layer_selection queryset = layer_selection.selected_features_or_values if\ layer_selection else \ feature_class.objects.none() if layer_selection and not (is_singleton_feature or kwargs.get( 'query_may_be_empty')) and queryset.count() == 0: raise Exception( "Unexpected empty queryset for layer_selection features: %s" % queryset.query) is_values_queryset = isinstance(queryset, ValuesQuerySet) #returns queryset ordered by the table id queryset = queryset.order_by('id') if is_values_queryset: join_feature_class = layer_selection.create_join_feature_class( ) if is_values_queryset else feature_class logger.info("Created join_feature_class: %s" % join_feature_class) # Force the queryset to our new class so that Tastypie can map the dict results to it queryset.model = join_feature_class return self.__class__.resolve_resource_class( join_feature_class, queryset=queryset, base_resource_class=self.join_feature_resource_class( join_feature_class), additional_fields_dict=dict( # Pass these to the feature resource to help it resolve # field mappings and add related fields (just need for join_feature_class) # Use the layer_selection if it exists since it might have filtered or extra query fields result_field_lookup=(layer_selection or db_entity).result_field_lookup if not metadata else {}, related_field_lookup=( layer_selection or db_entity).related_field_lookup if not metadata else {}, # We use these in the FeatureResource to create a unique id for each join Feature join_model_attributes=layer_selection and layer_selection.result_map.join_model_attributes), is_join_query=True, limit_fields=layer_selection.result_map['result_fields']) else: abstract_feature_resource_class = self.__class__ resource_class = abstract_feature_resource_class.resolve_resource_class( feature_class, queryset=queryset, # Give FeatureResource a reference to the layer_selection additional_fields_dict=merge( dict( # Pass this to the feature resource to help it resolve field mappings result_field_lookup=(layer_selection or db_entity). result_field_lookup if not metadata else {}), dict( # Not sure why it doesn't work to just stick this on the TemplateFeatureResource feature_fields=ListField( attribute='feature_fields', null=True, blank=True, readonly=True), feature_field_title_lookup=PickledDictField( attribute='feature_field_title_lookup', null=True, blank=True, readonly=True), ) if is_template_feature else dict()), for_template=is_template_feature) return resource_class return self