def importer(self, config_entity, db_entity, **kwargs):
        """
            Replaces the normal ImportProcessor importer with one to import a shapefile from disk
        """
        user = db_entity.creator

        if InformationSchema.objects.table_exists(db_entity.schema, db_entity.table):
            # The table already exists. Skip the import an log a warning
            logger.warn("The target table for the layer selection import already exists. Skipping table import.")
        else:
            feature_class_creator = FeatureClassCreator(config_entity, db_entity)
            origin_feature_class_configuration = db_entity.origin_instance.feature_class_configuration
            # Create the new DbEntity FeatureClassConfiguration from the origin's. Pass in what has already been
            # created for the new feature_class_configuration. This should have things like generated=True
            feature_class_configuration = feature_class_creator.complete_or_create_feature_class_configuration(
                origin_feature_class_configuration,
                **merge(db_entity.feature_class_configuration.__dict__, dict(generated=True)))
            # Update the DbEntity
            feature_class_creator.update_db_entity(feature_class_configuration)

            if feature_class_configuration.source_from_origin_layer_selection and \
               feature_class_configuration.origin_layer_id:
                # If desired, limit the layer clone to that of the source layer's current LayerSelection for the
                # User doing the update
                layer_selection_class = get_or_create_layer_selection_class_for_layer(
                    Layer.objects.get(id=feature_class_configuration.origin_layer_id), True)
                layer_selection = layer_selection_class.objects.get(user=user)
                features = layer_selection.selected_features
            else:
                # Leave blank to copy all features by default
                features = None

            DefaultImportProcessor().peer_importer(config_entity, db_entity, import_from_origin=True, source_queryset=features)
Example #2
0
def db_entity_defaults(db_entity, config_entity=None):

    # Instantiate a FeatureClassCreator to make the FeatureClassConfiguration
    feature_class_creator = FeatureClassCreator(config_entity, db_entity, no_ensure=True)
    if config_entity:
        # Find the database of the configured client
        connection = resolve_fixture(None, "init", InitFixture, config_entity.schema()).import_database()
        # Remove the _test suffix when running unit tests. Until we can auto-population the _test version
        # of the database, we want to simply rely on the manually configured source database
        if connection:
            connection['database'] = connection['database'].replace('test_', '')
    else:
        # No config_entity abstract DbEntity case
        connection = None

    return dict(
        # The name is passed in or the titleized version of key
        name=db_entity.name or titleize(db_entity.key),
        # Postgres URL for local sources, or possibly a remote url (e.g. for background layer sources)
        # Unless overridden, create the url according to this postgres url scheme
        url=db_entity.url or \
            ('postgres://{user}:{password}/{host}:{port}/{database}'.format(
                **merge(dict(port=5432), connection)) if connection else None),
        # Normally Equals the key, except for views of the table, like a Result DbEntity
        # Views leave this null and rely on query
        table=db_entity.table or (db_entity.key if not db_entity.query else None),
        # Query to create a "view" of the underlying data. Used by Result DbEntity instances
        query=db_entity.query,
        # How to group the features or query results. Not yet well hashed out
        group_by=db_entity.group_by,
        # The source DbEntity key if this DbEntity resulted from cloning a peer DbEntity
        source_db_entity_key=db_entity.source_db_entity_key,
        # Array used by remote data sources whose URLs have different host names
        # If so then the url will have a string variable for the host
        hosts=db_entity.hosts,
        # The User who created the DbEntity. TODO. Default should be an admin
        creator=db_entity.creator if hasattr(db_entity, 'creator') else get_user_model().objects.filter()[0],
        # The User who updated the DbEntity. TODO. Default should be an admin
        updater=db_entity.creator if hasattr(db_entity, 'creator') else get_user_model().objects.filter()[0],

        # The SRID of the Feature table
        srid=db_entity.srid,
        # This is a non-model object. So it is saved as a PickledObjectField
        # Whether the same instance is returned or not does not matter
        # If db_entity.feature_class_configuration is None, it will return None
        feature_class_configuration=feature_class_creator.complete_or_create_feature_class_configuration(
            db_entity.feature_class_configuration
        ),
        no_feature_class_configuration=db_entity.no_feature_class_configuration
        # feature_behavior is handled internally by DbEntity
    )
 def create_result_map(self, values_query_set):
     related_models = self.resolve_join_models()
     logger.debug("Creating result map for related models %s feature class %s" % (', '.join(map(lambda r: str(r), related_models)), self.feature_class))
     feature_class_creator = FeatureClassCreator.from_dynamic_model_class(self.feature_class)
     geography_scopes = feature_class_creator.geography_scopes()
     # Get the related model paths final segment. We want to map these to the db_entity_key names
     related_model_path_to_name = map_to_dict(
         lambda related_model:
         [resolve_related_model_path_via_geographies(
             self.feature_class.objects,
             related_model).split('__')[1],
          related_model.db_entity_key],
         related_models
     )
     return values_query_set.create_result_map(
         related_models=related_models,
         # map_path_segments maps related object paths to their model name,
         # and removes the geographies segment of the path
         map_path_segments=merge(
             # Map each geography scope to its corresponding field on the feature class
             map_to_dict(
                 lambda geography_scope: [
                     feature_class_creator.geographies_field(geography_scope).name,
                     None
                 ],
                 geography_scopes),
             related_model_path_to_name)
     )
Example #4
0
def annotated_related_feature_class_pk_via_geographies(manager, config_entity, db_entity_keys):
    """
        To join a related model by geographic join
    """
    from footprint.main.models.feature.feature_class_creator import FeatureClassCreator

    feature_class_creator = FeatureClassCreator.from_dynamic_model_class(manager.model)

    def resolve_related_model_pk(db_entity_key):
        related_model = config_entity.db_entity_feature_class(db_entity_key)
        # The common Geography class
        geography_class = feature_class_creator.common_geography_class(related_model)
        geography_scope = feature_class_creator.common_geography_scope(related_model)
        logger.warn("Resolved geography scope %s", geography_scope)
        # Find the geographies ManyToMany fields that relates this model to the related_model
        # via a Geography class. Which geography class depends on their common geography scope
        geographies_field = feature_class_creator.geographies_field(geography_scope)
        try:
            # Find the queryable field name from the geography class to the related model
            related_model_geographies_field_name = resolve_queryable_name_of_type(geography_class, related_model)
        except:
            # Sometimes the geography class hasn't had its fields cached properly. Fix here
            clear_many_cache(geography_class)
            related_model_geographies_field_name = resolve_queryable_name_of_type(geography_class, related_model)

        return "%s__%s__pk" % (geographies_field.name, related_model_geographies_field_name)

    pk_paths = map_to_dict(
        lambda db_entity_key: [db_entity_key, Min(resolve_related_model_pk(db_entity_key))], db_entity_keys
    )

    return manager.annotate(**pk_paths)
Example #5
0
def resolve_field_path_via_geographies(field_path, manager, related_models):
    """
        Resolve the given field path in case its not absolute.
        For instance, if it is 'block' and one of our related models accessible via geographies__relatedmodel has that property,
        return 'geographies_[scope_id]__relatedmodel__block'
        It will also be tested against the main manager after all related models fail,
        e.g. manager.values(field_path) if successful would simply return field_path
    :param field_path: django field path. e.g. du or built_form__name
    :param manager: The main manager by which the related models are resolved and by which the full path is computed
    :param related_models: models joined to the manager. For instance. manager.model is CanvasFeature, a related_model could be
        CensusBlock, which might be related to the former via 'geographies_[scope_id]__censusblock9rel'. The relationship is computed
        by assuming that the related model is related by geographies and looking for a field matching its type
    :return:
    """
    from footprint.main.models.feature.feature_class_creator import FeatureClassCreator

    feature_class_creator = FeatureClassCreator.from_dynamic_model_class(manager.model)
    for related_model in related_models:
        try:
            # See if the field_name resolves
            # There's probably a more efficient way to do this
            related_model.objects.values(field_path)
            resolved_field_path = field_path
        except:
            # See if the first segment matches the related_model db_entity_key
            first_segment = field_path.split("__")[0]
            if first_segment != related_model.db_entity_key:
                # If not, move on
                continue
            # Take all but the first segment
            resolved_field_path = "__".join(field_path.split("__")[1:])
        # Success, find the path to this model from geographies
        geography_class = feature_class_creator.common_geography_class(related_model)
        geographies_field = feature_class_creator.geographies_field(
            feature_class_creator.common_geography_scope(related_model)
        )
        geography_related_field_name = resolve_queryable_name_of_type(geography_class, related_model)
        return "%s__%s__%s" % (geographies_field.name, geography_related_field_name, resolved_field_path)
    # See if it matches the main model
    try:
        if field_path.split("__")[0] == manager.model.db_entity_key:
            # If the manager model db_entity_key was used in the path, just strip it out
            updated_field_path = "__".join(field_path.split("__")[1:])
            manager.values(updated_field_path)
        else:
            # Otherwise test query with the full path
            updated_field_path = field_path
            manager.values(updated_field_path)
        # Success, return the field_path
        return updated_field_path
    except:
        logger.exception(
            "Cannot resolve field path %s to the main model %s or any joined models %s",
            field_path,
            manager.model,
            related_models,
        )
        raise
    def importer(self, config_entity, db_entity, **kwargs):
        """
            Creates various GeojsonFeature classes by importing geojson and saving it to the database via a dynamic subclass of GeojsonFeature
        :schema: The optional schema to use for the dynamic subclass's meta db_table attribute, which will allow the class's table to be saved in the specified schema. Defaults to public
        :data: Optional python dict data to use instead of loading from the db_entity.url
        :return: a list of lists. Each list is a list of features of distinct subclass of GeoJsonFeature that is created dynamically. To persist these features, you must first create the subclass's table in the database using create_table_for_dynamic_class(). You should also register the table as a DbEntity.
        """
        if self.seed_data:
            data = geojson.loads(jsonify(self.seed_data), object_hook=geojson.GeoJSON.to_instance)
        else:
            fp = open(db_entity.url.replace('file://', ''))
            data = geojson.load(fp, object_hook=geojson.GeoJSON.to_instance)
        feature_class_creator = FeatureClassCreator(config_entity, db_entity)
        # find all unique properties
        feature_class_configuration = feature_class_creator.feature_class_configuration_from_geojson_introspection(data)
        feature_class_creator.update_db_entity(feature_class_configuration)
        feature_class = feature_class_creator.dynamic_model_class(base_only=True)
        # Create our base table. Normally this is done by the import, but we're just importing into memory
        create_tables_for_dynamic_classes(feature_class)
        # Now write each feature to our newly created table
        for feature in map(lambda feature: self.instantiate_sub_class(feature_class, feature), data.features):
            feature.save()
        # Create the rel table too
        rel_feature_class = feature_class_creator.dynamic_model_class()
        create_tables_for_dynamic_classes(rel_feature_class)

        # PostGIS 2 handles this for us now
        # if InformationSchema.objects.table_exists(db_entity.schema, db_entity.table):
        #     # Tell PostGIS about the new geometry column or the table
        #     sync_geometry_columns(db_entity.schema, db_entity.table)

        # Create association classes and tables and populate them with data
        create_and_populate_relations(config_entity, db_entity)
Example #7
0
def resolve_related_model_path_via_geographies(manager, related_model):
    """
        Returns the query string path 'geographies_[scope_id]__[field name of the related model form the main model]'
        The scope_id is the id of the ConfigEntity that both models share in common by ascending the ConfigEntity
        hierarchy starting at each models' geography_scope
    """
    from footprint.main.models.feature.feature_class_creator import FeatureClassCreator

    feature_class_creator = FeatureClassCreator.from_dynamic_model_class(manager.model)
    geography_scope = feature_class_creator.common_geography_scope(related_model)
    geographies_field = feature_class_creator.geographies_field(geography_scope)
    geography_class = feature_class_creator.common_geography_class(related_model)
    geography_related_field_name = resolve_queryable_name_of_type(geography_class, related_model)
    return "%s__%s" % (geographies_field.name, geography_related_field_name)
Example #8
0
 def result_map(cls):
     """
         Creates an caches a result map for the Feature class. The result_map has useful meta data about
         the class
     :param cls:
     :return:
     """
     if cls._result_map:
         return cls._result_map
     from footprint.main.models.feature.feature_class_creator import FeatureClassCreator
     feature_class_creator = FeatureClassCreator.from_dynamic_model_class(cls)
     if not feature_class_creator.dynamic_model_class_is_ready:
         return None
     cls._result_map = feature_class_creator.dynamic_model_class().objects.all().create_result_map()
     return cls._result_map
    def dynamic_resource_subclass(self, layer_selection=None, db_entity=None, feature_class=None, config_entity=None, metadata=None, params=None, **kwargs):
        """
            Creates the dynamic Feature Resource class by passing in a layer_selection, db_entity, or feature_class
        :param layer_selection: Required if db_entity or metadata aren't present
        :param db_entity: Required if layer_selection or metadata aren't present
        :param metadata: Required along with config_entity if layer_selection or db_entity aren't present
        :param kwargs:
        :return:
        """
        feature_class_configuration = None
        if layer_selection:
            # Coming in relative to a LayerSelection, which puts us in the context of the LayerSelection's
            # feature query for this Feature subclass
            layer = layer_selection.layer
            # If we pass in a ConfigEntity it means we want to scope the Feature class to its scope.
            # The ConfigEntity defaults to that of the Layer, but we can override it to be a lower
            # scope to make sure that we have access to lower DbEntities of performing joins
            config_entity = config_entity.subclassed if config_entity else layer.config_entity.subclassed
            logger.debug("Resolving FeatureResource subclass for layer_selection: {0}, config_entity: {1}".format(layer_selection.unique_id, config_entity.id))
            # Resolve the dynamic Feature class with the given config_entity so that we can access all DbEntities
            # of the ConfigEntity for joins
            feature_class = config_entity.db_entity_feature_class(layer.db_entity.key)
        elif db_entity:
            # Coming in relative to a DbEntity, meaning we don't care about a particular LayerSelection's
            # feature query for this Feature subclass
            config_entity = db_entity.config_entity
            logger.debug("Resolving FeatureResource subclass for db_entity: {0}, config_entity: {1}".format(db_entity.id, config_entity.id))
            # Resolve the dynamic Feature class with the given config_entity so that we can access all DbEntities
            # of the ConfigEntity for joins
            feature_class = config_entity.db_entity_feature_class(db_entity.key)
        elif metadata:
            # Coming in with metadata, meaning this is and uploaded or ArcGis table with no DbEntity yet
            # We need to construct a FeatureClass from the metadata
            logger.debug("Resolving FeatureResource subclass for metadata: {0}, config_entity: {1}".format(metadata, config_entity.id))
            feature_class_creator = FeatureClassCreator(
                config_entity
            )
            feature_class_configuration = feature_class_creator.feature_class_configuration_from_metadata(metadata['schema'])
            feature_class = FeatureClassCreator(
                config_entity,
                feature_class_configuration
            ).dynamic_model_class()

        if not feature_class_configuration:
            # If we didn't already ensure all dynamic model classes have been created
            # This only need to run once to get all dynamic feature subclasses into memory,
            # in case they are needed by an association, join, or something similar
            feature_class_creator = FeatureClassCreator.from_dynamic_model_class(feature_class)
            feature_class_creator.ensure_dynamic_models()

        logger.debug("Resolving resource for Feature subclass: {0}".format(feature_class))

        # Resolve the FeatureResource subclass based on the given Feature subclass
        # If self is already a subclass, just return self
        # Else, return a preconfigured subclass or one dynamically created. The latter will probably be the only way in the future.
        # If not already subclassed
        is_singleton_feature = issubclass(self.__class__, SingletonFeatureResourceMixin)
        is_template_feature = self.__class__ == TemplateFeatureResource
        if self.__class__ in [FeatureResource, TemplateFeatureResource, FeatureCategoryAttributeResource,
                              FeatureQuantitativeAttributeResource]:
            if is_singleton_feature or params.get('is_feature_attribute'):
                queryset = feature_class.objects.none()
            elif kwargs.get('method', None) == 'PATCH':
                # It's possible to PATCH with an active join query.
                # But we don't want to use a join query when patching
                queryset = feature_class.objects.all()
            else:
                # Get the queryset stored by the layer_selection or an empty query if we don't have a layer_selection
                queryset = layer_selection.selected_features_or_values if\
                    layer_selection else \
                    feature_class.objects.none()

                if layer_selection and not (is_singleton_feature or kwargs.get('query_may_be_empty')) and queryset.count()==0:
                    raise Exception(
                        "Unexpected empty queryset for layer_selection features: %s" %
                        queryset.query)
            is_values_queryset = isinstance(queryset, ValuesQuerySet)

            #returns queryset ordered by the table id
            queryset = queryset.order_by('id')

            if is_values_queryset:
                join_feature_class = layer_selection.create_join_feature_class() if is_values_queryset else feature_class
                logger.info("Created join_feature_class: %s" % join_feature_class)
                # Force the queryset to our new class so that Tastypie can map the dict results to it
                queryset.model = join_feature_class

                return self.__class__.resolve_resource_class(
                    join_feature_class,
                    queryset=queryset,
                    base_resource_class=self.join_feature_resource_class(join_feature_class),
                    additional_fields_dict=dict(
                        # Pass these to the feature resource to help it resolve
                        # field mappings and add related fields (just need for join_feature_class)
                        # Use the layer_selection if it exists since it might have filtered or extra query fields
                        result_field_lookup=(layer_selection or db_entity).result_field_lookup if not metadata else {},
                        related_field_lookup=(layer_selection or db_entity).related_field_lookup if not metadata else {},
                        # We use these in the FeatureResource to create a unique id for each join Feature
                        join_model_attributes=layer_selection and layer_selection.result_map.join_model_attributes
                    ),
                    is_join_query=True,
                    limit_fields=layer_selection.result_map['result_fields']
                )
            else:
                abstract_feature_resource_class = self.__class__
                resource_class = abstract_feature_resource_class.resolve_resource_class(
                    feature_class,
                    queryset=queryset,
                    # Give FeatureResource a reference to the layer_selection
                    additional_fields_dict=merge(
                        dict(
                            # Pass this to the feature resource to help it resolve field mappings
                            result_field_lookup=(layer_selection or db_entity).result_field_lookup if not metadata else {}
                        ),
                        dict(
                            # Not sure why it doesn't work to just stick this on the TemplateFeatureResource
                            feature_fields=ListField(attribute='feature_fields', null=True, blank=True, readonly=True),
                            feature_field_title_lookup=PickledDictField(attribute='feature_field_title_lookup', null=True, blank=True, readonly=True),
                        ) if is_template_feature else dict()
                    ),
                    for_template=is_template_feature
                )
                return resource_class
        return self
Example #10
0
    def importer(self, config_entity, db_entity, **kwargs):
        """
            Replaces the normal ImportProcessor importer with one to import a sql from disk
        """
        if InformationSchema.objects.table_exists(db_entity.schema, db_entity.table):
            # The table already exists. Skip the import an log a warning
            logger.warn("The target table for the feature table import already exists. Skipping table import.")
        else:
            # We don't store the upload_id alone, so pull it off the url
            upload_id = db_entity.url.replace('file:///tmp/', '').replace('.sql.zip', '')
            # Unpack the zipfile and return the path the sql file was placed at
            if db_entity.url.startswith('file://'):
                file_path = db_entity.url[len('file://'):]

            logger.warn(file_path)
            path = unpack_zipfile(file_path, upload_id)
            # The file is always the name of the table defined therein
            table_name = path.split('/')[-1].split('.')[0].lower()
            db_entity.url = 'file://%s' % path
            # Update the db_entity.url from the zip file url to the file_path
            # This lets ImportData find it.
            logger.info("Url of DbEntity is %s" % db_entity.url)
            db_entity.save()

            # Perform some sed updates to get the sql file ready for import
            regex_substitutions = []
            sql_file_path = file_url_to_path(db_entity.url)

            # Add IF EXISTS to the drop table to prevent an error if IF EXISTS doesn't exist yet
            regex_substitutions.append((r'DROP TABLE (?!IF EXISTS)', r'DROP TABLE IF EXISTS'))

            # TODO temp, fix an AC bug. It seems that using a capitalized column is problematic (?)
            # The suggested solution is to double quote it, but quotes cause other problems, so we simply lowercase
            regex_substitutions.append((r' OGC_FID ', ' ogc_fid ', (4, 4)))  # only line 4
            regex_substitutions.append((r'PRIMARY KEY \(ogc_fid\)', 'PRIMARY KEY (ogc_fid)', (4, 4)))  # only line 4
            # TODO end temp fix

            # Update the index name to include the schema. This format matches that created for preconfigured feature
            # tables (see import_data.py)
            spatial_index_name = '{schema}_{key}_geom_idx'.format(schema=db_entity.schema, key=db_entity.key)
            regex_substitutions.append((r'CREATE INDEX ".*" ON', 'CREATE INDEX "%s" ON' % spatial_index_name, (6, 6)))  # only line 6 6

            # Remove the reference to the geometry_columns, since we use a materialized view
            regex_substitutions.append((r'^DELETE FROM geometry_columns', '--DELETE FROM geometry_columns', (2, 2)))

            # Update the sql to have a unique table name which matches the DbEntity key
            # Also change public to our import schema to keep it from causing trouble in the public schema
            # Otherwise we run into all kinds of trouble trying to get the SQL into the system
            regex_substitutions.append((r'"public"."%s"' % table_name, '"import"."%s"' % db_entity.key))

            regex_substitutions.append((r"'%s'" % table_name, "'%s'" % db_entity.key, (2, 5)))

            regex_substitutions.append((r'"%s_pk"' % table_name, '"%s_pk"' % db_entity.key, (4, 4)))

            # Update public to the import schema
            regex_substitutions.append((r"AddGeometryColumn\('public'", "AddGeometryColumn('%s'" % settings.IMPORT_SCHEMA, (5, 5)))

            regex_substitutions.append((r'"%s_wkb_geometry_geom_idx"' % table_name, '"%s_wkb_geometry_geom_idx"' % db_entity.key, (6, 6)))

            for command in regex_substitutions:
                logger.info("Applying the following substitution %s" % ', '.join(command[0:2]))
            apply_regexes_to_file(sql_file_path, regex_substitutions)

            ImportData(config_entity=config_entity, db_entity_key=db_entity.key).run()

        # Add our normal primary key in the id column if negit eded
        add_primary_key_if_needed(db_entity)

        feature_class_creator = FeatureClassCreator(config_entity, db_entity)
        # Inspect the imported table to create the feature_class_configuration
        feature_class_configuration = feature_class_creator.feature_class_configuration_from_introspection()

        # Merge the created feature_class_configuration with the on already defined for the db_entity
        feature_class_creator.update_db_entity(feature_class_configuration)
        logger.info("Finished import for DbEntity: %s, feature_class_configuration: %s" % (db_entity, db_entity.feature_class_configuration))

        # Create association classes and tables and populate them with data
        create_and_populate_relations(config_entity, feature_class_creator.db_entity)