def read_stc(self, config_file_name): """ Reads provided config file :returns QDomDocument, QDomDocument.documentElement() :rtype tuple """ config_file_path = os.path.join( self.file_handler.localPath(), config_file_name ) config_file_path = QFile(config_file_path) config_file = os.path.basename(config_file_name) if self.check_config_file_exists(config_file): self.document = QDomDocument() status, msg, line, col = self.document.setContent(config_file_path) if not status: error_message = 'Configuration file cannot be loaded: {0}'. \ format(msg) self.append_log(str(error_message)) raise ConfigurationException(error_message) self.doc_element = self.document.documentElement()
def start_date(self, start_date_range): """ Set the minimum and maximum validity start dates. :param start_date_range: A tuple containing the minimum and maximum dates respectively. This will only be applied in the database if the columns have not yet been created. .. versionadded:: 1.5 :type start_date_range: tuple(min_start_date, max_start_date) """ if len(start_date_range) < 2: raise ConfigurationException( 'A tuple of minimum and maximum start dates expected.') min_date, max_date = start_date_range[0], start_date_range[1] if min_date > max_date: raise ConfigurationException( 'Minimum start date is greater than maximum start date.') self.validity_start_column.minimum = min_date self.validity_start_column.maximum = max_date
def featToDb(self, targettable, columnmatch, append, parentdialog, geomColumn=None, geomCode=-1, translator_manager=None): """ Performs the data import from the source layer to the STDM database. :param targettable: Destination table name :param columnmatch: Dictionary containing source columns as keys and target columns as the values. :param append: True to append, false to overwrite by deleting previous records :param parentdialog: A reference to the calling dialog. :param translator_manager: Instance of 'stdm.data.importexport.ValueTranslatorManager' containing value translators defined for the destination table columns. :type translator_manager: ValueTranslatorManager """ # Check current profile if self._current_profile is None: msg = QApplication.translate( 'OGRReader', 'The current profile could not be determined.\nPlease set it ' 'in the Options dialog or Configuration Wizard.') raise ConfigurationException(msg) if translator_manager is None: translator_manager = ValueTranslatorManager() # Delete existing rows in the target table if user has chosen to overwrite if not append: delete_table_data(targettable) # Container for mapping column names to their corresponding values lyr = self.getLayer() lyr.ResetReading() feat_defn = lyr.GetLayerDefn() numFeat = lyr.GetFeatureCount() # Configure progress dialog init_val = 0 progress = QProgressDialog("", "&Cancel", init_val, numFeat, parentdialog) progress.setWindowModality(Qt.WindowModal) lblMsgTemp = "Importing {0} of {1} to STDM..." # Set entity for use in translators destination_entity = self._data_source_entity(targettable) for feat in lyr: column_value_mapping = {} column_count = 0 progress.setValue(init_val) progressMsg = lblMsgTemp.format((init_val + 1), numFeat) progress.setLabelText(progressMsg) if progress.wasCanceled(): break # Reset source document manager for new records if destination_entity.supports_documents: if not self._source_doc_manager is None: self._source_doc_manager.reset() for f in range(feat_defn.GetFieldCount()): field_defn = feat_defn.GetFieldDefn(f) field_name = field_defn.GetNameRef() # Append value only if it has been defined by the user if field_name in columnmatch: dest_column = columnmatch[field_name] field_value = feat.GetField(f) # Create mapped class only once if self._mapped_cls is None: mapped_cls, mapped_doc_cls = self._get_mapped_class( targettable) if mapped_cls is None: msg = QApplication.translate( "OGRReader", "Something happened that caused the " "database table not to be mapped to the " "corresponding model class. Please contact" " your system administrator.") raise RuntimeError(msg) self._mapped_cls = mapped_cls self._mapped_doc_cls = mapped_doc_cls # Create source document manager if the entity supports them if destination_entity.supports_documents: self._source_doc_manager = SourceDocumentManager( destination_entity.supporting_doc, self._mapped_doc_cls) if geomColumn is not None: # Use geometry column SRID in the target table self._geomType, self._targetGeomColSRID = \ geometryType(targettable, geomColumn) ''' Check if there is a value translator defined for the specified destination column. ''' value_translator = translator_manager.translator( dest_column) if value_translator is not None: # Set destination table entity value_translator.entity = destination_entity source_col_names = value_translator.source_column_names( ) field_value_mappings = self._map_column_values( feat, feat_defn, source_col_names) # Set source document manager if required if value_translator.requires_source_document_manager: value_translator.source_document_manager = self._source_doc_manager field_value = value_translator.referencing_column_value( field_value_mappings) if not isinstance(field_value, IgnoreType): # Check column type and rename if multiple select for # SQLAlchemy compatibility col_obj = destination_entity.column(dest_column) if col_obj.TYPE_INFO == 'MULTIPLE_SELECT': lk_name = col_obj.value_list.name dest_column = '{0}_collection'.format(lk_name) column_value_mapping[dest_column] = field_value # Set supporting documents if destination_entity.supports_documents: column_value_mapping['documents'] = \ self._source_doc_manager.model_objects() column_count += 1 # Only insert geometry if it has been defined by the user if geomColumn is not None: geom = feat.GetGeometryRef() if geom is not None: # Check if the geometry types match layerGeomType = geom.GetGeometryName() # Convert polygon to multipolygon if the destination table is multi-polygon. geom_wkb, geom_type = self.auto_fix_geom_type( geom, layerGeomType, self._geomType) column_value_mapping[geomColumn] = "SRID={0!s};{1}".format( self._targetGeomColSRID, geom_wkb) if geom_type.lower() != self._geomType.lower(): raise TypeError( "The geometries of the source and destination columns do not match.\n" \ "Source Geometry Type: {0}, Destination Geometry Type: {1}".format( geom_type, self._geomType)) try: # Insert the record self._insertRow(targettable, column_value_mapping) except: progress.close() raise init_val += 1 progress.setValue(numFeat)
def _entity_select_column(entity, use_inner_join=False, join_parents=False, is_primary=False, foreign_key_parents=None, omit_view_columns=None, omit_join_statement_columns=None): # Check if the entity exists in the database if not pg_table_exists(entity.name): msg = u'{0} table does not exist, social tenure view will not be ' \ u'created.'.format(entity.name) LOGGER.debug(msg) raise ConfigurationException(msg) if omit_view_columns is None: omit_view_columns = [] if omit_join_statement_columns is None: omit_join_statement_columns = [] column_names = [] join_statements = [] columns = entity.columns.values() # Create foreign key parent collection if none is specified if foreign_key_parents is None: foreign_key_parents = {} str_entity = entity.profile.social_tenure for c in columns: if c.TYPE_INFO not in _exclude_view_column_types: normalized_entity_sname = entity.short_name.replace(' ', '_').lower() pseudo_column_name = u'{0}_{1}'.format(normalized_entity_sname, c.name) col_select_name = u'{0}.{1}'.format(entity.name, c.name) select_column_name = u'{0} AS {1}'.format(col_select_name, pseudo_column_name) if is_primary and c.name == 'id': # add row number id instead of party.id # if multi_party is allowed. if str_entity.multi_party: if not entity.has_geometry_column(): row_id = 'row_number() OVER () AS id' column_names.append(row_id) select_column_name = select_column_name else: # add spatial unit id as the id. select_column_name = col_select_name # add the social_tenure_relationship_id str_id = u'{0}.id AS {1}_id'.format( str_entity.name, str_entity.short_name.lower()) column_names.append(str_id) else: # add party_id on spatial unit view to use # [party]_supporting_document for # profiles with one party entity and no multi_party. if len(str_entity.parties) == 1 and not str_entity.multi_party and \ entity.has_geometry_column(): party_id = '{}_id'.format( str_entity.parties[0].short_name.lower().replace( ' ', '_')) str_party_id = u'{0}.{1} AS {1}'.format( str_entity.name, party_id) column_names.append(str_party_id) select_column_name = col_select_name # if entity has a geometry column, even if not multi_party # add social_tenure_relationship_id if entity.has_geometry_column(): # add the social_tenure_relationship_id str_id = u'{0}.id AS {1}_id'.format( str_entity.name, str_entity.short_name.lower()) column_names.append(str_id) # Use custom join flag use_custom_join = False if isinstance(c, ForeignKeyColumn) and join_parents: LOGGER.debug('Creating STR: Getting parent for %s column', c.name) fk_parent_entity = c.entity_relation.parent parent_table = c.entity_relation.parent.name LOGGER.debug('Parent found') select_column_name = '' # Handle renaming of parent table names to appropriate # pseudonames. if not parent_table in foreign_key_parents: foreign_key_parents[parent_table] = [] pseudo_names = foreign_key_parents.get(parent_table) # Get pseudoname to use table_pseudo_name = u'{0}_{1}'.format(parent_table, (len(pseudo_names) + 1)) pseudo_names.append(table_pseudo_name) # Map lookup and admin unit values by default if c.TYPE_INFO == 'LOOKUP': select_column_name = u'{0}.value AS {1}'.format( table_pseudo_name, pseudo_column_name) use_custom_join = True # Check if the column is for tenure type if c.name != 'tenure_type': use_inner_join = False elif c.TYPE_INFO == 'ADMIN_SPATIAL_UNIT': select_column_name = u'{0}.name AS {1}'.format( table_pseudo_name, pseudo_column_name) use_custom_join = True use_inner_join = False # These are outer joins join_type = 'LEFT JOIN' # Use inner join only if parent entity is an STR entity if use_inner_join and \ str_entity.is_str_entity(fk_parent_entity): join_type = 'INNER JOIN' if use_custom_join: join_statement = u'{0} {1} {2} ON {3} = {2}.{4}'.format( join_type, parent_table, table_pseudo_name, col_select_name, c.entity_relation.parent_column) else: join_statement = u'{0} {1} ON {2} = {1}.{3}'.format( join_type, parent_table, col_select_name, c.entity_relation.parent_column) # Assert if the column is in the list of omitted join columns if c.name not in omit_join_statement_columns: join_statements.append(join_statement) # Assert if the column is in the list of omitted view columns if c.name not in omit_view_columns: if select_column_name: column_names.append(select_column_name) return column_names, join_statements
def _entity_select_column(entity, use_inner_join=False, join_parents=False, is_primary=False, foreign_key_parents=None, omit_view_columns=None, omit_join_statement_columns=None, view_name=None): # Check if the entity exists in the database if not pg_table_exists(entity.name): msg = '{0} table does not exist, social tenure view will not be ' \ 'created.'.format(entity.name) LOGGER.debug(msg) raise ConfigurationException(msg) if omit_view_columns is None: omit_view_columns = [] if omit_join_statement_columns is None: omit_join_statement_columns = [] column_names = [] join_statements = [] columns = list(entity.columns.values()) # Create foreign key parent collection if none is specified if foreign_key_parents is None: foreign_key_parents = {} str_entity = entity.profile.social_tenure if entity in str_entity.custom_attribute_entities.values(): custom_tenure = True else: custom_tenure = False i = 0 parents = [] for c in columns: if c.TYPE_INFO not in _exclude_view_column_types: normalized_entity_sname = entity.short_name.replace(' ', '_').lower() pseudo_column_name = '{0}_{1}'.format(normalized_entity_sname, c.name) # use sudo name for custom tenure entity if custom_tenure: col_select_name = '{0}_1.{1}'.format(entity.name, c.name) else: col_select_name = '{0}.{1}'.format(entity.name, c.name) # Get pseudoname to use select_column_name = '{0} AS {1}'.format(col_select_name, pseudo_column_name) if is_primary and c.name == 'id': # add row number id instead of party.id # if multi_party is allowed. if str_entity.multi_party: # for party entity add use row number if not entity.has_geometry_column(): row_id = 'row_number() OVER () AS id' select_column_name = row_id else: # add spatial unit id as the id. col_spatial_unit_id = '{0}.{1} AS {1}'.format( entity.name, c.name) select_column_name = col_spatial_unit_id else: # add party id or spatial unit as id entity_id = '{0}.{1} AS {1}'.format(entity.name, c.name) select_column_name = entity_id # Use custom join flag use_custom_join = False if isinstance(c, ForeignKeyColumn) and join_parents: LOGGER.debug('Creating STR: Getting parent for %s column', c.name) fk_parent_entity = c.entity_relation.parent parent_table = c.entity_relation.parent.name LOGGER.debug('Parent found') # Handle renaming of parent table names to appropriate # pseudonames. if parent_table not in foreign_key_parents: foreign_key_parents[parent_table] = [] pseudo_names = foreign_key_parents.get(parent_table) # Get pseudoname to use table_pseudo_name = '{0}_{1}'.format(parent_table, (len(pseudo_names) + 1)) pseudo_names.append(table_pseudo_name) # Map lookup and admin unit values by default if c.TYPE_INFO == 'LOOKUP': lookup_model = entity_model(c.entity_relation.parent) lookup_model_obj = lookup_model() result = lookup_model_obj.queryObject().filter( lookup_model.code != '').filter( lookup_model.code is not None).all() if len(result) == 0: select_column_name = '{0}.value AS {1}'.format( table_pseudo_name, pseudo_column_name) else: value = '{0}.value'.format(table_pseudo_name) code = '{0}.code'.format(table_pseudo_name) select_column_name = "concat({0}, ' (', {1}, ')') AS {2}". \ format(value, code, pseudo_column_name) use_custom_join = True # Check if the column is for tenure type if c.name != 'tenure_type': use_inner_join = False elif c.TYPE_INFO == 'ADMIN_SPATIAL_UNIT': select_column_name = '{0}.name AS {1}'.format( table_pseudo_name, pseudo_column_name) use_custom_join = True use_inner_join = False elif c.TYPE_INFO == 'FOREIGN_KEY': if c.entity_relation.parent not in str_entity.parties and \ c.entity_relation.parent not in str_entity.spatial_units: if len(c.entity_relation.display_cols) > 0: display_col_names = [] for display_col in c.entity_relation.display_cols: name = '{0}.{1}'.format( table_pseudo_name, display_col) display_col_names.append(name) select_column_name = "concat_ws(' '::text, {0}) AS {1}".format( ', '.join(display_col_names), pseudo_column_name) else: if not custom_tenure: select_column_name = '{0}.id AS {1}'.format( table_pseudo_name, pseudo_column_name) use_custom_join = True use_inner_join = False else: QApplication.processEvents() # These are outer joins join_type = 'LEFT JOIN' # Use inner join only if parent entity is an STR entity and it is the current entity. # Other str entities should use left join. if str_entity.is_str_entity(fk_parent_entity) and \ fk_parent_entity.name in view_name: join_type = 'INNER JOIN' if use_custom_join: # exclude replace str name with custom tenure name in join. if custom_tenure: if c.name == 'social_tenure_relationship_id': i = i + 1 # pseudo_names = foreign_key_parents.get(parent_table) col_select_name = '{0}_{1}.{2}'.format( entity.name, str(i), c.name) # Get pseudoname to use table_pseudo_name = '{0}_{1}'.format( entity.name, str(i)) join_statement = '{0} {1} {2} ON {3} = {2}.{4}'.format( join_type, entity.name, table_pseudo_name, col_select_name, c.entity_relation.parent_column) join_statements = [join_statement ] + join_statements else: join_statement = '{0} {1} {2} ON {3} = {2}.{4}'.format( join_type, parent_table, table_pseudo_name, col_select_name, c.entity_relation.parent_column) join_statements.append(join_statement) else: join_statement = '{0} {1} {2} ON {3} = {2}.{4}'.format( join_type, parent_table, table_pseudo_name, col_select_name, c.entity_relation.parent_column) join_statements.append(join_statement) else: # Assert if the column is in the list of omitted join columns # # if c.name in omit_join_statement_columns: # if 'INNER JOIN' in join_statement: # join_statements.append(join_statement) # else: # This fix needs to be tested more ... if parent_table in foreign_key_parents and 'relationship' not in entity.name: parent_table_alias = foreign_key_parents[parent_table][ 0] join_statement = '{0} {1} {2} ON {3} = {2}.{4}'.format( join_type, parent_table, parent_table_alias, col_select_name, c.entity_relation.parent_column) else: join_statement = '{0} {1} ON {2} = {1}.{3}'.format( join_type, parent_table, col_select_name, c.entity_relation.parent_column) parents.append(parent_table) join_statements.append(join_statement) # Assert if the column is in the list of omitted view columns if c.name not in omit_view_columns: if select_column_name: column_names.append(select_column_name) QApplication.processEvents() return column_names, join_statements
def _entity_select_column(entity, use_inner_join=False, join_parents=False, is_primary=False, foreign_key_parents=None): #Check if the entity exists in the database if not pg_table_exists(entity.name): msg = u'{0} table does not exist, social tenure view will not be ' \ u'created.'.format(entity.name) LOGGER.debug(msg) raise ConfigurationException(msg) column_names = [] join_statements = [] columns = entity.columns.values() #Create foreign key parent collection if none is specified if foreign_key_parents is None: foreign_key_parents = {} for c in columns: if not c.TYPE_INFO in _exclude_view_column_types: normalized_entity_sname = entity.short_name.replace(' ', '_').lower() pseudo_column_name = u'{0}_{1}'.format(normalized_entity_sname, c.name) col_select_name = u'{0}.{1}'.format(entity.name, c.name) select_column_name = u'{0} AS {1}'.format(col_select_name, pseudo_column_name) if is_primary and c.name == 'id': select_column_name = col_select_name #Use custom join flag use_custom_join = False if isinstance(c, ForeignKeyColumn) and join_parents: LOGGER.debug('Creating STR: Getting parent for %s column', c.name) parent_table = c.entity_relation.parent.name LOGGER.debug('Parent found') select_column_name = '' #Handle renaming of parent table names to appropriate # pseudonames. if not parent_table in foreign_key_parents: foreign_key_parents[parent_table] = [] pseudo_names = foreign_key_parents.get(parent_table) #Get pseudoname to use table_pseudo_name = u'{0}_{1}'.format(parent_table, (len(pseudo_names) + 1)) pseudo_names.append(table_pseudo_name) #Map lookup and admin unit values by default if c.TYPE_INFO == 'LOOKUP': select_column_name = u'{0}.value AS {1}'.format( table_pseudo_name, pseudo_column_name) use_custom_join = True #Check if the column is for tenure type if c.name != 'tenure_type': use_inner_join = False elif c.TYPE_INFO == 'ADMIN_SPATIAL_UNIT': select_column_name = u'{0}.name AS {1}'.format( table_pseudo_name, pseudo_column_name) use_custom_join = True use_inner_join = False #These are outer joins join_type = 'LEFT JOIN' if use_inner_join: join_type = 'INNER JOIN' if use_custom_join: join_statement = u'{0} {1} {2} ON {3} = {2}.{4}'.format( join_type, parent_table, table_pseudo_name, col_select_name, c.entity_relation.parent_column) else: join_statement = u'{0} {1} ON {2} = {1}.{3}'.format( join_type, parent_table, col_select_name, c.entity_relation.parent_column) join_statements.append(join_statement) if select_column_name: column_names.append(select_column_name) return column_names, join_statements