class Save2DB: """ Class to insert entity data into db """ def __init__(self, entity, attributes, ids=None): """ Initialize class and class variable """ self.attributes = attributes self.form_entity = entity self.doc_model = None self._doc_manager = None self.entity = self.object_from_entity_name(self.form_entity) self.model = self.dbmodel_from_entity() self.key = 0 self.parents_ids = ids self.geom = 4326 self.entity_mapping = {} def object_from_entity_name(self, entity): """ :return: """ if entity == 'social_tenure': return current_profile().social_tenure else: user_entity = current_profile().entity_by_name(entity) return user_entity def entity_has_supporting_docs(self): """ Check if the entity has supporting document before importing :return: Bool """ if self.entity.supports_documents: return self.entity.supports_documents else: return None def entity_supported_document_types(self): """ Get the supported document types before importing so that they are captured during import process :return: List """ return self.entity.document_types_non_hex() def dbmodel_from_entity(self): """ Format model attributes from passed entity attributes :return: """ if self.entity_has_supporting_docs(): entity_object, self.doc_model = entity_model( self.entity, with_supporting_document=True) entity_object_model = entity_object() if hasattr(entity_object_model, 'documents'): if self.entity.TYPE_INFO == 'SOCIAL_TENURE': obj_doc_col = current_profile( ).social_tenure.supporting_doc else: obj_doc_col = self.entity.supporting_doc self._doc_manager = SourceDocumentManager( obj_doc_col, self.doc_model) else: entity_object = entity_model(self.entity) entity_object_model = entity_object() return entity_object_model def objects_from_supporting_doc(self, instance_file=None): """ Create supporting doc path instances based on the collected documents :return:paths :rtype: document object instance """ if instance_file: f_dir, file_name = os.path.split(instance_file) for document, val in self.attributes.iteritems(): if str(document).endswith('supporting_document'): if val != '': doc = self.format_document_name_from_attribute( document) doc_path = os.path.normpath(f_dir + '/' + val) abs_path = doc_path.replace('\\', '/').strip() if QFile.exists(abs_path): self.supporting_document_model(abs_path, doc) def supporting_document_model(self, doc_path, doc): """ :param doc_path: absolute document path :param doc: document name :type: str Construct supporting document model instance to add into the db :return: """ # Create document container doc_container = QVBoxLayout() supporting_doc_entity = self.entity.supporting_doc.document_type_entity document_type_id = entity_attr_to_id(supporting_doc_entity, 'value', doc, lower=False) # Register container self._doc_manager.registerContainer(doc_container, document_type_id) #Copy the document to STDM working directory self._doc_manager.insertDocumentFromFile(doc_path, document_type_id, self.entity) def format_document_name_from_attribute(self, doc): """ Get the type of document from attribute name So that supporting document class instance can save it in the right format :return: """ formatted_doc_list = self.entity_supported_document_types() default = 'General' doc_type = str(doc).split('_', 1) if doc_type[0].startswith( 'supporting') and formatted_doc_list[0] == default: return default elif doc_type[0].startswith( 'supporting') and formatted_doc_list[0] != default: return formatted_doc_list[0] elif not doc_type[0].startswith('supporting'): actual_doc_name = doc_type[0].replace('-', ' ') for doc_name in formatted_doc_list: if actual_doc_name in doc_name or doc_name.startswith( actual_doc_name): return doc_name else: return formatted_doc_list[0] else: return formatted_doc_list[0] def extract_social_tenure_entities(self): ''' We want to extract social tenure enities so that we know if it has multiple or single entities in the list :return: ''' party_ref_column = '' spatial_ref_column = '' if self.parents_ids is not None: print self.parents_ids if self.attributes.has_key('party'): full_party_ref_column = self.attributes.get('party') party_ref_column = full_party_ref_column + '_id' print 'party{}.'.format(party_ref_column) setattr(self.model, party_ref_column, self.parents_ids.get(full_party_ref_column)[0]) if self.attributes.has_key('spatial_unit'): full_spatial_ref_column = self.attributes.get('spatial_unit') spatial_ref_column = full_spatial_ref_column + '_id' print 'sp.{}.'.format(spatial_ref_column) setattr(self.model, spatial_ref_column, self.parents_ids.get(full_spatial_ref_column)[0]) return party_ref_column, spatial_ref_column def save_to_db(self): """ Format object attribute data from entity and save them into database :return: """ self.column_info() attributes = self.attributes try: if self.entity.short_name == 'social_tenure_relationship': #try: prefix = current_profile().prefix + '_' if self.attributes.has_key('party'): full_party_ref_column = self.attributes.get('party') party_ref_column = full_party_ref_column + '_id' self.attributes.pop('party') else: full_party_ref_column = current_profile( ).social_tenure.parties[0].name party_ref_column = full_party_ref_column.replace( prefix, '') + '_id' setattr(self.model, party_ref_column, self.parents_ids.get(full_party_ref_column)[0]) if self.attributes.has_key('spatial_unit'): full_spatial_ref_column = self.attributes.get( 'spatial_unit') spatial_ref_column = full_spatial_ref_column + '_id' self.attributes.pop('spatial_unit') else: full_spatial_ref_column = current_profile( ).social_tenure.spatial_units[0].name spatial_ref_column = full_spatial_ref_column.replace( prefix, '') + '_id' setattr(self.model, spatial_ref_column, self.parents_ids.get(full_spatial_ref_column)[0]) attributes = self.attributes['social_tenure'] except: pass for k, v in attributes.iteritems(): if hasattr(self.model, k): col_type = self.entity_mapping.get(k) col_prop = self.entity.columns[k] var = self.attribute_formatter(col_type, col_prop, v) setattr(self.model, k, var) if self.entity_has_supporting_docs(): self.model.documents = self._doc_manager.model_objects() self.model.save() return self.model.id def save_parent_to_db(self): """ Format object attribute data from entity and save them into database attribute :return: """ self.column_info() for k, v in self.attributes.iteritems(): if hasattr(self.model, k): col_type = self.entity_mapping.get(k) col_prop = self.entity.columns[k] #print "property{0}.... and type.{1}".format(col_prop, col_type) var = self.attribute_formatter(col_type, col_prop, v) setattr(self.model, k, var) if self.entity_has_supporting_docs(): self.model.documents = self._doc_manager.model_objects() self.model.save() self.key = self.model.id return self.key def save_foreign_key_table(self): """ Get the table with foreign keys only :return: """ for col, type_info in self.column_info().iteritems(): col_prop = self.entity.columns[col] var = self.attribute_formatter(type_info, col_prop, None) setattr(self.model, col, var) self.model.save() self.cleanup() def column_info(self): """ :return: """ self.entity_mapping = {} cols = self.entity.columns.values() for c in cols: self.entity_mapping[c.name] = c.TYPE_INFO def get_srid(self, srid): """ Let the user specify the coordinate system during data import :param srid: :return: """ self.geom = srid return self.geom def id_from_model_object(self, obj): """ We need to obtian id from object instance :param obj: :return: """ return obj.id def attribute_formatter(self, col_type, col_prop, var=None): """ Format geoodk attributes collected in the field to conform to STDM database contrains :return: """ if col_type == 'BOOL': if len(var) < 1: return None if len(var) > 1: if var == '' or var is None: return None if var == 'Yes' or var == True: return True if var == 'No' or var == False: return False else: return None if col_type == 'LOOKUP': if len(var) < 1 or var is None: return None if len(var) < 4: if var == 'Yes' or var == 'No': return entity_attr_to_model(col_prop.parent, 'value', var).id if var != 'Yes' and var != 'No': lk_code = entity_attr_to_id(col_prop.parent, "code", var) if not str(lk_code).isdigit(): return None else: return lk_code if len(var) > 3: if not str(entity_attr_to_id(col_prop.parent, 'code', var)).isdigit(): id_value = entity_attr_to_model(col_prop.parent, 'value', var) if id_value is not None: return id_value.id #return entity_attr_to_model(col_prop.parent, 'value', var).id else: lk_code = entity_attr_to_id(col_prop.parent, "code", var) if not str(lk_code).isdigit(): return None else: return lk_code else: return None elif col_type == 'ADMIN_SPATIAL_UNIT': var_code = None try: if len(var) < 1 or var is None: return None elif not len(var) > 3: var_code = entity_attr_to_id(col_prop.parent, "code", var) if var_code and var_code == var: return None else: return var_code elif len(var) > 3 and entity_attr_to_id( col_prop.parent, "name", var) is not None: var_code = entity_attr_to_id(col_prop.parent, "name", var) if var_code and var_code == var: return None else: return var_code else: if entity_attr_to_id(col_prop.parent, "name", var) is None: var_code = entity_attr_to_id(col_prop.parent, "code", var) if not var_code or var_code == var: return None except: pass elif col_type == 'MULTIPLE_SELECT': print 'multiple select {}'.format(var) if var == '' or var is None: return None else: col_parent = col_prop.association.first_parent lk_val_list = col_parent.values.values() choices_list = [] for code in lk_val_list: choices_list.append( entity_attr_to_id(col_parent.association.first_parent, 'value', code.value)) if len(choices_list) > 1: return choices_list else: return None elif col_type == 'GEOMETRY': defualt_srid = 0 if var: geom_provider = STDMGeometry(var) if isinstance(col_prop, GeometryColumn): defualt_srid = col_prop.srid if defualt_srid != 0: geom_provider.set_user_srid(defualt_srid) else: geom_provider.set_user_srid(GEOMPARAM) if col_prop.geometry_type() == 'POINT': return geom_provider.point_to_Wkt() if col_prop.geometry_type() == 'POLYGON': return geom_provider.polygon_to_Wkt() else: return None elif col_type == 'FOREIGN_KEY': ret_val = None for code, val in self.parents_ids.iteritems(): if col_prop.parent.name == code: ret_val = val[0] break return ret_val elif col_type == 'INT' or col_type == 'DOUBLE' or col_type == 'PERCENT': ret_val = None if var <> '': ret_val = var return ret_val elif col_type == 'DATETIME' or col_type == 'DATE': ret_val = None if var <> '': ret_val = var return ret_val else: return var def cleanup(self): """ Reset all the model and entity data before we process the next entity data :return: None z""" self.model = None self.entity = None self.attributes = None self._doc_manager = None
class OGRReader: def __init__(self, source_file): self._ds = ogr.Open(source_file) self._targetGeomColSRID = -1 self._geomType = '' self._dbSession = STDMDb.instance().session self._mapped_cls = None self._mapped_doc_cls = None self._current_profile = current_profile() self._source_doc_manager = None def getLayer(self): # Return the first layer in the data source if self.isValid(): numLayers = self._ds.GetLayerCount() if numLayers > 0: return self._ds.GetLayer(0) else: return None def getSpatialRefCode(self): # Get the EPSG code (More work required) if self.getLayer() != None: spRef = self.getLayer().GetSpatialRef() refCode = spRef.GetAttrValue("PRIMEM|AUTHORITY", 1) else: # Fallback to WGS84 refCode = 4326 return refCode def isValid(self): # Whether the open process succeeded or failed if self._ds is None: return False else: return True def reset(self): # Destroy self._ds = None self._geomType = "" self._targetGeomColSRID = -1 def getFields(self): # Return the data source's fields in a list fields = [] lyr = self.getLayer() lyr.ResetReading() feat_defn = lyr.GetLayerDefn() for l in range(feat_defn.GetFieldCount()): field_defn = feat_defn.GetFieldDefn(l) fields.append(str(field_defn.GetNameRef())) return fields def _data_source_entity(self, table_name): entity = self._current_profile.entity_by_name(table_name) return entity def entity_virtual_columns(self, table_name): """ :param table_name: Name of the target table. :type table_name: str :return: Returns a list of derived columns for the specified target table. :rtype: list """ entity = self._data_source_entity(table_name) if entity is None: return [] return entity.virtual_columns() def _get_mapped_class(self, table_name): # Get entity from the corresponding table name entity = self._data_source_entity(table_name) if entity is None: return None ent_model, doc_model = entity_model(entity, with_supporting_document=True) return ent_model, doc_model def auto_fix_percent(self, target_table, col_name, value): """ Fixes percent columns if empty and with a wrong format. :param target_table: The destination table name :type target_table: String :param col_name: The destination column name :type col_name: String :param value: Value to be saved to the DB :type value: Any :return: Converted value :rtype: Any """ entity = self._data_source_entity(target_table) if entity.columns[col_name].TYPE_INFO == 'PERCENT': if isinstance(value, str): if not bool(value.strip()) or value.strip().lower() == 'null': value = None if '%' in value: value = value.replace('%', '') try: if value is not None: value = float(value) except ValueError: value = None return value def auto_fix_float_integer(self, target_table, col_name, value): """ Fixes float and integer columns if empty and with a wrong format. :param target_table: The destination table name :type target_table: String :param col_name: The destination column name :type col_name: String :param value: Value to be saved to the DB :type value: Any :return: Converted value :rtype: Any """ entity = self._data_source_entity(target_table) integer_types = [ 'INT', 'LOOKUP', 'ADMIN_SPATIAL_UNIT', 'FOREIGN_KEY', 'DOUBLE' ] float_type = ['DOUBLE'] int_type = ['INT', 'LOOKUP', 'ADMIN_SPATIAL_UNIT', 'FOREIGN_KEY'] if col_name in entity.columns.keys(): if entity.columns[col_name].TYPE_INFO in integer_types: if isinstance(value, str): if not bool( value.strip()) or value.strip().lower() == 'null': value = None if entity.columns[col_name].TYPE_INFO in float_type: try: if value is not None: value = float(value) except ValueError: value = None elif entity.columns[col_name].TYPE_INFO in int_type: try: if value is not None: value = int(value) if isinstance(value, int): if value == 0: if entity.columns[col_name].TYPE_INFO in \ ['LOOKUP', 'ADMIN_SPATIAL_UNIT', 'FOREIGN_KEY']: value = None except ValueError: # TODO show warning to the user that # some values cannot be converted to integer. value = None return value def auto_fix_date(self, target_table, col_name, value): """ Fixes date and datetime columns if empty and with a wrong format. :param target_table: The destination table name :type target_table: String :param col_name: The destination column name :type col_name: String :param value: Value to be saved to the DB :type value: Any :return: Converted value :rtype: Any """ entity = self._data_source_entity(target_table) date_types = ['DATE', 'DATETIME'] if entity.columns[col_name].TYPE_INFO in date_types: if not bool(value) or value.lower() == 'null': value = None return value def auto_fix_yes_no(self, target_table, col_name, value): """ Fixes Yes_NO columns if empty and with a wrong format. :param target_table: The destination table name :type target_table: String :param col_name: The destination column name :type col_name: String :param value: Value to be saved to the DB :type value: Any :return: Converted value :rtype: Any """ entity = self._data_source_entity(target_table) yes_no_types = ['BOOL'] if entity.columns[col_name].TYPE_INFO in yes_no_types: if not bool(value.strip()) or value.strip().lower() == 'null': value = None elif value.strip().lower() == 'yes': value = True elif value.strip().lower() == 'no': value = False elif value.strip().lower() == 'true': value = True elif value.strip().lower() == 'false': value = False return value def _insertRow(self, target_table, columnValueMapping): """ Insert a new row using the mapped class instance then mapping column names to the corresponding column values. """ model_instance = self._mapped_cls() for col, value in columnValueMapping.items(): if hasattr(model_instance, col): # 'documents' is not a column so exclude it. if col != 'documents' and not 'collection' in col: value = self.auto_fix_float_integer( target_table, col, value) value = self.auto_fix_percent(target_table, col, value) value = self.auto_fix_date(target_table, col, value) value = self.auto_fix_yes_no(target_table, col, value) if not isinstance(value, IgnoreType): setattr(model_instance, col, value) try: self._dbSession.add(model_instance) self._dbSession.commit() except: self._dbSession.rollback() raise def auto_fix_geom_type(self, geom, source_geom_type, destination_geom_type): """ Converts single geometry type to multi type if the destination is multi type. :param geom: The OGR geometry :type geom: OGRGeometry :param source_geom_type: Source geometry type :type source_geom_type: String :param destination_geom_type: Destination geometry type :type destination_geom_type: String :return: WkB geometry and the output layer geometry type. :rtype: Tuple """ # Convert polygon to multipolygon if the destination table is multi-polygon. if source_geom_type.lower() == 'polygon' and \ destination_geom_type.lower() == 'multipolygon': geom_wkb, geom_type = self.to_ogr_multi_type( geom, ogr.wkbMultiPolygon) elif source_geom_type.lower() == 'linestring' and \ destination_geom_type.lower() == 'multilinestring': geom_wkb, geom_type = self.to_ogr_multi_type( geom, ogr.wkbMultiLineString) elif source_geom_type.lower() == 'point' and \ destination_geom_type.lower() == 'multipoint': geom_wkb, geom_type = self.to_ogr_multi_type( geom, ogr.wkbMultiPoint) else: geom_wkb = geom.ExportToWkt() geom_type = geom.GetGeometryName() return geom_wkb, geom_type def to_ogr_multi_type(self, geom, ogr_type): """ Convert ogr geometry to multi-type when supplied the ogr.type. :param geom: The ogr geometry :type geom: OGRGeometry :param ogr_type: The ogr geometry type :type ogr_type: String :return: WkB geometry and the output layer geometry type. :rtype: Tuple """ multi_geom = ogr.Geometry(ogr_type) multi_geom.AddGeometry(geom) geom_wkb = multi_geom.ExportToWkt() geom_type = multi_geom.GetGeometryName() return geom_wkb, geom_type def featToDb(self, targettable, columnmatch, append, parentdialog, geomColumn=None, geomCode=-1, translator_manager=None): """ Performs the data import from the source layer to the STDM database. :param targettable: Destination table name :param columnmatch: Dictionary containing source columns as keys and target columns as the values. :param append: True to append, false to overwrite by deleting previous records :param parentdialog: A reference to the calling dialog. :param translator_manager: Instance of 'stdm.data.importexport.ValueTranslatorManager' containing value translators defined for the destination table columns. :type translator_manager: ValueTranslatorManager """ # Check current profile if self._current_profile is None: msg = QApplication.translate( 'OGRReader', 'The current profile could not be determined.\nPlease set it ' 'in the Options dialog or Configuration Wizard.') raise ConfigurationException(msg) if translator_manager is None: translator_manager = ValueTranslatorManager() # Delete existing rows in the target table if user has chosen to overwrite if not append: delete_table_data(targettable) # Container for mapping column names to their corresponding values lyr = self.getLayer() lyr.ResetReading() feat_defn = lyr.GetLayerDefn() numFeat = lyr.GetFeatureCount() # Configure progress dialog init_val = 0 progress = QProgressDialog("", "&Cancel", init_val, numFeat, parentdialog) progress.setWindowModality(Qt.WindowModal) lblMsgTemp = "Importing {0} of {1} to STDM..." # Set entity for use in translators destination_entity = self._data_source_entity(targettable) for feat in lyr: column_value_mapping = {} column_count = 0 progress.setValue(init_val) progressMsg = lblMsgTemp.format((init_val + 1), numFeat) progress.setLabelText(progressMsg) if progress.wasCanceled(): break # Reset source document manager for new records if destination_entity.supports_documents: if not self._source_doc_manager is None: self._source_doc_manager.reset() for f in range(feat_defn.GetFieldCount()): field_defn = feat_defn.GetFieldDefn(f) field_name = field_defn.GetNameRef() # Append value only if it has been defined by the user if field_name in columnmatch: dest_column = columnmatch[field_name] field_value = feat.GetField(f) # Create mapped class only once if self._mapped_cls is None: mapped_cls, mapped_doc_cls = self._get_mapped_class( targettable) if mapped_cls is None: msg = QApplication.translate( "OGRReader", "Something happened that caused the " "database table not to be mapped to the " "corresponding model class. Please contact" " your system administrator.") raise RuntimeError(msg) self._mapped_cls = mapped_cls self._mapped_doc_cls = mapped_doc_cls # Create source document manager if the entity supports them if destination_entity.supports_documents: self._source_doc_manager = SourceDocumentManager( destination_entity.supporting_doc, self._mapped_doc_cls) if geomColumn is not None: # Use geometry column SRID in the target table self._geomType, self._targetGeomColSRID = \ geometryType(targettable, geomColumn) ''' Check if there is a value translator defined for the specified destination column. ''' value_translator = translator_manager.translator( dest_column) if value_translator is not None: # Set destination table entity value_translator.entity = destination_entity source_col_names = value_translator.source_column_names( ) field_value_mappings = self._map_column_values( feat, feat_defn, source_col_names) # Set source document manager if required if value_translator.requires_source_document_manager: value_translator.source_document_manager = self._source_doc_manager field_value = value_translator.referencing_column_value( field_value_mappings) if not isinstance(field_value, IgnoreType): # Check column type and rename if multiple select for # SQLAlchemy compatibility col_obj = destination_entity.column(dest_column) if col_obj.TYPE_INFO == 'MULTIPLE_SELECT': lk_name = col_obj.value_list.name dest_column = '{0}_collection'.format(lk_name) column_value_mapping[dest_column] = field_value # Set supporting documents if destination_entity.supports_documents: column_value_mapping['documents'] = \ self._source_doc_manager.model_objects() column_count += 1 # Only insert geometry if it has been defined by the user if geomColumn is not None: geom = feat.GetGeometryRef() if geom is not None: # Check if the geometry types match layerGeomType = geom.GetGeometryName() # Convert polygon to multipolygon if the destination table is multi-polygon. geom_wkb, geom_type = self.auto_fix_geom_type( geom, layerGeomType, self._geomType) column_value_mapping[geomColumn] = "SRID={0!s};{1}".format( self._targetGeomColSRID, geom_wkb) if geom_type.lower() != self._geomType.lower(): raise TypeError( "The geometries of the source and destination columns do not match.\n" \ "Source Geometry Type: {0}, Destination Geometry Type: {1}".format( geom_type, self._geomType)) try: # Insert the record self._insertRow(targettable, column_value_mapping) except: progress.close() raise init_val += 1 progress.setValue(numFeat) def _enumeration_column_type(self, column_name, value): """ Checks if the given column is of DeclEnumType. :param column_name: Name of the enumeration column. :type column_name: str :return: True if column is of DeclType and return Enum symbol; Else, False and None. :rtype: tuple """ try: # Get column type of the enumeration enum_col_type = self._mapped_cls.__mapper__.columns[ column_name].type except KeyError: return False, None if not hasattr(enum_col_type, "enum"): return False, None else: enum_obj = enum_col_type.enum try: if not isinstance(value, str) or not isinstance(value, str): value = str(value) enum_symbol = enum_obj.from_string(value.strip()) except ValueError: enum_symbol = IgnoreType() return True, enum_symbol def _map_column_values(self, feature, feature_defn, source_cols): """ Retrieves values for specific columns from the specified feature. :param feature: Input feature. :type feature: ogr.Feature :param feature_defn: Feature definition for the layer. :type feature_defn: ogr.FeatureDefn :param source_cols: List of columns whose respective values will be retrieved. :type source_cols: list :return: Collection containing pairs of column names and corresponding values. :rtype: dict """ col_values = {} if len(source_cols) == 0: return col_values for f in range(feature_defn.GetFieldCount()): field_defn = feature_defn.GetFieldDefn(f) field_name = field_defn.GetNameRef() match_idx = getIndex(source_cols, field_name) if match_idx != -1: field_value = feature.GetField(f) col_values[field_name] = field_value return col_values
class SupportingDocuments(ComponentUtility): onUploadDocument = pyqtSignal(list) def __init__(self, box, combobox, add_documents_btn, notification_bar, parent=None): """ Handles the supporting documents component loading. :param box: The layout holding the container widget. :type box: QVBoxLayout :param combobox: The combobox loading supporting document types. :type combobox: QComboBox :param add_documents_btn: The add supporting document button :type add_documents_btn: QPushButton :param notification_bar: The NotificationBar object that displays notification. :type notification_bar: Object :param parent: The container of the widget :type parent: QDialog or None """ ComponentUtility.__init__(self) self._parent = parent self.container_box = box self.doc_type_cbo = combobox self.notification_bar = notification_bar self.str_number = 1 self.add_documents_btn = add_documents_btn self.str_numbers = [1] self.current_party_count = None self.init_documents() def init_documents(self): """ Initializes the document type combobox by populating data. """ self.supporting_doc_manager = SourceDocumentManager( self.social_tenure.supporting_doc, self.str_doc_model, self._parent) self.create_doc_tab_populate_combobox() self.doc_type_cbo.currentIndexChanged.connect( self.match_doc_combo_to_tab) self.docs_tab.currentChanged.connect(self.match_doc_tab_to_combo) def party_count(self, count): """ A setter for current_party_count that is used to determined the number of copies for each supporting document. :param count: The number of currently added party records. :type count: Integer """ self.current_party_count = count def create_doc_tab_populate_combobox(self): """ Creates the supporting document component widget. """ self.doc_tab_data() self.docs_tab = QTabWidget() self.docs_tab_index = OrderedDict() for i, (id, doc) in enumerate(self.doc_types.iteritems()): self.docs_tab_index[doc] = i # the tab widget containing the document widget layout # and the child of the tab. tab_widget = QWidget() tab_widget.setObjectName(doc) # The layout of the tab widget cont_layout = QVBoxLayout(tab_widget) cont_layout.setObjectName(u'widget_layout_{}'.format(doc)) # the scroll area widget inside the tab widget. scroll_area = QScrollArea(tab_widget) scroll_area.setFrameShape(QFrame.NoFrame) scroll_area.setObjectName(u'tab_scroll_area_{}'.format(doc)) layout_widget = QWidget() # the widget the is under the scroll area content and # the widget containing the document widget layout # This widget is hidden and shown based on the STR number layout_widget.setObjectName(u'widget_{}'.format(doc)) doc_widget_layout = QVBoxLayout(layout_widget) doc_widget_layout.setObjectName( u'doc_widget_layout_{}'.format(doc)) doc_widget = QWidget() doc_widget.setObjectName(u'doc_widget_{}_{}'.format( doc, self.str_number)) doc_widget_layout.addWidget(doc_widget) # the layout containing document widget. ### This is the layout that is registered to add uploaded # supporting documents widgets into. tab_layout = QVBoxLayout(doc_widget) tab_layout.setObjectName(u'layout_{}_{}'.format( doc, self.str_number)) scroll_area.setWidgetResizable(True) scroll_area.setWidget(layout_widget) cont_layout.addWidget(scroll_area) # Add the tab widget with the document # type name to create a tab. self.docs_tab.addTab(tab_widget, doc) self.container_box.addWidget(self.docs_tab, 1) if len(self.str_numbers) == 1: self.doc_type_cbo.addItem(doc, id) def doc_tab_data(self): """ Sets the document types in the social tenure entity. """ doc_entity = self.social_tenure. \ supporting_doc.document_type_entity doc_type_model = entity_model(doc_entity) docs = doc_type_model() doc_type_list = docs.queryObject().all() self.doc_types = [(doc.id, doc.value) for doc in doc_type_list] self.doc_types = OrderedDict(self.doc_types) def match_doc_combo_to_tab(self): """ Changes the active tab based on the selected value of document type combobox. """ combo_text = self.doc_type_cbo.currentText() if combo_text is not None and len(combo_text) > 0: index = self.docs_tab_index[combo_text] self.docs_tab.setCurrentIndex(index) def match_doc_tab_to_combo(self): """ Changes the document type combobox value based on the selected tab. """ doc_tab_index = self.docs_tab.currentIndex() self.doc_type_cbo.setCurrentIndex(doc_tab_index) @staticmethod def hide_doc_widgets(widget, visibility): """ Hides or shows the visibility of the supporting document container widgets. :param widget: The widget to which the visibility is set. :type widget: QWidget :param visibility: A boolean to show or hide visibility. True hides widget and False shows it. :type visibility: Boolean """ widget.setHidden(visibility) def update_container(self, str_number): """ Update the current supporting document widget container to be used. :param str_number: The STR node number :type str_number: Integer """ doc_text = self.doc_type_cbo.currentText() cbo_index = self.doc_type_cbo.currentIndex() doc_id = self.doc_type_cbo.itemData(cbo_index) scroll_area = self.docs_tab.findChild( QScrollArea, u'tab_scroll_area_{}'.format(doc_text, str_number)) doc_widget = scroll_area.findChild( QWidget, u'doc_widget_{}_{}'.format(doc_text, str_number)) # If the doc widget doesn't exist create it for new STR instance if doc_widget is None: # find the doc_widget layout that contains # all STR doc widget layouts. Single # doc_widget_layout is created for each document type. # But all doc_widgets for each STR instance and # document types will be added here. doc_widget_layout = scroll_area.findChild( QVBoxLayout, u'doc_widget_layout_{}'.format(doc_text)) doc_widget = QWidget() doc_widget.setObjectName(u'doc_widget_{}_{}'.format( doc_text, str_number)) self.hide_all_other_widget(doc_text, str_number) doc_widget_layout.addWidget(doc_widget) # Create the layout so that layouts are registered in # which uploaded document widgets are added. layout = QVBoxLayout(doc_widget) layout.setObjectName(u'layout_{}_{}'.format(doc_text, str_number)) # If the doc widget exists, get the lowest # layout so that it is registered. else: # hide all other widgets self.hide_all_other_widget(doc_text, str_number) # show the current doc widget to display # the document widgets for the current tab. self.hide_doc_widgets(doc_widget, False) layout = doc_widget.findChild( QVBoxLayout, u'layout_{}_{}'.format(doc_text, str_number)) # register layout self.supporting_doc_manager.registerContainer(layout, doc_id) def hide_all_other_widget(self, doc_text, str_number): """ Hides all other supporting document widget except the current STR node widget. :param doc_text: The current document type selected. :type doc_text: String :param str_number: The STR node number :type str_number: Integer """ expression = QRegExp(u'doc_widget*') # hide all existing widgets in all layouts for widget in self.docs_tab.findChildren(QWidget, expression): if widget.objectName() != u'doc_widget_{}_{}'.format( doc_text, str_number): self.hide_doc_widgets(widget, True) def on_upload_document(self): ''' Slot raised when the user clicks to upload a supporting document. ''' document_str = QApplication.translate( "SupportingDocuments", "Specify the Document File Location") documents = self.select_file_dialog(document_str) cbo_index = self.doc_type_cbo.currentIndex() doc_id = self.doc_type_cbo.itemData(cbo_index) for doc in documents: self.supporting_doc_manager.insertDocumentFromFile( doc, doc_id, self.social_tenure, self.current_party_count) # Set last path if len(documents) > 0: doc = documents[0] fi = QFileInfo(doc) dir_path = fi.absolutePath() set_last_document_path(dir_path) model_objs = self.supporting_doc_manager.model_objects() self.onUploadDocument.emit(model_objs) def select_file_dialog(self, title): """ Displays a file dialog for a user to specify a source document :param title: The title of the file dialog :type title: String """ # Get last path for supporting documents last_path = last_document_path() if last_path is None: last_path = '/home' files = QFileDialog.getOpenFileNames( iface.mainWindow(), title, last_path, "Source Documents (*.jpg *.jpeg *.png *.bmp *.tiff *.svg)") return files
class Save2DB: """ Class to insert entity data into db """ def __init__(self, entity, attributes, ids=None): """ Initialize class and class variable """ self.attributes = attributes self.form_entity = entity self.doc_model = None self._doc_manager = None self.entity = self.object_from_entity_name(self.form_entity) self.model = self.dbmodel_from_entity() self.key = 0 self.parents_ids = ids self.geom = 4326 def object_from_entity_name(self, entity): """ :return: """ if entity == 'social_tenure': return current_profile().social_tenure else: user_entity = current_profile().entity_by_name(entity) return user_entity def entity_has_supporting_docs(self): """ Check if the entity has supporting document before importing :return: Bool """ return self.entity.supports_documents def entity_supported_document_types(self): """ Get the supported document types before importing so that they are captured during import process :return: List """ return self.entity.document_types_non_hex() def dbmodel_from_entity(self): """ Format model attributes from passed entity attributes :return: """ if self.entity_has_supporting_docs(): entity_object, self.doc_model = entity_model( self.entity, with_supporting_document=True) entity_object_model = entity_object() if hasattr(entity_object_model, 'documents'): if self.entity.TYPE_INFO == 'SOCIAL_TENURE': obj_doc_col = current_profile( ).social_tenure.supporting_doc else: obj_doc_col = self.entity.supporting_doc self._doc_manager = SourceDocumentManager( obj_doc_col, self.doc_model) else: entity_object = entity_model(self.entity) entity_object_model = entity_object() return entity_object_model def objects_from_supporting_doc(self, instance_file=None): """ Create supporting doc path instances based on the collected documents :return:paths :rtype: document object instance """ if instance_file: f_dir, file_name = os.path.split(instance_file) for document, val in self.attributes.iteritems(): if str(document).endswith('supporting_document'): if val != '': doc = self.format_document_name_from_attribute( document) doc_path = os.path.normpath(f_dir + '/' + val) abs_path = doc_path.replace('\\', '/').strip() if QFile.exists(abs_path): self.supporting_document_model(abs_path, doc) def supporting_document_model(self, doc_path, doc): """ Construct supporting document model instance to add into the db :return: """ # Create document container doc_container = QVBoxLayout() supporting_doc_entity = self.entity.supporting_doc.document_type_entity document_type_id = entity_attr_to_id(supporting_doc_entity, 'value', doc, lower=False) # Register container self._doc_manager.registerContainer(doc_container, document_type_id) #Copy the document to STDM working directory self._doc_manager.insertDocumentFromFile(doc_path, document_type_id, self.entity) def format_document_name_from_attribute(self, key_name): """ Get the type of document from attribute name So that supporting document class instance can save it in the right format :return: """ doc_list = self.entity_supported_document_types() default = 'General' doc_type = str(key_name).split('_') if len(doc_type) > 2: if doc_type[0] in doc_list: return doc_type[0] else: for doc in doc_list: if doc.startswith(doc_type[0]): return doc elif len(doc_type) < 2 and key_name != default: return key_name else: return default def save_to_db(self): """ Format object attribute data from entity and save them into database :return: """ try: if self.parents_ids is not None and self.entity.short_name == 'social_tenure_relationship': str_tables = current_profile().social_tenure setattr(self.model, str_tables.parties[0].short_name.lower() + '_id', self.parents_ids.get(str_tables.parties[0].name)[0]) setattr( self.model, str_tables.spatial_units[0].short_name.lower() + '_id', self.parents_ids.get(str_tables.spatial_units[0].name)[0]) except: pass for k, v in self.attributes.iteritems(): if hasattr(self.model, k): col_type = self.column_info().get(k) col_prop = self.entity.columns[k] var = self.attribute_formatter(col_type, col_prop, v) setattr(self.model, k, var) if self.entity_has_supporting_docs(): self.model.documents = self._doc_manager.model_objects() self.model.save() return self.model.id #self.cleanup() def save_parent_to_db(self): """ Format object attribute data from entity and save them into database attribute :return: """ for k, v in self.attributes.iteritems(): if hasattr(self.model, k): col_type = self.column_info().get(k) col_prop = self.entity.columns[k] var = self.attribute_formatter(col_type, col_prop, v) setattr(self.model, k, var) if self.entity_has_supporting_docs(): self.model.documents = self._doc_manager.model_objects() self.model.save() self.key = self.model.id return self.key #def model_object_formatter(self): def column_info(self): """ :return: """ type_mapping = {} cols = self.entity.columns.values() for c in cols: type_mapping[c.name] = c.TYPE_INFO return type_mapping def get_srid(self, srid): """ Let the user specify the coordinate system during data import :param srid: :return: """ self.geom = srid return self.geom def id_from_model_object(self, obj): """ We need to obtian id from object instance :param obj: :return: """ return obj.id def attribute_formatter(self, col_type, col_prop, var): """ :return: """ if col_type == 'LOOKUP': if var == '' or var is None: return None if var == 'Yes' or var == 'No': return entity_attr_to_model(col_prop.parent, 'value', var).id if not len(var) > 3 and var != 'Yes' and var != 'No': lk_code = entity_attr_to_id(col_prop.parent, "code", var) if not str(lk_code).isdigit(): return None else: return lk_code if len(var) > 3: if not str(entity_attr_to_id(col_prop.parent, 'code', var)).isdigit(): return entity_attr_to_model(col_prop.parent, 'value', var).id else: lk_code = entity_attr_to_id(col_prop.parent, "code", var) if not str(lk_code).isdigit(): return None else: return lk_code else: return None elif col_type == 'ADMIN_SPATIAL_UNIT': if not len(var) > 3: return entity_attr_to_id(col_prop.parent, "code", var) else: return entity_attr_to_id(col_prop.parent, "name", var) elif col_type == 'MULTIPLE_SELECT': if var == '' or var is None: return None if not len(var) > 3: return entity_attr_to_id(col_prop.association.first_parent, "code", var) elif len(var) > 3: if not str( entity_attr_to_id(col_prop.association.first_parent, "code", var)).isdigit(): return entity_attr_to_model( col_prop.association.first_parent, 'value', var).id else: return entity_attr_to_id(col_prop.association.first_parent, "code", var) elif col_type == 'GEOMETRY': defualt_srid = 0 geom_provider = STDMGeometry(var) if isinstance(col_prop, GeometryColumn): defualt_srid = col_prop.srid if defualt_srid != 0: geom_provider.set_user_srid(defualt_srid) else: geom_provider.set_user_srid(GEOMPARAM) if col_prop.geometry_type() == 'POINT': return geom_provider.point_to_Wkt() if col_prop.geometry_type() == 'POLYGON': return geom_provider.polygon_to_Wkt() elif col_type == 'FOREIGN_KEY': if self.parents_ids is None or len(self.parents_ids) < 0: return else: for code, val in self.parents_ids.iteritems(): if code is not None: if val[1] == GROUPCODE and col_prop.parent.name == code: return val[0] else: if col_prop.parent.name == code: return val[0] elif col_type == 'INT' or col_type == 'DOUBLE': if var == '': return 0 else: return var else: return var def cleanup(self): """ Reset all the model and entity data before we process the next entity data :return: None z""" self.model = None self.entity = None self.attributes = None self._doc_manager = None
class SupportingDocuments(ComponentUtility): onUploadDocument = pyqtSignal(list) def __init__(self, box, combobox, add_documents_btn, notification_bar, parent=None): """ Handles the supporting documents component loading. :param box: The layout holding the container widget. :type box: QVBoxLayout :param combobox: The combobox loading supporting document types. :type combobox: QComboBox :param add_documents_btn: The add supporting document button :type add_documents_btn: QPushButton :param notification_bar: The NotificationBar object that displays notification. :type notification_bar: Object :param parent: The container of the widget :type parent: QDialog or None """ ComponentUtility.__init__(self) self._parent = parent self.container_box = box self.doc_type_cbo = combobox self.notification_bar = notification_bar self.str_number = 1 self.add_documents_btn = add_documents_btn self.str_numbers = [1] self.current_party_count = None self.init_documents() def init_documents(self): """ Initializes the document type combobox by populating data. """ self.supporting_doc_manager = SourceDocumentManager( self.social_tenure.supporting_doc, self.str_doc_model, self._parent ) self.create_doc_tab_populate_combobox() self.doc_type_cbo.currentIndexChanged.connect( self.match_doc_combo_to_tab ) self.docs_tab.currentChanged.connect( self.match_doc_tab_to_combo ) def party_count(self, count): """ A setter for current_party_count that is used to determined the number of copies for each supporting document. :param count: The number of currently added party records. :type count: Integer """ self.current_party_count = count def create_doc_tab_populate_combobox(self): """ Creates the supporting document component widget. """ self.doc_tab_data() self.docs_tab = QTabWidget() self.docs_tab_index = OrderedDict() for i, (id, doc) in enumerate(self.doc_types.iteritems()): self.docs_tab_index[doc] = i # the tab widget containing the document widget layout # and the child of the tab. tab_widget = QWidget() tab_widget.setObjectName(doc) # The layout of the tab widget cont_layout = QVBoxLayout(tab_widget) cont_layout.setObjectName( u'widget_layout_{}'.format(doc) ) # the scroll area widget inside the tab widget. scroll_area = QScrollArea(tab_widget) scroll_area.setFrameShape(QFrame.NoFrame) scroll_area.setObjectName( u'tab_scroll_area_{}'.format(doc) ) layout_widget = QWidget() # the widget the is under the scroll area content and # the widget containing the document widget layout # This widget is hidden and shown based on the STR number layout_widget.setObjectName( u'widget_{}'.format(doc) ) doc_widget_layout = QVBoxLayout(layout_widget) doc_widget_layout.setObjectName( u'doc_widget_layout_{}'.format( doc ) ) doc_widget = QWidget() doc_widget.setObjectName( u'doc_widget_{}_{}'.format(doc, self.str_number) ) doc_widget_layout.addWidget(doc_widget) # the layout containing document widget. ### This is the layout that is registered to add uploaded # supporting documents widgets into. tab_layout = QVBoxLayout(doc_widget) tab_layout.setObjectName( u'layout_{}_{}'.format(doc, self.str_number) ) scroll_area.setWidgetResizable(True) scroll_area.setWidget(layout_widget) cont_layout.addWidget(scroll_area) # Add the tab widget with the document # type name to create a tab. self.docs_tab.addTab(tab_widget, doc) self.container_box.addWidget(self.docs_tab, 1) if len(self.str_numbers) == 1: self.doc_type_cbo.addItem(doc, id) def doc_tab_data(self): """ Sets the document types in the social tenure entity. """ doc_entity = self.social_tenure. \ supporting_doc.document_type_entity doc_type_model = entity_model(doc_entity) docs = doc_type_model() doc_type_list = docs.queryObject().all() self.doc_types = [(doc.id, doc.value) for doc in doc_type_list ] self.doc_types = OrderedDict(self.doc_types) def match_doc_combo_to_tab(self): """ Changes the active tab based on the selected value of document type combobox. """ combo_text = self.doc_type_cbo.currentText() if combo_text is not None and len(combo_text) > 0: index = self.docs_tab_index[combo_text] self.docs_tab.setCurrentIndex(index) def match_doc_tab_to_combo(self): """ Changes the document type combobox value based on the selected tab. """ doc_tab_index = self.docs_tab.currentIndex() self.doc_type_cbo.setCurrentIndex(doc_tab_index) @staticmethod def hide_doc_widgets(widget, visibility): """ Hides or shows the visibility of the supporting document container widgets. :param widget: The widget to which the visibility is set. :type widget: QWidget :param visibility: A boolean to show or hide visibility. True hides widget and False shows it. :type visibility: Boolean """ widget.setHidden(visibility) def update_container(self, str_number): """ Update the current supporting document widget container to be used. :param str_number: The STR node number :type str_number: Integer """ doc_text = self.doc_type_cbo.currentText() cbo_index = self.doc_type_cbo.currentIndex() doc_id = self.doc_type_cbo.itemData(cbo_index) scroll_area = self.docs_tab.findChild( QScrollArea, u'tab_scroll_area_{}'.format( doc_text, str_number ) ) doc_widget = scroll_area.findChild( QWidget, u'doc_widget_{}_{}'.format( doc_text, str_number ) ) # If the doc widget doesn't exist create it for new STR instance if doc_widget is None: # find the doc_widget layout that contains # all STR doc widget layouts. Single # doc_widget_layout is created for each document type. # But all doc_widgets for each STR instance and # document types will be added here. doc_widget_layout = scroll_area.findChild( QVBoxLayout, u'doc_widget_layout_{}'.format(doc_text) ) doc_widget = QWidget() doc_widget.setObjectName( u'doc_widget_{}_{}'.format(doc_text, str_number) ) self.hide_all_other_widget(doc_text, str_number) doc_widget_layout.addWidget(doc_widget) # Create the layout so that layouts are registered in # which uploaded document widgets are added. layout = QVBoxLayout(doc_widget) layout.setObjectName(u'layout_{}_{}'.format( doc_text, str_number ) ) # If the doc widget exists, get the lowest # layout so that it is registered. else: # hide all other widgets self.hide_all_other_widget(doc_text, str_number) # show the current doc widget to display # the document widgets for the current tab. self.hide_doc_widgets(doc_widget, False) layout = doc_widget.findChild( QVBoxLayout, u'layout_{}_{}'.format( doc_text, str_number ) ) # register layout self.supporting_doc_manager.registerContainer( layout, doc_id ) def hide_all_other_widget(self, doc_text, str_number): """ Hides all other supporting document widget except the current STR node widget. :param doc_text: The current document type selected. :type doc_text: String :param str_number: The STR node number :type str_number: Integer """ expression = QRegExp(u'doc_widget*') # hide all existing widgets in all layouts for widget in self.docs_tab.findChildren(QWidget, expression): if widget.objectName() != u'doc_widget_{}_{}'.format( doc_text, str_number): self.hide_doc_widgets(widget, True) def on_upload_document(self): ''' Slot raised when the user clicks to upload a supporting document. ''' document_str = QApplication.translate( "SupportingDocuments", "Specify the Document File Location" ) documents = self.select_file_dialog(document_str) cbo_index = self.doc_type_cbo.currentIndex() doc_id = self.doc_type_cbo.itemData(cbo_index) for doc in documents: self.supporting_doc_manager.insertDocumentFromFile( doc, doc_id, self.social_tenure, self.current_party_count ) # Set last path if len(documents) > 0: doc = documents[0] fi = QFileInfo(doc) dir_path = fi.absolutePath() set_last_document_path(dir_path) model_objs = self.supporting_doc_manager.model_objects() self.onUploadDocument.emit(model_objs) def select_file_dialog(self, title): """ Displays a file dialog for a user to specify a source document :param title: The title of the file dialog :type title: String """ # Get last path for supporting documents last_path = last_document_path() if last_path is None: last_path = '/home' files = QFileDialog.getOpenFileNames( iface.mainWindow(), title, last_path, "Source Documents (*.jpg *.jpeg *.png *.bmp *.tiff *.svg)" ) return files
class OGRReader(object): def __init__(self, source_file): self._ds = ogr.Open(source_file) self._targetGeomColSRID = -1 self._geomType = '' self._dbSession = STDMDb.instance().session self._mapped_cls = None self._mapped_doc_cls = None self._current_profile = current_profile() self._source_doc_manager = None def getLayer(self): #Return the first layer in the data source if self.isValid(): numLayers = self._ds.GetLayerCount() if numLayers > 0: return self._ds.GetLayer(0) else: return None def getSpatialRefCode(self): #Get the EPSG code (More work required) if self.getLayer() != None: spRef = self.getLayer().GetSpatialRef() refCode = spRef.GetAttrValue("PRIMEM|AUTHORITY", 1) else: #Fallback to WGS84 refCode = 4326 return refCode def isValid(self): #Whether the open process succeeded or failed if self._ds is None: return False else: return True def reset(self): #Destroy self._ds = None self._geomType = "" self._targetGeomColSRID = -1 def getFields(self): #Return the data source's fields in a list fields = [] lyr = self.getLayer() lyr.ResetReading() feat_defn = lyr.GetLayerDefn() for l in range(feat_defn.GetFieldCount()): field_defn = feat_defn.GetFieldDefn(l) fields.append(str(field_defn.GetNameRef())) return fields def _data_source_entity(self, table_name): entity = self._current_profile.entity_by_name(table_name) return entity def entity_virtual_columns(self, table_name): """ :param table_name: Name of the target table. :type table_name: str :return: Returns a list of derived columns for the specified target table. :rtype: list """ entity = self._data_source_entity(table_name) if entity is None: return [] return entity.virtual_columns() def _get_mapped_class(self, table_name): #Get entity from the corresponding table name entity = self._data_source_entity(table_name) if entity is None: return None ent_model, doc_model = entity_model(entity, with_supporting_document=True) return ent_model, doc_model def _insertRow(self, columnValueMapping): """ Insert a new row using the mapped class instance then mapping column names to the corresponding column values. """ model_instance = self._mapped_cls() for col, value in columnValueMapping.iteritems(): if hasattr(model_instance, col): ''' #Check if column type is enumeration and transform accordingly col_is_enum, enum_symbol = self._enumeration_column_type(col, value) if col_is_enum: value = enum_symbol ''' if not isinstance(value, IgnoreType): setattr(model_instance, col, value) try: self._dbSession.add(model_instance) self._dbSession.commit() except: self._dbSession.rollback() raise def featToDb(self, targettable, columnmatch, append, parentdialog, geomColumn=None, geomCode=-1, translator_manager=None): """ Performs the data import from the source layer to the STDM database. :param targettable: Destination table name :param columnmatch: Dictionary containing source columns as keys and target columns as the values. :param append: True to append, false to overwrite by deleting previous records :param parentdialog: A reference to the calling dialog. :param translator_manager: Instance of 'stdm.data.importexport.ValueTranslatorManager' containing value translators defined for the destination table columns. :type translator_manager: ValueTranslatorManager """ #Check current profile if self._current_profile is None: msg = QApplication.translate( 'OGRReader', 'The current profile could not be determined.\nPlease set it ' 'in the Options dialog or Configuration Wizard.') raise ConfigurationException(msg) if translator_manager is None: translator_manager = ValueTranslatorManager() #Delete existing rows in the target table if user has chosen to overwrite if not append: delete_table_data(targettable) #Container for mapping column names to their corresponding values column_value_mapping = {} lyr = self.getLayer() lyr.ResetReading() feat_defn = lyr.GetLayerDefn() numFeat = lyr.GetFeatureCount() #Configure progress dialog init_val = 0 progress = QProgressDialog("", "&Cancel", init_val, numFeat, parentdialog) progress.setWindowModality(Qt.WindowModal) lblMsgTemp = "Importing {0} of {1} to STDM..." #Set entity for use in translators destination_entity = self._data_source_entity(targettable) for feat in lyr: column_count = 0 progress.setValue(init_val) progressMsg = lblMsgTemp.format((init_val + 1), numFeat) progress.setLabelText(progressMsg) if progress.wasCanceled(): break #Reset source document manager for new records if destination_entity.supports_documents: if not self._source_doc_manager is None: self._source_doc_manager.reset() for f in range(feat_defn.GetFieldCount()): field_defn = feat_defn.GetFieldDefn(f) field_name = field_defn.GetNameRef() #Append value only if it has been defined by the user if field_name in columnmatch: dest_column = columnmatch[field_name] field_value = feat.GetField(f) #Create mapped class only once if self._mapped_cls is None: mapped_cls, mapped_doc_cls = self._get_mapped_class( targettable) if mapped_cls is None: msg = QApplication.translate( "OGRReader", "Something happened that caused the " "database table not to be mapped to the " "corresponding model class. Please contact" " your system administrator.") raise RuntimeError(msg) self._mapped_cls = mapped_cls self._mapped_doc_cls = mapped_doc_cls #Create source document manager if the entity supports them if destination_entity.supports_documents: self._source_doc_manager = SourceDocumentManager( destination_entity.supporting_doc, self._mapped_doc_cls) if geomColumn is not None: #Use geometry column SRID in the target table self._geomType,self._targetGeomColSRID = \ geometryType(targettable,geomColumn) ''' Check if there is a value translator defined for the specified destination column. ''' value_translator = translator_manager.translator( dest_column) if not value_translator is None: #Set destination table entity value_translator.entity = destination_entity source_col_names = value_translator.source_column_names( ) field_value_mappings = self._map_column_values( feat, feat_defn, source_col_names) #Set source document manager if required if value_translator.requires_source_document_manager: value_translator.source_document_manager = self._source_doc_manager field_value = value_translator.referencing_column_value( field_value_mappings) if not isinstance(field_value, IgnoreType): column_value_mapping[dest_column] = field_value #Set supporting documents if destination_entity.supports_documents: column_value_mapping['documents'] = \ self._source_doc_manager.model_objects() column_count += 1 #Only insert geometry if it has been defined by the user if geomColumn is not None: geom = feat.GetGeometryRef() if geom is not None: #Get geometry in WKT/WKB format geomWkb = geom.ExportToWkt() column_value_mapping[geomColumn] = "SRID={0!s};{1}".format( self._targetGeomColSRID, geomWkb) #Check if the geometry types match layerGeomType = geom.GetGeometryName() if layerGeomType.lower() != self._geomType.lower(): raise TypeError("The geometries of the source and destination columns do not match.\n" \ "Source Geometry Type: {0}, Destination Geometry Type: {1}".format(layerGeomType, self._geomType)) return try: #Insert the record self._insertRow(column_value_mapping) except: progress.close() raise init_val += 1 progress.setValue(numFeat) def _enumeration_column_type(self, column_name, value): """ Checks if the given column is of DeclEnumType. :param column_name: Name of the enumeration column. :type column_name: str :return: True if column is of DeclType and return Enum symbol; Else, False and None. :rtype: tuple """ try: #Get column type of the enumeration enum_col_type = self._mapped_cls.__mapper__.columns[ column_name].type except KeyError: return False, None if not hasattr(enum_col_type, "enum"): return False, None else: enum_obj = enum_col_type.enum try: if not isinstance(value, str) or not isinstance( value, unicode): value = unicode(value) enum_symbol = enum_obj.from_string(value.strip()) except ValueError: enum_symbol = IgnoreType() return True, enum_symbol def _map_column_values(self, feature, feature_defn, source_cols): """ Retrieves values for specific columns from the specified feature. :param feature: Input feature. :type feature: ogr.Feature :param feature_defn: Feature definition for the layer. :type feature_defn: ogr.FeatureDefn :param source_cols: List of columns whose respective values will be retrieved. :type source_cols: list :return: Collection containing pairs of column names and corresponding values. :rtype: dict """ col_values = {} if len(source_cols) == 0: return col_values for f in range(feature_defn.GetFieldCount()): field_defn = feature_defn.GetFieldDefn(f) field_name = field_defn.GetNameRef() match_idx = getIndex(source_cols, field_name) if match_idx != -1: field_value = feature.GetField(f) col_values[field_name] = field_value return col_values
class Save2DB: """ Class to insert entity data into db """ def __init__(self, entity, attributes, ids=None): """ Initialize class and class variable """ self.attributes = attributes self.form_entity = entity self.doc_model = None self._doc_manager =None self.entity = self.object_from_entity_name(self.form_entity) self.model = self.dbmodel_from_entity() self.key = 0 self.parents_ids = ids self.geom = 4326 self.entity_mapping = {} def object_from_entity_name(self, entity): """ :return: """ if entity == 'social_tenure': return current_profile().social_tenure else: user_entity = current_profile().entity_by_name(entity) return user_entity def entity_has_supporting_docs(self): """ Check if the entity has supporting document before importing :return: Bool """ if self.entity.supports_documents: return self.entity.supports_documents else: return None def entity_supported_document_types(self): """ Get the supported document types before importing so that they are captured during import process :return: List """ return self.entity.document_types_non_hex() def dbmodel_from_entity(self): """ Format model attributes from passed entity attributes :return: """ if self.entity_has_supporting_docs(): entity_object, self.doc_model = entity_model(self.entity, with_supporting_document=True) entity_object_model = entity_object() if hasattr(entity_object_model, 'documents'): if self.entity.TYPE_INFO == 'SOCIAL_TENURE': obj_doc_col = current_profile().social_tenure.supporting_doc else: obj_doc_col = self.entity.supporting_doc self._doc_manager = SourceDocumentManager( obj_doc_col, self.doc_model ) else: entity_object = entity_model(self.entity) entity_object_model = entity_object() return entity_object_model def objects_from_supporting_doc(self, instance_file = None): """ Create supporting doc path instances based on the collected documents :return:paths :rtype: document object instance """ if instance_file: f_dir, file_name = os.path.split(instance_file) for document, val in self.attributes.iteritems(): if str(document).endswith('supporting_document'): if val != '': doc = self.format_document_name_from_attribute(document) doc_path = os.path.normpath(f_dir+'/'+val) abs_path = doc_path.replace('\\','/').strip() if QFile.exists(abs_path): self.supporting_document_model(abs_path, doc) def supporting_document_model(self,doc_path,doc): """ :param doc_path: absolute document path :param doc: document name :type: str Construct supporting document model instance to add into the db :return: """ # Create document container doc_container = QVBoxLayout() supporting_doc_entity = self.entity.supporting_doc.document_type_entity document_type_id = entity_attr_to_id(supporting_doc_entity, 'value', doc, lower=False) # Register container self._doc_manager.registerContainer( doc_container, document_type_id ) #Copy the document to STDM working directory self._doc_manager.insertDocumentFromFile( doc_path, document_type_id, self.entity ) def format_document_name_from_attribute(self, doc): """ Get the type of document from attribute name So that supporting document class instance can save it in the right format :return: """ formatted_doc_list = self.entity_supported_document_types() default = 'General' doc_type = str(doc).split('_',1) if doc_type[0].startswith('supporting') and formatted_doc_list[0] == default: return default elif doc_type[0].startswith('supporting') and formatted_doc_list[0] != default: return formatted_doc_list[0] elif not doc_type[0].startswith('supporting'): actual_doc_name = doc_type[0].replace('-', ' ') for doc_name in formatted_doc_list: if actual_doc_name in doc_name or doc_name.startswith(actual_doc_name): return doc_name else: return formatted_doc_list[0] else: return formatted_doc_list[0] def extract_social_tenure_entities(self): ''' We want to extract social tenure enities so that we know if it has multiple or single entities in the list :return: ''' party_ref_column = '' spatial_ref_column = '' if self.parents_ids is not None: print self.parents_ids if self.attributes.has_key('party'): full_party_ref_column = self.attributes.get('party') party_ref_column = full_party_ref_column + '_id' print 'party{}.'.format(party_ref_column) setattr(self.model, party_ref_column, self.parents_ids.get(full_party_ref_column)[0]) if self.attributes.has_key('spatial_unit'): full_spatial_ref_column = self.attributes.get('spatial_unit') spatial_ref_column = full_spatial_ref_column + '_id' print 'sp.{}.'.format(spatial_ref_column) setattr(self.model, spatial_ref_column, self.parents_ids.get(full_spatial_ref_column)[0]) return party_ref_column, spatial_ref_column def save_to_db(self): """ Format object attribute data from entity and save them into database :return: """ self.column_info() try: if self.entity.short_name == 'social_tenure_relationship': #try: prefix = current_profile().prefix + '_' if self.attributes.has_key('party'): full_party_ref_column = self.attributes.get('party') party_ref_column = full_party_ref_column + '_id' self.attributes.pop('party') else: full_party_ref_column = current_profile().social_tenure.parties[0].name party_ref_column = full_party_ref_column + '_id' setattr(self.model, party_ref_column, self.parents_ids.get(prefix + full_party_ref_column)[0]) if self.attributes.has_key('spatial_unit'): full_spatial_ref_column = self.attributes.get('spatial_unit') spatial_ref_column = full_spatial_ref_column + '_id' self.attributes.pop('spatial_unit') else: full_spatial_ref_column = current_profile().social_tenure.spatial_units[0].name spatial_ref_column = full_spatial_ref_column + '_id' setattr(self.model, spatial_ref_column, self.parents_ids.get(prefix + full_spatial_ref_column)[0]) except: pass for k, v in self.attributes.iteritems(): if hasattr(self.model, k): col_type = self.entity_mapping.get(k) col_prop = self.entity.columns[k] var = self.attribute_formatter(col_type, col_prop, v) setattr(self.model, k, var) if self.entity_has_supporting_docs(): self.model.documents = self._doc_manager.model_objects() self.model.save() return self.model.id def save_parent_to_db(self): """ Format object attribute data from entity and save them into database attribute :return: """ self.column_info() for k, v in self.attributes.iteritems(): if hasattr(self.model, k): col_type = self.entity_mapping.get(k) col_prop = self.entity.columns[k] #print "property{0}.... and type.{1}".format(col_prop, col_type) var = self.attribute_formatter(col_type, col_prop, v) setattr(self.model, k, var) if self.entity_has_supporting_docs(): self.model.documents = self._doc_manager.model_objects() self.model.save() self.key = self.model.id return self.key def save_foreign_key_table(self): """ Get the table with foreign keys only :return: """ for col, type_info in self.column_info().iteritems(): col_prop = self.entity.columns[col] var = self.attribute_formatter(type_info, col_prop, None) setattr(self.model, col, var) self.model.save() self.cleanup() def column_info(self): """ :return: """ self.entity_mapping = {} cols = self.entity.columns.values() for c in cols: self.entity_mapping[c.name] = c.TYPE_INFO def get_srid(self, srid): """ Let the user specify the coordinate system during data import :param srid: :return: """ self.geom = srid return self.geom def id_from_model_object(self, obj): """ We need to obtian id from object instance :param obj: :return: """ return obj.id def attribute_formatter(self, col_type, col_prop, var=None): """ Format geoodk attributes collected in the field to conform to STDM database contrains :return: """ if col_type == 'BOOL': if len(var) < 1: return None if len(var)>1: if var == '' or var is None: return None if var == 'Yes' or var == True: return True if var == 'No' or var == False: return False else: return None if col_type == 'LOOKUP': if len(var) < 1 or var is None: return None if len(var) <4: if var == 'Yes' or var == 'No': return entity_attr_to_model(col_prop.parent, 'value', var).id if var != 'Yes' and var != 'No': lk_code = entity_attr_to_id(col_prop.parent, "code", var) if not str(lk_code).isdigit(): return None else: return lk_code if len(var) > 3: if not str(entity_attr_to_id(col_prop.parent, 'code', var)).isdigit(): id_value = entity_attr_to_model(col_prop.parent, 'value', var) if id_value is not None: return id_value.id #return entity_attr_to_model(col_prop.parent, 'value', var).id else: lk_code = entity_attr_to_id(col_prop.parent, "code", var) if not str(lk_code).isdigit(): return None else: return lk_code else: return None elif col_type == 'ADMIN_SPATIAL_UNIT': var_code = None try: if len(var) < 1 or var is None: return None elif not len(var) > 3: var_code = entity_attr_to_id(col_prop.parent, "code", var) if var_code and var_code == var: return None else: return var_code elif len(var) > 3 and entity_attr_to_id(col_prop.parent, "name", var) is not None: var_code = entity_attr_to_id(col_prop.parent, "name", var) if var_code and var_code == var: return None else: return var_code else: if entity_attr_to_id(col_prop.parent, "name", var) is None: var_code = entity_attr_to_id(col_prop.parent, "code", var) if not var_code or var_code == var: return None except: pass elif col_type == 'MULTIPLE_SELECT': print 'multiple select {}'.format(var) if var == '' or var is None: return None else: print var col_parent = col_prop.association.first_parent lk_val_list = col_parent.values.values() choices_list = [] for code in lk_val_list: choices_list.append(entity_attr_to_id( col_parent.association.first_parent, 'value', code.value)) print choices_list if len(choices_list) > 1: return choices_list else: return None elif col_type == 'GEOMETRY': defualt_srid = 0 if var: geom_provider = STDMGeometry(var) if isinstance(col_prop, GeometryColumn): defualt_srid = col_prop.srid if defualt_srid != 0: geom_provider.set_user_srid(defualt_srid) else: geom_provider.set_user_srid(GEOMPARAM) if col_prop.geometry_type() == 'POINT': return geom_provider.point_to_Wkt() if col_prop.geometry_type() == 'POLYGON': return geom_provider.polygon_to_Wkt() else: return None elif col_type == 'FOREIGN_KEY': if self.parents_ids is None or len(self.parents_ids) < 0: return None else: for code, val in self.parents_ids.iteritems(): if col_prop.parent.name == code: return val[0] else: return None elif col_type == 'INT' or col_type == 'DOUBLE' or col_type == 'PERCENT': if var == '': return None else: return var elif col_type == 'DATETIME' or col_type == 'DATE': if var is None: return None if var == '': return None else: return var else: return var def cleanup(self): """ Reset all the model and entity data before we process the next entity data :return: None z""" self.model = None self.entity = None self.attributes = None self._doc_manager = None