class Save2DB:
"""
Class to insert entity data into db
"""
def __init__(self, entity, attributes, ids=None):
"""
Initialize class and class variable
"""
self.attributes = attributes
self.form_entity = entity
self.doc_model = None
self._doc_manager = None
self.entity = self.object_from_entity_name(self.form_entity)
self.model = self.dbmodel_from_entity()
self.key = 0
self.parents_ids = ids
self.geom = 4326
self.entity_mapping = {}
def object_from_entity_name(self, entity):
"""
:return:
"""
if entity == 'social_tenure':
return current_profile().social_tenure
else:
user_entity = current_profile().entity_by_name(entity)
return user_entity
def entity_has_supporting_docs(self):
"""
Check if the entity has supporting document before importing
:return: Bool
"""
if self.entity.supports_documents:
return self.entity.supports_documents
else:
return None
def entity_supported_document_types(self):
"""
Get the supported document types before importing so that they are captured
during import process
:return: List
"""
return self.entity.document_types_non_hex()
def dbmodel_from_entity(self):
"""
Format model attributes from passed entity attributes
:return:
"""
if self.entity_has_supporting_docs():
entity_object, self.doc_model = entity_model(
self.entity, with_supporting_document=True)
entity_object_model = entity_object()
if hasattr(entity_object_model, 'documents'):
if self.entity.TYPE_INFO == 'SOCIAL_TENURE':
obj_doc_col = current_profile(
).social_tenure.supporting_doc
else:
obj_doc_col = self.entity.supporting_doc
self._doc_manager = SourceDocumentManager(
obj_doc_col, self.doc_model)
else:
entity_object = entity_model(self.entity)
entity_object_model = entity_object()
return entity_object_model
def objects_from_supporting_doc(self, instance_file=None):
"""
Create supporting doc path instances based on the collected documents
:return:paths
:rtype: document object instance
"""
if instance_file:
f_dir, file_name = os.path.split(instance_file)
for document, val in self.attributes.iteritems():
if str(document).endswith('supporting_document'):
if val != '':
doc = self.format_document_name_from_attribute(
document)
doc_path = os.path.normpath(f_dir + '/' + val)
abs_path = doc_path.replace('\\', '/').strip()
if QFile.exists(abs_path):
self.supporting_document_model(abs_path, doc)
def supporting_document_model(self, doc_path, doc):
"""
:param doc_path: absolute document path
:param doc: document name
:type: str
Construct supporting document model instance to add into the db
:return:
"""
# Create document container
doc_container = QVBoxLayout()
supporting_doc_entity = self.entity.supporting_doc.document_type_entity
document_type_id = entity_attr_to_id(supporting_doc_entity,
'value',
doc,
lower=False)
# Register container
self._doc_manager.registerContainer(doc_container, document_type_id)
#Copy the document to STDM working directory
self._doc_manager.insertDocumentFromFile(doc_path, document_type_id,
self.entity)
def format_document_name_from_attribute(self, doc):
"""
Get the type of document from attribute name
So that supporting document class instance can save it in the right format
:return:
"""
formatted_doc_list = self.entity_supported_document_types()
default = 'General'
doc_type = str(doc).split('_', 1)
if doc_type[0].startswith(
'supporting') and formatted_doc_list[0] == default:
return default
elif doc_type[0].startswith(
'supporting') and formatted_doc_list[0] != default:
return formatted_doc_list[0]
elif not doc_type[0].startswith('supporting'):
actual_doc_name = doc_type[0].replace('-', ' ')
for doc_name in formatted_doc_list:
if actual_doc_name in doc_name or doc_name.startswith(
actual_doc_name):
return doc_name
else:
return formatted_doc_list[0]
else:
return formatted_doc_list[0]
def extract_social_tenure_entities(self):
'''
We want to extract social tenure enities so that we know if it has multiple or single
entities in the list
:return:
'''
party_ref_column = ''
spatial_ref_column = ''
if self.parents_ids is not None:
print self.parents_ids
if self.attributes.has_key('party'):
full_party_ref_column = self.attributes.get('party')
party_ref_column = full_party_ref_column + '_id'
print 'party{}.'.format(party_ref_column)
setattr(self.model, party_ref_column,
self.parents_ids.get(full_party_ref_column)[0])
if self.attributes.has_key('spatial_unit'):
full_spatial_ref_column = self.attributes.get('spatial_unit')
spatial_ref_column = full_spatial_ref_column + '_id'
print 'sp.{}.'.format(spatial_ref_column)
setattr(self.model, spatial_ref_column,
self.parents_ids.get(full_spatial_ref_column)[0])
return party_ref_column, spatial_ref_column
def save_to_db(self):
"""
Format object attribute data from entity and save them into database
:return:
"""
self.column_info()
attributes = self.attributes
try:
if self.entity.short_name == 'social_tenure_relationship':
#try:
prefix = current_profile().prefix + '_'
if self.attributes.has_key('party'):
full_party_ref_column = self.attributes.get('party')
party_ref_column = full_party_ref_column + '_id'
self.attributes.pop('party')
else:
full_party_ref_column = current_profile(
).social_tenure.parties[0].name
party_ref_column = full_party_ref_column.replace(
prefix, '') + '_id'
setattr(self.model, party_ref_column,
self.parents_ids.get(full_party_ref_column)[0])
if self.attributes.has_key('spatial_unit'):
full_spatial_ref_column = self.attributes.get(
'spatial_unit')
spatial_ref_column = full_spatial_ref_column + '_id'
self.attributes.pop('spatial_unit')
else:
full_spatial_ref_column = current_profile(
).social_tenure.spatial_units[0].name
spatial_ref_column = full_spatial_ref_column.replace(
prefix, '') + '_id'
setattr(self.model, spatial_ref_column,
self.parents_ids.get(full_spatial_ref_column)[0])
attributes = self.attributes['social_tenure']
except:
pass
for k, v in attributes.iteritems():
if hasattr(self.model, k):
col_type = self.entity_mapping.get(k)
col_prop = self.entity.columns[k]
var = self.attribute_formatter(col_type, col_prop, v)
setattr(self.model, k, var)
if self.entity_has_supporting_docs():
self.model.documents = self._doc_manager.model_objects()
self.model.save()
return self.model.id
def save_parent_to_db(self):
"""
Format object attribute data from entity and save them into database
attribute
:return:
"""
self.column_info()
for k, v in self.attributes.iteritems():
if hasattr(self.model, k):
col_type = self.entity_mapping.get(k)
col_prop = self.entity.columns[k]
#print "property{0}.... and type.{1}".format(col_prop, col_type)
var = self.attribute_formatter(col_type, col_prop, v)
setattr(self.model, k, var)
if self.entity_has_supporting_docs():
self.model.documents = self._doc_manager.model_objects()
self.model.save()
self.key = self.model.id
return self.key
def save_foreign_key_table(self):
"""
Get the table with foreign keys only
:return:
"""
for col, type_info in self.column_info().iteritems():
col_prop = self.entity.columns[col]
var = self.attribute_formatter(type_info, col_prop, None)
setattr(self.model, col, var)
self.model.save()
self.cleanup()
def column_info(self):
"""
:return:
"""
self.entity_mapping = {}
cols = self.entity.columns.values()
for c in cols:
self.entity_mapping[c.name] = c.TYPE_INFO
def get_srid(self, srid):
"""
Let the user specify the coordinate system during data import
:param srid:
:return:
"""
self.geom = srid
return self.geom
def id_from_model_object(self, obj):
"""
We need to obtian id from object instance
:param obj:
:return:
"""
return obj.id
def attribute_formatter(self, col_type, col_prop, var=None):
"""
Format geoodk attributes collected in the field
to conform to STDM database contrains
:return:
"""
if col_type == 'BOOL':
if len(var) < 1:
return None
if len(var) > 1:
if var == '' or var is None:
return None
if var == 'Yes' or var == True:
return True
if var == 'No' or var == False:
return False
else:
return None
if col_type == 'LOOKUP':
if len(var) < 1 or var is None:
return None
if len(var) < 4:
if var == 'Yes' or var == 'No':
return entity_attr_to_model(col_prop.parent, 'value',
var).id
if var != 'Yes' and var != 'No':
lk_code = entity_attr_to_id(col_prop.parent, "code", var)
if not str(lk_code).isdigit():
return None
else:
return lk_code
if len(var) > 3:
if not str(entity_attr_to_id(col_prop.parent, 'code',
var)).isdigit():
id_value = entity_attr_to_model(col_prop.parent, 'value',
var)
if id_value is not None:
return id_value.id
#return entity_attr_to_model(col_prop.parent, 'value', var).id
else:
lk_code = entity_attr_to_id(col_prop.parent, "code", var)
if not str(lk_code).isdigit():
return None
else:
return lk_code
else:
return None
elif col_type == 'ADMIN_SPATIAL_UNIT':
var_code = None
try:
if len(var) < 1 or var is None:
return None
elif not len(var) > 3:
var_code = entity_attr_to_id(col_prop.parent, "code", var)
if var_code and var_code == var:
return None
else:
return var_code
elif len(var) > 3 and entity_attr_to_id(
col_prop.parent, "name", var) is not None:
var_code = entity_attr_to_id(col_prop.parent, "name", var)
if var_code and var_code == var:
return None
else:
return var_code
else:
if entity_attr_to_id(col_prop.parent, "name", var) is None:
var_code = entity_attr_to_id(col_prop.parent, "code",
var)
if not var_code or var_code == var:
return None
except:
pass
elif col_type == 'MULTIPLE_SELECT':
print 'multiple select {}'.format(var)
if var == '' or var is None:
return None
else:
col_parent = col_prop.association.first_parent
lk_val_list = col_parent.values.values()
choices_list = []
for code in lk_val_list:
choices_list.append(
entity_attr_to_id(col_parent.association.first_parent,
'value', code.value))
if len(choices_list) > 1:
return choices_list
else:
return None
elif col_type == 'GEOMETRY':
defualt_srid = 0
if var:
geom_provider = STDMGeometry(var)
if isinstance(col_prop, GeometryColumn):
defualt_srid = col_prop.srid
if defualt_srid != 0:
geom_provider.set_user_srid(defualt_srid)
else:
geom_provider.set_user_srid(GEOMPARAM)
if col_prop.geometry_type() == 'POINT':
return geom_provider.point_to_Wkt()
if col_prop.geometry_type() == 'POLYGON':
return geom_provider.polygon_to_Wkt()
else:
return None
elif col_type == 'FOREIGN_KEY':
ret_val = None
for code, val in self.parents_ids.iteritems():
if col_prop.parent.name == code:
ret_val = val[0]
break
return ret_val
elif col_type == 'INT' or col_type == 'DOUBLE' or col_type == 'PERCENT':
ret_val = None
if var <> '':
ret_val = var
return ret_val
elif col_type == 'DATETIME' or col_type == 'DATE':
ret_val = None
if var <> '':
ret_val = var
return ret_val
else:
return var
def cleanup(self):
"""
Reset all the model and entity data before we process
the next entity data
:return: None
z"""
self.model = None
self.entity = None
self.attributes = None
self._doc_manager = None
class OGRReader:
def __init__(self, source_file):
self._ds = ogr.Open(source_file)
self._targetGeomColSRID = -1
self._geomType = ''
self._dbSession = STDMDb.instance().session
self._mapped_cls = None
self._mapped_doc_cls = None
self._current_profile = current_profile()
self._source_doc_manager = None
def getLayer(self):
# Return the first layer in the data source
if self.isValid():
numLayers = self._ds.GetLayerCount()
if numLayers > 0:
return self._ds.GetLayer(0)
else:
return None
def getSpatialRefCode(self):
# Get the EPSG code (More work required)
if self.getLayer() != None:
spRef = self.getLayer().GetSpatialRef()
refCode = spRef.GetAttrValue("PRIMEM|AUTHORITY", 1)
else:
# Fallback to WGS84
refCode = 4326
return refCode
def isValid(self):
# Whether the open process succeeded or failed
if self._ds is None:
return False
else:
return True
def reset(self):
# Destroy
self._ds = None
self._geomType = ""
self._targetGeomColSRID = -1
def getFields(self):
# Return the data source's fields in a list
fields = []
lyr = self.getLayer()
lyr.ResetReading()
feat_defn = lyr.GetLayerDefn()
for l in range(feat_defn.GetFieldCount()):
field_defn = feat_defn.GetFieldDefn(l)
fields.append(str(field_defn.GetNameRef()))
return fields
def _data_source_entity(self, table_name):
entity = self._current_profile.entity_by_name(table_name)
return entity
def entity_virtual_columns(self, table_name):
"""
:param table_name: Name of the target table.
:type table_name: str
:return: Returns a list of derived columns for the specified target table.
:rtype: list
"""
entity = self._data_source_entity(table_name)
if entity is None:
return []
return entity.virtual_columns()
def _get_mapped_class(self, table_name):
# Get entity from the corresponding table name
entity = self._data_source_entity(table_name)
if entity is None:
return None
ent_model, doc_model = entity_model(entity,
with_supporting_document=True)
return ent_model, doc_model
def auto_fix_percent(self, target_table, col_name, value):
"""
Fixes percent columns if empty and with a wrong format.
:param target_table: The destination table name
:type target_table: String
:param col_name: The destination column name
:type col_name: String
:param value: Value to be saved to the DB
:type value: Any
:return: Converted value
:rtype: Any
"""
entity = self._data_source_entity(target_table)
if entity.columns[col_name].TYPE_INFO == 'PERCENT':
if isinstance(value, str):
if not bool(value.strip()) or value.strip().lower() == 'null':
value = None
if '%' in value:
value = value.replace('%', '')
try:
if value is not None:
value = float(value)
except ValueError:
value = None
return value
def auto_fix_float_integer(self, target_table, col_name, value):
"""
Fixes float and integer columns if empty and with a wrong format.
:param target_table: The destination table name
:type target_table: String
:param col_name: The destination column name
:type col_name: String
:param value: Value to be saved to the DB
:type value: Any
:return: Converted value
:rtype: Any
"""
entity = self._data_source_entity(target_table)
integer_types = [
'INT', 'LOOKUP', 'ADMIN_SPATIAL_UNIT', 'FOREIGN_KEY', 'DOUBLE'
]
float_type = ['DOUBLE']
int_type = ['INT', 'LOOKUP', 'ADMIN_SPATIAL_UNIT', 'FOREIGN_KEY']
if col_name in entity.columns.keys():
if entity.columns[col_name].TYPE_INFO in integer_types:
if isinstance(value, str):
if not bool(
value.strip()) or value.strip().lower() == 'null':
value = None
if entity.columns[col_name].TYPE_INFO in float_type:
try:
if value is not None:
value = float(value)
except ValueError:
value = None
elif entity.columns[col_name].TYPE_INFO in int_type:
try:
if value is not None:
value = int(value)
if isinstance(value, int):
if value == 0:
if entity.columns[col_name].TYPE_INFO in \
['LOOKUP', 'ADMIN_SPATIAL_UNIT',
'FOREIGN_KEY']:
value = None
except ValueError:
# TODO show warning to the user that
# some values cannot be converted to integer.
value = None
return value
def auto_fix_date(self, target_table, col_name, value):
"""
Fixes date and datetime columns if empty and with a wrong format.
:param target_table: The destination table name
:type target_table: String
:param col_name: The destination column name
:type col_name: String
:param value: Value to be saved to the DB
:type value: Any
:return: Converted value
:rtype: Any
"""
entity = self._data_source_entity(target_table)
date_types = ['DATE', 'DATETIME']
if entity.columns[col_name].TYPE_INFO in date_types:
if not bool(value) or value.lower() == 'null':
value = None
return value
def auto_fix_yes_no(self, target_table, col_name, value):
"""
Fixes Yes_NO columns if empty and with a wrong format.
:param target_table: The destination table name
:type target_table: String
:param col_name: The destination column name
:type col_name: String
:param value: Value to be saved to the DB
:type value: Any
:return: Converted value
:rtype: Any
"""
entity = self._data_source_entity(target_table)
yes_no_types = ['BOOL']
if entity.columns[col_name].TYPE_INFO in yes_no_types:
if not bool(value.strip()) or value.strip().lower() == 'null':
value = None
elif value.strip().lower() == 'yes':
value = True
elif value.strip().lower() == 'no':
value = False
elif value.strip().lower() == 'true':
value = True
elif value.strip().lower() == 'false':
value = False
return value
def _insertRow(self, target_table, columnValueMapping):
"""
Insert a new row using the mapped class instance then mapping column
names to the corresponding column values.
"""
model_instance = self._mapped_cls()
for col, value in columnValueMapping.items():
if hasattr(model_instance, col):
# 'documents' is not a column so exclude it.
if col != 'documents' and not 'collection' in col:
value = self.auto_fix_float_integer(
target_table, col, value)
value = self.auto_fix_percent(target_table, col, value)
value = self.auto_fix_date(target_table, col, value)
value = self.auto_fix_yes_no(target_table, col, value)
if not isinstance(value, IgnoreType):
setattr(model_instance, col, value)
try:
self._dbSession.add(model_instance)
self._dbSession.commit()
except:
self._dbSession.rollback()
raise
def auto_fix_geom_type(self, geom, source_geom_type,
destination_geom_type):
"""
Converts single geometry type to multi type if the destination is multi type.
:param geom: The OGR geometry
:type geom: OGRGeometry
:param source_geom_type: Source geometry type
:type source_geom_type: String
:param destination_geom_type: Destination geometry type
:type destination_geom_type: String
:return: WkB geometry and the output layer geometry type.
:rtype: Tuple
"""
# Convert polygon to multipolygon if the destination table is multi-polygon.
if source_geom_type.lower() == 'polygon' and \
destination_geom_type.lower() == 'multipolygon':
geom_wkb, geom_type = self.to_ogr_multi_type(
geom, ogr.wkbMultiPolygon)
elif source_geom_type.lower() == 'linestring' and \
destination_geom_type.lower() == 'multilinestring':
geom_wkb, geom_type = self.to_ogr_multi_type(
geom, ogr.wkbMultiLineString)
elif source_geom_type.lower() == 'point' and \
destination_geom_type.lower() == 'multipoint':
geom_wkb, geom_type = self.to_ogr_multi_type(
geom, ogr.wkbMultiPoint)
else:
geom_wkb = geom.ExportToWkt()
geom_type = geom.GetGeometryName()
return geom_wkb, geom_type
def to_ogr_multi_type(self, geom, ogr_type):
"""
Convert ogr geometry to multi-type when supplied the ogr.type.
:param geom: The ogr geometry
:type geom: OGRGeometry
:param ogr_type: The ogr geometry type
:type ogr_type: String
:return: WkB geometry and the output layer geometry type.
:rtype: Tuple
"""
multi_geom = ogr.Geometry(ogr_type)
multi_geom.AddGeometry(geom)
geom_wkb = multi_geom.ExportToWkt()
geom_type = multi_geom.GetGeometryName()
return geom_wkb, geom_type
def featToDb(self,
targettable,
columnmatch,
append,
parentdialog,
geomColumn=None,
geomCode=-1,
translator_manager=None):
"""
Performs the data import from the source layer to the STDM database.
:param targettable: Destination table name
:param columnmatch: Dictionary containing source columns as keys and target columns as the values.
:param append: True to append, false to overwrite by deleting previous records
:param parentdialog: A reference to the calling dialog.
:param translator_manager: Instance of 'stdm.data.importexport.ValueTranslatorManager'
containing value translators defined for the destination table columns.
:type translator_manager: ValueTranslatorManager
"""
# Check current profile
if self._current_profile is None:
msg = QApplication.translate(
'OGRReader',
'The current profile could not be determined.\nPlease set it '
'in the Options dialog or Configuration Wizard.')
raise ConfigurationException(msg)
if translator_manager is None:
translator_manager = ValueTranslatorManager()
# Delete existing rows in the target table if user has chosen to overwrite
if not append:
delete_table_data(targettable)
# Container for mapping column names to their corresponding values
lyr = self.getLayer()
lyr.ResetReading()
feat_defn = lyr.GetLayerDefn()
numFeat = lyr.GetFeatureCount()
# Configure progress dialog
init_val = 0
progress = QProgressDialog("", "&Cancel", init_val, numFeat,
parentdialog)
progress.setWindowModality(Qt.WindowModal)
lblMsgTemp = "Importing {0} of {1} to STDM..."
# Set entity for use in translators
destination_entity = self._data_source_entity(targettable)
for feat in lyr:
column_value_mapping = {}
column_count = 0
progress.setValue(init_val)
progressMsg = lblMsgTemp.format((init_val + 1), numFeat)
progress.setLabelText(progressMsg)
if progress.wasCanceled():
break
# Reset source document manager for new records
if destination_entity.supports_documents:
if not self._source_doc_manager is None:
self._source_doc_manager.reset()
for f in range(feat_defn.GetFieldCount()):
field_defn = feat_defn.GetFieldDefn(f)
field_name = field_defn.GetNameRef()
# Append value only if it has been defined by the user
if field_name in columnmatch:
dest_column = columnmatch[field_name]
field_value = feat.GetField(f)
# Create mapped class only once
if self._mapped_cls is None:
mapped_cls, mapped_doc_cls = self._get_mapped_class(
targettable)
if mapped_cls is None:
msg = QApplication.translate(
"OGRReader",
"Something happened that caused the "
"database table not to be mapped to the "
"corresponding model class. Please contact"
" your system administrator.")
raise RuntimeError(msg)
self._mapped_cls = mapped_cls
self._mapped_doc_cls = mapped_doc_cls
# Create source document manager if the entity supports them
if destination_entity.supports_documents:
self._source_doc_manager = SourceDocumentManager(
destination_entity.supporting_doc,
self._mapped_doc_cls)
if geomColumn is not None:
# Use geometry column SRID in the target table
self._geomType, self._targetGeomColSRID = \
geometryType(targettable, geomColumn)
'''
Check if there is a value translator defined for the
specified destination column.
'''
value_translator = translator_manager.translator(
dest_column)
if value_translator is not None:
# Set destination table entity
value_translator.entity = destination_entity
source_col_names = value_translator.source_column_names(
)
field_value_mappings = self._map_column_values(
feat, feat_defn, source_col_names)
# Set source document manager if required
if value_translator.requires_source_document_manager:
value_translator.source_document_manager = self._source_doc_manager
field_value = value_translator.referencing_column_value(
field_value_mappings)
if not isinstance(field_value, IgnoreType):
# Check column type and rename if multiple select for
# SQLAlchemy compatibility
col_obj = destination_entity.column(dest_column)
if col_obj.TYPE_INFO == 'MULTIPLE_SELECT':
lk_name = col_obj.value_list.name
dest_column = '{0}_collection'.format(lk_name)
column_value_mapping[dest_column] = field_value
# Set supporting documents
if destination_entity.supports_documents:
column_value_mapping['documents'] = \
self._source_doc_manager.model_objects()
column_count += 1
# Only insert geometry if it has been defined by the user
if geomColumn is not None:
geom = feat.GetGeometryRef()
if geom is not None:
# Check if the geometry types match
layerGeomType = geom.GetGeometryName()
# Convert polygon to multipolygon if the destination table is multi-polygon.
geom_wkb, geom_type = self.auto_fix_geom_type(
geom, layerGeomType, self._geomType)
column_value_mapping[geomColumn] = "SRID={0!s};{1}".format(
self._targetGeomColSRID, geom_wkb)
if geom_type.lower() != self._geomType.lower():
raise TypeError(
"The geometries of the source and destination columns do not match.\n" \
"Source Geometry Type: {0}, Destination Geometry Type: {1}".format(
geom_type,
self._geomType))
try:
# Insert the record
self._insertRow(targettable, column_value_mapping)
except:
progress.close()
raise
init_val += 1
progress.setValue(numFeat)
def _enumeration_column_type(self, column_name, value):
"""
Checks if the given column is of DeclEnumType.
:param column_name: Name of the enumeration column.
:type column_name: str
:return: True if column is of DeclType and return Enum symbol;
Else, False and None.
:rtype: tuple
"""
try:
# Get column type of the enumeration
enum_col_type = self._mapped_cls.__mapper__.columns[
column_name].type
except KeyError:
return False, None
if not hasattr(enum_col_type, "enum"):
return False, None
else:
enum_obj = enum_col_type.enum
try:
if not isinstance(value, str) or not isinstance(value, str):
value = str(value)
enum_symbol = enum_obj.from_string(value.strip())
except ValueError:
enum_symbol = IgnoreType()
return True, enum_symbol
def _map_column_values(self, feature, feature_defn, source_cols):
"""
Retrieves values for specific columns from the specified feature.
:param feature: Input feature.
:type feature: ogr.Feature
:param feature_defn: Feature definition for the layer.
:type feature_defn: ogr.FeatureDefn
:param source_cols: List of columns whose respective values will be
retrieved.
:type source_cols: list
:return: Collection containing pairs of column names and corresponding
values.
:rtype: dict
"""
col_values = {}
if len(source_cols) == 0:
return col_values
for f in range(feature_defn.GetFieldCount()):
field_defn = feature_defn.GetFieldDefn(f)
field_name = field_defn.GetNameRef()
match_idx = getIndex(source_cols, field_name)
if match_idx != -1:
field_value = feature.GetField(f)
col_values[field_name] = field_value
return col_values
class SupportingDocuments(ComponentUtility):
onUploadDocument = pyqtSignal(list)
def __init__(self,
box,
combobox,
add_documents_btn,
notification_bar,
parent=None):
"""
Handles the supporting documents component loading.
:param box: The layout holding the container widget.
:type box: QVBoxLayout
:param combobox: The combobox loading supporting document types.
:type combobox: QComboBox
:param add_documents_btn: The add supporting document button
:type add_documents_btn: QPushButton
:param notification_bar: The NotificationBar object that displays
notification.
:type notification_bar: Object
:param parent: The container of the widget
:type parent: QDialog or None
"""
ComponentUtility.__init__(self)
self._parent = parent
self.container_box = box
self.doc_type_cbo = combobox
self.notification_bar = notification_bar
self.str_number = 1
self.add_documents_btn = add_documents_btn
self.str_numbers = [1]
self.current_party_count = None
self.init_documents()
def init_documents(self):
"""
Initializes the document type combobox by
populating data.
"""
self.supporting_doc_manager = SourceDocumentManager(
self.social_tenure.supporting_doc, self.str_doc_model,
self._parent)
self.create_doc_tab_populate_combobox()
self.doc_type_cbo.currentIndexChanged.connect(
self.match_doc_combo_to_tab)
self.docs_tab.currentChanged.connect(self.match_doc_tab_to_combo)
def party_count(self, count):
"""
A setter for current_party_count that is used to determined
the number of copies for each supporting document.
:param count: The number of currently added party records.
:type count: Integer
"""
self.current_party_count = count
def create_doc_tab_populate_combobox(self):
"""
Creates the supporting document component widget.
"""
self.doc_tab_data()
self.docs_tab = QTabWidget()
self.docs_tab_index = OrderedDict()
for i, (id, doc) in enumerate(self.doc_types.iteritems()):
self.docs_tab_index[doc] = i
# the tab widget containing the document widget layout
# and the child of the tab.
tab_widget = QWidget()
tab_widget.setObjectName(doc)
# The layout of the tab widget
cont_layout = QVBoxLayout(tab_widget)
cont_layout.setObjectName(u'widget_layout_{}'.format(doc))
# the scroll area widget inside the tab widget.
scroll_area = QScrollArea(tab_widget)
scroll_area.setFrameShape(QFrame.NoFrame)
scroll_area.setObjectName(u'tab_scroll_area_{}'.format(doc))
layout_widget = QWidget()
# the widget the is under the scroll area content and
# the widget containing the document widget layout
# This widget is hidden and shown based on the STR number
layout_widget.setObjectName(u'widget_{}'.format(doc))
doc_widget_layout = QVBoxLayout(layout_widget)
doc_widget_layout.setObjectName(
u'doc_widget_layout_{}'.format(doc))
doc_widget = QWidget()
doc_widget.setObjectName(u'doc_widget_{}_{}'.format(
doc, self.str_number))
doc_widget_layout.addWidget(doc_widget)
# the layout containing document widget.
### This is the layout that is registered to add uploaded
# supporting documents widgets into.
tab_layout = QVBoxLayout(doc_widget)
tab_layout.setObjectName(u'layout_{}_{}'.format(
doc, self.str_number))
scroll_area.setWidgetResizable(True)
scroll_area.setWidget(layout_widget)
cont_layout.addWidget(scroll_area)
# Add the tab widget with the document
# type name to create a tab.
self.docs_tab.addTab(tab_widget, doc)
self.container_box.addWidget(self.docs_tab, 1)
if len(self.str_numbers) == 1:
self.doc_type_cbo.addItem(doc, id)
def doc_tab_data(self):
"""
Sets the document types in the social tenure entity.
"""
doc_entity = self.social_tenure. \
supporting_doc.document_type_entity
doc_type_model = entity_model(doc_entity)
docs = doc_type_model()
doc_type_list = docs.queryObject().all()
self.doc_types = [(doc.id, doc.value) for doc in doc_type_list]
self.doc_types = OrderedDict(self.doc_types)
def match_doc_combo_to_tab(self):
"""
Changes the active tab based on the
selected value of document type combobox.
"""
combo_text = self.doc_type_cbo.currentText()
if combo_text is not None and len(combo_text) > 0:
index = self.docs_tab_index[combo_text]
self.docs_tab.setCurrentIndex(index)
def match_doc_tab_to_combo(self):
"""
Changes the document type combobox value based on the
selected tab.
"""
doc_tab_index = self.docs_tab.currentIndex()
self.doc_type_cbo.setCurrentIndex(doc_tab_index)
@staticmethod
def hide_doc_widgets(widget, visibility):
"""
Hides or shows the visibility of the supporting document
container widgets.
:param widget: The widget to which the visibility is set.
:type widget: QWidget
:param visibility: A boolean to show or hide visibility.
True hides widget and False shows it.
:type visibility: Boolean
"""
widget.setHidden(visibility)
def update_container(self, str_number):
"""
Update the current supporting document widget container to be used.
:param str_number: The STR node number
:type str_number: Integer
"""
doc_text = self.doc_type_cbo.currentText()
cbo_index = self.doc_type_cbo.currentIndex()
doc_id = self.doc_type_cbo.itemData(cbo_index)
scroll_area = self.docs_tab.findChild(
QScrollArea, u'tab_scroll_area_{}'.format(doc_text, str_number))
doc_widget = scroll_area.findChild(
QWidget, u'doc_widget_{}_{}'.format(doc_text, str_number))
# If the doc widget doesn't exist create it for new STR instance
if doc_widget is None:
# find the doc_widget layout that contains
# all STR doc widget layouts. Single
# doc_widget_layout is created for each document type.
# But all doc_widgets for each STR instance and
# document types will be added here.
doc_widget_layout = scroll_area.findChild(
QVBoxLayout, u'doc_widget_layout_{}'.format(doc_text))
doc_widget = QWidget()
doc_widget.setObjectName(u'doc_widget_{}_{}'.format(
doc_text, str_number))
self.hide_all_other_widget(doc_text, str_number)
doc_widget_layout.addWidget(doc_widget)
# Create the layout so that layouts are registered in
# which uploaded document widgets are added.
layout = QVBoxLayout(doc_widget)
layout.setObjectName(u'layout_{}_{}'.format(doc_text, str_number))
# If the doc widget exists, get the lowest
# layout so that it is registered.
else:
# hide all other widgets
self.hide_all_other_widget(doc_text, str_number)
# show the current doc widget to display
# the document widgets for the current tab.
self.hide_doc_widgets(doc_widget, False)
layout = doc_widget.findChild(
QVBoxLayout, u'layout_{}_{}'.format(doc_text, str_number))
# register layout
self.supporting_doc_manager.registerContainer(layout, doc_id)
def hide_all_other_widget(self, doc_text, str_number):
"""
Hides all other supporting document widget except the current
STR node widget.
:param doc_text: The current document type selected.
:type doc_text: String
:param str_number: The STR node number
:type str_number: Integer
"""
expression = QRegExp(u'doc_widget*')
# hide all existing widgets in all layouts
for widget in self.docs_tab.findChildren(QWidget, expression):
if widget.objectName() != u'doc_widget_{}_{}'.format(
doc_text, str_number):
self.hide_doc_widgets(widget, True)
def on_upload_document(self):
'''
Slot raised when the user clicks
to upload a supporting document.
'''
document_str = QApplication.translate(
"SupportingDocuments", "Specify the Document File Location")
documents = self.select_file_dialog(document_str)
cbo_index = self.doc_type_cbo.currentIndex()
doc_id = self.doc_type_cbo.itemData(cbo_index)
for doc in documents:
self.supporting_doc_manager.insertDocumentFromFile(
doc, doc_id, self.social_tenure, self.current_party_count)
# Set last path
if len(documents) > 0:
doc = documents[0]
fi = QFileInfo(doc)
dir_path = fi.absolutePath()
set_last_document_path(dir_path)
model_objs = self.supporting_doc_manager.model_objects()
self.onUploadDocument.emit(model_objs)
def select_file_dialog(self, title):
"""
Displays a file dialog for a user to specify a source document
:param title: The title of the file dialog
:type title: String
"""
# Get last path for supporting documents
last_path = last_document_path()
if last_path is None:
last_path = '/home'
files = QFileDialog.getOpenFileNames(
iface.mainWindow(), title, last_path,
"Source Documents (*.jpg *.jpeg *.png *.bmp *.tiff *.svg)")
return files
class Save2DB:
"""
Class to insert entity data into db
"""
def __init__(self, entity, attributes, ids=None):
"""
Initialize class and class variable
"""
self.attributes = attributes
self.form_entity = entity
self.doc_model = None
self._doc_manager = None
self.entity = self.object_from_entity_name(self.form_entity)
self.model = self.dbmodel_from_entity()
self.key = 0
self.parents_ids = ids
self.geom = 4326
def object_from_entity_name(self, entity):
"""
:return:
"""
if entity == 'social_tenure':
return current_profile().social_tenure
else:
user_entity = current_profile().entity_by_name(entity)
return user_entity
def entity_has_supporting_docs(self):
"""
Check if the entity has supporting document before importing
:return: Bool
"""
return self.entity.supports_documents
def entity_supported_document_types(self):
"""
Get the supported document types before importing so that they are captured
during import process
:return: List
"""
return self.entity.document_types_non_hex()
def dbmodel_from_entity(self):
"""
Format model attributes from passed entity attributes
:return:
"""
if self.entity_has_supporting_docs():
entity_object, self.doc_model = entity_model(
self.entity, with_supporting_document=True)
entity_object_model = entity_object()
if hasattr(entity_object_model, 'documents'):
if self.entity.TYPE_INFO == 'SOCIAL_TENURE':
obj_doc_col = current_profile(
).social_tenure.supporting_doc
else:
obj_doc_col = self.entity.supporting_doc
self._doc_manager = SourceDocumentManager(
obj_doc_col, self.doc_model)
else:
entity_object = entity_model(self.entity)
entity_object_model = entity_object()
return entity_object_model
def objects_from_supporting_doc(self, instance_file=None):
"""
Create supporting doc path instances based on the collected documents
:return:paths
:rtype: document object instance
"""
if instance_file:
f_dir, file_name = os.path.split(instance_file)
for document, val in self.attributes.iteritems():
if str(document).endswith('supporting_document'):
if val != '':
doc = self.format_document_name_from_attribute(
document)
doc_path = os.path.normpath(f_dir + '/' + val)
abs_path = doc_path.replace('\\', '/').strip()
if QFile.exists(abs_path):
self.supporting_document_model(abs_path, doc)
def supporting_document_model(self, doc_path, doc):
"""
Construct supporting document model instance to add into the db
:return:
"""
# Create document container
doc_container = QVBoxLayout()
supporting_doc_entity = self.entity.supporting_doc.document_type_entity
document_type_id = entity_attr_to_id(supporting_doc_entity,
'value',
doc,
lower=False)
# Register container
self._doc_manager.registerContainer(doc_container, document_type_id)
#Copy the document to STDM working directory
self._doc_manager.insertDocumentFromFile(doc_path, document_type_id,
self.entity)
def format_document_name_from_attribute(self, key_name):
"""
Get the type of document from attribute name
So that supporting document class instance can save it in the right format
:return:
"""
doc_list = self.entity_supported_document_types()
default = 'General'
doc_type = str(key_name).split('_')
if len(doc_type) > 2:
if doc_type[0] in doc_list:
return doc_type[0]
else:
for doc in doc_list:
if doc.startswith(doc_type[0]):
return doc
elif len(doc_type) < 2 and key_name != default:
return key_name
else:
return default
def save_to_db(self):
"""
Format object attribute data from entity and save them into database
:return:
"""
try:
if self.parents_ids is not None and self.entity.short_name == 'social_tenure_relationship':
str_tables = current_profile().social_tenure
setattr(self.model,
str_tables.parties[0].short_name.lower() + '_id',
self.parents_ids.get(str_tables.parties[0].name)[0])
setattr(
self.model,
str_tables.spatial_units[0].short_name.lower() + '_id',
self.parents_ids.get(str_tables.spatial_units[0].name)[0])
except:
pass
for k, v in self.attributes.iteritems():
if hasattr(self.model, k):
col_type = self.column_info().get(k)
col_prop = self.entity.columns[k]
var = self.attribute_formatter(col_type, col_prop, v)
setattr(self.model, k, var)
if self.entity_has_supporting_docs():
self.model.documents = self._doc_manager.model_objects()
self.model.save()
return self.model.id
#self.cleanup()
def save_parent_to_db(self):
"""
Format object attribute data from entity and save them into database
attribute
:return:
"""
for k, v in self.attributes.iteritems():
if hasattr(self.model, k):
col_type = self.column_info().get(k)
col_prop = self.entity.columns[k]
var = self.attribute_formatter(col_type, col_prop, v)
setattr(self.model, k, var)
if self.entity_has_supporting_docs():
self.model.documents = self._doc_manager.model_objects()
self.model.save()
self.key = self.model.id
return self.key
#def model_object_formatter(self):
def column_info(self):
"""
:return:
"""
type_mapping = {}
cols = self.entity.columns.values()
for c in cols:
type_mapping[c.name] = c.TYPE_INFO
return type_mapping
def get_srid(self, srid):
"""
Let the user specify the coordinate system during data import
:param srid:
:return:
"""
self.geom = srid
return self.geom
def id_from_model_object(self, obj):
"""
We need to obtian id from object instance
:param obj:
:return:
"""
return obj.id
def attribute_formatter(self, col_type, col_prop, var):
"""
:return:
"""
if col_type == 'LOOKUP':
if var == '' or var is None:
return None
if var == 'Yes' or var == 'No':
return entity_attr_to_model(col_prop.parent, 'value', var).id
if not len(var) > 3 and var != 'Yes' and var != 'No':
lk_code = entity_attr_to_id(col_prop.parent, "code", var)
if not str(lk_code).isdigit():
return None
else:
return lk_code
if len(var) > 3:
if not str(entity_attr_to_id(col_prop.parent, 'code',
var)).isdigit():
return entity_attr_to_model(col_prop.parent, 'value',
var).id
else:
lk_code = entity_attr_to_id(col_prop.parent, "code", var)
if not str(lk_code).isdigit():
return None
else:
return lk_code
else:
return None
elif col_type == 'ADMIN_SPATIAL_UNIT':
if not len(var) > 3:
return entity_attr_to_id(col_prop.parent, "code", var)
else:
return entity_attr_to_id(col_prop.parent, "name", var)
elif col_type == 'MULTIPLE_SELECT':
if var == '' or var is None:
return None
if not len(var) > 3:
return entity_attr_to_id(col_prop.association.first_parent,
"code", var)
elif len(var) > 3:
if not str(
entity_attr_to_id(col_prop.association.first_parent,
"code", var)).isdigit():
return entity_attr_to_model(
col_prop.association.first_parent, 'value', var).id
else:
return entity_attr_to_id(col_prop.association.first_parent,
"code", var)
elif col_type == 'GEOMETRY':
defualt_srid = 0
geom_provider = STDMGeometry(var)
if isinstance(col_prop, GeometryColumn):
defualt_srid = col_prop.srid
if defualt_srid != 0:
geom_provider.set_user_srid(defualt_srid)
else:
geom_provider.set_user_srid(GEOMPARAM)
if col_prop.geometry_type() == 'POINT':
return geom_provider.point_to_Wkt()
if col_prop.geometry_type() == 'POLYGON':
return geom_provider.polygon_to_Wkt()
elif col_type == 'FOREIGN_KEY':
if self.parents_ids is None or len(self.parents_ids) < 0:
return
else:
for code, val in self.parents_ids.iteritems():
if code is not None:
if val[1] == GROUPCODE and col_prop.parent.name == code:
return val[0]
else:
if col_prop.parent.name == code:
return val[0]
elif col_type == 'INT' or col_type == 'DOUBLE':
if var == '':
return 0
else:
return var
else:
return var
def cleanup(self):
"""
Reset all the model and entity data before we process
the next entity data
:return: None
z"""
self.model = None
self.entity = None
self.attributes = None
self._doc_manager = None
class SupportingDocuments(ComponentUtility):
onUploadDocument = pyqtSignal(list)
def __init__(self, box, combobox, add_documents_btn, notification_bar, parent=None):
"""
Handles the supporting documents component loading.
:param box: The layout holding the container widget.
:type box: QVBoxLayout
:param combobox: The combobox loading supporting document types.
:type combobox: QComboBox
:param add_documents_btn: The add supporting document button
:type add_documents_btn: QPushButton
:param notification_bar: The NotificationBar object that displays
notification.
:type notification_bar: Object
:param parent: The container of the widget
:type parent: QDialog or None
"""
ComponentUtility.__init__(self)
self._parent = parent
self.container_box = box
self.doc_type_cbo = combobox
self.notification_bar = notification_bar
self.str_number = 1
self.add_documents_btn = add_documents_btn
self.str_numbers = [1]
self.current_party_count = None
self.init_documents()
def init_documents(self):
"""
Initializes the document type combobox by
populating data.
"""
self.supporting_doc_manager = SourceDocumentManager(
self.social_tenure.supporting_doc,
self.str_doc_model,
self._parent
)
self.create_doc_tab_populate_combobox()
self.doc_type_cbo.currentIndexChanged.connect(
self.match_doc_combo_to_tab
)
self.docs_tab.currentChanged.connect(
self.match_doc_tab_to_combo
)
def party_count(self, count):
"""
A setter for current_party_count that is used to determined
the number of copies for each supporting document.
:param count: The number of currently added party records.
:type count: Integer
"""
self.current_party_count = count
def create_doc_tab_populate_combobox(self):
"""
Creates the supporting document component widget.
"""
self.doc_tab_data()
self.docs_tab = QTabWidget()
self.docs_tab_index = OrderedDict()
for i, (id, doc) in enumerate(self.doc_types.iteritems()):
self.docs_tab_index[doc] = i
# the tab widget containing the document widget layout
# and the child of the tab.
tab_widget = QWidget()
tab_widget.setObjectName(doc)
# The layout of the tab widget
cont_layout = QVBoxLayout(tab_widget)
cont_layout.setObjectName(
u'widget_layout_{}'.format(doc)
)
# the scroll area widget inside the tab widget.
scroll_area = QScrollArea(tab_widget)
scroll_area.setFrameShape(QFrame.NoFrame)
scroll_area.setObjectName(
u'tab_scroll_area_{}'.format(doc)
)
layout_widget = QWidget()
# the widget the is under the scroll area content and
# the widget containing the document widget layout
# This widget is hidden and shown based on the STR number
layout_widget.setObjectName(
u'widget_{}'.format(doc)
)
doc_widget_layout = QVBoxLayout(layout_widget)
doc_widget_layout.setObjectName(
u'doc_widget_layout_{}'.format(
doc
)
)
doc_widget = QWidget()
doc_widget.setObjectName(
u'doc_widget_{}_{}'.format(doc, self.str_number)
)
doc_widget_layout.addWidget(doc_widget)
# the layout containing document widget.
### This is the layout that is registered to add uploaded
# supporting documents widgets into.
tab_layout = QVBoxLayout(doc_widget)
tab_layout.setObjectName(
u'layout_{}_{}'.format(doc, self.str_number)
)
scroll_area.setWidgetResizable(True)
scroll_area.setWidget(layout_widget)
cont_layout.addWidget(scroll_area)
# Add the tab widget with the document
# type name to create a tab.
self.docs_tab.addTab(tab_widget, doc)
self.container_box.addWidget(self.docs_tab, 1)
if len(self.str_numbers) == 1:
self.doc_type_cbo.addItem(doc, id)
def doc_tab_data(self):
"""
Sets the document types in the social tenure entity.
"""
doc_entity = self.social_tenure. \
supporting_doc.document_type_entity
doc_type_model = entity_model(doc_entity)
docs = doc_type_model()
doc_type_list = docs.queryObject().all()
self.doc_types = [(doc.id, doc.value)
for doc in doc_type_list
]
self.doc_types = OrderedDict(self.doc_types)
def match_doc_combo_to_tab(self):
"""
Changes the active tab based on the
selected value of document type combobox.
"""
combo_text = self.doc_type_cbo.currentText()
if combo_text is not None and len(combo_text) > 0:
index = self.docs_tab_index[combo_text]
self.docs_tab.setCurrentIndex(index)
def match_doc_tab_to_combo(self):
"""
Changes the document type combobox value based on the
selected tab.
"""
doc_tab_index = self.docs_tab.currentIndex()
self.doc_type_cbo.setCurrentIndex(doc_tab_index)
@staticmethod
def hide_doc_widgets(widget, visibility):
"""
Hides or shows the visibility of the supporting document
container widgets.
:param widget: The widget to which the visibility is set.
:type widget: QWidget
:param visibility: A boolean to show or hide visibility.
True hides widget and False shows it.
:type visibility: Boolean
"""
widget.setHidden(visibility)
def update_container(self, str_number):
"""
Update the current supporting document widget container to be used.
:param str_number: The STR node number
:type str_number: Integer
"""
doc_text = self.doc_type_cbo.currentText()
cbo_index = self.doc_type_cbo.currentIndex()
doc_id = self.doc_type_cbo.itemData(cbo_index)
scroll_area = self.docs_tab.findChild(
QScrollArea, u'tab_scroll_area_{}'.format(
doc_text, str_number
)
)
doc_widget = scroll_area.findChild(
QWidget, u'doc_widget_{}_{}'.format(
doc_text, str_number
)
)
# If the doc widget doesn't exist create it for new STR instance
if doc_widget is None:
# find the doc_widget layout that contains
# all STR doc widget layouts. Single
# doc_widget_layout is created for each document type.
# But all doc_widgets for each STR instance and
# document types will be added here.
doc_widget_layout = scroll_area.findChild(
QVBoxLayout, u'doc_widget_layout_{}'.format(doc_text)
)
doc_widget = QWidget()
doc_widget.setObjectName(
u'doc_widget_{}_{}'.format(doc_text, str_number)
)
self.hide_all_other_widget(doc_text, str_number)
doc_widget_layout.addWidget(doc_widget)
# Create the layout so that layouts are registered in
# which uploaded document widgets are added.
layout = QVBoxLayout(doc_widget)
layout.setObjectName(u'layout_{}_{}'.format(
doc_text, str_number
)
)
# If the doc widget exists, get the lowest
# layout so that it is registered.
else:
# hide all other widgets
self.hide_all_other_widget(doc_text, str_number)
# show the current doc widget to display
# the document widgets for the current tab.
self.hide_doc_widgets(doc_widget, False)
layout = doc_widget.findChild(
QVBoxLayout, u'layout_{}_{}'.format(
doc_text, str_number
)
)
# register layout
self.supporting_doc_manager.registerContainer(
layout, doc_id
)
def hide_all_other_widget(self, doc_text, str_number):
"""
Hides all other supporting document widget except the current
STR node widget.
:param doc_text: The current document type selected.
:type doc_text: String
:param str_number: The STR node number
:type str_number: Integer
"""
expression = QRegExp(u'doc_widget*')
# hide all existing widgets in all layouts
for widget in self.docs_tab.findChildren(QWidget, expression):
if widget.objectName() != u'doc_widget_{}_{}'.format(
doc_text, str_number):
self.hide_doc_widgets(widget, True)
def on_upload_document(self):
'''
Slot raised when the user clicks
to upload a supporting document.
'''
document_str = QApplication.translate(
"SupportingDocuments",
"Specify the Document File Location"
)
documents = self.select_file_dialog(document_str)
cbo_index = self.doc_type_cbo.currentIndex()
doc_id = self.doc_type_cbo.itemData(cbo_index)
for doc in documents:
self.supporting_doc_manager.insertDocumentFromFile(
doc,
doc_id,
self.social_tenure,
self.current_party_count
)
# Set last path
if len(documents) > 0:
doc = documents[0]
fi = QFileInfo(doc)
dir_path = fi.absolutePath()
set_last_document_path(dir_path)
model_objs = self.supporting_doc_manager.model_objects()
self.onUploadDocument.emit(model_objs)
def select_file_dialog(self, title):
"""
Displays a file dialog for a user to specify a source document
:param title: The title of the file dialog
:type title: String
"""
# Get last path for supporting documents
last_path = last_document_path()
if last_path is None:
last_path = '/home'
files = QFileDialog.getOpenFileNames(
iface.mainWindow(),
title,
last_path,
"Source Documents (*.jpg *.jpeg *.png *.bmp *.tiff *.svg)"
)
return files
class OGRReader(object):
def __init__(self, source_file):
self._ds = ogr.Open(source_file)
self._targetGeomColSRID = -1
self._geomType = ''
self._dbSession = STDMDb.instance().session
self._mapped_cls = None
self._mapped_doc_cls = None
self._current_profile = current_profile()
self._source_doc_manager = None
def getLayer(self):
#Return the first layer in the data source
if self.isValid():
numLayers = self._ds.GetLayerCount()
if numLayers > 0:
return self._ds.GetLayer(0)
else:
return None
def getSpatialRefCode(self):
#Get the EPSG code (More work required)
if self.getLayer() != None:
spRef = self.getLayer().GetSpatialRef()
refCode = spRef.GetAttrValue("PRIMEM|AUTHORITY", 1)
else:
#Fallback to WGS84
refCode = 4326
return refCode
def isValid(self):
#Whether the open process succeeded or failed
if self._ds is None:
return False
else:
return True
def reset(self):
#Destroy
self._ds = None
self._geomType = ""
self._targetGeomColSRID = -1
def getFields(self):
#Return the data source's fields in a list
fields = []
lyr = self.getLayer()
lyr.ResetReading()
feat_defn = lyr.GetLayerDefn()
for l in range(feat_defn.GetFieldCount()):
field_defn = feat_defn.GetFieldDefn(l)
fields.append(str(field_defn.GetNameRef()))
return fields
def _data_source_entity(self, table_name):
entity = self._current_profile.entity_by_name(table_name)
return entity
def entity_virtual_columns(self, table_name):
"""
:param table_name: Name of the target table.
:type table_name: str
:return: Returns a list of derived columns for the specified target table.
:rtype: list
"""
entity = self._data_source_entity(table_name)
if entity is None:
return []
return entity.virtual_columns()
def _get_mapped_class(self, table_name):
#Get entity from the corresponding table name
entity = self._data_source_entity(table_name)
if entity is None:
return None
ent_model, doc_model = entity_model(entity,
with_supporting_document=True)
return ent_model, doc_model
def _insertRow(self, columnValueMapping):
"""
Insert a new row using the mapped class instance then mapping column
names to the corresponding column values.
"""
model_instance = self._mapped_cls()
for col, value in columnValueMapping.iteritems():
if hasattr(model_instance, col):
'''
#Check if column type is enumeration and transform accordingly
col_is_enum, enum_symbol = self._enumeration_column_type(col, value)
if col_is_enum:
value = enum_symbol
'''
if not isinstance(value, IgnoreType):
setattr(model_instance, col, value)
try:
self._dbSession.add(model_instance)
self._dbSession.commit()
except:
self._dbSession.rollback()
raise
def featToDb(self,
targettable,
columnmatch,
append,
parentdialog,
geomColumn=None,
geomCode=-1,
translator_manager=None):
"""
Performs the data import from the source layer to the STDM database.
:param targettable: Destination table name
:param columnmatch: Dictionary containing source columns as keys and target columns as the values.
:param append: True to append, false to overwrite by deleting previous records
:param parentdialog: A reference to the calling dialog.
:param translator_manager: Instance of 'stdm.data.importexport.ValueTranslatorManager'
containing value translators defined for the destination table columns.
:type translator_manager: ValueTranslatorManager
"""
#Check current profile
if self._current_profile is None:
msg = QApplication.translate(
'OGRReader',
'The current profile could not be determined.\nPlease set it '
'in the Options dialog or Configuration Wizard.')
raise ConfigurationException(msg)
if translator_manager is None:
translator_manager = ValueTranslatorManager()
#Delete existing rows in the target table if user has chosen to overwrite
if not append:
delete_table_data(targettable)
#Container for mapping column names to their corresponding values
column_value_mapping = {}
lyr = self.getLayer()
lyr.ResetReading()
feat_defn = lyr.GetLayerDefn()
numFeat = lyr.GetFeatureCount()
#Configure progress dialog
init_val = 0
progress = QProgressDialog("", "&Cancel", init_val, numFeat,
parentdialog)
progress.setWindowModality(Qt.WindowModal)
lblMsgTemp = "Importing {0} of {1} to STDM..."
#Set entity for use in translators
destination_entity = self._data_source_entity(targettable)
for feat in lyr:
column_count = 0
progress.setValue(init_val)
progressMsg = lblMsgTemp.format((init_val + 1), numFeat)
progress.setLabelText(progressMsg)
if progress.wasCanceled():
break
#Reset source document manager for new records
if destination_entity.supports_documents:
if not self._source_doc_manager is None:
self._source_doc_manager.reset()
for f in range(feat_defn.GetFieldCount()):
field_defn = feat_defn.GetFieldDefn(f)
field_name = field_defn.GetNameRef()
#Append value only if it has been defined by the user
if field_name in columnmatch:
dest_column = columnmatch[field_name]
field_value = feat.GetField(f)
#Create mapped class only once
if self._mapped_cls is None:
mapped_cls, mapped_doc_cls = self._get_mapped_class(
targettable)
if mapped_cls is None:
msg = QApplication.translate(
"OGRReader",
"Something happened that caused the "
"database table not to be mapped to the "
"corresponding model class. Please contact"
" your system administrator.")
raise RuntimeError(msg)
self._mapped_cls = mapped_cls
self._mapped_doc_cls = mapped_doc_cls
#Create source document manager if the entity supports them
if destination_entity.supports_documents:
self._source_doc_manager = SourceDocumentManager(
destination_entity.supporting_doc,
self._mapped_doc_cls)
if geomColumn is not None:
#Use geometry column SRID in the target table
self._geomType,self._targetGeomColSRID = \
geometryType(targettable,geomColumn)
'''
Check if there is a value translator defined for the
specified destination column.
'''
value_translator = translator_manager.translator(
dest_column)
if not value_translator is None:
#Set destination table entity
value_translator.entity = destination_entity
source_col_names = value_translator.source_column_names(
)
field_value_mappings = self._map_column_values(
feat, feat_defn, source_col_names)
#Set source document manager if required
if value_translator.requires_source_document_manager:
value_translator.source_document_manager = self._source_doc_manager
field_value = value_translator.referencing_column_value(
field_value_mappings)
if not isinstance(field_value, IgnoreType):
column_value_mapping[dest_column] = field_value
#Set supporting documents
if destination_entity.supports_documents:
column_value_mapping['documents'] = \
self._source_doc_manager.model_objects()
column_count += 1
#Only insert geometry if it has been defined by the user
if geomColumn is not None:
geom = feat.GetGeometryRef()
if geom is not None:
#Get geometry in WKT/WKB format
geomWkb = geom.ExportToWkt()
column_value_mapping[geomColumn] = "SRID={0!s};{1}".format(
self._targetGeomColSRID, geomWkb)
#Check if the geometry types match
layerGeomType = geom.GetGeometryName()
if layerGeomType.lower() != self._geomType.lower():
raise TypeError("The geometries of the source and destination columns do not match.\n" \
"Source Geometry Type: {0}, Destination Geometry Type: {1}".format(layerGeomType,
self._geomType))
return
try:
#Insert the record
self._insertRow(column_value_mapping)
except:
progress.close()
raise
init_val += 1
progress.setValue(numFeat)
def _enumeration_column_type(self, column_name, value):
"""
Checks if the given column is of DeclEnumType.
:param column_name: Name of the enumeration column.
:type column_name: str
:return: True if column is of DeclType and return Enum symbol;
Else, False and None.
:rtype: tuple
"""
try:
#Get column type of the enumeration
enum_col_type = self._mapped_cls.__mapper__.columns[
column_name].type
except KeyError:
return False, None
if not hasattr(enum_col_type, "enum"):
return False, None
else:
enum_obj = enum_col_type.enum
try:
if not isinstance(value, str) or not isinstance(
value, unicode):
value = unicode(value)
enum_symbol = enum_obj.from_string(value.strip())
except ValueError:
enum_symbol = IgnoreType()
return True, enum_symbol
def _map_column_values(self, feature, feature_defn, source_cols):
"""
Retrieves values for specific columns from the specified feature.
:param feature: Input feature.
:type feature: ogr.Feature
:param feature_defn: Feature definition for the layer.
:type feature_defn: ogr.FeatureDefn
:param source_cols: List of columns whose respective values will be
retrieved.
:type source_cols: list
:return: Collection containing pairs of column names and corresponding
values.
:rtype: dict
"""
col_values = {}
if len(source_cols) == 0:
return col_values
for f in range(feature_defn.GetFieldCount()):
field_defn = feature_defn.GetFieldDefn(f)
field_name = field_defn.GetNameRef()
match_idx = getIndex(source_cols, field_name)
if match_idx != -1:
field_value = feature.GetField(f)
col_values[field_name] = field_value
return col_values
class Save2DB:
"""
Class to insert entity data into db
"""
def __init__(self, entity, attributes, ids=None):
"""
Initialize class and class variable
"""
self.attributes = attributes
self.form_entity = entity
self.doc_model = None
self._doc_manager =None
self.entity = self.object_from_entity_name(self.form_entity)
self.model = self.dbmodel_from_entity()
self.key = 0
self.parents_ids = ids
self.geom = 4326
self.entity_mapping = {}
def object_from_entity_name(self, entity):
"""
:return:
"""
if entity == 'social_tenure':
return current_profile().social_tenure
else:
user_entity = current_profile().entity_by_name(entity)
return user_entity
def entity_has_supporting_docs(self):
"""
Check if the entity has supporting document before importing
:return: Bool
"""
if self.entity.supports_documents:
return self.entity.supports_documents
else:
return None
def entity_supported_document_types(self):
"""
Get the supported document types before importing so that they are captured
during import process
:return: List
"""
return self.entity.document_types_non_hex()
def dbmodel_from_entity(self):
"""
Format model attributes from passed entity attributes
:return:
"""
if self.entity_has_supporting_docs():
entity_object, self.doc_model = entity_model(self.entity, with_supporting_document=True)
entity_object_model = entity_object()
if hasattr(entity_object_model, 'documents'):
if self.entity.TYPE_INFO == 'SOCIAL_TENURE':
obj_doc_col = current_profile().social_tenure.supporting_doc
else:
obj_doc_col = self.entity.supporting_doc
self._doc_manager = SourceDocumentManager(
obj_doc_col, self.doc_model
)
else:
entity_object = entity_model(self.entity)
entity_object_model = entity_object()
return entity_object_model
def objects_from_supporting_doc(self, instance_file = None):
"""
Create supporting doc path instances based on the collected documents
:return:paths
:rtype: document object instance
"""
if instance_file:
f_dir, file_name = os.path.split(instance_file)
for document, val in self.attributes.iteritems():
if str(document).endswith('supporting_document'):
if val != '':
doc = self.format_document_name_from_attribute(document)
doc_path = os.path.normpath(f_dir+'/'+val)
abs_path = doc_path.replace('\\','/').strip()
if QFile.exists(abs_path):
self.supporting_document_model(abs_path, doc)
def supporting_document_model(self,doc_path,doc):
"""
:param doc_path: absolute document path
:param doc: document name
:type: str
Construct supporting document model instance to add into the db
:return:
"""
# Create document container
doc_container = QVBoxLayout()
supporting_doc_entity = self.entity.supporting_doc.document_type_entity
document_type_id = entity_attr_to_id(supporting_doc_entity, 'value', doc, lower=False)
# Register container
self._doc_manager.registerContainer(
doc_container,
document_type_id
)
#Copy the document to STDM working directory
self._doc_manager.insertDocumentFromFile(
doc_path,
document_type_id,
self.entity
)
def format_document_name_from_attribute(self, doc):
"""
Get the type of document from attribute name
So that supporting document class instance can save it in the right format
:return:
"""
formatted_doc_list = self.entity_supported_document_types()
default = 'General'
doc_type = str(doc).split('_',1)
if doc_type[0].startswith('supporting') and formatted_doc_list[0] == default:
return default
elif doc_type[0].startswith('supporting') and formatted_doc_list[0] != default:
return formatted_doc_list[0]
elif not doc_type[0].startswith('supporting'):
actual_doc_name = doc_type[0].replace('-', ' ')
for doc_name in formatted_doc_list:
if actual_doc_name in doc_name or doc_name.startswith(actual_doc_name):
return doc_name
else:
return formatted_doc_list[0]
else:
return formatted_doc_list[0]
def extract_social_tenure_entities(self):
'''
We want to extract social tenure enities so that we know if it has multiple or single
entities in the list
:return:
'''
party_ref_column = ''
spatial_ref_column = ''
if self.parents_ids is not None:
print self.parents_ids
if self.attributes.has_key('party'):
full_party_ref_column = self.attributes.get('party')
party_ref_column = full_party_ref_column + '_id'
print 'party{}.'.format(party_ref_column)
setattr(self.model, party_ref_column, self.parents_ids.get(full_party_ref_column)[0])
if self.attributes.has_key('spatial_unit'):
full_spatial_ref_column = self.attributes.get('spatial_unit')
spatial_ref_column = full_spatial_ref_column + '_id'
print 'sp.{}.'.format(spatial_ref_column)
setattr(self.model, spatial_ref_column, self.parents_ids.get(full_spatial_ref_column)[0])
return party_ref_column, spatial_ref_column
def save_to_db(self):
"""
Format object attribute data from entity and save them into database
:return:
"""
self.column_info()
try:
if self.entity.short_name == 'social_tenure_relationship':
#try:
prefix = current_profile().prefix + '_'
if self.attributes.has_key('party'):
full_party_ref_column = self.attributes.get('party')
party_ref_column = full_party_ref_column + '_id'
self.attributes.pop('party')
else:
full_party_ref_column = current_profile().social_tenure.parties[0].name
party_ref_column = full_party_ref_column + '_id'
setattr(self.model, party_ref_column, self.parents_ids.get(prefix + full_party_ref_column)[0])
if self.attributes.has_key('spatial_unit'):
full_spatial_ref_column = self.attributes.get('spatial_unit')
spatial_ref_column = full_spatial_ref_column + '_id'
self.attributes.pop('spatial_unit')
else:
full_spatial_ref_column = current_profile().social_tenure.spatial_units[0].name
spatial_ref_column = full_spatial_ref_column + '_id'
setattr(self.model, spatial_ref_column, self.parents_ids.get(prefix + full_spatial_ref_column)[0])
except:
pass
for k, v in self.attributes.iteritems():
if hasattr(self.model, k):
col_type = self.entity_mapping.get(k)
col_prop = self.entity.columns[k]
var = self.attribute_formatter(col_type, col_prop, v)
setattr(self.model, k, var)
if self.entity_has_supporting_docs():
self.model.documents = self._doc_manager.model_objects()
self.model.save()
return self.model.id
def save_parent_to_db(self):
"""
Format object attribute data from entity and save them into database
attribute
:return:
"""
self.column_info()
for k, v in self.attributes.iteritems():
if hasattr(self.model, k):
col_type = self.entity_mapping.get(k)
col_prop = self.entity.columns[k]
#print "property{0}.... and type.{1}".format(col_prop, col_type)
var = self.attribute_formatter(col_type, col_prop, v)
setattr(self.model, k, var)
if self.entity_has_supporting_docs():
self.model.documents = self._doc_manager.model_objects()
self.model.save()
self.key = self.model.id
return self.key
def save_foreign_key_table(self):
"""
Get the table with foreign keys only
:return:
"""
for col, type_info in self.column_info().iteritems():
col_prop = self.entity.columns[col]
var = self.attribute_formatter(type_info, col_prop, None)
setattr(self.model, col, var)
self.model.save()
self.cleanup()
def column_info(self):
"""
:return:
"""
self.entity_mapping = {}
cols = self.entity.columns.values()
for c in cols:
self.entity_mapping[c.name] = c.TYPE_INFO
def get_srid(self, srid):
"""
Let the user specify the coordinate system during data import
:param srid:
:return:
"""
self.geom = srid
return self.geom
def id_from_model_object(self, obj):
"""
We need to obtian id from object instance
:param obj:
:return:
"""
return obj.id
def attribute_formatter(self, col_type, col_prop, var=None):
"""
Format geoodk attributes collected in the field
to conform to STDM database contrains
:return:
"""
if col_type == 'BOOL':
if len(var) < 1:
return None
if len(var)>1:
if var == '' or var is None:
return None
if var == 'Yes' or var == True:
return True
if var == 'No' or var == False:
return False
else:
return None
if col_type == 'LOOKUP':
if len(var) < 1 or var is None:
return None
if len(var) <4:
if var == 'Yes' or var == 'No':
return entity_attr_to_model(col_prop.parent, 'value', var).id
if var != 'Yes' and var != 'No':
lk_code = entity_attr_to_id(col_prop.parent, "code", var)
if not str(lk_code).isdigit():
return None
else:
return lk_code
if len(var) > 3:
if not str(entity_attr_to_id(col_prop.parent, 'code', var)).isdigit():
id_value = entity_attr_to_model(col_prop.parent, 'value', var)
if id_value is not None:
return id_value.id
#return entity_attr_to_model(col_prop.parent, 'value', var).id
else:
lk_code = entity_attr_to_id(col_prop.parent, "code", var)
if not str(lk_code).isdigit():
return None
else:
return lk_code
else:
return None
elif col_type == 'ADMIN_SPATIAL_UNIT':
var_code = None
try:
if len(var) < 1 or var is None:
return None
elif not len(var) > 3:
var_code = entity_attr_to_id(col_prop.parent, "code", var)
if var_code and var_code == var:
return None
else:
return var_code
elif len(var) > 3 and entity_attr_to_id(col_prop.parent, "name", var) is not None:
var_code = entity_attr_to_id(col_prop.parent, "name", var)
if var_code and var_code == var:
return None
else:
return var_code
else:
if entity_attr_to_id(col_prop.parent, "name", var) is None:
var_code = entity_attr_to_id(col_prop.parent, "code", var)
if not var_code or var_code == var:
return None
except:
pass
elif col_type == 'MULTIPLE_SELECT':
print 'multiple select {}'.format(var)
if var == '' or var is None:
return None
else:
print var
col_parent = col_prop.association.first_parent
lk_val_list = col_parent.values.values()
choices_list = []
for code in lk_val_list:
choices_list.append(entity_attr_to_id(
col_parent.association.first_parent, 'value', code.value))
print choices_list
if len(choices_list) > 1:
return choices_list
else:
return None
elif col_type == 'GEOMETRY':
defualt_srid = 0
if var:
geom_provider = STDMGeometry(var)
if isinstance(col_prop, GeometryColumn):
defualt_srid = col_prop.srid
if defualt_srid != 0:
geom_provider.set_user_srid(defualt_srid)
else:
geom_provider.set_user_srid(GEOMPARAM)
if col_prop.geometry_type() == 'POINT':
return geom_provider.point_to_Wkt()
if col_prop.geometry_type() == 'POLYGON':
return geom_provider.polygon_to_Wkt()
else:
return None
elif col_type == 'FOREIGN_KEY':
if self.parents_ids is None or len(self.parents_ids) < 0:
return None
else:
for code, val in self.parents_ids.iteritems():
if col_prop.parent.name == code:
return val[0]
else:
return None
elif col_type == 'INT' or col_type == 'DOUBLE' or col_type == 'PERCENT':
if var == '':
return None
else:
return var
elif col_type == 'DATETIME' or col_type == 'DATE':
if var is None:
return None
if var == '':
return None
else:
return var
else:
return var
def cleanup(self):
"""
Reset all the model and entity data before we process
the next entity data
:return: None
z"""
self.model = None
self.entity = None
self.attributes = None
self._doc_manager = None
