def read_stc(self, config_file_name):
"""
Reads provided config file
:returns QDomDocument, QDomDocument.documentElement()
:rtype tuple
"""
config_file_path = os.path.join(
self.file_handler.localPath(),
config_file_name
)
config_file_path = QFile(config_file_path)
config_file = os.path.basename(config_file_name)
if self.check_config_file_exists(config_file):
self.document = QDomDocument()
status, msg, line, col = self.document.setContent(config_file_path)
if not status:
error_message = 'Configuration file cannot be loaded: {0}'. \
format(msg)
self.append_log(str(error_message))
raise ConfigurationException(error_message)
self.doc_element = self.document.documentElement()
def start_date(self, start_date_range):
"""
Set the minimum and maximum validity start dates.
:param start_date_range: A tuple containing the minimum and maximum
dates respectively. This will only be applied in the database if the
columns have not yet been created.
.. versionadded:: 1.5
:type start_date_range: tuple(min_start_date, max_start_date)
"""
if len(start_date_range) < 2:
raise ConfigurationException(
'A tuple of minimum and maximum start dates expected.')
min_date, max_date = start_date_range[0], start_date_range[1]
if min_date > max_date:
raise ConfigurationException(
'Minimum start date is greater than maximum start date.')
self.validity_start_column.minimum = min_date
self.validity_start_column.maximum = max_date
def featToDb(self,
targettable,
columnmatch,
append,
parentdialog,
geomColumn=None,
geomCode=-1,
translator_manager=None):
"""
Performs the data import from the source layer to the STDM database.
:param targettable: Destination table name
:param columnmatch: Dictionary containing source columns as keys and target columns as the values.
:param append: True to append, false to overwrite by deleting previous records
:param parentdialog: A reference to the calling dialog.
:param translator_manager: Instance of 'stdm.data.importexport.ValueTranslatorManager'
containing value translators defined for the destination table columns.
:type translator_manager: ValueTranslatorManager
"""
# Check current profile
if self._current_profile is None:
msg = QApplication.translate(
'OGRReader',
'The current profile could not be determined.\nPlease set it '
'in the Options dialog or Configuration Wizard.')
raise ConfigurationException(msg)
if translator_manager is None:
translator_manager = ValueTranslatorManager()
# Delete existing rows in the target table if user has chosen to overwrite
if not append:
delete_table_data(targettable)
# Container for mapping column names to their corresponding values
lyr = self.getLayer()
lyr.ResetReading()
feat_defn = lyr.GetLayerDefn()
numFeat = lyr.GetFeatureCount()
# Configure progress dialog
init_val = 0
progress = QProgressDialog("", "&Cancel", init_val, numFeat,
parentdialog)
progress.setWindowModality(Qt.WindowModal)
lblMsgTemp = "Importing {0} of {1} to STDM..."
# Set entity for use in translators
destination_entity = self._data_source_entity(targettable)
for feat in lyr:
column_value_mapping = {}
column_count = 0
progress.setValue(init_val)
progressMsg = lblMsgTemp.format((init_val + 1), numFeat)
progress.setLabelText(progressMsg)
if progress.wasCanceled():
break
# Reset source document manager for new records
if destination_entity.supports_documents:
if not self._source_doc_manager is None:
self._source_doc_manager.reset()
for f in range(feat_defn.GetFieldCount()):
field_defn = feat_defn.GetFieldDefn(f)
field_name = field_defn.GetNameRef()
# Append value only if it has been defined by the user
if field_name in columnmatch:
dest_column = columnmatch[field_name]
field_value = feat.GetField(f)
# Create mapped class only once
if self._mapped_cls is None:
mapped_cls, mapped_doc_cls = self._get_mapped_class(
targettable)
if mapped_cls is None:
msg = QApplication.translate(
"OGRReader",
"Something happened that caused the "
"database table not to be mapped to the "
"corresponding model class. Please contact"
" your system administrator.")
raise RuntimeError(msg)
self._mapped_cls = mapped_cls
self._mapped_doc_cls = mapped_doc_cls
# Create source document manager if the entity supports them
if destination_entity.supports_documents:
self._source_doc_manager = SourceDocumentManager(
destination_entity.supporting_doc,
self._mapped_doc_cls)
if geomColumn is not None:
# Use geometry column SRID in the target table
self._geomType, self._targetGeomColSRID = \
geometryType(targettable, geomColumn)
'''
Check if there is a value translator defined for the
specified destination column.
'''
value_translator = translator_manager.translator(
dest_column)
if value_translator is not None:
# Set destination table entity
value_translator.entity = destination_entity
source_col_names = value_translator.source_column_names(
)
field_value_mappings = self._map_column_values(
feat, feat_defn, source_col_names)
# Set source document manager if required
if value_translator.requires_source_document_manager:
value_translator.source_document_manager = self._source_doc_manager
field_value = value_translator.referencing_column_value(
field_value_mappings)
if not isinstance(field_value, IgnoreType):
# Check column type and rename if multiple select for
# SQLAlchemy compatibility
col_obj = destination_entity.column(dest_column)
if col_obj.TYPE_INFO == 'MULTIPLE_SELECT':
lk_name = col_obj.value_list.name
dest_column = '{0}_collection'.format(lk_name)
column_value_mapping[dest_column] = field_value
# Set supporting documents
if destination_entity.supports_documents:
column_value_mapping['documents'] = \
self._source_doc_manager.model_objects()
column_count += 1
# Only insert geometry if it has been defined by the user
if geomColumn is not None:
geom = feat.GetGeometryRef()
if geom is not None:
# Check if the geometry types match
layerGeomType = geom.GetGeometryName()
# Convert polygon to multipolygon if the destination table is multi-polygon.
geom_wkb, geom_type = self.auto_fix_geom_type(
geom, layerGeomType, self._geomType)
column_value_mapping[geomColumn] = "SRID={0!s};{1}".format(
self._targetGeomColSRID, geom_wkb)
if geom_type.lower() != self._geomType.lower():
raise TypeError(
"The geometries of the source and destination columns do not match.\n" \
"Source Geometry Type: {0}, Destination Geometry Type: {1}".format(
geom_type,
self._geomType))
try:
# Insert the record
self._insertRow(targettable, column_value_mapping)
except:
progress.close()
raise
init_val += 1
progress.setValue(numFeat)
def _entity_select_column(entity,
use_inner_join=False,
join_parents=False,
is_primary=False,
foreign_key_parents=None,
omit_view_columns=None,
omit_join_statement_columns=None):
# Check if the entity exists in the database
if not pg_table_exists(entity.name):
msg = u'{0} table does not exist, social tenure view will not be ' \
u'created.'.format(entity.name)
LOGGER.debug(msg)
raise ConfigurationException(msg)
if omit_view_columns is None:
omit_view_columns = []
if omit_join_statement_columns is None:
omit_join_statement_columns = []
column_names = []
join_statements = []
columns = entity.columns.values()
# Create foreign key parent collection if none is specified
if foreign_key_parents is None:
foreign_key_parents = {}
str_entity = entity.profile.social_tenure
for c in columns:
if c.TYPE_INFO not in _exclude_view_column_types:
normalized_entity_sname = entity.short_name.replace(' ',
'_').lower()
pseudo_column_name = u'{0}_{1}'.format(normalized_entity_sname,
c.name)
col_select_name = u'{0}.{1}'.format(entity.name, c.name)
select_column_name = u'{0} AS {1}'.format(col_select_name,
pseudo_column_name)
if is_primary and c.name == 'id':
# add row number id instead of party.id
# if multi_party is allowed.
if str_entity.multi_party:
if not entity.has_geometry_column():
row_id = 'row_number() OVER () AS id'
column_names.append(row_id)
select_column_name = select_column_name
else:
# add spatial unit id as the id.
select_column_name = col_select_name
# add the social_tenure_relationship_id
str_id = u'{0}.id AS {1}_id'.format(
str_entity.name, str_entity.short_name.lower())
column_names.append(str_id)
else:
# add party_id on spatial unit view to use
# [party]_supporting_document for
# profiles with one party entity and no multi_party.
if len(str_entity.parties) == 1 and not str_entity.multi_party and \
entity.has_geometry_column():
party_id = '{}_id'.format(
str_entity.parties[0].short_name.lower().replace(
' ', '_'))
str_party_id = u'{0}.{1} AS {1}'.format(
str_entity.name, party_id)
column_names.append(str_party_id)
select_column_name = col_select_name
# if entity has a geometry column, even if not multi_party
# add social_tenure_relationship_id
if entity.has_geometry_column():
# add the social_tenure_relationship_id
str_id = u'{0}.id AS {1}_id'.format(
str_entity.name, str_entity.short_name.lower())
column_names.append(str_id)
# Use custom join flag
use_custom_join = False
if isinstance(c, ForeignKeyColumn) and join_parents:
LOGGER.debug('Creating STR: Getting parent for %s column',
c.name)
fk_parent_entity = c.entity_relation.parent
parent_table = c.entity_relation.parent.name
LOGGER.debug('Parent found')
select_column_name = ''
# Handle renaming of parent table names to appropriate
# pseudonames.
if not parent_table in foreign_key_parents:
foreign_key_parents[parent_table] = []
pseudo_names = foreign_key_parents.get(parent_table)
# Get pseudoname to use
table_pseudo_name = u'{0}_{1}'.format(parent_table,
(len(pseudo_names) + 1))
pseudo_names.append(table_pseudo_name)
# Map lookup and admin unit values by default
if c.TYPE_INFO == 'LOOKUP':
select_column_name = u'{0}.value AS {1}'.format(
table_pseudo_name, pseudo_column_name)
use_custom_join = True
# Check if the column is for tenure type
if c.name != 'tenure_type':
use_inner_join = False
elif c.TYPE_INFO == 'ADMIN_SPATIAL_UNIT':
select_column_name = u'{0}.name AS {1}'.format(
table_pseudo_name, pseudo_column_name)
use_custom_join = True
use_inner_join = False
# These are outer joins
join_type = 'LEFT JOIN'
# Use inner join only if parent entity is an STR entity
if use_inner_join and \
str_entity.is_str_entity(fk_parent_entity):
join_type = 'INNER JOIN'
if use_custom_join:
join_statement = u'{0} {1} {2} ON {3} = {2}.{4}'.format(
join_type, parent_table, table_pseudo_name,
col_select_name, c.entity_relation.parent_column)
else:
join_statement = u'{0} {1} ON {2} = {1}.{3}'.format(
join_type, parent_table, col_select_name,
c.entity_relation.parent_column)
# Assert if the column is in the list of omitted join columns
if c.name not in omit_join_statement_columns:
join_statements.append(join_statement)
# Assert if the column is in the list of omitted view columns
if c.name not in omit_view_columns:
if select_column_name:
column_names.append(select_column_name)
return column_names, join_statements
def _entity_select_column(entity,
use_inner_join=False,
join_parents=False,
is_primary=False,
foreign_key_parents=None,
omit_view_columns=None,
omit_join_statement_columns=None,
view_name=None):
# Check if the entity exists in the database
if not pg_table_exists(entity.name):
msg = '{0} table does not exist, social tenure view will not be ' \
'created.'.format(entity.name)
LOGGER.debug(msg)
raise ConfigurationException(msg)
if omit_view_columns is None:
omit_view_columns = []
if omit_join_statement_columns is None:
omit_join_statement_columns = []
column_names = []
join_statements = []
columns = list(entity.columns.values())
# Create foreign key parent collection if none is specified
if foreign_key_parents is None:
foreign_key_parents = {}
str_entity = entity.profile.social_tenure
if entity in str_entity.custom_attribute_entities.values():
custom_tenure = True
else:
custom_tenure = False
i = 0
parents = []
for c in columns:
if c.TYPE_INFO not in _exclude_view_column_types:
normalized_entity_sname = entity.short_name.replace(' ',
'_').lower()
pseudo_column_name = '{0}_{1}'.format(normalized_entity_sname,
c.name)
# use sudo name for custom tenure entity
if custom_tenure:
col_select_name = '{0}_1.{1}'.format(entity.name, c.name)
else:
col_select_name = '{0}.{1}'.format(entity.name, c.name)
# Get pseudoname to use
select_column_name = '{0} AS {1}'.format(col_select_name,
pseudo_column_name)
if is_primary and c.name == 'id':
# add row number id instead of party.id
# if multi_party is allowed.
if str_entity.multi_party:
# for party entity add use row number
if not entity.has_geometry_column():
row_id = 'row_number() OVER () AS id'
select_column_name = row_id
else:
# add spatial unit id as the id.
col_spatial_unit_id = '{0}.{1} AS {1}'.format(
entity.name, c.name)
select_column_name = col_spatial_unit_id
else:
# add party id or spatial unit as id
entity_id = '{0}.{1} AS {1}'.format(entity.name, c.name)
select_column_name = entity_id
# Use custom join flag
use_custom_join = False
if isinstance(c, ForeignKeyColumn) and join_parents:
LOGGER.debug('Creating STR: Getting parent for %s column',
c.name)
fk_parent_entity = c.entity_relation.parent
parent_table = c.entity_relation.parent.name
LOGGER.debug('Parent found')
# Handle renaming of parent table names to appropriate
# pseudonames.
if parent_table not in foreign_key_parents:
foreign_key_parents[parent_table] = []
pseudo_names = foreign_key_parents.get(parent_table)
# Get pseudoname to use
table_pseudo_name = '{0}_{1}'.format(parent_table,
(len(pseudo_names) + 1))
pseudo_names.append(table_pseudo_name)
# Map lookup and admin unit values by default
if c.TYPE_INFO == 'LOOKUP':
lookup_model = entity_model(c.entity_relation.parent)
lookup_model_obj = lookup_model()
result = lookup_model_obj.queryObject().filter(
lookup_model.code != '').filter(
lookup_model.code is not None).all()
if len(result) == 0:
select_column_name = '{0}.value AS {1}'.format(
table_pseudo_name, pseudo_column_name)
else:
value = '{0}.value'.format(table_pseudo_name)
code = '{0}.code'.format(table_pseudo_name)
select_column_name = "concat({0}, ' (', {1}, ')') AS {2}". \
format(value, code, pseudo_column_name)
use_custom_join = True
# Check if the column is for tenure type
if c.name != 'tenure_type':
use_inner_join = False
elif c.TYPE_INFO == 'ADMIN_SPATIAL_UNIT':
select_column_name = '{0}.name AS {1}'.format(
table_pseudo_name, pseudo_column_name)
use_custom_join = True
use_inner_join = False
elif c.TYPE_INFO == 'FOREIGN_KEY':
if c.entity_relation.parent not in str_entity.parties and \
c.entity_relation.parent not in str_entity.spatial_units:
if len(c.entity_relation.display_cols) > 0:
display_col_names = []
for display_col in c.entity_relation.display_cols:
name = '{0}.{1}'.format(
table_pseudo_name, display_col)
display_col_names.append(name)
select_column_name = "concat_ws(' '::text, {0}) AS {1}".format(
', '.join(display_col_names),
pseudo_column_name)
else:
if not custom_tenure:
select_column_name = '{0}.id AS {1}'.format(
table_pseudo_name, pseudo_column_name)
use_custom_join = True
use_inner_join = False
else:
QApplication.processEvents()
# These are outer joins
join_type = 'LEFT JOIN'
# Use inner join only if parent entity is an STR entity and it is the current entity.
# Other str entities should use left join.
if str_entity.is_str_entity(fk_parent_entity) and \
fk_parent_entity.name in view_name:
join_type = 'INNER JOIN'
if use_custom_join:
# exclude replace str name with custom tenure name in join.
if custom_tenure:
if c.name == 'social_tenure_relationship_id':
i = i + 1
# pseudo_names = foreign_key_parents.get(parent_table)
col_select_name = '{0}_{1}.{2}'.format(
entity.name, str(i), c.name)
# Get pseudoname to use
table_pseudo_name = '{0}_{1}'.format(
entity.name, str(i))
join_statement = '{0} {1} {2} ON {3} = {2}.{4}'.format(
join_type, entity.name, table_pseudo_name,
col_select_name,
c.entity_relation.parent_column)
join_statements = [join_statement
] + join_statements
else:
join_statement = '{0} {1} {2} ON {3} = {2}.{4}'.format(
join_type, parent_table, table_pseudo_name,
col_select_name,
c.entity_relation.parent_column)
join_statements.append(join_statement)
else:
join_statement = '{0} {1} {2} ON {3} = {2}.{4}'.format(
join_type, parent_table, table_pseudo_name,
col_select_name, c.entity_relation.parent_column)
join_statements.append(join_statement)
else:
# Assert if the column is in the list of omitted join columns
#
# if c.name in omit_join_statement_columns:
# if 'INNER JOIN' in join_statement:
# join_statements.append(join_statement)
# else:
# This fix needs to be tested more ...
if parent_table in foreign_key_parents and 'relationship' not in entity.name:
parent_table_alias = foreign_key_parents[parent_table][
0]
join_statement = '{0} {1} {2} ON {3} = {2}.{4}'.format(
join_type, parent_table, parent_table_alias,
col_select_name, c.entity_relation.parent_column)
else:
join_statement = '{0} {1} ON {2} = {1}.{3}'.format(
join_type, parent_table, col_select_name,
c.entity_relation.parent_column)
parents.append(parent_table)
join_statements.append(join_statement)
# Assert if the column is in the list of omitted view columns
if c.name not in omit_view_columns:
if select_column_name:
column_names.append(select_column_name)
QApplication.processEvents()
return column_names, join_statements
def _entity_select_column(entity,
use_inner_join=False,
join_parents=False,
is_primary=False,
foreign_key_parents=None):
#Check if the entity exists in the database
if not pg_table_exists(entity.name):
msg = u'{0} table does not exist, social tenure view will not be ' \
u'created.'.format(entity.name)
LOGGER.debug(msg)
raise ConfigurationException(msg)
column_names = []
join_statements = []
columns = entity.columns.values()
#Create foreign key parent collection if none is specified
if foreign_key_parents is None:
foreign_key_parents = {}
for c in columns:
if not c.TYPE_INFO in _exclude_view_column_types:
normalized_entity_sname = entity.short_name.replace(' ',
'_').lower()
pseudo_column_name = u'{0}_{1}'.format(normalized_entity_sname,
c.name)
col_select_name = u'{0}.{1}'.format(entity.name, c.name)
select_column_name = u'{0} AS {1}'.format(col_select_name,
pseudo_column_name)
if is_primary and c.name == 'id':
select_column_name = col_select_name
#Use custom join flag
use_custom_join = False
if isinstance(c, ForeignKeyColumn) and join_parents:
LOGGER.debug('Creating STR: Getting parent for %s column',
c.name)
parent_table = c.entity_relation.parent.name
LOGGER.debug('Parent found')
select_column_name = ''
#Handle renaming of parent table names to appropriate
# pseudonames.
if not parent_table in foreign_key_parents:
foreign_key_parents[parent_table] = []
pseudo_names = foreign_key_parents.get(parent_table)
#Get pseudoname to use
table_pseudo_name = u'{0}_{1}'.format(parent_table,
(len(pseudo_names) + 1))
pseudo_names.append(table_pseudo_name)
#Map lookup and admin unit values by default
if c.TYPE_INFO == 'LOOKUP':
select_column_name = u'{0}.value AS {1}'.format(
table_pseudo_name, pseudo_column_name)
use_custom_join = True
#Check if the column is for tenure type
if c.name != 'tenure_type':
use_inner_join = False
elif c.TYPE_INFO == 'ADMIN_SPATIAL_UNIT':
select_column_name = u'{0}.name AS {1}'.format(
table_pseudo_name, pseudo_column_name)
use_custom_join = True
use_inner_join = False
#These are outer joins
join_type = 'LEFT JOIN'
if use_inner_join:
join_type = 'INNER JOIN'
if use_custom_join:
join_statement = u'{0} {1} {2} ON {3} = {2}.{4}'.format(
join_type, parent_table, table_pseudo_name,
col_select_name, c.entity_relation.parent_column)
else:
join_statement = u'{0} {1} ON {2} = {1}.{3}'.format(
join_type, parent_table, col_select_name,
c.entity_relation.parent_column)
join_statements.append(join_statement)
if select_column_name:
column_names.append(select_column_name)
return column_names, join_statements