def do_operation(self): """ perform create mapping scheme operation """ # input/output verification already performed during set input/ouput survey_layer = self.inputs[0].value tax_field = self._tax_field # merge to create stats ms = MappingScheme(self._taxonomy) stats = Statistics(self._taxonomy) ms.assign(MappingSchemeZone('ALL'), stats) # loop through all input features tax_idx = layer_field_index(survey_layer, tax_field) area_idx = layer_field_index(survey_layer, AREA_FIELD_NAME) cost_idx = layer_field_index(survey_layer, COST_FIELD_NAME) for _f in layer_features(survey_layer): _tax_str = str(_f.attributeMap()[tax_idx].toString()) additional = {} _area = _f.attributeMap()[area_idx].toDouble()[0] if _area > 0: additional = {StatisticNode.AverageSize: _area} _cost = _f.attributeMap()[cost_idx].toDouble()[0] if _cost > 0: additional = {StatisticNode.UnitCost: _cost} try: stats.add_case(_tax_str, self._parse_order, self._parse_modifiers, additional) except TaxonomyParseError as perr: logAPICall.log("error parsing case %s, %s" % (str(_tax_str), str(perr)), logAPICall.WARNING) # store data in output stats.finalize() self.outputs[0].value = ms
def _create_grid(self, grid_name, grid_file, x_min, y_min, x_max, y_max, x_off, y_off): x_off2, y_off2 = x_off / 2.0, y_off / 2.0 x_min = floor(x_min / x_off) * x_off x_max = ceil(x_max / x_off) * x_off y_min = floor(y_min / y_off) * y_off y_max = ceil(y_max / y_off) * y_off xtotal = int((x_max - x_min) / x_off)+1 ytotal = int((y_max - y_min) / y_off)+1 logAPICall.log('x_min %f x_max %f y_min %f y_max %f x_off %f y_off %f xtotal %d, ytotal %d' % (x_min, x_max, y_min, y_max, x_off, y_off, xtotal, ytotal), logAPICall.DEBUG_L2) fields = { 0 : QgsField('GRID_GID', QVariant.String), } writer = QgsVectorFileWriter(grid_file, "utf-8", fields, QGis.WKBPolygon, self._crs, "ESRI Shapefile") f = QgsFeature() for x in range(xtotal): for y in range(ytotal): lon = x_min + (x * x_off) + (x_off2) lat = y_min + (y * y_off) + (y_off2) #out_geom = QgsGeometry.fromRect(QgsRectangle(lon-x_off2, lat-y_off2, # lon+x_off2, lat+y_off2)) f.setGeometry(self._outputGeometryFromLatLon(lat, lon)) f.addAttribute(0, QVariant(latlon_to_grid(lat, lon))) writer.addFeature(f) del writer return load_shapefile(grid_file, grid_name)
def build_exposure_steps(self): """ building exposure database from workflow """ if not self.workflow.ready: raise SIDDException('Cannot create exposure with current datasets. Please revise input') if not self.ms.is_valid: raise SIDDException('Current mapping scheme is not valid') for zone in self.ms.zones: zone.stats.refresh_leaves(with_modifier=True, order_attributes=True) if getattr(self, 'exposure', None) is not None: del self.exposure remove_shapefile(self.exposure_file) for op in self.workflow.nextstep(): yield op # when all steps are completed, set resulting exposure self.exposure = self.workflow.operator_data['exposure'].value self.exposure_file = self.workflow.operator_data['exposure_file'].value if self.workflow.operator_data.has_key('exposure_grid'): self.exposure_grid = self.workflow.operator_data['exposure_grid'].value logAPICall.log('exposure data created %s' % self.exposure_file, logAPICall.INFO)
def verify_result(self): """ run data quality tests """ builder = WorkflowBuilder(self.operator_options) try: verify_workflow = builder.build_verify_result_workflow(self) except WorkflowException as err: raise SIDDException( "error creating workflow for result verification\n%s" % err) # process workflow for step in verify_workflow.nextstep(): try: step.do_operation() except Exception as err: logAPICall.log(err, logAPICall.WARNING) pass self.quality_reports = {} if verify_workflow.operator_data.has_key('frag_report'): self.quality_reports[ 'fragmentation'] = verify_workflow.operator_data[ 'frag_report'].value if verify_workflow.operator_data.has_key('count_report'): self.quality_reports['count'] = verify_workflow.operator_data[ 'count_report'].value try: if self.zone_type == ZonesTypes.LanduseCount and self.output_type == OutputTypes.Grid: self.quality_reports['count']['_note'] = '' except: pass logAPICall.log('result verification completed', logAPICall.INFO)
def get_project_data(self, attrib): if self.db.has_key(attrib): logAPICall.log('read from db %s => %s ' % (attrib, str(self.db[attrib])[0:25]), logAPICall.DEBUG_L2) return self.db[attrib] else: logAPICall.log('%s does not exist in db' % attrib, logAPICall.DEBUG_L2) return None
def build_processing_chain(self, project, workflow): for [data, out_type, func] in self._process_chains: # match all inputs input_matches = True logAPICall.log( '\tcheck data input chain %s %s=> %s' % (data, out_type, func.__name__), logAPICall.DEBUG_L2) for data_item in data: if not workflow.operator_data.has_key(data_item): input_matches = False break if not input_matches: continue # match output type if out_type != project.output_type: continue # build process chain func(project, workflow) # done return # no match for input and output type # cannot do anything with given data raise WorkflowException(WorkflowErrors.NoActionDefined)
def do_operation(self): """ perform create mapping scheme operation """ # input/output verification already performed during set input/ouput zone_layer = self.inputs[0].value zone_field = self.inputs[1].value fp_layer = self.inputs[2].value # merge with zone to get assignment tmp_join = 'joined_%s' % get_unique_filename() tmp_join_file = '%s%s.shp' % (self._tmp_dir, tmp_join) analyzer = QgsOverlayAnalyzer() try: analyzer.intersection(fp_layer, zone_layer, tmp_join_file) tmp_join_layer = load_shapefile_verify(tmp_join_file, tmp_join,[zone_field]) except AssertionError as err: raise OperatorError(str(err), self.__class__) except Exception as err: raise OperatorError(str(err), self.__class__) fields = { 0 : QgsField(self._lon_field, QVariant.Double), 1 : QgsField(self._lat_field, QVariant.Double), 2 : QgsField(zone_field, QVariant.String), } zone_idx = layer_field_index(tmp_join_layer, zone_field) fp_layername = 'fpc_%s' % get_unique_filename() fp_file = '%s%s.shp' % (self._tmp_dir, fp_layername) try: writer = QgsVectorFileWriter(fp_file, "utf-8", fields, QGis.WKBPoint, self._crs, "ESRI Shapefile") f = QgsFeature() for _f in layer_features(tmp_join_layer): centroid = _f.geometry().centroid().asPoint() lon = centroid.x() lat = centroid.y() zone_str = str(_f.attributeMap()[zone_idx].toString()).upper() f.setGeometry(QgsGeometry.fromPoint(QgsPoint(lon, lat))) f.addAttribute(0, QVariant(lon)) f.addAttribute(1, QVariant(lat)) f.addAttribute(2, QVariant(zone_str)) writer.addFeature(f) del writer except Exception as err: logAPICall.log(err, logAPICall.ERROR) remove_shapefile(fp_file) raise OperatorError("error creating joined grid: %s" % err, self.__class__) # load shapefile as layer fp_layer = load_shapefile(fp_file, fp_layername) if not fp_layer: raise OperatorError('Error loading footprint centroid file' % (fp_file), self.__class__) # clean up del tmp_join_layer remove_shapefile(tmp_join_file) self.outputs[0].value = fp_layer self.outputs[1].value = fp_file
def verify_result(self): """ run data quality tests """ builder = WorkflowBuilder(self.operator_options) try: verify_workflow = builder.build_verify_result_workflow(self) except WorkflowException as err: raise SIDDException("error creating workflow for result verification\n%s" % err) # process workflow for step in verify_workflow.nextstep(): try: step.do_operation() except Exception as err: logAPICall.log(err, logAPICall.WARNING) pass self.quality_reports={} if verify_workflow.operator_data.has_key('frag_report'): self.quality_reports['fragmentation'] = verify_workflow.operator_data['frag_report'].value if verify_workflow.operator_data.has_key('count_report'): self.quality_reports['count'] = verify_workflow.operator_data['count_report'].value try: if self.zone_type == ZonesTypes.LanduseCount and self.output_type == OutputTypes.Grid: self.quality_reports['count']['_note'] = '' except: pass logAPICall.log('result verification completed', logAPICall.INFO)
def _create_grid(self, grid_name, grid_file, x_min, y_min, x_max, y_max, x_off, y_off): x_off2, y_off2 = x_off / 2.0, y_off / 2.0 x_min = floor(x_min / x_off) * x_off x_max = ceil(x_max / x_off) * x_off y_min = floor(y_min / y_off) * y_off y_max = ceil(y_max / y_off) * y_off xtotal = int((x_max - x_min) / x_off) + 1 ytotal = int((y_max - y_min) / y_off) + 1 logAPICall.log( "x_min %f x_max %f y_min %f y_max %f x_off %f y_off %f xtotal %d, ytotal %d" % (x_min, x_max, y_min, y_max, x_off, y_off, xtotal, ytotal), logAPICall.DEBUG_L2, ) fields = {0: QgsField("GRID_GID", QVariant.String)} writer = QgsVectorFileWriter(grid_file, "utf-8", fields, QGis.WKBPolygon, self._crs, "ESRI Shapefile") f = QgsFeature() for x in range(xtotal): for y in range(ytotal): lon = x_min + (x * x_off) + (x_off2) lat = y_min + (y * y_off) + (y_off2) # out_geom = QgsGeometry.fromRect(QgsRectangle(lon-x_off2, lat-y_off2, # lon+x_off2, lat+y_off2)) f.setGeometry(self._outputGeometryFromLatLon(lat, lon)) f.addAttribute(0, QVariant(latlon_to_grid(lat, lon))) writer.addFeature(f) del writer return load_shapefile(grid_file, grid_name)
def from_xml(self, xmlnode): """ construct node and children from XML """ self.name = get_node_attrib(xmlnode, 'attribute') self.value = get_node_attrib(xmlnode, 'value') self.level = int(get_node_attrib(xmlnode, 'level')) self.weight = float(get_node_attrib(xmlnode, 'weight')) self.count = self.weight self.is_default = str(get_node_attrib(xmlnode, 'is_default')).upper()=='TRUE' self.is_skipped = str(get_node_attrib(xmlnode, 'is_skipped')).upper()=='TRUE' for add_node in xmlnode.findall('additional'): for idx, label in enumerate(self.label_additional): add_value = get_node_attrib(add_node, label) if add_value != '': self.additional[idx]=add_value for mod_node in xmlnode.findall('modifiers/modifier'): mod = StatisticModifier() mod.from_xml(mod_node) self.modifiers.append(mod) for childnode in xmlnode.findall('children/node'): logAPICall.log('created new child with xmlnode %s' % childnode, logAPICall.DEBUG_L2) node = StatisticNode(self) node.from_xml(childnode) self.children.append(node)
def _write_grid_shapefile(self, path, x_min, y_min, x_max, y_max, x_off, y_off): x_off = self._x_off y_off = self._y_off x_min = floor(x_min / x_off) * x_off x_max = ceil(x_max / x_off) * x_off y_min = floor(y_min / y_off) * y_off y_max = ceil(y_max / y_off) * y_off xtotal = int((x_max - x_min) / x_off) ytotal = int((y_max - y_min) / y_off) logAPICall.log( 'x_min %f x_max %f y_min %f y_max %f x_off %f y_off %f xtotal %d, ytotal %d' % (x_min, x_max, y_min, y_max, x_off, y_off, xtotal, ytotal), logAPICall.DEBUG_L2) writer = QgsVectorFileWriter(path, "utf-8", self._fields, QGis.WKBPoint, self._crs, "ESRI Shapefile") f = QgsFeature() for x in range(xtotal): for y in range(ytotal): lon = x_min + (x * x_off) + (x_off / 2.0) lat = y_min + (y * y_off) + (y_off / 2.0) f.setGeometry(QgsGeometry.fromPoint(QgsPoint(lon, lat))) f.addAttribute(0, QVariant(lon)) f.addAttribute(1, QVariant(lat)) writer.addFeature(f) del writer
def build_processing_chain(self, project, workflow): for [data, out_type, func] in self._process_chains: # match all inputs input_matches = True logAPICall.log('\tcheck data input chain %s %s=> %s' % (data, out_type, func.__name__), logAPICall.DEBUG_L2) for data_item in data: if not workflow.operator_data.has_key(data_item): input_matches = False break if not input_matches: continue # match output type if out_type != project.output_type: continue # build process chain func(project, workflow) # done return # no match for input and output type # cannot do anything with given data raise WorkflowException(WorkflowErrors.NoActionDefined)
def build_exposure_steps(self): """ building exposure database from workflow """ if not self.workflow.ready: raise SIDDException( 'Cannot create exposure with current datasets. Please revise input' ) if not self.ms.is_valid: raise SIDDException('Current mapping scheme is not valid') for zone in self.ms.zones: zone.stats.refresh_leaves(with_modifier=True, order_attributes=True) if getattr(self, 'exposure', None) is not None: del self.exposure remove_shapefile(self.exposure_file) for op in self.workflow.nextstep(): yield op # when all steps are completed, set resulting exposure self.exposure = self.workflow.operator_data['exposure'].value self.exposure_file = self.workflow.operator_data['exposure_file'].value if self.workflow.operator_data.has_key('exposure_grid'): self.exposure_grid = self.workflow.operator_data[ 'exposure_grid'].value logAPICall.log('exposure data created %s' % self.exposure_file, logAPICall.INFO)
def _write_grid_shapefile(self, path, x_min, y_min, x_max, y_max, x_off, y_off): x_off = self._x_off y_off = self._y_off x_min = floor(x_min / x_off) * x_off x_max = ceil(x_max / x_off) * x_off y_min = floor(y_min / y_off) * y_off y_max = ceil(y_max / y_off) * y_off xtotal = int((x_max - x_min) / x_off) ytotal = int((y_max - y_min) / y_off) logAPICall.log('x_min %f x_max %f y_min %f y_max %f x_off %f y_off %f xtotal %d, ytotal %d' % (x_min, x_max, y_min, y_max, x_off, y_off, xtotal, ytotal), logAPICall.DEBUG_L2) writer = QgsVectorFileWriter(path, "utf-8", self._fields, QGis.WKBPoint, self._crs, "ESRI Shapefile") f = QgsFeature() for x in range(xtotal): for y in range(ytotal): lon = x_min + (x * x_off) + (x_off/2.0) lat = y_min + (y * y_off) + (y_off/2.0) f.setGeometry(QgsGeometry.fromPoint(QgsPoint(lon, lat))) f.addAttribute(0, QVariant(lon)) f.addAttribute(1, QVariant(lat)) writer.addFeature(f) del writer
def data(self, index, role): """ return data to be displayed in a cell """ if role == Qt.DisplayRole: logAPICall.log('row %s column %s ' %(index.row(), index.column()), logAPICall.DEBUG_L2) return QString("%s" % self.selected[index.row()][index.column()]) else: return QVariant()
def index(self, row, column, parent): """ provide index to data given a cell """ logAPICall.log('index row %s col %s parent %s' % (row, column, parent), logAPICall.DEBUG_L2) mod = self.get_modifier(row) if mod is not None: return self.createIndex(row, column, mod) else: return QModelIndex()
def save_project_data(self, attrib, value): if value is None: # delete logAPICall.log('delete from db %s ' % (attrib), logAPICall.DEBUG_L2) if self.db.has_key(attrib): del self.db[attrib] else: logAPICall.log('save to db %s => %s ' % (attrib, str(value)[0:25]), logAPICall.DEBUG_L2) self.db[attrib]=str(value)
def get_project_data(self, attrib): if self.db.has_key(attrib): logAPICall.log( 'read from db %s => %s ' % (attrib, str(self.db[attrib])[0:25]), logAPICall.DEBUG_L2) return self.db[attrib] else: logAPICall.log('%s does not exist in db' % attrib, logAPICall.DEBUG_L2) return None
def save_project_data(self, attrib, value): if value is None: # delete logAPICall.log('delete from db %s ' % (attrib), logAPICall.DEBUG_L2) if self.db.has_key(attrib): del self.db[attrib] else: logAPICall.log('save to db %s => %s ' % (attrib, str(value)[0:25]), logAPICall.DEBUG_L2) self.db[attrib] = str(value)
def build_ms_workflow(self, project, isEmpty=False): """ create mapping scheme creating workflow """ workflow = Workflow() logAPICall.log('creating survey loader ...', logAPICall.DEBUG_L2) if not isEmpty: self.load_survey(project, workflow, False) if project.zone_type == ZonesTypes.None: if isEmpty: logAPICall.log('creating empty mapping scheme ...', logAPICall.DEBUG_L2) ms_creator = EmptyMSCreator(self.operator_options) ms_creator.inputs = [] else: logAPICall.log('creating mapping scheme from survey only ...', logAPICall.DEBUG_L2) ms_creator = SurveyOnlyMSCreator(self.operator_options) ms_creator.inputs = [ workflow.operator_data['survey'], ] else: self.load_zone(project, workflow, False) if isEmpty: logAPICall.log('creating empty mapping scheme from zones ...', logAPICall.DEBUG_L2) ms_creator = EmptyZonesMSCreator(self.operator_options) ms_creator.inputs = [ workflow.operator_data['zone'], workflow.operator_data['zone_field'], ] else: logAPICall.log( 'creating mapping scheme from survey and zones ...', logAPICall.DEBUG_L2) workflow.operator_data['zone_field'] = OperatorData( OperatorDataTypes.StringAttribute, project.zone_field) ms_creator = SurveyZonesMSCreator(self.operator_options) ms_creator.inputs = [ workflow.operator_data['survey'], workflow.operator_data['zone'], workflow.operator_data['zone_field'], ] workflow.operator_data['ms'] = OperatorData( OperatorDataTypes.MappingScheme) ms_creator.outputs = [ workflow.operator_data['ms'], ] workflow.operators.append(ms_creator) workflow.ready = True return workflow
def verify_data(self): """ verify existing data and create workflow """ # build workflow based on current data builder = WorkflowBuilder(self.operator_options) self.workflow = builder.build_workflow(self) if self.workflow.ready: self.status = ProjectStatus.ReadyForExposure else: self.status = ProjectStatus.ReadyForMS self.errors = self.workflow.errors self.exposure = None logAPICall.log('input verification completed', logAPICall.INFO)
def set_project_path(self, project_file): try: if (not os.path.exists(project_file)): shutil.copyfile(FILE_PROJ_TEMPLATE, project_file) self.db = bsddb.btopen(project_file, 'c') self.version_major = self.get_project_data('version_major') self.version_minor = self.get_project_data('version_minor') logAPICall.log('opening project file version %s.%s' %(self.version_major, self.version_minor), logAPICall.INFO) self.project_file = project_file self.require_save = True except: raise SIDDProjectException(ProjectErrors.FileFormatError)
def export_data(self): """ export exposure data """ builder = WorkflowBuilder(self.operator_options) try: export_workflow = builder.build_export_workflow(self) except WorkflowException as err: raise SIDDException("error creating workflow for exporting data\n%s" % err) try: # process workflow export_workflow.process() logAPICall.log('data export completed', logAPICall.INFO) except Exception as err: raise SIDDException("error exporting data\n" % err)
def export_data(self): """ export exposure data """ builder = WorkflowBuilder(self.operator_options) try: export_workflow = builder.build_export_workflow(self) except WorkflowException as err: raise SIDDException( "error creating workflow for exporting data\n%s" % err) try: # process workflow export_workflow.process() logAPICall.log('data export completed', logAPICall.INFO) except Exception as err: raise SIDDException("error exporting data\n" % err)
def set_project_path(self, project_file): try: if (not os.path.exists(project_file)): shutil.copyfile(FILE_PROJ_TEMPLATE, project_file) self.db = bsddb.btopen(project_file, 'c') self.version_major = self.get_project_data('version_major') self.version_minor = self.get_project_data('version_minor') logAPICall.log( 'opening project file version %s.%s' % (self.version_major, self.version_minor), logAPICall.INFO) self.project_file = project_file self.require_save = True except: raise SIDDProjectException(ProjectErrors.FileFormatError)
def do_operation(self): """ perform footprint load operation """ grid_layer = self.inputs[0].value # make sure input is correct # NOTE: these checks cannot be performed at set input time # because the data layer maybe is not loaded yet self._test_layer_loaded(grid_layer) grid_fields = grid_layer.dataProvider().fields() output_layername = 'grid_%s' % get_unique_filename() output_file = self._tmp_dir + output_layername + '.shp' half_grid = DEFAULT_GRID_SIZE / 2.0 try: writer = QgsVectorFileWriter(output_file, "utf-8", grid_fields, QGis.WKBPolygon, grid_layer.crs(), "ESRI Shapefile") out_f = QgsFeature() for in_f in layer_features(grid_layer): in_point = in_f.geometry().asPoint() out_geom = QgsGeometry.fromRect( QgsRectangle(in_point.x() - half_grid, in_point.y() - half_grid, in_point.x() + half_grid, in_point.y() + half_grid)) out_f.setGeometry(out_geom) out_f.setAttributeMap(in_f.attributeMap()) writer.addFeature(out_f) del writer except Exception as err: logAPICall.log(str(err), logAPICall.ERROR) raise OperatorError('error writing out grid: %s' % err, self.__class__) # load shapefile as layer output_layer = load_shapefile(output_file, output_layername) if not output_layer: raise OperatorError( 'Error loading generated file %s' % (output_file), self.__class__) # store data in output self.outputs[0].value = output_layer self.outputs[1].value = output_file
def add(self, attr_vals, parse_order, level, additional_data={}): """ recursively update statistic @ node and @ child nodes using attr_val, defaults, skips at idx """ # increment count of current node self.count+=1 # the ending condition for the recursive call # NOTE: is_leaf is not used here, this process should work on a empty tree if (len(parse_order) <= level): # leaf node also aggregate additional data self.increment_additonal(self.AverageSize, additional_data) self.increment_additonal(self.UnitCost, additional_data) return logAPICall.log('processing %d %s' %(level, parse_order[level]), logAPICall.DEBUG) # get value to add/update children # NOTE: value for current node is already set by its parent # all processing/parsing is to work on its children attr_name = parse_order[level] value = None for val in attr_vals: if val.attribute.name == attr_name: value = val break # handle default cases is_default = False if value is None: is_default = True elif value is not None and (str(value) == value.attribute.default or str(value) == value.attribute.group.default): value = None is_default = True logAPICall.log('\tnode:%s' %(value), logAPICall.DEBUG_L2) child_found = False # find children and add value/modifier for child in self.children: if (child.value is None and value is None) or str(child.value) == str(value): logAPICall.log('found child with %s' % value, logAPICall.DEBUG_L2) child_found = True # recursive call to process next level child.add(attr_vals, parse_order, level+1, additional_data) return # if no children found, then add new node for value and add modifier if not child_found: logAPICall.log('create new child with %s' % value, logAPICall.DEBUG_L2) child = StatisticNode(self, attr_name, value, self.level+1, is_default, False) self.children.append(child) # recursive call to process next level child.add(attr_vals, parse_order, level+1, additional_data) return
def clean_up(self): """ cleanup """ try: logAPICall.log('attempt to delete project temp dir %s' % self.temp_dir, logAPICall.DEBUG) if os.path.exists(self.temp_dir): del self.workflow if self.exposure is not None: del self.exposure # must delete QGIS layer, otherwise exposure_file becomes locked # and will generate error on shutil.rmtree shutil.rmtree(self.temp_dir) except Exception as err: logAPICall.log('failed to delete temporary directory: %s' % str(err), logAPICall.WARNING) try: if self.project_file is not None and self.db is not None: self.db.close() except Exception: pass
def build_sampling_ms_workflow(self, project): """ create workflow for mapping scheme using stratified sampling methodology """ workflow = Workflow() logAPICall.log('creating survey loader ...', logAPICall.DEBUG_L2) # test to make sure all necessary data is available if project.fp_type != FootprintTypes.FootprintHt: workflow.add_error(WorkflowErrors.NeedsHeight) if project.zone_type == ZonesTypes.None: workflow.add_error(WorkflowErrors.NeedsZone) if project.survey_type == SurveyTypes.None: workflow.add_error(WorkflowErrors.NeedSurvey) if workflow.has_error(): return workflow # data loading self.load_footprint(project, workflow, True) self.load_survey(project, workflow, False) self.load_zone(project, workflow, False) # create operator for mapping scheme workflow.operator_data['zone_field'] = OperatorData( OperatorDataTypes.StringAttribute, project.zone_field) ms_creator = StratifiedMSCreator(self.operator_options) ms_creator.inputs = [ workflow.operator_data['fp'], OperatorData(OperatorDataTypes.StringAttribute, AREA_FIELD_NAME), OperatorData(OperatorDataTypes.StringAttribute, HT_FIELD_NAME), workflow.operator_data['zone'], workflow.operator_data['zone_field'], workflow.operator_data['survey'], ] workflow.operator_data['ms'] = OperatorData( OperatorDataTypes.MappingScheme) workflow.operator_data['zone_stats'] = OperatorData( OperatorDataTypes.ZoneStatistic) ms_creator.outputs = [ workflow.operator_data['ms'], workflow.operator_data['zone_stats'] ] workflow.operators.append(ms_creator) workflow.ready = True return workflow
def data(self, index, role): """ data for cells """ col, row = index.column(), index.row() logAPICall.log('data col %s row %s' % (row, col), logAPICall.DEBUG_L2) if role == Qt.DisplayRole: # construct data for display in table _mod = self.get_modifier(row) _idx = row - _mod[self.STR_INDEX] if (col < self.STR_INDEX): # for first 4 columns, only first row in new modifier # need to show the headings if (_idx == 0): return QVariant(_mod[col]) else: return QVariant() else: # for last 2 columns, show modifier value and associated percentage for _key in sorted(_mod[self.MOD_INDEX].keys()): if (_idx == 0): if (col == self.STR_INDEX): return QVariant(_key) else: return QVariant("%.2f" %_mod[self.MOD_INDEX].value(_key)) else: _idx -=1 elif role == Qt.ToolTipRole: # construct data for display in tooltip _mod = self.get_modifier(row) _idx = row - _mod[self.STR_INDEX] if col==1: if (_idx == 0): return build_attribute_tooltip(self.valid_codes, self.ms.taxonomy.parse(_mod[col])) else: return QVariant() elif col==2: _key = sorted(_mod[self.MOD_INDEX].keys())[_idx] if _key is not None: return build_attribute_tooltip(self.valid_codes, self.ms.taxonomy.parse(_key)) else: return QVariant("") else: return QVariant()
def clean_up(self): """ cleanup """ try: logAPICall.log( 'attempt to delete project temp dir %s' % self.temp_dir, logAPICall.DEBUG) if os.path.exists(self.temp_dir): del self.workflow if self.exposure is not None: del self.exposure # must delete QGIS layer, otherwise exposure_file becomes locked # and will generate error on shutil.rmtree shutil.rmtree(self.temp_dir) except Exception as err: logAPICall.log( 'failed to delete temporary directory: %s' % str(err), logAPICall.WARNING) try: if self.project_file is not None and self.db is not None: self.db.close() except Exception: pass
def export_ms(self, path, export_format): """ export mapping scheme according to given format see constants.MSExportTypes for type supported """ if self.ms is None: raise SIDDException('Mapping Scheme is required for this action') builder= WorkflowBuilder(self.operator_options) try: if export_format == MSExportTypes.XML: export_workflow = builder.build_export_ms_workflow(self, path) else: export_workflow = builder.build_export_distribution_workflow(self, path) export_workflow.process() logAPICall.log('data export completed', logAPICall.INFO) except WorkflowException: return False except Exception as err: logAPICall.log(err, logAPICall.ERROR) return False
def build_ms_workflow(self, project, isEmpty=False): """ create mapping scheme creating workflow """ workflow = Workflow() logAPICall.log('creating survey loader ...', logAPICall.DEBUG_L2) if not isEmpty: self.load_survey(project, workflow, False) if project.zone_type == ZonesTypes.None: if isEmpty: logAPICall.log('creating empty mapping scheme ...', logAPICall.DEBUG_L2) ms_creator = EmptyMSCreator(self.operator_options) ms_creator.inputs = [] else: logAPICall.log('creating mapping scheme from survey only ...', logAPICall.DEBUG_L2) ms_creator = SurveyOnlyMSCreator(self.operator_options) ms_creator.inputs = [workflow.operator_data['survey'],] else: self.load_zone(project, workflow, False) if isEmpty: logAPICall.log('creating empty mapping scheme from zones ...', logAPICall.DEBUG_L2) ms_creator = EmptyZonesMSCreator(self.operator_options) ms_creator.inputs = [ workflow.operator_data['zone'], workflow.operator_data['zone_field'],] else: logAPICall.log('creating mapping scheme from survey and zones ...', logAPICall.DEBUG_L2) workflow.operator_data['zone_field'] = OperatorData(OperatorDataTypes.StringAttribute, project.zone_field) ms_creator = SurveyZonesMSCreator(self.operator_options) ms_creator.inputs = [ workflow.operator_data['survey'], workflow.operator_data['zone'], workflow.operator_data['zone_field'],] workflow.operator_data['ms'] = OperatorData(OperatorDataTypes.MappingScheme) ms_creator.outputs = [workflow.operator_data['ms'],] workflow.operators.append(ms_creator) workflow.ready=True return workflow
def append_branch(self, node, branch): """ append a branch (from library) to a node in a mapping scheme tree """ stat_tree = None if type(node) == MappingSchemeZone: # selected node is zone # retrieve root node from stats tree stat_tree = self.get_assignment(node) node_to_attach = node.stats.get_tree() else: node_to_attach = node stat_tree = self.get_assignment_by_node(node_to_attach) if stat_tree is None: raise SIDDException( 'selected node does not belong to mapping scheme') # add branch as child if type(branch) == MappingSchemeZone: # branch starts from zone node, so it is a full stats tree # add only the child nodes logAPICall.log('branch is zone, add children', logAPICall.DEBUG_L2) # test to make sure append is valid # exception will be thrown is case of error for child in branch.stats.get_tree().children: stat_tree.test_repeated_attribute(node_to_attach, child) stat_tree.test_repeated_value(node_to_attach, child) for child in branch.stats.get_tree().children: stat_tree.add_branch(node_to_attach, child, test_repeating=False, update_stats=False) node_to_attach.balance_weights() else: # branch is from a tree # add branch as child node logAPICall.log('branch is node, add branch', logAPICall.DEBUG_L2) stat_tree.add_branch(node_to_attach, branch)
def export_ms(self, path, export_format): """ export mapping scheme according to given format see constants.MSExportTypes for type supported """ if self.ms is None: raise SIDDException('Mapping Scheme is required for this action') builder = WorkflowBuilder(self.operator_options) try: if export_format == MSExportTypes.XML: export_workflow = builder.build_export_ms_workflow(self, path) else: export_workflow = builder.build_export_distribution_workflow( self, path) export_workflow.process() logAPICall.log('data export completed', logAPICall.INFO) except WorkflowException: return False except Exception as err: logAPICall.log(err, logAPICall.ERROR) return False
def load_data(self, input_param, layer, output_file): input_file = getattr(self, input_param, None) if input_file is not None: builder = WorkflowBuilder(self.operator_options) # create workflow if input_param == 'fp_file': workflow = builder.build_load_fp_workflow(self) elif input_param == 'zone_file': workflow = builder.build_load_zones_workflow(self) elif input_param == 'survey_file': workflow = builder.build_load_survey_workflow(self) elif input_param == 'popgrid_file': workflow = builder.build_load_popgrid_workflow(self) else: raise Exception('Data Type Not Recognized %s' % input_param) if not workflow.ready: raise Exception('Cannot load data with %s' % input_param) workflow.process() logAPICall.log('data file %s loaded' % input_file, logAPICall.INFO) return workflow.operator_data[layer].value, workflow.operator_data[output_file].value
def do_operation(self): """ perform footprint load operation """ grid_layer = self.inputs[0].value # make sure input is correct # NOTE: these checks cannot be performed at set input time # because the data layer maybe is not loaded yet self._test_layer_loaded(grid_layer) grid_fields = grid_layer.dataProvider().fields() output_layername = 'grid_%s' % get_unique_filename() output_file = self._tmp_dir + output_layername + '.shp' half_grid = DEFAULT_GRID_SIZE / 2.0 try: writer = QgsVectorFileWriter(output_file, "utf-8", grid_fields, QGis.WKBPolygon, grid_layer.crs(), "ESRI Shapefile") out_f = QgsFeature() for in_f in layer_features(grid_layer): in_point = in_f.geometry().asPoint() out_geom = QgsGeometry.fromRect(QgsRectangle(in_point.x()-half_grid, in_point.y()-half_grid, in_point.x()+half_grid, in_point.y()+half_grid)) out_f.setGeometry(out_geom) out_f.setAttributeMap(in_f.attributeMap()) writer.addFeature(out_f) del writer except Exception as err: logAPICall.log(str(err), logAPICall.ERROR) raise OperatorError('error writing out grid: %s' % err, self.__class__) # load shapefile as layer output_layer = load_shapefile(output_file, output_layername) if not output_layer: raise OperatorError('Error loading generated file %s' % (output_file), self.__class__) # store data in output self.outputs[0].value = output_layer self.outputs[1].value = output_file
def build_sampling_ms_workflow(self, project): """ create workflow for mapping scheme using stratified sampling methodology """ workflow = Workflow() logAPICall.log('creating survey loader ...', logAPICall.DEBUG_L2) # test to make sure all necessary data is available if project.fp_type != FootprintTypes.FootprintHt: workflow.add_error(WorkflowErrors.NeedsHeight) if project.zone_type == ZonesTypes.None: workflow.add_error(WorkflowErrors.NeedsZone) if project.survey_type == SurveyTypes.None: workflow.add_error(WorkflowErrors.NeedSurvey) if workflow.has_error(): return workflow # data loading self.load_footprint(project, workflow, True) self.load_survey(project, workflow, False) self.load_zone(project, workflow, False) # create operator for mapping scheme workflow.operator_data['zone_field'] = OperatorData(OperatorDataTypes.StringAttribute, project.zone_field) ms_creator = StratifiedMSCreator(self.operator_options) ms_creator.inputs = [workflow.operator_data['fp'], OperatorData(OperatorDataTypes.StringAttribute, AREA_FIELD_NAME), OperatorData(OperatorDataTypes.StringAttribute, HT_FIELD_NAME), workflow.operator_data['zone'], workflow.operator_data['zone_field'], workflow.operator_data['survey'],] workflow.operator_data['ms'] = OperatorData(OperatorDataTypes.MappingScheme) workflow.operator_data['zone_stats'] = OperatorData(OperatorDataTypes.ZoneStatistic) ms_creator.outputs = [workflow.operator_data['ms'], workflow.operator_data['zone_stats']] workflow.operators.append(ms_creator) workflow.ready=True return workflow
def do_operation(self): """ perform create mapping scheme operation """ # input/output verification already performed during set input/ouput survey_layer = self.inputs[0].value tax_field = self._tax_field # merge to create stats ms = MappingScheme(self._taxonomy) stats = Statistics(self._taxonomy) ms.assign(MappingSchemeZone('ALL'), stats) # loop through all input features tax_idx = layer_field_index(survey_layer, tax_field) area_idx = layer_field_index(survey_layer, AREA_FIELD_NAME) cost_idx = layer_field_index(survey_layer, COST_FIELD_NAME) for _f in layer_features(survey_layer): _tax_str = str(_f.attributeMap()[tax_idx].toString()) additional = {} _area = _f.attributeMap()[area_idx].toDouble()[0] if _area > 0: additional = {StatisticNode.AverageSize: _area} _cost = _f.attributeMap()[cost_idx].toDouble()[0] if _cost > 0: additional = {StatisticNode.UnitCost: _cost} try: stats.add_case(_tax_str, self._parse_order, self._parse_modifiers, additional) except TaxonomyParseError as perr: logAPICall.log( "error parsing case %s, %s" % (str(_tax_str), str(perr)), logAPICall.WARNING) # store data in output stats.finalize() self.outputs[0].value = ms
def load_data(self, input_param, layer, output_file): input_file = getattr(self, input_param, None) if input_file is not None: builder = WorkflowBuilder(self.operator_options) # create workflow if input_param == 'fp_file': workflow = builder.build_load_fp_workflow(self) elif input_param == 'zone_file': workflow = builder.build_load_zones_workflow(self) elif input_param == 'survey_file': workflow = builder.build_load_survey_workflow(self) elif input_param == 'popgrid_file': workflow = builder.build_load_popgrid_workflow(self) else: raise Exception('Data Type Not Recognized %s' % input_param) if not workflow.ready: raise Exception('Cannot load data with %s' % input_param) workflow.process() logAPICall.log('data file %s loaded' % input_file, logAPICall.INFO) return workflow.operator_data[layer].value, workflow.operator_data[ output_file].value
def append_branch(self, node, branch): """ append a branch (from library) to a node in a mapping scheme tree """ stat_tree = None if type(node) == MappingSchemeZone: # selected node is zone # retrieve root node from stats tree stat_tree = self.get_assignment(node) node_to_attach = node.stats.get_tree() else: node_to_attach = node stat_tree = self.get_assignment_by_node(node_to_attach) if stat_tree is None: raise SIDDException('selected node does not belong to mapping scheme') # add branch as child if type(branch) == MappingSchemeZone: # branch starts from zone node, so it is a full stats tree # add only the child nodes logAPICall.log('branch is zone, add children', logAPICall.DEBUG_L2) # test to make sure append is valid # exception will be thrown is case of error for child in branch.stats.get_tree().children: stat_tree.test_repeated_attribute(node_to_attach, child) stat_tree.test_repeated_value(node_to_attach, child) for child in branch.stats.get_tree().children: stat_tree.add_branch(node_to_attach, child, test_repeating=False, update_stats=False) node_to_attach.balance_weights() else: # branch is from a tree # add branch as child node logAPICall.log('branch is node, add branch', logAPICall.DEBUG_L2) stat_tree.add_branch(node_to_attach, branch)
def do_build_ms(self, isEmpty=False, useSampling=False): """ create mapping scheme """ builder = WorkflowBuilder(self.operator_options) # force reload existing survey self.survey = None # create workflow if useSampling: ms_workflow = builder.build_sampling_ms_workflow(self) else: ms_workflow = builder.build_ms_workflow(self, isEmpty) if not ms_workflow.ready: raise SIDDException(ms_workflow.errors) # process workflow ms_workflow.process() self.ms = ms_workflow.operator_data['ms'].value if useSampling: self.zone_stats = ms_workflow.operator_data['zone_stats'].value for zone, stats in self.ms.assignments(): stats.refresh_leaves() logAPICall.log('mapping scheme created', logAPICall.INFO) self.require_save = True
def do_operation(self): """ perform footprint load operation """ # input/output data checking already done during property set zone_layer = self.inputs[0].value # make sure input is correct # NOTE: these checks cannot be performed at set input time # because the data layer maybe is not loaded yet self._test_layer_loaded(zone_layer) x_off = self._x_off y_off = self._y_off extent = zone_layer.extent() [x_min, y_min, x_max, y_max] = [extent.xMinimum(), extent.yMinimum(), extent.xMaximum(), extent.yMaximum()] # create grid based on extent of given region tmp_grid1 = 'grid_' + get_unique_filename() tmp_grid1_file = self._tmp_dir + tmp_grid1 + '.shp' try: self._write_grid_shapefile(tmp_grid1_file, x_min, y_min, x_max, y_max, x_off, y_off) except: remove_shapefile(tmp_grid1_file) raise OperatorError('error creating temporary grid', self.__class__) tmp_grid1_layer = load_shapefile(tmp_grid1_file, tmp_grid1) # temporary grid for joined shape with all grid points not within region removed tmp_grid2 = 'grid_' + get_unique_filename() tmp_grid2_file = self._tmp_dir + tmp_grid2 + '.shp' tmp_grid2_layer = None try: analyzer = QgsOverlayAnalyzer() analyzer.intersection(tmp_grid1_layer, zone_layer, tmp_grid2_file) tmp_grid2_layer = load_shapefile(tmp_grid2_file, tmp_grid2) except: raise OperatorError('error creating grid', self.__class__) # create result layer grid_layername = 'grid_%s' % get_unique_filename() grid_file = self._tmp_dir + grid_layername + '.shp' try: writer = QgsVectorFileWriter(grid_file, "utf-8", self._fields, QGis.WKBPoint, self._crs, "ESRI Shapefile") f = QgsFeature() lon_idx = layer_field_index(tmp_grid2_layer, self._lon_field) lat_idx = layer_field_index(tmp_grid2_layer, self._lat_field) for _f in layer_features(tmp_grid2_layer): lon = _f.attributeMap()[lon_idx].toDouble()[0] lat = _f.attributeMap()[lat_idx].toDouble()[0] f.setGeometry(QgsGeometry.fromPoint(QgsPoint(lon, lat))) f.addAttribute(0, QVariant(lon)) f.addAttribute(1, QVariant(lat)) writer.addFeature(f) del writer except Exception as err: logAPICall.log(str(err), logAPICall.ERROR) raise OperatorError('error writing out grid', self.__class__) grid_layer = load_shapefile(grid_file, grid_layername) if not grid_layer: raise OperatorError('Error loading result grid file' % (grid_file), self.__class__) # clean up del analyzer, tmp_grid1_layer, tmp_grid2_layer remove_shapefile(tmp_grid1_file) remove_shapefile(tmp_grid2_file) self.outputs[0].value = grid_layer self.outputs[1].value = grid_file
def do_operation(self): """ perform create mapping scheme operation """ # input/output verification already performed during set input/ouput fp_layer = self.inputs[0].value zone_field = self.inputs[1].value # aggregate footprint into grids logAPICall.log('aggregate statistic for grid ...', logAPICall.DEBUG) total_features = fp_layer.dataProvider().featureCount() if total_features > MAX_FEATURES_IN_MEMORY: # use bsddb to store temporary lat/lon tmp_db_file = '%sdb_%s.db' % (self._tmp_dir, get_unique_filename()) db = bsddb.btopen(tmp_db_file, 'c') use_db = True else: db = {} use_db = False zone_idx = layer_field_index(fp_layer, zone_field) for f in layer_features(fp_layer): geom = f.geometry() zone_str = str(f.attributeMap()[zone_idx].toString()) centroid = geom.centroid().asPoint() # use floor, this truncates all points within grid to grid's # bottom-left corner x = math.floor(centroid.x() / DEFAULT_GRID_SIZE) y = math.floor(centroid.y() / DEFAULT_GRID_SIZE) key = '%s %d %d' % (zone_str, x,y) if db.has_key(key): db[key] = str(int(db[key]) + 1) else: db[key] = '1' # output grid logAPICall.log('create grid ...', logAPICall.DEBUG) fields = { 0 : QgsField(self._lon_field, QVariant.Double), 1 : QgsField(self._lat_field, QVariant.Double), 2 : QgsField(CNT_FIELD_NAME, QVariant.Double), 3 : QgsField(zone_field, QVariant.String), } grid_layername = 'grid_%s' % get_unique_filename() grid_file = '%s%s.shp' % (self._tmp_dir, grid_layername) try: writer = QgsVectorFileWriter(grid_file, "utf-8", fields, QGis.WKBPoint , self._crs, "ESRI Shapefile") f = QgsFeature() for key, val in db.iteritems(): (zone_str, x, y) = key.split(' ') # point were aggregated to grid's bottom-left corner # add half grid size to place point at center of grid point = QgsPoint(int(x)*DEFAULT_GRID_SIZE+(DEFAULT_GRID_SIZE/2.0), int(y)*DEFAULT_GRID_SIZE+(DEFAULT_GRID_SIZE/2.0)) f.setGeometry(QgsGeometry.fromPoint(point)) f.addAttribute(0, QVariant(point.x())) f.addAttribute(1, QVariant(point.y())) f.addAttribute(2, QVariant(val)) f.addAttribute(3, QVariant(zone_str)) writer.addFeature(f) del writer except Exception as err: remove_shapefile(grid_file) raise OperatorError("error creating joined grid: " % err, self.__class__) grid_layer = load_shapefile(grid_file, grid_layername) if not grid_layer: raise OperatorError('Error loading created grid file' % (grid_file), self.__class__) # clean up if use_db: db.close() os.remove(tmp_db_file) # done self.outputs[0].value = grid_layer self.outputs[1].value = grid_file
def __eq__(self, other): """ compare two operator data object """ logAPICall.log( 'comparing self.type(%s) and other.type(%s)' % (self.type, other.type), logAPICall.DEBUG_L2) return self.type == other.type
def buildExposure(self): """ build exposure """ self.ui.statusbar.showMessage(get_ui_string("app.status.ms.processing")) # verify current dataset to make sure can process exposure self.project.verify_data() # can not proceed if project is not ready for exposure if self.project.status != ProjectStatus.ReadyForExposure: logUICall.log(get_ui_string("project.error.NotEnoughData"), logUICall.WARNING) # show result self.tab_datainput.showVerificationResults() self.ui.mainTabs.setCurrentIndex(self.TAB_DATA) return # close current results self.tab_result.closeResult() self.ui.mainTabs.setTabEnabled(self.TAB_RESULT, True) # reset progress dialog self.progress.setVisible(True) self.progress.ui.pb_progress.setRange(0, self.project.build_exposure_total_steps()) self.progress.ui.txt_progress.clear() self.progress.ui.txt_progress.appendPlainText(get_ui_string("app.status.processing")) self.qtapp.processEvents() cancelled = False error_occured = False error_message = "" curStep = 0 try: for step in self.project.build_exposure_steps(): if cancelled or error_occured: break # use introspection to get operator class class_name = str(step.__class__) # result of above call has format # <class '...'> where ... is the class name of interest class_name = class_name[class_name.find("'")+1:class_name.rfind("'")] # update UI logAPICall.log('\t %s' % step.name, logAPICall.DEBUG) self.progress.ui.txt_progress.appendPlainText(get_ui_string('message.%s'% class_name)) self.progress.ui.pb_progress.setValue(curStep) self.qtapp.processEvents() sleep(0.5) # perform operation step.do_operation() if not self.progress.isVisible(): cancelled = True # operation successful curStep+=1 except Exception as err: # exception are thrown if data is not ready for exposure error_message = err.message error_occured = True self.progress.setVisible(False) if error_occured: # processing cancelled logUICall.log(error_message, logUICall.WARNING) self.ui.statusbar.showMessage(get_ui_string("app.status.cancelled")) elif cancelled: # processing cancelled logUICall.log(get_ui_string("app.status.cancelled"), logUICall.WARNING) self.ui.statusbar.showMessage(get_ui_string("app.status.cancelled")) else: # processing completed self.project.verify_result() self.progress.setVisible(False) # show result self.tab_result.refreshResult() self.ui.mainTabs.setTabEnabled(self.TAB_RESULT, True) self.ui.mainTabs.setCurrentIndex(self.TAB_RESULT) self.ui.statusbar.showMessage(get_ui_string("app.status.exposure.created"))
def __initialize(self, db_path): """ prepare parser - load attributes and codes from underlying db """ if self.__initialized: return logAPICall.log('initialize taxonomy from database %s' % db_path, logAPICall.DEBUG) # load attributes / code from DB for parsing conn = sqlite3.connect(db_path) c = conn.cursor() # load attribute groups # attribute group default value for fill-in to missing groups sql = """ select a.attribute, a.default_value from dic_gem_attributes g inner join dic_gem_attribute_levels a on g.attribute=a.attribute where level=1 and order_in_basic <> '' """ c.execute(sql) default_values = {} for row in c: default_values[str(row[0])] = str(row[1]) sql = """ select order_in_extended, attribute from dic_gem_attributes g order by order_in_extended asc """ c.execute(sql) output_str = [] for row in c: output_str.append(str(row[1])) self._output_str = output_str[:3] + output_str sql = """ select g.order_in_basic, g.attribute, max(a.level) levels, g.format from dic_gem_attributes g inner join dic_gem_attribute_levels a on g.attribute=a.attribute where order_in_basic <> '' group by g.order_in_basic, g.attribute, g.format order by order_in_basic """ c.execute(sql) self.__attrGroups = [] for row in c: grp = GemTaxonomyAttributeGroup( str(row[1]).strip(), int(row[0]), int(row[2]), default_values[str(row[1])], int(row[3])) self.__attrGroups.append(grp) # load attributes sql = """ select a.name, a.level, g.attribute, a.format, a.lookup_table from dic_gem_attributes g inner join dic_gem_attribute_levels a on g.attribute=a.attribute where order_in_basic <> '' group by a.name, a.level, g.attribute, a.lookup_table, a.format order by g.order_in_basic, a.level """ c.execute(sql) self.__attrs = [] for row in c: grp = self.get_attribute_group_by_name(str(row[2]).strip()) attr = GemTaxonomyAttribute( str(row[0]).strip(), grp, int(row[1]), None, int(row[3])) attr.lookup_table = str(row[4]) grp.add_attribute(attr) self.__attrs.append(attr) # load codes sql = """select code, description, scope from %s""" self.__codes = {} for attr in self.__attrs: if attr.lookup_table == "": continue c.execute(sql % attr.lookup_table) for row in c: code_value = str(row[0]).strip() code = TaxonomyAttributeCode(attr, code_value, str(row[1]).strip(), str(row[2]).strip()) self.__codes[code_value] = code attr.add_code(code) # load rules sql = """ select parent_table, child_table, parent_code, child_code from GEM_RULES """ c.execute(sql) self.rules = {} for row in c: rule_key = '%s|%s' % (str(row[0]).strip(), str(row[1]).strip()) parent_code = self.__codes[str(row[2]).strip()] child_code = self.__codes[str(row[3]).strip()] if not self.rules.has_key(rule_key): self.rules[rule_key] = {} rule = self.rules[rule_key] if not rule.has_key(parent_code): rule[parent_code] = [] rule[parent_code].append(child_code) conn.close() self.__initialized = True
def do_operation(self): """ perform footprint load operation """ # input/output data checking already done during property set zone_layer = self.inputs[0].value # make sure input is correct # NOTE: these checks cannot be performed at set input time # because the data layer maybe is not loaded yet self._test_layer_loaded(zone_layer) x_off = self._x_off y_off = self._y_off extent = zone_layer.extent() [x_min, y_min, x_max, y_max] = [ extent.xMinimum(), extent.yMinimum(), extent.xMaximum(), extent.yMaximum() ] # create grid based on extent of given region tmp_grid1 = 'grid_' + get_unique_filename() tmp_grid1_file = self._tmp_dir + tmp_grid1 + '.shp' try: self._write_grid_shapefile(tmp_grid1_file, x_min, y_min, x_max, y_max, x_off, y_off) except: remove_shapefile(tmp_grid1_file) raise OperatorError('error creating temporary grid', self.__class__) tmp_grid1_layer = load_shapefile(tmp_grid1_file, tmp_grid1) # temporary grid for joined shape with all grid points not within region removed tmp_grid2 = 'grid_' + get_unique_filename() tmp_grid2_file = self._tmp_dir + tmp_grid2 + '.shp' tmp_grid2_layer = None try: analyzer = QgsOverlayAnalyzer() analyzer.intersection(tmp_grid1_layer, zone_layer, tmp_grid2_file) tmp_grid2_layer = load_shapefile(tmp_grid2_file, tmp_grid2) except: raise OperatorError('error creating grid', self.__class__) # create result layer grid_layername = 'grid_%s' % get_unique_filename() grid_file = self._tmp_dir + grid_layername + '.shp' try: writer = QgsVectorFileWriter(grid_file, "utf-8", self._fields, QGis.WKBPoint, self._crs, "ESRI Shapefile") f = QgsFeature() lon_idx = layer_field_index(tmp_grid2_layer, self._lon_field) lat_idx = layer_field_index(tmp_grid2_layer, self._lat_field) for _f in layer_features(tmp_grid2_layer): lon = _f.attributeMap()[lon_idx].toDouble()[0] lat = _f.attributeMap()[lat_idx].toDouble()[0] f.setGeometry(QgsGeometry.fromPoint(QgsPoint(lon, lat))) f.addAttribute(0, QVariant(lon)) f.addAttribute(1, QVariant(lat)) writer.addFeature(f) del writer except Exception as err: logAPICall.log(str(err), logAPICall.ERROR) raise OperatorError('error writing out grid', self.__class__) grid_layer = load_shapefile(grid_file, grid_layername) if not grid_layer: raise OperatorError('Error loading result grid file' % (grid_file), self.__class__) # clean up del analyzer, tmp_grid1_layer, tmp_grid2_layer remove_shapefile(tmp_grid1_file) remove_shapefile(tmp_grid2_file) self.outputs[0].value = grid_layer self.outputs[1].value = grid_file
def __initialize(self, db_path): """ prepare parser - load attributes and codes from underlying db """ if self.__initialized: return logAPICall.log('initialize taxonomy from database %s' % db_path, logAPICall.DEBUG) # load attributes / code from DB for parsing conn = sqlite3.connect(db_path) c = conn.cursor() # load attribute groups # attribute group default value for fill-in to missing groups sql = """ select a.attribute, a.default_value from dic_gem_attributes g inner join dic_gem_attribute_levels a on g.attribute=a.attribute where level=1 and order_in_basic <> '' """ c.execute(sql) default_values = {} for row in c: default_values[str(row[0])]=str(row[1]) sql = """ select order_in_extended, attribute from dic_gem_attributes g order by order_in_extended asc """ c.execute(sql) output_str = [] for row in c: output_str.append(str(row[1])) self._output_str = output_str[:3]+output_str sql = """ select g.order_in_basic, g.attribute, max(a.level) levels, g.format from dic_gem_attributes g inner join dic_gem_attribute_levels a on g.attribute=a.attribute where order_in_basic <> '' group by g.order_in_basic, g.attribute, g.format order by order_in_basic """ c.execute(sql) self.__attrGroups = [] for row in c: grp = GemTaxonomyAttributeGroup(str(row[1]).strip(), int(row[0]), int(row[2]), default_values[str(row[1])], int(row[3])) self.__attrGroups.append(grp) # load attributes sql = """ select a.name, a.level, g.attribute, a.format, a.lookup_table, a.default_value from dic_gem_attributes g inner join dic_gem_attribute_levels a on g.attribute=a.attribute where order_in_basic <> '' group by a.name, a.level, g.attribute, a.lookup_table, a.format, a.default_value order by g.order_in_basic, a.level """ c.execute(sql) self.__attrs = [] for row in c: grp = self.get_attribute_group_by_name(str(row[2]).strip()) attr = GemTaxonomyAttribute(str(row[0]).strip(), grp, int(row[1]), str(row[5]), int(row[3])) attr.lookup_table = str(row[4]) grp.add_attribute(attr) self.__attrs.append(attr) # load codes sql = """select code, description, scope from %s""" self.__codes = {} for attr in self.__attrs: if attr.lookup_table == "": continue c.execute(sql % attr.lookup_table) for row in c: code_value = str(row[0]).strip() code = TaxonomyAttributeCode(attr, code_value, str(row[1]).strip(), str(row[2]).strip()) self.__codes[code_value] = code attr.add_code(code) # load rules sql = """ select parent_table, child_table, parent_code, child_code from GEM_RULES """ c.execute(sql) self.rules = {} for row in c: rule_key = '%s|%s' % (str(row[0]).strip(), str(row[1]).strip()) parent_code = self.__codes[str(row[2]).strip()] child_code = self.__codes[str(row[3]).strip()] if not self.rules.has_key(rule_key): self.rules[rule_key] = {} rule = self.rules[rule_key] if not rule.has_key(parent_code): rule[parent_code] = [] rule[parent_code].append(child_code) conn.close() self.__initialized=True
def do_operation(self): """ perform create mapping scheme operation """ # input/output verification already performed during set input/ouput survey_layer = self.inputs[0].value zone_layer = self.inputs[1].value zone_field = self.inputs[2].value tax_field = self._tax_field logAPICall.log( 'survey %s, taxfield %s, zone %s, zone_field, %s' % (survey_layer.name(), tax_field, zone_layer.name(), zone_field), logAPICall.DEBUG) tmp_join_layername = 'join_%s' % get_unique_filename() tmp_join_file = self._tmp_dir + tmp_join_layername + '.shp' # load zone classes try: zone_classes = layer_field_stats(zone_layer, zone_field) except AssertionError as err: raise OperatorError(str(err), self.__class__) # merge to create stats logAPICall.log('merge survey & zone', logAPICall.DEBUG) analyzer = QgsOverlayAnalyzer() analyzer.intersection(survey_layer, zone_layer, tmp_join_file) tmp_join_layer = load_shapefile(tmp_join_file, tmp_join_layername) logAPICall.log('create mapping schemes', logAPICall.DEBUG) ms = MappingScheme(self._taxonomy) for _zone, _count in zone_classes.iteritems(): stats = Statistics(self._taxonomy) ms.assign(MappingSchemeZone(_zone), stats) # loop through all input features zone_idx = layer_field_index(tmp_join_layer, zone_field) tax_idx = layer_field_index(tmp_join_layer, tax_field) area_idx = layer_field_index(tmp_join_layer, AREA_FIELD_NAME) cost_idx = layer_field_index(tmp_join_layer, COST_FIELD_NAME) for _f in layer_features(tmp_join_layer): _zone_str = str(_f.attributeMap()[zone_idx].toString()) _tax_str = str(_f.attributeMap()[tax_idx].toString()) additional = {} _area = _f.attributeMap()[area_idx].toDouble()[0] if _area > 0: additional = {StatisticNode.AverageSize: _area} _cost = _f.attributeMap()[cost_idx].toDouble()[0] if _cost > 0: additional = {StatisticNode.UnitCost: _cost} logAPICall.log('zone %s => %s' % (_zone_str, _tax_str), logAPICall.DEBUG_L2) try: ms.get_assignment_by_name(_zone_str).add_case( _tax_str, self._parse_order, self._parse_modifiers, additional) except TaxonomyParseError as perr: logAPICall.log( "error parsing case %s, %s" % (str(_tax_str), str(perr)), logAPICall.WARNING) # store data in output for _zone, _stats in ms.assignments(): _stats.finalize() _stats.get_tree().value = _zone.name # clean up del tmp_join_layer, analyzer remove_shapefile(tmp_join_file) self.outputs[0].value = ms
def build_workflow(self, project): """ create workflow for creating exposure """ workflow = Workflow() still_needs_count = True still_needs_zone = True still_needs_ms = True # footprint loading logAPICall.log('checking footprint data', logAPICall.DEBUG) if project.fp_type == FootprintTypes.FootprintHt: self.load_footprint(project, workflow, True) still_needs_count = False elif project.fp_type == FootprintTypes.Footprint: self.load_footprint(project, workflow, False) still_needs_count = False # population grid loading if project.popgrid_type == PopGridTypes.Grid: self.load_popgrid(project, workflow) still_needs_count = False # survey loading logAPICall.log('checking survey data', logAPICall.DEBUG) if project.survey_type == SurveyTypes.CompleteSurvey: self.load_survey(project, workflow, True) still_needs_count = False still_needs_zone = False still_needs_ms = False elif project.survey_type == SurveyTypes.SampledSurvey: self.load_survey(project, workflow, False) # zones loading logAPICall.log('checking zone data', logAPICall.DEBUG) if project.zone_type == ZonesTypes.LanduseCount: self.load_zone(project, workflow, True) still_needs_count = False still_needs_zone = False elif project.zone_type == ZonesTypes.Landuse: self.load_zone(project, workflow, False) still_needs_zone = False # need to load MS logAPICall.log('checking ms', logAPICall.DEBUG) if getattr(project, 'ms', None) is not None and type( project.ms) == MappingScheme: workflow.operator_data['ms'] = OperatorData( OperatorDataTypes.MappingScheme, project.ms) still_needs_ms = False logAPICall.log('checking if dataset is complete', logAPICall.DEBUG) if still_needs_zone: workflow.add_error(WorkflowErrors.NeedsZone) if still_needs_count: workflow.add_error(WorkflowErrors.NeedsCount) if still_needs_ms: workflow.add_error(WorkflowErrors.NeedsMS) # if data set is not complete # return a workflow that is not ready if workflow.has_error(): return workflow logAPICall.log('add exposure building operation', logAPICall.DEBUG) # select appropriate operator(s) to apply mapping scheme try: self.build_processing_chain(project, workflow) workflow.ready = True except WorkflowException as err: # error building processing chain # workflow is not ready workflow.add_error(err.error) return workflow
def do_operation(self): # input/output verification not performed yet fp_layer = self.inputs[0].value area_field = self.inputs[1].value ht_field = self.inputs[2].value zone_layer = self.inputs[3].value zone_field = self.inputs[4].value svy_layer = self.inputs[5].value # make sure required data fields are populated area_idx = layer_field_index(fp_layer, area_field) if area_idx == -1: raise OperatorError( "Field %s does not exist in %s" % (area_field, fp_layer.name()), self.__class__) ht_idx = layer_field_index(fp_layer, ht_field) if ht_idx == -1: raise OperatorError( "Field %s does not exist in %s" % (ht_field, fp_layer.name()), self.__class__) zone_idx = layer_field_index(zone_layer, zone_field) if zone_idx == -1: raise OperatorError( "Field %s does not exist in %s" % (zone_field, zone_layer.name()), self.__class__) svy_samp_idx = layer_field_index(svy_layer, GRP_FIELD_NAME) if svy_samp_idx == -1: raise OperatorError( "Field %s does not exist in %s" % (GRP_FIELD_NAME, svy_layer.name()), self.__class__) svy_ht_idx = layer_field_index(svy_layer, HT_FIELD_NAME) if svy_ht_idx == -1: raise OperatorError( "Field %s does not exist in %s" % (HT_FIELD_NAME, svy_layer.name()), self.__class__) svy_size_idx = layer_field_index(svy_layer, AREA_FIELD_NAME) if svy_size_idx == -1: raise OperatorError("Field %s does not exist in %s" % (AREA_FIELD_NAME, svy_layer.name())) tax_idx = layer_field_index(svy_layer, TAX_FIELD_NAME) if tax_idx == -1: raise OperatorError("Field %s does not exist in %s" % (TAX_FIELD_NAME, svy_layer.name())) # load zone classes # the operations below must be performed for each zone try: zone_classes = layer_field_stats(zone_layer, zone_field) except AssertionError as err: raise OperatorError(str(err), self.__class__) # join survey with zones logAPICall.log('merge survey & zone', logAPICall.DEBUG) tmp_join_layername = 'join_%s' % get_unique_filename() tmp_join_file = self._tmp_dir + tmp_join_layername + '.shp' analyzer = QgsOverlayAnalyzer() analyzer.intersection(svy_layer, zone_layer, tmp_join_file) tmp_join_layer = load_shapefile(tmp_join_file, tmp_join_layername) logAPICall.log('compile zone statistics', logAPICall.DEBUG) zone_idx = layer_field_index(tmp_join_layer, zone_field) svy_samp_idx = layer_field_index(tmp_join_layer, GRP_FIELD_NAME) svy_ht_idx = layer_field_index(tmp_join_layer, HT_FIELD_NAME) svy_size_idx = layer_field_index(tmp_join_layer, AREA_FIELD_NAME) if svy_size_idx == -1: raise OperatorError("Field %s does not exist in %s" % (AREA_FIELD_NAME, svy_layer.name())) tax_idx = layer_field_index(tmp_join_layer, TAX_FIELD_NAME) if tax_idx == -1: raise OperatorError("Field %s does not exist in %s" % (TAX_FIELD_NAME, svy_layer.name())) # empty fields for holding the stats _zone_n_exp, _zone_p_exp, _zone_a_exp, _zone_e_exp = {}, {}, {}, {} _zone_group_counts, _zone_group_stories, _zone_group_weight = {}, {}, {} _zone_total_area, _zone_total_count, _zone_total_ht = {}, {}, {} for _zone in zone_classes.iterkeys(): _zone_n_exp[_zone] = {} _zone_p_exp[_zone] = {} _zone_a_exp[_zone] = {} _zone_e_exp[_zone] = {} _zone_group_counts[_zone] = {} _zone_group_stories[_zone] = {} _zone_group_weight[_zone] = {} _zone_total_area[_zone] = 0 _zone_total_count[_zone] = 0 _zone_total_ht[_zone] = 0 # associate group to ratio value for _rec in layer_features(tmp_join_layer): _ht = _rec.attributeMap()[svy_ht_idx].toInt()[0] _samp_grp = str(_rec.attributeMap()[svy_samp_idx].toString()) _tax_str = str(_rec.attributeMap()[tax_idx].toString()) try: self._taxonomy.parse(_tax_str) self.increment_dict(_zone_group_counts[_zone], _samp_grp, 1) self.increment_dict(_zone_group_stories[_zone], _samp_grp, _ht) except Exception as err: logAPICall.log("Error processing record %s" % err, logAPICall.WARNING) for _zone in zone_classes.iterkeys(): if len(_zone_group_counts[_zone]) != 3: raise OperatorError("Survey must have 3 sampling groups", self.__class__) cmp_value = -1 for _grp, _count in _zone_group_counts[_zone].iteritems(): if cmp_value == -1: cmp_value = _count if cmp_value != _count: raise OperatorError( "Survey groups must have same number of samples", self.__class__) # sort by stories group_stories_for_sort = {} for _grp, _ht in _zone_group_stories[_zone].iteritems(): group_stories_for_sort[_ht] = _grp sorted_keys = group_stories_for_sort.keys() sorted_keys.sort() # assign group to weight for idx, key in enumerate(sorted_keys): _zone_group_weight[_zone][ group_stories_for_sort[key]] = self.weights[idx] # aggregate values from survey for each building type # - count (n) # - floor area (p) # - total area (a) for _f in layer_features(tmp_join_layer): _zone_str = str(_f.attributeMap()[zone_idx].toString()) _tax_str = str(_f.attributeMap()[tax_idx].toString()) _sample_grp = str(_f.attributeMap()[svy_samp_idx].toString()) _sample_size = _f.attributeMap()[svy_size_idx].toDouble()[0] _sample_ht = _f.attributeMap()[svy_size_idx].toDouble()[0] group_weight = _zone_group_weight[_zone] try: self._taxonomy.parse(_tax_str) self.increment_dict(_zone_n_exp[_zone_str], _tax_str, group_weight[_sample_grp]) self.increment_dict(_zone_p_exp[_zone_str], _tax_str, _sample_size * group_weight[_sample_grp]) self.increment_dict( _zone_a_exp[_zone_str], _tax_str, _sample_size * _ht * group_weight[_sample_grp]) self.increment_dict(_zone_e_exp[_zone_str], _tax_str, 0) except Exception as err: logAPICall.log( "error processing sample with building type: %s" % _tax_str, logAPICall.WARNING) pass # adjust ratio using footprint ht/area tmp_join_layername2 = 'join_%s' % get_unique_filename() tmp_join_file2 = self._tmp_dir + tmp_join_layername2 + '.shp' analyzer = QgsOverlayAnalyzer() analyzer.intersection(fp_layer, zone_layer, tmp_join_file2) tmp_join_layer2 = load_shapefile(tmp_join_file2, tmp_join_layername) zone_idx = layer_field_index(tmp_join_layer2, zone_field) area_idx = layer_field_index(tmp_join_layer2, area_field) ht_idx = layer_field_index(tmp_join_layer2, ht_field) for _f in layer_features(tmp_join_layer2): _zone_str = str(_f.attributeMap()[zone_idx].toString()) _area = _f.attributeMap()[area_idx].toDouble()[0] _ht = _f.attributeMap()[ht_idx].toDouble()[0] _zone_total_area[_zone_str] += _area _zone_total_count[_zone_str] += 1 _zone_total_ht[_zone_str] += _ht # calculate building ratios for each zone for _zone in zone_classes.iterkeys(): # for total count (n) and area (a) e_nt_cluster_total = sum(_zone_n_exp[_zone].itervalues()) e_at_cluster_total = sum(_zone_a_exp[_zone].itervalues()) # E[A] estimated total building area for zone e_at_total = _zone_total_area[_zone] * _zone_total_ht[ _zone] / _zone_total_count[_zone] # calculate expected values for t, e_at_cluster in _zone_a_exp[_zone].iteritems(): e_nt_cluster = _zone_n_exp[_zone][t] if e_at_cluster == 0 or e_at_total == 0: # area is missing, use count instead _zone_e_exp[_zone][t] = e_nt_cluster / e_nt_cluster_total _zone_a_exp[_zone][t] = 0 else: # use ratio of area over total area # E[f(t)] building fraction based on sampled area e_ft_cluster = e_at_cluster / e_at_cluster_total # E[G(t)] average area per building e_gt_cluster = e_at_cluster / e_nt_cluster # E[A(t)] estimated total building area for zone for building type e_at = e_at_total * e_ft_cluster # E[N(t)] estimated total number of buildings zone-wide by type e_nt = e_at / e_gt_cluster _zone_e_exp[_zone][t] = e_nt _zone_a_exp[_zone][t] = e_ft_cluster # convert the building ratios logAPICall.log('create mapping scheme for zones', logAPICall.DEBUG) ms = MappingScheme(self._taxonomy) for _zone in zone_classes.iterkeys(): # create mapping scheme for zone stats = Statistics(self._taxonomy) # use building ratio to create statistic for _tax_str, _e_exp in _zone_e_exp[_zone].iteritems(): stats.add_case(_tax_str, self._parse_order, self._parse_modifiers, add_times=int(_e_exp * 1000)) # finalize call is required stats.finalize() ms.assign(MappingSchemeZone(_zone), stats) # clean up del tmp_join_layer, analyzer remove_shapefile(tmp_join_file) # assign output self.outputs[0].value = ms self.outputs[1].value = _zone_a_exp