def verify_result(self): """ run data quality tests """ builder = WorkflowBuilder(self.operator_options) try: verify_workflow = builder.build_verify_result_workflow(self) except WorkflowException as err: raise SIDDException( "error creating workflow for result verification\n%s" % err) # process workflow for step in verify_workflow.nextstep(): try: step.do_operation() except Exception as err: logAPICall.log(err, logAPICall.WARNING) pass self.quality_reports = {} if verify_workflow.operator_data.has_key('frag_report'): self.quality_reports[ 'fragmentation'] = verify_workflow.operator_data[ 'frag_report'].value if verify_workflow.operator_data.has_key('count_report'): self.quality_reports['count'] = verify_workflow.operator_data[ 'count_report'].value try: if self.zone_type == ZonesTypes.LanduseCount and self.output_type == OutputTypes.Grid: self.quality_reports['count']['_note'] = '' except: pass logAPICall.log('result verification completed', logAPICall.INFO)
def verify_result(self): """ run data quality tests """ builder = WorkflowBuilder(self.operator_options) try: verify_workflow = builder.build_verify_result_workflow(self) except WorkflowException as err: raise SIDDException("error creating workflow for result verification\n%s" % err) # process workflow for step in verify_workflow.nextstep(): try: step.do_operation() except Exception as err: logAPICall.log(err, logAPICall.WARNING) pass self.quality_reports={} if verify_workflow.operator_data.has_key('frag_report'): self.quality_reports['fragmentation'] = verify_workflow.operator_data['frag_report'].value if verify_workflow.operator_data.has_key('count_report'): self.quality_reports['count'] = verify_workflow.operator_data['count_report'].value try: if self.zone_type == ZonesTypes.LanduseCount and self.output_type == OutputTypes.Grid: self.quality_reports['count']['_note'] = '' except: pass logAPICall.log('result verification completed', logAPICall.INFO)
def export_data(self): """ export exposure data """ builder = WorkflowBuilder(self.operator_options) try: export_workflow = builder.build_export_workflow(self) except WorkflowException as err: raise SIDDException("error creating workflow for exporting data\n%s" % err) try: # process workflow export_workflow.process() logAPICall.log('data export completed', logAPICall.INFO) except Exception as err: raise SIDDException("error exporting data\n" % err)
def verify_data(self): """ verify existing data and create workflow """ # build workflow based on current data builder = WorkflowBuilder(self.operator_options) self.workflow = builder.build_workflow(self) if self.workflow.ready: self.status = ProjectStatus.ReadyForExposure else: self.status = ProjectStatus.ReadyForMS self.errors = self.workflow.errors self.exposure = None logAPICall.log('input verification completed', logAPICall.INFO)
def export_data(self): """ export exposure data """ builder = WorkflowBuilder(self.operator_options) try: export_workflow = builder.build_export_workflow(self) except WorkflowException as err: raise SIDDException( "error creating workflow for exporting data\n%s" % err) try: # process workflow export_workflow.process() logAPICall.log('data export completed', logAPICall.INFO) except Exception as err: raise SIDDException("error exporting data\n" % err)
def load_ms(self, path): """ load mapping scheme from XML """ if self.zone_type != ZonesTypes.None: self.create_empty_ms() builder= WorkflowBuilder(self.operator_options) ms_workflow = builder.build_load_ms_workflow(self, path) ms_workflow.process() ms = ms_workflow.operator_data['ms'].value if self.ms is not None: # include existing zones from current ms new_zones = [zone.name for zone in ms.zones] for existing_zone in self.ms.zones: try: new_zones.index(existing_zone.name) except: # not found statistics = Statistics(self.ms.taxonomy) zone = MappingSchemeZone(existing_zone.name) ms.assign(zone, statistics) self.ms = ms
def load_ms(self, path): """ load mapping scheme from XML """ if self.zone_type != ZonesTypes.None: self.create_empty_ms() builder = WorkflowBuilder(self.operator_options) ms_workflow = builder.build_load_ms_workflow(self, path) ms_workflow.process() ms = ms_workflow.operator_data['ms'].value if self.ms is not None: # include existing zones from current ms new_zones = [zone.name for zone in ms.zones] for existing_zone in self.ms.zones: try: new_zones.index(existing_zone.name) except: # not found statistics = Statistics(self.ms.taxonomy) zone = MappingSchemeZone(existing_zone.name) ms.assign(zone, statistics) self.ms = ms
def export_ms(self, path, export_format): """ export mapping scheme according to given format see constants.MSExportTypes for type supported """ if self.ms is None: raise SIDDException('Mapping Scheme is required for this action') builder= WorkflowBuilder(self.operator_options) try: if export_format == MSExportTypes.XML: export_workflow = builder.build_export_ms_workflow(self, path) else: export_workflow = builder.build_export_distribution_workflow(self, path) export_workflow.process() logAPICall.log('data export completed', logAPICall.INFO) except WorkflowException: return False except Exception as err: logAPICall.log(err, logAPICall.ERROR) return False
def load_data(self, input_param, layer, output_file): input_file = getattr(self, input_param, None) if input_file is not None: builder = WorkflowBuilder(self.operator_options) # create workflow if input_param == 'fp_file': workflow = builder.build_load_fp_workflow(self) elif input_param == 'zone_file': workflow = builder.build_load_zones_workflow(self) elif input_param == 'survey_file': workflow = builder.build_load_survey_workflow(self) elif input_param == 'popgrid_file': workflow = builder.build_load_popgrid_workflow(self) else: raise Exception('Data Type Not Recognized %s' % input_param) if not workflow.ready: raise Exception('Cannot load data with %s' % input_param) workflow.process() logAPICall.log('data file %s loaded' % input_file, logAPICall.INFO) return workflow.operator_data[layer].value, workflow.operator_data[output_file].value
def export_ms(self, path, export_format): """ export mapping scheme according to given format see constants.MSExportTypes for type supported """ if self.ms is None: raise SIDDException('Mapping Scheme is required for this action') builder = WorkflowBuilder(self.operator_options) try: if export_format == MSExportTypes.XML: export_workflow = builder.build_export_ms_workflow(self, path) else: export_workflow = builder.build_export_distribution_workflow( self, path) export_workflow.process() logAPICall.log('data export completed', logAPICall.INFO) except WorkflowException: return False except Exception as err: logAPICall.log(err, logAPICall.ERROR) return False
def do_build_ms(self, isEmpty=False, useSampling=False): """ create mapping scheme """ builder = WorkflowBuilder(self.operator_options) # force reload existing survey self.survey = None # create workflow if useSampling: ms_workflow = builder.build_sampling_ms_workflow(self) else: ms_workflow = builder.build_ms_workflow(self, isEmpty) if not ms_workflow.ready: raise SIDDException(ms_workflow.errors) # process workflow ms_workflow.process() self.ms = ms_workflow.operator_data['ms'].value if useSampling: self.zone_stats = ms_workflow.operator_data['zone_stats'].value for zone, stats in self.ms.assignments(): stats.refresh_leaves() logAPICall.log('mapping scheme created', logAPICall.INFO) self.require_save = True
def load_data(self, input_param, layer, output_file): input_file = getattr(self, input_param, None) if input_file is not None: builder = WorkflowBuilder(self.operator_options) # create workflow if input_param == 'fp_file': workflow = builder.build_load_fp_workflow(self) elif input_param == 'zone_file': workflow = builder.build_load_zones_workflow(self) elif input_param == 'survey_file': workflow = builder.build_load_survey_workflow(self) elif input_param == 'popgrid_file': workflow = builder.build_load_popgrid_workflow(self) else: raise Exception('Data Type Not Recognized %s' % input_param) if not workflow.ready: raise Exception('Cannot load data with %s' % input_param) workflow.process() logAPICall.log('data file %s loaded' % input_file, logAPICall.INFO) return workflow.operator_data[layer].value, workflow.operator_data[ output_file].value
def test_WorkflowBuilder(self): logging.debug('test_BuildWorkflow') def get_run_exception(func, param): try: func(param) except Exception as ex: import traceback traceback.print_exc() return ex return None # empty proj/ms should be enough for testing (proj, proj_file) = self.test_CreateProject(True) ms = MappingScheme(self.taxonomy) builder = WorkflowBuilder(self.operator_options) # test cases raising exception ################### # test case, empty project, should have errors NeedsZone, NeedsCount, NeedsMS workflow = builder.build_workflow(proj) self.assertTrue(not workflow.ready) self.assertEqual(len(workflow.errors), 3) self.assertListEqual(workflow.errors, [WorkflowErrors.NeedsZone, WorkflowErrors.NeedsCount, WorkflowErrors.NeedsMS]) # test case, only zone, should raise exception need count proj.set_zones(ZonesTypes.Landuse, self.zone2_path, self.zone2_field) workflow = builder.build_workflow(proj) self.assertTrue(not workflow.ready) self.assertEqual(len(workflow.errors), 2) self.assertListEqual(workflow.errors, [WorkflowErrors.NeedsCount, WorkflowErrors.NeedsMS]) # test case, zone / footprint, should raise exception need ms proj.set_footprint(FootprintTypes.Footprint, self.fp_path) workflow = builder.build_workflow(proj) self.assertTrue(not workflow.ready) self.assertEqual(len(workflow.errors), 1) self.assertListEqual(workflow.errors, [WorkflowErrors.NeedsMS]) # complete footprint / zone / ms to zone, no exception proj.ms = ms proj.set_output_type(OutputTypes.Zone) workflow = builder.build_workflow(proj) self.assertTrue(workflow.ready) self.assertEqual(len(workflow.errors), 0) # test cases no exception ################### # complete footprint / zone / ms to grid, no exception proj.set_output_type(OutputTypes.Grid) workflow = builder.build_workflow(proj) self.assertTrue(workflow.ready) self.assertEqual(len(workflow.errors), 0) # test case, zonecount and ms to grid, no exception proj.set_footprint(FootprintTypes.None) # remove footprint proj.set_zones(ZonesTypes.LanduseCount, self.zone_path, self.zone_field, self.bldgcount_field) proj.ms = ms proj.set_output_type(OutputTypes.Grid) workflow = builder.build_workflow(proj) self.assertTrue(workflow.ready) self.assertEqual(len(workflow.errors), 0) # test case, zonecount and ms to zone, no exception proj.set_output_type(OutputTypes.Zone) workflow = builder.build_workflow(proj) self.assertTrue(workflow.ready) self.assertEqual(len(workflow.errors), 0) # test case, complete survey, no exception proj.set_survey(SurveyTypes.CompleteSurvey, self.survey_path) workflow = builder.build_workflow(proj) self.assertTrue(workflow.ready) self.assertEqual(len(workflow.errors), 0) # clean up del proj os.remove(proj_file)