def verify_result(self):
"""
run data quality tests
"""
builder = WorkflowBuilder(self.operator_options)
try:
verify_workflow = builder.build_verify_result_workflow(self)
except WorkflowException as err:
raise SIDDException(
"error creating workflow for result verification\n%s" % err)
# process workflow
for step in verify_workflow.nextstep():
try:
step.do_operation()
except Exception as err:
logAPICall.log(err, logAPICall.WARNING)
pass
self.quality_reports = {}
if verify_workflow.operator_data.has_key('frag_report'):
self.quality_reports[
'fragmentation'] = verify_workflow.operator_data[
'frag_report'].value
if verify_workflow.operator_data.has_key('count_report'):
self.quality_reports['count'] = verify_workflow.operator_data[
'count_report'].value
try:
if self.zone_type == ZonesTypes.LanduseCount and self.output_type == OutputTypes.Grid:
self.quality_reports['count']['_note'] = ''
except:
pass
logAPICall.log('result verification completed', logAPICall.INFO)
def verify_result(self):
"""
run data quality tests
"""
builder = WorkflowBuilder(self.operator_options)
try:
verify_workflow = builder.build_verify_result_workflow(self)
except WorkflowException as err:
raise SIDDException("error creating workflow for result verification\n%s" % err)
# process workflow
for step in verify_workflow.nextstep():
try:
step.do_operation()
except Exception as err:
logAPICall.log(err, logAPICall.WARNING)
pass
self.quality_reports={}
if verify_workflow.operator_data.has_key('frag_report'):
self.quality_reports['fragmentation'] = verify_workflow.operator_data['frag_report'].value
if verify_workflow.operator_data.has_key('count_report'):
self.quality_reports['count'] = verify_workflow.operator_data['count_report'].value
try:
if self.zone_type == ZonesTypes.LanduseCount and self.output_type == OutputTypes.Grid:
self.quality_reports['count']['_note'] = ''
except:
pass
logAPICall.log('result verification completed', logAPICall.INFO)
def export_data(self):
""" export exposure data """
builder = WorkflowBuilder(self.operator_options)
try:
export_workflow = builder.build_export_workflow(self)
except WorkflowException as err:
raise SIDDException("error creating workflow for exporting data\n%s" % err)
try:
# process workflow
export_workflow.process()
logAPICall.log('data export completed', logAPICall.INFO)
except Exception as err:
raise SIDDException("error exporting data\n" % err)
def verify_data(self):
""" verify existing data and create workflow """
# build workflow based on current data
builder = WorkflowBuilder(self.operator_options)
self.workflow = builder.build_workflow(self)
if self.workflow.ready:
self.status = ProjectStatus.ReadyForExposure
else:
self.status = ProjectStatus.ReadyForMS
self.errors = self.workflow.errors
self.exposure = None
logAPICall.log('input verification completed', logAPICall.INFO)
def verify_data(self):
""" verify existing data and create workflow """
# build workflow based on current data
builder = WorkflowBuilder(self.operator_options)
self.workflow = builder.build_workflow(self)
if self.workflow.ready:
self.status = ProjectStatus.ReadyForExposure
else:
self.status = ProjectStatus.ReadyForMS
self.errors = self.workflow.errors
self.exposure = None
logAPICall.log('input verification completed', logAPICall.INFO)
def export_data(self):
""" export exposure data """
builder = WorkflowBuilder(self.operator_options)
try:
export_workflow = builder.build_export_workflow(self)
except WorkflowException as err:
raise SIDDException(
"error creating workflow for exporting data\n%s" % err)
try:
# process workflow
export_workflow.process()
logAPICall.log('data export completed', logAPICall.INFO)
except Exception as err:
raise SIDDException("error exporting data\n" % err)
def load_ms(self, path):
""" load mapping scheme from XML """
if self.zone_type != ZonesTypes.None:
self.create_empty_ms()
builder= WorkflowBuilder(self.operator_options)
ms_workflow = builder.build_load_ms_workflow(self, path)
ms_workflow.process()
ms = ms_workflow.operator_data['ms'].value
if self.ms is not None:
# include existing zones from current ms
new_zones = [zone.name for zone in ms.zones]
for existing_zone in self.ms.zones:
try:
new_zones.index(existing_zone.name)
except:
# not found
statistics = Statistics(self.ms.taxonomy)
zone = MappingSchemeZone(existing_zone.name)
ms.assign(zone, statistics)
self.ms = ms
def load_ms(self, path):
""" load mapping scheme from XML """
if self.zone_type != ZonesTypes.None:
self.create_empty_ms()
builder = WorkflowBuilder(self.operator_options)
ms_workflow = builder.build_load_ms_workflow(self, path)
ms_workflow.process()
ms = ms_workflow.operator_data['ms'].value
if self.ms is not None:
# include existing zones from current ms
new_zones = [zone.name for zone in ms.zones]
for existing_zone in self.ms.zones:
try:
new_zones.index(existing_zone.name)
except:
# not found
statistics = Statistics(self.ms.taxonomy)
zone = MappingSchemeZone(existing_zone.name)
ms.assign(zone, statistics)
self.ms = ms
def export_ms(self, path, export_format):
"""
export mapping scheme according to given format
see constants.MSExportTypes for type supported
"""
if self.ms is None:
raise SIDDException('Mapping Scheme is required for this action')
builder= WorkflowBuilder(self.operator_options)
try:
if export_format == MSExportTypes.XML:
export_workflow = builder.build_export_ms_workflow(self, path)
else:
export_workflow = builder.build_export_distribution_workflow(self, path)
export_workflow.process()
logAPICall.log('data export completed', logAPICall.INFO)
except WorkflowException:
return False
except Exception as err:
logAPICall.log(err, logAPICall.ERROR)
return False
def load_data(self, input_param, layer, output_file):
input_file = getattr(self, input_param, None)
if input_file is not None:
builder = WorkflowBuilder(self.operator_options)
# create workflow
if input_param == 'fp_file':
workflow = builder.build_load_fp_workflow(self)
elif input_param == 'zone_file':
workflow = builder.build_load_zones_workflow(self)
elif input_param == 'survey_file':
workflow = builder.build_load_survey_workflow(self)
elif input_param == 'popgrid_file':
workflow = builder.build_load_popgrid_workflow(self)
else:
raise Exception('Data Type Not Recognized %s' % input_param)
if not workflow.ready:
raise Exception('Cannot load data with %s' % input_param)
workflow.process()
logAPICall.log('data file %s loaded' % input_file, logAPICall.INFO)
return workflow.operator_data[layer].value, workflow.operator_data[output_file].value
def export_ms(self, path, export_format):
"""
export mapping scheme according to given format
see constants.MSExportTypes for type supported
"""
if self.ms is None:
raise SIDDException('Mapping Scheme is required for this action')
builder = WorkflowBuilder(self.operator_options)
try:
if export_format == MSExportTypes.XML:
export_workflow = builder.build_export_ms_workflow(self, path)
else:
export_workflow = builder.build_export_distribution_workflow(
self, path)
export_workflow.process()
logAPICall.log('data export completed', logAPICall.INFO)
except WorkflowException:
return False
except Exception as err:
logAPICall.log(err, logAPICall.ERROR)
return False
def do_build_ms(self, isEmpty=False, useSampling=False):
""" create mapping scheme """
builder = WorkflowBuilder(self.operator_options)
# force reload existing survey
self.survey = None
# create workflow
if useSampling:
ms_workflow = builder.build_sampling_ms_workflow(self)
else:
ms_workflow = builder.build_ms_workflow(self, isEmpty)
if not ms_workflow.ready:
raise SIDDException(ms_workflow.errors)
# process workflow
ms_workflow.process()
self.ms = ms_workflow.operator_data['ms'].value
if useSampling:
self.zone_stats = ms_workflow.operator_data['zone_stats'].value
for zone, stats in self.ms.assignments():
stats.refresh_leaves()
logAPICall.log('mapping scheme created', logAPICall.INFO)
self.require_save = True
def do_build_ms(self, isEmpty=False, useSampling=False):
""" create mapping scheme """
builder = WorkflowBuilder(self.operator_options)
# force reload existing survey
self.survey = None
# create workflow
if useSampling:
ms_workflow = builder.build_sampling_ms_workflow(self)
else:
ms_workflow = builder.build_ms_workflow(self, isEmpty)
if not ms_workflow.ready:
raise SIDDException(ms_workflow.errors)
# process workflow
ms_workflow.process()
self.ms = ms_workflow.operator_data['ms'].value
if useSampling:
self.zone_stats = ms_workflow.operator_data['zone_stats'].value
for zone, stats in self.ms.assignments():
stats.refresh_leaves()
logAPICall.log('mapping scheme created', logAPICall.INFO)
self.require_save = True
def load_data(self, input_param, layer, output_file):
input_file = getattr(self, input_param, None)
if input_file is not None:
builder = WorkflowBuilder(self.operator_options)
# create workflow
if input_param == 'fp_file':
workflow = builder.build_load_fp_workflow(self)
elif input_param == 'zone_file':
workflow = builder.build_load_zones_workflow(self)
elif input_param == 'survey_file':
workflow = builder.build_load_survey_workflow(self)
elif input_param == 'popgrid_file':
workflow = builder.build_load_popgrid_workflow(self)
else:
raise Exception('Data Type Not Recognized %s' % input_param)
if not workflow.ready:
raise Exception('Cannot load data with %s' % input_param)
workflow.process()
logAPICall.log('data file %s loaded' % input_file, logAPICall.INFO)
return workflow.operator_data[layer].value, workflow.operator_data[
output_file].value
def test_WorkflowBuilder(self):
logging.debug('test_BuildWorkflow')
def get_run_exception(func, param):
try:
func(param)
except Exception as ex:
import traceback
traceback.print_exc()
return ex
return None
# empty proj/ms should be enough for testing
(proj, proj_file) = self.test_CreateProject(True)
ms = MappingScheme(self.taxonomy)
builder = WorkflowBuilder(self.operator_options)
# test cases raising exception
###################
# test case, empty project, should have errors NeedsZone, NeedsCount, NeedsMS
workflow = builder.build_workflow(proj)
self.assertTrue(not workflow.ready)
self.assertEqual(len(workflow.errors), 3)
self.assertListEqual(workflow.errors, [WorkflowErrors.NeedsZone,
WorkflowErrors.NeedsCount,
WorkflowErrors.NeedsMS])
# test case, only zone, should raise exception need count
proj.set_zones(ZonesTypes.Landuse, self.zone2_path, self.zone2_field)
workflow = builder.build_workflow(proj)
self.assertTrue(not workflow.ready)
self.assertEqual(len(workflow.errors), 2)
self.assertListEqual(workflow.errors, [WorkflowErrors.NeedsCount,
WorkflowErrors.NeedsMS])
# test case, zone / footprint, should raise exception need ms
proj.set_footprint(FootprintTypes.Footprint, self.fp_path)
workflow = builder.build_workflow(proj)
self.assertTrue(not workflow.ready)
self.assertEqual(len(workflow.errors), 1)
self.assertListEqual(workflow.errors, [WorkflowErrors.NeedsMS])
# complete footprint / zone / ms to zone, no exception
proj.ms = ms
proj.set_output_type(OutputTypes.Zone)
workflow = builder.build_workflow(proj)
self.assertTrue(workflow.ready)
self.assertEqual(len(workflow.errors), 0)
# test cases no exception
###################
# complete footprint / zone / ms to grid, no exception
proj.set_output_type(OutputTypes.Grid)
workflow = builder.build_workflow(proj)
self.assertTrue(workflow.ready)
self.assertEqual(len(workflow.errors), 0)
# test case, zonecount and ms to grid, no exception
proj.set_footprint(FootprintTypes.None) # remove footprint
proj.set_zones(ZonesTypes.LanduseCount, self.zone_path, self.zone_field, self.bldgcount_field)
proj.ms = ms
proj.set_output_type(OutputTypes.Grid)
workflow = builder.build_workflow(proj)
self.assertTrue(workflow.ready)
self.assertEqual(len(workflow.errors), 0)
# test case, zonecount and ms to zone, no exception
proj.set_output_type(OutputTypes.Zone)
workflow = builder.build_workflow(proj)
self.assertTrue(workflow.ready)
self.assertEqual(len(workflow.errors), 0)
# test case, complete survey, no exception
proj.set_survey(SurveyTypes.CompleteSurvey, self.survey_path)
workflow = builder.build_workflow(proj)
self.assertTrue(workflow.ready)
self.assertEqual(len(workflow.errors), 0)
# clean up
del proj
os.remove(proj_file)