Exemplo n.º 1
0
    def verify_result(self):
        """
        run data quality tests 
        """
        builder = WorkflowBuilder(self.operator_options)
        try:
            verify_workflow = builder.build_verify_result_workflow(self)
        except WorkflowException as err:
            raise SIDDException(
                "error creating workflow for result verification\n%s" % err)
        # process workflow
        for step in verify_workflow.nextstep():
            try:
                step.do_operation()
            except Exception as err:
                logAPICall.log(err, logAPICall.WARNING)
                pass

        self.quality_reports = {}
        if verify_workflow.operator_data.has_key('frag_report'):
            self.quality_reports[
                'fragmentation'] = verify_workflow.operator_data[
                    'frag_report'].value
        if verify_workflow.operator_data.has_key('count_report'):
            self.quality_reports['count'] = verify_workflow.operator_data[
                'count_report'].value
            try:
                if self.zone_type == ZonesTypes.LanduseCount and self.output_type == OutputTypes.Grid:
                    self.quality_reports['count']['_note'] = ''
            except:
                pass

        logAPICall.log('result verification completed', logAPICall.INFO)
Exemplo n.º 2
0
    def verify_result(self):
        """
        run data quality tests 
        """
        builder = WorkflowBuilder(self.operator_options)
        try:
            verify_workflow = builder.build_verify_result_workflow(self)
        except WorkflowException as err:
            raise SIDDException("error creating workflow for result verification\n%s" % err)
        # process workflow
        for step in verify_workflow.nextstep():
            try:
                step.do_operation()
            except Exception as err:
                logAPICall.log(err, logAPICall.WARNING)
                pass                

        self.quality_reports={}
        if verify_workflow.operator_data.has_key('frag_report'):
            self.quality_reports['fragmentation'] = verify_workflow.operator_data['frag_report'].value
        if verify_workflow.operator_data.has_key('count_report'):
            self.quality_reports['count'] = verify_workflow.operator_data['count_report'].value
            try:
                if self.zone_type == ZonesTypes.LanduseCount and self.output_type == OutputTypes.Grid:
                    self.quality_reports['count']['_note'] = ''
            except:
                pass
                
        logAPICall.log('result verification completed', logAPICall.INFO)
Exemplo n.º 3
0
 def export_data(self):
     """ export exposure data """
     builder = WorkflowBuilder(self.operator_options)
     try:
         export_workflow = builder.build_export_workflow(self)
     except WorkflowException as err:
         raise SIDDException("error creating workflow for exporting data\n%s" % err)
     try:
         # process workflow 
         export_workflow.process()
         logAPICall.log('data export completed', logAPICall.INFO)            
     except Exception as err:
         raise SIDDException("error exporting data\n" % err)
Exemplo n.º 4
0
 def verify_data(self):
     """ verify existing data and create workflow """
     # build workflow based on current data
     builder = WorkflowBuilder(self.operator_options)
     self.workflow = builder.build_workflow(self)    
     
     if self.workflow.ready:
         self.status = ProjectStatus.ReadyForExposure
     else:
         self.status = ProjectStatus.ReadyForMS
     self.errors = self.workflow.errors
     self.exposure = None
     logAPICall.log('input verification completed', logAPICall.INFO)
Exemplo n.º 5
0
    def verify_data(self):
        """ verify existing data and create workflow """
        # build workflow based on current data
        builder = WorkflowBuilder(self.operator_options)
        self.workflow = builder.build_workflow(self)

        if self.workflow.ready:
            self.status = ProjectStatus.ReadyForExposure
        else:
            self.status = ProjectStatus.ReadyForMS
        self.errors = self.workflow.errors
        self.exposure = None
        logAPICall.log('input verification completed', logAPICall.INFO)
Exemplo n.º 6
0
 def export_data(self):
     """ export exposure data """
     builder = WorkflowBuilder(self.operator_options)
     try:
         export_workflow = builder.build_export_workflow(self)
     except WorkflowException as err:
         raise SIDDException(
             "error creating workflow for exporting data\n%s" % err)
     try:
         # process workflow
         export_workflow.process()
         logAPICall.log('data export completed', logAPICall.INFO)
     except Exception as err:
         raise SIDDException("error exporting data\n" % err)
Exemplo n.º 7
0
 def load_ms(self, path):
     """ load mapping scheme from XML """
     if self.zone_type != ZonesTypes.None:
         self.create_empty_ms()
     builder= WorkflowBuilder(self.operator_options)
     ms_workflow = builder.build_load_ms_workflow(self, path)        
     ms_workflow.process()
     ms = ms_workflow.operator_data['ms'].value  
     if self.ms is not None:              
         # include existing zones from current ms
         new_zones = [zone.name for zone in ms.zones]    
         for existing_zone in self.ms.zones:
             try:
                 new_zones.index(existing_zone.name)
             except:
                 # not found
                 statistics = Statistics(self.ms.taxonomy)
                 zone = MappingSchemeZone(existing_zone.name)
                 ms.assign(zone, statistics)
     self.ms = ms
Exemplo n.º 8
0
 def load_ms(self, path):
     """ load mapping scheme from XML """
     if self.zone_type != ZonesTypes.None:
         self.create_empty_ms()
     builder = WorkflowBuilder(self.operator_options)
     ms_workflow = builder.build_load_ms_workflow(self, path)
     ms_workflow.process()
     ms = ms_workflow.operator_data['ms'].value
     if self.ms is not None:
         # include existing zones from current ms
         new_zones = [zone.name for zone in ms.zones]
         for existing_zone in self.ms.zones:
             try:
                 new_zones.index(existing_zone.name)
             except:
                 # not found
                 statistics = Statistics(self.ms.taxonomy)
                 zone = MappingSchemeZone(existing_zone.name)
                 ms.assign(zone, statistics)
     self.ms = ms
Exemplo n.º 9
0
 def export_ms(self, path, export_format):
     """ 
     export mapping scheme according to given format
     see constants.MSExportTypes for type supported
     """
     if self.ms is None:
         raise SIDDException('Mapping Scheme is required for this action')
     
     builder= WorkflowBuilder(self.operator_options)
     try:
         if export_format == MSExportTypes.XML:
             export_workflow = builder.build_export_ms_workflow(self, path)
         else:
             export_workflow = builder.build_export_distribution_workflow(self, path)
         export_workflow.process()
         logAPICall.log('data export completed', logAPICall.INFO)
     except WorkflowException:
         return False
     except Exception as err:
         logAPICall.log(err, logAPICall.ERROR)
         return False
Exemplo n.º 10
0
 def load_data(self, input_param, layer, output_file):
     input_file = getattr(self, input_param, None)
     if input_file is not None:
         builder = WorkflowBuilder(self.operator_options)
         # create workflow
         if input_param == 'fp_file':
             workflow = builder.build_load_fp_workflow(self)
         elif input_param == 'zone_file':
             workflow = builder.build_load_zones_workflow(self)
         elif input_param == 'survey_file':
             workflow = builder.build_load_survey_workflow(self)
         elif input_param == 'popgrid_file':
             workflow = builder.build_load_popgrid_workflow(self)
         else:
             raise Exception('Data Type Not Recognized %s' % input_param)
         
         if not workflow.ready:
             raise Exception('Cannot load data with %s' % input_param)
         workflow.process()
         
         logAPICall.log('data file %s loaded' % input_file, logAPICall.INFO)
         return workflow.operator_data[layer].value, workflow.operator_data[output_file].value
Exemplo n.º 11
0
    def export_ms(self, path, export_format):
        """ 
        export mapping scheme according to given format
        see constants.MSExportTypes for type supported
        """
        if self.ms is None:
            raise SIDDException('Mapping Scheme is required for this action')

        builder = WorkflowBuilder(self.operator_options)
        try:
            if export_format == MSExportTypes.XML:
                export_workflow = builder.build_export_ms_workflow(self, path)
            else:
                export_workflow = builder.build_export_distribution_workflow(
                    self, path)
            export_workflow.process()
            logAPICall.log('data export completed', logAPICall.INFO)
        except WorkflowException:
            return False
        except Exception as err:
            logAPICall.log(err, logAPICall.ERROR)
            return False
Exemplo n.º 12
0
 def do_build_ms(self, isEmpty=False, useSampling=False):
     """ create mapping scheme """
     builder = WorkflowBuilder(self.operator_options)
     # force reload existing survey
     self.survey = None
     
     # create workflow 
     if useSampling:
         ms_workflow = builder.build_sampling_ms_workflow(self)
     else:
         ms_workflow = builder.build_ms_workflow(self, isEmpty)
     if not ms_workflow.ready:
         raise SIDDException(ms_workflow.errors)
     
     # process workflow 
     ms_workflow.process()
     self.ms = ms_workflow.operator_data['ms'].value
     if useSampling:
         self.zone_stats = ms_workflow.operator_data['zone_stats'].value
     for zone, stats in self.ms.assignments():
         stats.refresh_leaves()
         
     logAPICall.log('mapping scheme created', logAPICall.INFO)
     self.require_save = True
Exemplo n.º 13
0
    def do_build_ms(self, isEmpty=False, useSampling=False):
        """ create mapping scheme """
        builder = WorkflowBuilder(self.operator_options)
        # force reload existing survey
        self.survey = None

        # create workflow
        if useSampling:
            ms_workflow = builder.build_sampling_ms_workflow(self)
        else:
            ms_workflow = builder.build_ms_workflow(self, isEmpty)
        if not ms_workflow.ready:
            raise SIDDException(ms_workflow.errors)

        # process workflow
        ms_workflow.process()
        self.ms = ms_workflow.operator_data['ms'].value
        if useSampling:
            self.zone_stats = ms_workflow.operator_data['zone_stats'].value
        for zone, stats in self.ms.assignments():
            stats.refresh_leaves()

        logAPICall.log('mapping scheme created', logAPICall.INFO)
        self.require_save = True
Exemplo n.º 14
0
    def load_data(self, input_param, layer, output_file):
        input_file = getattr(self, input_param, None)
        if input_file is not None:
            builder = WorkflowBuilder(self.operator_options)
            # create workflow
            if input_param == 'fp_file':
                workflow = builder.build_load_fp_workflow(self)
            elif input_param == 'zone_file':
                workflow = builder.build_load_zones_workflow(self)
            elif input_param == 'survey_file':
                workflow = builder.build_load_survey_workflow(self)
            elif input_param == 'popgrid_file':
                workflow = builder.build_load_popgrid_workflow(self)
            else:
                raise Exception('Data Type Not Recognized %s' % input_param)

            if not workflow.ready:
                raise Exception('Cannot load data with %s' % input_param)
            workflow.process()

            logAPICall.log('data file %s loaded' % input_file, logAPICall.INFO)
            return workflow.operator_data[layer].value, workflow.operator_data[
                output_file].value
Exemplo n.º 15
0
    def test_WorkflowBuilder(self):
        logging.debug('test_BuildWorkflow')
        
        def get_run_exception(func, param):
            try:
                func(param)
            except Exception as ex:
                import traceback
                traceback.print_exc() 
                return ex
            return None

        # empty proj/ms should be enough for testing
        (proj, proj_file) = self.test_CreateProject(True)
        ms = MappingScheme(self.taxonomy) 
        
        builder = WorkflowBuilder(self.operator_options)

        # test cases raising exception
        ###################
        # test case, empty project, should have errors NeedsZone, NeedsCount, NeedsMS
        workflow = builder.build_workflow(proj)
        self.assertTrue(not workflow.ready)
        self.assertEqual(len(workflow.errors), 3)
        self.assertListEqual(workflow.errors, [WorkflowErrors.NeedsZone, 
                                               WorkflowErrors.NeedsCount, 
                                               WorkflowErrors.NeedsMS])
        
        # test case, only zone, should raise exception need count
        proj.set_zones(ZonesTypes.Landuse, self.zone2_path, self.zone2_field)
        workflow = builder.build_workflow(proj)        
        self.assertTrue(not workflow.ready)
        self.assertEqual(len(workflow.errors), 2)
        self.assertListEqual(workflow.errors, [WorkflowErrors.NeedsCount, 
                                               WorkflowErrors.NeedsMS])
        
        # test case, zone / footprint, should raise exception need ms 
        proj.set_footprint(FootprintTypes.Footprint, self.fp_path)
        workflow = builder.build_workflow(proj)        
        self.assertTrue(not workflow.ready)
        self.assertEqual(len(workflow.errors), 1)
        self.assertListEqual(workflow.errors, [WorkflowErrors.NeedsMS])

        # complete footprint / zone / ms to zone, no exception
        proj.ms = ms 
        proj.set_output_type(OutputTypes.Zone)
        workflow = builder.build_workflow(proj)
        self.assertTrue(workflow.ready)
        self.assertEqual(len(workflow.errors), 0)
        
        # test cases no exception
        ###################

        # complete footprint / zone / ms to grid, no exception 
        proj.set_output_type(OutputTypes.Grid)
        workflow = builder.build_workflow(proj)
        self.assertTrue(workflow.ready)
        self.assertEqual(len(workflow.errors), 0)

        # test case, zonecount and ms to grid, no exception
        proj.set_footprint(FootprintTypes.None) # remove footprint
        proj.set_zones(ZonesTypes.LanduseCount, self.zone_path, self.zone_field, self.bldgcount_field)
        proj.ms = ms
        proj.set_output_type(OutputTypes.Grid)
        workflow = builder.build_workflow(proj)
        self.assertTrue(workflow.ready)
        self.assertEqual(len(workflow.errors), 0)
        
        # test case, zonecount and ms to zone, no exception        
        proj.set_output_type(OutputTypes.Zone)
        workflow = builder.build_workflow(proj)
        self.assertTrue(workflow.ready)
        self.assertEqual(len(workflow.errors), 0)

        # test case, complete survey, no exception
        proj.set_survey(SurveyTypes.CompleteSurvey, self.survey_path)
        workflow = builder.build_workflow(proj)
        self.assertTrue(workflow.ready)
        self.assertEqual(len(workflow.errors), 0)
        
        # clean up
        del proj
        os.remove(proj_file)