示例#1
0
    def test_BuildMS(self, skipTest=False):
        import csv
        survey = csv.reader(open(self.survey_file, 'r'),
                            delimiter=',',
                            quotechar='"')
        # skip header, there is probably a better way to do this
        survey.next()

        stats = Statistics(self.taxonomy)
        _count = 0

        for row in survey:
            tax_string = row[2]
            stats.add_case(tax_string, parse_order=self.ms_parse_order)
        stats.finalize()

        ms = MappingScheme(self.taxonomy)
        ms_zone = MappingSchemeZone('ALL')
        ms.assign(ms_zone, stats)
        #ms.save(self.ms_file)
        if skipTest:
            return ms

        ms2 = MappingScheme(self.taxonomy)
        ms2.read(self.ms_file)

        self.assertEqual(
            ms.get_assignment_by_name("ALL").to_xml().strip().__len__(),
            ms2.get_assignment_by_name("ALL").to_xml().strip().__len__())
示例#2
0
    def do_operation(self):
        """ perform create mapping scheme operation """

        # input/output verification already performed during set input/ouput
        ms = MappingScheme(self._taxonomy)
        zone = MappingSchemeZone('ALL')
        stats = Statistics(self._taxonomy)
        stats.finalize()
        stats.get_tree().value = zone.name
        ms.assign(zone, stats)

        self.outputs[0].value = ms
示例#3
0
    def test_LoadMS(self, skipTest=False, statsOnly=True):
        ms = MappingScheme(self.taxonomy)
        ms.read(self.ms_file)

        if skipTest:
            if statsOnly:
                return ms.get_assignment_by_name("ALL")
            else:
                return ms

        stats = ms.get_assignment_by_name("ALL")
        attributes = stats.get_attributes(stats.get_tree())
        self.assertEqual(sorted(attributes), sorted(self.ms_parse_order))
示例#4
0
    def do_operation(self):
        """ perform create mapping scheme operation """

        # input/output verification already performed during set input/ouput
        zone_layer = self.inputs[0].value
        zone_field = self.inputs[1].value

        # load zone
        try:
            zone_classes = layer_field_stats(zone_layer, zone_field)
        except AssertionError as err:
            raise OperatorError(str(err), self.__class__)

        # merge to create stats
        ms = MappingScheme(self._taxonomy)
        for _zone, _count in zone_classes.iteritems():
            stats = Statistics(self._taxonomy)
            stats.finalize()
            stats.get_tree().value = _zone
            ms.assign(MappingSchemeZone(_zone), stats)

        self.outputs[0].value = ms
示例#5
0
    def do_operation(self):
        """ perform create mapping scheme operation """

        # input/output verification already performed during set input/ouput
        survey_layer = self.inputs[0].value
        tax_field = self._tax_field

        # merge to create stats
        ms = MappingScheme(self._taxonomy)
        stats = Statistics(self._taxonomy)
        ms.assign(MappingSchemeZone('ALL'), stats)

        # loop through all input features
        tax_idx = layer_field_index(survey_layer, tax_field)
        area_idx = layer_field_index(survey_layer, AREA_FIELD_NAME)
        cost_idx = layer_field_index(survey_layer, COST_FIELD_NAME)

        for _f in layer_features(survey_layer):
            _tax_str = str(_f.attributeMap()[tax_idx].toString())
            additional = {}
            _area = _f.attributeMap()[area_idx].toDouble()[0]
            if _area > 0:
                additional = {StatisticNode.AverageSize: _area}
            _cost = _f.attributeMap()[cost_idx].toDouble()[0]
            if _cost > 0:
                additional = {StatisticNode.UnitCost: _cost}
            try:
                stats.add_case(_tax_str, self._parse_order,
                               self._parse_modifiers, additional)
            except TaxonomyParseError as perr:
                logAPICall.log(
                    "error parsing case %s, %s" % (str(_tax_str), str(perr)),
                    logAPICall.WARNING)

        # store data in output
        stats.finalize()

        self.outputs[0].value = ms
示例#6
0
    def test_WorkflowBuilder(self):
        logging.debug('test_BuildWorkflow')
        
        def get_run_exception(func, param):
            try:
                func(param)
            except Exception as ex:
                import traceback
                traceback.print_exc() 
                return ex
            return None

        # empty proj/ms should be enough for testing
        (proj, proj_file) = self.test_CreateProject(True)
        ms = MappingScheme(self.taxonomy) 
        
        builder = WorkflowBuilder(self.operator_options)

        # test cases raising exception
        ###################
        # test case, empty project, should have errors NeedsZone, NeedsCount, NeedsMS
        workflow = builder.build_workflow(proj)
        self.assertTrue(not workflow.ready)
        self.assertEqual(len(workflow.errors), 3)
        self.assertListEqual(workflow.errors, [WorkflowErrors.NeedsZone, 
                                               WorkflowErrors.NeedsCount, 
                                               WorkflowErrors.NeedsMS])
        
        # test case, only zone, should raise exception need count
        proj.set_zones(ZonesTypes.Landuse, self.zone2_path, self.zone2_field)
        workflow = builder.build_workflow(proj)        
        self.assertTrue(not workflow.ready)
        self.assertEqual(len(workflow.errors), 2)
        self.assertListEqual(workflow.errors, [WorkflowErrors.NeedsCount, 
                                               WorkflowErrors.NeedsMS])
        
        # test case, zone / footprint, should raise exception need ms 
        proj.set_footprint(FootprintTypes.Footprint, self.fp_path)
        workflow = builder.build_workflow(proj)        
        self.assertTrue(not workflow.ready)
        self.assertEqual(len(workflow.errors), 1)
        self.assertListEqual(workflow.errors, [WorkflowErrors.NeedsMS])

        # complete footprint / zone / ms to zone, no exception
        proj.ms = ms 
        proj.set_output_type(OutputTypes.Zone)
        workflow = builder.build_workflow(proj)
        self.assertTrue(workflow.ready)
        self.assertEqual(len(workflow.errors), 0)
        
        # test cases no exception
        ###################

        # complete footprint / zone / ms to grid, no exception 
        proj.set_output_type(OutputTypes.Grid)
        workflow = builder.build_workflow(proj)
        self.assertTrue(workflow.ready)
        self.assertEqual(len(workflow.errors), 0)

        # test case, zonecount and ms to grid, no exception
        proj.set_footprint(FootprintTypes.None) # remove footprint
        proj.set_zones(ZonesTypes.LanduseCount, self.zone_path, self.zone_field, self.bldgcount_field)
        proj.ms = ms
        proj.set_output_type(OutputTypes.Grid)
        workflow = builder.build_workflow(proj)
        self.assertTrue(workflow.ready)
        self.assertEqual(len(workflow.errors), 0)
        
        # test case, zonecount and ms to zone, no exception        
        proj.set_output_type(OutputTypes.Zone)
        workflow = builder.build_workflow(proj)
        self.assertTrue(workflow.ready)
        self.assertEqual(len(workflow.errors), 0)

        # test case, complete survey, no exception
        proj.set_survey(SurveyTypes.CompleteSurvey, self.survey_path)
        workflow = builder.build_workflow(proj)
        self.assertTrue(workflow.ready)
        self.assertEqual(len(workflow.errors), 0)
        
        # clean up
        del proj
        os.remove(proj_file)
示例#7
0
    def do_operation(self):
        # input/output verification not performed yet
        fp_layer = self.inputs[0].value
        area_field = self.inputs[1].value
        ht_field = self.inputs[2].value
        zone_layer = self.inputs[3].value
        zone_field = self.inputs[4].value
        svy_layer = self.inputs[5].value

        # make sure required data fields are populated
        area_idx = layer_field_index(fp_layer, area_field)
        if area_idx == -1:
            raise OperatorError(
                "Field %s does not exist in %s" %
                (area_field, fp_layer.name()), self.__class__)
        ht_idx = layer_field_index(fp_layer, ht_field)
        if ht_idx == -1:
            raise OperatorError(
                "Field %s does not exist in %s" % (ht_field, fp_layer.name()),
                self.__class__)
        zone_idx = layer_field_index(zone_layer, zone_field)
        if zone_idx == -1:
            raise OperatorError(
                "Field %s does not exist in %s" %
                (zone_field, zone_layer.name()), self.__class__)
        svy_samp_idx = layer_field_index(svy_layer, GRP_FIELD_NAME)
        if svy_samp_idx == -1:
            raise OperatorError(
                "Field %s does not exist in %s" %
                (GRP_FIELD_NAME, svy_layer.name()), self.__class__)
        svy_ht_idx = layer_field_index(svy_layer, HT_FIELD_NAME)
        if svy_ht_idx == -1:
            raise OperatorError(
                "Field %s does not exist in %s" %
                (HT_FIELD_NAME, svy_layer.name()), self.__class__)
        svy_size_idx = layer_field_index(svy_layer, AREA_FIELD_NAME)
        if svy_size_idx == -1:
            raise OperatorError("Field %s does not exist in %s" %
                                (AREA_FIELD_NAME, svy_layer.name()))
        tax_idx = layer_field_index(svy_layer, TAX_FIELD_NAME)
        if tax_idx == -1:
            raise OperatorError("Field %s does not exist in %s" %
                                (TAX_FIELD_NAME, svy_layer.name()))

        # load zone classes
        # the operations below must be performed for each zone
        try:
            zone_classes = layer_field_stats(zone_layer, zone_field)
        except AssertionError as err:
            raise OperatorError(str(err), self.__class__)

        # join survey with zones
        logAPICall.log('merge survey & zone', logAPICall.DEBUG)
        tmp_join_layername = 'join_%s' % get_unique_filename()
        tmp_join_file = self._tmp_dir + tmp_join_layername + '.shp'
        analyzer = QgsOverlayAnalyzer()
        analyzer.intersection(svy_layer, zone_layer, tmp_join_file)
        tmp_join_layer = load_shapefile(tmp_join_file, tmp_join_layername)

        logAPICall.log('compile zone statistics', logAPICall.DEBUG)
        zone_idx = layer_field_index(tmp_join_layer, zone_field)
        svy_samp_idx = layer_field_index(tmp_join_layer, GRP_FIELD_NAME)
        svy_ht_idx = layer_field_index(tmp_join_layer, HT_FIELD_NAME)

        svy_size_idx = layer_field_index(tmp_join_layer, AREA_FIELD_NAME)
        if svy_size_idx == -1:
            raise OperatorError("Field %s does not exist in %s" %
                                (AREA_FIELD_NAME, svy_layer.name()))
        tax_idx = layer_field_index(tmp_join_layer, TAX_FIELD_NAME)
        if tax_idx == -1:
            raise OperatorError("Field %s does not exist in %s" %
                                (TAX_FIELD_NAME, svy_layer.name()))

        # empty fields for holding the stats
        _zone_n_exp, _zone_p_exp, _zone_a_exp, _zone_e_exp = {}, {}, {}, {}
        _zone_group_counts, _zone_group_stories, _zone_group_weight = {}, {}, {}
        _zone_total_area, _zone_total_count, _zone_total_ht = {}, {}, {}
        for _zone in zone_classes.iterkeys():
            _zone_n_exp[_zone] = {}
            _zone_p_exp[_zone] = {}
            _zone_a_exp[_zone] = {}
            _zone_e_exp[_zone] = {}
            _zone_group_counts[_zone] = {}
            _zone_group_stories[_zone] = {}
            _zone_group_weight[_zone] = {}
            _zone_total_area[_zone] = 0
            _zone_total_count[_zone] = 0
            _zone_total_ht[_zone] = 0

        # associate group to ratio value
        for _rec in layer_features(tmp_join_layer):
            _ht = _rec.attributeMap()[svy_ht_idx].toInt()[0]
            _samp_grp = str(_rec.attributeMap()[svy_samp_idx].toString())
            _tax_str = str(_rec.attributeMap()[tax_idx].toString())
            try:
                self._taxonomy.parse(_tax_str)
                self.increment_dict(_zone_group_counts[_zone], _samp_grp, 1)
                self.increment_dict(_zone_group_stories[_zone], _samp_grp, _ht)
            except Exception as err:
                logAPICall.log("Error processing record %s" % err,
                               logAPICall.WARNING)

        for _zone in zone_classes.iterkeys():
            if len(_zone_group_counts[_zone]) != 3:
                raise OperatorError("Survey must have 3 sampling groups",
                                    self.__class__)
            cmp_value = -1
            for _grp, _count in _zone_group_counts[_zone].iteritems():
                if cmp_value == -1:
                    cmp_value = _count
                if cmp_value != _count:
                    raise OperatorError(
                        "Survey groups must have same number of samples",
                        self.__class__)
            # sort by stories
            group_stories_for_sort = {}
            for _grp, _ht in _zone_group_stories[_zone].iteritems():
                group_stories_for_sort[_ht] = _grp
            sorted_keys = group_stories_for_sort.keys()
            sorted_keys.sort()
            # assign group to weight
            for idx, key in enumerate(sorted_keys):
                _zone_group_weight[_zone][
                    group_stories_for_sort[key]] = self.weights[idx]

        # aggregate values from survey for each building type
        # - count (n)
        # - floor area (p)
        # - total area (a)
        for _f in layer_features(tmp_join_layer):
            _zone_str = str(_f.attributeMap()[zone_idx].toString())
            _tax_str = str(_f.attributeMap()[tax_idx].toString())
            _sample_grp = str(_f.attributeMap()[svy_samp_idx].toString())
            _sample_size = _f.attributeMap()[svy_size_idx].toDouble()[0]
            _sample_ht = _f.attributeMap()[svy_size_idx].toDouble()[0]
            group_weight = _zone_group_weight[_zone]
            try:
                self._taxonomy.parse(_tax_str)
                self.increment_dict(_zone_n_exp[_zone_str], _tax_str,
                                    group_weight[_sample_grp])
                self.increment_dict(_zone_p_exp[_zone_str], _tax_str,
                                    _sample_size * group_weight[_sample_grp])
                self.increment_dict(
                    _zone_a_exp[_zone_str], _tax_str,
                    _sample_size * _ht * group_weight[_sample_grp])
                self.increment_dict(_zone_e_exp[_zone_str], _tax_str, 0)
            except Exception as err:
                logAPICall.log(
                    "error processing sample with building type: %s" %
                    _tax_str, logAPICall.WARNING)
                pass

        # adjust ratio using footprint ht/area
        tmp_join_layername2 = 'join_%s' % get_unique_filename()
        tmp_join_file2 = self._tmp_dir + tmp_join_layername2 + '.shp'
        analyzer = QgsOverlayAnalyzer()
        analyzer.intersection(fp_layer, zone_layer, tmp_join_file2)
        tmp_join_layer2 = load_shapefile(tmp_join_file2, tmp_join_layername)

        zone_idx = layer_field_index(tmp_join_layer2, zone_field)
        area_idx = layer_field_index(tmp_join_layer2, area_field)
        ht_idx = layer_field_index(tmp_join_layer2, ht_field)
        for _f in layer_features(tmp_join_layer2):
            _zone_str = str(_f.attributeMap()[zone_idx].toString())
            _area = _f.attributeMap()[area_idx].toDouble()[0]
            _ht = _f.attributeMap()[ht_idx].toDouble()[0]

            _zone_total_area[_zone_str] += _area
            _zone_total_count[_zone_str] += 1
            _zone_total_ht[_zone_str] += _ht

        # calculate building ratios for each zone
        for _zone in zone_classes.iterkeys():
            # for total count (n) and area (a)
            e_nt_cluster_total = sum(_zone_n_exp[_zone].itervalues())
            e_at_cluster_total = sum(_zone_a_exp[_zone].itervalues())
            # E[A] estimated total building area for zone
            e_at_total = _zone_total_area[_zone] * _zone_total_ht[
                _zone] / _zone_total_count[_zone]

            # calculate expected values
            for t, e_at_cluster in _zone_a_exp[_zone].iteritems():
                e_nt_cluster = _zone_n_exp[_zone][t]
                if e_at_cluster == 0 or e_at_total == 0:
                    # area is missing, use count instead
                    _zone_e_exp[_zone][t] = e_nt_cluster / e_nt_cluster_total
                    _zone_a_exp[_zone][t] = 0
                else:
                    # use ratio of area over total area
                    # E[f(t)] building fraction based on sampled area
                    e_ft_cluster = e_at_cluster / e_at_cluster_total
                    # E[G(t)] average area per building
                    e_gt_cluster = e_at_cluster / e_nt_cluster

                    # E[A(t)] estimated total building area for zone for building type
                    e_at = e_at_total * e_ft_cluster
                    # E[N(t)] estimated total number of buildings zone-wide by type
                    e_nt = e_at / e_gt_cluster

                    _zone_e_exp[_zone][t] = e_nt
                    _zone_a_exp[_zone][t] = e_ft_cluster

        # convert the building ratios
        logAPICall.log('create mapping scheme for zones', logAPICall.DEBUG)
        ms = MappingScheme(self._taxonomy)
        for _zone in zone_classes.iterkeys():
            # create mapping scheme for zone
            stats = Statistics(self._taxonomy)

            # use building ratio to create statistic
            for _tax_str, _e_exp in _zone_e_exp[_zone].iteritems():
                stats.add_case(_tax_str,
                               self._parse_order,
                               self._parse_modifiers,
                               add_times=int(_e_exp * 1000))
            # finalize call is required
            stats.finalize()
            ms.assign(MappingSchemeZone(_zone), stats)

        # clean up
        del tmp_join_layer, analyzer
        remove_shapefile(tmp_join_file)

        # assign output
        self.outputs[0].value = ms
        self.outputs[1].value = _zone_a_exp
示例#8
0
    def do_operation(self):
        """ perform create mapping scheme operation """

        # input/output verification already performed during set input/ouput
        survey_layer = self.inputs[0].value
        zone_layer = self.inputs[1].value
        zone_field = self.inputs[2].value
        tax_field = self._tax_field

        logAPICall.log(
            'survey %s, taxfield %s, zone %s, zone_field, %s' %
            (survey_layer.name(), tax_field, zone_layer.name(), zone_field),
            logAPICall.DEBUG)
        tmp_join_layername = 'join_%s' % get_unique_filename()
        tmp_join_file = self._tmp_dir + tmp_join_layername + '.shp'

        # load zone classes
        try:
            zone_classes = layer_field_stats(zone_layer, zone_field)
        except AssertionError as err:
            raise OperatorError(str(err), self.__class__)

        # merge to create stats
        logAPICall.log('merge survey & zone', logAPICall.DEBUG)
        analyzer = QgsOverlayAnalyzer()
        analyzer.intersection(survey_layer, zone_layer, tmp_join_file)
        tmp_join_layer = load_shapefile(tmp_join_file, tmp_join_layername)

        logAPICall.log('create mapping schemes', logAPICall.DEBUG)
        ms = MappingScheme(self._taxonomy)
        for _zone, _count in zone_classes.iteritems():
            stats = Statistics(self._taxonomy)
            ms.assign(MappingSchemeZone(_zone), stats)

        # loop through all input features
        zone_idx = layer_field_index(tmp_join_layer, zone_field)
        tax_idx = layer_field_index(tmp_join_layer, tax_field)
        area_idx = layer_field_index(tmp_join_layer, AREA_FIELD_NAME)
        cost_idx = layer_field_index(tmp_join_layer, COST_FIELD_NAME)

        for _f in layer_features(tmp_join_layer):
            _zone_str = str(_f.attributeMap()[zone_idx].toString())
            _tax_str = str(_f.attributeMap()[tax_idx].toString())
            additional = {}
            _area = _f.attributeMap()[area_idx].toDouble()[0]
            if _area > 0:
                additional = {StatisticNode.AverageSize: _area}
            _cost = _f.attributeMap()[cost_idx].toDouble()[0]
            if _cost > 0:
                additional = {StatisticNode.UnitCost: _cost}
            logAPICall.log('zone %s => %s' % (_zone_str, _tax_str),
                           logAPICall.DEBUG_L2)
            try:
                ms.get_assignment_by_name(_zone_str).add_case(
                    _tax_str, self._parse_order, self._parse_modifiers,
                    additional)
            except TaxonomyParseError as perr:
                logAPICall.log(
                    "error parsing case %s, %s" % (str(_tax_str), str(perr)),
                    logAPICall.WARNING)

        # store data in output
        for _zone, _stats in ms.assignments():
            _stats.finalize()
            _stats.get_tree().value = _zone.name

        # clean up
        del tmp_join_layer, analyzer
        remove_shapefile(tmp_join_file)

        self.outputs[0].value = ms
示例#9
0
    def sync(self, direction=SyncModes.Read):
        """ synchronize data with DB """
        if self.project_file is None or self.db is None:
            raise SIDDProjectException(ProjectErrors.FileNotSet)

        if (direction == SyncModes.Read):
            logAPICall.log("reading existing datasets from DB",
                           logAPICall.DEBUG)

            # load footprint
            fp_type = self.get_project_data('data.footprint')
            if fp_type is None:
                self.footprint = None
                self.fp_file = None
                self.fp_type = FootprintTypes.None
            else:
                if (fp_type == str(FootprintTypes.FootprintHt)):
                    self.set_footprint(
                        FootprintTypes.FootprintHt,
                        self.get_project_data('data.footprint.file'),
                        self.get_project_data('data.footprint.ht_field'))
                else:
                    self.set_footprint(
                        FootprintTypes.Footprint,
                        self.get_project_data('data.footprint.file'))
            # load survey
            survey_type = self.get_project_data('data.survey')
            if survey_type is None:
                self.survey = None
                self.survey_file = None
                self.survey_type = SurveyTypes.None
            else:
                if self.get_project_data('data.survey.is_complete') == 'True':
                    self.set_survey(SurveyTypes.CompleteSurvey,
                                    self.get_project_data('data.survey.file'))
                else:
                    self.set_survey(SurveyTypes.SampledSurvey,
                                    self.get_project_data('data.survey.file'))

            # load zone
            zone_type = self.get_project_data('data.zones')
            if zone_type is None:
                self.zones = None
                self.zone_file = None
                self.zone_type = ZonesTypes.None
            else:
                if zone_type == str(ZonesTypes.Landuse):
                    self.set_zones(
                        ZonesTypes.Landuse,
                        self.get_project_data('data.zones.file'),
                        self.get_project_data('data.zones.class_field'))
                else:
                    self.set_zones(
                        ZonesTypes.LanduseCount,
                        self.get_project_data('data.zones.file'),
                        self.get_project_data('data.zones.class_field'),
                        self.get_project_data('data.zones.count_field'),
                        self.get_project_data('data.zones.area_field'))

            # load popgrid
            pop_type = self.get_project_data('data.popgrid')
            if pop_type is None:
                self.popgrid = None
                self.popgrid_type = PopGridTypes.None
                self.popgrid_file = None
                self.pop_field = ''
            else:
                self.set_popgrid(
                    PopGridTypes.Grid,
                    self.get_project_data('data.popgrid.file'),
                    self.get_project_data('data.popgrid.pop_field'),
                    self.get_project_data('data.popgrid.pop_to_bldg'))

            # load output type
            output_type = self.get_project_data('data.output')
            if output_type == "Zone":
                self.output_type = OutputTypes.Zone
            else:
                self.output_type = OutputTypes.Grid

            # load mapping scheme
            ms_str = self.get_project_data('data.ms')
            if ms_str is not None:
                self.ms = MappingScheme(None)
                self.ms.from_text(ms_str)

            use_sampling = self.get_project_data('stratified.sampling')
            if use_sampling is None:
                self.operator_options[
                    'stratified.sampling'] = False  # default to not use sampling method
            else:
                self.operator_options['stratified.sampling'] = (
                    use_sampling == "True")

            # load taxonomy related options
            attr_order = self.get_project_data('attribute.order')
            if attr_order is not None:
                self.operator_options['attribute.order'] = json.loads(
                    attr_order)
            for attr in self.operator_options['taxonomy'].attributes:
                attr_options = self.get_project_data(attr.name)
                if attr_options is not None:
                    self.operator_options[attr.name] = json.loads(attr_options)

            extrapolation = self.get_project_data("proc.extrapolation")
            if extrapolation is not None:
                # NOTE: converting extrapolation to enum is required
                #       because comparison of str vs. enum is not valid
                self.operator_options["proc.extrapolation"] = makeEnum(
                    ExtrapolateOptions, extrapolation)
            else:
                self.operator_options[
                    "proc.extrapolation"] = ExtrapolateOptions.Fraction

            # load export settings
            export_type = self.get_project_data('export.type')
            if export_type is not None:
                self.export_type = makeEnum(ExportTypes, export_type)
            export_path = self.get_project_data('export.path')
            if export_path is not None:
                self.export_path = export_path

        else:
            logAPICall.log("store existing datasets into DB", logAPICall.DEBUG)
            # store footprint
            if self.fp_type == FootprintTypes.None:
                self.save_project_data('data.footprint', None)
                self.save_project_data('data.footprint.file', None)
                self.save_project_data('data.footprint.ht_field', None)
            else:
                self.save_project_data('data.footprint', self.fp_type)
                self.save_project_data('data.footprint.file', self.fp_file)
                if self.fp_type == FootprintTypes.FootprintHt:
                    self.save_project_data('data.footprint.ht_field',
                                           self.fp_ht_field)
                else:
                    self.save_project_data('data.footprint.ht_field', None)

            # store survey
            if self.survey_type == SurveyTypes.None:
                self.save_project_data('data.survey', None)
                self.save_project_data('data.survey.file', None)
            else:
                self.save_project_data('data.survey', self.survey_type)
                self.save_project_data('data.survey.file', self.survey_file)
                self.save_project_data(
                    'data.survey.is_complete',
                    (self.survey_type == SurveyTypes.CompleteSurvey))

            # store zone
            if self.zone_type == ZonesTypes.None:
                self.save_project_data('data.zones', None)
                self.save_project_data('data.zones.file', None)
                self.save_project_data('data.zones.class_field', None)
                self.save_project_data('data.zones.count_field', None)
            else:
                self.save_project_data('data.zones', self.zone_type)
                self.save_project_data('data.zones.file', self.zone_file)
                self.save_project_data('data.zones.class_field',
                                       self.zone_field)
                if self.zone_type == ZonesTypes.LanduseCount:
                    self.save_project_data('data.zones.count_field',
                                           self.zone_count_field)
                    self.save_project_data('data.zones.area_field',
                                           self.zone_area_field)
                else:
                    self.save_project_data('data.zones.count_field', None)
                    self.save_project_data('data.zones.area_field', None)

            # store popgrid
            if self.popgrid_type == PopGridTypes.None:
                self.save_project_data('data.popgrid', None)
                self.save_project_data('data.popgrid.file', None)
                self.save_project_data('data.popgrid.pop_field', None)
                self.save_project_data('data.popgrid.pop_to_bldg', None)
            else:
                self.save_project_data('data.popgrid', self.popgrid_type)
                self.save_project_data('data.popgrid.file', self.popgrid_file)
                self.save_project_data('data.popgrid.pop_field',
                                       self.pop_field)
                self.save_project_data('data.popgrid.pop_to_bldg',
                                       self.pop_to_bldg)

            # store output type
            self.save_project_data('data.output', self.output_type)

            # store mapping scheme
            if self.ms is None:
                self.save_project_data('data.ms', None)
            else:
                self.save_project_data('data.ms', self.ms.to_xml())

            if self.operator_options.has_key('stratified.sampling'):
                self.save_project_data(
                    'stratified.sampling',
                    self.operator_options['stratified.sampling'])

            # save taxonomy order
            if self.operator_options.has_key('attribute.order'):
                self.save_project_data(
                    'attribute.order',
                    json.dumps(self.operator_options['attribute.order']))
            for attr in self.operator_options['taxonomy'].attributes:
                if self.operator_options.has_key(attr.name):
                    self.save_project_data(
                        attr.name,
                        json.dumps(self.operator_options[attr.name]))

            # save processing attributes
            if self.operator_options.has_key("proc.extrapolation"):
                self.save_project_data(
                    "proc.extrapolation",
                    self.operator_options["proc.extrapolation"])

            # save export settings
            self.save_project_data('export.type',
                                   getattr(self, 'export_type', None))
            self.save_project_data('export.path',
                                   getattr(self, 'export_path', None))

            # flush to disk
            self.db.sync()

        # after each sync
        # project is same as db, so save no longer required
        self.require_save = False