def import_measure(headers, measure_row): """Create a score.""" print measure_row[headers.index('Tijdvak')] measure_type, measure_type_created = MeasureType.objects.get_or_create( code=measure_row[headers.index('MATCODE')]) if measure_row[headers.index('Tijdvak')].replace(' ', '') == '': period = None else: period = MeasurePeriod.objects.get( description=measure_row[headers.index('Tijdvak')]) import_raw_json = simplejson.dumps(measure_row, indent=4) initiator = Organization.objects.get( description=measure_row[headers.index('Initiatiefnemer')]) measure_kwargs = { # KRW matident => Measure.ident 'ident': measure_row[headers.index('MATIDENT')], 'is_KRW_measure': True, 'geometry': None, 'measure_type': measure_type, 'title': measure_row[headers.index('Maatregelnaam')], 'period': period, 'import_source': Measure.SOURCE_KRW_PORTAAL, 'datetime_in_source': None, 'import_raw': import_raw_json, 'aggregation_type': Measure.AGGREGATION_TYPE_MIN, 'description': "", 'value': measure_row[headers.index('Omvang')], 'investment_costs': None, 'exploitation_costs': None, 'executive': None, 'initiator': initiator, 'valid': True, 'in_sgbp': measure_row[headers.index('Rapporteren')] == 'SGBP', 'is_indicator': True} measure = Measure(**measure_kwargs) measure.save() return measure
def insert(self, krw_measures): """Insert new measures.""" count = 0 for krw_measure in krw_measures: matident = krw_measure.identificatie.matident measure = Measure.objects.filter(ident=matident) if measure.exists(): logger.warning("MATIDENT {} exists.".format(matident)) continue try: logger.debug("insert measure {}.".format(matident)) measure = Measure(**krw_measure.measureAsDict()) measure.save() measure.categories = krw_measure.krwcategories measure.areas = krw_measure.geldenVoorWaterbeheerGebied.modelAreas measure.waterbodies = krw_measure.geldenVoorWaterbeheerGebied.modelWaterBodies measure.save() count += 1 except: logger.exception("Cannot insert measure {}.".format(matident)) logger.info("Parsed measures {}.".format(len(krw_measures))) logger.info("Inserted measures {}.".format(count))
def import_measures(filename): for rec in _records(filename): measure_type, measure_type_created = MeasureType.objects.get_or_create( code=rec['matcode'], ) unit, unit_created = Unit.objects.get_or_create( code=rec['mateenh'], ) # vvv Decided from examination of screenshots from KRW portal initiator = Organization.objects.get(description=rec['uitvoerder']) datetime_in_source = datetime.datetime.strptime( rec['datum'], '%Y-%m-%d %H:%M:%S.%f', ) import_raw_json = simplejson.dumps( rec, indent=4, ) if rec['tijdvak'] == 'onbekend': period = None else: period = MeasurePeriod.objects.get(description=rec['tijdvak']) measure_kwargs = { # KRW matident => Measure.ident 'ident': rec['matident'], 'is_KRW_measure': True, 'geometry': None, 'measure_type': measure_type, 'title': rec['matnaam'], 'period': period, 'import_source': Measure.SOURCE_KRW_PORTAAL, 'datetime_in_source': datetime_in_source, 'import_raw': import_raw_json, 'aggregation_type': Measure.AGGREGATION_TYPE_MIN, 'description': rec['toelichting'], 'value': rec['matomv'], 'investment_costs': rec['investkosten'], 'exploitation_costs': rec['exploitkosten'], 'executive': None, 'initiator': initiator, 'valid': True, 'in_sgbp': rec['par_sgbp'], 'is_indicator': True, } measure = Measure(**measure_kwargs) measure.save() # Add waterbodies # They can be None, or single, or comma separated if rec['locatie'] is None: locations = [] else: locations = rec['locatie'].split(', ') for area_ident in locations: if area_ident.startswith('NL'): corrected_area_ident = area_ident else: corrected_area_ident = 'NL' + area_ident waterbody = WaterBody.objects.get( area_ident=corrected_area_ident, ) waterbody.save() measure.waterbodies.add(waterbody) data_set_set = set([w.area.data_set for w in measure.waterbodies.all()]) if None in data_set_set: data_set_set.remove(None) if len(data_set_set) == 1: # If dataset is unambiguously defined by the associated area's, # Use it for the measure, too. measure.data_set = data_set_set.pop() measure.save() # Add some categories krw_category, created = MeasureCategory.objects.get_or_create( name='KRW', ) measure.categories.add(krw_category) boolean_categories = [ 'wb21', 'n2000', 'gwb', ] for c in boolean_categories: if rec[c] == '1': category, category_created = _get_or_create( model=MeasureCategory, get_kwargs={'name': c}, ) measure.categories.add(category) if rec['thema'] is not None: category, category_created = _get_or_create( model=MeasureCategory, get_kwargs={'name': rec['thema']}, ) measure.categories.add(category) # Add a measurestatusmoment for the status at import measure_status = MeasureStatus.objects.get( name=rec['maatregelstatus'], ) measure_status_date = datetime.date(year=2010, month=1, day=1) measure_status_moment = MeasureStatusMoment( measure=measure, status=measure_status, planning_date=measure_status_date, description=MSM_DESCRIPTION_IMPORT, ) measure_status_moment.save() # Add fundingorganizations for n in ['1', '2', '3']: if rec['kostenpercent' + n] == '0': continue cost_carrier = rec['kostendrager' + n] cost_percentage = _to_float_or_none(rec['kostenpercent' + n]) organization = Organization.objects.get(description=cost_carrier) funding_organization = FundingOrganization( percentage=cost_percentage, organization=organization, measure=measure, ) funding_organization.save()