def handle(self, *args, **kwargs): project_field = kwargs.get("project_field") initiative_field = kwargs.get("initiative_field") filename = kwargs.get("matchfile") Project = apps.get_model("infrastructure", "Project") Initiative = apps.get_model("infrastructure", "Initiative") data = json.load(filename) for item in data: proj_name = clean_string(item.get(project_field, None)) init_name = clean_string(item.get(initiative_field, None)) if proj_name and init_name: self.stdout.write('Looking up "{}" for "{}"'.format(proj_name, init_name)) proj = None try: proj = Project.objects.get(name=proj_name) except Exception as e: self.stderr.write(str(e)) if proj: self.stdout.write('Found "{}"'.format(proj.name)) init = None try: init = Initiative.objects.get(name=init_name) except Exception as e: self.stderr.write(str(e)) if init: self.stdout.write('Found "{}", adding to "{}"'.format(init.name, proj.name)) if not proj.initiatives.filter(id__contains=init.id).exists(): proj.initiatives.add(init) self.stderr.write(self.style.SUCCESS("Matched!")) else: self.stderr.write(self.style.WARNING("Project already associated with initiative")) self.stdout.write("---")
def test_trim_whitespace(self): """Leading, trailing, and double spaces are removed.""" self.assertEqual(clean_string(' I love spaces'), 'I love spaces') self.assertEqual(clean_string('I love spaces '), 'I love spaces') self.assertEqual(clean_string('I love spaces'), 'I love spaces')
def test_strip_quotes(self): """Optionally quotes are stripped.""" self.assertEqual(clean_string('\'ok\''), 'ok') self.assertEqual(clean_string('\'ok\'', stripquotes=False), '\'ok\'') self.assertEqual(clean_string('"ok"'), 'ok') self.assertEqual(clean_string('"ok"', stripquotes=False), '"ok"')
def test_default_value(self): """Default value is used for non-string arguments.""" self.assertEqual(clean_string(None), '') self.assertEqual(clean_string(None, default='blip'), 'blip') self.assertEqual(clean_string(1.0), '') self.assertEqual(clean_string(1.0, default='invalid'), 'invalid')
def transform_project_data(item): name = clean_string(item.get("project_title"), default=None) return { "name": name, "slug": slugify(name, allow_unicode=True), "start_year": parse_int(item.get("project_start_year")), "start_month": parse_int(item.get("project_start_month")), "start_day": parse_int(item.get("project_start_day")), "status": project_status_from_statuses(item.get("project_status")), "sources": make_url_list(item.get("sources")), "notes": clean_string(item.get("notes")), "commencement_year": parse_int(item.get("commencement_year")), "commencement_month": parse_int(item.get("commencement_month")), "commencement_day": parse_int(item.get("commencement_day")), "total_cost": parse_int(item.get("total_project_cost")), "total_cost_currency": item.get("total_project_cost_currency"), "planned_completion_year": parse_int(item.get("planned_year_of_completion")), "planned_completion_month": parse_int(item.get("planned_month_of_completion")), "planned_completion_day": parse_int(item.get("planned_day_of_completion")), "new": evaluate_project_new_value(item.get("new")), "infrastructure_type": infrastructure_type_object(item.get('infrastructure_type')), "initiative": initiative_object(item.get('program_initiative')), "collection_stage": collection_stage_from_value(item.get('collection_stage')), "verified_path": coerce_to_boolean_or_null(item.get('verified_path')) }
def handle(self, *args, **kwargs): project_field = kwargs.get('project_field') initiative_field = kwargs.get('initiative_field') filename = kwargs.get('matchfile') Project = apps.get_model("infrastructure", "Project") Initiative = apps.get_model("infrastructure", "Initiative") data = json.load(filename) for item in data: proj_name = clean_string(item.get(project_field, None)) init_name = clean_string(item.get(initiative_field, None)) if proj_name and init_name: self.stdout.write('Looking up "{}" for "{}"'.format(proj_name, init_name)) proj = None try: proj = Project.objects.get(name=proj_name) except Exception as e: self.stderr.write(str(e)) if proj: self.stdout.write('Found "{}"'.format(proj.name)) init = None try: init = Initiative.objects.get(name=init_name) except Exception as e: self.stderr.write(str(e)) if init: self.stdout.write('Found "{}", adding to "{}"'.format(init.name, proj.name)) if not proj.initiatives.filter(id__contains=init.id).exists(): proj.initiatives.add(init) self.stderr.write(self.style.SUCCESS('Matched!')) else: self.stderr.write(self.style.WARNING('Project already associated with initiative')) self.stdout.write("---")
def test_strip_newlines(self): """Optionally newlines are stripped.""" self.assertEqual(clean_string('Hello\nWorld'), 'Hello World') self.assertEqual(clean_string('Hello\nWorld', stripnewlines=False), 'Hello\nWorld') self.assertEqual(clean_string('Hello\rWorld'), 'Hello World') self.assertEqual(clean_string('Hello\rWorld', stripnewlines=False), 'Hello\rWorld')
def handle(self, *args, **kwargs): infiles = kwargs.get('files') self.verbosity = kwargs.get('verbosity') self.dry_run = kwargs.get('dry_run') # Attempt to parse additional attributes raw_attrs = kwargs.get('attributes') self.attributes = self._create_attributes(raw_attrs) if raw_attrs else {} if self.verbosity > 1: self.stdout.write(self.style.NOTICE('3D geometries will be flattened!')) for f in infiles: path, filename = os.path.split(f) fname, fext = os.path.splitext(filename) self.stdout.write("Processing '{}'".format(fname)) self.attributes['source'] = clean_string(filename) try: geo_store = geostore_from_file( f, fname, verbosity=self.verbosity, geo_attributes=self.attributes, create=True if not self.dry_run else False ) if geo_store: self.stdout.write(self.style.SUCCESS("Created '{}'".format(geo_store.identifier))) else: self.stderr.write(self.style.ERROR("Unable to create Geostore for '{}'".format(filename))) except Exception as e: raise CommandError(e)
def transform_initiative_data(item): name = clean_string(item.get("program_initiative_name", item.get("program_initiative"))) return{ "name": name, "slug": slugify(name, allow_unicode=True), "initiative_type": initiative_type_object(item.get('initiative_type')), "appeared_year": parse_int(item.get('first_appearance_year')), "appeared_month": parse_int(item.get('first_appearance_month')), "appeared_day": parse_int(item.get('first_appearance_date')), }
def transform_initiative_data(item): name = clean_string( item.get("program_initiative_name", item.get("program_initiative"))) return { "name": name, "slug": slugify(name, allow_unicode=True), "initiative_type": initiative_type_object(item.get('initiative_type')), "appeared_year": parse_int(item.get('first_appearance_year')), "appeared_month": parse_int(item.get('first_appearance_month')), "appeared_day": parse_int(item.get('first_appearance_date')), }
def geostore_from_file(source, label, verbosity=0, geo_attributes={}, create=True): try: ds = DataSource(source) except Exception as e: logger.error(str(e)) raise e geo_store = None if create: geo_store = GeometryStore.objects.create(label=label[:LABEL_CHAR_LIMIT], attributes=geo_attributes) else: logger.info('Processing, but not saving geodata') if verbosity > 2: logger.info('3D geometries will be flattened!') for layer in ds: layer_name = layer.name.split('.')[0] if verbosity > 2: logger.info('Layer "{}": {} {}s'.format(layer_name, len(layer), layer.geom_type.name)) for feat in layer: # Remove 3rd dimension geom = feat.geom.clone() geom.coord_dim = 2 if create: # Create attributes dict from geom fields data = {f.name.lower(): f.value for f in feat} if 'name' in data: data['name'] = clean_string(data['name']) # Add source information to attributes data['layer'] = layer_name if verbosity > 2: logger.info('Feature "{}": {}'.format( data.get('name', layer_name), feat.geom_type)) if geom.geos: geom_label = data.get('name') or "{}: {}".format(label, layer_name) params = { 'label': geom_label[:LABEL_CHAR_LIMIT], 'geom': geom.geos, 'attributes': data } if geom.geom_type == 'Point': geo_store.points.create(**params) elif geom.geom_type == 'LineString': geo_store.lines.create(**params) elif geom.geom_type == 'Polygon': geo_store.polygons.create(**params) if verbosity > 1: logger.info("Geometry '{}' found and saved".format(geom_label)) elif verbosity > 1: logger.info("Geometry '{label}: {layer_name}' found, but not saved".format( label=label, layer_name=layer_name)) return geo_store
def transform_organization(item): if isinstance(name_field, typing.Iterable) and not isinstance(name_field, str): name_values = list(filter(lambda x: item.get(x), name_field)) if name_values: org_name = item.get(name_values[0]) else: return None else: org_name = clean_string(item.get(name_field)) return { 'name': org_name, 'slug': slugify(org_name, allow_unicode=True) }
def process_regions_data(value, recurse=True, inner_key='region_name'): if isinstance(value, str): if value.startswith('"') or value.startswith('\''): splitchar = value[0] regions_raw = [x.strip(',') for x in value.split(splitchar) if x] yield from process_regions_data(regions_raw, recurse=True) else: yield {'name': clean_string(value)} elif isinstance(value, list): if recurse: for item in value: yield from process_regions_data(item, recurse=False) elif isinstance(value, dict): yield from process_regions_data(value.get(inner_key, None), recurse=False)
def process_regions_data(value, recurse=True, inner_key='region_name'): if isinstance(value, str): if value.startswith('"') or value.startswith('\''): splitchar = value[0] regions_raw = [x.strip(',') for x in value.split(splitchar) if x] yield from process_regions_data(regions_raw, recurse=True) else: yield { 'name': clean_string(value) } elif isinstance(value, list): if recurse: for item in value: yield from process_regions_data(item, recurse=False) elif isinstance(value, dict): yield from process_regions_data(value.get(inner_key, None), recurse=False)
def handle(self, *args, **kwargs): infiles = kwargs.get('files') self.verbosity = kwargs.get('verbosity') self.dry_run = kwargs.get('dry_run') # Attempt to parse additional attributes raw_attrs = kwargs.get('attributes') self.attributes = self._create_attributes( raw_attrs) if raw_attrs else {} if self.verbosity > 1: self.stdout.write( self.style.NOTICE('3D geometries will be flattened!')) for f in infiles: path, filename = os.path.split(f) fname, fext = os.path.splitext(filename) self.stdout.write("Processing '{}'".format(fname)) self.attributes['source'] = clean_string(filename) try: geo_store = geostore_from_file( f, fname, verbosity=self.verbosity, geo_attributes=self.attributes, create=True if not self.dry_run else False) if geo_store: self.stdout.write( self.style.SUCCESS("Created '{}'".format( geo_store.identifier))) else: self.stderr.write( self.style.ERROR( "Unable to create Geostore for '{}'".format( filename))) except Exception as e: raise CommandError(e)
def transform_infrastructuretype_data(item): name = clean_string(item.get("infrastructure_type_name")) return { 'name': name, "slug": slugify(name, allow_unicode=True), }
def transform_initiative_type_data(item): value = item if isinstance(item, str) else item.get('initiative_type') if not value: return None name = clean_string(value) return {"name": name, 'slug': slugify(name, allow_unicode=True)}
def test_replace_question_marks(self): """Question marks should converted to dashes.""" self.assertEqual(clean_string('Why?'), 'Why–')
def test_replace_unicode_quotes(self): """Unicode left and right quotes should be replaced by single quotes.""" self.assertEqual(clean_string('This is \u2018great\u2019'), 'This is \'great\'')