def handle(self, *args, **kwargs):
project_field = kwargs.get("project_field")
initiative_field = kwargs.get("initiative_field")
filename = kwargs.get("matchfile")
Project = apps.get_model("infrastructure", "Project")
Initiative = apps.get_model("infrastructure", "Initiative")
data = json.load(filename)
for item in data:
proj_name = clean_string(item.get(project_field, None))
init_name = clean_string(item.get(initiative_field, None))
if proj_name and init_name:
self.stdout.write('Looking up "{}" for "{}"'.format(proj_name, init_name))
proj = None
try:
proj = Project.objects.get(name=proj_name)
except Exception as e:
self.stderr.write(str(e))
if proj:
self.stdout.write('Found "{}"'.format(proj.name))
init = None
try:
init = Initiative.objects.get(name=init_name)
except Exception as e:
self.stderr.write(str(e))
if init:
self.stdout.write('Found "{}", adding to "{}"'.format(init.name, proj.name))
if not proj.initiatives.filter(id__contains=init.id).exists():
proj.initiatives.add(init)
self.stderr.write(self.style.SUCCESS("Matched!"))
else:
self.stderr.write(self.style.WARNING("Project already associated with initiative"))
self.stdout.write("---")
def test_trim_whitespace(self):
"""Leading, trailing, and double spaces are removed."""
self.assertEqual(clean_string(' I love spaces'), 'I love spaces')
self.assertEqual(clean_string('I love spaces '), 'I love spaces')
self.assertEqual(clean_string('I love spaces'),
'I love spaces')
def test_strip_quotes(self):
"""Optionally quotes are stripped."""
self.assertEqual(clean_string('\'ok\''), 'ok')
self.assertEqual(clean_string('\'ok\'', stripquotes=False), '\'ok\'')
self.assertEqual(clean_string('"ok"'), 'ok')
self.assertEqual(clean_string('"ok"', stripquotes=False), '"ok"')
def test_default_value(self):
"""Default value is used for non-string arguments."""
self.assertEqual(clean_string(None), '')
self.assertEqual(clean_string(None, default='blip'), 'blip')
self.assertEqual(clean_string(1.0), '')
self.assertEqual(clean_string(1.0, default='invalid'), 'invalid')
def transform_project_data(item):
name = clean_string(item.get("project_title"), default=None)
return {
"name": name,
"slug": slugify(name, allow_unicode=True),
"start_year": parse_int(item.get("project_start_year")),
"start_month": parse_int(item.get("project_start_month")),
"start_day": parse_int(item.get("project_start_day")),
"status": project_status_from_statuses(item.get("project_status")),
"sources": make_url_list(item.get("sources")),
"notes": clean_string(item.get("notes")),
"commencement_year": parse_int(item.get("commencement_year")),
"commencement_month": parse_int(item.get("commencement_month")),
"commencement_day": parse_int(item.get("commencement_day")),
"total_cost": parse_int(item.get("total_project_cost")),
"total_cost_currency": item.get("total_project_cost_currency"),
"planned_completion_year": parse_int(item.get("planned_year_of_completion")),
"planned_completion_month": parse_int(item.get("planned_month_of_completion")),
"planned_completion_day": parse_int(item.get("planned_day_of_completion")),
"new": evaluate_project_new_value(item.get("new")),
"infrastructure_type": infrastructure_type_object(item.get('infrastructure_type')),
"initiative": initiative_object(item.get('program_initiative')),
"collection_stage": collection_stage_from_value(item.get('collection_stage')),
"verified_path": coerce_to_boolean_or_null(item.get('verified_path'))
}
def handle(self, *args, **kwargs):
project_field = kwargs.get('project_field')
initiative_field = kwargs.get('initiative_field')
filename = kwargs.get('matchfile')
Project = apps.get_model("infrastructure", "Project")
Initiative = apps.get_model("infrastructure", "Initiative")
data = json.load(filename)
for item in data:
proj_name = clean_string(item.get(project_field, None))
init_name = clean_string(item.get(initiative_field, None))
if proj_name and init_name:
self.stdout.write('Looking up "{}" for "{}"'.format(proj_name, init_name))
proj = None
try:
proj = Project.objects.get(name=proj_name)
except Exception as e:
self.stderr.write(str(e))
if proj:
self.stdout.write('Found "{}"'.format(proj.name))
init = None
try:
init = Initiative.objects.get(name=init_name)
except Exception as e:
self.stderr.write(str(e))
if init:
self.stdout.write('Found "{}", adding to "{}"'.format(init.name, proj.name))
if not proj.initiatives.filter(id__contains=init.id).exists():
proj.initiatives.add(init)
self.stderr.write(self.style.SUCCESS('Matched!'))
else:
self.stderr.write(self.style.WARNING('Project already associated with initiative'))
self.stdout.write("---")
def test_strip_newlines(self):
"""Optionally newlines are stripped."""
self.assertEqual(clean_string('Hello\nWorld'), 'Hello World')
self.assertEqual(clean_string('Hello\nWorld', stripnewlines=False),
'Hello\nWorld')
self.assertEqual(clean_string('Hello\rWorld'), 'Hello World')
self.assertEqual(clean_string('Hello\rWorld', stripnewlines=False),
'Hello\rWorld')
def handle(self, *args, **kwargs):
infiles = kwargs.get('files')
self.verbosity = kwargs.get('verbosity')
self.dry_run = kwargs.get('dry_run')
# Attempt to parse additional attributes
raw_attrs = kwargs.get('attributes')
self.attributes = self._create_attributes(raw_attrs) if raw_attrs else {}
if self.verbosity > 1:
self.stdout.write(self.style.NOTICE('3D geometries will be flattened!'))
for f in infiles:
path, filename = os.path.split(f)
fname, fext = os.path.splitext(filename)
self.stdout.write("Processing '{}'".format(fname))
self.attributes['source'] = clean_string(filename)
try:
geo_store = geostore_from_file(
f, fname,
verbosity=self.verbosity,
geo_attributes=self.attributes,
create=True if not self.dry_run else False
)
if geo_store:
self.stdout.write(self.style.SUCCESS("Created '{}'".format(geo_store.identifier)))
else:
self.stderr.write(self.style.ERROR("Unable to create Geostore for '{}'".format(filename)))
except Exception as e:
raise CommandError(e)
def transform_project_data(item):
name = clean_string(item.get("project_title"), default=None)
return {
"name":
name,
"slug":
slugify(name, allow_unicode=True),
"start_year":
parse_int(item.get("project_start_year")),
"start_month":
parse_int(item.get("project_start_month")),
"start_day":
parse_int(item.get("project_start_day")),
"status":
project_status_from_statuses(item.get("project_status")),
"sources":
make_url_list(item.get("sources")),
"notes":
clean_string(item.get("notes")),
"commencement_year":
parse_int(item.get("commencement_year")),
"commencement_month":
parse_int(item.get("commencement_month")),
"commencement_day":
parse_int(item.get("commencement_day")),
"total_cost":
parse_int(item.get("total_project_cost")),
"total_cost_currency":
item.get("total_project_cost_currency"),
"planned_completion_year":
parse_int(item.get("planned_year_of_completion")),
"planned_completion_month":
parse_int(item.get("planned_month_of_completion")),
"planned_completion_day":
parse_int(item.get("planned_day_of_completion")),
"new":
evaluate_project_new_value(item.get("new")),
"infrastructure_type":
infrastructure_type_object(item.get('infrastructure_type')),
"initiative":
initiative_object(item.get('program_initiative')),
"collection_stage":
collection_stage_from_value(item.get('collection_stage')),
"verified_path":
coerce_to_boolean_or_null(item.get('verified_path'))
}
def transform_initiative_data(item):
name = clean_string(item.get("program_initiative_name", item.get("program_initiative")))
return{
"name": name,
"slug": slugify(name, allow_unicode=True),
"initiative_type": initiative_type_object(item.get('initiative_type')),
"appeared_year": parse_int(item.get('first_appearance_year')),
"appeared_month": parse_int(item.get('first_appearance_month')),
"appeared_day": parse_int(item.get('first_appearance_date')),
}
def transform_initiative_data(item):
name = clean_string(
item.get("program_initiative_name", item.get("program_initiative")))
return {
"name": name,
"slug": slugify(name, allow_unicode=True),
"initiative_type": initiative_type_object(item.get('initiative_type')),
"appeared_year": parse_int(item.get('first_appearance_year')),
"appeared_month": parse_int(item.get('first_appearance_month')),
"appeared_day": parse_int(item.get('first_appearance_date')),
}
def geostore_from_file(source, label, verbosity=0, geo_attributes={}, create=True):
try:
ds = DataSource(source)
except Exception as e:
logger.error(str(e))
raise e
geo_store = None
if create:
geo_store = GeometryStore.objects.create(label=label[:LABEL_CHAR_LIMIT], attributes=geo_attributes)
else:
logger.info('Processing, but not saving geodata')
if verbosity > 2:
logger.info('3D geometries will be flattened!')
for layer in ds:
layer_name = layer.name.split('.')[0]
if verbosity > 2:
logger.info('Layer "{}": {} {}s'.format(layer_name, len(layer), layer.geom_type.name))
for feat in layer:
# Remove 3rd dimension
geom = feat.geom.clone()
geom.coord_dim = 2
if create:
# Create attributes dict from geom fields
data = {f.name.lower(): f.value for f in feat}
if 'name' in data:
data['name'] = clean_string(data['name'])
# Add source information to attributes
data['layer'] = layer_name
if verbosity > 2:
logger.info('Feature "{}": {}'.format(
data.get('name', layer_name), feat.geom_type))
if geom.geos:
geom_label = data.get('name') or "{}: {}".format(label, layer_name)
params = {
'label': geom_label[:LABEL_CHAR_LIMIT],
'geom': geom.geos,
'attributes': data
}
if geom.geom_type == 'Point':
geo_store.points.create(**params)
elif geom.geom_type == 'LineString':
geo_store.lines.create(**params)
elif geom.geom_type == 'Polygon':
geo_store.polygons.create(**params)
if verbosity > 1:
logger.info("Geometry '{}' found and saved".format(geom_label))
elif verbosity > 1:
logger.info("Geometry '{label}: {layer_name}' found, but not saved".format(
label=label, layer_name=layer_name))
return geo_store
def transform_organization(item):
if isinstance(name_field, typing.Iterable) and not isinstance(name_field, str):
name_values = list(filter(lambda x: item.get(x), name_field))
if name_values:
org_name = item.get(name_values[0])
else:
return None
else:
org_name = clean_string(item.get(name_field))
return {
'name': org_name,
'slug': slugify(org_name, allow_unicode=True)
}
def transform_organization(item):
if isinstance(name_field,
typing.Iterable) and not isinstance(name_field, str):
name_values = list(filter(lambda x: item.get(x), name_field))
if name_values:
org_name = item.get(name_values[0])
else:
return None
else:
org_name = clean_string(item.get(name_field))
return {
'name': org_name,
'slug': slugify(org_name, allow_unicode=True)
}
def process_regions_data(value, recurse=True, inner_key='region_name'):
if isinstance(value, str):
if value.startswith('"') or value.startswith('\''):
splitchar = value[0]
regions_raw = [x.strip(',') for x in value.split(splitchar) if x]
yield from process_regions_data(regions_raw, recurse=True)
else:
yield {'name': clean_string(value)}
elif isinstance(value, list):
if recurse:
for item in value:
yield from process_regions_data(item, recurse=False)
elif isinstance(value, dict):
yield from process_regions_data(value.get(inner_key, None),
recurse=False)
def process_regions_data(value, recurse=True, inner_key='region_name'):
if isinstance(value, str):
if value.startswith('"') or value.startswith('\''):
splitchar = value[0]
regions_raw = [x.strip(',') for x in value.split(splitchar) if x]
yield from process_regions_data(regions_raw, recurse=True)
else:
yield {
'name': clean_string(value)
}
elif isinstance(value, list):
if recurse:
for item in value:
yield from process_regions_data(item, recurse=False)
elif isinstance(value, dict):
yield from process_regions_data(value.get(inner_key, None), recurse=False)
def handle(self, *args, **kwargs):
infiles = kwargs.get('files')
self.verbosity = kwargs.get('verbosity')
self.dry_run = kwargs.get('dry_run')
# Attempt to parse additional attributes
raw_attrs = kwargs.get('attributes')
self.attributes = self._create_attributes(
raw_attrs) if raw_attrs else {}
if self.verbosity > 1:
self.stdout.write(
self.style.NOTICE('3D geometries will be flattened!'))
for f in infiles:
path, filename = os.path.split(f)
fname, fext = os.path.splitext(filename)
self.stdout.write("Processing '{}'".format(fname))
self.attributes['source'] = clean_string(filename)
try:
geo_store = geostore_from_file(
f,
fname,
verbosity=self.verbosity,
geo_attributes=self.attributes,
create=True if not self.dry_run else False)
if geo_store:
self.stdout.write(
self.style.SUCCESS("Created '{}'".format(
geo_store.identifier)))
else:
self.stderr.write(
self.style.ERROR(
"Unable to create Geostore for '{}'".format(
filename)))
except Exception as e:
raise CommandError(e)
def transform_infrastructuretype_data(item):
name = clean_string(item.get("infrastructure_type_name"))
return {
'name': name,
"slug": slugify(name, allow_unicode=True),
}
def transform_initiative_type_data(item):
value = item if isinstance(item, str) else item.get('initiative_type')
if not value:
return None
name = clean_string(value)
return {"name": name, 'slug': slugify(name, allow_unicode=True)}
def transform_infrastructuretype_data(item):
name = clean_string(item.get("infrastructure_type_name"))
return {
'name': name,
"slug": slugify(name, allow_unicode=True),
}
def transform_initiative_type_data(item):
value = item if isinstance(item, str) else item.get('initiative_type')
if not value:
return None
name = clean_string(value)
return {"name": name, 'slug': slugify(name, allow_unicode=True)}
def test_replace_question_marks(self):
"""Question marks should converted to dashes."""
self.assertEqual(clean_string('Why?'), 'Why–')
def test_replace_unicode_quotes(self):
"""Unicode left and right quotes should be replaced by single quotes."""
self.assertEqual(clean_string('This is \u2018great\u2019'),
'This is \'great\'')
