def handle(self, *args, **kwargs):
        project_field = kwargs.get("project_field")
        initiative_field = kwargs.get("initiative_field")
        filename = kwargs.get("matchfile")

        Project = apps.get_model("infrastructure", "Project")
        Initiative = apps.get_model("infrastructure", "Initiative")

        data = json.load(filename)

        for item in data:
            proj_name = clean_string(item.get(project_field, None))
            init_name = clean_string(item.get(initiative_field, None))
            if proj_name and init_name:
                self.stdout.write('Looking up "{}" for "{}"'.format(proj_name, init_name))
                proj = None
                try:
                    proj = Project.objects.get(name=proj_name)
                except Exception as e:
                    self.stderr.write(str(e))
                if proj:
                    self.stdout.write('Found "{}"'.format(proj.name))
                    init = None
                    try:
                        init = Initiative.objects.get(name=init_name)
                    except Exception as e:
                        self.stderr.write(str(e))
                    if init:
                        self.stdout.write('Found "{}", adding to "{}"'.format(init.name, proj.name))
                        if not proj.initiatives.filter(id__contains=init.id).exists():
                            proj.initiatives.add(init)
                            self.stderr.write(self.style.SUCCESS("Matched!"))
                        else:
                            self.stderr.write(self.style.WARNING("Project already associated with initiative"))
                self.stdout.write("---")
Ejemplo n.º 2
0
    def test_trim_whitespace(self):
        """Leading, trailing, and double spaces are removed."""

        self.assertEqual(clean_string('    I love spaces'), 'I love spaces')
        self.assertEqual(clean_string('I love spaces    '), 'I love spaces')
        self.assertEqual(clean_string('I     love     spaces'),
                         'I love spaces')
Ejemplo n.º 3
0
    def test_strip_quotes(self):
        """Optionally quotes are stripped."""

        self.assertEqual(clean_string('\'ok\''), 'ok')
        self.assertEqual(clean_string('\'ok\'', stripquotes=False), '\'ok\'')
        self.assertEqual(clean_string('"ok"'), 'ok')
        self.assertEqual(clean_string('"ok"', stripquotes=False), '"ok"')
Ejemplo n.º 4
0
    def test_default_value(self):
        """Default value is used for non-string arguments."""

        self.assertEqual(clean_string(None), '')
        self.assertEqual(clean_string(None, default='blip'), 'blip')
        self.assertEqual(clean_string(1.0), '')
        self.assertEqual(clean_string(1.0, default='invalid'), 'invalid')
Ejemplo n.º 5
0
def transform_project_data(item):
    name = clean_string(item.get("project_title"), default=None)
    return {
        "name": name,
        "slug": slugify(name, allow_unicode=True),
        "start_year": parse_int(item.get("project_start_year")),
        "start_month": parse_int(item.get("project_start_month")),
        "start_day": parse_int(item.get("project_start_day")),
        "status": project_status_from_statuses(item.get("project_status")),
        "sources": make_url_list(item.get("sources")),
        "notes": clean_string(item.get("notes")),
        "commencement_year": parse_int(item.get("commencement_year")),
        "commencement_month": parse_int(item.get("commencement_month")),
        "commencement_day": parse_int(item.get("commencement_day")),
        "total_cost": parse_int(item.get("total_project_cost")),
        "total_cost_currency": item.get("total_project_cost_currency"),
        "planned_completion_year": parse_int(item.get("planned_year_of_completion")),
        "planned_completion_month": parse_int(item.get("planned_month_of_completion")),
        "planned_completion_day": parse_int(item.get("planned_day_of_completion")),
        "new": evaluate_project_new_value(item.get("new")),
        "infrastructure_type": infrastructure_type_object(item.get('infrastructure_type')),
        "initiative": initiative_object(item.get('program_initiative')),
        "collection_stage": collection_stage_from_value(item.get('collection_stage')),
        "verified_path": coerce_to_boolean_or_null(item.get('verified_path'))
    }
    def handle(self, *args, **kwargs):
        project_field = kwargs.get('project_field')
        initiative_field = kwargs.get('initiative_field')
        filename = kwargs.get('matchfile')

        Project = apps.get_model("infrastructure", "Project")
        Initiative = apps.get_model("infrastructure", "Initiative")

        data = json.load(filename)

        for item in data:
            proj_name = clean_string(item.get(project_field, None))
            init_name = clean_string(item.get(initiative_field, None))
            if proj_name and init_name:
                self.stdout.write('Looking up "{}" for "{}"'.format(proj_name, init_name))
                proj = None
                try:
                    proj = Project.objects.get(name=proj_name)
                except Exception as e:
                    self.stderr.write(str(e))
                if proj:
                    self.stdout.write('Found "{}"'.format(proj.name))
                    init = None
                    try:
                        init = Initiative.objects.get(name=init_name)
                    except Exception as e:
                        self.stderr.write(str(e))
                    if init:
                        self.stdout.write('Found "{}", adding to "{}"'.format(init.name, proj.name))
                        if not proj.initiatives.filter(id__contains=init.id).exists():
                            proj.initiatives.add(init)
                            self.stderr.write(self.style.SUCCESS('Matched!'))
                        else:
                            self.stderr.write(self.style.WARNING('Project already associated with initiative'))
                self.stdout.write("---")
Ejemplo n.º 7
0
    def test_strip_newlines(self):
        """Optionally newlines are stripped."""

        self.assertEqual(clean_string('Hello\nWorld'), 'Hello World')
        self.assertEqual(clean_string('Hello\nWorld', stripnewlines=False),
                         'Hello\nWorld')
        self.assertEqual(clean_string('Hello\rWorld'), 'Hello World')
        self.assertEqual(clean_string('Hello\rWorld', stripnewlines=False),
                         'Hello\rWorld')
Ejemplo n.º 8
0
    def handle(self, *args, **kwargs):
        infiles = kwargs.get('files')
        self.verbosity = kwargs.get('verbosity')
        self.dry_run = kwargs.get('dry_run')
        # Attempt to parse additional attributes
        raw_attrs = kwargs.get('attributes')
        self.attributes = self._create_attributes(raw_attrs) if raw_attrs else {}

        if self.verbosity > 1:
            self.stdout.write(self.style.NOTICE('3D geometries will be flattened!'))

        for f in infiles:
            path, filename = os.path.split(f)
            fname, fext = os.path.splitext(filename)
            self.stdout.write("Processing '{}'".format(fname))
            self.attributes['source'] = clean_string(filename)

            try:
                geo_store = geostore_from_file(
                    f, fname,
                    verbosity=self.verbosity,
                    geo_attributes=self.attributes,
                    create=True if not self.dry_run else False
                )
                if geo_store:
                    self.stdout.write(self.style.SUCCESS("Created '{}'".format(geo_store.identifier)))
                else:
                    self.stderr.write(self.style.ERROR("Unable to create Geostore for '{}'".format(filename)))
            except Exception as e:
                raise CommandError(e)
Ejemplo n.º 9
0
def transform_project_data(item):
    name = clean_string(item.get("project_title"), default=None)
    return {
        "name":
        name,
        "slug":
        slugify(name, allow_unicode=True),
        "start_year":
        parse_int(item.get("project_start_year")),
        "start_month":
        parse_int(item.get("project_start_month")),
        "start_day":
        parse_int(item.get("project_start_day")),
        "status":
        project_status_from_statuses(item.get("project_status")),
        "sources":
        make_url_list(item.get("sources")),
        "notes":
        clean_string(item.get("notes")),
        "commencement_year":
        parse_int(item.get("commencement_year")),
        "commencement_month":
        parse_int(item.get("commencement_month")),
        "commencement_day":
        parse_int(item.get("commencement_day")),
        "total_cost":
        parse_int(item.get("total_project_cost")),
        "total_cost_currency":
        item.get("total_project_cost_currency"),
        "planned_completion_year":
        parse_int(item.get("planned_year_of_completion")),
        "planned_completion_month":
        parse_int(item.get("planned_month_of_completion")),
        "planned_completion_day":
        parse_int(item.get("planned_day_of_completion")),
        "new":
        evaluate_project_new_value(item.get("new")),
        "infrastructure_type":
        infrastructure_type_object(item.get('infrastructure_type')),
        "initiative":
        initiative_object(item.get('program_initiative')),
        "collection_stage":
        collection_stage_from_value(item.get('collection_stage')),
        "verified_path":
        coerce_to_boolean_or_null(item.get('verified_path'))
    }
Ejemplo n.º 10
0
def transform_initiative_data(item):
    name = clean_string(item.get("program_initiative_name", item.get("program_initiative")))
    return{
        "name": name,
        "slug": slugify(name, allow_unicode=True),
        "initiative_type": initiative_type_object(item.get('initiative_type')),
        "appeared_year": parse_int(item.get('first_appearance_year')),
        "appeared_month": parse_int(item.get('first_appearance_month')),
        "appeared_day": parse_int(item.get('first_appearance_date')),
    }
Ejemplo n.º 11
0
def transform_initiative_data(item):
    name = clean_string(
        item.get("program_initiative_name", item.get("program_initiative")))
    return {
        "name": name,
        "slug": slugify(name, allow_unicode=True),
        "initiative_type": initiative_type_object(item.get('initiative_type')),
        "appeared_year": parse_int(item.get('first_appearance_year')),
        "appeared_month": parse_int(item.get('first_appearance_month')),
        "appeared_day": parse_int(item.get('first_appearance_date')),
    }
Ejemplo n.º 12
0
def geostore_from_file(source, label, verbosity=0, geo_attributes={}, create=True):
    try:
        ds = DataSource(source)
    except Exception as e:
        logger.error(str(e))
        raise e

    geo_store = None
    if create:
        geo_store = GeometryStore.objects.create(label=label[:LABEL_CHAR_LIMIT], attributes=geo_attributes)
    else:
        logger.info('Processing, but not saving geodata')

    if verbosity > 2:
        logger.info('3D geometries will be flattened!')

    for layer in ds:
        layer_name = layer.name.split('.')[0]
        if verbosity > 2:
            logger.info('Layer "{}": {} {}s'.format(layer_name, len(layer), layer.geom_type.name))
        for feat in layer:
            # Remove 3rd dimension
            geom = feat.geom.clone()
            geom.coord_dim = 2
            if create:
                # Create attributes dict from geom fields
                data = {f.name.lower(): f.value for f in feat}
                if 'name' in data:
                    data['name'] = clean_string(data['name'])
                # Add source information to attributes
                data['layer'] = layer_name
                if verbosity > 2:
                    logger.info('Feature "{}": {}'.format(
                        data.get('name', layer_name), feat.geom_type))
                if geom.geos:
                    geom_label = data.get('name') or "{}: {}".format(label, layer_name)
                    params = {
                        'label': geom_label[:LABEL_CHAR_LIMIT],
                        'geom': geom.geos,
                        'attributes': data
                    }
                    if geom.geom_type == 'Point':
                        geo_store.points.create(**params)
                    elif geom.geom_type == 'LineString':
                        geo_store.lines.create(**params)
                    elif geom.geom_type == 'Polygon':
                        geo_store.polygons.create(**params)
                    if verbosity > 1:
                        logger.info("Geometry '{}' found and saved".format(geom_label))
            elif verbosity > 1:
                logger.info("Geometry '{label}: {layer_name}' found, but not saved".format(
                    label=label, layer_name=layer_name))

    return geo_store
Ejemplo n.º 13
0
 def transform_organization(item):
     if isinstance(name_field, typing.Iterable) and not isinstance(name_field, str):
         name_values = list(filter(lambda x: item.get(x), name_field))
         if name_values:
             org_name = item.get(name_values[0])
         else:
             return None
     else:
         org_name = clean_string(item.get(name_field))
     return {
         'name': org_name,
         'slug': slugify(org_name, allow_unicode=True)
     }
Ejemplo n.º 14
0
 def transform_organization(item):
     if isinstance(name_field,
                   typing.Iterable) and not isinstance(name_field, str):
         name_values = list(filter(lambda x: item.get(x), name_field))
         if name_values:
             org_name = item.get(name_values[0])
         else:
             return None
     else:
         org_name = clean_string(item.get(name_field))
     return {
         'name': org_name,
         'slug': slugify(org_name, allow_unicode=True)
     }
Ejemplo n.º 15
0
def process_regions_data(value, recurse=True, inner_key='region_name'):
    if isinstance(value, str):
        if value.startswith('"') or value.startswith('\''):
            splitchar = value[0]
            regions_raw = [x.strip(',') for x in value.split(splitchar) if x]
            yield from process_regions_data(regions_raw, recurse=True)
        else:
            yield {'name': clean_string(value)}
    elif isinstance(value, list):
        if recurse:
            for item in value:
                yield from process_regions_data(item, recurse=False)
    elif isinstance(value, dict):
        yield from process_regions_data(value.get(inner_key, None),
                                        recurse=False)
Ejemplo n.º 16
0
def process_regions_data(value, recurse=True, inner_key='region_name'):
    if isinstance(value, str):
        if value.startswith('"') or value.startswith('\''):
            splitchar = value[0]
            regions_raw = [x.strip(',') for x in value.split(splitchar) if x]
            yield from process_regions_data(regions_raw, recurse=True)
        else:
            yield {
                'name': clean_string(value)
            }
    elif isinstance(value, list):
        if recurse:
            for item in value:
                yield from process_regions_data(item, recurse=False)
    elif isinstance(value, dict):
        yield from process_regions_data(value.get(inner_key, None), recurse=False)
Ejemplo n.º 17
0
    def handle(self, *args, **kwargs):
        infiles = kwargs.get('files')
        self.verbosity = kwargs.get('verbosity')
        self.dry_run = kwargs.get('dry_run')
        # Attempt to parse additional attributes
        raw_attrs = kwargs.get('attributes')
        self.attributes = self._create_attributes(
            raw_attrs) if raw_attrs else {}

        if self.verbosity > 1:
            self.stdout.write(
                self.style.NOTICE('3D geometries will be flattened!'))

        for f in infiles:
            path, filename = os.path.split(f)
            fname, fext = os.path.splitext(filename)
            self.stdout.write("Processing '{}'".format(fname))
            self.attributes['source'] = clean_string(filename)

            try:
                geo_store = geostore_from_file(
                    f,
                    fname,
                    verbosity=self.verbosity,
                    geo_attributes=self.attributes,
                    create=True if not self.dry_run else False)
                if geo_store:
                    self.stdout.write(
                        self.style.SUCCESS("Created '{}'".format(
                            geo_store.identifier)))
                else:
                    self.stderr.write(
                        self.style.ERROR(
                            "Unable to create Geostore for '{}'".format(
                                filename)))
            except Exception as e:
                raise CommandError(e)
Ejemplo n.º 18
0
def transform_infrastructuretype_data(item):
    name = clean_string(item.get("infrastructure_type_name"))
    return {
        'name': name,
        "slug": slugify(name, allow_unicode=True),
    }
Ejemplo n.º 19
0
def transform_initiative_type_data(item):
    value = item if isinstance(item, str) else item.get('initiative_type')
    if not value:
        return None
    name = clean_string(value)
    return {"name": name, 'slug': slugify(name, allow_unicode=True)}
Ejemplo n.º 20
0
def transform_infrastructuretype_data(item):
    name = clean_string(item.get("infrastructure_type_name"))
    return {
        'name': name,
        "slug": slugify(name, allow_unicode=True),
    }
Ejemplo n.º 21
0
def transform_initiative_type_data(item):
    value = item if isinstance(item, str) else item.get('initiative_type')
    if not value:
        return None
    name = clean_string(value)
    return {"name": name, 'slug': slugify(name, allow_unicode=True)}
Ejemplo n.º 22
0
    def test_replace_question_marks(self):
        """Question marks should converted to dashes."""

        self.assertEqual(clean_string('Why?'), 'Why–')
Ejemplo n.º 23
0
    def test_replace_unicode_quotes(self):
        """Unicode left and right quotes should be replaced by single quotes."""

        self.assertEqual(clean_string('This is \u2018great\u2019'),
                         'This is \'great\'')