Beispiel #1
0
    def generate_elastic_mapping(self):

        # if self.resource.geometry in ('Point', 'MultiPoint'):
        #     geometry_mapping = {'type': 'geo_point', 'ignore_malformed': True}
        # else:
        geometry_mapping = {
            'type': 'geo_shape',
            'tree': 'quadtree',
            # 'precision': '',
            # 'tree_levels': '',
            # 'strategy': '',
            'distance_error_pct': 0,
            'orientation': 'counterclockwise',
            'points_only': False
        }

        props = {}
        for p in self.iter_properties():
            if not p.rejected:
                props[p.alias or p.name] = fetch_mapping(p)

        return clean_my_obj({
            self.name: {
                'properties': {
                    'geometry': geometry_mapping,
                    'lineage': {
                        'properties': {
                            'resource': {
                                'properties': {
                                    'name': not_searchable('keyword')
                                }
                            },
                            'source': {
                                'properties': {
                                    'protocol': not_searchable('keyword'),
                                    'uri': not_searchable('keyword')
                                }
                            }
                        }
                    },
                    'properties': {
                        'properties': props
                    }
                }
            }
        })
Beispiel #2
0
    def generate_elastic_mapping(self):

        props = {}
        for p in self.iter_properties():
            if not p.rejected:
                props[p.alias or p.name] = fetch_mapping(p)

        return clean_my_obj({
            self.name: {
                'properties': {
                    'lineage': {
                        'properties': {
                            # 'resource': {
                            #     'properties': {
                            #         'name': not_searchable('keyword')}},
                            'source': {
                                'properties': {
                                    'protocol': not_searchable('keyword'),
                                    'uri': not_searchable('keyword')}}}},
                    'properties': {'properties': props}}}})
Beispiel #3
0
    def generate_elastic_mapping(self):

        props = {}
        for p in self.iter_properties(ignore=['xml', 'uris']):
            if not p.rejected:
                props[p.alias or p.name] = fetch_mapping(p)

        return clean_my_obj({
            self.name: {
                'properties': {
                    'lineage': {
                        'properties': {
                            'resource': {
                                'properties': {
                                    'name': not_searchable('keyword')
                                }
                            },
                            'source': {
                                'properties': {
                                    'protocol': not_searchable('keyword'),
                                    'uri': not_searchable('keyword')
                                }
                            }
                        }
                    },
                    'properties': {
                        'properties': props
                    },
                    'uri': {
                        'properties': {
                            'protocol': not_searchable('keyword'),
                            'name': not_searchable('text'),
                            'description': not_searchable('text'),
                            'url': not_searchable('keyword')
                        }
                    },
                    'xml': not_searchable('text')
                }
            }
        })
Beispiel #4
0
    def get_collection(self, resource, step=10, id_record=[]):

        outputschema = tuple(
            tuple(k for v in l if v == resource.name)[0]
            for k, l in self.OUTPUSCHEMA.items() if resource.name in l)[0]

        params = {
            'cql': "type='{0}'".format(resource.name),
            'esn': 'full',
            'format': 'application/xml',
            'maxrecords': step,
            'outputschema': outputschema,
            'resulttype': 'results',
            'startposition': 0,
            'typenames': 'csw:Record'
        }

        if len(id_record) > 0:
            params['cql'] += " AND (identifier='{1}')".format(
                resource.name, "' OR identifier='".join(id_record))

        while True:
            self._csw.getrecords2(**params)
            records = list(self._csw.records.values())
            for rec in records:
                data = {}
                if rec.__class__.__name__ == 'MD_Metadata':

                    resolution = []
                    distance = rec.identification.distance
                    uom = rec.identification.uom
                    if len(distance) == len(uom):
                        for i in range(len(distance)):
                            resolution.append({
                                'uom': uom[i],
                                'distance': distance[i]
                            })

                    data.update(
                        **{
                            'abstract':
                            rec.identification.abstract,
                            'bbox':
                            rec.identification.bbox and {
                                'type':
                                'Polygon',
                                'coordinates':
                                [[[
                                    rec.identification.bbox.minx,
                                    rec.identification.bbox.miny
                                ],
                                  [
                                      rec.identification.bbox.maxx,
                                      rec.identification.bbox.miny
                                  ],
                                  [
                                      rec.identification.bbox.maxx,
                                      rec.identification.bbox.maxy
                                  ],
                                  [
                                      rec.identification.bbox.minx,
                                      rec.identification.bbox.maxy
                                  ],
                                  [
                                      rec.identification.bbox.minx,
                                      rec.identification.bbox.miny
                                  ]]]
                            },
                            'classification':
                            rec.identification.classification,
                            'contact':
                            rec.identification.contact and [
                                m.__dict__
                                for m in rec.identification.contact if
                                m.__class__.__name__ == 'CI_ResponsibleParty'
                            ],
                            'date_publication':
                            rec.identification.date and [
                                m.date for m in rec.identification.date
                                if m.__class__.__name__ == 'CI_Date'
                                and m.type == 'publication'
                            ],
                            'denominators':
                            rec.identification.denominators,
                            'identifier':
                            rec.identifier,
                            'keyword': [
                                y for x in [
                                    m.keywords
                                    for m in rec.identification.keywords2
                                    if m.__class__.__name__ == 'MD_Keywords'
                                ] for y in x
                            ],
                            'lineage':
                            rec.dataquality.lineage,
                            'parent_identifier':
                            rec.parentidentifier,
                            'resolution':
                            resolution,
                            'rights':
                            list(
                                itertools.chain(
                                    rec.identification.accessconstraints,
                                    rec.identification.securityconstraints,
                                    rec.identification.otherconstraints)),
                            'spatial_type':
                            rec.identification.spatialrepresentationtype,
                            'standard': {
                                'name': rec.stdname,
                                'version': rec.stdver
                            },
                            'title':
                            rec.identification.title,
                            'type':
                            rec.hierarchy,
                            'topic_category':
                            rec.identification.topiccategory,
                            'use_constraints':
                            rec.identification.useconstraints,
                            'use_limitation':
                            rec.identification.uselimitation,
                            'uris':
                            rec.distribution.online and [
                                m.__dict__ for m in rec.distribution.online
                                if m.__class__.__name__ == 'CI_OnlineResource'
                            ],
                            'xml':
                            rec.xml
                        })

                if rec.__class__.__name__ == 'CswRecord':
                    for col in resource.iter_columns():
                        try:
                            attr = getattr(rec, col['name'])
                        except AttributeError:
                            data[col['name']] = None
                            continue
                        if col['name'] == 'bbox_wgs84' \
                                and col['type'] == 'geo_shape' and attr:
                            attr = {
                                'type':
                                'Polygon',
                                'coordinates': [[[attr.minx, attr.miny],
                                                 [attr.maxx, attr.miny],
                                                 [attr.maxx, attr.maxy],
                                                 [attr.minx, attr.maxy],
                                                 [attr.minx, attr.miny]]]
                            }

                        data[col['name']] = \
                            isinstance(attr, bytes) and attr.decode() or attr

                data.update(schema=outputschema)

                yield clean_my_obj(data, fading=False)

            if len(records) < step:
                break
            params['startposition'] += step