def test_parse_daterange_end_full_iso(self): assert daterange_end('1984-06-07T00:00:00+00:00') == date(1984, 6, 7)
def test_parse_before_1900(self): assert daterange_start('1860') == date(1860, 1, 1) assert daterange_end('1860') == date(1860, 12, 31)
def test_parse_daterange_end_date(self): today = date.today() assert daterange_end(today) == today
def test_parse_daterange_end_month_february(self): self.assertEqual(daterange_end('1984-02'), date(1984, 2, 29)) self.assertEqual(daterange_end('1985-02'), date(1985, 2, 28))
def test_parse_before_1900(self): self.assertEqual(daterange_start('1860'), date(1860, 1, 1)) self.assertEqual(daterange_end('1860'), date(1860, 12, 31))
def remote_datasets(self): response = self.get('package_list') for name in response['result']: details = self.get('package_show', {'id': name})['result'] dataset = self.get_harvested(Dataset, details['id']) # Core attributes dataset.slug = details['name'] dataset.title = details['title'] dataset.description = details.get('notes', 'No description') dataset.license = License.objects(id=details['license_id']).first() or License.objects.get(id='notspecified') dataset.tags = [tag['name'].lower() for tag in details['tags']] dataset.frequency = self.map('frequency', details) or 'unknown' dataset.created_at = parse(details['metadata_created']) dataset.last_modified = parse(details['metadata_modified']) if any_field(details, 'territorial_coverage', 'territorial_coverage_granularity'): coverage = TerritorialCoverage( codes=[code.strip() for code in details.get('territorial_coverage', '').split(',') if code.strip()], granularity=self.map('territorial_coverage_granularity', details), ) dataset.extras['territorial_coverage'] = coverage try: dataset.spatial = territorial_to_spatial(dataset) except Exception as e: print 'Error while processing spatial coverage for {0}:'.format(dataset.title), e if all_field(details, 'temporal_coverage_from', 'temporal_coverage_to'): try: dataset.temporal_coverage = db.DateRange( start=daterange_start(details.get('temporal_coverage_from')), end=daterange_end(details.get('temporal_coverage_to')), ) except: log.error('Unable to parse temporal coverage for dataset %s', details['id']) # Organization if details.get('organization'): dataset.organization = self.get_harvested(Organization, details['organization']['id'], False) else: # Need to fetch user from roles roles = self.get('roles_show', {'domain_object': name})['result']['roles'] for role in roles: if role['role'] == 'admin' and role['context'] == 'Package': dataset.owner = self.get_harvested(User, role['user_id']) break # Supplier if details.get('supplier_id'): dataset.supplier = self.get_harvested(Organization, details['supplier_id'], False) # Remote URL if details.get('url'): dataset.extras['remote_url'] = details['url'] # Extras if 'extras' in details: extra_mapping = self.harvester.mapping.get('from_extras', {}) for extra in details['extras']: if extra['key'] in self.harvester.mapping: value = self.harvester.mapping[extra['key']].get(extra['value']) else: value = extra['value'] if extra['key'] in extra_mapping: setattr(dataset, extra_mapping[extra['key']], value) else: dataset.extras[extra['key']] = value # Resources for res in details['resources']: try: resource = get_by(dataset.resources, 'id', UUID(res['id'])) except: log.error('Unable to parse resource %s', res['id']) continue if not resource: resource = Resource(id=res['id']) dataset.resources.append(resource) resource.title = res.get('name', '') or '' resource.url = res['url'] resource.description = res.get('description') resource.format = res.get('format') resource.hash = res.get('hash') resource.created = parse(res['created']) resource.modified = parse(res['revision_timestamp']) resource.published = resource.published or resource.created yield dataset if dataset.id: followers = self.get('dataset_follower_list', {'id': name})['result'] for follower in followers: user = self.get_harvested(User, follower['id'], False) if user: follow, created = FollowDataset.objects.get_or_create(follower=user, following=dataset)
def test_parse_daterange_end_full(self): self.assertEqual(daterange_end('1984-06-07'), date(1984, 6, 7))
def test_parse_daterange_end_month(self): assert daterange_end('1984-06') == date(1984, 6, 30) assert daterange_end('1984-07') == date(1984, 7, 31)
def test_parse_daterange_end_month_february(self): assert daterange_end('1984-02') == date(1984, 2, 29) assert daterange_end('1985-02') == date(1985, 2, 28)
def test_parse_daterange_end_full(self): assert daterange_end('1984-06-07') == date(1984, 6, 7)
def test_parse_daterange_end_datetime(self): now = datetime.now() assert daterange_end(now) == now.date()
def test_parse_daterange_end_empty(self): assert daterange_end(None) is None assert daterange_end('') is None
def test_parse_daterange_end_year(self): assert daterange_end('1984') == date(1984, 12, 31)
def process(self, item): response = self.get_action('package_show', id=item.remote_id) data = self.validate(response['result'], schema) # Fix the remote_id: use real ID instead of not stable name item.remote_id = data['id'] # Skip if no resource if not len(data.get('resources', [])): msg = 'Dataset {0} has no record'.format(item.remote_id) raise HarvestSkipException(msg) dataset = self.get_dataset(item.remote_id) # Core attributes if not dataset.slug: dataset.slug = data['name'] dataset.title = data['title'] dataset.description = data['notes'] dataset.license = License.objects(id=data['license_id']).first() # dataset.license = license or License.objects.get(id='notspecified') dataset.tags = [t['name'] for t in data['tags']] dataset.created_at = data['metadata_created'] dataset.last_modified = data['metadata_modified'] dataset.extras['ckan:name'] = data['name'] temporal_start, temporal_end = None, None spatial_geom = None for extra in data['extras']: # GeoJSON representation (Polygon or Point) if extra['key'] == 'spatial': spatial_geom = json.loads(extra['value']) # Textual representation of the extent / location elif extra['key'] == 'spatial-text': log.debug('spatial-text value not handled') print 'spatial-text', extra['value'] # Linked Data URI representing the place name elif extra['key'] == 'spatial-uri': log.debug('spatial-uri value not handled') print 'spatial-uri', extra['value'] # Update frequency elif extra['key'] == 'frequency': print 'frequency', extra['value'] # Temporal coverage start elif extra['key'] == 'temporal_start': print 'temporal_start', extra['value'] temporal_start = daterange_start(extra['value']) continue # Temporal coverage end elif extra['key'] == 'temporal_end': print 'temporal_end', extra['value'] temporal_end = daterange_end(extra['value']) continue # else: # print extra['key'], extra['value'] dataset.extras[extra['key']] = extra['value'] if spatial_geom: dataset.spatial = SpatialCoverage() if spatial_geom['type'] == 'Polygon': coordinates = [spatial_geom['coordinates']] elif spatial_geom['type'] == 'MultiPolygon': coordinates = spatial_geom['coordinates'] else: HarvestException('Unsupported spatial geometry') dataset.spatial.geom = { 'type': 'MultiPolygon', 'coordinates': coordinates } if temporal_start and temporal_end: dataset.temporal_coverage = db.DateRange( start=temporal_start, end=temporal_end, ) # Remote URL if data.get('url'): dataset.extras['remote_url'] = data['url'] # Resources for res in data['resources']: if res['resource_type'] not in ALLOWED_RESOURCE_TYPES: continue try: resource = get_by(dataset.resources, 'id', UUID(res['id'])) except: log.error('Unable to parse resource ID %s', res['id']) continue if not resource: resource = Resource(id=res['id']) dataset.resources.append(resource) resource.title = res.get('name', '') or '' resource.description = res.get('description') resource.url = res['url'] resource.filetype = ('api' if res['resource_type'] == 'api' else 'remote') resource.format = res.get('format') resource.mime = res.get('mimetype') resource.hash = res.get('hash') resource.created = res['created'] resource.modified = res['last_modified'] resource.published = resource.published or resource.created return dataset
def test_parse_daterange_end_empty(self): self.assertIsNone(daterange_end(None)) self.assertIsNone(daterange_end(''))
def process(self, item): response = self.get_action('package_show', id=item.remote_id) data = self.validate(response['result'], self.schema) if type(data) == list: data = data[0] # Fix the remote_id: use real ID instead of not stable name item.remote_id = data['id'] # Skip if no resource if not len(data.get('resources', [])): msg = 'Dataset {0} has no record'.format(item.remote_id) raise HarvestSkipException(msg) dataset = self.get_dataset(item.remote_id) # Core attributes if not dataset.slug: dataset.slug = data['name'] dataset.title = data['title'] dataset.description = parse_html(data['notes']) # Detect Org organization_acronym = data['organization']['name'] orgObj = Organization.objects(acronym=organization_acronym).first() if orgObj: #print 'Found %s' % orgObj.acronym dataset.organization = orgObj else: orgObj = Organization() orgObj.acronym = organization_acronym orgObj.name = data['organization']['title'] orgObj.description = data['organization']['description'] orgObj.save() #print 'Created %s' % orgObj.acronym dataset.organization = orgObj # Detect license default_license = self.harvest_config.get('license', License.default()) dataset.license = License.guess(data['license_id'], data['license_title'], default=default_license) dataset.tags = [t['name'] for t in data['tags'] if t['name']] dataset.tags.append(urlparse(self.source.url).hostname) dataset.created_at = data['metadata_created'] dataset.last_modified = data['metadata_modified'] dataset.frequency = 'unknown' dataset.extras['ckan:name'] = data['name'] temporal_start, temporal_end = None, None spatial_geom = None for extra in data['extras']: # GeoJSON representation (Polygon or Point) if extra['key'] == 'spatial': spatial_geom = json.loads(extra['value']) # Textual representation of the extent / location elif extra['key'] == 'spatial-text': log.debug('spatial-text value not handled') # Linked Data URI representing the place name elif extra['key'] == 'spatial-uri': log.debug('spatial-uri value not handled') # Update frequency elif extra['key'] == 'frequency': print 'frequency', extra['value'] # Temporal coverage start elif extra['key'] == 'temporal_start': temporal_start = daterange_start(extra['value']) continue # Temporal coverage end elif extra['key'] == 'temporal_end': temporal_end = daterange_end(extra['value']) continue dataset.extras[extra['key']] = extra['value'] # We don't want spatial to be added on harvester if self.harvest_config.get('geozones', False): dataset.spatial = SpatialCoverage() dataset.spatial.zones = [] for zone in self.harvest_config.get('geozones'): geo_zone = GeoZone.objects.get(id=zone) dataset.spatial.zones.append(geo_zone) # # if spatial_geom: # dataset.spatial = SpatialCoverage() # if spatial_geom['type'] == 'Polygon': # coordinates = [spatial_geom['coordinates']] # elif spatial_geom['type'] == 'MultiPolygon': # coordinates = spatial_geom['coordinates'] # else: # HarvestException('Unsupported spatial geometry') # dataset.spatial.geom = { # 'type': 'MultiPolygon', # 'coordinates': coordinates # } if temporal_start and temporal_end: dataset.temporal_coverage = db.DateRange( start=temporal_start, end=temporal_end, ) # Remote URL if data.get('url'): try: url = uris.validate(data['url']) except uris.ValidationError: dataset.extras['remote_url'] = self.dataset_url(data['name']) dataset.extras['ckan:source'] = data['url'] else: dataset.extras['remote_url'] = url dataset.extras['harvest:name'] = self.source.name current_resources = [ str(resource.id) for resource in dataset.resources ] fetched_resources = [] # Resources for res in data['resources']: if res['resource_type'] not in ALLOWED_RESOURCE_TYPES: continue #Ignore invalid Resources try: url = uris.validate(res['url']) except uris.ValidationError: continue try: resource = get_by(dataset.resources, 'id', UUID(res['id'])) except Exception: log.error('Unable to parse resource ID %s', res['id']) continue fetched_resources.append(str(res['id'])) if not resource: resource = Resource(id=res['id']) dataset.resources.append(resource) resource.title = res.get('name', '') or '' resource.description = parse_html(res.get('description')) resource.url = res['url'] resource.filetype = 'remote' resource.format = res.get('format') resource.mime = res.get('mimetype') resource.hash = res.get('hash') resource.created = res['created'] resource.modified = res['last_modified'] resource.published = resource.published or resource.created # Clean up old resources removed from source for resource_id in current_resources: if resource_id not in fetched_resources: try: resource = get_by(dataset.resources, 'id', UUID(resource_id)) except Exception: log.error('Unable to parse resource ID %s', resource_id) continue else: if resource and not self.dryrun: dataset.resources.remove(resource) return dataset
def test_parse_daterange_end_month(self): self.assertEqual(daterange_end('1984-06'), date(1984, 6, 30)) self.assertEqual(daterange_end('1984-07'), date(1984, 7, 31))
def test_parse_daterange_end_year(self): self.assertEqual(daterange_end('1984'), date(1984, 12, 31))
def process(self, item): response = self.get_action('package_show', id=item.remote_id) data = self.validate(response['result'], self.schema) if type(data) == list: data = data[0] # Fix the remote_id: use real ID instead of not stable name item.remote_id = data['id'] # Skip if no resource if not len(data.get('resources', [])): msg = 'Dataset {0} has no record'.format(item.remote_id) raise HarvestSkipException(msg) dataset = self.get_dataset(item.remote_id) # Core attributes if not dataset.slug: dataset.slug = data['name'] dataset.title = data['title'] dataset.description = parse_html(data['notes']) # Detect license default_license = dataset.license or License.default() dataset.license = License.guess(data['license_id'], data['license_title'], default=default_license) dataset.tags = [t['name'] for t in data['tags'] if t['name']] dataset.created_at = data['metadata_created'] dataset.last_modified = data['metadata_modified'] dataset.extras['ckan:name'] = data['name'] temporal_start, temporal_end = None, None spatial_geom, spatial_zone = None, None for extra in data['extras']: key = extra['key'] value = extra['value'] if value is None or (isinstance(value, str) and not value.strip()): # Skip empty extras continue elif key == 'spatial': # GeoJSON representation (Polygon or Point) spatial_geom = json.loads(value) elif key == 'spatial-text': # Textual representation of the extent / location qs = GeoZone.objects(db.Q(name=value) | db.Q(slug=value)) qs = qs.valid_at(datetime.now()) if qs.count() == 1: spatial_zone = qs.first() else: dataset.extras['ckan:spatial-text'] = value log.debug('spatial-text value not handled: %s', value) elif key == 'spatial-uri': # Linked Data URI representing the place name dataset.extras['ckan:spatial-uri'] = value log.debug('spatial-uri value not handled: %s', value) elif key == 'frequency': # Update frequency freq = frequency_from_rdf(value) if freq: dataset.frequency = freq elif value in UPDATE_FREQUENCIES: dataset.frequency = value else: dataset.extras['ckan:frequency'] = value log.debug('frequency value not handled: %s', value) # Temporal coverage start elif key == 'temporal_start': temporal_start = daterange_start(value) # Temporal coverage end elif key == 'temporal_end': temporal_end = daterange_end(value) else: dataset.extras[extra['key']] = value if spatial_geom or spatial_zone: dataset.spatial = SpatialCoverage() if spatial_zone: dataset.spatial.zones = [spatial_zone] if spatial_geom: if spatial_geom['type'] == 'Polygon': coordinates = [spatial_geom['coordinates']] elif spatial_geom['type'] == 'MultiPolygon': coordinates = spatial_geom['coordinates'] else: raise HarvestException('Unsupported spatial geometry') dataset.spatial.geom = { 'type': 'MultiPolygon', 'coordinates': coordinates } if temporal_start and temporal_end: dataset.temporal_coverage = db.DateRange( start=temporal_start, end=temporal_end, ) # Remote URL dataset.extras['remote_url'] = self.dataset_url(data['name']) if data.get('url'): try: url = uris.validate(data['url']) except uris.ValidationError: dataset.extras['ckan:source'] = data['url'] else: # use declared `url` as `remote_url` if any dataset.extras['remote_url'] = url # Resources for res in data['resources']: if res['resource_type'] not in ALLOWED_RESOURCE_TYPES: continue try: resource = get_by(dataset.resources, 'id', UUID(res['id'])) except Exception: log.error('Unable to parse resource ID %s', res['id']) continue if not resource: resource = Resource(id=res['id']) dataset.resources.append(resource) resource.title = res.get('name', '') or '' resource.description = parse_html(res.get('description')) resource.url = res['url'] resource.filetype = 'remote' resource.format = res.get('format') resource.mime = res.get('mimetype') resource.hash = res.get('hash') resource.created = res['created'] resource.modified = res['last_modified'] resource.published = resource.published or resource.created return dataset
def process(self, item): response = self.get_action('package_show', id=item.remote_id) data = self.validate(response['result'], schema) # Fix the remote_id: use real ID instead of not stable name item.remote_id = data['id'] # Skip if no resource if not len(data.get('resources', [])): msg = 'Dataset {0} has no record'.format(item.remote_id) raise HarvestSkipException(msg) dataset = self.get_dataset(item.remote_id) # Core attributes if not dataset.slug: dataset.slug = data['name'] dataset.title = data['title'] dataset.description = data['notes'] dataset.license = License.objects(id=data['license_id']).first() # dataset.license = license or License.objects.get(id='notspecified') dataset.tags = [t['name'] for t in data['tags'] if t['name']] dataset.created_at = data['metadata_created'] dataset.last_modified = data['metadata_modified'] dataset.extras['ckan:name'] = data['name'] temporal_start, temporal_end = None, None spatial_geom = None for extra in data['extras']: # GeoJSON representation (Polygon or Point) if extra['key'] == 'spatial': spatial_geom = json.loads(extra['value']) # Textual representation of the extent / location elif extra['key'] == 'spatial-text': log.debug('spatial-text value not handled') print 'spatial-text', extra['value'] # Linked Data URI representing the place name elif extra['key'] == 'spatial-uri': log.debug('spatial-uri value not handled') print 'spatial-uri', extra['value'] # Update frequency elif extra['key'] == 'frequency': print 'frequency', extra['value'] # Temporal coverage start elif extra['key'] == 'temporal_start': print 'temporal_start', extra['value'] temporal_start = daterange_start(extra['value']) continue # Temporal coverage end elif extra['key'] == 'temporal_end': print 'temporal_end', extra['value'] temporal_end = daterange_end(extra['value']) continue # else: # print extra['key'], extra['value'] dataset.extras[extra['key']] = extra['value'] if spatial_geom: dataset.spatial = SpatialCoverage() if spatial_geom['type'] == 'Polygon': coordinates = [spatial_geom['coordinates']] elif spatial_geom['type'] == 'MultiPolygon': coordinates = spatial_geom['coordinates'] else: HarvestException('Unsupported spatial geometry') dataset.spatial.geom = { 'type': 'MultiPolygon', 'coordinates': coordinates } if temporal_start and temporal_end: dataset.temporal_coverage = db.DateRange( start=temporal_start, end=temporal_end, ) # Remote URL if data.get('url'): dataset.extras['remote_url'] = data['url'] # Resources for res in data['resources']: if res['resource_type'] not in ALLOWED_RESOURCE_TYPES: continue try: resource = get_by(dataset.resources, 'id', UUID(res['id'])) except: log.error('Unable to parse resource ID %s', res['id']) continue if not resource: resource = Resource(id=res['id']) dataset.resources.append(resource) resource.title = res.get('name', '') or '' resource.description = res.get('description') resource.url = res['url'] resource.filetype = ('api' if res['resource_type'] == 'api' else 'remote') resource.format = res.get('format') resource.mime = res.get('mimetype') resource.hash = res.get('hash') resource.created = res['created'] resource.modified = res['last_modified'] resource.published = resource.published or resource.created return dataset
def remote_datasets(self): response = self.get('package_list') for name in response['result']: details = self.get('package_show', {'id': name})['result'] dataset = self.get_harvested(Dataset, details['id']) # Core attributes dataset.slug = details['name'] dataset.title = details['title'] dataset.description = details.get('notes', 'No description') dataset.license = License.objects( id=details['license_id']).first() or License.objects.get( id='notspecified') dataset.tags = [tag['name'].lower() for tag in details['tags']] dataset.frequency = self.map('frequency', details) or 'unknown' dataset.created_at = parse(details['metadata_created']) dataset.last_modified = parse(details['metadata_modified']) if any_field(details, 'territorial_coverage', 'territorial_coverage_granularity'): coverage = TerritorialCoverage( codes=[ code.strip() for code in details.get( 'territorial_coverage', '').split(',') if code.strip() ], granularity=self.map('territorial_coverage_granularity', details), ) dataset.extras['territorial_coverage'] = coverage try: dataset.spatial = territorial_to_spatial(dataset) except Exception as e: print 'Error while processing spatial coverage for {0}:'.format( dataset.title), e if all_field(details, 'temporal_coverage_from', 'temporal_coverage_to'): try: dataset.temporal_coverage = db.DateRange( start=daterange_start( details.get('temporal_coverage_from')), end=daterange_end(details.get('temporal_coverage_to')), ) except: log.error( 'Unable to parse temporal coverage for dataset %s', details['id']) # Organization if details.get('organization'): dataset.organization = self.get_harvested( Organization, details['organization']['id'], False) else: # Need to fetch user from roles roles = self.get('roles_show', {'domain_object': name})['result']['roles'] for role in roles: if role['role'] == 'admin' and role['context'] == 'Package': dataset.owner = self.get_harvested( User, role['user_id']) break # Supplier if details.get('supplier_id'): dataset.supplier = self.get_harvested(Organization, details['supplier_id'], False) # Remote URL if details.get('url'): dataset.extras['remote_url'] = details['url'] # Extras if 'extras' in details: extra_mapping = self.harvester.mapping.get('from_extras', {}) for extra in details['extras']: if extra['key'] in self.harvester.mapping: value = self.harvester.mapping[extra['key']].get( extra['value']) else: value = extra['value'] if extra['key'] in extra_mapping: setattr(dataset, extra_mapping[extra['key']], value) else: dataset.extras[extra['key']] = value # Resources for res in details['resources']: try: resource = get_by(dataset.resources, 'id', UUID(res['id'])) except: log.error('Unable to parse resource %s', res['id']) continue if not resource: resource = Resource(id=res['id']) dataset.resources.append(resource) resource.title = res.get('name', '') or '' resource.url = res['url'] resource.description = res.get('description') resource.format = res.get('format') resource.hash = res.get('hash') resource.created = parse(res['created']) resource.modified = parse(res['revision_timestamp']) resource.published = resource.published or resource.created yield dataset if dataset.id: followers = self.get('dataset_follower_list', {'id': name})['result'] for follower in followers: user = self.get_harvested(User, follower['id'], False) if user: follow, created = FollowDataset.objects.get_or_create( follower=user, following=dataset)