def import_package(package): row = {} #Store a copy of the metadata row['metadata'] = package row[u'Stadt'] = city row[u'Dateibezeichnung'] = package['title'] row[u'Beschreibung'] = package['notes'] row[u'url'] = package['url'] #Get resources and formats if ('resources' in package and len(package['resources']) > 0): formats = [] files = [] for resource in package['resources']: files.append(resource['url']) formats.append(resource['format']) row[u'Format'] = formats row[u'geo'] = metautils.isgeo(formats) row[u'files'] = files if 'temporal_coverage_from' in package['extras'] and len( package['extras']['temporal_coverage_from']) > 3: row[u'Zeitlicher Bezug'] = package['extras']['temporal_coverage_from'][ 0:4] if ('terms_of_use' in package['extras'] and len(package['extras']['terms_of_use']) > 0): row[u'Lizenz'] = package['extras']['terms_of_use']['licence_id'] row['categories'] = map(lambda x: metautils.govDataShortToODM(x)[0], package.get('groups', [])) return row
def mapData(package): files = [] resources = [] row = {} if ('res_url' in package): resources = package['res_url'] for file in resources: files.append(file) row[u'filelist'] = files row[u'Stadt'] = city row[u'url'] = 'http://www.opendata-hro.de/dataset/' + package['id'] if ('res_format' in package): row[u'Format'] = list(set(package['res_format'])) row[u'geo'] = metautils.isgeo(row[u'Format']) row[u'Dateibezeichnung'] = package['title'] if 'notes' in package: row[u'Beschreibung'] = package['notes'] if 'license_id' in package: row[u'Lizenz'] = package['license_id'] if 'maintainer' in package: row[u'Veröffentlichende Stelle'] = package['maintainer'] row['categories'] = map(lambda x: metautils.govDataShortToODM(x)[0], package.get('groups', [])) row['metadata'] = package return row
def import_package(part): row = {} package = {} # Simplify JSON package['title'] = part['title']['description'] package['notes'] = part['notes']['description'] package['author'] = part['author']['description'] package['url'] = part['url'] package['groups'] = [part['subgroups']['items']['description']] if 'resources' in part: package['resources'] = [] for theresource in part['resources']['items']: resource = {} resource['url'] = theresource['properties']['url']['description'] resource['format'] = theresource['properties']['format'][ 'description'].split('/')[1].upper() if 'moers.de' not in resource['url']: resource['url'] = 'http://www.moers.de' + package['url'] if resource['format'] == 'NSF': resource['format'] = 'XML' package['resources'].append(resource) package['extras'] = {} package['extras']['temporal_coverage_from'] = part['extras']['properties'][ 'dates']['items']['properties']['date']['description'][6:10] package['extras']['terms_of_use'] = {} package['extras']['terms_of_use']['licence_id'] = part['license_id'][ 'description'] # Store a copy of the metadata row['metadata'] = part row[u'Stadt'] = city row[u'Dateibezeichnung'] = package['title'] row[u'Beschreibung'] = package['notes'] row[u'URL PARENT'] = package['url'] # Get resources and formats if ('resources' in package and len(package['resources']) > 0): formats = [] files = [] for resource in package['resources']: files.append(resource['url']) formats.append(resource['format']) row[u'Format'] = list(set(formats)) row[u'geo'] = metautils.isgeo(row[u'Format']) row[u'files'] = files if 'temporal_coverage_from' in package['extras'] and len( package['extras']['temporal_coverage_from']) > 3: row[u'Zeitlicher Bezug'] = package['extras']['temporal_coverage_from'][ 0:4] if ('terms_of_use' in package['extras'] and len(package['extras']['terms_of_use']) > 0): row[u'Lizenz'] = package['extras']['terms_of_use']['licence_id'] row['categories'] = map(lambda x: metautils.govDataLongToODM(x)[0], package.get('groups', [])) return row
def import_package(package): row = {} #Store a copy of the metadata row['metadata'] = package row[u'Stadt'] = city row[u'Dateibezeichnung'] = package['title'] row[u'Beschreibung'] = package['notes'] row[u'url'] = package['url'] #Get resources and formats if ('resources' in package and len(package['resources']) > 0): formats = [] files = [] for resource in package['resources']: files.append(resource['url']) formats.append(resource['format']) row[u'Format'] = formats row[u'geo'] = metautils.isgeo(formats) row[u'files'] = files if 'temporal_coverage_from' in package['extras'] and len(package['extras']['temporal_coverage_from']) > 3: row[u'Zeitlicher Bezug'] = package['extras']['temporal_coverage_from'][0:4] if ('terms_of_use' in package['extras'] and len(package['extras']['terms_of_use']) > 0): row[u'Lizenz'] = package['extras']['terms_of_use']['licence_id'] row['categories'] = map(lambda x: metautils.govDataShortToODM(x)[0], package.get('groups', [])) return row
def import_package(part): row = {} package = {} # Simplify JSON package['title'] = part['title']['description'] package['notes'] = part['notes']['description'] package['author'] = part['author']['description'] package['url'] = part['url'] package['groups'] = [part['subgroups']['items']['description']] if 'resources' in part: package['resources'] = [] for theresource in part['resources']['items']: resource = {} resource['url'] = theresource['properties']['url']['description'] resource['format'] = theresource['properties']['format']['description'].split('/')[1].upper() if 'moers.de' not in resource['url']: resource['url'] = 'http://www.moers.de' + package['url'] if resource['format'] == 'NSF': resource['format'] = 'XML' package['resources'].append(resource) package['extras'] = {} package['extras']['temporal_coverage_from'] = part['extras']['properties']['dates']['items']['properties']['date']['description'][6:10] package['extras']['terms_of_use'] = {} package['extras']['terms_of_use']['licence_id'] = part['license_id']['description'] # Store a copy of the metadata row['metadata'] = part row[u'Stadt'] = city row[u'Dateibezeichnung'] = package['title'] row[u'Beschreibung'] = package['notes'] row[u'URL PARENT'] = package['url'] # Get resources and formats if ('resources' in package and len(package['resources']) > 0): formats = [] files = [] for resource in package['resources']: files.append(resource['url']) formats.append(resource['format']) row[u'Format'] = list(set(formats)) row[u'geo'] = metautils.isgeo(row[u'Format']) row[u'files'] = files if 'temporal_coverage_from' in package['extras'] and len(package['extras']['temporal_coverage_from']) > 3: row[u'Zeitlicher Bezug'] = package['extras']['temporal_coverage_from'][0:4] if ('terms_of_use' in package['extras'] and len(package['extras']['terms_of_use']) > 0): row[u'Lizenz'] = package['extras']['terms_of_use']['licence_id'] row['categories'] = map(lambda x: metautils.govDataLongToODM(x)[0], package.get('groups', [])) return row
def mapData(data, city=True): returndata = [] for result in data: files = [] resources = [] package = result['item'] row = {} if ('res_url' in package): resources = package['res_url'] for file in resources: files.append(file) row[u'filelist'] = files if city: row[u'Stadt'] = metautils.getShortCityName( result['city']['originalname']) else: row[u'Stadt'] = 'rheinlandpfalz' row[u'url'] = 'http://www.daten.rlp.de/dataset/' + package['id'] if ('res_format' in package): row[u'Format'] = list(set(package['res_format'])) row[u'geo'] = metautils.isgeo(row[u'Format']) else: row[u'Format'] = None row[u'geo'] = None row[u'Dateibezeichnung'] = package['title'] if 'notes' in package: row[u'Beschreibung'] = package['notes'] else: row[u'Beschreibung'] = '' if 'license_id' in package: row[u'Lizenz'] = package['license_id'] else: row[u'Lizenz'] = '' if 'maintainer' in package: row[u'Veröffentlichende Stelle'] = package['maintainer'] else: row[u'Veröffentlichende Stelle'] = '' row['categories'] = map(lambda x: metautils.govDataShortToODM(x)[0], package.get('groups', [])) row['metadata'] = package returndata.append(row) return returndata