def import_package(package):
row = {}
#Store a copy of the metadata
row['metadata'] = package
row[u'Stadt'] = city
row[u'Dateibezeichnung'] = package['title']
row[u'Beschreibung'] = package['notes']
row[u'url'] = package['url']
#Get resources and formats
if ('resources' in package and len(package['resources']) > 0):
formats = []
files = []
for resource in package['resources']:
files.append(resource['url'])
formats.append(resource['format'])
row[u'Format'] = formats
row[u'geo'] = metautils.isgeo(formats)
row[u'files'] = files
if 'temporal_coverage_from' in package['extras'] and len(
package['extras']['temporal_coverage_from']) > 3:
row[u'Zeitlicher Bezug'] = package['extras']['temporal_coverage_from'][
0:4]
if ('terms_of_use' in package['extras']
and len(package['extras']['terms_of_use']) > 0):
row[u'Lizenz'] = package['extras']['terms_of_use']['licence_id']
row['categories'] = map(lambda x: metautils.govDataShortToODM(x)[0],
package.get('groups', []))
return row
def mapData(package):
files = []
resources = []
row = {}
if ('res_url' in package):
resources = package['res_url']
for file in resources:
files.append(file)
row[u'filelist'] = files
row[u'Stadt'] = city
row[u'url'] = 'http://www.opendata-hro.de/dataset/' + package['id']
if ('res_format' in package):
row[u'Format'] = list(set(package['res_format']))
row[u'geo'] = metautils.isgeo(row[u'Format'])
row[u'Dateibezeichnung'] = package['title']
if 'notes' in package:
row[u'Beschreibung'] = package['notes']
if 'license_id' in package:
row[u'Lizenz'] = package['license_id']
if 'maintainer' in package:
row[u'Veröffentlichende Stelle'] = package['maintainer']
row['categories'] = map(lambda x: metautils.govDataShortToODM(x)[0], package.get('groups', []))
row['metadata'] = package
return row
def import_package(part):
row = {}
package = {}
# Simplify JSON
package['title'] = part['title']['description']
package['notes'] = part['notes']['description']
package['author'] = part['author']['description']
package['url'] = part['url']
package['groups'] = [part['subgroups']['items']['description']]
if 'resources' in part:
package['resources'] = []
for theresource in part['resources']['items']:
resource = {}
resource['url'] = theresource['properties']['url']['description']
resource['format'] = theresource['properties']['format'][
'description'].split('/')[1].upper()
if 'moers.de' not in resource['url']:
resource['url'] = 'http://www.moers.de' + package['url']
if resource['format'] == 'NSF': resource['format'] = 'XML'
package['resources'].append(resource)
package['extras'] = {}
package['extras']['temporal_coverage_from'] = part['extras']['properties'][
'dates']['items']['properties']['date']['description'][6:10]
package['extras']['terms_of_use'] = {}
package['extras']['terms_of_use']['licence_id'] = part['license_id'][
'description']
# Store a copy of the metadata
row['metadata'] = part
row[u'Stadt'] = city
row[u'Dateibezeichnung'] = package['title']
row[u'Beschreibung'] = package['notes']
row[u'URL PARENT'] = package['url']
# Get resources and formats
if ('resources' in package and len(package['resources']) > 0):
formats = []
files = []
for resource in package['resources']:
files.append(resource['url'])
formats.append(resource['format'])
row[u'Format'] = list(set(formats))
row[u'geo'] = metautils.isgeo(row[u'Format'])
row[u'files'] = files
if 'temporal_coverage_from' in package['extras'] and len(
package['extras']['temporal_coverage_from']) > 3:
row[u'Zeitlicher Bezug'] = package['extras']['temporal_coverage_from'][
0:4]
if ('terms_of_use' in package['extras']
and len(package['extras']['terms_of_use']) > 0):
row[u'Lizenz'] = package['extras']['terms_of_use']['licence_id']
row['categories'] = map(lambda x: metautils.govDataLongToODM(x)[0],
package.get('groups', []))
return row
def import_package(package):
row = {}
#Store a copy of the metadata
row['metadata'] = package
row[u'Stadt'] = city
row[u'Dateibezeichnung'] = package['title']
row[u'Beschreibung'] = package['notes']
row[u'url'] = package['url']
#Get resources and formats
if ('resources' in package and len(package['resources']) > 0):
formats = []
files = []
for resource in package['resources']:
files.append(resource['url'])
formats.append(resource['format'])
row[u'Format'] = formats
row[u'geo'] = metautils.isgeo(formats)
row[u'files'] = files
if 'temporal_coverage_from' in package['extras'] and len(package['extras']['temporal_coverage_from']) > 3:
row[u'Zeitlicher Bezug'] = package['extras']['temporal_coverage_from'][0:4]
if ('terms_of_use' in package['extras'] and len(package['extras']['terms_of_use']) > 0):
row[u'Lizenz'] = package['extras']['terms_of_use']['licence_id']
row['categories'] = map(lambda x: metautils.govDataShortToODM(x)[0], package.get('groups', []))
return row
def import_package(part):
row = {}
package = {}
# Simplify JSON
package['title'] = part['title']['description']
package['notes'] = part['notes']['description']
package['author'] = part['author']['description']
package['url'] = part['url']
package['groups'] = [part['subgroups']['items']['description']]
if 'resources' in part:
package['resources'] = []
for theresource in part['resources']['items']:
resource = {}
resource['url'] = theresource['properties']['url']['description']
resource['format'] = theresource['properties']['format']['description'].split('/')[1].upper()
if 'moers.de' not in resource['url']:
resource['url'] = 'http://www.moers.de' + package['url']
if resource['format'] == 'NSF': resource['format'] = 'XML'
package['resources'].append(resource)
package['extras'] = {}
package['extras']['temporal_coverage_from'] = part['extras']['properties']['dates']['items']['properties']['date']['description'][6:10]
package['extras']['terms_of_use'] = {}
package['extras']['terms_of_use']['licence_id'] = part['license_id']['description']
# Store a copy of the metadata
row['metadata'] = part
row[u'Stadt'] = city
row[u'Dateibezeichnung'] = package['title']
row[u'Beschreibung'] = package['notes']
row[u'URL PARENT'] = package['url']
# Get resources and formats
if ('resources' in package and len(package['resources']) > 0):
formats = []
files = []
for resource in package['resources']:
files.append(resource['url'])
formats.append(resource['format'])
row[u'Format'] = list(set(formats))
row[u'geo'] = metautils.isgeo(row[u'Format'])
row[u'files'] = files
if 'temporal_coverage_from' in package['extras'] and len(package['extras']['temporal_coverage_from']) > 3:
row[u'Zeitlicher Bezug'] = package['extras']['temporal_coverage_from'][0:4]
if ('terms_of_use' in package['extras'] and len(package['extras']['terms_of_use']) > 0):
row[u'Lizenz'] = package['extras']['terms_of_use']['licence_id']
row['categories'] = map(lambda x: metautils.govDataLongToODM(x)[0], package.get('groups', []))
return row
def mapData(data, city=True):
returndata = []
for result in data:
files = []
resources = []
package = result['item']
row = {}
if ('res_url' in package):
resources = package['res_url']
for file in resources:
files.append(file)
row[u'filelist'] = files
if city:
row[u'Stadt'] = metautils.getShortCityName(
result['city']['originalname'])
else:
row[u'Stadt'] = 'rheinlandpfalz'
row[u'url'] = 'http://www.daten.rlp.de/dataset/' + package['id']
if ('res_format' in package):
row[u'Format'] = list(set(package['res_format']))
row[u'geo'] = metautils.isgeo(row[u'Format'])
else:
row[u'Format'] = None
row[u'geo'] = None
row[u'Dateibezeichnung'] = package['title']
if 'notes' in package:
row[u'Beschreibung'] = package['notes']
else:
row[u'Beschreibung'] = ''
if 'license_id' in package:
row[u'Lizenz'] = package['license_id']
else:
row[u'Lizenz'] = ''
if 'maintainer' in package:
row[u'Veröffentlichende Stelle'] = package['maintainer']
else:
row[u'Veröffentlichende Stelle'] = ''
row['categories'] = map(lambda x: metautils.govDataShortToODM(x)[0],
package.get('groups', []))
row['metadata'] = package
returndata.append(row)
return returndata