def mapData(data, nocity=False):
returndata = []
uniquecities = set()
for result in data:
files = []
resources = []
if cityimport == 'rlp':
package = result['item']
else:
package = result
row = metautils.getBlankRow()
if ('res_url' in package):
resources = package['res_url']
for file in resources:
files.append(file)
row[u'files'] = files
if cityimport == 'rlp':
if not nocity:
row[u'Stadt'] = metautils.getShortCityName(result['city']['originalname'])
uniquecities.add(result['city']['originalname'])
else:
row[u'Stadt'] = 'rheinlandpfalz'
row[u'URL PARENT'] = 'http://www.daten.rlp.de/dataset/' + package['id']
elif cityimport == 'rostock':
row[u'Stadt'] = 'rostock'
row[u'URL PARENT'] = 'http://www.opendata-hro.de/dataset/' + package['id']
if ('res_format' in package):
[formattext, geo] = metautils.processListOfFormats(package['res_format'])
row[u'Format'] = formattext
row[u'geo'] = geo
row[u'Dateibezeichnung'] = package['title']
if 'notes' in package:
row[u'Beschreibung'] = package['notes']
if 'license_id' in package:
row[u'Lizenz'] = package['license_id']
if 'maintainer' in package:
row[u'Veröffentlichende Stelle'] = package['maintainer']
if 'groups' in package:
for group in package['groups']:
odm_cats = metautils.govDataShortToODM(group)
if len(odm_cats) > 0:
for cat in odm_cats:
row[cat] = 'x'
row[u'Noch nicht kategorisiert'] = ''
returndata.append(row)
return [returndata, uniquecities]
for text in jtexts:
jtext = ('[{\"name\"' + text[0:len(text)-7] + ']').replace('application\\', 'application/').replace('\r', '').replace('\n', '').replace('},"license_id"', ']},"license_id"').replace('"description": "Ressourcen: Folgende Felder können für jede Ressource individuell angegeben werden.","type": "array","items": {','"description": "Ressourcen: Folgende Felder können für jede Ressource individuell angegeben werden.","type": "array","items": [{')
try:
package = json.loads(jtext)
except:
jtext = jtext[0:len(jtext)-1] + '}}]'
package = json.loads(jtext)
packages.append(package[0])
#Save the fixed JSON
with open('../metadata/moers/catalog.json', 'wb') as outfile:
json.dump(packages, outfile)
datafordb = []
for part in packages:
row = metautils.getBlankRow()
if city == 'moers':
package = {}
#Simplify JSON
package['title'] = part['title']['description']
package['notes'] = part['notes']['description']
package['author'] = part['author']['description']
package['url'] = part['url']['description']
package['groups'] = [part['subgroups']['items']['description']]
if 'resources' in part:
package['resources'] = []
for theresource in part['resources']['items']:
resource = {}
resource['url'] = theresource['properties']['url']['description']
resource['format'] = theresource['properties']['format']['description'].split('/')[1].upper()
#Separate out communal data
allcities = metautils.getCities()
#First take the Verbandsgemeinde
cities = metautils.getCities(alternativeFile='verbandsgemeinderlp.csv')
#Then all settlements in RLP
cities.extend(metautils.filterCitiesByLand(allcities, 'Rheinland-Pfalz'))
beforefilter = len(data['results'])
[data, notcitydata] = metautils.findOnlyCityData(data['results'], cities)
afterfilter = len(data)
print 'Of the total ' + str(beforefilter) + ' records, ' + str(afterfilter) + ' appear to be related to a city'
print 'The rest (' + str(len(notcitydata)) + ') will be assigned to Rheinland-Pfalz as a Land'
else:
data = data['results']
#Map and write the data. Still wondering how much of this can/should be pulled out to metautils
row = metautils.getBlankRow()
datafordb = []
[returnData, uniquecities] = mapData(data)
datafordb.extend(returnData)
if cityimport == 'rlp':
[returnData, ignoreuniquecities] = mapData(notcitydata, nocity=True)
datafordb.extend(returnData)
#Write data to the DB
metautils.setsettings(settings)
if cityimport == 'rlp':
#Update city list
metautils.addCities(uniquecities, 'Rheinland-Pfalz')
#Remove this catalog's data
metautils.removeDataFromPortal('daten.rlp.de')