def import_data(rec): rec['originating_portal'] = portalname rec['city'] = city rec['source'] = 'd' rec['publisher'] = '' rec['description'] = None rec['costs'] = None rec['metadata_xml'] = None rec['spatial'] = False rec['categories'] = [category_to_odm_map[rec['categories']]] rec['filelist'] = [] rec['metadata'] = '' # according to http://www.arnsberg.de/open-data/nutzungsbedingungen.php # nothing seems to be marked different rec['licenseshort'] = 'dl-de-zero-2.0' rec['open'] = metautils.isopen(rec['licenseshort']) # If a year of the 21st century is in the title use it as the temporalextent # insted of the date the file was added. # This is inconsistend but still better? t = re.search(r'20\d\d', rec['title']) if t: rec['temporalextent'] = t.group(0) return rec
def recordToDB(rec): db = {} db['city'] = 'braunschweig' db['source'] = 'd' db['costs'] = None db['url'] = rec['url'] db['title'] = rec['title'] db['description'] = rec['abstract'] db['temporalextent'] = rec['created'] db['publisher'] = rec['organisation'] db['filelist'] = rec['filelist'] db['formats'] = formatsToODM(rec['formats']) db['categories'] = categoriesToODM(rec['topic category']) db['licenseshort'] = licenseToODM(rec['rights']) db['open'] = metautils.isopen(db['licenseshort']) db['spatial'] = isSpatialFormat(db['formats']) additionalMetadata = [ 'accessRights', 'modified', 'spatials', 'type', 'subjects', 'categoriesB' ] db['metadata'] = dict(db.items() + {key: rec[key] for key in additionalMetadata}.items()) # xml metadata only includes data from catalog api?! db['metadata_xml'] = rec['xml'] return db
def import_data(self, d): d = metautils.gerToEngKeys(d) d['originating_portal'] = portalname d['accepted'] = True d['source'] = 'd' d['metadata_xml'] = None d['costs'] = None d['spatial'] = None d['open'] = metautils.isopen(d.get('licenseshort', '').strip()) d['temporalextent'] = '' # have a look if its there return d
def import_data(self, rec): d = imp_rec(rec) d = metautils.gerToEngKeys(d) d['open'] = metautils.isopen(d.get('licenseshort', '').strip()) d['json'] = '' d['publisher'] = '' d['originating_portal'] = 'daten.ulm.de' d['accepted'] = True d['source'] = 'd' d[u'metadata_xml'] = rec.get('metadata_xml', '') return d
def import_data(self, d): d = import_package(d) d = metautils.gerToEngKeys(d) d = dict(d) d['originating_portal'] = portalname d['accepted'] = True d['source'] = 'd' d['metadata_xml'] = None d['costs'] = None d['spatial'] = None d['open'] = metautils.isopen(d['licenseshort'].strip()) d['publisher'] = '' # actually its in the data d['filelist'] = d['files'] return d
def import_data(self, d): d = import_package(d) d = metautils.gerToEngKeys(d) d['originating_portal'] = portalname d['accepted'] = True d['costs'] = None d['open'] = metautils.isopen(d['licenseshort']) d['publisher'] = None d['spatial'] = None d['source'] = 'd' d['metadata_xml'] = None d['filelist'] = d['files'] return d
def import_data(self, d): d = importCity(self.city, self.url, d) if d != {}: d = metautils.gerToEngKeys(d) d = dict(d) d['originating_portal'] = self.portalname d['accepted'] = True d['costs'] = None d['spatial'] = None d['source'] = 'd' d['metadata_xml'] = None d['formats'] = list(d['formats']) d['open'] = metautils.isopen(d['licenseshort'].strip()) if 'categoies' not in d: d['categories'] = [] d['filelist'] = d['files'] return d
def import_data(self, d): d = importCity(self.city, self.url, d) if d != {}: d = metautils.gerToEngKeys(d) d = dict(d) d['city'] = self.city d['originating_portal'] = self.portalname d['accepted'] = True d['costs'] = None d['spatial'] = None d['source'] = 'd' d['metadata_xml'] = None d['formats'] = list(d['formats']) d['open'] = metautils.isopen(d['licenseshort'].strip()) if 'categories' not in d: d['categories'] = [] d['filelist'] = d['files'] return d
def toDB(rec): db = {} db['city'] = 'badenwuerttemberg' # Baden-Württenberg is not a city ?! db['source'] = 'd' db['costs'] = None db['categories'] = categoryToODM(rec['category']) db['url'] = rec['url'] db['title'] = rec['title'] db['description'] = rec['description'] db['publisher'] = rec['herausgeber'] db['filelist'] = [extractUrl(rec['file-url'])] db['formats'] = formatToODM(rec['format']) db['licenseshort'] = licenseToODM(rec['nutzungsbedingungen']) temps = filter(lambda x: x != "", [rec['zeitraum'], rec['stichtag'], rec['publiziert am']]) db['temporalextent'] = temps[0] if temps else None db['open'] = metautils.isopen(db['licenseshort']) db['spatial'] = False db['metadata'] = '' db['metadata_xml'] = None return db
def toDB(rec): db = {} db['city'] = 'badenwuerttemberg' # Baden-Württenberg is not a city ?! db['source'] = 'd' db['costs'] = None db['categories'] = categoryToODM(rec['category']) db['url'] = rec['url'] db['title'] = rec['title'] db['description'] = rec['description'] db['publisher'] = rec['herausgeber'] db['filelist'] = [extractUrl(rec['file-url'])] db['formats'] = formatToODM(rec['format']) db['licenseshort'] = licenseToODM(rec['nutzungsbedingungen']) temps = filter(lambda x: x != "", [rec['zeitraum'], rec['stichtag'], rec['publiziert am']]) db['temporalextent'] = temps[0] if temps else None db['open'] = metautils.isopen(db['licenseshort']) db['spatial'] = False db['metadata'] = '' db['metadata_xml'] = None return db
def importCity(cityname, url, package): if cityname == 'hamburg': # Only take 'open data' if package['type'] != 'dataset' or 'forward-reference' in package['title']: return {} resources = [] formats = set() files = [] # Key for the file link in the resource urlkeys = ['url'] formatkey = 'format' if ('resources' in package): resources = package['resources'] for file in resources: for urlkey in urlkeys: if (file[urlkey] not in [None, '']): if '://' not in file[urlkey]: files.append(url + file[urlkey]) else: files.append(file[urlkey]) break if formatkey in file and file[formatkey] not in [None, '']: format = file[formatkey] formats.add(format.upper()) row = {} row[u'Stadt'] = cityname row[u'Dateibezeichnung'] = package['title'] row[u'URL PARENT'] = url + '/dataset/' + package['name'] if cityname in ('hamburg', 'koeln', 'frankfurt', 'aachen', 'berlin', 'muenchen'): if cityname in ('hamburg', 'frankfurt', 'aachen'): licensekey = 'license_id' vstellekey = 'author' catskey = 'groups' catssubkey = 'title' elif cityname == 'muenchen': licensekey = 'license_id' vstellekey = 'maintainer' catskey = 'groups' catssubkey = 'title' elif cityname in ('koeln', 'berlin'): licensekey = 'license_title' vstellekey = 'maintainer' if cityname == 'koeln': catskey = 'tags' elif cityname == 'berlin': catskey = 'groups' catssubkey = 'name' # Generate URL for the catalog page if 'notes' in package and package['notes'] != None: row[u'Beschreibung'] = package['notes'] if cityname == 'koeln': soup = BeautifulSoup(row[u'Beschreibung']) row[u'Beschreibung'] = soup.getText('\n') else: row[u'Beschreibung'] = '' row[u'Zeitlicher Bezug'] = '' if licensekey in package and package[licensekey] != None: row[u'Lizenz'] = package[licensekey] # if not already short, try to convert if metautils.isopen(row[u'Lizenz']) is 'Unbekannt': row[u'Lizenz'] = metautils.long_license_to_short(row[u'Lizenz']) else: row[u'Lizenz'] = 'nicht bekannt' if vstellekey in package and package[vstellekey] != None: row[u'Veröffentlichende Stelle'] = package[vstellekey] else: row[u'Veröffentlichende Stelle'] = '' if 'extras' in package: print 'WARNING: No author/maintainer/publisher, checking extras' for extra in package['extras']: if extra['key'] == 'contacts': print 'WARNING: No author, but amazingly there is possibly data in the contacts: ' + extra['value'] for group in metautils.setofvaluesasarray(package[catskey], catssubkey): if cityname != 'berlin': odm_cats = metautils.govDataLongToODM(group) else: odm_cats = berlin_to_odm(group) row[u'categories'] = odm_cats # Bonn is just different enough to do it separately. TODO: Consider combining into above. elif cityname == 'bonn': row[u'Beschreibung'] = package.get('description', '') for timeattempt in ['temporal', 'modified']: if timeattempt in package and package[timeattempt] not in [None, '']: row[u'Zeitlicher Bezug'] = package[timeattempt] break row[u'Zeitlicher Bezug'] = row.get(u'Zeitlicher Bezug', '') row[u'Lizenz'] = package.get('license', False) if not row[u'Lizenz']: row[u'Lizenz'] = package['license_title'] row[u'Veröffentlichende Stelle'] = package.get('publisher', '') cats = package.get('keyword', []) odm_cats = map(lambda x: metautils.govDataLongToODM(x, checkAll=True), cats) resources = package.get(u'distribution', []) for r in resources: files.append(r[u'accessURL']) formats.append(r[u'format']) row[u'Format'] = formats row[u'files'] = files row['metadata'] = package return row
def get_categorie_content(category_link): # Get the page allrecords = [] parser = etree.HTMLParser(encoding='utf-8') data = etree.parse(rooturl + category_link, parser) # Get the category category = data.xpath('/html/body/div/div[5]/div/div[1]//h1/text()')[0].strip() # category = urllib.unquote(category).decode('utf8') if (verbose): print 'Category: ' + ascii_only(category) datasets = get_datasets(data) numdatasets = len(datasets) if (verbose): print 'There are ' + str(numdatasets) + ' datasets' # Now get the html for each one. This is painful. # The bit of html concerning the datasets: corehtml = data.xpath('//div[@id=\'ContentBlock\']')[0] # First try to split by the horizontal rules. This usually works, but not always datasetparts = etree.tostring(corehtml).split('<hr id="hr') if (verbose): print 'Found ' + str(len(datasetparts)) + ' datasets by splitting by hr elements with ids' if len(datasetparts) != numdatasets: if (verbose): print 'This doesn\'t match. Trying with links to TOC' # If there is TOC, this works. There isn\'t always one. datasetparts = etree.tostring(corehtml).split('nach oben') del datasetparts[len(datasetparts) - 1] for index in range(0, len(datasetparts)): datasetparts[index] = datasetparts[index] + '</a>' if (verbose): print 'Found ' + str(len(datasetparts)) + ' datasets by splitting by links to TOC' if len(datasetparts) != numdatasets: if (verbose): print 'Well, that didn\'t work either. Giving up' print 'Exciting because of a serious error - turn on verbose in the code to find out what dataset is causing the problem' exit() else: if numdatasets > 1: for index in range(1, len(datasetparts)): # That split makes for bad HTML. Make it better. datasetparts[index] = '<hr id="hr' + datasetparts[index] count = 1 for datasetpart in datasetparts: data = etree.HTML(datasetpart) record = {} record['city'] = 'bochum' record['categories'] = [] record['categories'].append(category) datasets = get_datasets(data) record['title'] = datasets[0] if (verbose): print 'Parsing dataset ' + ascii_only(record['title']) if 'noch im Aufbau' in record['title']: # Nothing to see here if (verbose): print 'Empty category' continue record['url'] = rooturl + category_link + '#par' + str(count) count += 1 datatables, filetables = findfilesanddata(data) if len(datatables) == 0: if (verbose): print 'This record contains no data... checking for link to another page...' checkforsubpage = data.xpath('//span//a') for link in checkforsubpage: if (verbose): print etree.tostring(link) if len(link.xpath('text()')) > 0 and u'zu den Daten' in link.xpath('text()')[0]: testurl = link.xpath('@href')[0] if (verbose): print 'Following/updating URL: ' + rooturl + testurl record['url'] = rooturl + testurl datatables, filetables = findfilesanddata(html.parse(rooturl + testurl)) # get the data on the files, and get each link in it record['filelist'] = [] for table in filetables: record['filelist'].extend([(rooturl + x) for x in etree.HTML(table).xpath('//a/@href')]) record['formats'] = set() record['spatial'] = False for file in record['filelist']: formatarray = file.split('/')[-1].split('.') format = 'Unknown' if len(formatarray)>1: format = formatarray[1].upper().split('?')[0] elif 'WMS' in formatarray[0]: format = 'WMS' elif 'WFS' in formatarray[0]: format = 'WFS' record['formats'].add(format) if (format.upper() in metautils.geoformats): record['spatial'] = True record['formats'] = list(record['formats']) if len(datatables) > 1: if (verbose): print 'ERROR: More than one data table' print 'Exciting because of a serious error - turn on verbose in the code to find out what dataset is causing the problem' exit() elif len(datatables) == 0: if (verbose): print 'ERROR: No data table' print 'Exciting because of a serious error - turn on verbose in the code to find out what dataset is causing the problem' exit() # parse the data table by row if (verbose): print 'Reading datatable...' rowelements = etree.HTML(datatables[0]).xpath('//tr') for row in rowelements: if len(row.xpath('td[1]/text()')) == 0: continue key = row.xpath('td[1]/text()')[0] if (verbose): print ascii_only(key) if len(row.xpath('td[2]/text()')) != 0: val = row.xpath('td[2]/text()')[0] elif len(row.xpath('td[2]//a')) != 0: val = row.xpath('td[2]//a/text()')[0] else: if (verbose): print 'ERROR: Missing value' print 'Exciting because of a serious error - turn on verbose in the code to find out what dataset is causing the problem' exit() if (verbose): print ascii_only('Parsing key ' + key.replace(':', '') + ' with value ' + val) if u'veröffentlicht' in key: record['publisher'] = val elif u'geändert' in key: record['temporalextent'] = val.split(' ')[2] elif u'Lizenz' in key: record['licenseshort'] = metautils.long_license_to_short(val) record['open'] = metautils.isopen(record['licenseshort']) elif u'Webseite' in key: record['website'] = row.xpath('td[2]//a/@href')[0] # keep, as 'original' metadata if 'http://' not in record['website']: record['website'] = rooturl + record['website'] elif u'Kontakt' in key: record['contact'] = rooturl + row.xpath('td[2]//a/@href')[0] allrecords.append(record) return allrecords
def test_is_open_uppercase(self): assert metautils.isopen('CC BY 3.0 DE') == 'Offen'
def test_isopen(self): assert metautils.isopen('dl-de-by-2.0') == 'Offen'
def importCity(cityname, url, package): if cityname == 'hamburg': # Only take 'open data' if package['type'] != 'dataset' or 'forward-reference' in package[ 'title']: return {} #There is a version of CKAN that can output private datasets! but DKAN is using this field for different purposes if package['private'] and cityname not in dkanCities: return {} resources = [] formats = set() files = [] # Key for the file link in the resource urlkeys = ['url'] formatkey = 'format' if ('resources' in package): resources = package['resources'] for file in resources: for urlkey in urlkeys: if (file[urlkey] not in [None, '']): if '://' not in file[urlkey]: files.append(url + file[urlkey]) else: files.append(file[urlkey]) break if formatkey in file and file[formatkey] not in [None, '']: format = file[formatkey] formats.add(format.upper()) row = {} row[u'Stadt'] = cityname row[u'Dateibezeichnung'] = package['title'] if 'name' in package: row[u'URL PARENT'] = url + '/dataset/' + package['name'] elif 'url' in package: row[u'URL PARENT'] = package['url'] else: row[u'URL PARENT'] = '' if cityname in v3cities: licensekey = 'license_id' vstellekey = 'author' catskey = 'groups' catssubkey = 'title' if cityname == 'berlin': catssubkey = 'name' elif cityname == 'muenchen': licensekey = 'license_id' vstellekey = 'maintainer' catskey = 'groups' catssubkey = 'title' elif cityname in dkanCities: licensekey = 'license_title' vstellekey = 'maintainer' catskey = 'tags' catssubkey = 'name' # Generate URL for the catalog page if 'notes' in package and package['notes'] != None: row[u'Beschreibung'] = package['notes'] if cityname == 'koeln': soup = BeautifulSoup(row[u'Beschreibung']) row[u'Beschreibung'] = soup.getText('\n') else: row[u'Beschreibung'] = '' row[u'Zeitlicher Bezug'] = '' if licensekey in package and package[licensekey] != None: row[u'Lizenz'] = package[licensekey] # if not already short, try to convert if metautils.isopen(row[u'Lizenz']) is 'Unbekannt': row[u'Lizenz'] = metautils.long_license_to_short(row[u'Lizenz']) else: row[u'Lizenz'] = 'nicht bekannt' if vstellekey in package and package[vstellekey] != None: row[u'Veröffentlichende Stelle'] = package[vstellekey] else: row[u'Veröffentlichende Stelle'] = '' if 'extras' in package: print 'WARNING: No author/maintainer/publisher, checking extras' for extra in package['extras']: if extra['key'] == 'contacts': print 'WARNING: No author, but amazingly there is possibly data in the contacts: ' + extra[ 'value'] cat_groups = metautils.setofvaluesasarray(package[catskey], catssubkey) if cityname != 'berlin': odm_cats = metautils.matchCategories(cat_groups) else: for group in cat_groups: odm_cats = berlin_to_odm(group) row[u'categories'] = odm_cats row[u'Format'] = formats row[u'files'] = files row['metadata'] = package row[u'original_metadata'] = { u'metadata_created': package['metadata_created'], u'metadata_modified': package['metadata_modified'] } return row