def package_to_dcat(package):
return converters.ckan_to_dcat(package)
# pkg = ckanops.get_package('ciclones')
# print pkg
# print package_to_dcat(pkg)
# print 'Datasets'
# print list_datasets()
# print 'Groups'
# print list_groups()
catalog = ckanops.dcat_to_utf8_dict(
"http://adela.datos.gob.mx/sedesol/catalogo.json")
print catalog.get('title')
for dataset in catalog.get('dataset', []):
d = converters.dcat_to_ckan(dataset)
d['name'] = munge.munge_title_to_name(d['title'])
print "Creating dataset '%s'" % d['title'], "with %d resources" % len(
d['resources'])
print "Name: %s" % d['name']
print "Org: %s" % d['owner_org']
new_dataset = ckanops.upsert_dataset(demo, d)
if new_dataset:
print 'Dataset upserted'
else:
print 'Something went wrong'
sys.exit()
elif opt in ("-d", "--datasets"):
datasets = remote.action.package_list()
for d in datasets:
print d
elif opt in ("-s", "--harvest", "-p", "--purge-harvest"):
catalog = dcat_to_utf8_dict(arg)
# If purge mode is activated, then delete all org's datasets
if opt in ("-p", "--purge-harvest"):
org_name = catalog['dataset'][0]['publisher']['name']
org_id = munge.munge_name(org_name)
delete_organization_datasets(remote, org_id)
for dcat_dataset in catalog.get('dataset', []):
ckan_dataset = converters.dcat_to_ckan(dcat_dataset)
ckan_dataset['name'] = munge.munge_title_to_name(ckan_dataset['title'])
ckan_dataset['state'] = 'active'
print 'Dataset "%s"' % ckan_dataset['title'], 'with %d resources' % len(ckan_dataset['resources'])
print datetime.datetime.utcnow()
new_dataset = upsert_dataset(remote, ckan_dataset)
if new_dataset:
print 'Dataset upserted'
else:
print 'Something went wrong'
elif opt in ("-q", "--find-datasets"):
datasets = find_datasets_with_query(remote, arg)
for d in datasets['results']:
print d['title']
elif opt in ("-f", "--find"):
resources = find_resources_with_query(remote, arg)
def package_to_dcat(package):
return converters.ckan_to_dcat(package)
# pkg = ckanops.get_package('ciclones')
# print pkg
# print package_to_dcat(pkg)
# print 'Datasets'
# print list_datasets()
# print 'Groups'
# print list_groups()
catalog = ckanops.dcat_to_utf8_dict("http://adela.datos.gob.mx/sedesol/catalogo.json")
print catalog.get('title')
for dataset in catalog.get('dataset', []):
d = converters.dcat_to_ckan(dataset)
d['name'] = munge.munge_title_to_name(d['title'])
print "Creating dataset '%s'" % d['title'], "with %d resources" % len(d['resources'])
print "Name: %s" % d['name']
print "Org: %s" % d['owner_org']
new_dataset = ckanops.upsert_dataset(demo, d)
if new_dataset:
print 'Dataset upserted'
else:
print 'Something went wrong'
def test_DataNorgeConverter(self):
dcat_format={"title":"Kvalitet p\u00e5 nett - resultatliste 2010","description":[{"language":"nb","value":"\u003Cp\u003EResultatliste fra kvalitetsvurderingen av n\u00e6r 700 offentlige nettsteder 2010. Vurderingen er utf\u00f8rt av Direktoratet for forvaltning og IKT. Detaljerte vurderinger for hvert enkelt nettsted finnes i eget datasett. Se http:\/\/kvalitet.difi.no for mer informasjon om vurderingsarbeidet.\u003C\/p\u003E"}],"landingPage":"http:\/\/kvalitet.difi.no","issued":"2015-03-19","modified":"2016-09-23T14:11","language":["NOR"],"publisher":{"name":"Direktoratet for forvaltning og IKT","mbox":"*****@*****.**"},"keyword":["Forvaltning og offentlig sektor"],"distribution":[{"title":"API","description":[{"language":"nb","value":"Kvalitet p\u00e5 nett - Resultatliste 2010"}],"format":"API","downloadURL":"http:\/\/hotell.difi.no\/download\/difi\/kvalitet\/oversikt2010?download","accessURL":"http:\/\/hotell.difi.no\/?dataset=difi\/kvalitet\/oversikt2010","webserviceURL":"http:\/\/hotell.difi.no\/application.wadl","license":"http:\/\/data.norge.no\/nlod\/"}]}
ckan_format={'tags': [{'name': 'Forvaltning_og_offentlig_sektor'}], 'url': 'http:\\/\\/kvalitet.difi.no', 'notes': '\\u003Cp\\u003EResultatliste fra kvalitetsvurderingen av n\\u00e6r 700 offentlige nettsteder 2010. Vurderingen er utf\\u00f8rt av Direktoratet for forvaltning og IKT. Detaljerte vurderinger for hvert enkelt nettsted finnes i eget datasett. Se http:\\/\\/kvalitet.difi.no for mer informasjon om vurderingsarbeidet.\\u003C\\/p\\u003E', 'title': 'Kvalitet p\\u00e5 nett - resultatliste 2010', 'extras': [{'value': '2015-03-19', 'key': 'dcat_issued'}, {'value': '2016-09-23T14:11', 'key': 'dcat_modified'}, {'value': None, 'key': 'guid'}, {'value': 'Direktoratet for forvaltning og IKT', 'key': 'dcat_publisher_name'}, {'value': '*****@*****.**', 'key': 'dcat_publisher_email'}, {'value': 'NOR', 'key': 'language'}, {'value': '[]', 'key': 'metadata_provenance'}], 'resources': [{'url': 'http:\\/\\/hotell.difi.no\\/download\\/difi\\/kvalitet\\/oversikt2010?download', 'name': 'API', 'format': 'API'}]}
self.assertEqual(ckan_format,converters.dcat_to_ckan(dcat_format))
sys.exit()
elif opt in ("-d", "--datasets"):
datasets = remote.action.package_list()
for d in datasets:
print d
elif opt in ("-s", "--harvest", "-p", "--purge-harvest"):
catalog = dcat_to_utf8_dict(arg)
# If purge mode is activated, then delete all org's datasets
if opt in ("-p", "--purge-harvest"):
org_name = catalog['dataset'][0]['publisher']['name']
org_id = munge.munge_name(org_name)
delete_organization_datasets(remote, org_id)
for dcat_dataset in catalog.get('dataset', []):
ckan_dataset = converters.dcat_to_ckan(dcat_dataset)
ckan_dataset['name'] = munge.munge_title_to_name(ckan_dataset['title'])
ckan_dataset['state'] = 'active'
print 'Dataset "%s"' % ckan_dataset['title'], 'with %d resources' % len(ckan_dataset['resources'])
print datetime.datetime.utcnow()
new_dataset = upsert_dataset(remote, ckan_dataset)
if new_dataset:
print 'Dataset upserted'
else:
print 'Something went wrong'
elif opt in ("-q", "--find-datasets"):
datasets = find_datasets_with_query(remote, arg)
for d in datasets['results']:
print d['title']
elif opt in ("-f", "--find"):
resources = find_resources_with_query(remote, arg)
