def get_existing_datasets_ckan():
# Return list of strings (dataset names)
r = make_ckan_api_call("api/action/package_list", {'all_fields': True})
if r is not None:
return r['result']
else:
return None
def purge_dataset(dataset_name_or_id):
r = make_ckan_api_call("api/action/dataset_purge", {"id": dataset_name_or_id})
if not r["success"]:
raise CKANAPIException(
{"message": "Impossible to create dataset", "dataset": dataset_name_or_id, "error": r["error"]}
)
def create_dataset(dataset, all_organizations):
params = {
"title": dataset.title,
"name": dataset_title_to_name(dataset.title),
"notes": dataset.description,
"owner_org": all_organizations[dataset.publishing_organization_key].name,
"url": urlparse.urljoin("http://www.gbif.org/dataset/", dataset.uuid),
# Having difficulties adding extras to the dataset.
# So far, it works IF the extras parameter is not named extras (myextras is good), and a dict
# (not a list of dicts) is passed. It is, however, not shown in the web interface later...
#'extras': [{'dataset_type': dataset.dataset_type}]
"gbif_uuid": dataset.uuid,
# A Heavy but perfectly working solution: add the field via a plugin like in the tutorial:
# http://docs.ckan.org/en/latest/extensions/adding-custom-fields.html
# Then pass the parameter as a first-class one (title, name, ...) (no list of dicts: just a key and value)
"dataset_type": dataset.dataset_type,
"administrative_contact": dataset.administrative_contact_full,
"administrative_contact_name": dataset.administrative_contact_name,
"metadata_contact": dataset.metadata_contact,
}
if dataset.dwca_url:
params["dwca_url"] = dataset.dwca_url
if dataset.website:
params["dataset_website"] = dataset.website
r = make_ckan_api_call("api/action/package_create", params)
if not r["success"]:
raise CKANAPIException({"message": "Impossible to create dataset", "dataset": dataset, "error": r["error"]})
def create_in_ckan(self):
extras = []
if self.homepages:
extras.append({"key": "Homepage(s)", "value": ",".join(self.homepages)})
if self.city:
extras.append({"key": "City", "value": self.city})
if self.lat:
extras.append({"key": "Latitude", "value": self.lat})
if self.lon:
extras.append({"key": "Longitude", "value": self.lon})
for c in self.contacts:
contact_type, contact_details = c.for_display()
# TODO: do we have an error if several contact have the same contact type?
extras.append({"key": contact_type, "value": contact_details})
params = {
"name": self.name,
"id": self.key,
"title": self.title,
"image_url": ORGANIZATION_LOGOS.get(self.key, ""),
# API documentation about extras is unclear, but this works:
"extras": extras,
}
if self.description:
params["description"] = self.description
r = make_ckan_api_call("api/action/organization_create", params)
return r["success"]
def get_from_ckan(self):
params = {'id': self.name}
r = make_ckan_api_call("api/action/vocabulary_show", params)
if r is not None and r['success'] is not None and r['success'] and r[
'result'] is not None:
return Vocabulary(id=r['result']['id'],
name=r['result']['name'],
tags=r['result']['tags'])
def get_existing_vocabularies_ckan(cls):
r = make_ckan_api_call("api/action/vocabulary_list",
{'all_fields': True})
if r is not None:
return [
Vocabulary(id=res['id'], name=res['name'], tags=res['tags'])
for res in r['result']
]
def purge_ckan(self):
r = make_ckan_api_call("api/action/vocabulary_delete", {'id': self.id})
if not r['success']:
raise CKANAPIException({
"message": "Impossible to purge vocabulary",
"vocabulary_id": self.id,
"reason": r['error']['message']
})
def purge_ckan(self):
r = make_ckan_api_call("api/action/organization_purge",
{'id': self.key})
if not r['success']:
raise CKANAPIException({
"message": "Impossible to purge organization",
"organization_key": self.key,
"reason": r['error']['message']
})
def purge_ckan(cls, id):
r = make_ckan_api_call("api/action/dataset_purge", {'id': id})
if not r['success']:
raise CKANAPIException({
"message": "Impossible to purge dataset",
"dataset": id,
"error": r['error']
})
def purge_ckan(self):
r = make_ckan_api_call("api/action/organization_purge", {"id": self.key})
if not r["success"]:
raise CKANAPIException(
{
"message": "Impossible to purge organization",
"organization_key": self.key,
"reason": r["error"]["message"],
}
)
def create_or_update_in_ckan(self):
existing_vocab = self.get_from_ckan()
if existing_vocab is not None:
self.id = existing_vocab.id
params = {'id': self.id, 'name': self.name} #,
# 'tags': existing_vocab.tags + [{'name': k.name, 'vocabulary_id': self.id} for k in self.tags]}
r = make_ckan_api_call("api/action/vocabulary_update", params)
else:
params = {
'name': self.name
} #, 'tags': [{'name': k.name} for k in self.tags]}
r = make_ckan_api_call("api/action/vocabulary_create", params)
if r is not None:
if not r['success']:
print("Couldn't create/update vocabulary " + self.name)
else:
id = r['result']['id']
self.id = id
return self
def purge_dataset(dataset_name_or_id):
r = make_ckan_api_call("api/action/dataset_purge",
{'id': dataset_name_or_id})
if not r['success']:
raise CKANAPIException({
"message": "Impossible to create dataset",
"dataset": dataset_name_or_id,
"error": r['error']
})
def create_in_ckan(self):
# Document is incorrect regarding packages: we need an id parameter, that in fact receive the dataset name... confusing.
params = {
"name": self.name,
"title": self.title,
"packages": [{"id": dataset_title_to_name(dataset.title)} for dataset in self.attached_datasets],
"image_url": self.logo_url,
}
r = make_ckan_api_call("api/action/group_create", params)
return r["success"]
def get_existing_keywords_ckan(cls):
r = make_ckan_api_call("api/action/vocabulary_list",
{'all_fields': True})
result = []
if r is not None:
for res in r['result']:
for tag in res['tags']:
result.append(
Keyword(name=tag['name'],
id=tag["id"],
vocabulary_id=tag['vocabulary_id']))
return result
def create_in_ckan(self):
params = {}
r = None
if self.name is not None:
if self.vocabulary_id is not None:
v = Vocabulary(name=self.vocabulary_id,
tags=[self],
id=self.vocabulary_id)
vocabulary = v.create_or_update_in_ckan()
self.vocabulary_id = vocabulary.id # move from a name-based vocabulary_id to a uuid based one.
params = {
'vocabulary_id': self.vocabulary_id,
'name': self.name
}
r = make_ckan_api_call("api/action/tag_create", params)
return self
else:
params = {'name': self.name}
r = make_ckan_api_call("api/action/tag_create", params)
return self
else:
print("Couldn't create keyword as it is empty")
return self
def create_in_ckan(self):
# Document is incorrect regarding packages: we need an id parameter, that in fact receive the dataset name... confusing.
params = {
'name':
self.name,
'title':
self.title,
'packages': [{
'id': dataset_title_to_name(dataset.title)
} for dataset in self.attached_datasets],
'image_url':
self.logo_url
}
try:
r = make_ckan_api_call("api/action/group_create", params)
return r['success']
except ValueError:
#FIXME: why does we sometimes (only in prod...) get a JSONDecodeError at this stage?
print("Error decoding JSON")
return True
def create_in_ckan(self):
extras = []
if self.homepages:
extras.append({
'key': 'Homepage(s)',
'value': ','.join(self.homepages)
})
if self.city:
extras.append({'key': 'City', 'value': self.city})
if self.lat:
extras.append({'key': 'Latitude', 'value': self.lat})
if self.lon:
extras.append({'key': 'Longitude', 'value': self.lon})
for c in self.contacts:
contact_type, contact_details = c.to_string()
# TODO: do we have an error if several contact have the same contact type?
extras.append({'key': contact_type, 'value': contact_details})
params = {
'name': self.name,
'id': self.key,
'title': self.title,
'image_url': ORGANIZATION_LOGOS.get(self.key, ''),
# API documentation about extras is unclear, but this works:
'extras': extras
}
if self.description:
params['description'] = self.description
r = make_ckan_api_call("api/action/organization_create", params)
if r is not None:
return r['success']
def create_in_ckan(self):
params = {'package_id': self.package_id, 'url': self.url}
if self.revision_id:
params['revision_id'] = self.revision_id
if self.description:
params['description'] = self.description
if self.format:
params['format'] = self.format
if self.hash:
params['hash'] = self.hash
if self.mimetype:
params['mimetype'] = self.mimetype
if self.size:
params['size'] = self.size
if self.created:
params['created'] = self.created.partition(".")[0]
if self.last_modified:
params['last_modified'] = self.last_modified.partition(".")[0]
r = make_ckan_api_call("api/action/resource_create", params)
if r is not None:
return r['success']
def create_dataset(dataset, all_organizations):
params = {
'title': dataset.title,
'name': dataset_title_to_name(dataset.title),
'notes': dataset.description,
'owner_org':
all_organizations[dataset.publishing_organization_key].name,
'url': urljoin("http://www.gbif.org/dataset/", dataset.uuid),
# Having difficulties adding extras to the dataset.
# So far, it works IF the extras parameter is not named extras (myextras is good), and a dict
# (not a list of dicts) is passed. It is, however, not shown in the web interface later...
#'extras': [{'dataset_type': dataset.dataset_type}]
'gbif_uuid': dataset.uuid,
# A Heavy but perfectly working solution: add the field via a plugin like in the tutorial:
# http://docs.ckan.org/en/latest/extensions/adding-custom-fields.html
# Then pass the parameter as a first-class one (title, name, ...) (no list of dicts: just a key and value)
'dataset_type': dataset.dataset_type,
'administrative_contact': dataset.administrative_contact_full,
'administrative_contact_name': dataset.administrative_contact_name,
'metadata_contact': dataset.metadata_contact,
}
if dataset.dwca_url:
params['dwca_url'] = dataset.dwca_url
if dataset.website:
params['dataset_website'] = dataset.website
r = make_ckan_api_call("api/action/package_create", params)
if not r['success']:
raise CKANAPIException({
"message": "Impossible to create dataset",
"dataset": dataset,
"error": r['error']
})
def get_existing_organizations_ckan(cls):
r = make_ckan_api_call("api/action/organization_list", {"all_fields": True})
return [cls(res["id"], res["title"]) for res in r["result"]]
def create_in_ckan(self, all_organizations):
params = {
'title': self.title,
'name': self.id, # ie the IPT identifier
'id': self.
gbif_uuid, # ie the GBIF uuio becomes the internal CKAN identifier
'dataset_type': self.dataset_type,
'notes': self.description,
'owner_org':
all_organizations[self.publishing_organization_key].name,
'url': urljoin("http://www.gbif.org/dataset/", self.gbif_uuid),
'administrative_contact_full': self.administrative_contact_full,
'administrative_contact_name': self.administrative_contact_name,
'metadata_contact_full': self.metadata_contact_full,
'metadata_contact_name': self.metadata_contact_name,
'originator_full': self.originator_full,
'originator_name': self.originator_name,
'eml_created': self.metadata_created,
'eml_modified': self.metadata_modified,
'maintenance_frequency': self.maintenance_frequency,
'start_datetime': self.start_datetime,
'end_datetime': self.end_datetime,
'geo_desc': self.geo_desc,
#'occurrences': jsonpickle.encode(self.occurrences),
'occurrence_count': self.occurrence_count,
'doi': self.doi,
'doi_gbif': self.doi_gbif,
'study_extent': self.study_extent,
'quality_control': self.quality_control
}
if self.dwca_url:
params['dwca_url'] = self.dwca_url
if self.website:
params['dataset_website'] = self.website
geo_json = self.bounds_to_geojson(northbound_lat=self.northbound_lat,
southbound_lat=self.southbound_lat,
eastbound_lon=self.eastbound_lon,
westbound_lon=self.westbound_lon,
geo_desc=self.geo_desc)
if geo_json is not None:
params['extras'] = [{
'key':
'spatial',
'value':
'{"type": "Polygon","coordinates": ' + geo_json + '}'
}]
params['tags'] = []
for keyword in self.keywords:
keyword = Keyword.create_in_ckan(keyword)
if keyword is not None and keyword.name is not None and not "".__eq__(
keyword.name.strip()):
if keyword.vocabulary_id is None:
params['tags'].append({'name': keyword.name})
else:
params['tags'].append({
'name':
keyword.name,
'vocabulary_id':
keyword.vocabulary_id
})
if license is not None:
params['license_id'] = self.license_id
if self.method_steps is not None:
r = ''
for i in range(len(self.method_steps)):
r = r + str(i + 1) + '. ' + self.method_steps[i] + '\n'
if r != '':
params['method_steps'] = r
r = make_ckan_api_call("api/action/package_create", params)
if r is not None:
if not r['success']:
raise CKANAPIException({
"message": "Impossible to create dataset",
"dataset": self,
"error": r['error']
})
for resource in self.resources:
Resource.create_in_ckan(resource)
def get_existing_groups_ckan(cls):
r = make_ckan_api_call("api/action/group_list", {"all_fields": True})
return [cls(res["title"]) for res in r["result"]]
def purge_ckan(self):
# Purge the group whose name is self.name
r = make_ckan_api_call("api/action/group_purge", {"id": self.name})
return r["success"]
def purge_ckan(self):
# Purge the group whose name is self.name
r = make_ckan_api_call("api/action/group_purge", {'id': self.name})
return r['success']
def get_existing_groups_ckan(cls):
r = make_ckan_api_call("api/action/group_list", {'all_fields': True})
if r is not None:
return [cls(res['title']) for res in r['result']]
def get_existing_datasets_ckan():
# Return list of strings (dataset names)
r = make_ckan_api_call("api/action/package_list", {"all_fields": True})
return r["result"]
def get_existing_organizations_ckan(cls):
r = make_ckan_api_call("api/action/organization_list",
{'all_fields': True})
if r is not None:
return [cls(res['id'], res['title']) for res in r['result']]
