def _read_from_hdx(self, object_type, value, fieldname='id',
action=None, **kwargs):
# type: (str, str, str, Optional[str], Any) -> Tuple[bool, Union[dict, str]]
"""Makes a read call to HDX passing in given parameter.
Args:
object_type (str): Description of HDX object type (for messages)
value (str): Value of HDX field
fieldname (str): HDX field name. Defaults to id.
action (Optional[str]): Replacement CKAN action url to use. Defaults to None.
**kwargs: Other fields to pass to CKAN.
Returns:
Tuple[bool, Union[dict, str]]: (True/False, HDX object metadata/Error)
"""
if not fieldname:
raise HDXError("Empty %s field name!" % object_type)
if action is None:
action = self.actions()['show']
data = {fieldname: value}
data.update(kwargs)
try:
result = Configuration.remoteckan().call_action(action, data,
requests_kwargs={
'auth': Configuration.read()._get_credentials()})
return True, result
except NotFound:
return False, "%s=%s: not found!" % (fieldname, value)
except Exception as e:
six.raise_from(HDXError('Failed when trying to read: %s=%s! (POST)' % (fieldname, value)), e)
def get_HDX_code_from_location(location, configuration=None):
# type: (str, Optional[Configuration]) -> Tuple[Optional[str], bool]
"""Get HDX code for location
Args:
location (str): Location for which to get HDX code
configuration (Optional[Configuration]): HDX configuration. Defaults to global configuration.
Returns:
Tuple[Optional[str], bool]: HDX code and if the match is strong or (None, False) for no match
"""
if configuration is None:
configuration = Configuration.read()
locationlower = location.lower()
for locdict in configuration.validlocations():
locationcode = locdict['name']
if locationlower == locationcode.lower():
return locationcode, True
for locdict in configuration.validlocations():
if locationlower == locdict['title'].lower():
return locdict['name'], True
for locdict in configuration.validlocations():
locationname = locdict['title'].lower()
if locationlower in locationname or locationname in locationlower:
return locdict['name'], False
return None, False
def _write_to_hdx(self, action, data, id_field_name, file_to_upload=None):
# type: (str, dict, str, Optional[str]) -> dict
"""Creates or updates an HDX object in HDX and return HDX object metadata dict
Args:
action (str): Action to perform eg. 'create', 'update'
data (dict): Data to write to HDX
id_field_name (str): Name of field containing HDX object identifier or None
file_to_upload (Optional[str]): File to upload to HDX
Returns:
dict: HDX object metadata
"""
file = None
try:
if file_to_upload:
file = open(file_to_upload, 'rb')
files = [('upload', file)]
else:
files = None
return Configuration.remoteckan().call_action(self.actions()[action], data, files=files,
requests_kwargs={
'auth': Configuration.read()._get_credentials()})
except Exception as e:
six.raise_from(HDXError('Failed when trying to %s %s! (POST)' % (action, self.data[id_field_name])), e)
finally:
if file_to_upload and file:
file.close()
def __init__(self, initial_data, configuration=None):
# type: (dict, Optional[Configuration]) -> None
super(HDXObject, self).__init__(initial_data)
self.old_data = None
if configuration is None:
self.configuration = Configuration.read()
else:
self.configuration = configuration
def _check_required_fields(self, object_type, ignore_fields):
# type: (str, List[str]) -> None
"""Helper method to check that metadata for HDX object is complete
Args:
ignore_fields (List[str]): Any fields to ignore in the check
Returns:
None
"""
for field in Configuration.read()['%s' % object_type]['required_fields']:
if field not in self.data and field not in ignore_fields:
raise HDXError("Field %s is missing in %s!" % (field, object_type))
def create_in_hdx(self):
# type: () -> None
"""Check if dataset exists in HDX and if so, update it, otherwise create it
Returns:
None
"""
self.check_required_fields()
loadedid = None
if 'id' in self.data:
if self._dataset_load_from_hdx(self.data['id']):
loadedid = self.data['id']
else:
logger.warning('Failed to load dataset with id %s' %
self.data['id'])
if not loadedid:
if self._dataset_load_from_hdx(self.data['name']):
loadedid = self.data['name']
if loadedid:
logger.warning('Dataset exists. Updating %s' % loadedid)
self._dataset_merge_hdx_update(True, True)
return
filestore_resources = list()
if self.resources:
for resource in self.resources:
resource.check_required_fields(ignore_dataset_id=True)
if resource.get_file_to_upload():
filestore_resources.append(resource)
self.data['resources'] = self._convert_hdxobjects(self.resources)
self._save_to_hdx('create', 'name')
for resource in filestore_resources:
for created_resource in self.data['resources']:
if resource['name'] == created_resource['name']:
merge_two_dictionaries(resource.data, created_resource)
resource.update_in_hdx()
merge_two_dictionaries(created_resource, resource.data)
break
self.init_resources()
self.separate_resources()
if self.include_gallery:
self.old_data['gallery'] = self._copy_hdxobjects(
self.gallery, GalleryItem)
galleryitem_dataset_id = Configuration.read(
)['galleryitem']['dataset_id']
for i, galleryitem in enumerate(self.gallery):
galleryitem[galleryitem_dataset_id] = self.data['id']
galleryitem.check_required_fields()
galleryitem.create_in_hdx()
def check_required_fields(self, ignore_dataset_id=False) -> None:
"""Check that metadata for dataset and its resources and gallery is complete. (ignore_dataset_id is not used.)
Returns:
None
"""
for field in Configuration.read()['dataset']['required_fields']:
if field not in self.data:
raise HDXError("Field %s is missing in dataset!" % field)
for resource in self.resources:
resource.check_required_fields(ignore_dataset_id=True)
for galleryitem in self.gallery:
galleryitem.check_required_fields(ignore_dataset_id=True)
def get_location_from_HDX_code(code, configuration=None):
# type: (str, Optional[Configuration]) -> Optional[str]
"""Get location from HDX location code
Args:
code (str): code for which to get location name
configuration (Optional[Configuration]): HDX configuration. Defaults to global configuration.
Returns:
Optional[str]: location name
"""
if configuration is None:
configuration = Configuration.read()
for locdict in configuration.validlocations():
if code.lower() == locdict['name'].lower():
return locdict['title']
def _merge_hdx_update(self, object_type, id_field_name, file_to_upload=None):
# type: (str, str, Optional[str]) -> None
"""Helper method to check if HDX object exists and update it
Args:
object_type (str): Description of HDX object type (for messages)
id_field_name (str): Name of field containing HDX object identifier
file_to_upload (Optional[str]): File to upload to HDX
Returns:
None
"""
merge_two_dictionaries(self.data, self.old_data)
ignore_dataset_id = Configuration.read()['%s' % object_type].get('ignore_dataset_id_on_update', False)
self.check_required_fields(ignore_dataset_id=ignore_dataset_id)
self._save_to_hdx('update', id_field_name, file_to_upload)
def check_required_fields(self, ignore_dataset_id=False):
# type: (Optional[bool]) -> None
"""Check that metadata for gallery item is complete. The parameter ignore_dataset_id should
be set to True if you intend to add the object to a Dataset object (where it will be created during dataset
creation).
Args:
ignore_dataset_id (bool): Whether to ignore the dataset id. Default is False.
Returns:
None
"""
if ignore_dataset_id:
ignore_fields = [Configuration.read()['galleryitem']['dataset_id']]
else:
ignore_fields = list()
self._check_required_fields('galleryitem', ignore_fields)
def check_required_fields(self, ignore_dataset_id=False):
# type: (Optional[bool]) -> None
"""Check that metadata for resource is complete and add resource_type and url_type if not supplied.
The parameter ignore_dataset_id should
be set to True if you intend to add the object to a Dataset object (where it will be created during dataset
creation).
Args:
ignore_dataset_id (bool): Whether to ignore the dataset id. Default is False.
Returns:
None
"""
if self.file_to_upload is None:
if 'url' in self.data:
if 'resource_type' not in self.data:
self.data['resource_type'] = 'api'
if 'url_type' not in self.data:
self.data['url_type'] = 'api'
else:
raise HDXError(
'Either a url or a file to upload must be supplied!')
else:
if 'url' not in self.data:
self.data[
'url'] = 'ignore' # must be set even though overwritten
if 'resource_type' not in self.data:
self.data['resource_type'] = 'file.upload'
if 'url_type' not in self.data:
self.data['url_type'] = 'upload'
if 'tracking_summary' in self.data:
del self.data['tracking_summary']
if ignore_dataset_id:
ignore_fields = [Configuration.read()['resource']['dataset_id']]
else:
ignore_fields = list()
self._check_required_fields('resource', ignore_fields)
def my_excfn():
testresult.actual_result = Configuration.read().get_hdx_site_url()
raise ValueError('Some failure!')
def my_testfn():
testresult.actual_result = Configuration.read().get_hdx_site_url()
def _dataset_merge_hdx_update(self, update_resources, update_gallery):
# type: (bool, bool) -> None
"""Helper method to check if dataset or its resources or gallery items exist and update them
Args:
update_resources (bool): Whether to update resources
update_gallery (bool): Whether to update gallery
Returns:
None
"""
merge_two_dictionaries(self.data, self.old_data)
if 'resources' in self.data:
del self.data['resources']
if 'gallery' in self.data:
del self.data['gallery']
old_resources = self.old_data.get('resources', None)
filestore_resources = list()
if update_resources and old_resources:
resource_names = set()
for resource in self.resources:
resource_name = resource['name']
resource_names.add(resource_name)
for old_resource in old_resources:
if resource_name == old_resource['name']:
logger.warning('Resource exists. Updating %s' %
resource_name)
merge_two_dictionaries(resource, old_resource)
if old_resource.get_file_to_upload():
resource.set_file_to_upload(
old_resource.get_file_to_upload())
filestore_resources.append(resource)
resource.check_required_fields(ignore_dataset_id=True)
break
for old_resource in old_resources:
if not old_resource['name'] in resource_names:
old_resource.check_required_fields(ignore_dataset_id=True)
self.resources.append(old_resource)
if old_resource.get_file_to_upload():
filestore_resources.append(old_resource)
old_gallery = self.old_data.get('gallery', None)
if self.resources:
self.data['resources'] = self._convert_hdxobjects(self.resources)
self._save_to_hdx('update', 'id')
for resource in filestore_resources:
for created_resource in self.data['resources']:
if resource['name'] == created_resource['name']:
merge_two_dictionaries(resource.data, created_resource)
resource.update_in_hdx()
merge_two_dictionaries(created_resource, resource.data)
break
if self.include_gallery and update_gallery and old_gallery:
self.old_data['gallery'] = self._copy_hdxobjects(
self.gallery, GalleryItem)
galleryitem_titles = set()
galleryitem_dataset_id = Configuration.read(
)['galleryitem']['dataset_id']
for i, galleryitem in enumerate(self.gallery):
galleryitem_title = galleryitem['title']
galleryitem_titles.add(galleryitem_title)
for old_galleryitem in old_gallery:
if galleryitem_title == old_galleryitem['title']:
logger.warning('Gallery item exists. Updating %s' %
galleryitem_title)
merge_two_dictionaries(galleryitem, old_galleryitem)
galleryitem.check_required_fields(
ignore_dataset_id=True)
galleryitem.update_in_hdx()
for old_galleryitem in old_gallery:
if not old_galleryitem['title'] in galleryitem_titles:
old_galleryitem[galleryitem_dataset_id] = self.data['id']
old_galleryitem.check_required_fields()
old_galleryitem.create_in_hdx()
self.gallery.append(old_galleryitem)
