def check_url_filetoupload(self):
# type: () -> None
"""Check if url or file to upload provided for resource and add resource_type and url_type if not supplied
Returns:
None
"""
if self.file_to_upload is None:
if 'url' in self.data:
if 'resource_type' not in self.data:
self.data['resource_type'] = 'api'
if 'url_type' not in self.data:
self.data['url_type'] = 'api'
else:
raise HDXError(
'Either a url or a file to upload must be supplied!')
else:
if 'url' in self.data:
if self.data[
'url'] != hdx.data.filestore_helper.FilestoreHelper.temporary_url:
raise HDXError(
'Either a url or a file to upload must be supplied not both!'
)
if 'resource_type' not in self.data:
self.data['resource_type'] = 'file.upload'
if 'url_type' not in self.data:
self.data['url_type'] = 'upload'
if 'tracking_summary' in self.data:
del self.data['tracking_summary']
def reorder_resource_views(self, resource_views):
# type: (List[Union[ResourceView,Dict,str]]) -> None
"""Order resource views in resource.
Args:
resource_views (List[Union[ResourceView,Dict,str]]): A list of either resource view ids or resource views metadata from ResourceView objects or dictionaries
Returns:
None
"""
if not isinstance(resource_views, list):
raise HDXError('ResourceViews should be a list!')
ids = list()
for resource_view in resource_views:
if isinstance(resource_view, str):
resource_view_id = resource_view
else:
resource_view_id = resource_view['id']
if is_valid_uuid(resource_view_id) is False:
raise HDXError('%s is not a valid resource view id!' %
resource_view)
ids.append(resource_view_id)
_, result = self._read_from_hdx('resource view',
self.data['id'],
'id',
ResourceView.actions()['reorder'],
order=ids)
def add_update_resource(self, resource, ignore_datasetid=False):
# type: (Union[Resource,dict,str], Optional[bool]) -> None
"""Add new or update existing resource in dataset with new metadata
Args:
resource (Union[Resource,dict,str]): Either resource id or resource metadata from a Resource object or a dictionary
ignore_datasetid (Optional[bool]): Whether to ignore dataset id in the resource
Returns:
None
"""
if isinstance(resource, str):
resource = Resource.read_from_hdx(resource,
configuration=self.configuration)
elif isinstance(resource, dict):
resource = Resource(resource, configuration=self.configuration)
if isinstance(resource, Resource):
if 'package_id' in resource:
if not ignore_datasetid:
raise HDXError(
'Resource %s being added already has a dataset id!' %
(resource['name']))
resource_updated = self._addupdate_hdxobject(
self.resources, 'name', resource)
resource_updated.set_file_to_upload(resource.get_file_to_upload())
return
raise HDXError('Type %s cannot be added as a resource!' %
type(resource).__name__)
def set_dataset_date(self,
dataset_date: str,
date_format: Optional[str] = None) -> None:
"""Set dataset date from string using specified format. If no format is supplied, the function will guess.
For unambiguous formats, this should be fine.
Args:
dataset_date (str): Dataset date string
date_format (Optional[str]): Date format. If None is given, will attempt to guess. Defaults to None.
Returns:
None
"""
if date_format is None:
try:
parsed_date = parser.parse(dataset_date)
except (ValueError, OverflowError) as e:
raise HDXError('Invalid dataset date!') from e
else:
try:
parsed_date = datetime.datetime.strptime(
dataset_date, date_format)
except ValueError as e:
raise HDXError('Invalid dataset date!') from e
self.set_dataset_date_from_datetime(parsed_date)
def delete_resource_view(self, resource_view):
# type: (Union[ResourceView,Dict,str]) -> None
"""Delete a resource view from the resource and HDX
Args:
resource_view (Union[ResourceView,Dict,str]): Either a resource view id or resource view metadata either from a ResourceView object or a dictionary
Returns:
None
"""
if isinstance(resource_view, str):
if is_valid_uuid(resource_view) is False:
raise HDXError('%s is not a valid resource view id!' %
resource_view)
resource_view = ResourceView({'id': resource_view},
configuration=self.configuration)
else:
resource_view = self._get_resource_view(resource_view)
if 'id' not in resource_view:
found = False
title = resource_view.get('title')
for rv in self.get_resource_views():
if resource_view['title'] == rv['title']:
resource_view = rv
found = True
break
if not found:
raise HDXError(
'No resource views have title %s in this resource!' %
title)
resource_view.delete_from_hdx()
def add_country_location(self, country):
# type: (str) -> None
"""Add a country. If an iso 3 code is not provided, value is parsed and if it is a valid country name,
converted to an iso 3 code. If the country is already added, it is ignored.
Args:
country (str): Country to add
Returns:
None
"""
iso3, match = Location.get_iso3_country_code(country)
if iso3 is None:
raise HDXError('Country: %s - cannot find iso3 code!' % country)
hdx_code, match = Location.get_HDX_code_from_location(
iso3, self.configuration)
if hdx_code is None:
raise HDXError(
'Country: %s with iso3: %s could not be found in HDX list!' %
(country, iso3))
groups = self.data.get('groups', None)
if groups:
if hdx_code in [x['name'] for x in groups]:
return
else:
groups = list()
groups.append({'name': hdx_code})
self.data['groups'] = groups
def update_in_hdx(self,
update_resources: Optional[bool] = True,
update_gallery: Optional[bool] = True) -> None:
"""Check if dataset exists in HDX and if so, update it
Args:
update_resources (Optional[bool]): Whether to update resources. Defaults to True.
update_gallery (Optional[bool]): Whether to update resources. Defaults to True.
Returns:
None
"""
loaded = False
if 'id' in self.data:
self._check_existing_object('dataset', 'id')
if self._dataset_load_from_hdx(self.data['id']):
loaded = True
else:
logger.warning('Failed to load dataset with id %s' %
self.data['id'])
if not loaded:
self._check_existing_object('dataset', 'name')
if not self._dataset_load_from_hdx(self.data['name']):
raise HDXError('No existing dataset to update!')
self._dataset_merge_hdx_update(update_resources, update_gallery)
def add_update_user(self, user, capacity=None):
# type: (Union[User,dict,str],Optional[str]) -> None
"""Add new or update existing users item in organization with new metadata. Capacity eg. member, admin
must be supplied either within the User object or dictionary or using the capacity argument (which takes
precedence)
Args:
user (Union[User,dict,str]): Either a user id or user metadata either from a User object or a dictionary
capacity (Optional[str]): Capacity of user eg. member, admin. Defaults to None.
Returns:
None
"""
if isinstance(user, str):
user = User.read_from_hdx(user, configuration=self.configuration)
if isinstance(user, User):
user = user.data
if isinstance(user, dict):
users = self.data.get('users')
if users is None:
users = list()
self.data['users'] = users
if capacity is not None:
user['capacity'] = capacity
self._addupdate_hdxobject(users, 'name', user)
return
raise HDXError('Type %s cannot be added as a user!' %
type(user).__name__)
def check_required_fields(self,
ignore_fields=list(),
allow_no_resources=False):
# type: (List[str], Optional[bool]) -> None
"""Check that metadata for dataset and its resources is complete. The parameter ignore_fields
should be set if required to any fields that should be ignored for the particular operation.
Args:
ignore_fields (List[str]): Fields to ignore. Default is [].
allow_no_resources (Optional[bool]): Whether to allow no resources. Defaults to False.
Returns:
None
"""
if self.get_requestable():
self._check_required_fields('dataset-requestable', ignore_fields)
else:
self._check_required_fields('dataset', ignore_fields)
if len(self.resources) == 0 and not allow_no_resources:
raise HDXError(
'There are no resources! Please add at least one resource!'
)
for resource in self.resources:
ignore_fields = ['package_id']
resource.check_required_fields(ignore_fields=ignore_fields)
def check_required_fields(self, ignore_fields=list()):
# type: (List[str]) -> None
"""Check that metadata for resource is complete and add resource_type and url_type if not supplied.
The parameter ignore_fields should be set if required to any fields that should be ignored for the particular
operation.
Args:
ignore_fields (List[str]): Fields to ignore. Default is [].
Returns:
None
"""
if self.file_to_upload is None:
if 'url' in self.data:
if 'resource_type' not in self.data:
self.data['resource_type'] = 'api'
if 'url_type' not in self.data:
self.data['url_type'] = 'api'
else:
raise HDXError('Either a url or a file to upload must be supplied!')
else:
if 'url' not in self.data:
self.data['url'] = 'ignore' # must be set even though overwritten
if 'resource_type' not in self.data:
self.data['resource_type'] = 'file.upload'
if 'url_type' not in self.data:
self.data['url_type'] = 'upload'
if 'tracking_summary' in self.data:
del self.data['tracking_summary']
self._check_required_fields('resource', ignore_fields)
def get_all_datasets(configuration=None, **kwargs):
# type: (Optional[Configuration], ...) -> List['Dataset']
"""Get all datasets in HDX
Args:
configuration (Optional[Configuration]): HDX configuration. Defaults to global configuration.
**kwargs: See below
limit (int): Number of rows to return. Defaults to all datasets (sys.maxsize).
offset (int): Offset in the complete result for where the set of returned datasets should begin
Returns:
List[Dataset]: List of all datasets in HDX
"""
dataset = Dataset(configuration=configuration)
dataset['id'] = 'all datasets' # only for error message if produced
total_rows = kwargs.get('limit', max_int)
start = kwargs.get('offset', 0)
all_datasets = None
attempts = 0
while attempts < max_attempts and all_datasets is None: # if the dataset names vary for multiple calls, then must redo query
all_datasets = list()
for page in range(total_rows // page_size + 1):
pagetimespagesize = page * page_size
kwargs['offset'] = start + pagetimespagesize
rows_left = total_rows - pagetimespagesize
rows = min(rows_left, page_size)
kwargs['limit'] = rows
result = dataset._write_to_hdx('all', kwargs, 'id')
datasets = list()
if result:
no_results = len(result)
for datasetdict in result:
dataset = Dataset(configuration=configuration)
dataset.old_data = dict()
dataset.data = datasetdict
dataset._dataset_create_resources()
datasets.append(dataset)
all_datasets += datasets
if no_results < rows:
break
else:
logger.debug(result)
names_list = [dataset['name'] for dataset in all_datasets]
names = set(names_list)
if len(names_list) != len(
names): # check for duplicates (shouldn't happen)
all_datasets = None
attempts += 1
# elif total_rows == max_int:
# all_names = set(Dataset.get_all_dataset_names()) # check dataset names match package_list
# if names != all_names:
# all_datasets = None
# attempts += 1
if attempts == max_attempts and all_datasets is None:
raise HDXError(
'Maximum attempts reached for getting all datasets!')
return all_datasets
def update_in_hdx(self, **kwargs):
# type: (Any) -> None
"""Check if resource view exists in HDX and if so, update resource view
Returns:
None
"""
if not self._update_resource_view(**kwargs):
raise HDXError('No existing resource view to update!')
def __setitem__(self, key: Any, value: Any) -> None:
"""Set dictionary items but do not allow setting of resources or gallery
Args:
key (Any): Key in dictionary
value (Any): Value to put in dictionary
Returns:
None
"""
if key == 'resources':
raise HDXError(
'Add resource using add_resource or resources using add_resources!'
)
if key == 'gallery':
raise HDXError(
'Add gallery item using add_galleryitem or gallery using add_gallery!'
)
super(Dataset, self).__setitem__(key, value)
def get_dataset(self):
# type: () -> hdx.data.dataset.Dataset
"""Return dataset containing this resource
Returns:
hdx.data.dataset.Dataset: Dataset containing this resource
"""
package_id = self.data.get('package_id')
if package_id is None:
raise HDXError('Resource has no package id!')
return hdx.data.dataset.Dataset.read_from_hdx(package_id)
def add_update_resource(self, resource: Any) -> None:
"""Add new or update existing resource in dataset with new metadata
Args:
resource (Any): Resource metadata either from a Resource object or a dictionary
Returns:
None
"""
if isinstance(resource, dict):
resource = Resource(self.configuration, resource)
if isinstance(resource, Resource):
if 'package_id' in resource:
raise HDXError(
"Resource %s being added already has a dataset id!" %
(resource['name']))
self._addupdate_hdxobject(self.resources, 'name',
self._underlying_object, resource)
return
raise HDXError("Type %s cannot be added as a resource!" %
type(resource).__name__)
def copy(self, resource_view):
# type: (Union[ResourceView,Dict,str]) -> None
"""Copies all fields except id, resource_id and package_id from another resource view.
Args:
resource_view (Union[ResourceView,Dict,str]): Either a resource view id or resource view metadata either from a ResourceView object or a dictionary
Returns:
None
"""
if isinstance(resource_view, str):
if is_valid_uuid(resource_view) is False:
raise HDXError('%s is not a valid resource view id!' %
resource_view)
resource_view = ResourceView.read_from_hdx(resource_view)
if not isinstance(resource_view, dict) and not isinstance(
resource_view, ResourceView):
raise HDXError('%s is not a valid resource view!' % resource_view)
for key in resource_view:
if key not in ('id', 'resource_id', 'package_id'):
self.data[key] = resource_view[key]
def add_update_galleryitem(self, galleryitem) -> None:
"""Add new or update existing gallery item in dataset with new metadata
Args:
galleryitem (Any): Gallery item metadata either from a GalleryItem object or a dictionary
Returns:
None
"""
if isinstance(galleryitem, dict):
galleryitem = GalleryItem(galleryitem)
if isinstance(galleryitem, GalleryItem):
if 'dataset_id' in galleryitem:
raise HDXError(
"Gallery item %s being added already has a dataset id!" %
(galleryitem['name']))
self._addupdate_hdxobject(self.gallery, 'title', galleryitem)
return
raise HDXError("Type %s cannot be added as a gallery item!" %
type(galleryitem).__name__)
def add_update_resources(self, resources: List[Any]) -> None:
"""Add new or update existing resources with new metadata to the dataset
Args:
resources (List[Any]): Resources metadata from a list of either Resource objects or dictionaries
Returns:
None
"""
if not isinstance(resources, list):
raise HDXError('Resources should be a list!')
for resource in resources:
self.add_update_resource(resource)
def add_update_gallery(self, gallery):
"""Add new or update existing gallery items with new metadata to the dataset
Args:
gallery (List[Any]): Gallery metadata from a list of either GalleryItem objects or dictionaries
Returns:
None
"""
if not isinstance(gallery, list):
raise HDXError('Gallery should be a list!')
for galleryitem in gallery:
self.add_update_galleryitem(galleryitem)
def add_update_resource(self, resource):
# type: (Any) -> None
"""Add new or update existing resource in dataset with new metadata
Args:
resource (Any): Resource metadata either from a Resource object or a dictionary
Returns:
None
"""
if isinstance(resource, dict):
resource = Resource(resource)
if isinstance(resource, Resource):
if 'package_id' in resource:
raise HDXError(
"Resource %s being added already has a dataset id!" %
(resource['name']))
resource_updated = self._addupdate_hdxobject(
self.resources, 'name', resource)
resource_updated.set_file_to_upload(resource.get_file_to_upload())
return
raise HDXError("Type %s cannot be added as a resource!" %
type(resource).__name__)
def add_update_resource_views(self, resource_views):
# type: (List[Union[ResourceView,Dict]]) -> None
"""Add new or update existing resource views in resource with new metadata.
Args:
resource_views (List[Union[ResourceView,Dict]]): A list of resource views metadata from ResourceView objects or dictionaries
Returns:
None
"""
if not isinstance(resource_views, list):
raise HDXError('ResourceViews should be a list!')
for resource_view in resource_views:
self.add_update_resource_view(resource_view)
def check_required_fields(self, ignore_dataset_id=False) -> None:
"""Check that metadata for dataset and its resources and gallery is complete. (ignore_dataset_id is not used.)
Returns:
None
"""
for field in Configuration.read()['dataset']['required_fields']:
if field not in self.data:
raise HDXError("Field %s is missing in dataset!" % field)
for resource in self.resources:
resource.check_required_fields(ignore_dataset_id=True)
for galleryitem in self.gallery:
galleryitem.check_required_fields(ignore_dataset_id=True)
def add_other_location(self,
location,
exact=True,
alterror=None,
locations=None):
# type: (str, Optional[bool], Optional[str], Optional[List[str]]) -> bool
"""Add a location which is not a country or continent. Value is parsed and compared to existing locations in
HDX. If the location is already added, it is ignored.
Args:
location (str): Location to add
exact (Optional[bool]): True for exact matching or False to allow fuzzy matching. Defaults to True.
alterror (Optional[str]): Alternative error message to builtin if location not found. Defaults to None.
locations (Optional[List[str]]): Valid locations list. Defaults to list downloaded from HDX.
Returns:
bool: True if location added or False if location already present
"""
hdx_code, match = Locations.get_HDX_code_from_location(
location,
exact=exact,
locations=locations,
configuration=self.configuration)
if hdx_code is None:
if alterror is None:
raise HDXError('Location: %s - cannot find in HDX!' % location)
else:
raise HDXError(alterror)
groups = self.data.get('groups', None)
if groups:
if hdx_code in [x['name'] for x in groups]:
return False
else:
groups = list()
groups.append({'name': hdx_code})
self.data['groups'] = groups
return True
def add_update_resources(self, resources, ignore_datasetid=False):
# type: (List[Union[Resource,dict,str]]) -> None
"""Add new or update existing resources with new metadata to the dataset
Args:
resources (List[Union[Resource,dict,str]]): A list of either resource ids or resources metadata from either Resource objects or dictionaries
ignore_datasetid (Optional[bool]): Whether to ignore dataset id in the resource
Returns:
None
"""
if not isinstance(resources, list):
raise HDXError('Resources should be a list!')
for resource in resources:
self.add_update_resource(resource, ignore_datasetid)
def add_update_users(self, users, capacity=None):
# type: (List[Union[User,dict,str]]) -> None
"""Add new or update existing users items with new metadata to the organization
Args:
users (List[Union[User,dict,str]]): A list of either user ids or users metadata from User objects or dictionaries
capacity (Optional[str]): Capacity of users eg. member, admin. Defaults to None.
Returns:
None
"""
if not isinstance(users, list):
raise HDXError('Users should be a list!')
for user in users:
self.add_update_user(user, capacity)
def set_maintainer(self, maintainer):
# type: (Any) -> None
"""Set the dataset's maintainer.
Args:
maintainer (Any): Set the dataset's maintainer either from a User object or a str.
Returns:
None
"""
if isinstance(maintainer, str):
self.data['maintainer'] = maintainer
elif isinstance(maintainer, User):
self.data['maintainer'] = maintainer['name']
else:
raise HDXError('Type %s cannot be added as a maintainer!' %
type(maintainer).__name__)
def add_update_users(self, users, capacity=None):
# type: (List[Union[hdx.data.user.User,Dict,str]],Optional[str]) -> None
"""Add new or update existing users in organization with new metadata. Capacity eg. member, admin
must be supplied either within the User object or dictionary or using the capacity argument (which takes
precedence).
Args:
users (List[Union[User,Dict,str]]): A list of either user ids or users metadata from User objects or dictionaries
capacity (Optional[str]): Capacity of users eg. member, admin. Defaults to None.
Returns:
None
"""
if not isinstance(users, list):
raise HDXError('Users should be a list!')
for user in users:
self.add_update_user(user, capacity)
def set_expected_update_frequency(self, update_frequency: str) -> None:
"""Set expected update frequency
Args:
update_frequency (str): Update frequency
Returns:
None
"""
try:
int(update_frequency)
except ValueError:
update_frequency = Dataset.transform_update_frequency(
update_frequency)
if not update_frequency:
raise HDXError('Invalid update frequency supplied!')
self.data['data_update_frequency'] = update_frequency
def _get_resource_view(self, resource_view):
# type: (Union[ResourceView,Dict]) -> ResourceView
"""Get resource view id
Args:
resource_view (Union[ResourceView,Dict]): ResourceView metadata from a ResourceView object or dictionary
Returns:
ResourceView: ResourceView object
"""
if isinstance(resource_view, dict):
resource_view = ResourceView(resource_view,
configuration=self.configuration)
if isinstance(resource_view, ResourceView):
return resource_view
raise HDXError('Type %s is not a valid resource view!' %
type(resource_view).__name__)
def download(self, folder=None):
# type: (Optional[str]) -> Tuple[str, str]
"""Download resource store to provided folder or temporary folder if no folder supplied
Args:
folder (Optional[str]): Folder to download resource to. Defaults to None.
Returns:
Tuple[str, str]: (URL downloaded, Path to downloaded file)
"""
# Download the resource
url = self.data.get('url', None)
if not url:
raise HDXError('No URL to download!')
logger.debug('Downloading %s' % url)
with Download() as download:
path = download.download_file(url, folder)
return url, path