def _separate_hdxobjects(self, hdxobjects: List[HDXObjectUpperBound], hdxobjects_name: str, id_field: str,
hdxobjectclass: type) -> None:
"""Helper function to take a list of HDX objects contained in the internal dictionary and add them to a
supplied list of HDX objects or update existing metadata if any objects already exist in the list. The list in
the internal dictionary is then deleted.
Args:
hdxobjects (list[T <= HDXObject]): List of HDX objects to which to add new objects or update existing ones
hdxobjects_name (str): Name of key in internal dictionary from which to obtain list of HDX objects
id_field (str): Field on which to match to determine if object already exists in list
hdxobjectclass (type): Type of the HDX Object to be added/updated
Returns:
None
"""
new_hdxobjects = self.data.get(hdxobjects_name, None)
if new_hdxobjects:
hdxobject_names = set()
for hdxobject in hdxobjects:
hdxobject_name = hdxobject[id_field]
hdxobject_names.add(hdxobject_name)
for new_hdxobject in new_hdxobjects:
if hdxobject_name == new_hdxobject[id_field]:
merge_two_dictionaries(hdxobject, new_hdxobject)
break
for new_hdxobject in new_hdxobjects:
if not new_hdxobject[id_field] in hdxobject_names:
hdxobjects.append(hdxobjectclass(self.configuration, new_hdxobject))
del self.data[hdxobjects_name]
def mockreturn(url, data, headers, files, allow_redirects, auth):
datadict = json.loads(data.decode('utf-8'))
if 'show' in url:
return mockshow(url, datadict)
if 'update' not in url:
return MockResponse(
404,
'{"success": false, "error": {"message": "TEST ERROR: Not update", "__type": "TEST ERROR: Not Update Error"}, "help": "http://test-data.humdata.org/api/3/action/help_show?name=related_update"}'
)
resultdictcopy = copy.deepcopy(resultdict)
merge_two_dictionaries(resultdictcopy, datadict)
result = json.dumps(resultdictcopy)
if datadict['title'] == 'MyGalleryItem1':
return MockResponse(
200,
'{"success": true, "result": %s, "help": "http://test-data.humdata.org/api/3/action/help_show?name=related_update"}'
% result)
if datadict['title'] == 'MyGalleryItem2':
return MockResponse(
404,
'{"success": false, "error": {"message": "Not found", "__type": "Not Found Error"}, "help": "http://test-data.humdata.org/api/3/action/help_show?name=related_update"}'
)
if datadict['title'] == 'MyGalleryItem3':
return MockResponse(
200,
'{"success": false, "error": {"message": "Not found", "__type": "Not Found Error"}, "help": "http://test-data.humdata.org/api/3/action/help_show?name=related_update"}'
)
return MockResponse(
404,
'{"success": false, "error": {"message": "Not found", "__type": "Not Found Error"}, "help": "http://test-data.humdata.org/api/3/action/help_show?name=related_update"}'
)
def _separate_hdxobjects(self, hdxobjects: List[HDXObjectUpperBound], hdxobjects_name: str, id_field: str,
hdxobjectclass: type) -> None:
"""Helper function to take a list of HDX objects contained in the internal dictionary and add them to a
supplied list of HDX objects or update existing metadata if any objects already exist in the list. The list in
the internal dictionary is then deleted.
Args:
hdxobjects (list[T <= HDXObject]): List of HDX objects to which to add new objects or update existing ones
hdxobjects_name (str): Name of key in internal dictionary from which to obtain list of HDX objects
id_field (str): Field on which to match to determine if object already exists in list
hdxobjectclass (type): Type of the HDX Object to be added/updated
Returns:
None
"""
new_hdxobjects = self.data.get(hdxobjects_name, list())
""":type : List[HDXObjectUpperBound]"""
if new_hdxobjects:
hdxobject_names = set()
for hdxobject in hdxobjects:
hdxobject_name = hdxobject[id_field]
hdxobject_names.add(hdxobject_name)
for new_hdxobject in new_hdxobjects:
if hdxobject_name == new_hdxobject[id_field]:
merge_two_dictionaries(hdxobject, new_hdxobject)
break
for new_hdxobject in new_hdxobjects:
if not new_hdxobject[id_field] in hdxobject_names:
hdxobjects.append(hdxobjectclass(self.configuration, new_hdxobject))
del self.data[hdxobjects_name]
def mockreturn(url, data, headers, files, allow_redirects, auth):
datadict = json.loads(data.decode('utf-8'))
if 'show' in url or 'related_list' in url:
return mockshow(url, datadict)
if 'update' not in url:
return MockResponse(404,
'{"success": false, "error": {"message": "TEST ERROR: Not update", "__type": "TEST ERROR: Not Update Error"}, "help": "http://test-data.humdata.org/api/3/action/help_show?name=dataset_update"}')
if 'related' in url:
result = json.dumps(TestDataset.gallerydict)
return MockResponse(200,
'{"success": true, "result": %s, "help": "http://test-data.humdata.org/api/3/action/help_show?name=related_create"}' % result)
else:
resultdictcopy = copy.deepcopy(resultdict)
merge_two_dictionaries(resultdictcopy, datadict)
for i, resource in enumerate(resultdictcopy['resources']):
for j, resource2 in enumerate(resultdictcopy['resources']):
if i != j:
if resource == resource2:
del resultdictcopy['resources'][j]
break
result = json.dumps(resultdictcopy)
if datadict['name'] == 'MyDataset1':
return MockResponse(200,
'{"success": true, "result": %s, "help": "http://test-data.humdata.org/api/3/action/help_show?name=dataset_update"}' % result)
if datadict['name'] == 'MyDataset2':
return MockResponse(404,
'{"success": false, "error": {"message": "Not found", "__type": "Not Found Error"}, "help": "http://test-data.humdata.org/api/3/action/help_show?name=dataset_update"}')
if datadict['name'] == 'MyDataset3':
return MockResponse(200,
'{"success": false, "error": {"message": "Not found", "__type": "Not Found Error"}, "help": "http://test-data.humdata.org/api/3/action/help_show?name=dataset_update"}')
return MockResponse(404,
'{"success": false, "error": {"message": "Not found", "__type": "Not Found Error"}, "help": "http://test-data.humdata.org/api/3/action/help_show?name=dataset_update"}')
def _merge_hdx_update(self, object_type: str, id_field_name: str) -> None:
"""Helper method to check if HDX object exists and update it
Args:
object_type (str): Description of HDX object type (for messages)
id_field_name (str): Name of field containing HDX object identifier
Returns:
None
"""
merge_two_dictionaries(self.data, self.old_data)
self.check_required_fields(self.configuration['%s' % object_type].get('ignore_on_update', []))
self._save_to_hdx('update', id_field_name)
def _merge_hdx_update(self, object_type: str, id_field_name: str) -> None:
"""Helper method to check if HDX object exists and update it
Args:
object_type (str): Description of HDX object type (for messages)
id_field_name (str): Name of field containing HDX object identifier
Returns:
None
"""
merge_two_dictionaries(self.data, self.old_data)
self.check_required_fields(self.configuration['%s' % object_type].get('ignore_on_update', []))
self._save_to_hdx('update', id_field_name)
def mockreturn(url, data, headers, files, allow_redirects, auth):
datadict = json.loads(data.decode('utf-8'))
if 'show' in url or 'related_list' in url:
return mockshow(url, datadict)
if 'update' not in url:
return MockResponse(
404,
'{"success": false, "error": {"message": "TEST ERROR: Not update", "__type": "TEST ERROR: Not Update Error"}, "help": "http://test-data.humdata.org/api/3/action/help_show?name=dataset_update"}'
)
if 'related' in url:
result = json.dumps(TestDataset.gallerydict)
return MockResponse(
200,
'{"success": true, "result": %s, "help": "http://test-data.humdata.org/api/3/action/help_show?name=related_create"}'
% result)
else:
resultdictcopy = copy.deepcopy(resultdict)
merge_two_dictionaries(resultdictcopy, datadict)
for i, resource in enumerate(resultdictcopy['resources']):
for j, resource2 in enumerate(resultdictcopy['resources']):
if i != j:
if resource == resource2:
del resultdictcopy['resources'][j]
break
result = json.dumps(resultdictcopy)
if datadict['name'] == 'MyDataset1':
return MockResponse(
200,
'{"success": true, "result": %s, "help": "http://test-data.humdata.org/api/3/action/help_show?name=dataset_update"}'
% result)
if datadict['name'] == 'MyDataset2':
return MockResponse(
404,
'{"success": false, "error": {"message": "Not found", "__type": "Not Found Error"}, "help": "http://test-data.humdata.org/api/3/action/help_show?name=dataset_update"}'
)
if datadict['name'] == 'MyDataset3':
return MockResponse(
200,
'{"success": false, "error": {"message": "Not found", "__type": "Not Found Error"}, "help": "http://test-data.humdata.org/api/3/action/help_show?name=dataset_update"}'
)
return MockResponse(
404,
'{"success": false, "error": {"message": "Not found", "__type": "Not Found Error"}, "help": "http://test-data.humdata.org/api/3/action/help_show?name=dataset_update"}'
)
def load_json_into_existing_dict(data: dict, path: str) -> dict:
"""Merge JSON file into existing dictionary
Args:
data (dict): Dictionary to merge into
path (str): JSON file to load and merge
Returns:
dict: JSON file merged into dictionary
"""
jsondict = load_json(path)
return merge_two_dictionaries(data, jsondict)
def load_yaml_into_existing_dict(data: dict, path: str) -> dict:
"""Merge YAML file into existing dictionary
Args:
data (dict): Dictionary to merge into
path (str): YAML file to load and merge
Returns:
dict: YAML file merged into dictionary
"""
yamldict = load_yaml(path)
return merge_two_dictionaries(data, yamldict)
def _addupdate_hdxobject(self, hdxobjects: List[HDXObjectUpperBound], id_field: str, hdxobjectclass: type,
new_hdxobject: HDXObjectUpperBound) -> None:
"""Helper function to add a new HDX object to a supplied list of HDX objects or update existing metadata if the object
already exists in the list
Args:
hdxobjects (list[T <= HDXObject]): List of HDX objects to which to add new objects or update existing ones
id_field (str): Field on which to match to determine if object already exists in list
hdxobjectclass (type): Type of the HDX Object to be added/updated
new_hdxobject (T <= HDXObject): The HDX object to be added/updated
Returns:
None
"""
found = False
for hdxobject in hdxobjects:
if hdxobject[id_field] == new_hdxobject[id_field]:
merge_two_dictionaries(hdxobject, new_hdxobject)
found = True
break
if not found:
hdxobjects.append(hdxobjectclass(self.configuration, new_hdxobject))
def _addupdate_hdxobject(self, hdxobjects: List[HDXObjectUpperBound], id_field: str, hdxobjectclass: type,
new_hdxobject: HDXObjectUpperBound) -> None:
"""Helper function to add a new HDX object to a supplied list of HDX objects or update existing metadata if the object
already exists in the list
Args:
hdxobjects (list[T <= HDXObject]): List of HDX objects to which to add new objects or update existing ones
id_field (str): Field on which to match to determine if object already exists in list
hdxobjectclass (type): Type of the HDX Object to be added/updated
new_hdxobject (T <= HDXObject): The HDX object to be added/updated
Returns:
None
"""
found = False
for hdxobject in hdxobjects:
if hdxobject[id_field] == new_hdxobject[id_field]:
merge_two_dictionaries(hdxobject, new_hdxobject)
found = True
break
if not found:
hdxobjects.append(hdxobjectclass(self.configuration, new_hdxobject))
def mockreturn(url, data, headers, files, allow_redirects, auth):
datadict = json.loads(data.decode('utf-8'))
if 'show' in url:
return mockshow(url, datadict)
if 'update' not in url:
return MockResponse(404,
'{"success": false, "error": {"message": "TEST ERROR: Not update", "__type": "TEST ERROR: Not Update Error"}, "help": "http://test-data.humdata.org/api/3/action/help_show?name=resource_update"}')
resultdictcopy = copy.deepcopy(resultdict)
merge_two_dictionaries(resultdictcopy, datadict)
result = json.dumps(resultdictcopy)
if datadict['name'] == 'MyResource1':
return MockResponse(200,
'{"success": true, "result": %s, "help": "http://test-data.humdata.org/api/3/action/help_show?name=resource_update"}' % result)
if datadict['name'] == 'MyResource2':
return MockResponse(404,
'{"success": false, "error": {"message": "Not found", "__type": "Not Found Error"}, "help": "http://test-data.humdata.org/api/3/action/help_show?name=resource_update"}')
if datadict['name'] == 'MyResource3':
return MockResponse(200,
'{"success": false, "error": {"message": "Not found", "__type": "Not Found Error"}, "help": "http://test-data.humdata.org/api/3/action/help_show?name=resource_update"}')
return MockResponse(404,
'{"success": false, "error": {"message": "Not found", "__type": "Not Found Error"}, "help": "http://test-data.humdata.org/api/3/action/help_show?name=resource_update"}')
def _dataset_merge_hdx_update(self, update_resources: bool,
update_gallery: bool) -> None:
"""Helper method to check if dataset or its resources or gallery items exist and update them
Args:
update_resources (bool): Whether to update resources
update_gallery (bool): Whether to update gallery
Returns:
None
"""
merge_two_dictionaries(self.data, self.old_data)
if 'resources' in self.data:
del self.data['resources']
if 'gallery' in self.data:
del self.data['gallery']
old_resources = self.old_data.get('resources', None)
if update_resources and old_resources:
resource_dataset_id = [
self.configuration['resource']['dataset_id']
]
resource_names = set()
for resource in self.resources:
resource_name = resource['name']
resource_names.add(resource_name)
for old_resource in old_resources:
if resource_name == old_resource['name']:
logger.warning('Resource exists. Updating %s' %
resource_name)
merge_two_dictionaries(resource, old_resource)
resource.check_required_fields(resource_dataset_id)
break
for old_resource in old_resources:
if not old_resource['name'] in resource_names:
old_resource.check_required_fields(resource_dataset_id)
self.resources.append(old_resource)
old_gallery = self.old_data.get('gallery', None)
if self.resources:
self.data['resources'] = self._convert_hdxobjects(self.resources)
self._save_to_hdx('update', 'id')
self.init_resources()
self.separate_resources()
if self.include_gallery and update_gallery and old_gallery:
self.old_data['gallery'] = self._copy_hdxobjects(
self.gallery, GalleryItem)
galleryitem_titles = set()
galleryitem_dataset_id = self.configuration['galleryitem'][
'dataset_id']
for i, galleryitem in enumerate(self.gallery):
galleryitem_title = galleryitem['title']
galleryitem_titles.add(galleryitem_title)
for old_galleryitem in old_gallery:
if galleryitem_title == old_galleryitem['title']:
logger.warning('Gallery item exists. Updating %s' %
galleryitem_title)
merge_two_dictionaries(galleryitem, old_galleryitem)
galleryitem.check_required_fields(
[galleryitem_dataset_id])
galleryitem.update_in_hdx()
for old_galleryitem in old_gallery:
if not old_galleryitem['title'] in galleryitem_titles:
old_galleryitem[galleryitem_dataset_id] = self.data['id']
old_galleryitem.check_required_fields()
old_galleryitem.create_in_hdx()
self.gallery.append(old_galleryitem)
def __init__(self, **kwargs):
super(Configuration, self).__init__()
hdx_config_found = False
hdx_config_dict = kwargs.get('hdx_config_dict', None)
if hdx_config_dict:
hdx_config_found = True
logger.info('Loading HDX configuration from dictionary')
hdx_config_json = kwargs.get('hdx_config_json', '')
if hdx_config_json:
if hdx_config_found:
raise ConfigurationError('More than one HDX configuration file given!')
hdx_config_found = True
logger.info('Loading HDX configuration from: %s' % hdx_config_json)
hdx_config_dict = load_json(hdx_config_json)
hdx_config_yaml = kwargs.get('hdx_config_yaml', '')
if hdx_config_found:
if hdx_config_yaml:
raise ConfigurationError('More than one HDX configuration file given!')
else:
if not hdx_config_yaml:
logger.info('No HDX configuration parameter. Using default.')
hdx_config_yaml = script_dir_plus_file('hdx_configuration.yml', Configuration)
logger.info('Loading HDX configuration from: %s' % hdx_config_yaml)
hdx_config_dict = load_yaml(hdx_config_yaml)
project_config_found = False
project_config_dict = kwargs.get('project_config_dict', None)
if project_config_dict is not None:
project_config_found = True
logger.info('Loading project configuration from dictionary')
project_config_json = kwargs.get('project_config_json', '')
if project_config_json:
if project_config_found:
raise ConfigurationError('More than one project configuration file given!')
project_config_found = True
logger.info('Loading project configuration from: %s' % project_config_json)
project_config_dict = load_json(project_config_json)
project_config_yaml = kwargs.get('project_config_yaml', '')
if project_config_found:
if project_config_yaml:
raise ConfigurationError('More than one project configuration file given!')
else:
if not project_config_yaml:
logger.info('No project configuration parameter. Using default.')
project_config_yaml = join('config', 'project_configuration.yml')
logger.info('Loading project configuration from: %s' % project_config_yaml)
project_config_dict = load_yaml(project_config_yaml)
self.data = merge_two_dictionaries(hdx_config_dict, project_config_dict)
hdx_key_file = kwargs.get('hdx_key_file', join(expanduser("~"), '.hdxkey'))
self.data['api_key'] = self.load_api_key(hdx_key_file)
self.hdx_site = 'hdx_%s_site' % kwargs.get('hdx_site', 'test')
if self.hdx_site not in self.data:
raise ConfigurationError('%s not defined in configuration!' % self.hdx_site)