def _separate_hdxobjects(self, hdxobjects, hdxobjects_name, id_field, hdxobjectclass):
# type: (List[HDXObjectUpperBound], str, str, type) -> None
"""Helper function to take a list of HDX objects contained in the internal dictionary and add them to a
supplied list of HDX objects or update existing metadata if any objects already exist in the list. The list in
the internal dictionary is then deleted.
Args:
hdxobjects (List[T <= HDXObject]): list of HDX objects to which to add new objects or update existing ones
hdxobjects_name (str): Name of key in internal dictionary from which to obtain list of HDX objects
id_field (str): Field on which to match to determine if object already exists in list
hdxobjectclass (type): Type of the HDX Object to be added/updated
Returns:
None
"""
new_hdxobjects = self.data.get(hdxobjects_name, list())
""":type : List[HDXObjectUpperBound]"""
if new_hdxobjects:
hdxobject_names = set()
for hdxobject in hdxobjects:
hdxobject_name = hdxobject[id_field]
hdxobject_names.add(hdxobject_name)
for new_hdxobject in new_hdxobjects:
if hdxobject_name == new_hdxobject[id_field]:
merge_two_dictionaries(hdxobject, new_hdxobject)
break
for new_hdxobject in new_hdxobjects:
if not new_hdxobject[id_field] in hdxobject_names:
hdxobjects.append(hdxobjectclass(new_hdxobject, configuration=self.configuration))
del self.data[hdxobjects_name]
def post(url, data, headers, files, allow_redirects, auth):
datadict = json.loads(data.decode('utf-8'))
if url.endswith('show') or 'list' in url:
return mockshow(url, datadict)
if 'update' not in url:
return MockResponse(
404,
'{"success": false, "error": {"message": "TEST ERROR: Not update", "__type": "TEST ERROR: Not Update Error"}, "help": "http://test-data.humdata.org/api/3/action/help_show?name=ckanext_showcase_update"}'
)
resultdictcopy = copy.deepcopy(showcase_resultdict)
merge_two_dictionaries(resultdictcopy, datadict)
result = json.dumps(resultdictcopy)
if datadict['title'] == 'MyShowcase1':
return MockResponse(
200,
'{"success": true, "result": %s, "help": "http://test-data.humdata.org/api/3/action/help_show?name=ckanext_showcase_update"}'
% result)
if datadict['title'] == 'MyShowcase2':
return MockResponse(
404,
'{"success": false, "error": {"message": "Not found", "__type": "Not Found Error"}, "help": "http://test-data.humdata.org/api/3/action/help_show?name=ckanext_showcase_update"}'
)
if datadict['title'] == 'MyShowcase3':
return MockResponse(
200,
'{"success": false, "error": {"message": "Not found", "__type": "Not Found Error"}, "help": "http://test-data.humdata.org/api/3/action/help_show?name=ckanext_showcase_update"}'
)
return MockResponse(
404,
'{"success": false, "error": {"message": "Not found", "__type": "Not Found Error"}, "help": "http://test-data.humdata.org/api/3/action/help_show?name=ckanext_showcase_update"}'
)
def post(url, data, headers, files, allow_redirects, auth=None):
if isinstance(data, dict):
datadict = {k.decode('utf8'): v.decode('utf8') for k, v in data.items()}
else:
datadict = json.loads(data.decode('utf-8'))
if 'show' in url:
return mockshow(url, datadict)
if 'resource_id' in datadict:
if datadict['resource_id'] == '74b74ae1-df0c-4716-829f-4f939a046811':
return MockResponse(200,
'{"success": true, "result": {"fields": [{"type": "text", "id": "code"}, {"type": "text", "id": "title"}, {"type": "float", "id": "value"}, {"type": "timestamp", "id": "latest_date"}, {"type": "text", "id": "source"}, {"type": "text", "id": "source_link"}, {"type": "text", "id": "notes"}, {"type": "text", "id": "explore"}, {"type": "text", "id": "units"}], "method": "insert", "primary_key": "code", "resource_id": "bfa6b55f-10b6-4ba2-8470-33bb9a5194a5"}, "help": "http://test-data.humdata.org/api/3/action/help_show?name=datastore_create"}')
if 'update' not in url:
return MockResponse(404,
'{"success": false, "error": {"message": "TEST ERROR: Not update", "__type": "TEST ERROR: Not Update Error"}, "help": "http://test-data.humdata.org/api/3/action/help_show?name=resource_update"}')
if datadict['name'] == 'MyResource1':
resultdictcopy = copy.deepcopy(resultdict)
merge_two_dictionaries(resultdictcopy, datadict)
if files is not None:
resultdictcopy['url_type'] = 'upload'
resultdictcopy['resource_type'] = 'file.upload'
filename = os.path.basename(files[0][1].name)
resultdictcopy[
'url'] = 'http://test-data.humdata.org/dataset/6f36a41c-f126-4b18-aaaf-6c2ddfbc5d4d/resource/de6549d8-268b-4dfe-adaf-a4ae5c8510d5/download/%s' % filename
result = json.dumps(resultdictcopy)
return MockResponse(200,
'{"success": true, "result": %s, "help": "http://test-data.humdata.org/api/3/action/help_show?name=resource_update"}' % result)
if datadict['name'] == 'MyResource2':
return MockResponse(404,
'{"success": false, "error": {"message": "Not found", "__type": "Not Found Error"}, "help": "http://test-data.humdata.org/api/3/action/help_show?name=resource_update"}')
if datadict['name'] == 'MyResource3':
return MockResponse(200,
'{"success": false, "error": {"message": "Not found", "__type": "Not Found Error"}, "help": "http://test-data.humdata.org/api/3/action/help_show?name=resource_update"}')
return MockResponse(404,
'{"success": false, "error": {"message": "Not found", "__type": "Not Found Error"}, "help": "http://test-data.humdata.org/api/3/action/help_show?name=resource_update"}')
def post(url, data, headers, files, allow_redirects, auth=None):
datadict = json.loads(data.decode('utf-8'))
if 'show' in url:
return organization_mockshow(url, datadict)
if 'update' not in url:
return MockResponse(
404,
'{"success": false, "error": {"message": "TEST ERROR: Not update", "__type": "TEST ERROR: Not Update Error"}, "help": "http://test-data.humdata.org/api/3/action/help_show?name=organization_update"}'
)
resultdictcopy = copy.deepcopy(resultdict)
merge_two_dictionaries(resultdictcopy, datadict)
result = json.dumps(resultdictcopy)
if datadict['name'] == 'MyOrganization1':
return MockResponse(
200,
'{"success": true, "result": %s, "help": "http://test-data.humdata.org/api/3/action/help_show?name=organization_update"}'
% result)
if datadict['name'] == 'MyOrganization2':
return MockResponse(
404,
'{"success": false, "error": {"message": "Not found", "__type": "Not Found Error"}, "help": "http://test-data.humdata.org/api/3/action/help_show?name=organization_update"}'
)
if datadict['name'] == 'MyOrganization3':
return MockResponse(
200,
'{"success": false, "error": {"message": "Not found", "__type": "Not Found Error"}, "help": "http://test-data.humdata.org/api/3/action/help_show?name=organization_update"}'
)
return MockResponse(
404,
'{"success": false, "error": {"message": "Not found", "__type": "Not Found Error"}, "help": "http://test-data.humdata.org/api/3/action/help_show?name=organization_update"}'
)
def post(url, data, headers, files, allow_redirects, auth=None):
if isinstance(data, dict):
datadict = {k.decode('utf8'): v.decode('utf8') for k, v in data.items()}
else:
datadict = json.loads(data.decode('utf-8'))
if 'show' in url:
return mockshow(url, datadict)
if 'resource_id' in datadict:
if datadict['resource_id'] == '74b74ae1-df0c-4716-829f-4f939a046811':
return MockResponse(200,
'{"success": true, "result": {"fields": [{"type": "text", "id": "code"}, {"type": "text", "id": "title"}, {"type": "float", "id": "value"}, {"type": "timestamp", "id": "latest_date"}, {"type": "text", "id": "source"}, {"type": "text", "id": "source_link"}, {"type": "text", "id": "notes"}, {"type": "text", "id": "explore"}, {"type": "text", "id": "units"}], "method": "insert", "primary_key": "code", "resource_id": "bfa6b55f-10b6-4ba2-8470-33bb9a5194a5"}, "help": "http://test-data.humdata.org/api/3/action/help_show?name=datastore_create"}')
if 'update' not in url:
return MockResponse(404,
'{"success": false, "error": {"message": "TEST ERROR: Not update", "__type": "TEST ERROR: Not Update Error"}, "help": "http://test-data.humdata.org/api/3/action/help_show?name=resource_update"}')
if datadict['name'] == 'MyResource1':
resultdictcopy = copy.deepcopy(resultdict)
merge_two_dictionaries(resultdictcopy, datadict)
if files is not None:
resultdictcopy['url_type'] = 'upload'
resultdictcopy['resource_type'] = 'file.upload'
filename = os.path.basename(files[0][1].name)
resultdictcopy[
'url'] = 'http://test-data.humdata.org/dataset/6f36a41c-f126-4b18-aaaf-6c2ddfbc5d4d/resource/de6549d8-268b-4dfe-adaf-a4ae5c8510d5/download/%s' % filename
result = json.dumps(resultdictcopy)
return MockResponse(200,
'{"success": true, "result": %s, "help": "http://test-data.humdata.org/api/3/action/help_show?name=resource_update"}' % result)
if datadict['name'] == 'MyResource2':
return MockResponse(404,
'{"success": false, "error": {"message": "Not found", "__type": "Not Found Error"}, "help": "http://test-data.humdata.org/api/3/action/help_show?name=resource_update"}')
if datadict['name'] == 'MyResource3':
return MockResponse(200,
'{"success": false, "error": {"message": "Not found", "__type": "Not Found Error"}, "help": "http://test-data.humdata.org/api/3/action/help_show?name=resource_update"}')
return MockResponse(404,
'{"success": false, "error": {"message": "Not found", "__type": "Not Found Error"}, "help": "http://test-data.humdata.org/api/3/action/help_show?name=resource_update"}')
def post(url, data, headers, files, allow_redirects, auth):
if isinstance(data, dict):
datadict = {
k.decode('utf8'): v.decode('utf8')
for k, v in data.items()
}
else:
datadict = json.loads(data.decode('utf-8'))
if 'show' in url or 'related_list' in url:
return mockshow(url, datadict)
if 'related' in url:
result = json.dumps(TestDataset.gallerydict)
return MockResponse(
200,
'{"success": true, "result": %s, "help": "http://test-data.humdata.org/api/3/action/help_show?name=related_create"}'
% result)
if 'resource' in url:
result = json.dumps(TestDataset.resources_data[0])
return MockResponse(
200,
'{"success": true, "result": %s, "help": "http://test-data.humdata.org/api/3/action/help_show?name=resource_update"}'
% result)
else:
if 'update' not in url:
return MockResponse(
404,
'{"success": false, "error": {"message": "TEST ERROR: Not update", "__type": "TEST ERROR: Not Update Error"}, "help": "http://test-data.humdata.org/api/3/action/help_show?name=dataset_update"}'
)
resultdictcopy = copy.deepcopy(resultdict)
merge_two_dictionaries(resultdictcopy, datadict)
for i, resource in enumerate(resultdictcopy['resources']):
for j, resource2 in enumerate(
resultdictcopy['resources']):
if i != j:
if resource == resource2:
del resultdictcopy['resources'][j]
break
result = json.dumps(resultdictcopy)
if datadict['name'] == 'MyDataset1':
return MockResponse(
200,
'{"success": true, "result": %s, "help": "http://test-data.humdata.org/api/3/action/help_show?name=dataset_update"}'
% result)
if datadict['name'] == 'MyDataset2':
return MockResponse(
404,
'{"success": false, "error": {"message": "Not found", "__type": "Not Found Error"}, "help": "http://test-data.humdata.org/api/3/action/help_show?name=dataset_update"}'
)
if datadict['name'] == 'MyDataset3':
return MockResponse(
200,
'{"success": false, "error": {"message": "Not found", "__type": "Not Found Error"}, "help": "http://test-data.humdata.org/api/3/action/help_show?name=dataset_update"}'
)
return MockResponse(
404,
'{"success": false, "error": {"message": "Not found", "__type": "Not Found Error"}, "help": "http://test-data.humdata.org/api/3/action/help_show?name=dataset_update"}'
)
def create_in_hdx(self):
# type: () -> None
"""Check if dataset exists in HDX and if so, update it, otherwise create it
Returns:
None
"""
self.check_required_fields()
loadedid = None
if 'id' in self.data:
if self._dataset_load_from_hdx(self.data['id']):
loadedid = self.data['id']
else:
logger.warning('Failed to load dataset with id %s' %
self.data['id'])
if not loadedid:
if self._dataset_load_from_hdx(self.data['name']):
loadedid = self.data['name']
if loadedid:
logger.warning('Dataset exists. Updating %s' % loadedid)
self._dataset_merge_hdx_update(True, True)
return
filestore_resources = list()
if self.resources:
ignore_fields = [self.configuration['resource']['dataset_id']]
for resource in self.resources:
resource.check_required_fields(ignore_fields=ignore_fields)
if resource.get_file_to_upload():
filestore_resources.append(resource)
self.data['resources'] = self._convert_hdxobjects(self.resources)
self._save_to_hdx('create', 'name')
for resource in filestore_resources:
for created_resource in self.data['resources']:
if resource['name'] == created_resource['name']:
merge_two_dictionaries(resource.data, created_resource)
resource.update_in_hdx()
merge_two_dictionaries(created_resource, resource.data)
break
self.init_resources()
self.separate_resources()
if self.include_gallery:
self.old_data['gallery'] = self._copy_hdxobjects(
self.gallery, GalleryItem)
galleryitem_dataset_id = self.configuration['galleryitem'][
'dataset_id']
for i, galleryitem in enumerate(self.gallery):
galleryitem[galleryitem_dataset_id] = self.data['id']
galleryitem.check_required_fields()
galleryitem.create_in_hdx()
def update_in_hdx(self, **kwargs):
# type: (Any) -> None
"""Check if showcase exists in HDX and if so, update it
Returns:
None
"""
self._check_load_existing_object('showcase', 'name')
# We load an existing object even though it may well have been loaded already
# to prevent an admittedly unlikely race condition where someone has updated
# the object in the intervening time
merge_two_dictionaries(self.data, self.old_data)
self.clean_tags()
self._hdx_update('showcase', 'name', force_active=True, **kwargs)
self._update_in_hdx('showcase', 'name', **kwargs)
def resource_view_mockcreate(url, datadict):
if 'create' not in url:
return MockResponse(
404,
'{"success": false, "error": {"message": "TEST ERROR: Not create", "__type": "TEST ERROR: Not Create Error"}, "help": "http://test-data.humdata.org/api/3/action/help_show?name=resource_view_create"}'
)
if datadict['title'] == 'A Preview':
result = json.dumps(resultdict)
return MockResponse(
200,
'{"success": true, "result": %s, "help": "http://test-data.humdata.org/api/3/action/help_show?name=resource_view_create"}'
% result)
if datadict['title'] == 'Quick Charts':
resultdictcopy = copy.deepcopy(resultdict)
result = json.dumps(merge_two_dictionaries(resultdictcopy, datadict))
return MockResponse(
200,
'{"success": true, "result": %s, "help": "http://test-data.humdata.org/api/3/action/help_show?name=resource_view_create"}'
% result)
if datadict['title'] == 'XXX':
return MockResponse(
404,
'{"success": false, "error": {"message": "Not found", "__type": "Not Found Error"}, "help": "http://test-data.humdata.org/api/3/action/help_show?name=resource_view_create"}'
)
if datadict['title'] == 'YYY':
return MockResponse(
200,
'{"success": false, "error": {"message": "Not found", "__type": "Not Found Error"}, "help": "http://test-data.humdata.org/api/3/action/help_show?name=resource_view_create"}'
)
return MockResponse(
404,
'{"success": false, "error": {"message": "Not found", "__type": "Not Found Error"}, "help": "http://test-data.humdata.org/api/3/action/help_show?name=resource_view_create"}'
)
def _merge_hdx_update(self, object_type, id_field_name, file_to_upload=None):
# type: (str, str, Optional[str]) -> None
"""Helper method to check if HDX object exists and update it
Args:
object_type (str): Description of HDX object type (for messages)
id_field_name (str): Name of field containing HDX object identifier
file_to_upload (Optional[str]): File to upload to HDX
Returns:
None
"""
merge_two_dictionaries(self.data, self.old_data)
ignore_dataset_id = Configuration.read()['%s' % object_type].get('ignore_dataset_id_on_update', False)
self.check_required_fields(ignore_dataset_id=ignore_dataset_id)
self._save_to_hdx('update', id_field_name, file_to_upload)
def _merge_hdx_update(self, object_type, id_field_name, file_to_upload=None, force_active=False, **kwargs):
# type: (str, str, Optional[str], bool, Any) -> None
"""Helper method to check if HDX object exists and update it
Args:
object_type (str): Description of HDX object type (for messages)
id_field_name (str): Name of field containing HDX object identifier
file_to_upload (Optional[str]): File to upload to HDX
force_active (bool): Make object state active. Defaults to False.
**kwargs: See below
operation (string): Operation to perform eg. patch. Defaults to update.
ignore_field (string): Any field to ignore when checking dataset metadata. Defaults to None.
Returns:
None
"""
merge_two_dictionaries(self.data, self.old_data)
self._hdx_update(object_type, id_field_name, file_to_upload=file_to_upload, force_active=force_active, **kwargs)
def create_in_hdx(self, allow_no_resources=False):
# type: (Optional[bool]) -> None
"""Check if dataset exists in HDX and if so, update it, otherwise create it
Args:
allow_no_resources (Optional[bool]): Whether to allow no resources. Defaults to False.
Returns:
None
"""
self.check_required_fields(allow_no_resources=allow_no_resources)
loadedid = None
if 'id' in self.data:
if self._dataset_load_from_hdx(self.data['id']):
loadedid = self.data['id']
else:
logger.warning('Failed to load dataset with id %s' %
self.data['id'])
if not loadedid:
if self._dataset_load_from_hdx(self.data['name']):
loadedid = self.data['name']
if loadedid:
logger.warning('Dataset exists. Updating %s' % loadedid)
self._dataset_merge_hdx_update(True)
return
filestore_resources = list()
if self.resources:
ignore_fields = ['package_id']
for resource in self.resources:
resource.check_required_fields(ignore_fields=ignore_fields)
if resource.get_file_to_upload():
filestore_resources.append(resource)
self.data['resources'] = self._convert_hdxobjects(self.resources)
self._save_to_hdx('create', 'name')
for resource in filestore_resources:
for created_resource in self.data['resources']:
if resource['name'] == created_resource['name']:
merge_two_dictionaries(resource.data, created_resource)
resource.update_in_hdx()
merge_two_dictionaries(created_resource, resource.data)
break
self.init_resources()
self.separate_resources()
def _addupdate_hdxobject(self, hdxobjects, id_field, new_hdxobject):
# type: (List[HDXObjectUpperBound], str, HDXObjectUpperBound) -> HDXObjectUpperBound
"""Helper function to add a new HDX object to a supplied list of HDX objects or update existing metadata if the object
already exists in the list
Args:
hdxobjects (List[T <= HDXObject]): list of HDX objects to which to add new objects or update existing ones
id_field (str): Field on which to match to determine if object already exists in list
new_hdxobject (T <= HDXObject): The HDX object to be added/updated
Returns:
T <= HDXObject: The HDX object which was added or updated
"""
for hdxobject in hdxobjects:
if hdxobject[id_field] == new_hdxobject[id_field]:
merge_two_dictionaries(hdxobject, new_hdxobject)
return hdxobject
hdxobjects.append(new_hdxobject)
return new_hdxobject
def create_in_hdx(self, **kwargs):
# type: (Any) -> None
"""Check if showcase exists in HDX and if so, update it, otherwise create it
Returns:
None
"""
if 'ignore_check' not in kwargs: # allow ignoring of field checks
self.check_required_fields()
if 'name' in self.data and self._load_from_hdx('showcase', self.data['name']):
logger.warning('%s exists. Updating %s' % ('showcase', self.data['name']))
merge_two_dictionaries(self.data, self.old_data)
self.clean_tags()
self._hdx_update('showcase', 'name', force_active=True, **kwargs)
else:
self.clean_tags()
self._save_to_hdx('create', 'title', force_active=True)
self._create_in_hdx('showcase', 'name', 'title', **kwargs)
def add_filestore_resources(resources_list, filestore_resources):
# type: (List[Dict], List[hdx.data.resource.Resource]) -> None
"""Helper method to create files in filestore by updating resources.
Args:
resources_list (List[Dict]): List of resources from dataset dict's resources key
filestore_resources (List[hdx.data.resource.Resource]): List of resources that use filestore (to be appended to)
Returns:
None
"""
for resource in filestore_resources:
for created_resource in resources_list:
if resource['name'] == created_resource['name']:
merge_two_dictionaries(resource.data, created_resource)
del resource['url']
resource.update_in_hdx()
merge_two_dictionaries(created_resource, resource.data)
break
def _dataset_merge_hdx_update(self, update_resources):
# type: (bool) -> None
"""Helper method to check if dataset or its resources exist and update them
Args:
update_resources (bool): Whether to update resources
Returns:
None
"""
merge_two_dictionaries(self.data, self.old_data)
if 'resources' in self.data:
del self.data['resources']
old_resources = self.old_data.get('resources', None)
filestore_resources = list()
if update_resources and old_resources:
ignore_fields = ['package_id']
resource_names = set()
for resource in self.resources:
resource_name = resource['name']
resource_names.add(resource_name)
for old_resource in old_resources:
if resource_name == old_resource['name']:
logger.warning('Resource exists. Updating %s' %
resource_name)
merge_two_dictionaries(resource, old_resource)
if old_resource.get_file_to_upload():
resource.set_file_to_upload(
old_resource.get_file_to_upload())
filestore_resources.append(resource)
resource.check_required_fields(
ignore_fields=ignore_fields)
break
for old_resource in old_resources:
if not old_resource['name'] in resource_names:
old_resource.check_required_fields(
ignore_fields=ignore_fields)
self.resources.append(old_resource)
if old_resource.get_file_to_upload():
filestore_resources.append(old_resource)
if self.resources:
self.data['resources'] = self._convert_hdxobjects(self.resources)
self._save_to_hdx('update', 'id')
for resource in filestore_resources:
for created_resource in self.data['resources']:
if resource['name'] == created_resource['name']:
merge_two_dictionaries(resource.data, created_resource)
resource.update_in_hdx()
merge_two_dictionaries(created_resource, resource.data)
break
def load_json_into_existing_dict(data: dict, path: str) -> dict:
"""Merge JSON file into existing dictionary
Args:
data (dict): Dictionary to merge into
path (str): JSON file to load and merge
Returns:
dict: JSON file merged into dictionary
"""
jsondict = load_json(path)
return merge_two_dictionaries(data, jsondict)
def load_yaml_into_existing_dict(data: dict, path: str) -> dict:
"""Merge YAML file into existing dictionary
Args:
data (dict): Dictionary to merge into
path (str): YAML file to load and merge
Returns:
dict: YAML file merged into dictionary
"""
yamldict = load_yaml(path)
return merge_two_dictionaries(data, yamldict)
def dataset_merge_filestore_resource(cls, resource, updated_resource,
filestore_resources, ignore_fields):
# type: (hdx.data.resource.Resource, hdx.data.resource.Resource, List[hdx.data.resource.Resource], List[str]) -> None
"""Helper method to merge updated resource from dataset into HDX resource read from HDX including filestore.
Args:
resource (hdx.data.resource.Resource): Resource read from HDX
updated_resource (hdx.data.resource.Resource): Updated resource from dataset
filestore_resources (List[hdx.data.resource.Resource]): List of resources that use filestore (to be appended to)
ignore_fields (List[str]): List of fields to ignore when checking resource
Returns:
None
"""
if updated_resource.get_file_to_upload():
resource.set_file_to_upload(updated_resource.get_file_to_upload())
filestore_resources.append(resource)
merge_two_dictionaries(resource, updated_resource)
resource.check_required_fields(ignore_fields=ignore_fields)
if resource.get_file_to_upload():
resource['url'] = cls.temporary_url
def post(url, data, headers, files, allow_redirects, auth=None):
datadict = json.loads(data.decode('utf-8'))
if 'show' in url:
return organization_mockshow(url, datadict)
if 'update' not in url:
return MockResponse(404,
'{"success": false, "error": {"message": "TEST ERROR: Not update", "__type": "TEST ERROR: Not Update Error"}, "help": "http://test-data.humdata.org/api/3/action/help_show?name=organization_update"}')
resultdictcopy = copy.deepcopy(resultdict)
merge_two_dictionaries(resultdictcopy, datadict)
result = json.dumps(resultdictcopy)
if datadict['name'] == 'MyOrganization1':
return MockResponse(200,
'{"success": true, "result": %s, "help": "http://test-data.humdata.org/api/3/action/help_show?name=organization_update"}' % result)
if datadict['name'] == 'MyOrganization2':
return MockResponse(404,
'{"success": false, "error": {"message": "Not found", "__type": "Not Found Error"}, "help": "http://test-data.humdata.org/api/3/action/help_show?name=organization_update"}')
if datadict['name'] == 'MyOrganization3':
return MockResponse(200,
'{"success": false, "error": {"message": "Not found", "__type": "Not Found Error"}, "help": "http://test-data.humdata.org/api/3/action/help_show?name=organization_update"}')
return MockResponse(404,
'{"success": false, "error": {"message": "Not found", "__type": "Not Found Error"}, "help": "http://test-data.humdata.org/api/3/action/help_show?name=organization_update"}')
def load_json_into_existing_dict(data: dict,
path: str,
encoding: str = "utf-8") -> Dict:
"""Merge JSON file into existing dictionary
Args:
data (dict): Dictionary to merge into
path (str): JSON file to load and merge
encoding (str): Encoding of file. Defaults to utf-8.
Returns:
dict: JSON file merged into dictionary
"""
jsondict = load_json(path, encoding=encoding)
return merge_two_dictionaries(data, jsondict)
def _dataset_merge_hdx_update(self, update_resources, update_gallery):
# type: (bool, bool) -> None
"""Helper method to check if dataset or its resources or gallery items exist and update them
Args:
update_resources (bool): Whether to update resources
update_gallery (bool): Whether to update gallery
Returns:
None
"""
merge_two_dictionaries(self.data, self.old_data)
if 'resources' in self.data:
del self.data['resources']
if 'gallery' in self.data:
del self.data['gallery']
old_resources = self.old_data.get('resources', None)
filestore_resources = list()
if update_resources and old_resources:
ignore_fields = [self.configuration['resource']['dataset_id']]
resource_names = set()
for resource in self.resources:
resource_name = resource['name']
resource_names.add(resource_name)
for old_resource in old_resources:
if resource_name == old_resource['name']:
logger.warning('Resource exists. Updating %s' %
resource_name)
merge_two_dictionaries(resource, old_resource)
if old_resource.get_file_to_upload():
resource.set_file_to_upload(
old_resource.get_file_to_upload())
filestore_resources.append(resource)
resource.check_required_fields(
ignore_fields=ignore_fields)
break
for old_resource in old_resources:
if not old_resource['name'] in resource_names:
old_resource.check_required_fields(
ignore_fields=ignore_fields)
self.resources.append(old_resource)
if old_resource.get_file_to_upload():
filestore_resources.append(old_resource)
old_gallery = self.old_data.get('gallery', None)
if self.resources:
self.data['resources'] = self._convert_hdxobjects(self.resources)
self._save_to_hdx('update', 'id')
for resource in filestore_resources:
for created_resource in self.data['resources']:
if resource['name'] == created_resource['name']:
merge_two_dictionaries(resource.data, created_resource)
resource.update_in_hdx()
merge_two_dictionaries(created_resource, resource.data)
break
if self.include_gallery and update_gallery and old_gallery:
self.old_data['gallery'] = self._copy_hdxobjects(
self.gallery, GalleryItem)
ignore_fields = [self.configuration['galleryitem']['dataset_id']]
galleryitem_titles = set()
galleryitem_dataset_id = self.configuration['galleryitem'][
'dataset_id']
for i, galleryitem in enumerate(self.gallery):
galleryitem_title = galleryitem['title']
galleryitem_titles.add(galleryitem_title)
for old_galleryitem in old_gallery:
if galleryitem_title == old_galleryitem['title']:
logger.warning('Gallery item exists. Updating %s' %
galleryitem_title)
merge_two_dictionaries(galleryitem, old_galleryitem)
galleryitem.check_required_fields(
ignore_fields=ignore_fields)
galleryitem.update_in_hdx()
for old_galleryitem in old_gallery:
if not old_galleryitem['title'] in galleryitem_titles:
old_galleryitem[galleryitem_dataset_id] = self.data['id']
old_galleryitem.check_required_fields()
old_galleryitem.create_in_hdx()
self.gallery.append(old_galleryitem)
def __init__(self, **kwargs):
# type: (Any) -> None
super(Configuration, self).__init__()
self._remoteckan = None
self._emailer = None
hdx_base_config_found = False
hdx_base_config_dict = kwargs.get('hdx_base_config_dict', None)
if hdx_base_config_dict:
hdx_base_config_found = True
logger.info('Loading HDX base configuration from dictionary')
hdx_base_config_json = kwargs.get('hdx_base_config_json', '')
if hdx_base_config_json:
if hdx_base_config_found:
raise ConfigurationError('More than one HDX base configuration given!')
hdx_base_config_found = True
logger.info('Loading HDX base configuration from: %s' % hdx_base_config_json)
hdx_base_config_dict = load_json(hdx_base_config_json)
hdx_base_config_yaml = kwargs.get('hdx_base_config_yaml', '')
if hdx_base_config_found:
if hdx_base_config_yaml:
raise ConfigurationError('More than one HDX base configuration given!')
else:
if not hdx_base_config_yaml:
hdx_base_config_yaml = Configuration.default_hdx_base_config_yaml
logger.info('No HDX base configuration parameter. Using default base configuration file: %s.' % hdx_base_config_yaml)
logger.info('Loading HDX base configuration from: %s' % hdx_base_config_yaml)
hdx_base_config_dict = load_yaml(hdx_base_config_yaml)
hdx_config_found = False
hdx_config_dict = kwargs.get('hdx_config_dict', None)
if hdx_config_dict:
hdx_config_found = True
logger.info('Loading HDX configuration from dictionary')
hdx_config_json = kwargs.get('hdx_config_json', '')
if hdx_config_json:
if hdx_config_found:
raise ConfigurationError('More than one HDX configuration given!')
hdx_config_found = True
logger.info('Loading HDX configuration from: %s' % hdx_config_json)
hdx_config_dict = load_json(hdx_config_json)
hdx_config_yaml = kwargs.get('hdx_config_yaml', '')
if hdx_config_found:
if hdx_config_yaml:
raise ConfigurationError('More than one HDX configuration given!')
else:
if not hdx_config_yaml:
hdx_config_yaml = Configuration.default_hdx_config_yaml
if isfile(hdx_config_yaml):
logger.info('No HDX configuration parameter. Using default configuration file: %s.' % hdx_config_yaml)
else:
logger.info('No HDX configuration parameter and no configuration file at default path: %s.' % hdx_config_yaml)
hdx_config_yaml = None
hdx_config_dict = dict()
if hdx_config_yaml:
logger.info('Loading HDX configuration from: %s' % hdx_config_yaml)
hdx_config_dict = load_yaml(hdx_config_yaml)
self.data = merge_two_dictionaries(hdx_base_config_dict, hdx_config_dict)
project_config_found = False
project_config_dict = kwargs.get('project_config_dict', None)
if project_config_dict is not None:
project_config_found = True
logger.info('Loading project configuration from dictionary')
project_config_json = kwargs.get('project_config_json', '')
if project_config_json:
if project_config_found:
raise ConfigurationError('More than one project configuration given!')
project_config_found = True
logger.info('Loading project configuration from: %s' % project_config_json)
project_config_dict = load_json(project_config_json)
project_config_yaml = kwargs.get('project_config_yaml', '')
if project_config_found:
if project_config_yaml:
raise ConfigurationError('More than one project configuration given!')
else:
if project_config_yaml:
logger.info('Loading project configuration from: %s' % project_config_yaml)
project_config_dict = load_yaml(project_config_yaml)
else:
project_config_dict = dict()
self.data = merge_two_dictionaries(hdx_base_config_dict, project_config_dict)
ua = kwargs.get('full_agent')
if ua:
self.user_agent = ua
else:
try:
self.user_agent = UserAgent.get(prefix=Configuration.prefix, **kwargs)
except UserAgentError:
self.user_agent = UserAgent.get(prefix=Configuration.prefix, **self.data)
self.hdx_read_only = kwargs.get('hdx_read_only', self.data.get('hdx_read_only', False))
logger.info('Read only access to HDX: %s' % str(self.hdx_read_only))
self.hdx_key = kwargs.get('hdx_key', self.data.get('hdx_key'))
if not self.hdx_key and not self.hdx_read_only:
raise ConfigurationError('No HDX API key supplied as a parameter or in configuration!')
hdx_url = kwargs.get('hdx_url', self.data.get('hdx_url'))
if hdx_url:
self.hdx_site = 'hdx_custom_site'
self.data[self.hdx_site] = {'url': hdx_url}
else:
self.hdx_site = 'hdx_%s_site' % kwargs.get('hdx_site', self.data.get('hdx_site', 'test'))
if self.hdx_site not in self.data:
raise ConfigurationError('%s not defined in configuration!' % self.hdx_site)
def __init__(self, **kwargs):
# type: (Any) -> None
super(Configuration, self).__init__()
self._remoteckan = None
self._emailer = None
hdx_base_config_found = False
hdx_base_config_dict = kwargs.get('hdx_base_config_dict', None)
if hdx_base_config_dict:
hdx_base_config_found = True
logger.info('Loading HDX base configuration from dictionary')
hdx_base_config_json = kwargs.get('hdx_base_config_json', '')
if hdx_base_config_json:
if hdx_base_config_found:
raise ConfigurationError(
'More than one HDX base configuration given!')
hdx_base_config_found = True
logger.info('Loading HDX base configuration from: %s' %
hdx_base_config_json)
hdx_base_config_dict = load_json(hdx_base_config_json)
hdx_base_config_yaml = kwargs.get('hdx_base_config_yaml', '')
if hdx_base_config_found:
if hdx_base_config_yaml:
raise ConfigurationError(
'More than one HDX base configuration given!')
else:
if not hdx_base_config_yaml:
hdx_base_config_yaml = Configuration.default_hdx_base_config_yaml
logger.info(
'No HDX base configuration parameter. Using default base configuration file: %s.'
% hdx_base_config_yaml)
logger.info('Loading HDX base configuration from: %s' %
hdx_base_config_yaml)
hdx_base_config_dict = load_yaml(hdx_base_config_yaml)
hdx_config_found = False
hdx_config_dict = kwargs.get('hdx_config_dict', None)
if hdx_config_dict:
hdx_config_found = True
logger.info('Loading HDX configuration from dictionary')
hdx_config_json = kwargs.get('hdx_config_json', '')
if hdx_config_json:
if hdx_config_found:
raise ConfigurationError(
'More than one HDX configuration given!')
hdx_config_found = True
logger.info('Loading HDX configuration from: %s' % hdx_config_json)
hdx_config_dict = load_json(hdx_config_json)
hdx_config_yaml = kwargs.get('hdx_config_yaml', '')
if hdx_config_found:
if hdx_config_yaml:
raise ConfigurationError(
'More than one HDX configuration given!')
else:
if not hdx_config_yaml:
hdx_config_yaml = Configuration.default_hdx_config_yaml
if isfile(hdx_config_yaml):
logger.info(
'No HDX configuration parameter. Using default configuration file: %s.'
% hdx_config_yaml)
else:
logger.info(
'No HDX configuration parameter and no configuration file at default path: %s.'
% hdx_config_yaml)
hdx_config_yaml = None
hdx_config_dict = dict()
if hdx_config_yaml:
logger.info('Loading HDX configuration from: %s' %
hdx_config_yaml)
hdx_config_dict = load_yaml(hdx_config_yaml)
self.data = merge_two_dictionaries(hdx_base_config_dict,
hdx_config_dict)
project_config_found = False
project_config_dict = kwargs.get('project_config_dict', None)
if project_config_dict is not None:
project_config_found = True
logger.info('Loading project configuration from dictionary')
project_config_json = kwargs.get('project_config_json', '')
if project_config_json:
if project_config_found:
raise ConfigurationError(
'More than one project configuration given!')
project_config_found = True
logger.info('Loading project configuration from: %s' %
project_config_json)
project_config_dict = load_json(project_config_json)
project_config_yaml = kwargs.get('project_config_yaml', '')
if project_config_found:
if project_config_yaml:
raise ConfigurationError(
'More than one project configuration given!')
else:
if project_config_yaml:
logger.info('Loading project configuration from: %s' %
project_config_yaml)
project_config_dict = load_yaml(project_config_yaml)
else:
project_config_dict = dict()
self.data = merge_two_dictionaries(hdx_base_config_dict,
project_config_dict)
ua = kwargs.get('full_agent')
if ua:
self.user_agent = ua
else:
try:
self.user_agent = UserAgent.get(prefix=Configuration.prefix,
**kwargs)
except UserAgentError:
self.user_agent = UserAgent.get(prefix=Configuration.prefix,
**self.data)
self.hdx_read_only = kwargs.get('hdx_read_only',
self.data.get('hdx_read_only', False))
logger.info('Read only access to HDX: %s' % str(self.hdx_read_only))
self.hdx_key = kwargs.get('hdx_key', self.data.get('hdx_key'))
if not self.hdx_key and not self.hdx_read_only:
raise ConfigurationError(
'No HDX API key supplied as a parameter or in configuration!')
hdx_url = kwargs.get('hdx_url', self.data.get('hdx_url'))
if hdx_url:
self.hdx_site = 'hdx_custom_site'
hdx_url = hdx_url.rstrip('/')
self.data[self.hdx_site] = {'url': hdx_url}
else:
self.hdx_site = 'hdx_%s_site' % kwargs.get(
'hdx_site', self.data.get('hdx_site', 'test'))
if self.hdx_site not in self.data:
raise ConfigurationError('%s not defined in configuration!' %
self.hdx_site)
def __init__(self, **kwargs):
# type: (Any) -> None
super(Configuration, self).__init__()
hdx_config_found = False
hdx_config_dict = kwargs.get('hdx_config_dict', None)
if hdx_config_dict:
hdx_config_found = True
logger.info('Loading HDX configuration from dictionary')
hdx_config_json = kwargs.get('hdx_config_json', '')
if hdx_config_json:
if hdx_config_found:
raise ConfigurationError(
'More than one HDX configuration file given!')
hdx_config_found = True
logger.info('Loading HDX configuration from: %s' % hdx_config_json)
hdx_config_dict = load_json(hdx_config_json)
hdx_config_yaml = kwargs.get('hdx_config_yaml', '')
if hdx_config_found:
if hdx_config_yaml:
raise ConfigurationError(
'More than one HDX configuration file given!')
else:
if not hdx_config_yaml:
logger.info('No HDX configuration parameter. Using default.')
hdx_config_yaml = script_dir_plus_file('hdx_configuration.yml',
Configuration)
logger.info('Loading HDX configuration from: %s' % hdx_config_yaml)
hdx_config_dict = load_yaml(hdx_config_yaml)
project_config_found = False
project_config_dict = kwargs.get('project_config_dict', None)
if project_config_dict is not None:
project_config_found = True
logger.info('Loading project configuration from dictionary')
project_config_json = kwargs.get('project_config_json', '')
if project_config_json:
if project_config_found:
raise ConfigurationError(
'More than one project configuration file given!')
project_config_found = True
logger.info('Loading project configuration from: %s' %
project_config_json)
project_config_dict = load_json(project_config_json)
project_config_yaml = kwargs.get('project_config_yaml', '')
if project_config_found:
if project_config_yaml:
raise ConfigurationError(
'More than one project configuration file given!')
else:
if project_config_yaml:
logger.info('Loading project configuration from: %s' %
project_config_yaml)
project_config_dict = load_yaml(project_config_yaml)
else:
project_config_dict = dict()
self.data = merge_two_dictionaries(hdx_config_dict,
project_config_dict)
self.hdx_read_only = kwargs.get('hdx_read_only', False)
if not self.hdx_read_only:
if 'hdx_key' in kwargs:
self.data['api_key'] = kwargs.get('hdx_key')
else:
hdx_key_file = kwargs.get('hdx_key_file',
join(expanduser('~'), '.hdxkey'))
""" :type : str"""
self.data['api_key'] = self.load_api_key(hdx_key_file)
self.hdx_site = 'hdx_%s_site' % kwargs.get('hdx_site', 'test')
if self.hdx_site not in self.data:
raise ConfigurationError('%s not defined in configuration!' %
self.hdx_site)