def _separate_hdxobjects(self, hdxobjects: List[HDXObjectUpperBound], hdxobjects_name: str, id_field: str, hdxobjectclass: type) -> None: """Helper function to take a list of HDX objects contained in the internal dictionary and add them to a supplied list of HDX objects or update existing metadata if any objects already exist in the list. The list in the internal dictionary is then deleted. Args: hdxobjects (list[T <= HDXObject]): List of HDX objects to which to add new objects or update existing ones hdxobjects_name (str): Name of key in internal dictionary from which to obtain list of HDX objects id_field (str): Field on which to match to determine if object already exists in list hdxobjectclass (type): Type of the HDX Object to be added/updated Returns: None """ new_hdxobjects = self.data.get(hdxobjects_name, None) if new_hdxobjects: hdxobject_names = set() for hdxobject in hdxobjects: hdxobject_name = hdxobject[id_field] hdxobject_names.add(hdxobject_name) for new_hdxobject in new_hdxobjects: if hdxobject_name == new_hdxobject[id_field]: merge_two_dictionaries(hdxobject, new_hdxobject) break for new_hdxobject in new_hdxobjects: if not new_hdxobject[id_field] in hdxobject_names: hdxobjects.append(hdxobjectclass(self.configuration, new_hdxobject)) del self.data[hdxobjects_name]
def mockreturn(url, data, headers, files, allow_redirects, auth): datadict = json.loads(data.decode('utf-8')) if 'show' in url: return mockshow(url, datadict) if 'update' not in url: return MockResponse( 404, '{"success": false, "error": {"message": "TEST ERROR: Not update", "__type": "TEST ERROR: Not Update Error"}, "help": "http://test-data.humdata.org/api/3/action/help_show?name=related_update"}' ) resultdictcopy = copy.deepcopy(resultdict) merge_two_dictionaries(resultdictcopy, datadict) result = json.dumps(resultdictcopy) if datadict['title'] == 'MyGalleryItem1': return MockResponse( 200, '{"success": true, "result": %s, "help": "http://test-data.humdata.org/api/3/action/help_show?name=related_update"}' % result) if datadict['title'] == 'MyGalleryItem2': return MockResponse( 404, '{"success": false, "error": {"message": "Not found", "__type": "Not Found Error"}, "help": "http://test-data.humdata.org/api/3/action/help_show?name=related_update"}' ) if datadict['title'] == 'MyGalleryItem3': return MockResponse( 200, '{"success": false, "error": {"message": "Not found", "__type": "Not Found Error"}, "help": "http://test-data.humdata.org/api/3/action/help_show?name=related_update"}' ) return MockResponse( 404, '{"success": false, "error": {"message": "Not found", "__type": "Not Found Error"}, "help": "http://test-data.humdata.org/api/3/action/help_show?name=related_update"}' )
def _separate_hdxobjects(self, hdxobjects: List[HDXObjectUpperBound], hdxobjects_name: str, id_field: str, hdxobjectclass: type) -> None: """Helper function to take a list of HDX objects contained in the internal dictionary and add them to a supplied list of HDX objects or update existing metadata if any objects already exist in the list. The list in the internal dictionary is then deleted. Args: hdxobjects (list[T <= HDXObject]): List of HDX objects to which to add new objects or update existing ones hdxobjects_name (str): Name of key in internal dictionary from which to obtain list of HDX objects id_field (str): Field on which to match to determine if object already exists in list hdxobjectclass (type): Type of the HDX Object to be added/updated Returns: None """ new_hdxobjects = self.data.get(hdxobjects_name, list()) """:type : List[HDXObjectUpperBound]""" if new_hdxobjects: hdxobject_names = set() for hdxobject in hdxobjects: hdxobject_name = hdxobject[id_field] hdxobject_names.add(hdxobject_name) for new_hdxobject in new_hdxobjects: if hdxobject_name == new_hdxobject[id_field]: merge_two_dictionaries(hdxobject, new_hdxobject) break for new_hdxobject in new_hdxobjects: if not new_hdxobject[id_field] in hdxobject_names: hdxobjects.append(hdxobjectclass(self.configuration, new_hdxobject)) del self.data[hdxobjects_name]
def mockreturn(url, data, headers, files, allow_redirects, auth): datadict = json.loads(data.decode('utf-8')) if 'show' in url or 'related_list' in url: return mockshow(url, datadict) if 'update' not in url: return MockResponse(404, '{"success": false, "error": {"message": "TEST ERROR: Not update", "__type": "TEST ERROR: Not Update Error"}, "help": "http://test-data.humdata.org/api/3/action/help_show?name=dataset_update"}') if 'related' in url: result = json.dumps(TestDataset.gallerydict) return MockResponse(200, '{"success": true, "result": %s, "help": "http://test-data.humdata.org/api/3/action/help_show?name=related_create"}' % result) else: resultdictcopy = copy.deepcopy(resultdict) merge_two_dictionaries(resultdictcopy, datadict) for i, resource in enumerate(resultdictcopy['resources']): for j, resource2 in enumerate(resultdictcopy['resources']): if i != j: if resource == resource2: del resultdictcopy['resources'][j] break result = json.dumps(resultdictcopy) if datadict['name'] == 'MyDataset1': return MockResponse(200, '{"success": true, "result": %s, "help": "http://test-data.humdata.org/api/3/action/help_show?name=dataset_update"}' % result) if datadict['name'] == 'MyDataset2': return MockResponse(404, '{"success": false, "error": {"message": "Not found", "__type": "Not Found Error"}, "help": "http://test-data.humdata.org/api/3/action/help_show?name=dataset_update"}') if datadict['name'] == 'MyDataset3': return MockResponse(200, '{"success": false, "error": {"message": "Not found", "__type": "Not Found Error"}, "help": "http://test-data.humdata.org/api/3/action/help_show?name=dataset_update"}') return MockResponse(404, '{"success": false, "error": {"message": "Not found", "__type": "Not Found Error"}, "help": "http://test-data.humdata.org/api/3/action/help_show?name=dataset_update"}')
def _merge_hdx_update(self, object_type: str, id_field_name: str) -> None: """Helper method to check if HDX object exists and update it Args: object_type (str): Description of HDX object type (for messages) id_field_name (str): Name of field containing HDX object identifier Returns: None """ merge_two_dictionaries(self.data, self.old_data) self.check_required_fields(self.configuration['%s' % object_type].get('ignore_on_update', [])) self._save_to_hdx('update', id_field_name)
def _merge_hdx_update(self, object_type: str, id_field_name: str) -> None: """Helper method to check if HDX object exists and update it Args: object_type (str): Description of HDX object type (for messages) id_field_name (str): Name of field containing HDX object identifier Returns: None """ merge_two_dictionaries(self.data, self.old_data) self.check_required_fields(self.configuration['%s' % object_type].get('ignore_on_update', [])) self._save_to_hdx('update', id_field_name)
def mockreturn(url, data, headers, files, allow_redirects, auth): datadict = json.loads(data.decode('utf-8')) if 'show' in url or 'related_list' in url: return mockshow(url, datadict) if 'update' not in url: return MockResponse( 404, '{"success": false, "error": {"message": "TEST ERROR: Not update", "__type": "TEST ERROR: Not Update Error"}, "help": "http://test-data.humdata.org/api/3/action/help_show?name=dataset_update"}' ) if 'related' in url: result = json.dumps(TestDataset.gallerydict) return MockResponse( 200, '{"success": true, "result": %s, "help": "http://test-data.humdata.org/api/3/action/help_show?name=related_create"}' % result) else: resultdictcopy = copy.deepcopy(resultdict) merge_two_dictionaries(resultdictcopy, datadict) for i, resource in enumerate(resultdictcopy['resources']): for j, resource2 in enumerate(resultdictcopy['resources']): if i != j: if resource == resource2: del resultdictcopy['resources'][j] break result = json.dumps(resultdictcopy) if datadict['name'] == 'MyDataset1': return MockResponse( 200, '{"success": true, "result": %s, "help": "http://test-data.humdata.org/api/3/action/help_show?name=dataset_update"}' % result) if datadict['name'] == 'MyDataset2': return MockResponse( 404, '{"success": false, "error": {"message": "Not found", "__type": "Not Found Error"}, "help": "http://test-data.humdata.org/api/3/action/help_show?name=dataset_update"}' ) if datadict['name'] == 'MyDataset3': return MockResponse( 200, '{"success": false, "error": {"message": "Not found", "__type": "Not Found Error"}, "help": "http://test-data.humdata.org/api/3/action/help_show?name=dataset_update"}' ) return MockResponse( 404, '{"success": false, "error": {"message": "Not found", "__type": "Not Found Error"}, "help": "http://test-data.humdata.org/api/3/action/help_show?name=dataset_update"}' )
def load_json_into_existing_dict(data: dict, path: str) -> dict: """Merge JSON file into existing dictionary Args: data (dict): Dictionary to merge into path (str): JSON file to load and merge Returns: dict: JSON file merged into dictionary """ jsondict = load_json(path) return merge_two_dictionaries(data, jsondict)
def load_yaml_into_existing_dict(data: dict, path: str) -> dict: """Merge YAML file into existing dictionary Args: data (dict): Dictionary to merge into path (str): YAML file to load and merge Returns: dict: YAML file merged into dictionary """ yamldict = load_yaml(path) return merge_two_dictionaries(data, yamldict)
def _addupdate_hdxobject(self, hdxobjects: List[HDXObjectUpperBound], id_field: str, hdxobjectclass: type, new_hdxobject: HDXObjectUpperBound) -> None: """Helper function to add a new HDX object to a supplied list of HDX objects or update existing metadata if the object already exists in the list Args: hdxobjects (list[T <= HDXObject]): List of HDX objects to which to add new objects or update existing ones id_field (str): Field on which to match to determine if object already exists in list hdxobjectclass (type): Type of the HDX Object to be added/updated new_hdxobject (T <= HDXObject): The HDX object to be added/updated Returns: None """ found = False for hdxobject in hdxobjects: if hdxobject[id_field] == new_hdxobject[id_field]: merge_two_dictionaries(hdxobject, new_hdxobject) found = True break if not found: hdxobjects.append(hdxobjectclass(self.configuration, new_hdxobject))
def _addupdate_hdxobject(self, hdxobjects: List[HDXObjectUpperBound], id_field: str, hdxobjectclass: type, new_hdxobject: HDXObjectUpperBound) -> None: """Helper function to add a new HDX object to a supplied list of HDX objects or update existing metadata if the object already exists in the list Args: hdxobjects (list[T <= HDXObject]): List of HDX objects to which to add new objects or update existing ones id_field (str): Field on which to match to determine if object already exists in list hdxobjectclass (type): Type of the HDX Object to be added/updated new_hdxobject (T <= HDXObject): The HDX object to be added/updated Returns: None """ found = False for hdxobject in hdxobjects: if hdxobject[id_field] == new_hdxobject[id_field]: merge_two_dictionaries(hdxobject, new_hdxobject) found = True break if not found: hdxobjects.append(hdxobjectclass(self.configuration, new_hdxobject))
def mockreturn(url, data, headers, files, allow_redirects, auth): datadict = json.loads(data.decode('utf-8')) if 'show' in url: return mockshow(url, datadict) if 'update' not in url: return MockResponse(404, '{"success": false, "error": {"message": "TEST ERROR: Not update", "__type": "TEST ERROR: Not Update Error"}, "help": "http://test-data.humdata.org/api/3/action/help_show?name=resource_update"}') resultdictcopy = copy.deepcopy(resultdict) merge_two_dictionaries(resultdictcopy, datadict) result = json.dumps(resultdictcopy) if datadict['name'] == 'MyResource1': return MockResponse(200, '{"success": true, "result": %s, "help": "http://test-data.humdata.org/api/3/action/help_show?name=resource_update"}' % result) if datadict['name'] == 'MyResource2': return MockResponse(404, '{"success": false, "error": {"message": "Not found", "__type": "Not Found Error"}, "help": "http://test-data.humdata.org/api/3/action/help_show?name=resource_update"}') if datadict['name'] == 'MyResource3': return MockResponse(200, '{"success": false, "error": {"message": "Not found", "__type": "Not Found Error"}, "help": "http://test-data.humdata.org/api/3/action/help_show?name=resource_update"}') return MockResponse(404, '{"success": false, "error": {"message": "Not found", "__type": "Not Found Error"}, "help": "http://test-data.humdata.org/api/3/action/help_show?name=resource_update"}')
def _dataset_merge_hdx_update(self, update_resources: bool, update_gallery: bool) -> None: """Helper method to check if dataset or its resources or gallery items exist and update them Args: update_resources (bool): Whether to update resources update_gallery (bool): Whether to update gallery Returns: None """ merge_two_dictionaries(self.data, self.old_data) if 'resources' in self.data: del self.data['resources'] if 'gallery' in self.data: del self.data['gallery'] old_resources = self.old_data.get('resources', None) if update_resources and old_resources: resource_dataset_id = [ self.configuration['resource']['dataset_id'] ] resource_names = set() for resource in self.resources: resource_name = resource['name'] resource_names.add(resource_name) for old_resource in old_resources: if resource_name == old_resource['name']: logger.warning('Resource exists. Updating %s' % resource_name) merge_two_dictionaries(resource, old_resource) resource.check_required_fields(resource_dataset_id) break for old_resource in old_resources: if not old_resource['name'] in resource_names: old_resource.check_required_fields(resource_dataset_id) self.resources.append(old_resource) old_gallery = self.old_data.get('gallery', None) if self.resources: self.data['resources'] = self._convert_hdxobjects(self.resources) self._save_to_hdx('update', 'id') self.init_resources() self.separate_resources() if self.include_gallery and update_gallery and old_gallery: self.old_data['gallery'] = self._copy_hdxobjects( self.gallery, GalleryItem) galleryitem_titles = set() galleryitem_dataset_id = self.configuration['galleryitem'][ 'dataset_id'] for i, galleryitem in enumerate(self.gallery): galleryitem_title = galleryitem['title'] galleryitem_titles.add(galleryitem_title) for old_galleryitem in old_gallery: if galleryitem_title == old_galleryitem['title']: logger.warning('Gallery item exists. Updating %s' % galleryitem_title) merge_two_dictionaries(galleryitem, old_galleryitem) galleryitem.check_required_fields( [galleryitem_dataset_id]) galleryitem.update_in_hdx() for old_galleryitem in old_gallery: if not old_galleryitem['title'] in galleryitem_titles: old_galleryitem[galleryitem_dataset_id] = self.data['id'] old_galleryitem.check_required_fields() old_galleryitem.create_in_hdx() self.gallery.append(old_galleryitem)
def __init__(self, **kwargs): super(Configuration, self).__init__() hdx_config_found = False hdx_config_dict = kwargs.get('hdx_config_dict', None) if hdx_config_dict: hdx_config_found = True logger.info('Loading HDX configuration from dictionary') hdx_config_json = kwargs.get('hdx_config_json', '') if hdx_config_json: if hdx_config_found: raise ConfigurationError('More than one HDX configuration file given!') hdx_config_found = True logger.info('Loading HDX configuration from: %s' % hdx_config_json) hdx_config_dict = load_json(hdx_config_json) hdx_config_yaml = kwargs.get('hdx_config_yaml', '') if hdx_config_found: if hdx_config_yaml: raise ConfigurationError('More than one HDX configuration file given!') else: if not hdx_config_yaml: logger.info('No HDX configuration parameter. Using default.') hdx_config_yaml = script_dir_plus_file('hdx_configuration.yml', Configuration) logger.info('Loading HDX configuration from: %s' % hdx_config_yaml) hdx_config_dict = load_yaml(hdx_config_yaml) project_config_found = False project_config_dict = kwargs.get('project_config_dict', None) if project_config_dict is not None: project_config_found = True logger.info('Loading project configuration from dictionary') project_config_json = kwargs.get('project_config_json', '') if project_config_json: if project_config_found: raise ConfigurationError('More than one project configuration file given!') project_config_found = True logger.info('Loading project configuration from: %s' % project_config_json) project_config_dict = load_json(project_config_json) project_config_yaml = kwargs.get('project_config_yaml', '') if project_config_found: if project_config_yaml: raise ConfigurationError('More than one project configuration file given!') else: if not project_config_yaml: logger.info('No project configuration parameter. Using default.') project_config_yaml = join('config', 'project_configuration.yml') logger.info('Loading project configuration from: %s' % project_config_yaml) project_config_dict = load_yaml(project_config_yaml) self.data = merge_two_dictionaries(hdx_config_dict, project_config_dict) hdx_key_file = kwargs.get('hdx_key_file', join(expanduser("~"), '.hdxkey')) self.data['api_key'] = self.load_api_key(hdx_key_file) self.hdx_site = 'hdx_%s_site' % kwargs.get('hdx_site', 'test') if self.hdx_site not in self.data: raise ConfigurationError('%s not defined in configuration!' % self.hdx_site)