def _separate_hdxobjects(self, hdxobjects, hdxobjects_name, id_field, hdxobjectclass): # type: (List[HDXObjectUpperBound], str, str, type) -> None """Helper function to take a list of HDX objects contained in the internal dictionary and add them to a supplied list of HDX objects or update existing metadata if any objects already exist in the list. The list in the internal dictionary is then deleted. Args: hdxobjects (List[T <= HDXObject]): list of HDX objects to which to add new objects or update existing ones hdxobjects_name (str): Name of key in internal dictionary from which to obtain list of HDX objects id_field (str): Field on which to match to determine if object already exists in list hdxobjectclass (type): Type of the HDX Object to be added/updated Returns: None """ new_hdxobjects = self.data.get(hdxobjects_name, list()) """:type : List[HDXObjectUpperBound]""" if new_hdxobjects: hdxobject_names = set() for hdxobject in hdxobjects: hdxobject_name = hdxobject[id_field] hdxobject_names.add(hdxobject_name) for new_hdxobject in new_hdxobjects: if hdxobject_name == new_hdxobject[id_field]: merge_two_dictionaries(hdxobject, new_hdxobject) break for new_hdxobject in new_hdxobjects: if not new_hdxobject[id_field] in hdxobject_names: hdxobjects.append(hdxobjectclass(new_hdxobject, configuration=self.configuration)) del self.data[hdxobjects_name]
def post(url, data, headers, files, allow_redirects, auth): datadict = json.loads(data.decode('utf-8')) if url.endswith('show') or 'list' in url: return mockshow(url, datadict) if 'update' not in url: return MockResponse( 404, '{"success": false, "error": {"message": "TEST ERROR: Not update", "__type": "TEST ERROR: Not Update Error"}, "help": "http://test-data.humdata.org/api/3/action/help_show?name=ckanext_showcase_update"}' ) resultdictcopy = copy.deepcopy(showcase_resultdict) merge_two_dictionaries(resultdictcopy, datadict) result = json.dumps(resultdictcopy) if datadict['title'] == 'MyShowcase1': return MockResponse( 200, '{"success": true, "result": %s, "help": "http://test-data.humdata.org/api/3/action/help_show?name=ckanext_showcase_update"}' % result) if datadict['title'] == 'MyShowcase2': return MockResponse( 404, '{"success": false, "error": {"message": "Not found", "__type": "Not Found Error"}, "help": "http://test-data.humdata.org/api/3/action/help_show?name=ckanext_showcase_update"}' ) if datadict['title'] == 'MyShowcase3': return MockResponse( 200, '{"success": false, "error": {"message": "Not found", "__type": "Not Found Error"}, "help": "http://test-data.humdata.org/api/3/action/help_show?name=ckanext_showcase_update"}' ) return MockResponse( 404, '{"success": false, "error": {"message": "Not found", "__type": "Not Found Error"}, "help": "http://test-data.humdata.org/api/3/action/help_show?name=ckanext_showcase_update"}' )
def post(url, data, headers, files, allow_redirects, auth=None): if isinstance(data, dict): datadict = {k.decode('utf8'): v.decode('utf8') for k, v in data.items()} else: datadict = json.loads(data.decode('utf-8')) if 'show' in url: return mockshow(url, datadict) if 'resource_id' in datadict: if datadict['resource_id'] == '74b74ae1-df0c-4716-829f-4f939a046811': return MockResponse(200, '{"success": true, "result": {"fields": [{"type": "text", "id": "code"}, {"type": "text", "id": "title"}, {"type": "float", "id": "value"}, {"type": "timestamp", "id": "latest_date"}, {"type": "text", "id": "source"}, {"type": "text", "id": "source_link"}, {"type": "text", "id": "notes"}, {"type": "text", "id": "explore"}, {"type": "text", "id": "units"}], "method": "insert", "primary_key": "code", "resource_id": "bfa6b55f-10b6-4ba2-8470-33bb9a5194a5"}, "help": "http://test-data.humdata.org/api/3/action/help_show?name=datastore_create"}') if 'update' not in url: return MockResponse(404, '{"success": false, "error": {"message": "TEST ERROR: Not update", "__type": "TEST ERROR: Not Update Error"}, "help": "http://test-data.humdata.org/api/3/action/help_show?name=resource_update"}') if datadict['name'] == 'MyResource1': resultdictcopy = copy.deepcopy(resultdict) merge_two_dictionaries(resultdictcopy, datadict) if files is not None: resultdictcopy['url_type'] = 'upload' resultdictcopy['resource_type'] = 'file.upload' filename = os.path.basename(files[0][1].name) resultdictcopy[ 'url'] = 'http://test-data.humdata.org/dataset/6f36a41c-f126-4b18-aaaf-6c2ddfbc5d4d/resource/de6549d8-268b-4dfe-adaf-a4ae5c8510d5/download/%s' % filename result = json.dumps(resultdictcopy) return MockResponse(200, '{"success": true, "result": %s, "help": "http://test-data.humdata.org/api/3/action/help_show?name=resource_update"}' % result) if datadict['name'] == 'MyResource2': return MockResponse(404, '{"success": false, "error": {"message": "Not found", "__type": "Not Found Error"}, "help": "http://test-data.humdata.org/api/3/action/help_show?name=resource_update"}') if datadict['name'] == 'MyResource3': return MockResponse(200, '{"success": false, "error": {"message": "Not found", "__type": "Not Found Error"}, "help": "http://test-data.humdata.org/api/3/action/help_show?name=resource_update"}') return MockResponse(404, '{"success": false, "error": {"message": "Not found", "__type": "Not Found Error"}, "help": "http://test-data.humdata.org/api/3/action/help_show?name=resource_update"}')
def post(url, data, headers, files, allow_redirects, auth=None): datadict = json.loads(data.decode('utf-8')) if 'show' in url: return organization_mockshow(url, datadict) if 'update' not in url: return MockResponse( 404, '{"success": false, "error": {"message": "TEST ERROR: Not update", "__type": "TEST ERROR: Not Update Error"}, "help": "http://test-data.humdata.org/api/3/action/help_show?name=organization_update"}' ) resultdictcopy = copy.deepcopy(resultdict) merge_two_dictionaries(resultdictcopy, datadict) result = json.dumps(resultdictcopy) if datadict['name'] == 'MyOrganization1': return MockResponse( 200, '{"success": true, "result": %s, "help": "http://test-data.humdata.org/api/3/action/help_show?name=organization_update"}' % result) if datadict['name'] == 'MyOrganization2': return MockResponse( 404, '{"success": false, "error": {"message": "Not found", "__type": "Not Found Error"}, "help": "http://test-data.humdata.org/api/3/action/help_show?name=organization_update"}' ) if datadict['name'] == 'MyOrganization3': return MockResponse( 200, '{"success": false, "error": {"message": "Not found", "__type": "Not Found Error"}, "help": "http://test-data.humdata.org/api/3/action/help_show?name=organization_update"}' ) return MockResponse( 404, '{"success": false, "error": {"message": "Not found", "__type": "Not Found Error"}, "help": "http://test-data.humdata.org/api/3/action/help_show?name=organization_update"}' )
def post(url, data, headers, files, allow_redirects, auth=None): if isinstance(data, dict): datadict = {k.decode('utf8'): v.decode('utf8') for k, v in data.items()} else: datadict = json.loads(data.decode('utf-8')) if 'show' in url: return mockshow(url, datadict) if 'resource_id' in datadict: if datadict['resource_id'] == '74b74ae1-df0c-4716-829f-4f939a046811': return MockResponse(200, '{"success": true, "result": {"fields": [{"type": "text", "id": "code"}, {"type": "text", "id": "title"}, {"type": "float", "id": "value"}, {"type": "timestamp", "id": "latest_date"}, {"type": "text", "id": "source"}, {"type": "text", "id": "source_link"}, {"type": "text", "id": "notes"}, {"type": "text", "id": "explore"}, {"type": "text", "id": "units"}], "method": "insert", "primary_key": "code", "resource_id": "bfa6b55f-10b6-4ba2-8470-33bb9a5194a5"}, "help": "http://test-data.humdata.org/api/3/action/help_show?name=datastore_create"}') if 'update' not in url: return MockResponse(404, '{"success": false, "error": {"message": "TEST ERROR: Not update", "__type": "TEST ERROR: Not Update Error"}, "help": "http://test-data.humdata.org/api/3/action/help_show?name=resource_update"}') if datadict['name'] == 'MyResource1': resultdictcopy = copy.deepcopy(resultdict) merge_two_dictionaries(resultdictcopy, datadict) if files is not None: resultdictcopy['url_type'] = 'upload' resultdictcopy['resource_type'] = 'file.upload' filename = os.path.basename(files[0][1].name) resultdictcopy[ 'url'] = 'http://test-data.humdata.org/dataset/6f36a41c-f126-4b18-aaaf-6c2ddfbc5d4d/resource/de6549d8-268b-4dfe-adaf-a4ae5c8510d5/download/%s' % filename result = json.dumps(resultdictcopy) return MockResponse(200, '{"success": true, "result": %s, "help": "http://test-data.humdata.org/api/3/action/help_show?name=resource_update"}' % result) if datadict['name'] == 'MyResource2': return MockResponse(404, '{"success": false, "error": {"message": "Not found", "__type": "Not Found Error"}, "help": "http://test-data.humdata.org/api/3/action/help_show?name=resource_update"}') if datadict['name'] == 'MyResource3': return MockResponse(200, '{"success": false, "error": {"message": "Not found", "__type": "Not Found Error"}, "help": "http://test-data.humdata.org/api/3/action/help_show?name=resource_update"}') return MockResponse(404, '{"success": false, "error": {"message": "Not found", "__type": "Not Found Error"}, "help": "http://test-data.humdata.org/api/3/action/help_show?name=resource_update"}')
def post(url, data, headers, files, allow_redirects, auth): if isinstance(data, dict): datadict = { k.decode('utf8'): v.decode('utf8') for k, v in data.items() } else: datadict = json.loads(data.decode('utf-8')) if 'show' in url or 'related_list' in url: return mockshow(url, datadict) if 'related' in url: result = json.dumps(TestDataset.gallerydict) return MockResponse( 200, '{"success": true, "result": %s, "help": "http://test-data.humdata.org/api/3/action/help_show?name=related_create"}' % result) if 'resource' in url: result = json.dumps(TestDataset.resources_data[0]) return MockResponse( 200, '{"success": true, "result": %s, "help": "http://test-data.humdata.org/api/3/action/help_show?name=resource_update"}' % result) else: if 'update' not in url: return MockResponse( 404, '{"success": false, "error": {"message": "TEST ERROR: Not update", "__type": "TEST ERROR: Not Update Error"}, "help": "http://test-data.humdata.org/api/3/action/help_show?name=dataset_update"}' ) resultdictcopy = copy.deepcopy(resultdict) merge_two_dictionaries(resultdictcopy, datadict) for i, resource in enumerate(resultdictcopy['resources']): for j, resource2 in enumerate( resultdictcopy['resources']): if i != j: if resource == resource2: del resultdictcopy['resources'][j] break result = json.dumps(resultdictcopy) if datadict['name'] == 'MyDataset1': return MockResponse( 200, '{"success": true, "result": %s, "help": "http://test-data.humdata.org/api/3/action/help_show?name=dataset_update"}' % result) if datadict['name'] == 'MyDataset2': return MockResponse( 404, '{"success": false, "error": {"message": "Not found", "__type": "Not Found Error"}, "help": "http://test-data.humdata.org/api/3/action/help_show?name=dataset_update"}' ) if datadict['name'] == 'MyDataset3': return MockResponse( 200, '{"success": false, "error": {"message": "Not found", "__type": "Not Found Error"}, "help": "http://test-data.humdata.org/api/3/action/help_show?name=dataset_update"}' ) return MockResponse( 404, '{"success": false, "error": {"message": "Not found", "__type": "Not Found Error"}, "help": "http://test-data.humdata.org/api/3/action/help_show?name=dataset_update"}' )
def create_in_hdx(self): # type: () -> None """Check if dataset exists in HDX and if so, update it, otherwise create it Returns: None """ self.check_required_fields() loadedid = None if 'id' in self.data: if self._dataset_load_from_hdx(self.data['id']): loadedid = self.data['id'] else: logger.warning('Failed to load dataset with id %s' % self.data['id']) if not loadedid: if self._dataset_load_from_hdx(self.data['name']): loadedid = self.data['name'] if loadedid: logger.warning('Dataset exists. Updating %s' % loadedid) self._dataset_merge_hdx_update(True, True) return filestore_resources = list() if self.resources: ignore_fields = [self.configuration['resource']['dataset_id']] for resource in self.resources: resource.check_required_fields(ignore_fields=ignore_fields) if resource.get_file_to_upload(): filestore_resources.append(resource) self.data['resources'] = self._convert_hdxobjects(self.resources) self._save_to_hdx('create', 'name') for resource in filestore_resources: for created_resource in self.data['resources']: if resource['name'] == created_resource['name']: merge_two_dictionaries(resource.data, created_resource) resource.update_in_hdx() merge_two_dictionaries(created_resource, resource.data) break self.init_resources() self.separate_resources() if self.include_gallery: self.old_data['gallery'] = self._copy_hdxobjects( self.gallery, GalleryItem) galleryitem_dataset_id = self.configuration['galleryitem'][ 'dataset_id'] for i, galleryitem in enumerate(self.gallery): galleryitem[galleryitem_dataset_id] = self.data['id'] galleryitem.check_required_fields() galleryitem.create_in_hdx()
def update_in_hdx(self, **kwargs): # type: (Any) -> None """Check if showcase exists in HDX and if so, update it Returns: None """ self._check_load_existing_object('showcase', 'name') # We load an existing object even though it may well have been loaded already # to prevent an admittedly unlikely race condition where someone has updated # the object in the intervening time merge_two_dictionaries(self.data, self.old_data) self.clean_tags() self._hdx_update('showcase', 'name', force_active=True, **kwargs) self._update_in_hdx('showcase', 'name', **kwargs)
def resource_view_mockcreate(url, datadict): if 'create' not in url: return MockResponse( 404, '{"success": false, "error": {"message": "TEST ERROR: Not create", "__type": "TEST ERROR: Not Create Error"}, "help": "http://test-data.humdata.org/api/3/action/help_show?name=resource_view_create"}' ) if datadict['title'] == 'A Preview': result = json.dumps(resultdict) return MockResponse( 200, '{"success": true, "result": %s, "help": "http://test-data.humdata.org/api/3/action/help_show?name=resource_view_create"}' % result) if datadict['title'] == 'Quick Charts': resultdictcopy = copy.deepcopy(resultdict) result = json.dumps(merge_two_dictionaries(resultdictcopy, datadict)) return MockResponse( 200, '{"success": true, "result": %s, "help": "http://test-data.humdata.org/api/3/action/help_show?name=resource_view_create"}' % result) if datadict['title'] == 'XXX': return MockResponse( 404, '{"success": false, "error": {"message": "Not found", "__type": "Not Found Error"}, "help": "http://test-data.humdata.org/api/3/action/help_show?name=resource_view_create"}' ) if datadict['title'] == 'YYY': return MockResponse( 200, '{"success": false, "error": {"message": "Not found", "__type": "Not Found Error"}, "help": "http://test-data.humdata.org/api/3/action/help_show?name=resource_view_create"}' ) return MockResponse( 404, '{"success": false, "error": {"message": "Not found", "__type": "Not Found Error"}, "help": "http://test-data.humdata.org/api/3/action/help_show?name=resource_view_create"}' )
def _merge_hdx_update(self, object_type, id_field_name, file_to_upload=None): # type: (str, str, Optional[str]) -> None """Helper method to check if HDX object exists and update it Args: object_type (str): Description of HDX object type (for messages) id_field_name (str): Name of field containing HDX object identifier file_to_upload (Optional[str]): File to upload to HDX Returns: None """ merge_two_dictionaries(self.data, self.old_data) ignore_dataset_id = Configuration.read()['%s' % object_type].get('ignore_dataset_id_on_update', False) self.check_required_fields(ignore_dataset_id=ignore_dataset_id) self._save_to_hdx('update', id_field_name, file_to_upload)
def _merge_hdx_update(self, object_type, id_field_name, file_to_upload=None, force_active=False, **kwargs): # type: (str, str, Optional[str], bool, Any) -> None """Helper method to check if HDX object exists and update it Args: object_type (str): Description of HDX object type (for messages) id_field_name (str): Name of field containing HDX object identifier file_to_upload (Optional[str]): File to upload to HDX force_active (bool): Make object state active. Defaults to False. **kwargs: See below operation (string): Operation to perform eg. patch. Defaults to update. ignore_field (string): Any field to ignore when checking dataset metadata. Defaults to None. Returns: None """ merge_two_dictionaries(self.data, self.old_data) self._hdx_update(object_type, id_field_name, file_to_upload=file_to_upload, force_active=force_active, **kwargs)
def create_in_hdx(self, allow_no_resources=False): # type: (Optional[bool]) -> None """Check if dataset exists in HDX and if so, update it, otherwise create it Args: allow_no_resources (Optional[bool]): Whether to allow no resources. Defaults to False. Returns: None """ self.check_required_fields(allow_no_resources=allow_no_resources) loadedid = None if 'id' in self.data: if self._dataset_load_from_hdx(self.data['id']): loadedid = self.data['id'] else: logger.warning('Failed to load dataset with id %s' % self.data['id']) if not loadedid: if self._dataset_load_from_hdx(self.data['name']): loadedid = self.data['name'] if loadedid: logger.warning('Dataset exists. Updating %s' % loadedid) self._dataset_merge_hdx_update(True) return filestore_resources = list() if self.resources: ignore_fields = ['package_id'] for resource in self.resources: resource.check_required_fields(ignore_fields=ignore_fields) if resource.get_file_to_upload(): filestore_resources.append(resource) self.data['resources'] = self._convert_hdxobjects(self.resources) self._save_to_hdx('create', 'name') for resource in filestore_resources: for created_resource in self.data['resources']: if resource['name'] == created_resource['name']: merge_two_dictionaries(resource.data, created_resource) resource.update_in_hdx() merge_two_dictionaries(created_resource, resource.data) break self.init_resources() self.separate_resources()
def _addupdate_hdxobject(self, hdxobjects, id_field, new_hdxobject): # type: (List[HDXObjectUpperBound], str, HDXObjectUpperBound) -> HDXObjectUpperBound """Helper function to add a new HDX object to a supplied list of HDX objects or update existing metadata if the object already exists in the list Args: hdxobjects (List[T <= HDXObject]): list of HDX objects to which to add new objects or update existing ones id_field (str): Field on which to match to determine if object already exists in list new_hdxobject (T <= HDXObject): The HDX object to be added/updated Returns: T <= HDXObject: The HDX object which was added or updated """ for hdxobject in hdxobjects: if hdxobject[id_field] == new_hdxobject[id_field]: merge_two_dictionaries(hdxobject, new_hdxobject) return hdxobject hdxobjects.append(new_hdxobject) return new_hdxobject
def create_in_hdx(self, **kwargs): # type: (Any) -> None """Check if showcase exists in HDX and if so, update it, otherwise create it Returns: None """ if 'ignore_check' not in kwargs: # allow ignoring of field checks self.check_required_fields() if 'name' in self.data and self._load_from_hdx('showcase', self.data['name']): logger.warning('%s exists. Updating %s' % ('showcase', self.data['name'])) merge_two_dictionaries(self.data, self.old_data) self.clean_tags() self._hdx_update('showcase', 'name', force_active=True, **kwargs) else: self.clean_tags() self._save_to_hdx('create', 'title', force_active=True) self._create_in_hdx('showcase', 'name', 'title', **kwargs)
def add_filestore_resources(resources_list, filestore_resources): # type: (List[Dict], List[hdx.data.resource.Resource]) -> None """Helper method to create files in filestore by updating resources. Args: resources_list (List[Dict]): List of resources from dataset dict's resources key filestore_resources (List[hdx.data.resource.Resource]): List of resources that use filestore (to be appended to) Returns: None """ for resource in filestore_resources: for created_resource in resources_list: if resource['name'] == created_resource['name']: merge_two_dictionaries(resource.data, created_resource) del resource['url'] resource.update_in_hdx() merge_two_dictionaries(created_resource, resource.data) break
def _dataset_merge_hdx_update(self, update_resources): # type: (bool) -> None """Helper method to check if dataset or its resources exist and update them Args: update_resources (bool): Whether to update resources Returns: None """ merge_two_dictionaries(self.data, self.old_data) if 'resources' in self.data: del self.data['resources'] old_resources = self.old_data.get('resources', None) filestore_resources = list() if update_resources and old_resources: ignore_fields = ['package_id'] resource_names = set() for resource in self.resources: resource_name = resource['name'] resource_names.add(resource_name) for old_resource in old_resources: if resource_name == old_resource['name']: logger.warning('Resource exists. Updating %s' % resource_name) merge_two_dictionaries(resource, old_resource) if old_resource.get_file_to_upload(): resource.set_file_to_upload( old_resource.get_file_to_upload()) filestore_resources.append(resource) resource.check_required_fields( ignore_fields=ignore_fields) break for old_resource in old_resources: if not old_resource['name'] in resource_names: old_resource.check_required_fields( ignore_fields=ignore_fields) self.resources.append(old_resource) if old_resource.get_file_to_upload(): filestore_resources.append(old_resource) if self.resources: self.data['resources'] = self._convert_hdxobjects(self.resources) self._save_to_hdx('update', 'id') for resource in filestore_resources: for created_resource in self.data['resources']: if resource['name'] == created_resource['name']: merge_two_dictionaries(resource.data, created_resource) resource.update_in_hdx() merge_two_dictionaries(created_resource, resource.data) break
def load_json_into_existing_dict(data: dict, path: str) -> dict: """Merge JSON file into existing dictionary Args: data (dict): Dictionary to merge into path (str): JSON file to load and merge Returns: dict: JSON file merged into dictionary """ jsondict = load_json(path) return merge_two_dictionaries(data, jsondict)
def load_yaml_into_existing_dict(data: dict, path: str) -> dict: """Merge YAML file into existing dictionary Args: data (dict): Dictionary to merge into path (str): YAML file to load and merge Returns: dict: YAML file merged into dictionary """ yamldict = load_yaml(path) return merge_two_dictionaries(data, yamldict)
def dataset_merge_filestore_resource(cls, resource, updated_resource, filestore_resources, ignore_fields): # type: (hdx.data.resource.Resource, hdx.data.resource.Resource, List[hdx.data.resource.Resource], List[str]) -> None """Helper method to merge updated resource from dataset into HDX resource read from HDX including filestore. Args: resource (hdx.data.resource.Resource): Resource read from HDX updated_resource (hdx.data.resource.Resource): Updated resource from dataset filestore_resources (List[hdx.data.resource.Resource]): List of resources that use filestore (to be appended to) ignore_fields (List[str]): List of fields to ignore when checking resource Returns: None """ if updated_resource.get_file_to_upload(): resource.set_file_to_upload(updated_resource.get_file_to_upload()) filestore_resources.append(resource) merge_two_dictionaries(resource, updated_resource) resource.check_required_fields(ignore_fields=ignore_fields) if resource.get_file_to_upload(): resource['url'] = cls.temporary_url
def post(url, data, headers, files, allow_redirects, auth=None): datadict = json.loads(data.decode('utf-8')) if 'show' in url: return organization_mockshow(url, datadict) if 'update' not in url: return MockResponse(404, '{"success": false, "error": {"message": "TEST ERROR: Not update", "__type": "TEST ERROR: Not Update Error"}, "help": "http://test-data.humdata.org/api/3/action/help_show?name=organization_update"}') resultdictcopy = copy.deepcopy(resultdict) merge_two_dictionaries(resultdictcopy, datadict) result = json.dumps(resultdictcopy) if datadict['name'] == 'MyOrganization1': return MockResponse(200, '{"success": true, "result": %s, "help": "http://test-data.humdata.org/api/3/action/help_show?name=organization_update"}' % result) if datadict['name'] == 'MyOrganization2': return MockResponse(404, '{"success": false, "error": {"message": "Not found", "__type": "Not Found Error"}, "help": "http://test-data.humdata.org/api/3/action/help_show?name=organization_update"}') if datadict['name'] == 'MyOrganization3': return MockResponse(200, '{"success": false, "error": {"message": "Not found", "__type": "Not Found Error"}, "help": "http://test-data.humdata.org/api/3/action/help_show?name=organization_update"}') return MockResponse(404, '{"success": false, "error": {"message": "Not found", "__type": "Not Found Error"}, "help": "http://test-data.humdata.org/api/3/action/help_show?name=organization_update"}')
def load_json_into_existing_dict(data: dict, path: str, encoding: str = "utf-8") -> Dict: """Merge JSON file into existing dictionary Args: data (dict): Dictionary to merge into path (str): JSON file to load and merge encoding (str): Encoding of file. Defaults to utf-8. Returns: dict: JSON file merged into dictionary """ jsondict = load_json(path, encoding=encoding) return merge_two_dictionaries(data, jsondict)
def _dataset_merge_hdx_update(self, update_resources, update_gallery): # type: (bool, bool) -> None """Helper method to check if dataset or its resources or gallery items exist and update them Args: update_resources (bool): Whether to update resources update_gallery (bool): Whether to update gallery Returns: None """ merge_two_dictionaries(self.data, self.old_data) if 'resources' in self.data: del self.data['resources'] if 'gallery' in self.data: del self.data['gallery'] old_resources = self.old_data.get('resources', None) filestore_resources = list() if update_resources and old_resources: ignore_fields = [self.configuration['resource']['dataset_id']] resource_names = set() for resource in self.resources: resource_name = resource['name'] resource_names.add(resource_name) for old_resource in old_resources: if resource_name == old_resource['name']: logger.warning('Resource exists. Updating %s' % resource_name) merge_two_dictionaries(resource, old_resource) if old_resource.get_file_to_upload(): resource.set_file_to_upload( old_resource.get_file_to_upload()) filestore_resources.append(resource) resource.check_required_fields( ignore_fields=ignore_fields) break for old_resource in old_resources: if not old_resource['name'] in resource_names: old_resource.check_required_fields( ignore_fields=ignore_fields) self.resources.append(old_resource) if old_resource.get_file_to_upload(): filestore_resources.append(old_resource) old_gallery = self.old_data.get('gallery', None) if self.resources: self.data['resources'] = self._convert_hdxobjects(self.resources) self._save_to_hdx('update', 'id') for resource in filestore_resources: for created_resource in self.data['resources']: if resource['name'] == created_resource['name']: merge_two_dictionaries(resource.data, created_resource) resource.update_in_hdx() merge_two_dictionaries(created_resource, resource.data) break if self.include_gallery and update_gallery and old_gallery: self.old_data['gallery'] = self._copy_hdxobjects( self.gallery, GalleryItem) ignore_fields = [self.configuration['galleryitem']['dataset_id']] galleryitem_titles = set() galleryitem_dataset_id = self.configuration['galleryitem'][ 'dataset_id'] for i, galleryitem in enumerate(self.gallery): galleryitem_title = galleryitem['title'] galleryitem_titles.add(galleryitem_title) for old_galleryitem in old_gallery: if galleryitem_title == old_galleryitem['title']: logger.warning('Gallery item exists. Updating %s' % galleryitem_title) merge_two_dictionaries(galleryitem, old_galleryitem) galleryitem.check_required_fields( ignore_fields=ignore_fields) galleryitem.update_in_hdx() for old_galleryitem in old_gallery: if not old_galleryitem['title'] in galleryitem_titles: old_galleryitem[galleryitem_dataset_id] = self.data['id'] old_galleryitem.check_required_fields() old_galleryitem.create_in_hdx() self.gallery.append(old_galleryitem)
def __init__(self, **kwargs): # type: (Any) -> None super(Configuration, self).__init__() self._remoteckan = None self._emailer = None hdx_base_config_found = False hdx_base_config_dict = kwargs.get('hdx_base_config_dict', None) if hdx_base_config_dict: hdx_base_config_found = True logger.info('Loading HDX base configuration from dictionary') hdx_base_config_json = kwargs.get('hdx_base_config_json', '') if hdx_base_config_json: if hdx_base_config_found: raise ConfigurationError('More than one HDX base configuration given!') hdx_base_config_found = True logger.info('Loading HDX base configuration from: %s' % hdx_base_config_json) hdx_base_config_dict = load_json(hdx_base_config_json) hdx_base_config_yaml = kwargs.get('hdx_base_config_yaml', '') if hdx_base_config_found: if hdx_base_config_yaml: raise ConfigurationError('More than one HDX base configuration given!') else: if not hdx_base_config_yaml: hdx_base_config_yaml = Configuration.default_hdx_base_config_yaml logger.info('No HDX base configuration parameter. Using default base configuration file: %s.' % hdx_base_config_yaml) logger.info('Loading HDX base configuration from: %s' % hdx_base_config_yaml) hdx_base_config_dict = load_yaml(hdx_base_config_yaml) hdx_config_found = False hdx_config_dict = kwargs.get('hdx_config_dict', None) if hdx_config_dict: hdx_config_found = True logger.info('Loading HDX configuration from dictionary') hdx_config_json = kwargs.get('hdx_config_json', '') if hdx_config_json: if hdx_config_found: raise ConfigurationError('More than one HDX configuration given!') hdx_config_found = True logger.info('Loading HDX configuration from: %s' % hdx_config_json) hdx_config_dict = load_json(hdx_config_json) hdx_config_yaml = kwargs.get('hdx_config_yaml', '') if hdx_config_found: if hdx_config_yaml: raise ConfigurationError('More than one HDX configuration given!') else: if not hdx_config_yaml: hdx_config_yaml = Configuration.default_hdx_config_yaml if isfile(hdx_config_yaml): logger.info('No HDX configuration parameter. Using default configuration file: %s.' % hdx_config_yaml) else: logger.info('No HDX configuration parameter and no configuration file at default path: %s.' % hdx_config_yaml) hdx_config_yaml = None hdx_config_dict = dict() if hdx_config_yaml: logger.info('Loading HDX configuration from: %s' % hdx_config_yaml) hdx_config_dict = load_yaml(hdx_config_yaml) self.data = merge_two_dictionaries(hdx_base_config_dict, hdx_config_dict) project_config_found = False project_config_dict = kwargs.get('project_config_dict', None) if project_config_dict is not None: project_config_found = True logger.info('Loading project configuration from dictionary') project_config_json = kwargs.get('project_config_json', '') if project_config_json: if project_config_found: raise ConfigurationError('More than one project configuration given!') project_config_found = True logger.info('Loading project configuration from: %s' % project_config_json) project_config_dict = load_json(project_config_json) project_config_yaml = kwargs.get('project_config_yaml', '') if project_config_found: if project_config_yaml: raise ConfigurationError('More than one project configuration given!') else: if project_config_yaml: logger.info('Loading project configuration from: %s' % project_config_yaml) project_config_dict = load_yaml(project_config_yaml) else: project_config_dict = dict() self.data = merge_two_dictionaries(hdx_base_config_dict, project_config_dict) ua = kwargs.get('full_agent') if ua: self.user_agent = ua else: try: self.user_agent = UserAgent.get(prefix=Configuration.prefix, **kwargs) except UserAgentError: self.user_agent = UserAgent.get(prefix=Configuration.prefix, **self.data) self.hdx_read_only = kwargs.get('hdx_read_only', self.data.get('hdx_read_only', False)) logger.info('Read only access to HDX: %s' % str(self.hdx_read_only)) self.hdx_key = kwargs.get('hdx_key', self.data.get('hdx_key')) if not self.hdx_key and not self.hdx_read_only: raise ConfigurationError('No HDX API key supplied as a parameter or in configuration!') hdx_url = kwargs.get('hdx_url', self.data.get('hdx_url')) if hdx_url: self.hdx_site = 'hdx_custom_site' self.data[self.hdx_site] = {'url': hdx_url} else: self.hdx_site = 'hdx_%s_site' % kwargs.get('hdx_site', self.data.get('hdx_site', 'test')) if self.hdx_site not in self.data: raise ConfigurationError('%s not defined in configuration!' % self.hdx_site)
def __init__(self, **kwargs): # type: (Any) -> None super(Configuration, self).__init__() self._remoteckan = None self._emailer = None hdx_base_config_found = False hdx_base_config_dict = kwargs.get('hdx_base_config_dict', None) if hdx_base_config_dict: hdx_base_config_found = True logger.info('Loading HDX base configuration from dictionary') hdx_base_config_json = kwargs.get('hdx_base_config_json', '') if hdx_base_config_json: if hdx_base_config_found: raise ConfigurationError( 'More than one HDX base configuration given!') hdx_base_config_found = True logger.info('Loading HDX base configuration from: %s' % hdx_base_config_json) hdx_base_config_dict = load_json(hdx_base_config_json) hdx_base_config_yaml = kwargs.get('hdx_base_config_yaml', '') if hdx_base_config_found: if hdx_base_config_yaml: raise ConfigurationError( 'More than one HDX base configuration given!') else: if not hdx_base_config_yaml: hdx_base_config_yaml = Configuration.default_hdx_base_config_yaml logger.info( 'No HDX base configuration parameter. Using default base configuration file: %s.' % hdx_base_config_yaml) logger.info('Loading HDX base configuration from: %s' % hdx_base_config_yaml) hdx_base_config_dict = load_yaml(hdx_base_config_yaml) hdx_config_found = False hdx_config_dict = kwargs.get('hdx_config_dict', None) if hdx_config_dict: hdx_config_found = True logger.info('Loading HDX configuration from dictionary') hdx_config_json = kwargs.get('hdx_config_json', '') if hdx_config_json: if hdx_config_found: raise ConfigurationError( 'More than one HDX configuration given!') hdx_config_found = True logger.info('Loading HDX configuration from: %s' % hdx_config_json) hdx_config_dict = load_json(hdx_config_json) hdx_config_yaml = kwargs.get('hdx_config_yaml', '') if hdx_config_found: if hdx_config_yaml: raise ConfigurationError( 'More than one HDX configuration given!') else: if not hdx_config_yaml: hdx_config_yaml = Configuration.default_hdx_config_yaml if isfile(hdx_config_yaml): logger.info( 'No HDX configuration parameter. Using default configuration file: %s.' % hdx_config_yaml) else: logger.info( 'No HDX configuration parameter and no configuration file at default path: %s.' % hdx_config_yaml) hdx_config_yaml = None hdx_config_dict = dict() if hdx_config_yaml: logger.info('Loading HDX configuration from: %s' % hdx_config_yaml) hdx_config_dict = load_yaml(hdx_config_yaml) self.data = merge_two_dictionaries(hdx_base_config_dict, hdx_config_dict) project_config_found = False project_config_dict = kwargs.get('project_config_dict', None) if project_config_dict is not None: project_config_found = True logger.info('Loading project configuration from dictionary') project_config_json = kwargs.get('project_config_json', '') if project_config_json: if project_config_found: raise ConfigurationError( 'More than one project configuration given!') project_config_found = True logger.info('Loading project configuration from: %s' % project_config_json) project_config_dict = load_json(project_config_json) project_config_yaml = kwargs.get('project_config_yaml', '') if project_config_found: if project_config_yaml: raise ConfigurationError( 'More than one project configuration given!') else: if project_config_yaml: logger.info('Loading project configuration from: %s' % project_config_yaml) project_config_dict = load_yaml(project_config_yaml) else: project_config_dict = dict() self.data = merge_two_dictionaries(hdx_base_config_dict, project_config_dict) ua = kwargs.get('full_agent') if ua: self.user_agent = ua else: try: self.user_agent = UserAgent.get(prefix=Configuration.prefix, **kwargs) except UserAgentError: self.user_agent = UserAgent.get(prefix=Configuration.prefix, **self.data) self.hdx_read_only = kwargs.get('hdx_read_only', self.data.get('hdx_read_only', False)) logger.info('Read only access to HDX: %s' % str(self.hdx_read_only)) self.hdx_key = kwargs.get('hdx_key', self.data.get('hdx_key')) if not self.hdx_key and not self.hdx_read_only: raise ConfigurationError( 'No HDX API key supplied as a parameter or in configuration!') hdx_url = kwargs.get('hdx_url', self.data.get('hdx_url')) if hdx_url: self.hdx_site = 'hdx_custom_site' hdx_url = hdx_url.rstrip('/') self.data[self.hdx_site] = {'url': hdx_url} else: self.hdx_site = 'hdx_%s_site' % kwargs.get( 'hdx_site', self.data.get('hdx_site', 'test')) if self.hdx_site not in self.data: raise ConfigurationError('%s not defined in configuration!' % self.hdx_site)
def __init__(self, **kwargs): # type: (Any) -> None super(Configuration, self).__init__() hdx_config_found = False hdx_config_dict = kwargs.get('hdx_config_dict', None) if hdx_config_dict: hdx_config_found = True logger.info('Loading HDX configuration from dictionary') hdx_config_json = kwargs.get('hdx_config_json', '') if hdx_config_json: if hdx_config_found: raise ConfigurationError( 'More than one HDX configuration file given!') hdx_config_found = True logger.info('Loading HDX configuration from: %s' % hdx_config_json) hdx_config_dict = load_json(hdx_config_json) hdx_config_yaml = kwargs.get('hdx_config_yaml', '') if hdx_config_found: if hdx_config_yaml: raise ConfigurationError( 'More than one HDX configuration file given!') else: if not hdx_config_yaml: logger.info('No HDX configuration parameter. Using default.') hdx_config_yaml = script_dir_plus_file('hdx_configuration.yml', Configuration) logger.info('Loading HDX configuration from: %s' % hdx_config_yaml) hdx_config_dict = load_yaml(hdx_config_yaml) project_config_found = False project_config_dict = kwargs.get('project_config_dict', None) if project_config_dict is not None: project_config_found = True logger.info('Loading project configuration from dictionary') project_config_json = kwargs.get('project_config_json', '') if project_config_json: if project_config_found: raise ConfigurationError( 'More than one project configuration file given!') project_config_found = True logger.info('Loading project configuration from: %s' % project_config_json) project_config_dict = load_json(project_config_json) project_config_yaml = kwargs.get('project_config_yaml', '') if project_config_found: if project_config_yaml: raise ConfigurationError( 'More than one project configuration file given!') else: if project_config_yaml: logger.info('Loading project configuration from: %s' % project_config_yaml) project_config_dict = load_yaml(project_config_yaml) else: project_config_dict = dict() self.data = merge_two_dictionaries(hdx_config_dict, project_config_dict) self.hdx_read_only = kwargs.get('hdx_read_only', False) if not self.hdx_read_only: if 'hdx_key' in kwargs: self.data['api_key'] = kwargs.get('hdx_key') else: hdx_key_file = kwargs.get('hdx_key_file', join(expanduser('~'), '.hdxkey')) """ :type : str""" self.data['api_key'] = self.load_api_key(hdx_key_file) self.hdx_site = 'hdx_%s_site' % kwargs.get('hdx_site', 'test') if self.hdx_site not in self.data: raise ConfigurationError('%s not defined in configuration!' % self.hdx_site)