def add_update_resource(self, resource, ignore_datasetid=False):
# type: (Union[Resource,dict,str], Optional[bool]) -> None
"""Add new or update existing resource in dataset with new metadata
Args:
resource (Union[Resource,dict,str]): Either resource id or resource metadata from a Resource object or a dictionary
ignore_datasetid (Optional[bool]): Whether to ignore dataset id in the resource
Returns:
None
"""
if isinstance(resource, str):
resource = Resource.read_from_hdx(resource,
configuration=self.configuration)
elif isinstance(resource, dict):
resource = Resource(resource, configuration=self.configuration)
if isinstance(resource, Resource):
if 'package_id' in resource:
if not ignore_datasetid:
raise HDXError(
'Resource %s being added already has a dataset id!' %
(resource['name']))
resource_updated = self._addupdate_hdxobject(
self.resources, 'name', resource)
resource_updated.set_file_to_upload(resource.get_file_to_upload())
return
raise HDXError('Type %s cannot be added as a resource!' %
type(resource).__name__)
def test_search_in_hdx(self, configuration, search):
resources = Resource.search_in_hdx('name:ACLED')
assert len(resources) == 4
resources = Resource.search_in_hdx('name:ajyhgr')
assert len(resources) == 0
with pytest.raises(HDXError):
Resource.search_in_hdx('fail')
def test_search_in_hdx(self, configuration, search):
resources = Resource.search_in_hdx('name:ACLED')
assert len(resources) == 4
resources = Resource.search_in_hdx('name:ajyhgr')
assert len(resources) == 0
with pytest.raises(HDXError):
Resource.search_in_hdx('fail')
def test_datastore(self, configuration, post_datastore, topline_yaml,
topline_json):
resource = Resource.read_from_hdx('TEST1')
resource2 = Resource.read_from_hdx('TEST5')
TestResource.datastore = None
resource.create_datastore(delete_first=0)
assert TestResource.datastore == 'create'
TestResource.datastore = None
resource.create_datastore(delete_first=1)
assert TestResource.datastore == 'create'
TestResource.datastore = None
resource.create_datastore(delete_first=2)
assert TestResource.datastore == 'create'
TestResource.datastore = None
with pytest.raises(HDXError):
resource.create_datastore(delete_first=3)
resource.update_datastore()
assert TestResource.datastore == 'create'
TestResource.datastore = None
resource.update_datastore_for_topline()
assert TestResource.datastore == 'create'
TestResource.datastore = None
resource.update_datastore_from_dict_schema({
"schema": [
{
"id": "code",
"type": "text"
},
],
"primary_key":
"code"
})
assert TestResource.datastore == 'create'
TestResource.datastore = None
resource.update_datastore_from_yaml_schema(topline_yaml)
assert TestResource.datastore == 'create'
TestResource.datastore = None
filefordatastore = join('tests', 'fixtures', 'test_data.csv')
resource.update_datastore_from_json_schema(topline_json,
path=filefordatastore)
assert TestResource.datastore == 'create'
TestResource.datastore = None
filefordatastore = join('tests', 'fixtures', 'test_data.zip')
resource.update_datastore_from_json_schema(topline_json,
path=filefordatastore)
assert TestResource.datastore == 'create'
TestResource.datastore = None
filefordatastore = join(
'tests', 'fixtures', 'datastore',
'ACLED-All-Africa-File_20170101-to-20170708.xlsx')
resource.update_datastore(path=filefordatastore)
assert TestResource.datastore == 'create'
with pytest.raises(HDXError):
resource2.update_datastore_from_json_schema(topline_json)
resource.delete_datastore()
assert TestResource.datastore == 'delete'
TestResource.datastore = None
with pytest.raises(HDXError):
del resource['url']
resource.create_datastore()
def generate_dataset(configuration):
url = configuration['base_url'] + configuration['api']
loaData.writeData(url)
name = 'Africa health facilities'
title = 'Africa health facilities data'
slugified_name = slugify(name).lower()
dataset = Dataset(configuration, {})
dataset['name'] = slugified_name
dataset['title'] = title
date = time.strftime("%d/%m/%Y")
dataset['dataset_date'] = date
dataset.add_continent_location('AF')
rName = "sen-healthfacilities"
resource = Resource()
resource['name'] = rName
resource['format'] = 'geojson'
resource['url'] = url
resource['description'] = configuration['base_url']
resource['url_type'] = 'api'
resource['resource_type'] = 'api'
resource.set_file_to_upload(configuration['data_folder'] +
'sen-healthfacilities.geojson')
dataset.add_update_resource(resource)
return dataset
def test_update_in_hdx(self, configuration, post_update):
dataset = Dataset()
dataset['id'] = 'NOTEXIST'
with pytest.raises(HDXError):
dataset.update_in_hdx()
dataset['name'] = 'LALA'
with pytest.raises(HDXError):
dataset.update_in_hdx()
dataset = Dataset.read_from_hdx('TEST1')
assert dataset['id'] == '6f36a41c-f126-4b18-aaaf-6c2ddfbc5d4d'
assert dataset['dataset_date'] == '06/04/2016'
dataset['dataset_date'] = '02/26/2016'
dataset['id'] = 'TEST1'
dataset['name'] = 'MyDataset1'
dataset.update_in_hdx()
assert dataset['id'] == 'TEST1'
assert dataset['dataset_date'] == '02/26/2016'
dataset['id'] = 'NOTEXIST'
with pytest.raises(HDXError):
dataset.update_in_hdx()
del dataset['id']
with pytest.raises(HDXError):
dataset.update_in_hdx()
dataset_data = copy.deepcopy(TestDataset.dataset_data)
gallery_data = copy.deepcopy(TestDataset.gallery_data)
dataset_data['name'] = 'MyDataset1'
dataset_data['id'] = 'TEST1'
dataset = Dataset(dataset_data)
dataset.add_update_gallery(gallery_data)
dataset.create_in_hdx()
assert dataset['id'] == 'TEST1'
assert dataset['dataset_date'] == '03/23/2016'
assert len(dataset.resources) == 2
assert len(dataset.gallery) == 1
dataset.update_in_hdx()
assert len(dataset.resources) == 2
assert len(dataset.gallery) == 1
dataset = Dataset.read_from_hdx('TEST4')
del gallery_data[0]['id']
dataset.add_update_gallery(gallery_data)
dataset['id'] = 'TEST4'
dataset.update_in_hdx()
assert len(dataset.resources) == 2
assert len(dataset.gallery) == 1
dataset = Dataset.read_from_hdx('TEST4')
resources_data = copy.deepcopy(TestDataset.resources_data)
resource = Resource(resources_data[0])
file = tempfile.NamedTemporaryFile(delete=False)
resource.set_file_to_upload(file.name)
dataset.add_update_resource(resource)
dataset.update_in_hdx()
os.unlink(file.name)
assert len(dataset.resources) == 2
assert len(dataset.gallery) == 0
def test_get_dataset(self, configuration, post_dataset):
resource_data = copy.deepcopy(TestResource.resource_data)
resource = Resource(resource_data)
dataset = resource.get_dataset()
assert dataset['id'] == '6f36a41c-f126-4b18-aaaf-6c2ddfbc5d4d'
del resource['package_id']
with pytest.raises(HDXError):
resource.get_dataset()
def test_update_json(self, configuration, static_json):
resource_data = copy.deepcopy(TestResource.resource_data)
resource = Resource(resource_data)
assert resource['name'] == 'MyResource1'
assert resource['format'] == 'xlsx'
resource.update_from_json(static_json)
assert resource['name'] == 'MyResource1'
assert resource['format'] == 'zipped csv'
def test_get_dataset(self, configuration, post_dataset):
resource_data = copy.deepcopy(TestResource.resource_data)
resource = Resource(resource_data)
dataset = resource.get_dataset()
assert dataset['id'] == '6f36a41c-f126-4b18-aaaf-6c2ddfbc5d4d'
del resource['package_id']
with pytest.raises(HDXError):
resource.get_dataset()
def test_update_yaml(self, configuration, static_yaml):
resource_data = copy.deepcopy(TestResource.resource_data)
resource = Resource(configuration, resource_data)
assert resource['name'] == 'MyResource1'
assert resource['format'] == 'xlsx'
resource.update_from_yaml(static_yaml)
assert resource['name'] == 'MyResource1'
assert resource['format'] == 'csv'
def test_update_json(self, configuration, static_json):
resource_data = copy.deepcopy(TestResource.resource_data)
resource = Resource(resource_data)
assert resource['name'] == 'MyResource1'
assert resource.get_file_type() == 'xlsx'
resource.update_from_json(static_json)
assert resource['name'] == 'MyResource1'
assert resource.get_file_type() == 'zipped csv'
def test_update_json(self, configuration, static_json):
resource_data = copy.deepcopy(TestResource.resource_data)
resource = Resource(configuration, resource_data)
assert resource['name'] == 'MyResource1'
assert resource['format'] == 'xlsx'
resource.update_json(static_json)
assert resource['name'] == 'MyResource1'
assert resource['format'] == 'zipped csv'
def test_read_from_hdx(self, configuration, read):
resource = Resource.read_from_hdx('TEST1')
assert resource['id'] == 'de6549d8-268b-4dfe-adaf-a4ae5c8510d5'
assert resource['name'] == 'MyResource1'
assert resource['package_id'] == '6f36a41c-f126-4b18-aaaf-6c2ddfbc5d4d'
resource = Resource.read_from_hdx('TEST2')
assert resource is None
resource = Resource.read_from_hdx('TEST3')
assert resource is None
def test_read_from_hdx(self, configuration, read):
resource = Resource.read_from_hdx(configuration, 'TEST1')
assert resource['id'] == 'de6549d8-268b-4dfe-adaf-a4ae5c8510d5'
assert resource['name'] == 'MyResource1'
assert resource['package_id'] == '6f36a41c-f126-4b18-aaaf-6c2ddfbc5d4d'
resource = Resource.read_from_hdx(configuration, 'TEST2')
assert resource is None
resource = Resource.read_from_hdx(configuration, 'TEST3')
assert resource is None
def test_check_url_filetoupload(self, configuration):
resource_data = copy.deepcopy(TestResource.resource_data)
resource = Resource(resource_data)
resource.check_url_filetoupload()
resource.set_file_to_upload('abc')
resource.check_url_filetoupload()
resource['url'] = 'lala'
with pytest.raises(HDXError):
resource.check_url_filetoupload()
def test_download(self, configuration, read):
resource = Resource.read_from_hdx(configuration, 'TEST1')
url, path = resource.download()
unlink(path)
assert url == 'https://raw.githubusercontent.com/OCHA-DAP/hdx-python-api/master/tests/fixtures/test_data.csv'
resource['url'] = ''
with pytest.raises(HDXError):
resource.download()
resource = Resource.read_from_hdx(configuration, 'TEST4')
with pytest.raises(DownloadError):
resource.download()
def test_read_from_hdx(self, configuration, read):
resource = Resource.read_from_hdx('74b74ae1-df0c-4716-829f-4f939a046811')
assert resource['id'] == 'de6549d8-268b-4dfe-adaf-a4ae5c8510d5'
assert resource['name'] == 'MyResource1'
assert resource['package_id'] == '6f36a41c-f126-4b18-aaaf-6c2ddfbc5d4d'
resource = Resource.read_from_hdx('74b74ae1-df0c-4716-829f-4f939a046812')
assert resource is None
resource = Resource.read_from_hdx('74b74ae1-df0c-4716-829f-4f939a046813')
assert resource is None
with pytest.raises(HDXError):
Resource.read_from_hdx('ABC')
def test_read_from_hdx(self, configuration, read):
resource = Resource.read_from_hdx('74b74ae1-df0c-4716-829f-4f939a046811')
assert resource['id'] == 'de6549d8-268b-4dfe-adaf-a4ae5c8510d5'
assert resource['name'] == 'MyResource1'
assert resource['package_id'] == '6f36a41c-f126-4b18-aaaf-6c2ddfbc5d4d'
resource = Resource.read_from_hdx('74b74ae1-df0c-4716-829f-4f939a046812')
assert resource is None
resource = Resource.read_from_hdx('74b74ae1-df0c-4716-829f-4f939a046813')
assert resource is None
with pytest.raises(HDXError):
Resource.read_from_hdx('ABC')
def test_download(self, configuration, read):
resource = Resource.read_from_hdx('74b74ae1-df0c-4716-829f-4f939a046811')
resource2 = Resource.read_from_hdx('74b74ae1-df0c-4716-829f-4f939a046814')
url, path = resource.download()
remove(path)
assert url == 'https://raw.githubusercontent.com/OCHA-DAP/hdx-python-api/master/tests/fixtures/test_data.csv'
assert basename(path) == 'MyResource1.csv'
resource['url'] = ''
with pytest.raises(HDXError):
resource.download()
with pytest.raises(DownloadError):
resource2.download()
def test_download(self, configuration, read):
resource = Resource.read_from_hdx('74b74ae1-df0c-4716-829f-4f939a046811')
resource2 = Resource.read_from_hdx('74b74ae1-df0c-4716-829f-4f939a046814')
url, path = resource.download()
remove(path)
assert url == 'https://raw.githubusercontent.com/OCHA-DAP/hdx-python-api/master/tests/fixtures/test_data.csv'
assert basename(path) == 'MyResource1.csv'
resource['url'] = ''
with pytest.raises(HDXError):
resource.download()
with pytest.raises(DownloadError):
resource2.download()
def test_check_url_filetoupload(self, configuration):
resource_data = copy.deepcopy(TestResource.resource_data)
resource = Resource(resource_data)
resource.check_url_filetoupload()
resource.set_file_to_upload('abc')
resource.check_url_filetoupload()
resource['url'] = 'lala'
with pytest.raises(HDXError):
resource.check_url_filetoupload()
def test_datastore(self, configuration, post_datastore, topline_yaml,
topline_json):
resource = Resource.read_from_hdx(configuration, 'TEST1')
resource.create_datastore(delete_first=0)
resource.create_datastore(delete_first=1)
resource.create_datastore(delete_first=2)
with pytest.raises(HDXError):
resource.create_datastore(delete_first=3)
resource.update_datastore()
resource.update_datastore_for_topline()
resource.update_datastore_from_yaml_schema(topline_yaml)
resource.update_datastore_from_json_schema(topline_json)
resource = Resource.read_from_hdx(configuration, 'TEST5')
with pytest.raises(HDXError):
resource.update_datastore_from_json_schema(topline_json)
def test_datastore(self, configuration, post_datastore, topline_yaml, topline_json, monkeypatch):
resource = Resource.read_from_hdx('TEST1')
resource2 = Resource.read_from_hdx('TEST5')
monkeypatch.undo()
resource.create_datastore(delete_first=0)
resource.create_datastore(delete_first=1)
resource.create_datastore(delete_first=2)
with pytest.raises(HDXError):
resource.create_datastore(delete_first=3)
resource.update_datastore()
resource.update_datastore_for_topline()
resource.update_datastore_from_yaml_schema(topline_yaml)
resource.update_datastore_from_json_schema(topline_json, path='fixtures/downloader/test_data.csv')
with pytest.raises(HDXError):
resource2.update_datastore_from_json_schema(topline_json)
def test_add_update_delete_resources(self, configuration, post_delete):
dataset_data = copy.deepcopy(TestDataset.dataset_data)
resources_data = copy.deepcopy(TestDataset.resources_data)
dataset = Dataset(dataset_data)
dataset.add_update_resources(resources_data)
assert len(dataset.resources) == 2
dataset.delete_resource('NOTEXIST')
assert len(dataset.resources) == 2
dataset.delete_resource('de6549d8-268b-4dfe-adaf-a4ae5c8510d5')
assert len(dataset.resources) == 1
resources_data = copy.deepcopy(TestDataset.resources_data)
resource = Resource(resources_data[0])
resource.set_file_to_upload('lala')
dataset.add_update_resource(resource)
assert dataset.resources[1].get_file_to_upload() == 'lala'
def main(configuration):
resources = Resource.search_in_hdx(configuration, 'name:')
last_modified_check = list()
for resource in resources:
resource_id = resource['id']
url = resource['url']
if 'data.humdata.org' in url or 'manage.hdx.rwlabs.org' in url or 'proxy.hxlstandard.org' in url or \
'scraperwiki.com' in url or 'ourairports.com' in url:
continue
last_modified_check.append((url, resource_id))
last_modified_check = sorted(
last_modified_check)[:NUMBER_OF_URLS_TO_PROCESS]
# run_grequests(last_modified_check)
run_aiohttp(last_modified_check)
def add_update_resource(self, resource):
# type: (Any) -> None
"""Add new or update existing resource in dataset with new metadata
Args:
resource (Any): Resource metadata either from a Resource object or a dictionary
Returns:
None
"""
if isinstance(resource, dict):
resource = Resource(resource, configuration=self.configuration)
if isinstance(resource, Resource):
if 'package_id' in resource:
raise HDXError(
"Resource %s being added already has a dataset id!" %
(resource['name']))
resource_updated = self._addupdate_hdxobject(
self.resources, 'name', resource)
resource_updated.set_file_to_upload(resource.get_file_to_upload())
return
raise HDXError("Type %s cannot be added as a resource!" %
type(resource).__name__)
def generateDatasetBykey(key, countryName):
metadata = yaml.load(open('config/metadata.yml', 'r'))
title = '%s - ' % countryName + metadata[key]['title']
name = metadata[key]['name']
desc = metadata[key]['notes']
slugified_name = slugify(name).lower()
dataset = Dataset({
'name': slugified_name,
'title': title,
'description': desc
})
dataset.set_dataset_year_range(1985, 2017)
dataset.set_expected_update_frequency('Every year')
dataset.set_subnational(1)
dataset.add_country_location(countryName)
resource = Resource()
rName = ''
upCountry = countryName.upper()
if key == 'education':
dataset.add_tag('EDUCATION')
rName = 'UNECA %s - Education' % countryName
resource.set_file_to_upload('data/%s-education.csv' % upCountry)
if key == 'health':
dataset.add_tag('health')
rName = 'UNECA %s - Health' % countryName
resource.set_file_to_upload('data/%s-health.csv' % upCountry)
if key == 'population_and_migration':
dataset.add_tags(['population', 'migration'])
rName = 'UNECA %s - Population and Migration' % countryName
resource.set_file_to_upload('data/%s-population_and_migration.csv' %
upCountry)
resource['name'] = rName
resource['description'] = 'UNECA %s data' % countryName
resource['format'] = 'csv'
# resource.check_required_fields(['notes'])
dataset.add_update_resource(resource)
print("==================== %s dataset generated ====================" %
key)
return dataset
def generate_dataset_and_showcase(countryName, countryISO2):
title = '%s - Demographic, Health, Education and Transport indicators' % countryName
logger.info('Creating dataset: %s' % title)
name = 'unhabitat-%s-indicators' % countryISO2
slugified_name = slugify(name).lower()
dataset = Dataset({
'name': slugified_name,
'title': title,
})
# dataset.set_dataset_date(date, dataset_end_date=)
dataset.set_dataset_year_range(1950, 2050)
dataset.set_expected_update_frequency('Every year')
dataset.set_subnational(1)
dataset.add_country_location(getCountryISO3Code(countryISO2))
dataset.add_tags(['EDUCATION', 'POPULATION', 'HEALTH', 'TRANSPORT', 'HXL'])
if os.path.isfile('data/indicator_data_' + countryISO2 + '.csv'):
resource = Resource()
resource['name'] = 'Indicators_data_%s' % countryISO2
resource[
'description'] = '%s - Demographic, Health, Education and Transport indicators' % countryName
resource['format'] = 'csv'
resource.set_file_to_upload('data/indicator_data_' + countryISO2 +
'.csv')
resource.check_required_fields(['group', 'package_id'])
dataset.add_update_resource(resource)
showcase_name = slugify('unhabitat-%s' % countryName +
' indacators-data').lower()
showcase = Showcase({
'name':
showcase_name,
'title':
'Explore %s' % countryName + ' indicators',
'notes':
'Explore %s' % countryName + ' indicators',
'url':
'http://urbandata.unhabitat.org/data-country/?countries=%s' %
countryISO2 +
'&indicators=total_length_road,rural_population,urban_population_countries,urban_slum_population_countries,population,income_gini_coefficient_countries',
'image_url':
'https://centre.humdata.org/wp-content/uploads/2018/09/unhabitat-showcase.png'
})
showcase.add_tags(['EDUCATION', 'POPULATION', 'HEALTH', 'TRANSPORT'])
return dataset, showcase
def hdx_acap_connector():
"""Connects to HDX, and fetches acaps covid 19 government measures dataset
Arguments:
None
Returns:
pandas.DataFrame
"""
setup_logging()
Configuration.create(hdx_site='prod',
user_agent='CoronaWhy',
hdx_read_only=True)
dataset = Dataset.read_from_hdx(
'acaps-covid19-government-measures-dataset')
logger.info("Dataset Fetched from: %s", dataset.get_hdx_url())
logger.info('Expected Update Frequency: %s',
dataset.get_expected_update_frequency())
resources = dataset.get_resources()
logger.info('Description: %s', resources[0]['description'])
logger.info('Last Modified: %s, Revision Last Updated: %s',
resources[0]['last_modified'],
resources[0]['revision_last_updated'])
logger.info('Size: %sMb', resources[0]['size'] / (1024**2))
logger.info('Dataset Url: %s', resources[0]['url'])
logger.info('Tags: %s', dataset.get_tags())
resource = Resource.read_from_hdx(resources[0]['id'])
url, absolute_path = resource.download('./')
logger.info('Downloaded dataset at path: %s', absolute_path)
xl = pd.ExcelFile(absolute_path)
logger.info(xl.sheet_names)
df = xl.parse('Database')
return df
def test_update_in_hdx(self, configuration, post_update):
resource = Resource()
resource['id'] = 'NOTEXIST'
with pytest.raises(HDXError):
resource.update_in_hdx()
resource['name'] = 'LALA'
with pytest.raises(HDXError):
resource.update_in_hdx()
resource = Resource.read_from_hdx('74b74ae1-df0c-4716-829f-4f939a046811')
assert resource['id'] == 'de6549d8-268b-4dfe-adaf-a4ae5c8510d5'
assert resource.get_file_type() == 'csv'
resource.set_file_type('XLSX')
resource['id'] = '74b74ae1-df0c-4716-829f-4f939a046811'
resource['name'] = 'MyResource1'
resource.update_in_hdx()
assert resource['id'] == '74b74ae1-df0c-4716-829f-4f939a046811'
assert resource['format'] == 'xlsx'
assert resource.get_file_type() == 'xlsx'
assert resource['url_type'] == 'api'
assert resource['resource_type'] == 'api'
assert resource[
'url'] == 'https://raw.githubusercontent.com/OCHA-DAP/hdx-python-api/master/tests/fixtures/test_data.csv'
filetoupload = join('tests', 'fixtures', 'test_data.csv')
resource.set_file_to_upload(filetoupload)
resource.update_in_hdx()
assert resource['url_type'] == 'upload'
assert resource['resource_type'] == 'file.upload'
assert resource[
'url'] == 'http://test-data.humdata.org/dataset/6f36a41c-f126-4b18-aaaf-6c2ddfbc5d4d/resource/de6549d8-268b-4dfe-adaf-a4ae5c8510d5/download/test_data.csv'
resource['id'] = 'NOTEXIST'
with pytest.raises(HDXError):
resource.update_in_hdx()
del resource['id']
with pytest.raises(HDXError):
resource.update_in_hdx()
resource.data = dict()
with pytest.raises(HDXError):
resource.update_in_hdx()
resource_data = copy.deepcopy(TestResource.resource_data)
resource_data['name'] = 'MyResource1'
resource_data['id'] = '74b74ae1-df0c-4716-829f-4f939a046811'
resource = Resource(resource_data)
resource.create_in_hdx()
assert resource['id'] == '74b74ae1-df0c-4716-829f-4f939a046811'
assert resource.get_file_type() == 'xlsx'
def test_delete_from_hdx(self, configuration, post_delete):
resource = Resource.read_from_hdx(configuration, 'TEST1')
resource.delete_from_hdx()
del resource['id']
with pytest.raises(HDXError):
resource.delete_from_hdx()
def test_delete_from_hdx(self, configuration, post_delete):
resource = Resource.read_from_hdx('74b74ae1-df0c-4716-829f-4f939a046811')
resource.delete_from_hdx()
del resource['id']
with pytest.raises(HDXError):
resource.delete_from_hdx()
def test_create_in_hdx(self, configuration, post_create):
resource = Resource(configuration)
with pytest.raises(HDXError):
resource.create_in_hdx()
resource['id'] = 'TEST1'
resource['name'] = 'LALA'
with pytest.raises(HDXError):
resource.create_in_hdx()
resource_data = copy.deepcopy(TestResource.resource_data)
resource = Resource(configuration, resource_data)
resource.create_in_hdx()
assert resource['id'] == 'de6549d8-268b-4dfe-adaf-a4ae5c8510d5'
resource_data['name'] = 'MyResource2'
resource = Resource(configuration, resource_data)
with pytest.raises(HDXError):
resource.create_in_hdx()
resource_data['name'] = 'MyResource3'
resource = Resource(configuration, resource_data)
with pytest.raises(HDXError):
resource.create_in_hdx()
def test_update_in_hdx(self, configuration, post_update):
resource = Resource(configuration)
resource['id'] = 'NOTEXIST'
with pytest.raises(HDXError):
resource.update_in_hdx()
resource['name'] = 'LALA'
with pytest.raises(HDXError):
resource.update_in_hdx()
resource = Resource.read_from_hdx(configuration, 'TEST1')
assert resource['id'] == 'de6549d8-268b-4dfe-adaf-a4ae5c8510d5'
assert resource['format'] == 'XLSX'
resource['format'] = 'CSV'
resource['id'] = 'TEST1'
resource['name'] = 'MyResource1'
resource.update_in_hdx()
assert resource['id'] == 'TEST1'
assert resource['format'] == 'CSV'
resource['id'] = 'NOTEXIST'
with pytest.raises(HDXError):
resource.update_in_hdx()
del resource['id']
with pytest.raises(HDXError):
resource.update_in_hdx()
resource_data = copy.deepcopy(TestResource.resource_data)
resource_data['name'] = 'MyResource1'
resource_data['id'] = 'TEST1'
resource = Resource(configuration, resource_data)
resource.create_in_hdx()
assert resource['id'] == 'TEST1'
assert resource['format'] == 'xlsx'
def test_check_required_fields(self, configuration):
resource_data = copy.deepcopy(TestResource.resource_data)
resource = Resource(resource_data)
resource.check_url_filetoupload()
resource.check_required_fields()
def test_patch(self, configuration, post_patch):
resource = Resource()
resource['id'] = '74b74ae1-df0c-4716-829f-4f939a046811'
resource.update_in_hdx(operation='patch', batch_mode='KEEP_OLD', skip_validation=True)
assert resource['id'] == 'de6549d8-268b-4dfe-adaf-a4ae5c8510d5'
def test_datastore(self, configuration, post_datastore, topline_yaml, topline_json):
resource_ids = Resource.get_all_resource_ids_in_datastore()
assert resource_ids == ['f9228459-d808-4b51-948f-68a5850abfde', 'af618a0b-09b8-42c8-836f-2be597e1ea34', '748b40dd-7bd3-40a3-941b-e76f0bfbe0eb', '91c78d24-eab3-40b5-ba91-6b29bcda7178', '9320cfce-4620-489a-bcbe-25c73867d4fc', 'b9d2eb36-e65c-417a-bc28-f4dadb149302', 'ca6a0891-8395-4d58-9168-6c44e17e0193']
resource = Resource.read_from_hdx('74b74ae1-df0c-4716-829f-4f939a046811')
resource2 = Resource.read_from_hdx('74b74ae1-df0c-4716-829f-4f939a046815')
TestResource.datastore = None
resource.create_datastore(delete_first=0)
assert TestResource.datastore == 'create'
TestResource.datastore = None
resource.create_datastore(delete_first=1)
assert TestResource.datastore == 'create'
TestResource.datastore = None
resource.create_datastore(delete_first=2)
assert TestResource.datastore == 'create'
TestResource.datastore = None
with pytest.raises(HDXError):
resource.create_datastore(delete_first=3)
resource.update_datastore()
assert TestResource.datastore == 'create'
TestResource.datastore = None
resource.update_datastore_for_topline()
assert TestResource.datastore == 'create'
TestResource.datastore = None
resource.update_datastore_from_dict_schema({
"schema": [
{
"id": "code",
"type": "text"
},
],
"primary_key": "code"
})
assert TestResource.datastore == 'create'
TestResource.datastore = None
resource.update_datastore_from_yaml_schema(topline_yaml)
assert TestResource.datastore == 'create'
TestResource.datastore = None
filefordatastore = join('tests', 'fixtures', 'test_data.csv')
resource.update_datastore_from_json_schema(topline_json, path=filefordatastore)
assert TestResource.datastore == 'create'
TestResource.datastore = None
assert resource.has_datastore() is True
assert TestResource.datastore == 'create'
TestResource.datastore = None
assert resource2.has_datastore() is False
TestResource.datastore = None
filefordatastore = join('tests', 'fixtures', 'datastore', 'ACLED-All-Africa-File_20170101-to-20170708.xlsx')
resource.update_datastore(path=filefordatastore)
assert TestResource.datastore == 'create'
with pytest.raises(HDXError):
resource2.update_datastore_from_json_schema(topline_json)
resource.delete_datastore()
assert TestResource.datastore == 'delete'
TestResource.datastore = None
with pytest.raises(HDXError):
del resource['url']
resource.create_datastore()
if six.PY3:
filefordatastore = join('tests', 'fixtures', 'test_data.zip')
resource.update_datastore_from_json_schema(topline_json, path=filefordatastore)
assert TestResource.datastore == 'create'
def test_datastore(self, configuration, post_datastore, topline_yaml, topline_json):
resource_ids = Resource.get_all_resource_ids_in_datastore()
assert resource_ids == ['f9228459-d808-4b51-948f-68a5850abfde', 'af618a0b-09b8-42c8-836f-2be597e1ea34', '748b40dd-7bd3-40a3-941b-e76f0bfbe0eb', '91c78d24-eab3-40b5-ba91-6b29bcda7178', '9320cfce-4620-489a-bcbe-25c73867d4fc', 'b9d2eb36-e65c-417a-bc28-f4dadb149302', 'ca6a0891-8395-4d58-9168-6c44e17e0193']
resource = Resource.read_from_hdx('74b74ae1-df0c-4716-829f-4f939a046811')
resource2 = Resource.read_from_hdx('74b74ae1-df0c-4716-829f-4f939a046815')
TestResource.datastore = None
resource.create_datastore(delete_first=0)
assert TestResource.datastore == 'create'
TestResource.datastore = None
resource.create_datastore(delete_first=1)
assert TestResource.datastore == 'create'
TestResource.datastore = None
resource.create_datastore(delete_first=2)
assert TestResource.datastore == 'create'
TestResource.datastore = None
with pytest.raises(HDXError):
resource.create_datastore(delete_first=3)
resource.update_datastore()
assert TestResource.datastore == 'create'
TestResource.datastore = None
resource.update_datastore_for_topline()
assert TestResource.datastore == 'create'
TestResource.datastore = None
resource.update_datastore_from_dict_schema({
"schema": [
{
"id": "code",
"type": "text"
},
],
"primary_key": "code"
})
assert TestResource.datastore == 'create'
TestResource.datastore = None
resource.update_datastore_from_yaml_schema(topline_yaml)
assert TestResource.datastore == 'create'
TestResource.datastore = None
filefordatastore = join('tests', 'fixtures', 'test_data.csv')
resource.update_datastore_from_json_schema(topline_json, path=filefordatastore)
assert TestResource.datastore == 'create'
TestResource.datastore = None
assert resource.has_datastore() is True
assert TestResource.datastore == 'create'
TestResource.datastore = None
assert resource2.has_datastore() is False
TestResource.datastore = None
filefordatastore = join('tests', 'fixtures', 'datastore', 'ACLED-All-Africa-File_20170101-to-20170708.xlsx')
resource.update_datastore(path=filefordatastore)
assert TestResource.datastore == 'create'
with pytest.raises(HDXError):
resource2.update_datastore_from_json_schema(topline_json)
resource.delete_datastore()
assert TestResource.datastore == 'delete'
TestResource.datastore = None
with pytest.raises(HDXError):
del resource['url']
resource.create_datastore()
if six.PY3:
filefordatastore = join('tests', 'fixtures', 'test_data.zip')
resource.update_datastore_from_json_schema(topline_json, path=filefordatastore)
assert TestResource.datastore == 'create'
def test_create_in_hdx(self, configuration, post_create):
resource = Resource()
with pytest.raises(HDXError):
resource.create_in_hdx()
resource['id'] = 'TEST1'
resource['name'] = 'LALA'
with pytest.raises(HDXError):
resource.create_in_hdx()
resource_data = copy.deepcopy(TestResource.resource_data)
resource = Resource(resource_data)
resource.create_in_hdx()
assert resource['id'] == 'de6549d8-268b-4dfe-adaf-a4ae5c8510d5'
assert resource['url_type'] == 'api'
assert resource['resource_type'] == 'api'
assert resource[
'url'] == 'https://raw.githubusercontent.com/OCHA-DAP/hdx-python-api/master/tests/fixtures/test_data.csv'
resource_data = copy.deepcopy(TestResource.resource_data)
resource = Resource(resource_data)
filetoupload = join('tests', 'fixtures', 'test_data.csv')
resource.set_file_to_upload(filetoupload)
assert resource.get_file_to_upload() == filetoupload
resource.create_in_hdx()
assert resource['url_type'] == 'upload'
assert resource['resource_type'] == 'file.upload'
assert resource[
'url'] == 'http://test-data.humdata.org/dataset/6f36a41c-f126-4b18-aaaf-6c2ddfbc5d4d/resource/de6549d8-268b-4dfe-adaf-a4ae5c8510d5/download/test_data.csv'
resource_data['name'] = 'MyResource2'
resource = Resource(resource_data)
with pytest.raises(HDXError):
resource.create_in_hdx()
resource_data['name'] = 'MyResource3'
resource = Resource(resource_data)
with pytest.raises(HDXError):
resource.create_in_hdx()
def test_resource_views(self, configuration, post_resourceview):
resource = Resource({'id': '25982d1c-f45a-45e1-b14e-87d367413045'})
with pytest.raises(HDXError):
resource.add_update_resource_view('123')
resource_view = copy.deepcopy(resource_view_list[0])
del resource_view['id']
del resource_view['package_id']
resource.add_update_resource_view(resource_view)
resource_view = copy.deepcopy(resource_view_list[1])
del resource_view['id']
del resource_view['package_id']
with pytest.raises(HDXError):
resource.add_update_resource_views('123')
resource.add_update_resource_views([resource_view])
resource_views = resource.get_resource_views()
assert resource_views[0]['id'] == 'd80301b5-4abd-49bd-bf94-fa4af7b6e7a4'
assert resource_views[1]['id'] == 'c06b5a0d-1d41-4a74-a196-41c251c76023'
with pytest.raises(HDXError):
resource.delete_resource_view('123')
resource.delete_resource_view('d80301b5-4abd-49bd-bf94-fa4af7b6e7a4')
resource.delete_resource_view(resource_view)
resource_view['title'] = 'XXX'
with pytest.raises(HDXError):
resource.delete_resource_view(resource_view)
with pytest.raises(HDXError):
resource.reorder_resource_views('123')
resource.reorder_resource_views(['c06b5a0d-1d41-4a74-a196-41c251c76023', 'd80301b5-4abd-49bd-bf94-fa4af7b6e7a4'])
resource.reorder_resource_views(resource_view_list)
resource_view = copy.deepcopy(resource_view_list[0])
resource_view['id'] = '123'
with pytest.raises(HDXError):
resource.reorder_resource_views([resource_view_list[1], resource_view])
def test_patch(self, configuration, post_patch):
resource = Resource()
resource['id'] = '74b74ae1-df0c-4716-829f-4f939a046811'
resource.update_in_hdx(operation='patch', batch_mode='KEEP_OLD', skip_validation=True)
assert resource['id'] == 'de6549d8-268b-4dfe-adaf-a4ae5c8510d5'
def test_delete_from_hdx(self, configuration, post_delete):
resource = Resource.read_from_hdx('74b74ae1-df0c-4716-829f-4f939a046811')
resource.delete_from_hdx()
del resource['id']
with pytest.raises(HDXError):
resource.delete_from_hdx()
def test_update_in_hdx(self, configuration, post_update):
resource = Resource()
resource['id'] = 'NOTEXIST'
with pytest.raises(HDXError):
resource.update_in_hdx()
resource['name'] = 'LALA'
with pytest.raises(HDXError):
resource.update_in_hdx()
resource = Resource.read_from_hdx('74b74ae1-df0c-4716-829f-4f939a046811')
assert resource['id'] == 'de6549d8-268b-4dfe-adaf-a4ae5c8510d5'
assert resource.get_file_type() == 'csv'
resource.set_file_type('XLSX')
resource['id'] = '74b74ae1-df0c-4716-829f-4f939a046811'
resource['name'] = 'MyResource1'
resource.update_in_hdx()
assert resource['id'] == '74b74ae1-df0c-4716-829f-4f939a046811'
assert resource['format'] == 'xlsx'
assert resource.get_file_type() == 'xlsx'
assert resource['url_type'] == 'api'
assert resource['resource_type'] == 'api'
assert resource[
'url'] == 'https://raw.githubusercontent.com/OCHA-DAP/hdx-python-api/master/tests/fixtures/test_data.csv'
assert resource['state'] == 'active'
filetoupload = join('tests', 'fixtures', 'test_data.csv')
resource.set_file_to_upload(filetoupload)
resource.update_in_hdx()
assert resource['url_type'] == 'upload'
assert resource['resource_type'] == 'file.upload'
assert resource[
'url'] == 'http://test-data.humdata.org/dataset/6f36a41c-f126-4b18-aaaf-6c2ddfbc5d4d/resource/de6549d8-268b-4dfe-adaf-a4ae5c8510d5/download/test_data.csv'
assert resource['state'] == 'active'
resource['id'] = 'NOTEXIST'
with pytest.raises(HDXError):
resource.update_in_hdx()
del resource['id']
with pytest.raises(HDXError):
resource.update_in_hdx()
resource.data = dict()
with pytest.raises(HDXError):
resource.update_in_hdx()
resource_data = copy.deepcopy(TestResource.resource_data)
resource_data['name'] = 'MyResource1'
resource_data['id'] = '74b74ae1-df0c-4716-829f-4f939a046811'
resource = Resource(resource_data)
resource.create_in_hdx()
assert resource['id'] == '74b74ae1-df0c-4716-829f-4f939a046811'
assert resource.get_file_type() == 'xlsx'
assert resource['state'] == 'active'
def test_check_required_fields(self, configuration):
resource_data = copy.deepcopy(TestResource.resource_data)
resource = Resource(resource_data)
resource.check_url_filetoupload()
resource.check_required_fields()
def test_resource_views(self, configuration, post_resourceview):
resource = Resource({'id': '25982d1c-f45a-45e1-b14e-87d367413045'})
with pytest.raises(HDXError):
resource.add_update_resource_view('123')
resource_view = copy.deepcopy(resource_view_list[0])
del resource_view['id']
del resource_view['package_id']
resource.add_update_resource_view(resource_view)
resource_view = copy.deepcopy(resource_view_list[1])
del resource_view['id']
del resource_view['package_id']
with pytest.raises(HDXError):
resource.add_update_resource_views('123')
resource.add_update_resource_views([resource_view])
resource_views = resource.get_resource_views()
assert resource_views[0]['id'] == 'd80301b5-4abd-49bd-bf94-fa4af7b6e7a4'
assert resource_views[1]['id'] == 'c06b5a0d-1d41-4a74-a196-41c251c76023'
with pytest.raises(HDXError):
resource.delete_resource_view('123')
resource.delete_resource_view('d80301b5-4abd-49bd-bf94-fa4af7b6e7a4')
resource.delete_resource_view(resource_view)
resource_view['title'] = 'XXX'
with pytest.raises(HDXError):
resource.delete_resource_view(resource_view)
with pytest.raises(HDXError):
resource.reorder_resource_views('123')
resource.reorder_resource_views(['c06b5a0d-1d41-4a74-a196-41c251c76023', 'd80301b5-4abd-49bd-bf94-fa4af7b6e7a4'])
resource.reorder_resource_views(resource_view_list)
resource_view = copy.deepcopy(resource_view_list[0])
resource_view['id'] = '123'
with pytest.raises(HDXError):
resource.reorder_resource_views([resource_view_list[1], resource_view])
def update_resources(resource_updates):
for resource_info in resource_updates.values():
resource = Resource.read_from_hdx(resource_info['id'])
resource.set_file_to_upload(resource_info['path'])
resource.update_in_hdx()
def test_delete_from_hdx(self, configuration, post_delete):
resource = Resource.read_from_hdx('TEST1')
resource.delete_from_hdx()
del resource['id']
with pytest.raises(HDXError):
resource.delete_from_hdx()
