def test_update_json(self, configuration, static_json):
org_data = copy.deepcopy(organization_data)
organization = Organization(org_data)
assert organization['name'] == 'MyOrganization1'
assert organization['description'] == 'We do humanitarian work'
organization.update_from_json(static_json)
assert organization['name'] == 'MyOrg1'
assert organization['description'] == 'other'
def test_read_from_hdx(self, configuration, read, mocksmtp):
organization = Organization.read_from_hdx('b67e6c74-c185-4f43-b561-0e114a736f19')
assert organization['id'] == 'b67e6c74-c185-4f43-b561-0e114a736f19'
assert organization['name'] == 'acled'
organization = Organization.read_from_hdx('TEST2')
assert organization is None
organization = Organization.read_from_hdx('TEST3')
assert organization is None
def test_read_from_hdx(self, configuration, read, mocksmtp):
organization = Organization.read_from_hdx('TEST1')
assert organization['id'] == 'b67e6c74-c185-4f43-b561-0e114a736f19'
assert organization['name'] == 'acled'
organization = Organization.read_from_hdx('TEST2')
assert organization is None
organization = Organization.read_from_hdx('TEST3')
assert organization is None
def test_update_json(self, configuration, static_json):
org_data = copy.deepcopy(organization_data)
organization = Organization(org_data)
assert organization['name'] == 'MyOrganization1'
assert organization['description'] == 'We do humanitarian work'
organization.update_from_json(static_json)
assert organization['name'] == 'MyOrg1'
assert organization['description'] == 'other'
def __init__(
self,
site_url: str,
users: Optional[List[User]] = None,
organizations: Optional[List[Organization]] = None,
):
self.site_url = site_url
if users is None: # pragma: no cover
users = User.get_all_users()
self.users: Dict[str, User] = dict()
self.sysadmins = dict()
for user in users:
userid = user["id"]
self.users[userid] = user
if user["sysadmin"]:
self.sysadmins[userid] = user
self.organizations: Dict = dict()
if organizations is None: # pragma: no cover
organizations: List = Organization.get_all_organization_names(
all_fields=True, include_users=True
)
for organization in organizations:
users_per_capacity = dict()
for user in organization["users"]:
dict_of_lists_add(
users_per_capacity, user["capacity"], user["id"]
)
self.organizations[organization["id"]] = users_per_capacity
def test_delete_from_hdx(self, configuration, post_delete):
organization = Organization.read_from_hdx(
'b67e6c74-c185-4f43-b561-0e114a736f19')
organization.delete_from_hdx()
del organization['id']
with pytest.raises(HDXError):
organization.delete_from_hdx()
def test_get_all_organizations(self, configuration, post_list):
organizations = Organization.get_all_organization_names()
assert len(organizations) == 10
def test_delete_from_hdx(self, configuration, post_delete):
organization = Organization.read_from_hdx('TEST1')
organization.delete_from_hdx()
del organization['id']
with pytest.raises(HDXError):
organization.delete_from_hdx()
def test_create_in_hdx(self, configuration, post_create):
organization = Organization()
with pytest.raises(HDXError):
organization.create_in_hdx()
organization['id'] = 'b67e6c74-c185-4f43-b561-0e114a736f19'
organization['name'] = 'LALA'
with pytest.raises(HDXError):
organization.create_in_hdx()
org_data = copy.deepcopy(organization_data)
organization = Organization(org_data)
organization.create_in_hdx()
assert organization['id'] == 'b67e6c74-c185-4f43-b561-0e114a736f19'
assert organization['state'] == 'active'
org_data['name'] = 'MyOrganization2'
organization = Organization(org_data)
with pytest.raises(HDXError):
organization.create_in_hdx()
org_data['name'] = 'MyOrganization3'
organization = Organization(org_data)
with pytest.raises(HDXError):
organization.create_in_hdx()
def test_update_in_hdx(self, configuration, post_update):
organization = Organization()
organization['id'] = 'NOTEXIST'
with pytest.raises(HDXError):
organization.update_in_hdx()
organization['name'] = 'LALA'
with pytest.raises(HDXError):
organization.update_in_hdx()
organization = Organization.read_from_hdx(
'b67e6c74-c185-4f43-b561-0e114a736f19')
assert organization['id'] == 'b67e6c74-c185-4f43-b561-0e114a736f19'
assert organization['name'] == 'acled'
organization['description'] = 'Humanitarian work'
organization['id'] = 'b67e6c74-c185-4f43-b561-0e114a736f19'
organization['name'] = 'MyOrganization1'
organization.update_in_hdx()
assert organization['id'] == 'b67e6c74-c185-4f43-b561-0e114a736f19'
assert organization['description'] == 'Humanitarian work'
assert organization['state'] == 'active'
organization['id'] = 'NOTEXIST'
with pytest.raises(HDXError):
organization.update_in_hdx()
del organization['id']
with pytest.raises(HDXError):
organization.update_in_hdx()
org_data = copy.deepcopy(organization_data)
org_data['name'] = 'MyOrganization1'
org_data['id'] = 'b67e6c74-c185-4f43-b561-0e114a736f19'
organization = Organization(org_data)
organization.create_in_hdx()
assert organization['id'] == 'b67e6c74-c185-4f43-b561-0e114a736f19'
assert organization['name'] == 'MyOrganization1'
assert organization['state'] == 'active'
def test_users(self, configuration, user_read):
org_data = copy.deepcopy(resultdict)
organization = Organization(org_data)
organization['users'][0]['name'] = 'MyUser1'
users = organization.get_users()
assert len(users) == 1
assert users[0]['name'] == 'MyUser1'
organization.remove_user('9f3e9973-7dbe-4c65-8820-f48578e3ffea')
users = organization.get_users()
assert len(users) == 0
userdata_copy = copy.deepcopy(user_data)
userdata_copy['id'] = '9f3e9973-7dbe-4c65-8820-f48578e3ffea'
user = User(userdata_copy)
user['name'] = 'TEST1'
user['capacity'] = 'member'
del organization['users']
organization.add_update_users([user])
users = organization.get_users('member')
assert len(users) == 1
organization.remove_user(users[0])
users = organization.get_users()
assert len(users) == 0
organization.add_update_users([user])
users = organization.get_users('member')
assert len(users) == 1
organization.add_update_users(['MyUser1'], 'member')
users = organization.get_users('member')
assert len(users) == 2
assert users[0]['name'] == 'MyUser1'
organization.remove_user(users[0].data)
users = organization.get_users('member')
assert len(users) == 1
organization.add_update_users([user], 'editor')
organization.remove_user('')
users = organization.get_users('editor')
assert len(users) == 1
organization.remove_user('NOT_EXIST')
users = organization.get_users('editor')
assert len(users) == 1
users = organization.get_users('admin')
assert len(users) == 0
del organization['users'][0]['id']
users = organization.get_users('member')
assert len(users) == 1
organization.add_update_user({'name': 'TEST1'}, 'member')
users = organization.get_users('member')
assert len(users) == 2
with pytest.raises(HDXError):
organization.add_update_users(123)
with pytest.raises(HDXError):
organization.add_update_user(123)
with pytest.raises(HDXError):
organization.remove_user(123)
def test_get_datasets(self, configuration, datasets_get):
org_data = copy.deepcopy(resultdict)
organization = Organization(org_data)
datasets = organization.get_datasets()
assert len(datasets) == 10
def generate_dataset(self, metadata):
title = metadata["DatasetTitle"]
is_requestdata_type = metadata["is_requestdata_type"]
if not is_requestdata_type:
if metadata["Total"] == 0:
self.errors.add(
f"Ignoring dataset: {title} which has no resources!")
return None, None
if not metadata["Source"]:
self.errors.add(f"Dataset: {title} has no source!")
logger.info(f"Creating dataset: {title}")
cod_level = "cod-standard"
if metadata["is_enhanced_cod"]:
cod_level = "cod-enhanced"
theme = metadata["Theme"]
if not theme:
self.errors.add(f"Dataset: {title} has no theme!")
location = metadata["Location"]
if theme == "COD_AB" and (location == ["MMR"] or location == ["mmr"]):
name = slugify(title)
else:
name = slugify(f"{theme} {' '.join(location)}")
dataset = Dataset({
"name": name[:99],
"title": title,
"notes": metadata["DatasetDescription"],
"dataset_source": metadata["Source"],
"methodology": metadata["Methodology"],
"methodology_other": metadata["Methodology_Other"],
"license_id": metadata["License"],
"license_other": metadata["License_Other"],
"caveats": metadata["Caveats"],
"data_update_frequency": metadata["FrequencyUpdates"],
"cod_level": cod_level,
})
licence = metadata["License"]
if licence == "Other":
dataset["license_id"] = "hdx-other"
dataset["license_other"] = metadata["License_Other"]
else:
dataset["license_id"] = licence
methodology = metadata["Methodology"]
if methodology == "Other":
dataset["methodology"] = "Other"
methodology_other = metadata["Methodology_Other"]
if not methodology_other:
self.errors.add(f"Dataset: {title} has no methodology!")
if methodology_other:
dataset["methodology_other"] = methodology_other
else:
dataset["methodology"] = methodology
dataset.set_maintainer("196196be-6037-4488-8b71-d786adf4c081")
organization = Organization.autocomplete(metadata["Contributor"])
if len(organization) == 0:
organization = Organization.autocomplete(
metadata["Contributor"].replace(" ", "-"))
organization_id = None
batch = None
try:
organization_id = organization[0]["id"]
except IndexError:
self.errors.add(
f"Dataset: {title} has an invalid organization {metadata['Contributor']}!"
)
if organization_id:
dataset.set_organization(organization_id)
batch = self.batches_by_org.get(organization_id, get_uuid())
self.batches_by_org[organization_id] = batch
dataset.set_subnational(True)
try:
dataset.add_country_locations(location)
except HDXError:
self.errors.add(
f"Dataset: {title} has an invalid location {location}!")
dataset.add_tags(metadata["Tags"])
if len(dataset.get_tags()) < len(metadata["Tags"]):
self.errors.add(f"Dataset: {title} has invalid tags!")
if "common operational dataset - cod" not in dataset.get_tags():
dataset.add_tag("common operational dataset - cod")
if is_requestdata_type:
dataset["dataset_date"] = metadata["DatasetDate"]
dataset["is_requestdata_type"] = True
dataset["file_types"] = metadata["file_types"]
dataset["field_names"] = metadata["field_names"]
num_of_rows = metadata.get("num_of_rows")
if num_of_rows:
dataset["num_of_rows"] = num_of_rows
else:
startdate = None
enddate = None
ongoing = False
resources = list()
for resource_metadata in metadata["Resources"]:
resource_daterange = resource_metadata["daterange_for_data"]
format = resource_metadata["Format"]
if format == "VectorTile":
format = "MBTiles"
logger.error(
f"Dataset: {title} is using file type VectorTile instead of MBTiles"
)
resourcedata = {
"name": resource_metadata["ResourceItemTitle"],
"description":
resource_metadata["ResourceItemDescription"],
"url": resource_metadata["DownloadURL"],
"format": format,
"daterange_for_data": resource_daterange,
"grouping": resource_metadata["Version"],
}
date_info = DateHelper.get_date_info(resource_daterange)
resource_startdate = date_info["startdate"]
resource_enddate = date_info["enddate"]
resource_ongoing = date_info["ongoing"]
if startdate is None or resource_startdate < startdate:
startdate = resource_startdate
if enddate is None or resource_enddate > enddate:
enddate = resource_enddate
ongoing = resource_ongoing
resource = Resource(resourcedata)
resources.append(resource)
if ongoing:
enddate = "*"
try:
dataset.add_update_resources(resources)
except HDXError as ex:
self.errors.add(
f"Dataset: {title} resources could not be added. Error: {ex}"
)
dataset.set_date_of_dataset(startdate, enddate)
if len(self.errors.errors) > 0:
return None, None
else:
return dataset, batch
def test_users(self, configuration, user_read):
org_data = copy.deepcopy(resultdict)
organization = Organization(org_data)
organization['users'][0]['name'] = 'MyUser1'
users = organization.get_users()
assert len(users) == 1
assert users[0]['name'] == 'MyUser1'
organization.remove_user('9f3e9973-7dbe-4c65-8820-f48578e3ffea')
users = organization.get_users()
assert len(users) == 0
userdata_copy = copy.deepcopy(user_data)
userdata_copy['id'] = '9f3e9973-7dbe-4c65-8820-f48578e3ffea'
user = User(userdata_copy)
user['name'] = 'TEST1'
user['capacity'] = 'member'
del organization['users']
organization.add_update_users([user])
users = organization.get_users('member')
assert len(users) == 1
organization.remove_user(users[0])
users = organization.get_users()
assert len(users) == 0
organization.add_update_users([user])
users = organization.get_users('member')
assert len(users) == 1
organization.add_update_users(['MyUser1'], 'member')
users = organization.get_users('member')
assert len(users) == 2
assert users[0]['name'] == 'MyUser1'
organization.remove_user(users[0].data)
users = organization.get_users('member')
assert len(users) == 1
organization.add_update_users([user], 'editor')
organization.remove_user('')
users = organization.get_users('editor')
assert len(users) == 1
organization.remove_user('NOT_EXIST')
users = organization.get_users('editor')
assert len(users) == 1
users = organization.get_users('admin')
assert len(users) == 0
del organization['users'][0]['id']
users = organization.get_users('member')
assert len(users) == 1
organization.add_update_user({'name': 'TEST1'}, 'member')
users = organization.get_users('member')
assert len(users) == 2
with pytest.raises(HDXError):
organization.add_update_users(123)
with pytest.raises(HDXError):
organization.add_update_user(123)
with pytest.raises(HDXError):
organization.remove_user(123)
def test_delete_from_hdx(self, configuration, post_delete):
organization = Organization.read_from_hdx('b67e6c74-c185-4f43-b561-0e114a736f19')
organization.delete_from_hdx()
del organization['id']
with pytest.raises(HDXError):
organization.delete_from_hdx()
def test_update_in_hdx(self, configuration, post_update):
organization = Organization()
organization['id'] = 'NOTEXIST'
with pytest.raises(HDXError):
organization.update_in_hdx()
organization['name'] = 'LALA'
with pytest.raises(HDXError):
organization.update_in_hdx()
organization = Organization.read_from_hdx('b67e6c74-c185-4f43-b561-0e114a736f19')
assert organization['id'] == 'b67e6c74-c185-4f43-b561-0e114a736f19'
assert organization['name'] == 'acled'
organization['description'] = 'Humanitarian work'
organization['id'] = 'b67e6c74-c185-4f43-b561-0e114a736f19'
organization['name'] = 'MyOrganization1'
organization.update_in_hdx()
assert organization['id'] == 'b67e6c74-c185-4f43-b561-0e114a736f19'
assert organization['description'] == 'Humanitarian work'
organization['id'] = 'NOTEXIST'
with pytest.raises(HDXError):
organization.update_in_hdx()
del organization['id']
with pytest.raises(HDXError):
organization.update_in_hdx()
org_data = copy.deepcopy(organization_data)
org_data['name'] = 'MyOrganization1'
org_data['id'] = 'b67e6c74-c185-4f43-b561-0e114a736f19'
organization = Organization(org_data)
organization.create_in_hdx()
assert organization['id'] == 'b67e6c74-c185-4f43-b561-0e114a736f19'
assert organization['name'] == 'MyOrganization1'
def test_create_in_hdx(self, configuration, post_create):
organization = Organization()
with pytest.raises(HDXError):
organization.create_in_hdx()
organization['id'] = 'b67e6c74-c185-4f43-b561-0e114a736f19'
organization['name'] = 'LALA'
with pytest.raises(HDXError):
organization.create_in_hdx()
org_data = copy.deepcopy(organization_data)
organization = Organization(org_data)
organization.create_in_hdx()
assert organization['id'] == 'b67e6c74-c185-4f43-b561-0e114a736f19'
org_data['name'] = 'MyOrganization2'
organization = Organization(org_data)
with pytest.raises(HDXError):
organization.create_in_hdx()
org_data['name'] = 'MyOrganization3'
organization = Organization(org_data)
with pytest.raises(HDXError):
organization.create_in_hdx()
def test_get_datasets(self, configuration, datasets_get):
org_data = copy.deepcopy(resultdict)
organization = Organization(org_data)
datasets = organization.get_datasets()
assert len(datasets) == 10
def test_get_all_organizations(self, configuration, post_list):
organizations = Organization.get_all_organization_names()
assert len(organizations) == 10