def test_source_import_success(self):
OrganizationFactory(mnemonic='DemoOrg')
self.assertFalse(Source.objects.filter(mnemonic='DemoSource').exists())
data = {
"type": "Source",
"id": "DemoSource",
"short_code": "DemoSource",
"name": "OCL Demo Source",
"full_name": "OCL Demo Source",
"owner_type": "Organization",
"owner": "DemoOrg",
"description": "Source used for demo purposes",
"default_locale": "en",
"source_type": "Dictionary",
"public_access": "View",
"supported_locales": "en",
"custom_validation_schema": "None"
}
importer = BulkImportInline(json.dumps(data), 'ocladmin', True)
importer.run()
self.assertTrue(
Source.objects.filter(mnemonic='DemoSource',
version='HEAD').exists())
self.assertEqual(importer.processed, 1)
self.assertEqual(len(importer.created), 1)
self.assertEqual(importer.created[0], data)
self.assertTrue(importer.elapsed_seconds > 0)
def test_collection_and_version_import(self):
OrganizationFactory(mnemonic='PEPFAR')
self.assertFalse(
Collection.objects.filter(
mnemonic='MER-R-MOH-Facility-FY19').exists())
data = '{"type": "Collection", "id": "MER-R-MOH-Facility-FY19", "name": "MER R: MOH Facility Based FY19", ' \
'"default_locale": "en", "short_code": "MER-R-MOH-Facility-FY19", "external_id": "OBhi1PUW3OL", ' \
'"extras": {"Period": "FY19", "Period Description": "COP18 (FY19Q1)", "datim_sync_moh_fy19": true, ' \
'"DHIS2-Dataset-Code": "MER_R_MOH"}, "collection_type": "Code List", "full_name": ' \
'"MER Results: MOH Facility Based FY19", "owner": "PEPFAR", "public_access": "View", ' \
'"owner_type": "Organization", "supported_locales": "en"}\n' \
'{"type": "Collection Version", "id": "FY19.v0", ' \
'"description": "Initial release of FY19 DATIM-MOH definitions", ' \
'"collection": "MER-R-MOH-Facility-FY19", "released": true, "owner": "PEPFAR", ' \
'"owner_type": "Organization"}'
importer = BulkImportInline(data, 'ocladmin', True)
importer.run()
self.assertTrue(
Collection.objects.filter(mnemonic='MER-R-MOH-Facility-FY19',
version='HEAD').exists())
self.assertTrue(
Collection.objects.filter(mnemonic='MER-R-MOH-Facility-FY19',
version='FY19.v0').exists())
self.assertEqual(importer.processed, 2)
self.assertEqual(len(importer.created), 1)
self.assertEqual(len(importer.updated), 1)
self.assertEqual(importer.failed, [])
self.assertTrue(importer.elapsed_seconds > 0)
def test_source_and_version_import(self):
OrganizationFactory(mnemonic='DemoOrg')
self.assertFalse(Source.objects.filter(mnemonic='DemoSource').exists())
data = '{"type": "Source", "id": "DemoSource", "short_code": "DemoSource", "name": "OCL Demo Source", ' \
'"full_name": "OCL Demo Source", "owner_type": "Organization", "owner": "DemoOrg", "description": ' \
'"Source used for demo purposes", "default_locale": "en", "source_type": "Dictionary", ' \
'"public_access": "View", "supported_locales": "en", "custom_validation_schema": "None"}\n' \
'{"type": "Source Version", "id": "initial", "source": "DemoSource", "description": "Initial empty ' \
'repository version", "released": true, "owner": "DemoOrg", "owner_type": "Organization"} '
importer = BulkImportInline(data, 'ocladmin', True)
importer.run()
self.assertTrue(
Source.objects.filter(mnemonic='DemoSource',
version='HEAD').exists())
self.assertTrue(
Source.objects.filter(mnemonic='DemoSource',
version='initial').exists())
self.assertEqual(importer.processed, 2)
self.assertEqual(len(importer.created), 1)
self.assertEqual(len(importer.updated), 1)
self.assertEqual(importer.failed, [])
self.assertTrue(importer.elapsed_seconds > 0)
def test_source_import_failed(self):
self.assertFalse(Source.objects.filter(mnemonic='DemoSource').exists())
data = {
"type": "Source",
"id": "DemoSource",
"short_code": "DemoSource",
"name": "OCL Demo Source",
"full_name": "OCL Demo Source",
"owner_type": "Organization",
"owner": "DemoOrg",
"description": "Source used for demo purposes",
"default_locale": "en",
"source_type": "Dictionary",
"public_access": "View",
"supported_locales": "en",
"custom_validation_schema": "None"
}
importer = BulkImportInline(json.dumps(data), 'ocladmin', True)
importer.run()
self.assertFalse(Source.objects.filter(mnemonic='DemoSource').exists())
self.assertEqual(importer.processed, 1)
self.assertEqual(len(importer.created), 0)
self.assertEqual(len(importer.failed), 1)
self.assertEqual(importer.failed[0], {
**data, 'errors': {
'parent': 'Parent resource cannot be None.'
}
})
self.assertTrue(importer.elapsed_seconds > 0)
def test_pepfar_import(self):
importer = BulkImportInline(
open(
os.path.join(os.path.dirname(__file__), '..',
'samples/pepfar_datim_moh_fy19.json'),
'r').read(), 'ocladmin', True)
importer.run()
self.assertEqual(importer.processed, 413)
self.assertEqual(len(importer.created), 411)
self.assertEqual(len(importer.exists), 0)
self.assertEqual(len(importer.updated), 2)
self.assertEqual(len(importer.failed), 0)
self.assertEqual(len(importer.invalid), 0)
self.assertEqual(len(importer.others), 0)
def test_sample_import(self):
importer = BulkImportInline(
open(
os.path.join(os.path.dirname(__file__), '..',
'samples/sample_ocldev.json'), 'r').read(),
'ocladmin', True)
importer.run()
self.assertEqual(importer.processed, 64)
self.assertEqual(len(importer.created), 47)
self.assertEqual(len(importer.exists), 3)
self.assertEqual(len(importer.updated), 14)
self.assertEqual(len(importer.failed), 0)
self.assertEqual(len(importer.invalid), 0)
self.assertEqual(len(importer.others), 0)
def bulk_import_parts_inline(self, input_list, username, update_if_exists):
from core.importers.models import BulkImportInline
return BulkImportInline(content=None,
username=username,
update_if_exists=update_if_exists,
input_list=input_list,
self_task_id=self.request.id).run()
def test_collection_import_failed(self):
self.assertFalse(
Collection.objects.filter(
mnemonic='MER-R-MOH-Facility-FY19').exists())
data = {
"type": "Collection",
"id": "MER-R-MOH-Facility-FY19",
"name": "MER R: MOH Facility Based FY19",
"default_locale": "en",
"short_code": "MER-R-MOH-Facility-FY19",
"external_id": "OBhi1PUW3OL",
"extras": {
"Period": "FY19",
"Period Description": "COP18 (FY19Q1)",
"datim_sync_moh_fy19": True,
"DHIS2-Dataset-Code": "MER_R_MOH"
},
"collection_type": "Code List",
"full_name": "MER Results: MOH Facility Based FY19",
"owner": "PEPFAR",
"public_access": "View",
"owner_type": "Organization",
"supported_locales": "en"
}
importer = BulkImportInline(json.dumps(data), 'ocladmin', True)
importer.run()
self.assertFalse(
Collection.objects.filter(
mnemonic='MER-R-MOH-Facility-FY19').exists())
self.assertEqual(importer.processed, 1)
self.assertEqual(len(importer.created), 0)
self.assertEqual(len(importer.failed), 1)
self.assertEqual(importer.failed[0], {
**data, 'errors': {
'parent': 'Parent resource cannot be None.'
}
})
self.assertTrue(importer.elapsed_seconds > 0)
def test_org_import(self):
self.assertFalse(
Organization.objects.filter(mnemonic='DemoOrg').exists())
data = {
"type": "Organization",
"id": "DemoOrg",
"website": "",
"name": "OCL Demo Organization",
"company": "",
"public_access": "View"
}
importer = BulkImportInline(json.dumps(data), 'ocladmin', True)
importer.run()
self.assertTrue(
Organization.objects.filter(mnemonic='DemoOrg').exists())
self.assertEqual(importer.processed, 1)
self.assertEqual(len(importer.created), 1)
self.assertEqual(importer.created[0], data)
self.assertTrue(importer.elapsed_seconds > 0)
data = {
"type": "Organization",
"id": "DemoOrg",
"website": "",
"name": "OCL Demo Organization",
"company": "",
"public_access": "View"
}
importer = BulkImportInline(json.dumps(data), 'ocladmin', True)
importer.run()
self.assertEqual(importer.processed, 1)
self.assertEqual(len(importer.created), 0)
self.assertEqual(len(importer.failed), 0)
self.assertEqual(len(importer.exists), 1)
self.assertEqual(importer.exists[0], data)
self.assertTrue(importer.elapsed_seconds > 0)
def test_reference_import(self):
importer = BulkImportInline(
open(
os.path.join(os.path.dirname(__file__), '..',
'samples/sample_collection_references.json'),
'r').read(), 'ocladmin', True)
importer.run()
self.assertEqual(importer.processed, 9)
self.assertEqual(len(importer.created), 9)
self.assertEqual(len(importer.exists), 0)
self.assertEqual(len(importer.updated), 0)
self.assertEqual(len(importer.failed), 0)
self.assertEqual(len(importer.invalid), 0)
self.assertEqual(len(importer.others), 0)
collection = Collection.objects.filter(
uri='/orgs/PEPFAR/collections/MER-R-MOH-Facility-FY19/').first()
self.assertEqual(collection.concepts.count(), 4)
self.assertEqual(collection.mappings.count(), 0)
self.assertEqual(collection.references.count(), 4)
# duplicate run
importer = BulkImportInline(
open(
os.path.join(os.path.dirname(__file__), '..',
'samples/sample_collection_references.json'),
'r').read(), 'ocladmin', True)
importer.run()
self.assertEqual(importer.processed, 9)
self.assertEqual(len(importer.created), 2)
self.assertEqual(len(importer.exists), 3)
self.assertEqual(len(importer.updated), 4)
self.assertEqual(len(importer.failed), 0)
self.assertEqual(len(importer.invalid), 0)
self.assertEqual(len(importer.others), 0)
collection = Collection.objects.filter(
uri='/orgs/PEPFAR/collections/MER-R-MOH-Facility-FY19/').first()
self.assertEqual(collection.concepts.count(), 4)
self.assertEqual(collection.mappings.count(), 0)
self.assertEqual(collection.references.count(), 4)
def bulk_import_inline(to_import, username, update_if_exists):
from core.importers.models import BulkImportInline
return BulkImportInline(content=to_import,
username=username,
update_if_exists=update_if_exists).run()
def test_mapping_import(self):
self.assertEqual(Mapping.objects.count(), 0)
source = OrganizationSourceFactory(
organization=(OrganizationFactory(mnemonic='DemoOrg')),
mnemonic='DemoSource',
version='HEAD')
ConceptFactory(parent=source, mnemonic='Corn')
ConceptFactory(parent=source, mnemonic='Vegetable')
data = {
"to_concept_url":
"/orgs/DemoOrg/sources/DemoSource/concepts/Corn/",
"from_concept_url":
"/orgs/DemoOrg/sources/DemoSource/concepts/Vegetable/",
"type": "Mapping",
"source": "DemoSource",
"extras": None,
"owner": "DemoOrg",
"map_type": "Has Child",
"owner_type": "Organization",
"external_id": None
}
importer = BulkImportInline(json.dumps(data), 'ocladmin', True)
importer.run()
self.assertEqual(
Mapping.objects.filter(map_type='Has Child').count(), 2)
self.assertEqual(
Mapping.objects.filter(
map_type='Has Child',
id=F('versioned_object_id')).first().versions.count(), 1)
self.assertTrue(
Mapping.objects.filter(map_type='Has Child',
is_latest_version=True).exists())
self.assertEqual(importer.processed, 1)
self.assertEqual(len(importer.created), 1)
self.assertEqual(importer.failed, [])
self.assertTrue(importer.elapsed_seconds > 0)
data = {
"to_concept_url":
"/orgs/DemoOrg/sources/DemoSource/concepts/Corn/",
"from_concept_url":
"/orgs/DemoOrg/sources/DemoSource/concepts/Vegetable/",
"type": "Mapping",
"source": "DemoSource",
"extras": {
"foo": "bar"
},
"owner": "DemoOrg",
"map_type": "Has Child",
"owner_type": "Organization",
"external_id": None
}
importer = BulkImportInline(json.dumps(data), 'ocladmin', True)
importer.run()
self.assertEqual(
Mapping.objects.filter(
map_type='Has Child',
id=F('versioned_object_id')).first().versions.count(), 2)
self.assertEqual(importer.processed, 1)
self.assertEqual(len(importer.created), 0)
self.assertEqual(len(importer.updated), 1)
self.assertEqual(importer.failed, [])
self.assertTrue(importer.elapsed_seconds > 0)
def test_concept_import(self):
self.assertFalse(Concept.objects.filter(mnemonic='Food').exists())
OrganizationSourceFactory(
organization=(OrganizationFactory(mnemonic='DemoOrg')),
mnemonic='DemoSource',
version='HEAD')
data = {
"type":
"Concept",
"id":
"Food",
"concept_class":
"Root",
"datatype":
"None",
"source":
"DemoSource",
"owner":
"DemoOrg",
"owner_type":
"Organization",
"names": [{
"name": "Food",
"locale": "en",
"locale_preferred": "True",
"name_type": "Fully Specified"
}],
"descriptions": [],
}
importer = BulkImportInline(json.dumps(data), 'ocladmin', True)
importer.run()
self.assertEqual(Concept.objects.filter(mnemonic='Food').count(), 2)
self.assertEqual(
Concept.objects.filter(
mnemonic='Food',
id=F('versioned_object_id')).first().versions.count(), 1)
self.assertTrue(
Concept.objects.filter(mnemonic='Food',
is_latest_version=True).exists())
self.assertEqual(importer.processed, 1)
self.assertEqual(len(importer.created), 1)
self.assertEqual(importer.failed, [])
self.assertTrue(importer.elapsed_seconds > 0)
data = {
"type":
"Concept",
"id":
"Food",
"concept_class":
"Root",
"datatype":
"Rule",
"source":
"DemoSource",
"owner":
"DemoOrg",
"owner_type":
"Organization",
"names": [{
"name": "Food",
"locale": "en",
"locale_preferred": "True",
"name_type": "Fully Specified"
}],
"descriptions": [],
}
importer = BulkImportInline(json.dumps(data), 'ocladmin', True)
importer.run()
self.assertEqual(
Concept.objects.filter(
mnemonic='Food',
id=F('versioned_object_id')).first().versions.count(), 2)
self.assertTrue(
Concept.objects.filter(mnemonic='Food',
is_latest_version=True,
datatype='Rule').exists())
self.assertEqual(importer.processed, 1)
self.assertEqual(len(importer.created), 0)
self.assertEqual(len(importer.updated), 1)
self.assertEqual(importer.failed, [])
self.assertTrue(importer.elapsed_seconds > 0)
def test_org_import(self):
self.assertFalse(
Organization.objects.filter(mnemonic='DATIM-MOH-BI-FY19').exists())
OrganizationFactory(mnemonic='DATIM-MOH-BI-FY19', location='blah')
self.assertTrue(
Organization.objects.filter(mnemonic='DATIM-MOH-BI-FY19').exists())
data = '{"type": "Organization", "__action": "DELETE", "id": "DATIM-MOH-BI-FY19"}\n' \
'{"name": "DATIM MOH Burundi", "extras": {"datim_moh_country_code": "BI", "datim_moh_period": "FY19",' \
' "datim_moh_object": true}, "location": "Burundi", "public_access": "None", "type": "Organization",' \
' "id": "DATIM-MOH-BI-FY19"}'
importer = BulkImportInline(data, 'ocladmin', True)
importer.run()
self.assertTrue(
Organization.objects.filter(mnemonic='DATIM-MOH-BI-FY19').exists())
self.assertEqual(importer.processed, 2)
self.assertEqual(len(importer.created), 1)
self.assertEqual(len(importer.deleted), 1)
self.assertTrue(importer.elapsed_seconds > 0)
data = {
"name": "DATIM MOH Burundi",
"extras": {
"datim_moh_country_code": "BI",
"datim_moh_period": "FY19",
"datim_moh_object": True
},
"location": "Burundi",
"public_access": "None",
"type": "Organization",
"id": "DATIM-MOH-BI-FY19"
}
importer = BulkImportInline(json.dumps(data), 'ocladmin', True)
importer.run()
self.assertEqual(importer.processed, 1)
self.assertEqual(len(importer.created), 0)
self.assertEqual(len(importer.failed), 0)
self.assertEqual(len(importer.deleted), 0)
self.assertEqual(len(importer.exists), 1)
self.assertEqual(importer.exists[0], data)
self.assertTrue(importer.elapsed_seconds > 0)
data = {"type": "Organization", "__action": "DELETE", "id": "FOOBAR"}
importer = BulkImportInline(json.dumps(data), 'ocladmin', True)
importer.run()
self.assertEqual(importer.processed, 1)
self.assertEqual(len(importer.created), 0)
self.assertEqual(len(importer.failed), 0)
self.assertEqual(len(importer.deleted), 0)
self.assertEqual(len(importer.exists), 0)
self.assertEqual(len(importer.not_found), 1)
self.assertEqual(importer.not_found[0], data)
self.assertTrue(importer.elapsed_seconds > 0)