def setUp(self):
management.call_command('flush', interactive=False, verbosity=0)
self.datasetSyncer = DatasetSyncer()
iati_factory.LanguageFactory.create(code='en', name='English')
iati_factory.VersionFactory.create(code='2.02', name='2.02')
iati_factory.OrganisationTypeFactory.create(code='22',
name='Multilateral')
def test_remove_publisher_duplicates(self):
publisher = synchroniser_factory.PublisherFactory.create(org_id='NL-1')
publisher_duplicate = synchroniser_factory.PublisherFactory.create(
org_id='NL-1')
synchroniser_factory.DatasetFactory.create(
ref='first_set',
source_url='http://www.nourl.com/test1.xml',
publisher=publisher)
synchroniser_factory.DatasetFactory.create(
ref='second_set',
source_url='http://www.nourl.com/test2.xml',
publisher=publisher_duplicate)
syncer = DatasetSyncer()
syncer.remove_publisher_duplicates('NL-1')
# publisher duplicate should be removed, all datasets should be under the first publisher
self.assertEqual(publisher,
Publisher.objects.filter(org_id='NL-1')[0],
"first publisher should still be in the database")
self.assertEqual(
1, Publisher.objects.count(),
"publisher duplicate should be removed from the database")
self.assertEqual(2,
publisher.iatixmlsource_set.all().count(),
"Both XML sources should still be in the database")
self.assertEqual(2, IatiXmlSource.objects.count(),
"Both XML sources should still be in the database")
def test_if_publisher_assigned_to_source(self):
"""
Test if correct publisher assigned to IatiXmlSource
"""
syncer = DatasetSyncer()
with open('iati_synchroniser/fixtures/test_activity.json') as fixture:
data = json.load(fixture).get('results', [{}, ])[0]
syncer.parse_json_line(data, 1)
source = IatiXmlSource.objects.get(ref="cic-sl")
publisher = Publisher.objects.get(org_id="GB-CHC-1020488")
self.assertEqual(publisher, source.publisher,
"IatiXmlSource should have correct publisher")
def test_parsed_published_data_integrity(self):
"""
Test if publisher data parsed correctly
"""
syncer = DatasetSyncer()
with open('iati_synchroniser/fixtures/test_activity.json') as fixture:
data = json.load(fixture).get('results', [{}, ])[0]
syncer.parse_json_line(data, 1)
publisher = Publisher.objects.all()[0]
self.assertEqual("GB-CHC-1020488", publisher.org_id)
self.assertEqual("", publisher.org_abbreviate)
self.assertEqual("Children in Crisis", publisher.org_name)
def setUp(self):
# XXX: previously, django's 'flush' management command was called to
# flush the database, but it breaks tests ('no table blah blah exists')
# and etc., so let's just manually remove objects which were created
# during previous fixtures.
# TODO: get rid of fixtures and use factory-boy everywhere.
Publisher.objects.all().delete()
self.datasetSyncer = DatasetSyncer()
iati_factory.LanguageFactory.create(code='en', name='English')
iati_factory.VersionFactory.create(code='2.02', name='2.02')
iati_factory.OrganisationTypeFactory.create(code='22',
name='Multilateral')
self.BASE_DIR = os.path.dirname(
os.path.dirname(os.path.realpath(__file__)))
def test_if_publisher_assigned_to_source(self):
"""
Test if correct publisher assigned to IatiXmlSource
"""
syncer = DatasetSyncer()
with open('iati_synchroniser/fixtures/test_activity.json') as fixture:
data = json.load(fixture).get('results', [
{},
])[0]
syncer.parse_json_line(data)
source = IatiXmlSource.objects.get(ref="cic-sl")
publisher = Publisher.objects.get(org_id="GB-CHC-1020488")
self.assertEqual(publisher, source.publisher,
"IatiXmlSource should have correct publisher")
def test_parsed_published_data_integrity(self):
"""
Test if publisher data parsed correctly
"""
syncer = DatasetSyncer()
with open('iati_synchroniser/fixtures/test_activity.json') as fixture:
data = json.load(fixture).get('results', [
{},
])[0]
syncer.parse_json_line(data)
publisher = Publisher.objects.all()[0]
self.assertEqual("GB-CHC-1020488", publisher.org_id)
self.assertEqual("cic", publisher.org_abbreviate)
self.assertEqual("Children in Crisis", publisher.org_name)
def test_line_parser(self):
"""
Test if activity source data parsed
"""
syncer = DatasetSyncer()
publishers_count = Publisher.objects.count()
sources_count = IatiXmlSource.objects.count()
with open('iati_synchroniser/fixtures/test_activity.json') as fixture:
data = json.load(fixture).get('results', [{}, ])[0]
syncer.parse_json_line(data, 1)
self.assertNotEqual(publishers_count, Publisher.objects.count(),
"New publisher should be added into database")
self.assertNotEqual(sources_count, IatiXmlSource.objects.count(),
"New IatiXmlSource should be added into database")
def test_line_parser(self):
"""
Test if activity source data parsed
"""
syncer = DatasetSyncer()
publishers_count = Publisher.objects.count()
sources_count = IatiXmlSource.objects.count()
with open('iati_synchroniser/fixtures/test_activity.json') as fixture:
data = json.load(fixture).get('results', [
{},
])[0]
syncer.parse_json_line(data)
self.assertNotEqual(publishers_count, Publisher.objects.count(),
"New publisher should be added into database")
self.assertNotEqual(sources_count, IatiXmlSource.objects.count(),
"New IatiXmlSource should be added into database")
def test_parsed_xml_source_data_integrity(self):
"""
Test if activity source data parsed correctly
"""
syncer = DatasetSyncer()
with open('iati_synchroniser/fixtures/test_activity.json') as fixture:
data = json.load(fixture).get('results', [{}, ])[0]
syncer.parse_json_line(data, 1)
source = IatiXmlSource.objects.all()[0]
self.assertEqual("cic-sl", source.ref)
self.assertEqual("http://aidstream.org/files/xml/cic-sl.xml",
source.source_url)
self.assertEqual("Children in Crisis Activity File for Sierra leone",
source.title)
self.assertFalse(source.is_parsed,
"New IatiXmlSource should not be marked as parsed")
def test_parsed_xml_source_data_integrity(self):
"""
Test if activity source data parsed correctly
"""
syncer = DatasetSyncer()
with open('iati_synchroniser/fixtures/test_activity.json') as fixture:
data = json.load(fixture).get('results', [
{},
])[0]
syncer.parse_json_line(data)
source = IatiXmlSource.objects.all()[0]
self.assertEqual("cic-sl", source.ref)
self.assertEqual("http://aidstream.org/files/xml/cic-sl.xml",
source.source_url)
self.assertEqual("Children in Crisis Activity File for Sierra leone",
source.title)
self.assertFalse(source.is_parsed,
"New IatiXmlSource should not be marked as parsed")
def setUp(self):
# XXX: previously, django's 'flush' management command was called to
# flush the database, but it breaks tests ('no table blah blah exists')
# and etc., so let's just manually remove objects which were created
# during previous fixtures.
# TODO: get rid of fixtures and use factory-boy everywhere.
Publisher.objects.all().delete()
self.datasetSyncer = DatasetSyncer()
iati_factory.LanguageFactory.create(code='en', name='English')
iati_factory.VersionFactory.create(code='2.02', name='2.02')
iati_factory.OrganisationTypeFactory.create(
code='22', name='Multilateral')
self.BASE_DIR = os.path.dirname(
os.path.dirname(os.path.realpath(__file__))
)
class DatasetSyncerTestCase(TestCase):
"""
Test DatasetSyncer functionality
"""
def setUp(self):
management.call_command('flush', interactive=False, verbosity=0)
self.datasetSyncer = DatasetSyncer()
def test_get_val_in_list_of_dicts(self):
"""
test if returns correct key's data
"""
input_list = [
{
"key": "filetype",
"value": "organisation"
},
{
"key": "version",
"value": "2.02"
},
]
keyval = self.datasetSyncer.get_val_in_list_of_dicts(
"filetype", input_list)
self.assertEqual(keyval, input_list[0])
# @unittest.skip("Not implemented")
def test_synchronize_with_iati_api(self):
"""
"""
with open('iati_synchroniser/fixtures/test_publisher.json') as fixture:
publisher = json.load(fixture).get('result')[0]
with open('iati_synchroniser/fixtures/test_dataset.json') as fixture:
dataset = json.load(fixture)['result']['results'][0]
self.datasetSyncer.get_data = MagicMock(side_effect=[
{
'result': [publisher]
}, # first occurance, return 1 publisher
{
'result': []
}, # 2nd, return empty publisher db return
{
'result': {
'results': [dataset]
}
}, # 3rd, return 1 dataset
{
'result': {
'results': []
}
} # 4th, return empty dataset db return
])
self.datasetSyncer.synchronize_with_iati_api()
self.assertEqual(Publisher.objects.count(), 1)
self.assertEqual(Dataset.objects.count(), 1)
def test_update_or_create_publisher(self):
"""
check if dataset is saved as expected
"""
with open('iati_synchroniser/fixtures/test_publisher.json') as fixture:
data = json.load(fixture).get('result')[0]
self.datasetSyncer.update_or_create_publisher(data)
publisher = Publisher.objects.all()[0]
self.assertEqual("NP-SWC-27693", publisher.publisher_iati_id)
self.assertEqual("Aasaman Nepal", publisher.display_name)
self.assertEqual("aasaman", publisher.name)
def test_update_or_create_dataset(self):
"""
"""
publisher = Publisher(
id="8797b894-9858-492e-a109-dc45b75ce27b",
publisher_iati_id="",
display_name="jica",
name="Japan International Cooperation Agency (JICA)")
publisher.save()
with open('iati_synchroniser/fixtures/test_dataset.json') as fixture:
data = json.load(fixture)['result']['results'][0]
self.datasetSyncer.update_or_create_dataset(data)
dataset = Dataset.objects.all()[0]
print dataset.publisher
self.assertEqual("43aa0616-58a4-4d16-b0a9-1181e3871827", dataset.id)
self.assertEqual("cic-sl", dataset.name)
self.assertEqual("088States Ex-Yugoslavia unspecified2013",
dataset.title)
self.assertEqual(publisher, dataset.publisher)
self.assertEqual("http://aidstream.org/files/xml/cic-sl.xml",
dataset.source_url)
self.assertEqual(1, dataset.filetype)
class DatasetSyncerTestCase(TestCase):
"""
Test DatasetSyncer functionality
"""
def setUp(self):
# XXX: previously, django's 'flush' management command was called to
# flush the database, but it breaks tests ('no table blah blah exists')
# and etc., so let's just manually remove objects which were created
# during previous fixtures.
# TODO: get rid of fixtures and use factory-boy everywhere.
Publisher.objects.all().delete()
self.datasetSyncer = DatasetSyncer()
iati_factory.LanguageFactory.create(code='en', name='English')
iati_factory.VersionFactory.create(code='2.02', name='2.02')
iati_factory.OrganisationTypeFactory.create(
code='22', name='Multilateral')
self.BASE_DIR = os.path.dirname(
os.path.dirname(os.path.realpath(__file__))
)
def test_get_val_in_list_of_dicts(self):
"""
test if returns correct key's data
"""
input_list = [
{"key": "filetype", "value": "organisation"},
{"key": "version", "value": "2.02"},
]
keyval = self.datasetSyncer.get_val_in_list_of_dicts(
"filetype", input_list)
self.assertEqual(keyval, input_list[0])
def test_synchronize_with_iati_api(self):
"""
"""
with open(self.BASE_DIR + '/fixtures/test_publisher.json') as fixture:
publisher = json.load(fixture).get('result')[0]
with open(self.BASE_DIR + '/fixtures/test_dataset.json') as fixture:
dataset = json.load(fixture)['result']['results'][0]
self.datasetSyncer.get_data = MagicMock(side_effect=[
{'result': [publisher]}, # first occurance, return 1 publisher
{'result': []}, # 2nd, return empty publisher db return
{'result': {'results': [dataset]}}, # 3rd, return 1 dataset
{'result': {'results': []}} # 4th, return empty dataset db return
])
self.datasetSyncer.synchronize_with_iati_api()
self.assertEqual(Publisher.objects.count(), 1)
self.assertEqual(Dataset.objects.count(), 1)
def test_update_or_create_publisher(self):
"""
check if dataset is saved as expected
"""
with open(self.BASE_DIR + '/fixtures/test_publisher.json') as fixture:
data = json.load(fixture).get('result')[0]
self.datasetSyncer.update_or_create_publisher(data)
publisher = Publisher.objects.last()
self.assertEqual("NP-SWC-27693", publisher.publisher_iati_id)
self.assertEqual("Aasaman Nepal", publisher.display_name)
self.assertEqual("aasaman", publisher.name)
def test_update_or_create_dataset(self):
"""
"""
publisher = Publisher(
iati_id="85d72513-66b6-4642-a526-214b1081fff1",
publisher_iati_id="",
display_name="jica",
name="Japan International Cooperation Agency (JICA)")
publisher.save()
with open(self.BASE_DIR + '/fixtures/test_dataset.json') as fixture:
data = json.load(fixture)['result']['results'][0]
self.datasetSyncer.update_or_create_dataset(data)
dataset = Dataset.objects.all()[0]
self.assertEqual(
"43aa0616-58a4-4d16-b0a9-1181e3871827", dataset.iati_id)
self.assertEqual("cic-sl", dataset.name)
self.assertEqual(
"088States Ex-Yugoslavia unspecified2013", dataset.title)
self.assertEqual(publisher, dataset.publisher)
self.assertEqual(
"http://aidstream.org/files/xml/cic-sl.xml", dataset.source_url)
self.assertEqual(1, dataset.filetype)
def handle(self, *args, **options):
ds = DatasetSyncer()
ds.synchronize_with_iati_api()
def handle(self, *args, **options):
ds = DatasetSyncer()
ds.synchronize_with_iati_api(1)
class DatasetSyncerTestCase(TestCase):
"""
Test DatasetSyncer functionality
"""
def setUp(self):
# XXX: previously, django's 'flush' management command was called to
# flush the database, but it breaks tests ('no table blah blah exists')
# and etc., so let's just manually remove objects which were created
# during previous fixtures.
# TODO: get rid of fixtures and use factory-boy everywhere.
Publisher.objects.all().delete()
self.datasetSyncer = DatasetSyncer()
iati_factory.LanguageFactory.create(code='en', name='English')
iati_factory.VersionFactory.create(code='2.02', name='2.02')
iati_factory.OrganisationTypeFactory.create(code='22',
name='Multilateral')
self.BASE_DIR = os.path.dirname(
os.path.dirname(os.path.realpath(__file__)))
def test_get_val_in_list_of_dicts(self):
"""
test if returns correct key's data
"""
input_list = [
{
"key": "filetype",
"value": "organisation"
},
{
"key": "version",
"value": "2.02"
},
]
keyval = self.datasetSyncer.get_val_in_list_of_dicts(
"filetype", input_list)
self.assertEqual(keyval, input_list[0])
@unittest.skip("Not implemented")
def test_synchronize_with_iati_api(self):
"""
"""
with open(self.BASE_DIR + '/fixtures/test_publisher.json') as fixture:
publisher = json.load(fixture).get('result')[0]
with open(self.BASE_DIR + '/fixtures/test_dataset.json') as fixture:
dataset = json.load(fixture)['result']['results'][0]
self.datasetSyncer.get_data = MagicMock(side_effect=[
{
'result': [publisher]
}, # first occurance, return 1 publisher
{
'result': []
}, # 2nd, return empty publisher db return
{
'result': {
'results': [dataset]
}
}, # 3rd, return 1 dataset
{
'result': {
'results': []
}
} # 4th, return empty dataset db return
])
self.datasetSyncer.synchronize_with_iati_api()
self.assertEqual(Publisher.objects.count(), 1)
self.assertEqual(Dataset.objects.count(), 1)
@unittest.skip("Not implemented")
def test_update_or_create_publisher(self):
"""
check if dataset is saved as expected
"""
with open(self.BASE_DIR + '/fixtures/test_publisher.json') as fixture:
data = json.load(fixture).get('result')[0]
self.datasetSyncer.update_or_create_publisher(data)
publisher = Publisher.objects.last()
self.assertEqual("NP-SWC-27693", publisher.publisher_iati_id)
self.assertEqual("Aasaman Nepal", publisher.display_name)
self.assertEqual("aasaman", publisher.name)
@unittest.skip("Not implemented")
def test_update_or_create_dataset(self):
"""
"""
publisher = Publisher(
iati_id="85d72513-66b6-4642-a526-214b1081fff1",
publisher_iati_id="",
display_name="jica",
name="Japan International Cooperation Agency (JICA)")
publisher.save()
with open(self.BASE_DIR + '/fixtures/test_dataset.json') as fixture:
data = json.load(fixture)['result']['results'][0]
self.datasetSyncer.update_or_create_dataset(data)
dataset = Dataset.objects.all()[0]
self.assertEqual("43aa0616-58a4-4d16-b0a9-1181e3871827",
dataset.iati_id)
self.assertEqual("cic-sl", dataset.name)
self.assertEqual("088States Ex-Yugoslavia unspecified2013",
dataset.title)
self.assertEqual(publisher, dataset.publisher)
self.assertEqual("http://aidstream.org/files/xml/cic-sl.xml",
dataset.source_url)
self.assertEqual(1, dataset.filetype)
def handle(self, *args, **options):
ds = DatasetSyncer()
ds.synchronize_with_iati_api(is_download_datasets=True)
def setUp(self):
management.call_command('flush', interactive=False, verbosity=0)
self.datasetSyncer = DatasetSyncer()
def get_new_sources_from_iati_api():
from iati_synchroniser.dataset_syncer import DatasetSyncer
ds = DatasetSyncer()
ds.synchronize_with_iati_api(1)
def sync_dataset_with_iati_api(self):
syncer = DatasetSyncer()
syncer.synchronize_with_iati_api(self.type)
def get_new_sources_from_iati_api():
from iati_synchroniser.dataset_syncer import DatasetSyncer
ds = DatasetSyncer()
ds.synchronize_with_iati_api(1)
def get_new_organisations_from_iati_api():
from iati_synchroniser.dataset_syncer import DatasetSyncer
ds = DatasetSyncer()
ds.synchronize_with_iati_api(2)
