def test_custom_config(self): config = { 'next_key': 'next_page', 'results_key': 'items', 'fields': ['classification', 'origin_id', 'name', 'parent'], 'field_config': { 'classification': { 'data_type': 'link', } } } importer = RestAPIImporter('http://fake.url/', config) self.assertEqual(importer.next_key, 'next_page') self.assertEqual(importer.results_key, 'items') expected_field_config = { 'parent': { 'data_type': 'link', }, 'origin_id': { 'data_type': 'str_lower', }, 'classification': { 'data_type': 'link', } } self.assertDictEqual(importer.field_config, expected_field_config)
def handle(self, *args, **options): url = options['url'] config = {} if options['rename_data_source']: rename_data_source = {} for item in options['rename_data_source']: old_name, new_name = self._parse_rename_data_source(item) rename_data_source[old_name] = new_name config['rename_data_source'] = rename_data_source importer = RestAPIImporter(url, config) try: importer.import_data() except DataImportError as e: self.stderr.write('Import failed: {0}'.format(e)) else: self.stdout.write('Import completed successfully')
def setUp(self): self.config = { 'next_key': None, 'results_key': None, 'fields': [ 'origin_id', 'classification', 'name', 'parent', ], 'update_fields': [ 'classification', 'name', 'parent', ], 'field_config': { 'parent': { 'source_field': 'parent_id', 'data_type': 'value', 'optional': True, }, 'origin_id': { 'source_field': 'id', 'data_type': 'value', }, 'classification': { 'source_field': 'organization_type', 'optional': True, }, 'name': { 'source_field': 'name_fi', 'optional': True, } }, 'default_data_source': 'tprek' } self.importer = RestAPIImporter('http://fake.tprek.url/department/', self.config)
class TestTprekRestApiImporter(TestRestApiImporter): # here we test all the features of the TPREK import that are different from default REST import def setUp(self): self.config = { 'next_key': None, 'results_key': None, 'fields': [ 'origin_id', 'classification', 'name', 'parent', ], 'update_fields': [ 'classification', 'name', 'parent', ], 'field_config': { 'parent': { 'source_field': 'parent_id', 'data_type': 'value', 'optional': True, }, 'origin_id': { 'source_field': 'id', 'data_type': 'value', }, 'classification': { 'source_field': 'organization_type', 'optional': True, }, 'name': { 'source_field': 'name_fi', 'optional': True, } }, 'default_data_source': 'tprek' } self.importer = RestAPIImporter('http://fake.tprek.url/department/', self.config) @patch('requests.get', MagicMock(side_effect=mock_tprek_request_get)) def test_get_field_value_related_fields(self): value = self.importer._get_field_value({'data_source': 'tprek'}, 'data_source', {}) self.assertEqual(value.id, 'tprek') value = self.importer._get_field_value( tprek_organization_1, 'classification', self.config['field_config']['classification']) self.assertEqual(value.id, 'tprek:TEST_TYPE_1') value = self.importer._get_field_value( tprek_organization_1, 'parent', self.config['field_config']['parent']) self.assertEqual(value.id, 'tprek:222') @patch('requests.get', MagicMock(side_effect=mock_tprek_request_get)) def test_data_iter(self): url = 'http://fake.tprek.url/department/' iterator = self.importer._data_iter(url) iter_len = len(list(iterator)) self.assertEqual(iter_len, 3) url = 'http://not-exist.url/organizations/' self.assertRaises(DataImportError, list, self.importer._data_iter(url)) @patch('requests.get', MagicMock(side_effect=mock_tprek_request_get)) def test_import_data(self): self.importer.import_data() data_source_model = get_data_source_model() self.assertEqual(Organization.objects.count(), 3) self.assertEqual(data_source_model.objects.count(), 1) self.assertEqual(OrganizationClass.objects.count(), 2) def test_import_organization_class_with_simple_string(self): organization_class = self.importer._import_organization_class( 'class-with-no-source') qs = OrganizationClass.objects.all() self.assertQuerysetEqual(qs, [repr(organization_class)]) self.assertEqual(organization_class.id, 'tprek:class-with-no-source') def test_import_organization_class_with_dict_data(self): pass def test_import_organization_class_with_simple_dict_data(self): pass def test_import_organization_update_existing(self): organization = OrganizationFactory( name='existing-organization', origin_id=tprek_organization_2['id'], data_source=self.importer._import_data_source('tprek'), ) self.importer._import_organization(tprek_organization_2) organization.refresh_from_db() self.assertQuerysetEqual(Organization.objects.all(), [repr(organization)]) self.assertEqual(organization.name, 'Organization-2') @patch('requests.get', MagicMock(side_effect=mock_tprek_request_get)) def test_import_organization_with_parent(self): organization = self.importer._import_organization(tprek_organization_1) qs = Organization.objects.all() # also created parent organization, but the name is empty at the moment self.assertQuerysetEqual( qs, [repr(organization.parent), repr(organization)], ordered=False) self.assertEqual(organization.name, 'Organization-1') self.assertNotEqual(organization.id, 111) self.assertEqual(organization.parent.name, '') self.assertNotEqual(organization.parent_id, 222) @patch('requests.get', MagicMock(side_effect=mock_tprek_request_get)) def test_import_organization_without_parent(self): organization = self.importer._import_organization(tprek_organization_2) qs = Organization.objects.all() self.assertQuerysetEqual(qs, [repr(organization)]) self.assertEqual(organization.name, 'Organization-2') self.assertNotEqual(organization.id, 222) def test_import_organization_with_string(self): organization = self.importer._import_organization('test-value') qs = Organization.objects.all() # created empty organization with only the id self.assertQuerysetEqual(qs, [repr(organization)]) self.assertEqual(organization.name, '') self.assertNotEqual(organization.id, 'test-value')
def setUp(self): self.importer = RestAPIImporter( 'http://fake.url/organizations/?page=1')
class TestRestApiImporter(TestCase): def setUp(self): self.importer = RestAPIImporter( 'http://fake.url/organizations/?page=1') def test_custom_config(self): config = { 'next_key': 'next_page', 'results_key': 'items', 'fields': ['classification', 'origin_id', 'name', 'parent'], 'field_config': { 'classification': { 'data_type': 'link', } } } importer = RestAPIImporter('http://fake.url/', config) self.assertEqual(importer.next_key, 'next_page') self.assertEqual(importer.results_key, 'items') expected_field_config = { 'parent': { 'data_type': 'link', }, 'origin_id': { 'data_type': 'str_lower', }, 'classification': { 'data_type': 'link', } } self.assertDictEqual(importer.field_config, expected_field_config) def test_get_organization_class(self): data = {'id': 'test-source:test-class'} organization_class = self.importer._get_organization_class(data) self.assertEqual(organization_class.id, 'test-source:test-class') self.assertEqual(OrganizationClass.objects.count(), 1) self.importer._get_organization_class(data) self.assertEqual(OrganizationClass.objects.count(), 1) # fetched from cached def test_get_data_source(self): data = {'id': 'test-source'} data_source = self.importer._get_data_source(data) self.assertEqual(data_source.id, 'test-source') data_source_model = get_data_source_model() self.assertEqual(data_source_model.objects.count(), 1) self.importer._get_data_source(data) self.assertEqual(data_source_model.objects.count(), 1) # fetched from cached self.importer.config['rename_data_source'] = { 'test-source': 'new-source-name' } data_source = self.importer._get_data_source(data) self.assertEqual(data_source.id, 'new-source-name') self.assertEqual(data_source_model.objects.count(), 2) @patch('requests.get', MagicMock(side_effect=mock_request_get)) def test_import_data(self): self.importer.import_data() data_source_model = get_data_source_model() self.assertEqual(Organization.objects.count(), 3) self.assertEqual(data_source_model.objects.count(), 2) self.assertEqual(OrganizationClass.objects.count(), 2) @patch('requests.get', MagicMock(side_effect=mock_request_get)) def test_import_organization_with_parent(self): organization = self.importer._import_organization(organization_1) qs = Organization.objects.all() # also created parent organization self.assertQuerysetEqual( qs, [repr(organization.parent), repr(organization)], ordered=False) self.assertEqual(organization.name, 'Organization-1') self.assertNotEqual(organization.id, 111) self.assertEqual(organization.parent.name, 'Organization-2') self.assertNotEqual(organization.parent_id, 222) @patch('requests.get', MagicMock(side_effect=mock_request_get)) def test_import_organization_without_parent(self): organization = self.importer._import_organization(organization_2) qs = Organization.objects.all() self.assertQuerysetEqual(qs, [repr(organization)]) self.assertEqual(organization.name, 'Organization-2') self.assertNotEqual(organization.id, 222) def test_import_organization_with_string(self): self.assertRaises(DataImportError, self.importer._import_organization, 'test-value') def test_import_organization_update_existing(self): organization = OrganizationFactory( name='existing-organization', origin_id=organization_2['origin_id'], data_source=self.importer._import_data_source( organization_2['data_source']), ) self.importer._import_organization(organization_2) organization.refresh_from_db() self.assertQuerysetEqual(Organization.objects.all(), [repr(organization)]) self.assertEqual(organization.name, 'Organization-2') def test_import_data_source_with_string(self): data_source = self.importer._import_data_source('test-data-source') data_source_model = get_data_source_model() qs = data_source_model.objects.all() self.assertQuerysetEqual(qs, [repr(data_source)]) self.assertEqual(data_source.id, 'test-data-source') def test_import_data_source_with_dict_data(self): data = {'id': 'test-data-source'} data_source = self.importer._import_data_source(data) data_source_model = get_data_source_model() qs = data_source_model.objects.all() self.assertQuerysetEqual(qs, [repr(data_source)]) self.assertEqual(data_source.id, 'test-data-source') def test_import_organization_class_with_string(self): organization_class = self.importer._import_organization_class( 'test-source-1:test-org-class') qs = OrganizationClass.objects.all() self.assertQuerysetEqual(qs, [repr(organization_class)]) self.assertEqual(organization_class.id, 'test-source-1:test-org-class') def test_import_organization_class_with_simple_string(self): organization_class = self.importer._import_organization_class( 'class-with-no-source') qs = OrganizationClass.objects.all() self.assertQuerysetEqual(qs, [repr(organization_class)]) self.assertEqual(organization_class.id, 'OpenDecisionAPI:class-with-no-source') def test_import_organization_class_with_dict_data(self): data = { 'id': 999, 'origin_id': 'test-org-class', 'name': 'test-org-class', 'data_source': 'test-source-1', } organization_class = self.importer._import_organization_class(data) qs = OrganizationClass.objects.all() self.assertQuerysetEqual(qs, [repr(organization_class)]) self.assertEqual(organization_class.name, 'test-org-class') self.assertEqual(organization_class.id, 'test-source-1:test-org-class') def test_import_organization_class_with_simple_dict_data(self): data = {'id': 999, 'name': 'test-org-class'} organization_class = self.importer._import_organization_class(data) qs = OrganizationClass.objects.all() self.assertQuerysetEqual(qs, [repr(organization_class)]) self.assertEqual(organization_class.name, 'test-org-class') self.assertEqual(organization_class.id, 'OpenDecisionAPI:999') @patch('requests.get', MagicMock(side_effect=mock_request_get)) def test_data_iter(self): url = 'http://fake.url/organizations/?page=1' iterator = self.importer._data_iter(url) iter_len = len(list(iterator)) self.assertEqual(iter_len, 3) url = 'http://fake.url/organizations/?page=2' iterator = self.importer._data_iter(url) iter_len = len(list(iterator)) self.assertEqual(iter_len, 1) url = 'http://not-exist.url/organizations/' self.assertRaises(DataImportError, list, self.importer._data_iter(url)) def test_get_field_value_empty_value(self): value = self.importer._get_field_value(organization_2, 'dissolution_date', {}) self.assertIsNone(value) def test_get_field_value_different_source_field(self): config = { 'source_field': 'id', } value = self.importer._get_field_value(organization_1, 'origin_id', config) self.assertEqual(value, 111) def test_get_field_value_not_exist_field(self): config = { 'source_field': 'not-exist-field', } self.assertRaises( DataImportError, self.importer._get_field_value, organization_1, 'origin_id', config, ) def test_get_field_value_not_exist_data_type(self): config = { 'data_type': 'not-exist-data-type', } self.assertRaises( DataImportError, self.importer._get_field_value, organization_1, 'origin_id', config, ) @patch('requests.get', MagicMock(return_value=MockResponse('test-get-return'))) def test_get_field_value_link_data_type(self): config = { 'data_type': 'link', } value = self.importer._get_field_value({'name': 'http://fake.url/'}, 'name', config) self.assertEqual(value, 'test-get-return') def test_get_field_value_str_lower_data_type(self): config = { 'data_type': 'str_lower', } value = self.importer._get_field_value(organization_1, 'origin_id', config) self.assertEqual(value, 'abc-123') def test_get_field_value_regex_data_type(self): config = { 'data_type': 'regex', } # test raise DataImportError if no pattern is provided self.assertRaises( DataImportError, self.importer._get_field_value, organization_1, 'origin_id', config, ) config = { 'data_type': 'regex', 'pattern': r'\w+\-(\d+)', } value = self.importer._get_field_value(organization_1, 'origin_id', config) self.assertEqual(value, '123') @patch('requests.get', MagicMock(side_effect=mock_request_get)) def test_get_field_value_related_fields(self): value = self.importer._get_field_value(organization_1, 'data_source', {}) self.assertEqual(value.id, 'test-source-1') value = self.importer._get_field_value(organization_1, 'classification', {}) self.assertEqual(value.id, 'test-source-1:test-class-1') value = self.importer._get_field_value(organization_1, 'parent', {'data_type': 'link'}) self.assertEqual(value.name, 'Organization-2') def test_get_regex_data(self): pattern = r'\w+\-(\d+)' self.assertRaises( DataImportError, self.importer._get_regex_data, '123', pattern, ) data = self.importer._get_regex_data('abc-123', pattern) self.assertEqual(data, '123') @patch('requests.get', MagicMock(side_effect=mock_request_get)) def test_get_link_data(self): invalid_url = 'abc.123' self.assertRaises(DataImportError, self.importer._get_link_data, invalid_url) url = 'http://not-exist.url/organizations/111/' self.assertRaises(DataImportError, self.importer._get_link_data, url) url = 'http://fake.url/organizations/111/' data = self.importer._get_link_data(url) self.assertEqual(data, organization_1)