def test_default_tags_invalid(self): config = {'default_tags': ['geo']} # should be list of dicts with assert_raises(toolkit.ValidationError) as harvest_context: run_harvest(url='http://localhost:%s' % mock_ckan.PORT, harvester=CKANHarvester(), config=json.dumps(config)) assert_in('default_tags must be a list of dictionaries', str(harvest_context.exception))
def test_default_tags_invalid(self): config = {'default_tags': ['geo']} # should be list of dicts with assert_raises(toolkit.ValidationError) as harvest_context: run_harvest( url='http://localhost:%s' % mock_ckan.PORT, harvester=CKANHarvester(), config=json.dumps(config)) assert_in('default_tags must be a list of dictionaries', str(harvest_context.exception))
def test_default_extras_invalid(self): config = { 'default_extras': 'utf8', # value should be a dict } with assert_raises(toolkit.ValidationError) as harvest_context: run_harvest(url='http://localhost:%s' % mock_ckan.PORT, harvester=CKANHarvester(), config=json.dumps(config)) assert_in('default_extras must be a dictionary', str(harvest_context.exception))
def test_default_extras_invalid(self): config = { 'default_extras': 'utf8', # value should be a dict } with assert_raises(toolkit.ValidationError) as harvest_context: run_harvest( url='http://localhost:%s' % mock_ckan.PORT, harvester=CKANHarvester(), config=json.dumps(config)) assert_in('default_extras must be a dictionary', str(harvest_context.exception))
def test_default_groups_invalid(self): Group(id='group2-id', name='group2') # should be list of strings config = {'default_groups': [{'name': 'group2'}]} with assert_raises(toolkit.ValidationError) as harvest_context: run_harvest(url='http://localhost:%s' % mock_ckan.PORT, harvester=CKANHarvester(), config=json.dumps(config)) assert_in('default_groups must be a list of group names/ids', str(harvest_context.exception))
def test_default_groups_invalid(self): Group(id='group2-id', name='group2') # should be list of strings config = {'default_groups': [{'name': 'group2'}]} with assert_raises(toolkit.ValidationError) as harvest_context: run_harvest( url='http://localhost:%s' % mock_ckan.PORT, harvester=CKANHarvester(), config=json.dumps(config)) assert_in('default_groups must be a list of group names/ids', str(harvest_context.exception))
def test_update_dataset(self): guid = 'obj-update' MockHarvester._set_test_params(guid=guid) # create the original harvest_object and dataset run_harvest(url='http://some-url.com', harvester=MockHarvester()) # update it results_by_guid = run_harvest(url='http://some-url.com', harvester=MockHarvester()) result = results_by_guid[guid] assert_equal(result['state'], 'COMPLETE') assert_equal(result['report_status'], 'updated') assert_equal(result['errors'], [])
def test_harvest_not_modified(self): run_harvest(url='http://localhost:%s/' % mock_ckan.PORT, harvester=CKANHarvester()) results_by_guid = run_harvest(url='http://localhost:%s/' % mock_ckan.PORT, harvester=CKANHarvester()) # The metadata_modified was the same for this dataset so the import # would have returned 'unchanged' result = results_by_guid[mock_ckan.DATASETS[1]['name']] assert_equal(result['state'], 'COMPLETE') assert_equal(result['report_status'], 'not modified') assert 'dataset' not in result assert_equal(result['errors'], [])
def test_default_extras(self): config = { 'default_extras': { 'encoding': 'utf8', 'harvest_url': '{harvest_source_url}/dataset/{dataset_id}' } } tmp_c = toolkit.c try: # c.user is used by the validation (annoying), # however patch doesn't work because it's a weird # StackedObjectProxy, so we swap it manually toolkit.c = MagicMock(user='') results_by_guid = run_harvest(url='http://localhost:%s' % mock_ckan.PORT, harvester=CKANHarvester(), config=json.dumps(config)) finally: toolkit.c = tmp_c assert_equal(results_by_guid['dataset1-id']['errors'], []) extras = results_by_guid['dataset1-id']['dataset']['extras'] extras_dict = dict((e['key'], e['value']) for e in extras) assert_equal(extras_dict['encoding'], 'utf8') assert_equal(extras_dict['harvest_url'], 'http://localhost:8998/dataset/dataset1-id')
def test_default_groups(self): Group(id='group1-id', name='group1') Group(id='group2-id', name='group2') Group(id='group3-id', name='group3') config = {'default_groups': ['group2-id', 'group3'], 'remote_groups': 'only_local'} tmp_c = toolkit.c try: # c.user is used by the validation (annoying), # however patch doesn't work because it's a weird # StackedObjectProxy, so we swap it manually toolkit.c = MagicMock(user='') results_by_guid = run_harvest( url='http://localhost:%s' % mock_ckan.PORT, harvester=CKANHarvester(), config=json.dumps(config)) finally: toolkit.c = tmp_c assert_equal(results_by_guid['dataset1-id']['errors'], []) groups = results_by_guid['dataset1-id']['dataset']['groups'] group_names = set(group['name'] for group in groups) # group1 comes from the harvested dataset # group2 & 3 come from the default_groups assert_equal(group_names, set(('group1', 'group2', 'group3')))
def test_harvest_not_modified(self): run_harvest( url='http://localhost:%s/' % mock_ckan.PORT, harvester=CKANHarvester()) results_by_guid = run_harvest( url='http://localhost:%s/' % mock_ckan.PORT, harvester=CKANHarvester()) # The metadata_modified was the same for this dataset so the import # would have returned 'unchanged' result = results_by_guid[mock_ckan.DATASETS[1]['id']] assert_equal(result['state'], 'COMPLETE') assert_equal(result['report_status'], 'not modified') assert 'dataset' not in result assert_equal(result['errors'], [])
def test_harvest_not_modified(self): run_harvest(url='http://localhost:%s/' % mock_ckan.PORT, harvester=CKANHarvester()) results_by_guid = run_harvest(url='http://localhost:%s/' % mock_ckan.PORT, harvester=CKANHarvester()) # The metadata_modified was the same for this dataset so the import # would have returned 'unchanged' result = results_by_guid[mock_ckan.DATASETS[1]['id']] assert result['state'] == 'COMPLETE' assert result['report_status'] == 'not modified' assert 'dataset' not in result assert result['errors'] == [] assert was_last_job_considered_error_free()
def test_default_groups(self): Group(id='group1-id', name='group1') Group(id='group2-id', name='group2') Group(id='group3-id', name='group3') config = { 'default_groups': ['group2-id', 'group3'], 'remote_groups': 'only_local' } tmp_c = toolkit.c try: # c.user is used by the validation (annoying), # however patch doesn't work because it's a weird # StackedObjectProxy, so we swap it manually toolkit.c = MagicMock(user='') results_by_guid = run_harvest(url='http://localhost:%s' % mock_ckan.PORT, harvester=CKANHarvester(), config=json.dumps(config)) finally: toolkit.c = tmp_c assert_equal(results_by_guid['dataset1-id']['errors'], []) groups = results_by_guid['dataset1-id']['dataset']['groups'] group_names = set(group['name'] for group in groups) # group1 comes from the harvested dataset # group2 & 3 come from the default_groups assert_equal(group_names, set(('group1', 'group2', 'group3')))
def test_include_groups(self): config = {'groups_filter_include': ['group1']} results_by_guid = run_harvest(url='http://localhost:%s' % mock_ckan.PORT, harvester=CKANHarvester(), config=json.dumps(config)) assert 'dataset1-id' in results_by_guid assert mock_ckan.DATASETS[1]['id'] not in results_by_guid
def test_harvest_whilst_datasets_added(self): results_by_guid = run_harvest( url='http://localhost:%s/datasets_added' % mock_ckan.PORT, harvester=CKANHarvester()) assert_equal(sorted(results_by_guid.keys()), [mock_ckan.DATASETS[1]['id'], mock_ckan.DATASETS[0]['id']])
def test_exclude_organizations(self): config = {'organizations_filter_exclude': ['org1']} results_by_guid = run_harvest(url='http://localhost:%s' % mock_ckan.PORT, harvester=CKANHarvester(), config=json.dumps(config)) assert 'dataset1-id' not in results_by_guid assert mock_ckan.DATASETS[1]['id'] in results_by_guid
def test_update_dataset(self): guid = 'obj-update' MockHarvester._set_test_params(guid=guid) # create the original harvest_object and dataset run_harvest( url='http://some-url.com', harvester=MockHarvester()) # update it results_by_guid = run_harvest( url='http://some-url.com', harvester=MockHarvester()) result = results_by_guid[guid] assert_equal(result['state'], 'COMPLETE') assert_equal(result['report_status'], 'updated') assert_equal(result['errors'], [])
def test_harvest_whilst_datasets_added(self): results_by_guid = run_harvest( url='http://localhost:%s/datasets_added' % mock_ckan.PORT, harvester=CKANHarvester()) assert_equal( sorted(results_by_guid.keys()), [mock_ckan.DATASETS[1]['id'], mock_ckan.DATASETS[0]['id']])
def test_include_organizations(self): config = {'organizations_filter_include': ['org1']} results_by_guid = run_harvest( url='http://localhost:%s' % mock_ckan.PORT, harvester=CKANHarvester(), config=json.dumps(config)) assert 'dataset1-id' in results_by_guid assert mock_ckan.DATASETS[1]['id'] not in results_by_guid
def test_exclude_groups(self): config = {'groups_filter_exclude': ['group1']} results_by_guid = run_harvest( url='http://localhost:%s' % mock_ckan.PORT, harvester=CKANHarvester(), config=json.dumps(config)) assert 'dataset1-id' not in results_by_guid assert mock_ckan.DATASETS[1]['id'] in results_by_guid
def test_harvest_twice(self): run_harvest( url='http://localhost:%s/' % mock_ckan.PORT, harvester=CKANHarvester()) results_by_guid = run_harvest( url='http://localhost:%s/' % mock_ckan.PORT, harvester=CKANHarvester()) # updated the dataset which has revisions result = results_by_guid['dataset1'] assert_equal(result['state'], 'COMPLETE') assert_equal(result['report_status'], 'updated') assert_equal(result['dataset']['name'], mock_ckan.DATASETS[0]['name']) assert_equal(result['errors'], []) # the other dataset is unchanged and not harvested assert mock_ckan.DATASETS[1]['name'] not in result
def test_default_tags(self): config = {'default_tags': [{'name': 'geo'}]} results_by_guid = run_harvest(url='http://localhost:%s' % mock_ckan.PORT, harvester=CKANHarvester(), config=json.dumps(config)) tags = results_by_guid['dataset1-id']['dataset']['tags'] tag_names = [tag['name'] for tag in tags] assert 'geo' in tag_names
def test_default_tags(self): config = {'default_tags': [{'name': 'geo'}]} results_by_guid = run_harvest( url='http://localhost:%s' % mock_ckan.PORT, harvester=CKANHarvester(), config=json.dumps(config)) tags = results_by_guid['dataset1-id']['dataset']['tags'] tag_names = [tag['name'] for tag in tags] assert 'geo' in tag_names
def test_remote_groups_create(self): config = {'remote_groups': 'create'} results_by_guid = run_harvest(url='http://localhost:%s' % mock_ckan.PORT, harvester=CKANHarvester(), config=json.dumps(config)) assert 'dataset1-id' in results_by_guid # Check that the remote group was created locally call_action('group_show', {}, id=mock_ckan.GROUPS[0]['id'])
def test_remote_groups_create(self): config = {'remote_groups': 'create'} results_by_guid = run_harvest( url='http://localhost:%s' % mock_ckan.PORT, harvester=CKANHarvester(), config=json.dumps(config)) assert 'dataset1-id' in results_by_guid # Check that the remote group was created locally call_action('group_show', {}, id=mock_ckan.GROUPS[0]['id'])
def test_delete_dataset(self): guid = 'obj-delete' MockHarvester._set_test_params(guid=guid) # create the original harvest_object and dataset run_harvest( url='http://some-url.com', harvester=MockHarvester()) MockHarvester._set_test_params(guid=guid, delete=True) # delete it results_by_guid = run_harvest( url='http://some-url.com', harvester=MockHarvester()) result = results_by_guid[guid] assert result['state'] == 'COMPLETE' assert result['report_status'] == 'deleted' assert result['errors'] == []
def test_harvest_invalid_tag(self): from nose.plugins.skip import SkipTest; raise SkipTest() results_by_guid = run_harvest( url='http://localhost:%s/invalid_tag' % mock_ckan.PORT, harvester=CKANHarvester()) result = results_by_guid['dataset1-id'] assert_equal(result['state'], 'COMPLETE') assert_equal(result['report_status'], 'added') assert_equal(result['dataset']['name'], mock_ckan.DATASETS[0]['name'])
def test_import_unchanged(self): guid = 'obj-error' MockHarvester._set_test_params(guid=guid, import_object_unchanged=True) results_by_guid = run_harvest(url='http://some-url.com', harvester=MockHarvester()) result = results_by_guid[guid] assert_equal(result['state'], 'COMPLETE') assert_equal(result['report_status'], 'not modified') assert_equal(result['errors'], [])
def test_obj_error(self): guid = 'obj-error' MockHarvester._set_test_params(guid=guid, object_error=True) results_by_guid = run_harvest(url='http://some-url.com', harvester=MockHarvester()) result = results_by_guid[guid] assert_equal(result['state'], 'ERROR') assert_equal(result['report_status'], 'errored') assert_equal(result['errors'], [])
def test_create_dataset(self): guid = 'obj-create' MockHarvester._set_test_params(guid=guid) results_by_guid = run_harvest(url='http://some-url.com', harvester=MockHarvester()) result = results_by_guid[guid] assert_equal(result['state'], 'COMPLETE') assert_equal(result['report_status'], 'added') assert_equal(result['errors'], [])
def test_obj_error(self): guid = 'obj-error' MockHarvester._set_test_params(guid=guid, object_error=True) results_by_guid = run_harvest( url='http://some-url.com', harvester=MockHarvester()) result = results_by_guid[guid] assert_equal(result['state'], 'ERROR') assert_equal(result['report_status'], 'errored') assert_equal(result['errors'], [])
def test_import_unchanged(self): guid = 'obj-error' MockHarvester._set_test_params(guid=guid, import_object_unchanged=True) results_by_guid = run_harvest( url='http://some-url.com', harvester=MockHarvester()) result = results_by_guid[guid] assert_equal(result['state'], 'COMPLETE') assert_equal(result['report_status'], 'not modified') assert_equal(result['errors'], [])
def test_create_dataset(self): guid = 'obj-create' MockHarvester._set_test_params(guid=guid) results_by_guid = run_harvest( url='http://some-url.com', harvester=MockHarvester()) result = results_by_guid[guid] assert_equal(result['state'], 'COMPLETE') assert_equal(result['report_status'], 'added') assert_equal(result['errors'], [])
def test_harvest_twice(self): run_harvest(url='http://localhost:%s/' % mock_ckan.PORT, harvester=CKANHarvester()) # change the modified date datasets = copy.deepcopy(mock_ckan.DATASETS) datasets[1]['metadata_modified'] = '2050-05-09T22:00:01.486366' with patch('ckanext.harvest.tests.harvesters.mock_ckan.DATASETS', datasets): results_by_guid = run_harvest(url='http://localhost:%s/' % mock_ckan.PORT, harvester=CKANHarvester()) # updated the dataset which has revisions result = results_by_guid[mock_ckan.DATASETS[1]['name']] assert_equal(result['state'], 'COMPLETE') assert_equal(result['report_status'], 'updated') assert_equal(result['dataset']['name'], mock_ckan.DATASETS[1]['name']) assert_equal(result['errors'], []) # the other dataset is unchanged and not harvested assert mock_ckan.DATASETS[1]['name'] not in result
def test_harvest_info_in_package_show(self): results_by_guid = run_harvest(url='http://localhost:%s' % mock_ckan.PORT, harvester=CKANHarvester()) assert 'dataset1-id' in results_by_guid # Check that the dataset extras has the harvest_object_id, harvest_source_id, and harvest_source_title dataset = call_action('package_show', {"for_view": True}, id=mock_ckan.DATASETS[0]['id']) extras_dict = dict((e['key'], e['value']) for e in dataset['extras']) assert 'harvest_object_id' in extras_dict assert 'harvest_source_id' in extras_dict assert 'harvest_source_title' in extras_dict
def test_harvest_twice(self): run_harvest( url='http://localhost:%s/' % mock_ckan.PORT, harvester=CKANHarvester()) # change the modified date datasets = copy.deepcopy(mock_ckan.DATASETS) datasets[1]['metadata_modified'] = '2050-05-09T22:00:01.486366' with patch('ckanext.harvest.tests.harvesters.mock_ckan.DATASETS', datasets): results_by_guid = run_harvest( url='http://localhost:%s/' % mock_ckan.PORT, harvester=CKANHarvester()) # updated the dataset which has revisions result = results_by_guid[mock_ckan.DATASETS[1]['id']] assert_equal(result['state'], 'COMPLETE') assert_equal(result['report_status'], 'updated') assert_equal(result['dataset']['name'], mock_ckan.DATASETS[1]['name']) assert_equal(result['errors'], []) # the other dataset is unchanged and not harvested assert mock_ckan.DATASETS[1]['name'] not in result
def test_harvest(self): results_by_guid = run_harvest(url='http://localhost:%s/' % mock_ckan.PORT, harvester=CKANHarvester()) result = results_by_guid['dataset1-id'] assert_equal(result['state'], 'COMPLETE') assert_equal(result['report_status'], 'added') assert_equal(result['dataset']['name'], mock_ckan.DATASETS[0]['name']) assert_equal(result['errors'], []) result = results_by_guid[mock_ckan.DATASETS[1]['id']] assert_equal(result['state'], 'COMPLETE') assert_equal(result['report_status'], 'added') assert_equal(result['dataset']['name'], mock_ckan.DATASETS[1]['name']) assert_equal(result['errors'], [])
def test_harvest(self): results_by_guid = run_harvest( url='http://localhost:%s/' % mock_ckan.PORT, harvester=CKANHarvester()) result = results_by_guid['dataset1-id'] assert_equal(result['state'], 'COMPLETE') assert_equal(result['report_status'], 'added') assert_equal(result['dataset']['name'], mock_ckan.DATASETS[0]['name']) assert_equal(result['errors'], []) result = results_by_guid[mock_ckan.DATASETS[1]['id']] assert_equal(result['state'], 'COMPLETE') assert_equal(result['report_status'], 'added') assert_equal(result['dataset']['name'], mock_ckan.DATASETS[1]['name']) assert_equal(result['errors'], [])
def test_default_extras(self): config = { 'default_extras': { 'encoding': 'utf8', 'harvest_url': '{harvest_source_url}/dataset/{dataset_id}' }} results_by_guid = run_harvest( url='http://localhost:%s' % mock_ckan.PORT, harvester=CKANHarvester(), config=json.dumps(config)) assert_equal(results_by_guid['dataset1-id']['errors'], []) extras = results_by_guid['dataset1-id']['dataset']['extras'] extras_dict = dict((e['key'], e['value']) for e in extras) assert_equal(extras_dict['encoding'], 'utf8') assert_equal(extras_dict['harvest_url'], 'http://localhost:8998/dataset/dataset1-id')
def test_harvest(self): results_by_guid = run_harvest(url='http://localhost:%s/' % mock_ckan.PORT, harvester=CKANHarvester()) result = results_by_guid['dataset1-id'] assert result['state'] == 'COMPLETE' assert result['report_status'] == 'added' assert result['dataset']['name'] == mock_ckan.DATASETS[0]['name'] assert result['errors'] == [] result = results_by_guid[mock_ckan.DATASETS[1]['id']] assert result['state'] == 'COMPLETE' assert result['report_status'] == 'added' assert result['dataset']['name'] == mock_ckan.DATASETS[1]['name'] assert result['errors'] == [] assert was_last_job_considered_error_free()
def test_remote_groups_only_local(self): # Create an existing group Group(id='group1-id', name='group1') config = {'remote_groups': 'only_local'} results_by_guid = run_harvest( url='http://localhost:%s' % mock_ckan.PORT, harvester=CKANHarvester(), config=json.dumps(config)) assert 'dataset1-id' in results_by_guid # Check that the dataset was added to the existing local group dataset = call_action('package_show', {}, id=mock_ckan.DATASETS[0]['id']) assert_equal(dataset['groups'][0]['id'], mock_ckan.DATASETS[0]['groups'][0]['id']) # Check that the other remote group was not created locally assert_raises(toolkit.ObjectNotFound, call_action, 'group_show', {}, id='remote-group')
def test_default_extras(self): config = { 'default_extras': { 'encoding': 'utf8', 'harvest_url': '{harvest_source_url}/dataset/{dataset_id}' }} tmp_c = toolkit.c try: # c.user is used by the validation (annoying), # however patch doesn't work because it's a weird # StackedObjectProxy, so we swap it manually toolkit.c = MagicMock(user='') results_by_guid = run_harvest( url='http://localhost:%s' % mock_ckan.PORT, harvester=CKANHarvester(), config=json.dumps(config)) finally: toolkit.c = tmp_c assert_equal(results_by_guid['dataset1-id']['errors'], []) extras = results_by_guid['dataset1-id']['dataset']['extras'] extras_dict = dict((e['key'], e['value']) for e in extras) assert_equal(extras_dict['encoding'], 'utf8') assert_equal(extras_dict['harvest_url'], 'http://localhost:8998/dataset/dataset1-id')
def test_OdgovltHarvester(app, db, mocker): sync = IvpkIrsSync(db) mocker.patch('odgovlt.IvpkIrsSync', return_value=sync) db.execute( sync.t.rinkmena.insert(), { 'PAVADINIMAS': 'Testinė rinkmena nr. 1', 'SANTRAUKA': 'Testas nr. 1', 'TINKLAPIS': 'http://www.testas1.lt', 'R_ZODZIAI': 'Šilumos tiekimo licencijas turinčių įmonių sąrašas,' 'šiluma,' 'šilumos tiekėjai,' 'licencijos,' 'licencijuojamos veiklos teritorija', 'K_EMAIL': '*****@*****.**', 'STATUSAS': 'U', 'USER_ID': 1, 'istaiga_id': 1, }) db.execute( sync.t.rinkmena.insert(), { 'PAVADINIMAS': 'Testinė rinkmena nr. 2', 'SANTRAUKA': 'Testas nr. 2', 'TINKLAPIS': 'http://www.testas2.lt', 'R_ZODZIAI': 'keliai,eismo intensyvumas,"e"', 'K_EMAIL': '*****@*****.**', 'STATUSAS': 'U', 'USER_ID': 2, 'istaiga_id': 2, }) db.execute( sync.t.user.insert(), { 'LOGIN': '******', 'PASS': '******', 'EMAIL': '*****@*****.**', 'TELEFONAS': '+37000000000', 'FIRST_NAME': 'Jonas', 'LAST_NAME': 'Jonaitis', }) db.execute( sync.t.user.insert(), { 'LOGIN': '******', 'PASS': '******', 'EMAIL': '*****@*****.**', 'TELEFONAS': '+37000000000', 'FIRST_NAME': 'Tomas', 'LAST_NAME': 'Tomauskas', }) db.execute( sync.t.istaiga.insert(), { 'PAVADINIMAS': 'Testinė organizacija nr. 1', 'KODAS': 888, 'ADRESAS': 'Testinė g. 9' }) db.execute( sync.t.istaiga.insert(), { 'PAVADINIMAS': 'Testinė organizacija nr. 2', 'KODAS': 777, 'ADRESAS': 'Testinė g. 91' }) db.execute(sync.t.kategorija.insert(), { 'PAVADINIMAS': 'testas1', 'KATEGORIJA_ID': 0, 'LYGIS': 1 }) db.execute(sync.t.kategorija.insert(), { 'PAVADINIMAS': 'testas2', 'KATEGORIJA_ID': 0, 'LYGIS': 1 }) db.execute(sync.t.kategorija.insert(), { 'PAVADINIMAS': 'testas3', 'KATEGORIJA_ID': 1, 'LYGIS': 2 }) db.execute(sync.t.kategorija.insert(), { 'PAVADINIMAS': 'testas4', 'KATEGORIJA_ID': 2, 'LYGIS': 2 }) db.execute(sync.t.kategorija.insert(), { 'PAVADINIMAS': 'testas5', 'KATEGORIJA_ID': 3, 'LYGIS': 3 }) db.execute(sync.t.kategorija.insert(), { 'PAVADINIMAS': 'testas6', 'KATEGORIJA_ID': 4, 'LYGIS': 3 }) db.execute(sync.t.kategorija.insert(), { 'PAVADINIMAS': 'testas7', 'KATEGORIJA_ID': 4, 'LYGIS': 3 }) db.execute(sync.t.kategorija_rinkmena.insert(), { 'KATEGORIJA_ID': 1, 'RINKMENA_ID': 1 }) db.execute(sync.t.kategorija_rinkmena.insert(), { 'KATEGORIJA_ID': 3, 'RINKMENA_ID': 2 }) ckanapi = CkanAPI({'user': '******'}) source = HarvestSourceObj(url='sqlite://', source_type='opendata-gov-lt') job = HarvestJobObj(source=source) harvester = OdgovltHarvester() obj_ids = harvester.gather_stage(job) assert job.gather_errors == [] assert ckanapi.group_list() == [ 'testas1-1', 'testas2-2', 'testas3-3', 'testas4-4', 'testas5-5', 'testas6-6', 'testas7-7', ] def subgroups(name): return [g['name'] for g in ckanapi.group_show(id=name)['groups']] assert subgroups('testas1-1') == ['testas3-3'] assert subgroups('testas2-2') == ['testas4-4'] assert subgroups('testas3-3') == ['testas5-5'] assert subgroups('testas4-4') == ['testas6-6', 'testas7-7'] assert subgroups('testas5-5') == [] assert subgroups('testas6-6') == [] assert subgroups('testas7-7') == [] assert [ json.loads( ckanext.harvest.model.HarvestObject.get(x).content)['PAVADINIMAS'] for x in obj_ids ] == [ 'Testinė rinkmena nr. 1', 'Testinė rinkmena nr. 2', ] database_data_list = [ dict(row) for row in db.execute(sync.t.rinkmena.select()) ] user1 = sync.sync_user(1) user2 = sync.sync_user(2) user3 = sync.sync_user(3) database_data_list[0]['VARDAS'] = user1['fullname'] database_data_list[1]['VARDAS'] = user2['fullname'] organization1 = sync.sync_organization(1) organization2 = sync.sync_organization(2) organization3 = sync.sync_organization(3) database_data_list[0]['ORGANIZACIJA'] = organization1['name'] database_data_list[1]['ORGANIZACIJA'] = organization2['name'] database_data_list[0]['KATEGORIJA_RINKMENA'] = json.dumps([{ 'ID': 1, 'KATEGORIJA_ID': 1, 'RINKMENA_ID': 1, }]) database_data_list[1]['KATEGORIJA_RINKMENA'] = json.dumps([{ 'ID': 2, 'KATEGORIJA_ID': 1, 'RINKMENA_ID': 2, }]) obj1 = HarvestObjectObj(guid=database_data_list[0]['ID'], job=job, content=json.dumps(database_data_list[0], cls=DatetimeEncoder)) result = harvester.fetch_stage(obj1) assert obj1.errors == [] assert result obj2 = HarvestObjectObj(guid=database_data_list[1]['ID'], job=job, content=json.dumps(database_data_list[1], cls=DatetimeEncoder)) result = harvester.fetch_stage(obj2) assert obj2.errors == [] assert result create_or_update = harvester.import_stage(obj1) assert create_or_update create_or_update = harvester.import_stage(obj2) assert create_or_update assert obj1.package_id assert obj2.package_id reset_db() sync = IvpkIrsSync(db) mocker.patch('odgovlt.IvpkIrsSync', return_value=sync) results_by_guid = run_harvest(url='sqlite://', harvester=OdgovltHarvester()) result = results_by_guid['1'] assert result['state'] == 'COMPLETE' assert result['report_status'] == 'added' assert result['errors'] == [] result = results_by_guid['2'] assert result['state'] == 'COMPLETE' assert result['report_status'] == 'added' assert result['errors'] == [] assert was_last_job_considered_error_free() ids = ckanapi.package_list() assert len(ids) == 3 package1 = ckanapi.package_show(id=ids[0]) package2 = ckanapi.package_show(id=ids[1]) assert package1['title'] == 'Testinė rinkmena nr. 1' assert package1['notes'] == 'Testas nr. 1' assert package1['url'] == 'http://www.testas1.lt' assert package1['maintainer'] == 'Jonas Jonaitis' assert package1['maintainer_email'] == '*****@*****.**' assert package1['organization']['title'] == 'Testinė organizacija nr. 1' assert package1['groups'] == [{ 'display_name': 'testas1', 'description': '', 'image_display_url': '', 'title': 'testas1', 'id': package1['groups'][0]['id'], 'name': 'testas1-1', }] assert package2['title'] == 'Testinė rinkmena nr. 2' assert package2['notes'] == 'Testas nr. 2' assert package2['url'] == 'http://www.testas2.lt' assert package2['maintainer'] == 'Tomas Tomauskas' assert package2['maintainer_email'] == '*****@*****.**' assert package2['organization']['title'] == 'Testinė organizacija nr. 2' assert package2['groups'] == [{ 'display_name': 'testas3', 'description': '', 'image_display_url': '', 'title': 'testas3', 'id': package2['groups'][0]['id'], 'name': 'testas3-3', }] assert user3['fullname'] == 'Unknown User' assert organization3['title'] == 'Unknown organization' fixcase_test = fixcase('Testas9') assert fixcase_test == 'testas9' tags_test = get_package_tags('aaaaaaaaaaaaaaaaaaaaaaaaaaaaaa' 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaa' 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaa' 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaa,' 'testas2 testas3, testas4 testas5; testas6') assert tags_test == [ 'testas2 testas3', 'testas4 testas5', 'testas6', ] tags1 = package1['tags'] tags2 = package2['tags'] assert sorted([x['name'] for x in tags1]) == [ 'licencijos', 'licencijuojamos veiklos teritorija', 'šiluma', 'šilumos tiekimo licencijas turinčių įmonių sąrašas', 'šilumos tiekėjai', ] assert sorted([x['name'] for x in tags2]) == [ 'eismo intensyvumas', 'keliai', ]
def test_harvest_site_down(self): results_by_guid = run_harvest(url='http://localhost:%s/site_down' % mock_ckan.PORT, harvester=CKANHarvester()) assert not results_by_guid assert not was_last_job_considered_error_free()
def test_harvest_site_down(self): results_by_guid = run_harvest( url='http://localhost:%s/site_down' % mock_ckan.PORT, harvester=CKANHarvester()) assert not results_by_guid assert not was_last_job_considered_error_free()