def test_default_tags_invalid(self):
config = {'default_tags': ['geo']} # should be list of dicts
with assert_raises(toolkit.ValidationError) as harvest_context:
run_harvest(url='http://localhost:%s' % mock_ckan.PORT,
harvester=CKANHarvester(),
config=json.dumps(config))
assert_in('default_tags must be a list of dictionaries',
str(harvest_context.exception))
def test_default_tags_invalid(self):
config = {'default_tags': ['geo']} # should be list of dicts
with assert_raises(toolkit.ValidationError) as harvest_context:
run_harvest(
url='http://localhost:%s' % mock_ckan.PORT,
harvester=CKANHarvester(),
config=json.dumps(config))
assert_in('default_tags must be a list of dictionaries',
str(harvest_context.exception))
def test_default_extras_invalid(self):
config = {
'default_extras': 'utf8', # value should be a dict
}
with assert_raises(toolkit.ValidationError) as harvest_context:
run_harvest(url='http://localhost:%s' % mock_ckan.PORT,
harvester=CKANHarvester(),
config=json.dumps(config))
assert_in('default_extras must be a dictionary',
str(harvest_context.exception))
def test_default_extras_invalid(self):
config = {
'default_extras': 'utf8', # value should be a dict
}
with assert_raises(toolkit.ValidationError) as harvest_context:
run_harvest(
url='http://localhost:%s' % mock_ckan.PORT,
harvester=CKANHarvester(),
config=json.dumps(config))
assert_in('default_extras must be a dictionary',
str(harvest_context.exception))
def test_default_groups_invalid(self):
Group(id='group2-id', name='group2')
# should be list of strings
config = {'default_groups': [{'name': 'group2'}]}
with assert_raises(toolkit.ValidationError) as harvest_context:
run_harvest(url='http://localhost:%s' % mock_ckan.PORT,
harvester=CKANHarvester(),
config=json.dumps(config))
assert_in('default_groups must be a list of group names/ids',
str(harvest_context.exception))
def test_default_groups_invalid(self):
Group(id='group2-id', name='group2')
# should be list of strings
config = {'default_groups': [{'name': 'group2'}]}
with assert_raises(toolkit.ValidationError) as harvest_context:
run_harvest(
url='http://localhost:%s' % mock_ckan.PORT,
harvester=CKANHarvester(),
config=json.dumps(config))
assert_in('default_groups must be a list of group names/ids',
str(harvest_context.exception))
def test_update_dataset(self):
guid = 'obj-update'
MockHarvester._set_test_params(guid=guid)
# create the original harvest_object and dataset
run_harvest(url='http://some-url.com', harvester=MockHarvester())
# update it
results_by_guid = run_harvest(url='http://some-url.com',
harvester=MockHarvester())
result = results_by_guid[guid]
assert_equal(result['state'], 'COMPLETE')
assert_equal(result['report_status'], 'updated')
assert_equal(result['errors'], [])
def test_harvest_not_modified(self):
run_harvest(url='http://localhost:%s/' % mock_ckan.PORT,
harvester=CKANHarvester())
results_by_guid = run_harvest(url='http://localhost:%s/' %
mock_ckan.PORT,
harvester=CKANHarvester())
# The metadata_modified was the same for this dataset so the import
# would have returned 'unchanged'
result = results_by_guid[mock_ckan.DATASETS[1]['name']]
assert_equal(result['state'], 'COMPLETE')
assert_equal(result['report_status'], 'not modified')
assert 'dataset' not in result
assert_equal(result['errors'], [])
def test_default_extras(self):
config = {
'default_extras': {
'encoding': 'utf8',
'harvest_url': '{harvest_source_url}/dataset/{dataset_id}'
}
}
tmp_c = toolkit.c
try:
# c.user is used by the validation (annoying),
# however patch doesn't work because it's a weird
# StackedObjectProxy, so we swap it manually
toolkit.c = MagicMock(user='')
results_by_guid = run_harvest(url='http://localhost:%s' %
mock_ckan.PORT,
harvester=CKANHarvester(),
config=json.dumps(config))
finally:
toolkit.c = tmp_c
assert_equal(results_by_guid['dataset1-id']['errors'], [])
extras = results_by_guid['dataset1-id']['dataset']['extras']
extras_dict = dict((e['key'], e['value']) for e in extras)
assert_equal(extras_dict['encoding'], 'utf8')
assert_equal(extras_dict['harvest_url'],
'http://localhost:8998/dataset/dataset1-id')
def test_default_groups(self):
Group(id='group1-id', name='group1')
Group(id='group2-id', name='group2')
Group(id='group3-id', name='group3')
config = {'default_groups': ['group2-id', 'group3'],
'remote_groups': 'only_local'}
tmp_c = toolkit.c
try:
# c.user is used by the validation (annoying),
# however patch doesn't work because it's a weird
# StackedObjectProxy, so we swap it manually
toolkit.c = MagicMock(user='')
results_by_guid = run_harvest(
url='http://localhost:%s' % mock_ckan.PORT,
harvester=CKANHarvester(),
config=json.dumps(config))
finally:
toolkit.c = tmp_c
assert_equal(results_by_guid['dataset1-id']['errors'], [])
groups = results_by_guid['dataset1-id']['dataset']['groups']
group_names = set(group['name'] for group in groups)
# group1 comes from the harvested dataset
# group2 & 3 come from the default_groups
assert_equal(group_names, set(('group1', 'group2', 'group3')))
def test_harvest_not_modified(self):
run_harvest(
url='http://localhost:%s/' % mock_ckan.PORT,
harvester=CKANHarvester())
results_by_guid = run_harvest(
url='http://localhost:%s/' % mock_ckan.PORT,
harvester=CKANHarvester())
# The metadata_modified was the same for this dataset so the import
# would have returned 'unchanged'
result = results_by_guid[mock_ckan.DATASETS[1]['id']]
assert_equal(result['state'], 'COMPLETE')
assert_equal(result['report_status'], 'not modified')
assert 'dataset' not in result
assert_equal(result['errors'], [])
def test_harvest_not_modified(self):
run_harvest(url='http://localhost:%s/' % mock_ckan.PORT,
harvester=CKANHarvester())
results_by_guid = run_harvest(url='http://localhost:%s/' %
mock_ckan.PORT,
harvester=CKANHarvester())
# The metadata_modified was the same for this dataset so the import
# would have returned 'unchanged'
result = results_by_guid[mock_ckan.DATASETS[1]['id']]
assert result['state'] == 'COMPLETE'
assert result['report_status'] == 'not modified'
assert 'dataset' not in result
assert result['errors'] == []
assert was_last_job_considered_error_free()
def test_default_groups(self):
Group(id='group1-id', name='group1')
Group(id='group2-id', name='group2')
Group(id='group3-id', name='group3')
config = {
'default_groups': ['group2-id', 'group3'],
'remote_groups': 'only_local'
}
tmp_c = toolkit.c
try:
# c.user is used by the validation (annoying),
# however patch doesn't work because it's a weird
# StackedObjectProxy, so we swap it manually
toolkit.c = MagicMock(user='')
results_by_guid = run_harvest(url='http://localhost:%s' %
mock_ckan.PORT,
harvester=CKANHarvester(),
config=json.dumps(config))
finally:
toolkit.c = tmp_c
assert_equal(results_by_guid['dataset1-id']['errors'], [])
groups = results_by_guid['dataset1-id']['dataset']['groups']
group_names = set(group['name'] for group in groups)
# group1 comes from the harvested dataset
# group2 & 3 come from the default_groups
assert_equal(group_names, set(('group1', 'group2', 'group3')))
def test_include_groups(self):
config = {'groups_filter_include': ['group1']}
results_by_guid = run_harvest(url='http://localhost:%s' %
mock_ckan.PORT,
harvester=CKANHarvester(),
config=json.dumps(config))
assert 'dataset1-id' in results_by_guid
assert mock_ckan.DATASETS[1]['id'] not in results_by_guid
def test_harvest_whilst_datasets_added(self):
results_by_guid = run_harvest(
url='http://localhost:%s/datasets_added' % mock_ckan.PORT,
harvester=CKANHarvester())
assert_equal(sorted(results_by_guid.keys()),
[mock_ckan.DATASETS[1]['id'],
mock_ckan.DATASETS[0]['id']])
def test_exclude_organizations(self):
config = {'organizations_filter_exclude': ['org1']}
results_by_guid = run_harvest(url='http://localhost:%s' %
mock_ckan.PORT,
harvester=CKANHarvester(),
config=json.dumps(config))
assert 'dataset1-id' not in results_by_guid
assert mock_ckan.DATASETS[1]['id'] in results_by_guid
def test_update_dataset(self):
guid = 'obj-update'
MockHarvester._set_test_params(guid=guid)
# create the original harvest_object and dataset
run_harvest(
url='http://some-url.com',
harvester=MockHarvester())
# update it
results_by_guid = run_harvest(
url='http://some-url.com',
harvester=MockHarvester())
result = results_by_guid[guid]
assert_equal(result['state'], 'COMPLETE')
assert_equal(result['report_status'], 'updated')
assert_equal(result['errors'], [])
def test_harvest_whilst_datasets_added(self):
results_by_guid = run_harvest(
url='http://localhost:%s/datasets_added' % mock_ckan.PORT,
harvester=CKANHarvester())
assert_equal(
sorted(results_by_guid.keys()),
[mock_ckan.DATASETS[1]['id'], mock_ckan.DATASETS[0]['id']])
def test_include_organizations(self):
config = {'organizations_filter_include': ['org1']}
results_by_guid = run_harvest(
url='http://localhost:%s' % mock_ckan.PORT,
harvester=CKANHarvester(),
config=json.dumps(config))
assert 'dataset1-id' in results_by_guid
assert mock_ckan.DATASETS[1]['id'] not in results_by_guid
def test_exclude_groups(self):
config = {'groups_filter_exclude': ['group1']}
results_by_guid = run_harvest(
url='http://localhost:%s' % mock_ckan.PORT,
harvester=CKANHarvester(),
config=json.dumps(config))
assert 'dataset1-id' not in results_by_guid
assert mock_ckan.DATASETS[1]['id'] in results_by_guid
def test_harvest_twice(self):
run_harvest(
url='http://localhost:%s/' % mock_ckan.PORT,
harvester=CKANHarvester())
results_by_guid = run_harvest(
url='http://localhost:%s/' % mock_ckan.PORT,
harvester=CKANHarvester())
# updated the dataset which has revisions
result = results_by_guid['dataset1']
assert_equal(result['state'], 'COMPLETE')
assert_equal(result['report_status'], 'updated')
assert_equal(result['dataset']['name'], mock_ckan.DATASETS[0]['name'])
assert_equal(result['errors'], [])
# the other dataset is unchanged and not harvested
assert mock_ckan.DATASETS[1]['name'] not in result
def test_default_tags(self):
config = {'default_tags': [{'name': 'geo'}]}
results_by_guid = run_harvest(url='http://localhost:%s' %
mock_ckan.PORT,
harvester=CKANHarvester(),
config=json.dumps(config))
tags = results_by_guid['dataset1-id']['dataset']['tags']
tag_names = [tag['name'] for tag in tags]
assert 'geo' in tag_names
def test_default_tags(self):
config = {'default_tags': [{'name': 'geo'}]}
results_by_guid = run_harvest(
url='http://localhost:%s' % mock_ckan.PORT,
harvester=CKANHarvester(),
config=json.dumps(config))
tags = results_by_guid['dataset1-id']['dataset']['tags']
tag_names = [tag['name'] for tag in tags]
assert 'geo' in tag_names
def test_remote_groups_create(self):
config = {'remote_groups': 'create'}
results_by_guid = run_harvest(url='http://localhost:%s' %
mock_ckan.PORT,
harvester=CKANHarvester(),
config=json.dumps(config))
assert 'dataset1-id' in results_by_guid
# Check that the remote group was created locally
call_action('group_show', {}, id=mock_ckan.GROUPS[0]['id'])
def test_remote_groups_create(self):
config = {'remote_groups': 'create'}
results_by_guid = run_harvest(
url='http://localhost:%s' % mock_ckan.PORT,
harvester=CKANHarvester(),
config=json.dumps(config))
assert 'dataset1-id' in results_by_guid
# Check that the remote group was created locally
call_action('group_show', {}, id=mock_ckan.GROUPS[0]['id'])
def test_delete_dataset(self):
guid = 'obj-delete'
MockHarvester._set_test_params(guid=guid)
# create the original harvest_object and dataset
run_harvest(
url='http://some-url.com',
harvester=MockHarvester())
MockHarvester._set_test_params(guid=guid, delete=True)
# delete it
results_by_guid = run_harvest(
url='http://some-url.com',
harvester=MockHarvester())
result = results_by_guid[guid]
assert result['state'] == 'COMPLETE'
assert result['report_status'] == 'deleted'
assert result['errors'] == []
def test_harvest_invalid_tag(self):
from nose.plugins.skip import SkipTest; raise SkipTest()
results_by_guid = run_harvest(
url='http://localhost:%s/invalid_tag' % mock_ckan.PORT,
harvester=CKANHarvester())
result = results_by_guid['dataset1-id']
assert_equal(result['state'], 'COMPLETE')
assert_equal(result['report_status'], 'added')
assert_equal(result['dataset']['name'], mock_ckan.DATASETS[0]['name'])
def test_harvest_invalid_tag(self):
from nose.plugins.skip import SkipTest; raise SkipTest()
results_by_guid = run_harvest(
url='http://localhost:%s/invalid_tag' % mock_ckan.PORT,
harvester=CKANHarvester())
result = results_by_guid['dataset1-id']
assert_equal(result['state'], 'COMPLETE')
assert_equal(result['report_status'], 'added')
assert_equal(result['dataset']['name'], mock_ckan.DATASETS[0]['name'])
def test_import_unchanged(self):
guid = 'obj-error'
MockHarvester._set_test_params(guid=guid, import_object_unchanged=True)
results_by_guid = run_harvest(url='http://some-url.com',
harvester=MockHarvester())
result = results_by_guid[guid]
assert_equal(result['state'], 'COMPLETE')
assert_equal(result['report_status'], 'not modified')
assert_equal(result['errors'], [])
def test_obj_error(self):
guid = 'obj-error'
MockHarvester._set_test_params(guid=guid, object_error=True)
results_by_guid = run_harvest(url='http://some-url.com',
harvester=MockHarvester())
result = results_by_guid[guid]
assert_equal(result['state'], 'ERROR')
assert_equal(result['report_status'], 'errored')
assert_equal(result['errors'], [])
def test_create_dataset(self):
guid = 'obj-create'
MockHarvester._set_test_params(guid=guid)
results_by_guid = run_harvest(url='http://some-url.com',
harvester=MockHarvester())
result = results_by_guid[guid]
assert_equal(result['state'], 'COMPLETE')
assert_equal(result['report_status'], 'added')
assert_equal(result['errors'], [])
def test_obj_error(self):
guid = 'obj-error'
MockHarvester._set_test_params(guid=guid, object_error=True)
results_by_guid = run_harvest(
url='http://some-url.com',
harvester=MockHarvester())
result = results_by_guid[guid]
assert_equal(result['state'], 'ERROR')
assert_equal(result['report_status'], 'errored')
assert_equal(result['errors'], [])
def test_import_unchanged(self):
guid = 'obj-error'
MockHarvester._set_test_params(guid=guid, import_object_unchanged=True)
results_by_guid = run_harvest(
url='http://some-url.com',
harvester=MockHarvester())
result = results_by_guid[guid]
assert_equal(result['state'], 'COMPLETE')
assert_equal(result['report_status'], 'not modified')
assert_equal(result['errors'], [])
def test_create_dataset(self):
guid = 'obj-create'
MockHarvester._set_test_params(guid=guid)
results_by_guid = run_harvest(
url='http://some-url.com',
harvester=MockHarvester())
result = results_by_guid[guid]
assert_equal(result['state'], 'COMPLETE')
assert_equal(result['report_status'], 'added')
assert_equal(result['errors'], [])
def test_harvest_twice(self):
run_harvest(url='http://localhost:%s/' % mock_ckan.PORT,
harvester=CKANHarvester())
# change the modified date
datasets = copy.deepcopy(mock_ckan.DATASETS)
datasets[1]['metadata_modified'] = '2050-05-09T22:00:01.486366'
with patch('ckanext.harvest.tests.harvesters.mock_ckan.DATASETS',
datasets):
results_by_guid = run_harvest(url='http://localhost:%s/' %
mock_ckan.PORT,
harvester=CKANHarvester())
# updated the dataset which has revisions
result = results_by_guid[mock_ckan.DATASETS[1]['name']]
assert_equal(result['state'], 'COMPLETE')
assert_equal(result['report_status'], 'updated')
assert_equal(result['dataset']['name'], mock_ckan.DATASETS[1]['name'])
assert_equal(result['errors'], [])
# the other dataset is unchanged and not harvested
assert mock_ckan.DATASETS[1]['name'] not in result
def test_harvest_info_in_package_show(self):
results_by_guid = run_harvest(url='http://localhost:%s' %
mock_ckan.PORT,
harvester=CKANHarvester())
assert 'dataset1-id' in results_by_guid
# Check that the dataset extras has the harvest_object_id, harvest_source_id, and harvest_source_title
dataset = call_action('package_show', {"for_view": True},
id=mock_ckan.DATASETS[0]['id'])
extras_dict = dict((e['key'], e['value']) for e in dataset['extras'])
assert 'harvest_object_id' in extras_dict
assert 'harvest_source_id' in extras_dict
assert 'harvest_source_title' in extras_dict
def test_harvest_twice(self):
run_harvest(
url='http://localhost:%s/' % mock_ckan.PORT,
harvester=CKANHarvester())
# change the modified date
datasets = copy.deepcopy(mock_ckan.DATASETS)
datasets[1]['metadata_modified'] = '2050-05-09T22:00:01.486366'
with patch('ckanext.harvest.tests.harvesters.mock_ckan.DATASETS',
datasets):
results_by_guid = run_harvest(
url='http://localhost:%s/' % mock_ckan.PORT,
harvester=CKANHarvester())
# updated the dataset which has revisions
result = results_by_guid[mock_ckan.DATASETS[1]['id']]
assert_equal(result['state'], 'COMPLETE')
assert_equal(result['report_status'], 'updated')
assert_equal(result['dataset']['name'], mock_ckan.DATASETS[1]['name'])
assert_equal(result['errors'], [])
# the other dataset is unchanged and not harvested
assert mock_ckan.DATASETS[1]['name'] not in result
def test_harvest(self):
results_by_guid = run_harvest(url='http://localhost:%s/' %
mock_ckan.PORT,
harvester=CKANHarvester())
result = results_by_guid['dataset1-id']
assert_equal(result['state'], 'COMPLETE')
assert_equal(result['report_status'], 'added')
assert_equal(result['dataset']['name'], mock_ckan.DATASETS[0]['name'])
assert_equal(result['errors'], [])
result = results_by_guid[mock_ckan.DATASETS[1]['id']]
assert_equal(result['state'], 'COMPLETE')
assert_equal(result['report_status'], 'added')
assert_equal(result['dataset']['name'], mock_ckan.DATASETS[1]['name'])
assert_equal(result['errors'], [])
def test_harvest(self):
results_by_guid = run_harvest(
url='http://localhost:%s/' % mock_ckan.PORT,
harvester=CKANHarvester())
result = results_by_guid['dataset1-id']
assert_equal(result['state'], 'COMPLETE')
assert_equal(result['report_status'], 'added')
assert_equal(result['dataset']['name'], mock_ckan.DATASETS[0]['name'])
assert_equal(result['errors'], [])
result = results_by_guid[mock_ckan.DATASETS[1]['id']]
assert_equal(result['state'], 'COMPLETE')
assert_equal(result['report_status'], 'added')
assert_equal(result['dataset']['name'], mock_ckan.DATASETS[1]['name'])
assert_equal(result['errors'], [])
def test_default_extras(self):
config = {
'default_extras': {
'encoding': 'utf8',
'harvest_url': '{harvest_source_url}/dataset/{dataset_id}'
}}
results_by_guid = run_harvest(
url='http://localhost:%s' % mock_ckan.PORT,
harvester=CKANHarvester(),
config=json.dumps(config))
assert_equal(results_by_guid['dataset1-id']['errors'], [])
extras = results_by_guid['dataset1-id']['dataset']['extras']
extras_dict = dict((e['key'], e['value']) for e in extras)
assert_equal(extras_dict['encoding'], 'utf8')
assert_equal(extras_dict['harvest_url'],
'http://localhost:8998/dataset/dataset1-id')
def test_harvest(self):
results_by_guid = run_harvest(url='http://localhost:%s/' %
mock_ckan.PORT,
harvester=CKANHarvester())
result = results_by_guid['dataset1-id']
assert result['state'] == 'COMPLETE'
assert result['report_status'] == 'added'
assert result['dataset']['name'] == mock_ckan.DATASETS[0]['name']
assert result['errors'] == []
result = results_by_guid[mock_ckan.DATASETS[1]['id']]
assert result['state'] == 'COMPLETE'
assert result['report_status'] == 'added'
assert result['dataset']['name'] == mock_ckan.DATASETS[1]['name']
assert result['errors'] == []
assert was_last_job_considered_error_free()
def test_remote_groups_only_local(self):
# Create an existing group
Group(id='group1-id', name='group1')
config = {'remote_groups': 'only_local'}
results_by_guid = run_harvest(
url='http://localhost:%s' % mock_ckan.PORT,
harvester=CKANHarvester(),
config=json.dumps(config))
assert 'dataset1-id' in results_by_guid
# Check that the dataset was added to the existing local group
dataset = call_action('package_show', {}, id=mock_ckan.DATASETS[0]['id'])
assert_equal(dataset['groups'][0]['id'], mock_ckan.DATASETS[0]['groups'][0]['id'])
# Check that the other remote group was not created locally
assert_raises(toolkit.ObjectNotFound, call_action, 'group_show', {},
id='remote-group')
def test_remote_groups_only_local(self):
# Create an existing group
Group(id='group1-id', name='group1')
config = {'remote_groups': 'only_local'}
results_by_guid = run_harvest(
url='http://localhost:%s' % mock_ckan.PORT,
harvester=CKANHarvester(),
config=json.dumps(config))
assert 'dataset1-id' in results_by_guid
# Check that the dataset was added to the existing local group
dataset = call_action('package_show', {}, id=mock_ckan.DATASETS[0]['id'])
assert_equal(dataset['groups'][0]['id'], mock_ckan.DATASETS[0]['groups'][0]['id'])
# Check that the other remote group was not created locally
assert_raises(toolkit.ObjectNotFound, call_action, 'group_show', {},
id='remote-group')
def test_default_extras(self):
config = {
'default_extras': {
'encoding': 'utf8',
'harvest_url': '{harvest_source_url}/dataset/{dataset_id}'
}}
tmp_c = toolkit.c
try:
# c.user is used by the validation (annoying),
# however patch doesn't work because it's a weird
# StackedObjectProxy, so we swap it manually
toolkit.c = MagicMock(user='')
results_by_guid = run_harvest(
url='http://localhost:%s' % mock_ckan.PORT,
harvester=CKANHarvester(),
config=json.dumps(config))
finally:
toolkit.c = tmp_c
assert_equal(results_by_guid['dataset1-id']['errors'], [])
extras = results_by_guid['dataset1-id']['dataset']['extras']
extras_dict = dict((e['key'], e['value']) for e in extras)
assert_equal(extras_dict['encoding'], 'utf8')
assert_equal(extras_dict['harvest_url'],
'http://localhost:8998/dataset/dataset1-id')
def test_OdgovltHarvester(app, db, mocker):
sync = IvpkIrsSync(db)
mocker.patch('odgovlt.IvpkIrsSync', return_value=sync)
db.execute(
sync.t.rinkmena.insert(), {
'PAVADINIMAS':
'Testinė rinkmena nr. 1',
'SANTRAUKA':
'Testas nr. 1',
'TINKLAPIS':
'http://www.testas1.lt',
'R_ZODZIAI':
'Šilumos tiekimo licencijas turinčių įmonių sąrašas,'
'šiluma,'
'šilumos tiekėjai,'
'licencijos,'
'licencijuojamos veiklos teritorija',
'K_EMAIL':
'*****@*****.**',
'STATUSAS':
'U',
'USER_ID':
1,
'istaiga_id':
1,
})
db.execute(
sync.t.rinkmena.insert(), {
'PAVADINIMAS': 'Testinė rinkmena nr. 2',
'SANTRAUKA': 'Testas nr. 2',
'TINKLAPIS': 'http://www.testas2.lt',
'R_ZODZIAI': 'keliai,eismo intensyvumas,"e"',
'K_EMAIL': '*****@*****.**',
'STATUSAS': 'U',
'USER_ID': 2,
'istaiga_id': 2,
})
db.execute(
sync.t.user.insert(), {
'LOGIN': '******',
'PASS': '******',
'EMAIL': '*****@*****.**',
'TELEFONAS': '+37000000000',
'FIRST_NAME': 'Jonas',
'LAST_NAME': 'Jonaitis',
})
db.execute(
sync.t.user.insert(), {
'LOGIN': '******',
'PASS': '******',
'EMAIL': '*****@*****.**',
'TELEFONAS': '+37000000000',
'FIRST_NAME': 'Tomas',
'LAST_NAME': 'Tomauskas',
})
db.execute(
sync.t.istaiga.insert(), {
'PAVADINIMAS': 'Testinė organizacija nr. 1',
'KODAS': 888,
'ADRESAS': 'Testinė g. 9'
})
db.execute(
sync.t.istaiga.insert(), {
'PAVADINIMAS': 'Testinė organizacija nr. 2',
'KODAS': 777,
'ADRESAS': 'Testinė g. 91'
})
db.execute(sync.t.kategorija.insert(), {
'PAVADINIMAS': 'testas1',
'KATEGORIJA_ID': 0,
'LYGIS': 1
})
db.execute(sync.t.kategorija.insert(), {
'PAVADINIMAS': 'testas2',
'KATEGORIJA_ID': 0,
'LYGIS': 1
})
db.execute(sync.t.kategorija.insert(), {
'PAVADINIMAS': 'testas3',
'KATEGORIJA_ID': 1,
'LYGIS': 2
})
db.execute(sync.t.kategorija.insert(), {
'PAVADINIMAS': 'testas4',
'KATEGORIJA_ID': 2,
'LYGIS': 2
})
db.execute(sync.t.kategorija.insert(), {
'PAVADINIMAS': 'testas5',
'KATEGORIJA_ID': 3,
'LYGIS': 3
})
db.execute(sync.t.kategorija.insert(), {
'PAVADINIMAS': 'testas6',
'KATEGORIJA_ID': 4,
'LYGIS': 3
})
db.execute(sync.t.kategorija.insert(), {
'PAVADINIMAS': 'testas7',
'KATEGORIJA_ID': 4,
'LYGIS': 3
})
db.execute(sync.t.kategorija_rinkmena.insert(), {
'KATEGORIJA_ID': 1,
'RINKMENA_ID': 1
})
db.execute(sync.t.kategorija_rinkmena.insert(), {
'KATEGORIJA_ID': 3,
'RINKMENA_ID': 2
})
ckanapi = CkanAPI({'user': '******'})
source = HarvestSourceObj(url='sqlite://', source_type='opendata-gov-lt')
job = HarvestJobObj(source=source)
harvester = OdgovltHarvester()
obj_ids = harvester.gather_stage(job)
assert job.gather_errors == []
assert ckanapi.group_list() == [
'testas1-1',
'testas2-2',
'testas3-3',
'testas4-4',
'testas5-5',
'testas6-6',
'testas7-7',
]
def subgroups(name):
return [g['name'] for g in ckanapi.group_show(id=name)['groups']]
assert subgroups('testas1-1') == ['testas3-3']
assert subgroups('testas2-2') == ['testas4-4']
assert subgroups('testas3-3') == ['testas5-5']
assert subgroups('testas4-4') == ['testas6-6', 'testas7-7']
assert subgroups('testas5-5') == []
assert subgroups('testas6-6') == []
assert subgroups('testas7-7') == []
assert [
json.loads(
ckanext.harvest.model.HarvestObject.get(x).content)['PAVADINIMAS']
for x in obj_ids
] == [
'Testinė rinkmena nr. 1',
'Testinė rinkmena nr. 2',
]
database_data_list = [
dict(row) for row in db.execute(sync.t.rinkmena.select())
]
user1 = sync.sync_user(1)
user2 = sync.sync_user(2)
user3 = sync.sync_user(3)
database_data_list[0]['VARDAS'] = user1['fullname']
database_data_list[1]['VARDAS'] = user2['fullname']
organization1 = sync.sync_organization(1)
organization2 = sync.sync_organization(2)
organization3 = sync.sync_organization(3)
database_data_list[0]['ORGANIZACIJA'] = organization1['name']
database_data_list[1]['ORGANIZACIJA'] = organization2['name']
database_data_list[0]['KATEGORIJA_RINKMENA'] = json.dumps([{
'ID':
1,
'KATEGORIJA_ID':
1,
'RINKMENA_ID':
1,
}])
database_data_list[1]['KATEGORIJA_RINKMENA'] = json.dumps([{
'ID':
2,
'KATEGORIJA_ID':
1,
'RINKMENA_ID':
2,
}])
obj1 = HarvestObjectObj(guid=database_data_list[0]['ID'],
job=job,
content=json.dumps(database_data_list[0],
cls=DatetimeEncoder))
result = harvester.fetch_stage(obj1)
assert obj1.errors == []
assert result
obj2 = HarvestObjectObj(guid=database_data_list[1]['ID'],
job=job,
content=json.dumps(database_data_list[1],
cls=DatetimeEncoder))
result = harvester.fetch_stage(obj2)
assert obj2.errors == []
assert result
create_or_update = harvester.import_stage(obj1)
assert create_or_update
create_or_update = harvester.import_stage(obj2)
assert create_or_update
assert obj1.package_id
assert obj2.package_id
reset_db()
sync = IvpkIrsSync(db)
mocker.patch('odgovlt.IvpkIrsSync', return_value=sync)
results_by_guid = run_harvest(url='sqlite://',
harvester=OdgovltHarvester())
result = results_by_guid['1']
assert result['state'] == 'COMPLETE'
assert result['report_status'] == 'added'
assert result['errors'] == []
result = results_by_guid['2']
assert result['state'] == 'COMPLETE'
assert result['report_status'] == 'added'
assert result['errors'] == []
assert was_last_job_considered_error_free()
ids = ckanapi.package_list()
assert len(ids) == 3
package1 = ckanapi.package_show(id=ids[0])
package2 = ckanapi.package_show(id=ids[1])
assert package1['title'] == 'Testinė rinkmena nr. 1'
assert package1['notes'] == 'Testas nr. 1'
assert package1['url'] == 'http://www.testas1.lt'
assert package1['maintainer'] == 'Jonas Jonaitis'
assert package1['maintainer_email'] == '*****@*****.**'
assert package1['organization']['title'] == 'Testinė organizacija nr. 1'
assert package1['groups'] == [{
'display_name': 'testas1',
'description': '',
'image_display_url': '',
'title': 'testas1',
'id': package1['groups'][0]['id'],
'name': 'testas1-1',
}]
assert package2['title'] == 'Testinė rinkmena nr. 2'
assert package2['notes'] == 'Testas nr. 2'
assert package2['url'] == 'http://www.testas2.lt'
assert package2['maintainer'] == 'Tomas Tomauskas'
assert package2['maintainer_email'] == '*****@*****.**'
assert package2['organization']['title'] == 'Testinė organizacija nr. 2'
assert package2['groups'] == [{
'display_name': 'testas3',
'description': '',
'image_display_url': '',
'title': 'testas3',
'id': package2['groups'][0]['id'],
'name': 'testas3-3',
}]
assert user3['fullname'] == 'Unknown User'
assert organization3['title'] == 'Unknown organization'
fixcase_test = fixcase('Testas9')
assert fixcase_test == 'testas9'
tags_test = get_package_tags('aaaaaaaaaaaaaaaaaaaaaaaaaaaaaa'
'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaa'
'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaa'
'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaa,'
'testas2 testas3, testas4 testas5; testas6')
assert tags_test == [
'testas2 testas3',
'testas4 testas5',
'testas6',
]
tags1 = package1['tags']
tags2 = package2['tags']
assert sorted([x['name'] for x in tags1]) == [
'licencijos',
'licencijuojamos veiklos teritorija',
'šiluma',
'šilumos tiekimo licencijas turinčių įmonių sąrašas',
'šilumos tiekėjai',
]
assert sorted([x['name'] for x in tags2]) == [
'eismo intensyvumas',
'keliai',
]
def test_harvest_site_down(self):
results_by_guid = run_harvest(url='http://localhost:%s/site_down' %
mock_ckan.PORT,
harvester=CKANHarvester())
assert not results_by_guid
assert not was_last_job_considered_error_free()
def test_harvest_site_down(self):
results_by_guid = run_harvest(
url='http://localhost:%s/site_down' % mock_ckan.PORT,
harvester=CKANHarvester())
assert not results_by_guid
assert not was_last_job_considered_error_free()
