def test_gather_normal(self):
source = HarvestSourceObj(url='http://localhost:%s/' % mock_ckan.PORT)
job = HarvestJobObj(source=source)
harvester = CKANHarvester()
obj_ids = harvester.gather_stage(job)
assert_equal(type(obj_ids), list)
assert_equal(len(obj_ids), len(mock_ckan.DATASETS))
harvest_object = harvest_model.HarvestObject.get(obj_ids[0])
assert_equal(harvest_object.guid, mock_ckan.DATASETS[0]['id'])
def test_fetch_normal(self):
source = HarvestSourceObj(url='http://localhost:%s/' % mock_ckan.PORT)
job = HarvestJobObj(source=source)
harvest_object = HarvestObjectObj(guid=mock_ckan.DATASETS[0]['id'],
job=job,
content=json.dumps(
mock_ckan.DATASETS[0]))
harvester = CKANHarvester()
result = harvester.fetch_stage(harvest_object)
assert_equal(harvest_object.errors, [])
assert_equal(result, True)
def test_gather_normal(self):
source = HarvestSourceObj(url='http://localhost:%s/' % mock_ckan.PORT)
job = HarvestJobObj(source=source)
harvester = CKANHarvester()
obj_ids = harvester.gather_stage(job)
assert job.gather_errors == []
assert type(obj_ids) == list
assert len(obj_ids) == len(mock_ckan.DATASETS)
harvest_object = harvest_model.HarvestObject.get(obj_ids[0])
assert harvest_object.guid == mock_ckan.DATASETS[0]['id']
assert json.loads(harvest_object.content) == mock_ckan.DATASETS[0]
def run_harvest(url, harvester, config=''):
'''Runs a harvest and returns the results.
This allows you to test a harvester.
Queues are avoided as they are a pain in tests.
'''
# User creates a harvest source
source = HarvestSourceObj(url=url, config=config)
# User triggers a harvest, which is the creation of a harvest job.
# We set run=False so that it doesn't put it on the gather queue.
job = HarvestJobObj(source=source, run=False)
return run_harvest_job(job, harvester)
def test_OdgovltHarvester(app, db, mocker):
sync = IvpkIrsSync(db)
mocker.patch('odgovlt.IvpkIrsSync', return_value=sync)
db.execute(
sync.t.rinkmena.insert(), {
'PAVADINIMAS':
'Testinė rinkmena nr. 1',
'SANTRAUKA':
'Testas nr. 1',
'TINKLAPIS':
'http://www.testas1.lt',
'R_ZODZIAI':
'Šilumos tiekimo licencijas turinčių įmonių sąrašas,'
'šiluma,'
'šilumos tiekėjai,'
'licencijos,'
'licencijuojamos veiklos teritorija',
'K_EMAIL':
'*****@*****.**',
'STATUSAS':
'U',
'USER_ID':
1,
'istaiga_id':
1,
})
db.execute(
sync.t.rinkmena.insert(), {
'PAVADINIMAS': 'Testinė rinkmena nr. 2',
'SANTRAUKA': 'Testas nr. 2',
'TINKLAPIS': 'http://www.testas2.lt',
'R_ZODZIAI': 'keliai,eismo intensyvumas,"e"',
'K_EMAIL': '*****@*****.**',
'STATUSAS': 'U',
'USER_ID': 2,
'istaiga_id': 2,
})
db.execute(
sync.t.user.insert(), {
'LOGIN': '******',
'PASS': '******',
'EMAIL': '*****@*****.**',
'TELEFONAS': '+37000000000',
'FIRST_NAME': 'Jonas',
'LAST_NAME': 'Jonaitis',
})
db.execute(
sync.t.user.insert(), {
'LOGIN': '******',
'PASS': '******',
'EMAIL': '*****@*****.**',
'TELEFONAS': '+37000000000',
'FIRST_NAME': 'Tomas',
'LAST_NAME': 'Tomauskas',
})
db.execute(
sync.t.istaiga.insert(), {
'PAVADINIMAS': 'Testinė organizacija nr. 1',
'KODAS': 888,
'ADRESAS': 'Testinė g. 9'
})
db.execute(
sync.t.istaiga.insert(), {
'PAVADINIMAS': 'Testinė organizacija nr. 2',
'KODAS': 777,
'ADRESAS': 'Testinė g. 91'
})
db.execute(sync.t.kategorija.insert(), {
'PAVADINIMAS': 'testas1',
'KATEGORIJA_ID': 0,
'LYGIS': 1
})
db.execute(sync.t.kategorija.insert(), {
'PAVADINIMAS': 'testas2',
'KATEGORIJA_ID': 0,
'LYGIS': 1
})
db.execute(sync.t.kategorija.insert(), {
'PAVADINIMAS': 'testas3',
'KATEGORIJA_ID': 1,
'LYGIS': 2
})
db.execute(sync.t.kategorija.insert(), {
'PAVADINIMAS': 'testas4',
'KATEGORIJA_ID': 2,
'LYGIS': 2
})
db.execute(sync.t.kategorija.insert(), {
'PAVADINIMAS': 'testas5',
'KATEGORIJA_ID': 3,
'LYGIS': 3
})
db.execute(sync.t.kategorija.insert(), {
'PAVADINIMAS': 'testas6',
'KATEGORIJA_ID': 4,
'LYGIS': 3
})
db.execute(sync.t.kategorija.insert(), {
'PAVADINIMAS': 'testas7',
'KATEGORIJA_ID': 4,
'LYGIS': 3
})
db.execute(sync.t.kategorija_rinkmena.insert(), {
'KATEGORIJA_ID': 1,
'RINKMENA_ID': 1
})
db.execute(sync.t.kategorija_rinkmena.insert(), {
'KATEGORIJA_ID': 3,
'RINKMENA_ID': 2
})
ckanapi = CkanAPI({'user': '******'})
source = HarvestSourceObj(url='sqlite://', source_type='opendata-gov-lt')
job = HarvestJobObj(source=source)
harvester = OdgovltHarvester()
obj_ids = harvester.gather_stage(job)
assert job.gather_errors == []
assert ckanapi.group_list() == [
'testas1-1',
'testas2-2',
'testas3-3',
'testas4-4',
'testas5-5',
'testas6-6',
'testas7-7',
]
def subgroups(name):
return [g['name'] for g in ckanapi.group_show(id=name)['groups']]
assert subgroups('testas1-1') == ['testas3-3']
assert subgroups('testas2-2') == ['testas4-4']
assert subgroups('testas3-3') == ['testas5-5']
assert subgroups('testas4-4') == ['testas6-6', 'testas7-7']
assert subgroups('testas5-5') == []
assert subgroups('testas6-6') == []
assert subgroups('testas7-7') == []
assert [
json.loads(
ckanext.harvest.model.HarvestObject.get(x).content)['PAVADINIMAS']
for x in obj_ids
] == [
'Testinė rinkmena nr. 1',
'Testinė rinkmena nr. 2',
]
database_data_list = [
dict(row) for row in db.execute(sync.t.rinkmena.select())
]
user1 = sync.sync_user(1)
user2 = sync.sync_user(2)
user3 = sync.sync_user(3)
database_data_list[0]['VARDAS'] = user1['fullname']
database_data_list[1]['VARDAS'] = user2['fullname']
organization1 = sync.sync_organization(1)
organization2 = sync.sync_organization(2)
organization3 = sync.sync_organization(3)
database_data_list[0]['ORGANIZACIJA'] = organization1['name']
database_data_list[1]['ORGANIZACIJA'] = organization2['name']
database_data_list[0]['KATEGORIJA_RINKMENA'] = json.dumps([{
'ID':
1,
'KATEGORIJA_ID':
1,
'RINKMENA_ID':
1,
}])
database_data_list[1]['KATEGORIJA_RINKMENA'] = json.dumps([{
'ID':
2,
'KATEGORIJA_ID':
1,
'RINKMENA_ID':
2,
}])
obj1 = HarvestObjectObj(guid=database_data_list[0]['ID'],
job=job,
content=json.dumps(database_data_list[0],
cls=DatetimeEncoder))
result = harvester.fetch_stage(obj1)
assert obj1.errors == []
assert result
obj2 = HarvestObjectObj(guid=database_data_list[1]['ID'],
job=job,
content=json.dumps(database_data_list[1],
cls=DatetimeEncoder))
result = harvester.fetch_stage(obj2)
assert obj2.errors == []
assert result
create_or_update = harvester.import_stage(obj1)
assert create_or_update
create_or_update = harvester.import_stage(obj2)
assert create_or_update
assert obj1.package_id
assert obj2.package_id
reset_db()
sync = IvpkIrsSync(db)
mocker.patch('odgovlt.IvpkIrsSync', return_value=sync)
results_by_guid = run_harvest(url='sqlite://',
harvester=OdgovltHarvester())
result = results_by_guid['1']
assert result['state'] == 'COMPLETE'
assert result['report_status'] == 'added'
assert result['errors'] == []
result = results_by_guid['2']
assert result['state'] == 'COMPLETE'
assert result['report_status'] == 'added'
assert result['errors'] == []
assert was_last_job_considered_error_free()
ids = ckanapi.package_list()
assert len(ids) == 3
package1 = ckanapi.package_show(id=ids[0])
package2 = ckanapi.package_show(id=ids[1])
assert package1['title'] == 'Testinė rinkmena nr. 1'
assert package1['notes'] == 'Testas nr. 1'
assert package1['url'] == 'http://www.testas1.lt'
assert package1['maintainer'] == 'Jonas Jonaitis'
assert package1['maintainer_email'] == '*****@*****.**'
assert package1['organization']['title'] == 'Testinė organizacija nr. 1'
assert package1['groups'] == [{
'display_name': 'testas1',
'description': '',
'image_display_url': '',
'title': 'testas1',
'id': package1['groups'][0]['id'],
'name': 'testas1-1',
}]
assert package2['title'] == 'Testinė rinkmena nr. 2'
assert package2['notes'] == 'Testas nr. 2'
assert package2['url'] == 'http://www.testas2.lt'
assert package2['maintainer'] == 'Tomas Tomauskas'
assert package2['maintainer_email'] == '*****@*****.**'
assert package2['organization']['title'] == 'Testinė organizacija nr. 2'
assert package2['groups'] == [{
'display_name': 'testas3',
'description': '',
'image_display_url': '',
'title': 'testas3',
'id': package2['groups'][0]['id'],
'name': 'testas3-3',
}]
assert user3['fullname'] == 'Unknown User'
assert organization3['title'] == 'Unknown organization'
fixcase_test = fixcase('Testas9')
assert fixcase_test == 'testas9'
tags_test = get_package_tags('aaaaaaaaaaaaaaaaaaaaaaaaaaaaaa'
'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaa'
'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaa'
'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaa,'
'testas2 testas3, testas4 testas5; testas6')
assert tags_test == [
'testas2 testas3',
'testas4 testas5',
'testas6',
]
tags1 = package1['tags']
tags2 = package2['tags']
assert sorted([x['name'] for x in tags1]) == [
'licencijos',
'licencijuojamos veiklos teritorija',
'šiluma',
'šilumos tiekimo licencijas turinčių įmonių sąrašas',
'šilumos tiekėjai',
]
assert sorted([x['name'] for x in tags2]) == [
'eismo intensyvumas',
'keliai',
]