def test_harvest_sources_job_history_clear(self):
# prepare
data_dict = SOURCE_DICT.copy()
source_1 = factories.HarvestSourceObj(**data_dict)
data_dict['name'] = 'another-source'
data_dict['url'] = 'http://another-url'
source_2 = factories.HarvestSourceObj(**data_dict)
job_1 = factories.HarvestJobObj(source=source_1)
dataset_1 = ckan_factories.Dataset()
object_1_ = factories.HarvestObjectObj(job=job_1,
source=source_1,
package_id=dataset_1['id'])
job_2 = factories.HarvestJobObj(source=source_2)
dataset_2 = ckan_factories.Dataset()
object_2_ = factories.HarvestObjectObj(job=job_2,
source=source_2,
package_id=dataset_2['id'])
# execute
context = {
'model': model,
'session': model.Session,
'ignore_auth': True,
'user': ''
}
result = toolkit.get_action('harvest_sources_job_history_clear')(
context, {})
# verify
assert_equal(sorted(result),
sorted([{
'id': source_1.id
}, {
'id': source_2.id
}]))
source_1 = harvest_model.HarvestSource.get(source_1.id)
assert source_1
assert_equal(harvest_model.HarvestJob.get(job_1.id), None)
assert_equal(harvest_model.HarvestObject.get(object_1_.id), None)
dataset_from_db_1 = model.Package.get(dataset_1['id'])
assert dataset_from_db_1, 'is None'
assert_equal(dataset_from_db_1.id, dataset_1['id'])
source_2 = harvest_model.HarvestSource.get(source_1.id)
assert source_2
assert_equal(harvest_model.HarvestJob.get(job_2.id), None)
assert_equal(harvest_model.HarvestObject.get(object_2_.id), None)
dataset_from_db_2 = model.Package.get(dataset_2['id'])
assert dataset_from_db_2, 'is None'
assert_equal(dataset_from_db_2.id, dataset_2['id'])
def test_harvest_jobs_run_does_not_timeout_if_timeout_not_set(
self, mock_error_log):
harvest_source = factories.HarvestSourceObj(**SOURCE_DICT.copy())
harvest_job = factories.HarvestJobObj(source=harvest_source, run=True)
# date in the past, assumes ckan.harvest.timeout has been set to 5 minutes
harvest_job.created = '2020-05-29 10:00:00.0'
harvest_job.save()
context = {
'model': model,
'session': model.Session,
'ignore_auth': True,
'user': ''
}
data_dict = {
'guid': 'guid',
'content': 'content',
'job_id': harvest_job.id,
'source_id': harvest_source.id
}
job_obj = HarvestJob.get(harvest_job.id)
job = toolkit.get_action('harvest_jobs_run')(context, data_dict)
assert not mock_error_log.called
status = toolkit.get_action('harvest_source_show_status')(
context, {
'id': harvest_source.id
})
assert status['last_job']['status'] == 'Running'
assert status['last_job']['stats']['errored'] == 0
def test_harvest_jobs_run_does_not_timeout_if_within_time(
self, mock_error_log):
harvest_source = factories.HarvestSourceObj(**SOURCE_DICT.copy())
harvest_job = factories.HarvestJobObj(source=harvest_source, run=True)
# job has just been created, so no timeout expected
context = {
'model': model,
'session': model.Session,
'ignore_auth': True,
'user': ''
}
data_dict = {
'guid': 'guid',
'content': 'content',
'job_id': harvest_job.id,
'source_id': harvest_source.id
}
job_obj = HarvestJob.get(harvest_job.id)
job = toolkit.get_action('harvest_jobs_run')(context, data_dict)
assert not mock_error_log.called
status = toolkit.get_action('harvest_source_show_status')(
context, {
'id': harvest_source.id
})
assert status['last_job']['status'] == 'Running'
assert status['last_job']['stats']['errored'] == 0
def test_harvest_jobs_run_times_out(self, mock_error_log):
harvest_source = factories.HarvestSourceObj(**SOURCE_DICT.copy())
harvest_job = factories.HarvestJobObj(source=harvest_source, run=True)
# date in the past, ckan.harvest.timeout has been set to 5 minutes in test-nose.ini
harvest_job.created = '2020-05-29 10:00:00.0'
harvest_job.save()
context = {
'model': model,
'session': model.Session,
'ignore_auth': True,
'user': ''
}
data_dict = {
'guid': 'guid',
'content': 'content',
'job_id': harvest_job.id,
'source_id': harvest_source.id
}
job = toolkit.get_action('harvest_jobs_run')(context, data_dict)
msg, = mock_error_log.call_args[0]
assert mock_error_log.called
assert msg == 'Job timeout: {} is taking longer than 5 minutes'.format(
harvest_job.id)
status = toolkit.get_action('harvest_source_show_status')(
context, {
'id': harvest_source.id
})
assert status['last_job']['status'] == 'Finished'
assert status['last_job']['stats']['errored'] == 1
def test_harvest_source_job_history_clear(self):
# prepare
source = factories.HarvestSourceObj(**SOURCE_DICT.copy())
job = factories.HarvestJobObj(source=source)
dataset = ckan_factories.Dataset()
object_ = factories.HarvestObjectObj(job=job,
source=source,
package_id=dataset['id'])
# execute
context = {
'model': model,
'session': model.Session,
'ignore_auth': True,
'user': ''
}
result = toolkit.get_action('harvest_source_job_history_clear')(
context, {
'id': source.id
})
# verify
assert_equal(result, {'id': source.id})
source = harvest_model.HarvestSource.get(source.id)
assert source
assert_equal(harvest_model.HarvestJob.get(job.id), None)
assert_equal(harvest_model.HarvestObject.get(object_.id), None)
dataset_from_db = model.Package.get(dataset['id'])
assert dataset_from_db, 'is None'
assert_equal(dataset_from_db.id, dataset['id'])
def test_create_bad_parameters(self):
source_a = factories.HarvestSourceObj()
job = factories.HarvestJobObj()
context = {
'model': model,
'session': model.Session,
'ignore_auth': True,
}
data_dict = {'job_id': job.id, 'source_id': source_a.id, 'extras': 1}
harvest_object_create = toolkit.get_action('harvest_object_create')
self.assertRaises(toolkit.ValidationError, harvest_object_create,
context, data_dict)
data_dict['extras'] = {'test': 1}
self.assertRaises(toolkit.ValidationError, harvest_object_create,
context, data_dict)
def test_create(self):
job = factories.HarvestJobObj()
context = {
'model': model,
'session': model.Session,
'ignore_auth': True,
}
data_dict = {
'guid': 'guid',
'content': 'content',
'job_id': job.id,
'extras': {'a key': 'a value'},
}
harvest_object = toolkit.get_action('harvest_object_create')(
context, data_dict)
# fetch the object from database to check it was created
created_object = harvest_model.HarvestObject.get(harvest_object['id'])
assert created_object.guid == harvest_object['guid'] == data_dict['guid']
def test_harvest_source_clear(self):
source = factories.HarvestSourceObj(**SOURCE_DICT)
job = factories.HarvestJobObj(source=source)
dataset = ckan_factories.Dataset()
object_ = factories.HarvestObjectObj(job=job,
source=source,
package_id=dataset['id'])
context = {
'model': model,
'session': model.Session,
'ignore_auth': True,
'user': ''
}
result = toolkit.get_action('harvest_source_clear')(context, {
'id': source.id
})
assert_equal(result, {'id': source.id})
source = harvest_model.HarvestSource.get(source.id)
assert source
assert_equal(harvest_model.HarvestJob.get(job.id), None)
assert_equal(harvest_model.HarvestObject.get(object_.id), None)
assert_equal(model.Package.get(dataset['id']), None)
