def test_harvest_sources_job_history_clear(self): # prepare data_dict = SOURCE_DICT.copy() source_1 = factories.HarvestSourceObj(**data_dict) data_dict['name'] = 'another-source' data_dict['url'] = 'http://another-url' source_2 = factories.HarvestSourceObj(**data_dict) job_1 = factories.HarvestJobObj(source=source_1) dataset_1 = ckan_factories.Dataset() object_1_ = factories.HarvestObjectObj(job=job_1, source=source_1, package_id=dataset_1['id']) job_2 = factories.HarvestJobObj(source=source_2) dataset_2 = ckan_factories.Dataset() object_2_ = factories.HarvestObjectObj(job=job_2, source=source_2, package_id=dataset_2['id']) # execute context = { 'model': model, 'session': model.Session, 'ignore_auth': True, 'user': '' } result = toolkit.get_action('harvest_sources_job_history_clear')( context, {}) # verify assert_equal(sorted(result), sorted([{ 'id': source_1.id }, { 'id': source_2.id }])) source_1 = harvest_model.HarvestSource.get(source_1.id) assert source_1 assert_equal(harvest_model.HarvestJob.get(job_1.id), None) assert_equal(harvest_model.HarvestObject.get(object_1_.id), None) dataset_from_db_1 = model.Package.get(dataset_1['id']) assert dataset_from_db_1, 'is None' assert_equal(dataset_from_db_1.id, dataset_1['id']) source_2 = harvest_model.HarvestSource.get(source_1.id) assert source_2 assert_equal(harvest_model.HarvestJob.get(job_2.id), None) assert_equal(harvest_model.HarvestObject.get(object_2_.id), None) dataset_from_db_2 = model.Package.get(dataset_2['id']) assert dataset_from_db_2, 'is None' assert_equal(dataset_from_db_2.id, dataset_2['id'])
def test_harvest_jobs_run_does_not_timeout_if_timeout_not_set( self, mock_error_log): harvest_source = factories.HarvestSourceObj(**SOURCE_DICT.copy()) harvest_job = factories.HarvestJobObj(source=harvest_source, run=True) # date in the past, assumes ckan.harvest.timeout has been set to 5 minutes harvest_job.created = '2020-05-29 10:00:00.0' harvest_job.save() context = { 'model': model, 'session': model.Session, 'ignore_auth': True, 'user': '' } data_dict = { 'guid': 'guid', 'content': 'content', 'job_id': harvest_job.id, 'source_id': harvest_source.id } job_obj = HarvestJob.get(harvest_job.id) job = toolkit.get_action('harvest_jobs_run')(context, data_dict) assert not mock_error_log.called status = toolkit.get_action('harvest_source_show_status')( context, { 'id': harvest_source.id }) assert status['last_job']['status'] == 'Running' assert status['last_job']['stats']['errored'] == 0
def test_harvest_jobs_run_does_not_timeout_if_within_time( self, mock_error_log): harvest_source = factories.HarvestSourceObj(**SOURCE_DICT.copy()) harvest_job = factories.HarvestJobObj(source=harvest_source, run=True) # job has just been created, so no timeout expected context = { 'model': model, 'session': model.Session, 'ignore_auth': True, 'user': '' } data_dict = { 'guid': 'guid', 'content': 'content', 'job_id': harvest_job.id, 'source_id': harvest_source.id } job_obj = HarvestJob.get(harvest_job.id) job = toolkit.get_action('harvest_jobs_run')(context, data_dict) assert not mock_error_log.called status = toolkit.get_action('harvest_source_show_status')( context, { 'id': harvest_source.id }) assert status['last_job']['status'] == 'Running' assert status['last_job']['stats']['errored'] == 0
def test_harvest_jobs_run_times_out(self, mock_error_log): harvest_source = factories.HarvestSourceObj(**SOURCE_DICT.copy()) harvest_job = factories.HarvestJobObj(source=harvest_source, run=True) # date in the past, ckan.harvest.timeout has been set to 5 minutes in test-nose.ini harvest_job.created = '2020-05-29 10:00:00.0' harvest_job.save() context = { 'model': model, 'session': model.Session, 'ignore_auth': True, 'user': '' } data_dict = { 'guid': 'guid', 'content': 'content', 'job_id': harvest_job.id, 'source_id': harvest_source.id } job = toolkit.get_action('harvest_jobs_run')(context, data_dict) msg, = mock_error_log.call_args[0] assert mock_error_log.called assert msg == 'Job timeout: {} is taking longer than 5 minutes'.format( harvest_job.id) status = toolkit.get_action('harvest_source_show_status')( context, { 'id': harvest_source.id }) assert status['last_job']['status'] == 'Finished' assert status['last_job']['stats']['errored'] == 1
def test_harvest_source_job_history_clear(self): # prepare source = factories.HarvestSourceObj(**SOURCE_DICT.copy()) job = factories.HarvestJobObj(source=source) dataset = ckan_factories.Dataset() object_ = factories.HarvestObjectObj(job=job, source=source, package_id=dataset['id']) # execute context = { 'model': model, 'session': model.Session, 'ignore_auth': True, 'user': '' } result = toolkit.get_action('harvest_source_job_history_clear')( context, { 'id': source.id }) # verify assert_equal(result, {'id': source.id}) source = harvest_model.HarvestSource.get(source.id) assert source assert_equal(harvest_model.HarvestJob.get(job.id), None) assert_equal(harvest_model.HarvestObject.get(object_.id), None) dataset_from_db = model.Package.get(dataset['id']) assert dataset_from_db, 'is None' assert_equal(dataset_from_db.id, dataset['id'])
def test_create_bad_parameters(self): source_a = factories.HarvestSourceObj() job = factories.HarvestJobObj() context = { 'model': model, 'session': model.Session, 'ignore_auth': True, } data_dict = {'job_id': job.id, 'source_id': source_a.id, 'extras': 1} harvest_object_create = toolkit.get_action('harvest_object_create') self.assertRaises(toolkit.ValidationError, harvest_object_create, context, data_dict) data_dict['extras'] = {'test': 1} self.assertRaises(toolkit.ValidationError, harvest_object_create, context, data_dict)
def test_create(self): job = factories.HarvestJobObj() context = { 'model': model, 'session': model.Session, 'ignore_auth': True, } data_dict = { 'guid': 'guid', 'content': 'content', 'job_id': job.id, 'extras': {'a key': 'a value'}, } harvest_object = toolkit.get_action('harvest_object_create')( context, data_dict) # fetch the object from database to check it was created created_object = harvest_model.HarvestObject.get(harvest_object['id']) assert created_object.guid == harvest_object['guid'] == data_dict['guid']
def test_harvest_source_clear(self): source = factories.HarvestSourceObj(**SOURCE_DICT) job = factories.HarvestJobObj(source=source) dataset = ckan_factories.Dataset() object_ = factories.HarvestObjectObj(job=job, source=source, package_id=dataset['id']) context = { 'model': model, 'session': model.Session, 'ignore_auth': True, 'user': '' } result = toolkit.get_action('harvest_source_clear')(context, { 'id': source.id }) assert_equal(result, {'id': source.id}) source = harvest_model.HarvestSource.get(source.id) assert source assert_equal(harvest_model.HarvestJob.get(job.id), None) assert_equal(harvest_model.HarvestObject.get(object_.id), None) assert_equal(model.Package.get(dataset['id']), None)