def fetchable(cls, job_type, form):
meta = form.get_meta()
Job.create(job_type=job_type,
targets=form.urls.data,
extract=form.extract.data,
javascript=form.javascript.data,
meta=meta)
return bottle.redirect('/jobs/')
def test_mark_finished(self, dispatch):
job = Job.create(targets=self.standalone_targets,
job_type=Job.STANDALONE,
origin=self.origin)
assert job.is_finished is False
job.mark_finished()
assert job.is_finished is True
def test_mark_queued(self, dispatch):
job = Job.create(targets=self.standalone_targets,
job_type=Job.STANDALONE,
origin=self.origin)
job.mark_erred() # jobs are queued by default, so make it erred
assert job.is_queued is False
job.mark_queued()
assert job.is_queued is True
def standalone(cls, job_type, form):
folder_name = str(uuid.uuid4())
media_root = settings.BOTTLE_CONFIG.get('web.media_root', '')
upload_dir = os.path.join(media_root, folder_name)
os.makedirs(upload_dir)
targets = []
for uploaded_file in bottle.request.files.getlist('files'):
upload_path = os.path.join(upload_dir, uploaded_file.filename)
uploaded_file.save(upload_path)
targets.append(upload_path)
meta = form.get_meta()
Job.create(job_type=job_type,
targets=targets,
origin=form.origin.data,
meta=meta)
return bottle.redirect('/jobs/')
def jobs_new():
job_type = bottle.request.query.get('type')
if Job.is_valid_type(job_type):
form_cls = CreateJobController.forms[job_type]
form = form_cls()
return bottle.jinja2_template('job_{0}.html'.format(job_type.lower()),
form=form,
job_type=job_type)
return bottle.jinja2_template('job_wizard.html')
def test_create_standalone_job(self, dispatch):
job = Job.create(targets=self.standalone_targets,
job_type=Job.STANDALONE,
origin=self.origin)
assert job.status == Job.QUEUED
assert job.job_type == Job.STANDALONE
assert job.scheduled == job.updated
assert job.options['origin'] == self.origin
dispatch.assert_called_once_with({'type': job.job_type,
'id': job.job_id})
self.assert_tasks(job, self.standalone_targets)
def test_retry(self, dispatch):
job = Job.create(targets=self.standalone_targets,
job_type=Job.STANDALONE,
origin=self.origin)
job.mark_erred() # jobs are queued by default, so make it erred
assert job.is_queued is False
job.retry()
assert job.is_queued is True
job_data = {'type': job.job_type, 'id': job.job_id}
# called twice, first when the job is created, next when it's retried
dispatch.assert_has_calls([mock.call(job_data), mock.call(job_data)])
def test_create_fetchable_job(self, dispatch):
job = Job.create(targets=self.fetchable_targets,
job_type=Job.FETCHABLE,
extract=True,
javascript=True)
assert job.status == Job.QUEUED
assert job.job_type == Job.FETCHABLE
assert job.scheduled == job.updated
assert job.options['extract'] is True
assert job.options['javascript'] is True
dispatch.assert_called_once_with({'type': job.job_type,
'id': job.job_id})
self.assert_tasks(job, self.fetchable_targets)
def test_run_success(self, mark_processing, mark_erred, mark_finished,
process_task, *args):
job = Job.create(targets=self.targets,
job_type=Job.FETCHABLE,
extract=True,
javascript=True)
# make second task already finished, so it should be skipped
job.tasks[1].mark_finished()
handler = BaseJobHandler()
handler.run({'type': job.job_type, 'id': job.job_id})
mark_processing.assert_called_once_with()
mark_finished.assert_called_once_with()
assert not mark_erred.called
calls = [mock.call(job.tasks[0], job.options)]
process_task.assert_has_calls(calls)
def jobs_create():
job_type = bottle.request.forms.get('type')
if Job.is_valid_type(job_type):
form_cls = CreateJobController.forms[job_type]
form_data = bottle.request.forms.decode()
form_data.update(bottle.request.files)
form = form_cls(form_data)
if form.validate():
handler = CreateJobController.get_handler(job_type.lower())
return handler(job_type, form)
template_name = 'job_{0}.html'.format(job_type.lower())
return bottle.jinja2_template(template_name,
form=form,
job_type=job_type)
return bottle.jinja2_template('job_wizard.html')
def test_run_failure(self, mark_processing, mark_erred, mark_finished,
process_task, *args):
job = Job.create(targets=self.targets,
job_type=Job.FETCHABLE,
extract=True,
javascript=True)
# make second task failed, as real processing is skipped, it will
# trigger failure on the whole process
job.tasks[1].mark_failed("failed")
handler = BaseJobHandler()
handler.run({'type': job.job_type, 'id': job.job_id})
mark_processing.assert_called_once_with()
mark_erred.assert_called_once_with()
assert not mark_finished.called
calls = [mock.call(task, job.options) for task in job.tasks]
process_task.assert_has_calls(calls)
def test_is_valid_type(self):
assert Job.is_valid_type(Job.STANDALONE) is True
assert Job.is_valid_type(Job.FETCHABLE) is True
assert Job.is_valid_type('INVALID') is False