def test_get_queue_default_queue(self):
u'''
Test that the default queue is returned if no queue is given.
'''
q = jobs.get_queue()
assert_equal(jobs.remove_queue_name_prefix(q.name),
jobs.DEFAULT_QUEUE_NAME)
def test_get_queue_default_queue(self):
u'''
Test that the default queue is returned if no queue is given.
'''
q = jobs.get_queue()
assert_equal(jobs.remove_queue_name_prefix(q.name),
jobs.DEFAULT_QUEUE_NAME)
def broken_links():
QUEUE_NAME = 'default'
queue = jobs.get_queue(QUEUE_NAME)
queue._default_timeout = 3600 * 24
try:
toolkit.check_access('sysadmin', {'user': g.user, model: model})
except toolkit.NotAuthorized:
return toolkit.abort(403)
filepath = toolkit.config['spc.report.broken_links_filepath']
try:
last_check = datetime.fromtimestamp(os.stat(filepath).st_mtime)
except OSError:
last_check = None
active_jobs_count = jobs.get_queue(QUEUE_NAME).count
if request.method == 'POST':
action = request.form.get('action')
if action == 'download' and last_check:
return send_file(
filepath,
as_attachment=True,
attachment_filename='SPC-BrokenLinks-{:%Y-%m-%d}.csv'.format(
last_check))
elif action == 'start':
jobs.enqueue(broken_links_report,
kwargs={'recepients': [g.user]},
queue=QUEUE_NAME)
h.flash_notice('Report generation in progress. '
'You will recieve email notification '
'as soon as report finished')
return h.redirect_to('spc_admin.broken_links')
if active_jobs_count:
h.flash_error('There are unfinished '
'report generation processes in progress. '
'You will not be able to manually start checker '
'until they are finished.')
extra_vars = {
'last_check': last_check,
'active_jobs_count': active_jobs_count
}
return toolkit.render('admin/broken_links.html', extra_vars)
def _print_status(self):
try:
import ckan.lib.jobs as rq_jobs
except ImportError:
import ckanext.rq.jobs as rq_jobs
jobs = rq_jobs.get_queue().jobs
if not jobs:
print('No jobs currently queued')
for job in jobs:
job_params = eval(job.description.replace(
'ckanext.xloader.jobs.xloader_data_into_datastore', ''))
job_metadata = job_params['metadata']
print('{id} Enqueued={enqueued:%Y-%m-%d %H:%M} res_id={res_id} '
'url={url}'.format(
id=job._id,
enqueued=job.enqueued_at,
res_id=job_metadata['resource_id'],
url=job_metadata['original_url'],
))
def custom_enqueue(fn,
args=None,
kwargs=None,
title=None,
queue=DEFAULT_QUEUE_NAME,
timeout=DEFAULT_JOB_TIMEOUT):
u'''
Enqueue a job to be run in the background.
:param function fn: Function to be executed in the background
:param list args: List of arguments to be passed to the function.
Pass an empty list if there are no arguments (default).
:param dict kwargs: Dict of keyword arguments to be passed to the
function. Pass an empty dict if there are no keyword arguments
(default).
:param string title: Optional human-readable title of the job.
:param string queue: Name of the queue. If not given then the
default queue is used.
:param integer timeout: The timeout, in seconds, to be passed
to the background job via rq.
:returns: The enqueued job.
:rtype: ``rq.job.Job``
'''
if args is None:
args = []
if kwargs is None:
kwargs = {}
job = jobs.get_queue(queue).enqueue_call(func=fn,
args=args,
kwargs=kwargs,
timeout=timeout)
job.meta[u'title'] = title
job.save()
msg = u'Added background job {}'.format(job.id)
if title:
msg = u'{} ("{}")'.format(msg, title)
msg = u'{} to queue "{}"'.format(msg, queue)
log.info(msg)
return job
def job_clear(context, data_dict):
'''Clear background job queues.
Does not affect jobs that are already being processed.
:param list queues: The queues to clear. If not given then ALL
queues are cleared.
:returns: The cleared queues.
:rtype: list
.. versionadded:: 2.7
'''
_check_access(u'job_clear', context, data_dict)
queues = data_dict.get(u'queues')
if queues:
queues = [jobs.get_queue(q) for q in queues]
else:
queues = jobs.get_all_queues()
names = [jobs.remove_queue_name_prefix(queue.name) for queue in queues]
for queue, name in zip(queues, names):
queue.empty()
log.info(u'Cleared background job queue "{}"'.format(name))
return names
def job_clear(context, data_dict):
'''Clear background job queues.
Does not affect jobs that are already being processed.
:param list queues: The queues to clear. If not given then ALL
queues are cleared.
:returns: The cleared queues.
:rtype: list
.. versionadded:: 2.7
'''
_check_access(u'job_clear', context, data_dict)
queues = data_dict.get(u'queues')
if queues:
queues = [jobs.get_queue(q) for q in queues]
else:
queues = jobs.get_all_queues()
names = [jobs.remove_queue_name_prefix(queue.name) for queue in queues]
for queue, name in zip(queues, names):
queue.empty()
log.info(u'Cleared background job queue "{}"'.format(name))
return names
def test_get_queue_other_queue(self):
u"""
Test that a different queue can be given.
"""
q = jobs.get_queue(u"my_queue")
assert jobs.remove_queue_name_prefix(q.name) == u"my_queue"
def test_get_queue_default_queue(self):
u"""
Test that the default queue is returned if no queue is given.
"""
q = jobs.get_queue()
assert jobs.remove_queue_name_prefix(q.name) == jobs.DEFAULT_QUEUE_NAME
def test_get_queue_other_queue(self):
u'''
Test that a different queue can be given.
'''
q = jobs.get_queue(u'my_queue')
assert_equal(jobs.remove_queue_name_prefix(q.name), u'my_queue')
def test_get_queue_other_queue(self):
u'''
Test that a different queue can be given.
'''
q = jobs.get_queue(u'my_queue')
assert_equal(jobs.remove_queue_name_prefix(q.name), u'my_queue')
def test_get_queue_other_queue(self):
u"""
Test that a different queue can be given.
"""
q = jobs.get_queue(u"my_queue")
assert_equal(jobs.remove_queue_name_prefix(q.name), u"my_queue")