def schedule(self, dt, func, interval=0, repeat=0, *args, **kwargs):
"""
Add the job to the scheduler for the specified time, interval, and number of repeats.
Repeat of None with a specified interval means the job will repeat forever at that
interval.
"""
if not isinstance(dt, datetime):
raise ValueError("Time argument must be a datetime object.")
if not interval and repeat != 0:
raise ValueError(
"Must specify an interval if the task is repeating")
if isinstance(func, Job):
job = func
# else, turn it into a job first.
else:
job = Job(func, *args, **kwargs)
job.state = State.SCHEDULED
with self.session_scope() as session:
scheduled_job = ScheduledJob(
id=job.job_id,
queue=self.queue.name,
interval=interval,
repeat=repeat,
scheduled_time=dt,
obj=job,
)
session.merge(scheduled_job)
return job.job_id
def test_can_get_first_job_queued(self, defaultbackend):
job1 = Job(open)
job2 = Job(open)
job1_id = defaultbackend.enqueue_job(job1, QUEUE)
defaultbackend.enqueue_job(job2, QUEUE)
assert defaultbackend.get_next_queued_job([QUEUE]).job_id == job1_id
def test_task_cancel(self, queue_mock):
queue_mock.fetch_job.return_value = Job(state=State.CANCELED,
func=lambda: None)
response = self.client.post(reverse("kolibri:core:task-canceltask"),
{"task_id": "1"},
format="json")
self.assertEqual(response.data, {})
def enqueue(self, func, *args, **kwargs):
"""
Enqueues a function func for execution.
One special parameter is track_progress. If passed in and not None, the func will be passed in a
keyword parameter called update_progress:
def update_progress(progress, total_progress, stage=""):
The running function can call the update_progress function to notify interested parties of the function's
current progress.
Another special parameter is the "cancellable" keyword parameter. When passed in and not None, a special
"check_for_cancel" parameter is passed in. When called, it raises an error when the user has requested a job
to be cancelled.
The caller can also pass in any pickleable object into the "extra_metadata" parameter. This data is stored
within the job and can be retrieved when the job status is queried.
All other parameters are directly passed to the function when it starts running.
:type func: callable or str
:param func: A callable object that will be scheduled for running.
:return: a string representing the job_id.
"""
# if the func is already a job object, just schedule that directly.
if isinstance(func, Job):
job = func
# else, turn it into a job first.
else:
job = Job(func, *args, **kwargs)
job_id = self.storage.enqueue_job(job, self.name)
return job_id
def test_enqueue_job_writes_to_storage_on_success(self, worker):
with patch.object(worker.storage,
"complete_job",
wraps=worker.storage.complete_job) as spy:
# this job should never fail.
job = Job(id, 9)
worker.storage.enqueue_job(job, QUEUE)
while job.state == State.QUEUED:
job = worker.storage.get_job(job.job_id)
time.sleep(0.5)
try:
# Get the future, or pass if it has already been cleaned up.
future = worker.future_job_mapping[job.job_id]
future.result()
except KeyError:
pass
# verify that we sent a message through our backend
assert spy.call_count == 1
call_args = spy.call_args
job_id = call_args[0][0]
# verify that we're setting the correct job_id
assert job_id == job.job_id
def restart_job(self, job_id):
"""
Given a job_id, restart the job for that id. A job will only be restarted if
in CANCELED or FAILED state.
This first clears the job then creates a new job with the same job_id as
the cleared one.
"""
old_job = self.fetch_job(job_id)
if old_job.state in [State.CANCELED, State.FAILED]:
self.clear_job(job_id)
job = Job(old_job)
job.job_id = job_id
return self.enqueue(job)
else:
raise JobNotRestartable("Cannot restart job with state={}".format(
old_job.state))
def test_enqueue_job_runs_job(self, worker):
job = Job(id, 9)
worker.storage.enqueue_job(job, QUEUE)
while job.state == State.QUEUED:
job = worker.storage.get_job(job.job_id)
time.sleep(0.5)
try:
# Get the future, or pass if it has already been cleaned up.
future = worker.future_job_mapping[job.job_id]
future.result()
except KeyError:
pass
assert job.state == State.COMPLETED
def test_can_handle_unicode_exceptions(self, worker):
# Make sure task exception info is not an object, but is either a string or None.
# See Storage.mark_job_as_failed in kolibri.core.tasks.storage for more details on why we do this.
# create a job that triggers an exception
job = Job("kolibri.core.tasks.test.taskrunner.test_worker.error_func")
job_id = worker.storage.enqueue_job(job, QUEUE)
while job.state == State.QUEUED:
job = worker.storage.get_job(job.job_id)
time.sleep(0.5)
returned_job = worker.storage.get_job(job_id)
assert returned_job.state == "FAILED"
assert isinstance(returned_job.exception, TypeError)
assert returned_job.exception.args[0] == error_text
class JobTest(TestCase):
def setUp(self):
self.job = Job(id)
self.job.storage = mock.MagicMock()
def test_job_save_as_cancellable(self):
cancellable = not self.job.cancellable
self.job.save_as_cancellable(cancellable=cancellable)
self.job.storage.save_job_as_cancellable.assert_called_once_with(
self.job.job_id, cancellable=cancellable
)
def test_job_save_as_cancellable__skip(self):
cancellable = self.job.cancellable
self.job.save_as_cancellable(cancellable=cancellable)
self.job.storage.save_job_as_cancellable.assert_not_called()
def test_job_save_as_cancellable__no_storage(self):
cancellable = not self.job.cancellable
self.job.storage = None
with self.assertRaises(ReferenceError):
self.job.save_as_cancellable(cancellable=cancellable)
def simplejob():
return Job(id)
def simplejob():
return Job("builtins.id")
def request_soud_sync(server, user=None, queue_id=None, ttl=10):
"""
Make a request to the serverurl endpoint to sync this SoUD (Subset of Users Device)
- If the server says "sync now" immediately queue a sync task for the server
- If the server responds with an identifier and interval, schedule itself to run
again in the future with that identifier as an argument, at the interval specified
"""
if queue_id is None:
endpoint = reverse("kolibri:core:syncqueue-list")
else:
endpoint = reverse("kolibri:core:syncqueue-detail",
kwargs={"pk": queue_id})
server_url = "{server}{endpoint}".format(server=server, endpoint=endpoint)
try:
if queue_id is None:
data = {"user": user}
response = requests.post(server_url, json=data)
else:
data = {"pk": queue_id}
response = requests.put(server_url, json=data)
except requests.exceptions.ConnectionError:
# Algorithm to try several times if the server is not responding
# Randomly it can be trying it up to 1560 seconds (26 minutes)
# before desisting
ttl -= 1
if ttl == 0:
logger.error("Give up trying to connect to the server")
return
interval = random.randint(1, 30 * (10 - ttl))
job = Job(request_soud_sync, server, user, queue_id, ttl)
dt = datetime.timedelta(seconds=interval)
scheduler.enqueue_in(dt, job)
logger.warn(
"The server has some trouble. Trying to connect in {} seconds".
format(interval))
return
response_content = (response.content.decode() if isinstance(
response.content, bytes) else response.content)
server_response = json.loads(response_content or "{}")
if response.status_code == status.HTTP_404_NOT_FOUND:
return # Request done to a server not owning this user's data
if response.status_code == status.HTTP_200_OK:
if server_response["action"] == SYNC:
job_id = startpeerfacilitysync(server, user)
logger.info("Enqueuing a sync task for user {} in job {}".format(
user, job_id))
elif server_response["action"] == QUEUED:
pk = server_response["id"]
time_alive = server_response["keep_alive"]
dt = datetime.timedelta(seconds=int(time_alive))
job = Job(request_soud_sync, server, user, pk)
scheduler.enqueue_in(dt, job)
logger.info(
"Server busy, will try again in {} seconds with pk={}".format(
time_alive, pk))
def setUp(self):
self.job = Job(id)
self.job.storage = mock.MagicMock()