def schedule(self, dt, func, interval=0, repeat=0, *args, **kwargs): """ Add the job to the scheduler for the specified time, interval, and number of repeats. Repeat of None with a specified interval means the job will repeat forever at that interval. """ if not isinstance(dt, datetime): raise ValueError("Time argument must be a datetime object.") if not interval and repeat != 0: raise ValueError( "Must specify an interval if the task is repeating") if isinstance(func, Job): job = func # else, turn it into a job first. else: job = Job(func, *args, **kwargs) job.state = State.SCHEDULED with self.session_scope() as session: scheduled_job = ScheduledJob( id=job.job_id, queue=self.queue.name, interval=interval, repeat=repeat, scheduled_time=dt, obj=job, ) session.merge(scheduled_job) return job.job_id
def test_can_get_first_job_queued(self, defaultbackend): job1 = Job(open) job2 = Job(open) job1_id = defaultbackend.enqueue_job(job1, QUEUE) defaultbackend.enqueue_job(job2, QUEUE) assert defaultbackend.get_next_queued_job([QUEUE]).job_id == job1_id
def test_task_cancel(self, queue_mock): queue_mock.fetch_job.return_value = Job(state=State.CANCELED, func=lambda: None) response = self.client.post(reverse("kolibri:core:task-canceltask"), {"task_id": "1"}, format="json") self.assertEqual(response.data, {})
def enqueue(self, func, *args, **kwargs): """ Enqueues a function func for execution. One special parameter is track_progress. If passed in and not None, the func will be passed in a keyword parameter called update_progress: def update_progress(progress, total_progress, stage=""): The running function can call the update_progress function to notify interested parties of the function's current progress. Another special parameter is the "cancellable" keyword parameter. When passed in and not None, a special "check_for_cancel" parameter is passed in. When called, it raises an error when the user has requested a job to be cancelled. The caller can also pass in any pickleable object into the "extra_metadata" parameter. This data is stored within the job and can be retrieved when the job status is queried. All other parameters are directly passed to the function when it starts running. :type func: callable or str :param func: A callable object that will be scheduled for running. :return: a string representing the job_id. """ # if the func is already a job object, just schedule that directly. if isinstance(func, Job): job = func # else, turn it into a job first. else: job = Job(func, *args, **kwargs) job_id = self.storage.enqueue_job(job, self.name) return job_id
def test_enqueue_job_writes_to_storage_on_success(self, worker): with patch.object(worker.storage, "complete_job", wraps=worker.storage.complete_job) as spy: # this job should never fail. job = Job(id, 9) worker.storage.enqueue_job(job, QUEUE) while job.state == State.QUEUED: job = worker.storage.get_job(job.job_id) time.sleep(0.5) try: # Get the future, or pass if it has already been cleaned up. future = worker.future_job_mapping[job.job_id] future.result() except KeyError: pass # verify that we sent a message through our backend assert spy.call_count == 1 call_args = spy.call_args job_id = call_args[0][0] # verify that we're setting the correct job_id assert job_id == job.job_id
def restart_job(self, job_id): """ Given a job_id, restart the job for that id. A job will only be restarted if in CANCELED or FAILED state. This first clears the job then creates a new job with the same job_id as the cleared one. """ old_job = self.fetch_job(job_id) if old_job.state in [State.CANCELED, State.FAILED]: self.clear_job(job_id) job = Job(old_job) job.job_id = job_id return self.enqueue(job) else: raise JobNotRestartable("Cannot restart job with state={}".format( old_job.state))
def test_enqueue_job_runs_job(self, worker): job = Job(id, 9) worker.storage.enqueue_job(job, QUEUE) while job.state == State.QUEUED: job = worker.storage.get_job(job.job_id) time.sleep(0.5) try: # Get the future, or pass if it has already been cleaned up. future = worker.future_job_mapping[job.job_id] future.result() except KeyError: pass assert job.state == State.COMPLETED
def test_can_handle_unicode_exceptions(self, worker): # Make sure task exception info is not an object, but is either a string or None. # See Storage.mark_job_as_failed in kolibri.core.tasks.storage for more details on why we do this. # create a job that triggers an exception job = Job("kolibri.core.tasks.test.taskrunner.test_worker.error_func") job_id = worker.storage.enqueue_job(job, QUEUE) while job.state == State.QUEUED: job = worker.storage.get_job(job.job_id) time.sleep(0.5) returned_job = worker.storage.get_job(job_id) assert returned_job.state == "FAILED" assert isinstance(returned_job.exception, TypeError) assert returned_job.exception.args[0] == error_text
class JobTest(TestCase): def setUp(self): self.job = Job(id) self.job.storage = mock.MagicMock() def test_job_save_as_cancellable(self): cancellable = not self.job.cancellable self.job.save_as_cancellable(cancellable=cancellable) self.job.storage.save_job_as_cancellable.assert_called_once_with( self.job.job_id, cancellable=cancellable ) def test_job_save_as_cancellable__skip(self): cancellable = self.job.cancellable self.job.save_as_cancellable(cancellable=cancellable) self.job.storage.save_job_as_cancellable.assert_not_called() def test_job_save_as_cancellable__no_storage(self): cancellable = not self.job.cancellable self.job.storage = None with self.assertRaises(ReferenceError): self.job.save_as_cancellable(cancellable=cancellable)
def simplejob(): return Job(id)
def simplejob(): return Job("builtins.id")
def request_soud_sync(server, user=None, queue_id=None, ttl=10): """ Make a request to the serverurl endpoint to sync this SoUD (Subset of Users Device) - If the server says "sync now" immediately queue a sync task for the server - If the server responds with an identifier and interval, schedule itself to run again in the future with that identifier as an argument, at the interval specified """ if queue_id is None: endpoint = reverse("kolibri:core:syncqueue-list") else: endpoint = reverse("kolibri:core:syncqueue-detail", kwargs={"pk": queue_id}) server_url = "{server}{endpoint}".format(server=server, endpoint=endpoint) try: if queue_id is None: data = {"user": user} response = requests.post(server_url, json=data) else: data = {"pk": queue_id} response = requests.put(server_url, json=data) except requests.exceptions.ConnectionError: # Algorithm to try several times if the server is not responding # Randomly it can be trying it up to 1560 seconds (26 minutes) # before desisting ttl -= 1 if ttl == 0: logger.error("Give up trying to connect to the server") return interval = random.randint(1, 30 * (10 - ttl)) job = Job(request_soud_sync, server, user, queue_id, ttl) dt = datetime.timedelta(seconds=interval) scheduler.enqueue_in(dt, job) logger.warn( "The server has some trouble. Trying to connect in {} seconds". format(interval)) return response_content = (response.content.decode() if isinstance( response.content, bytes) else response.content) server_response = json.loads(response_content or "{}") if response.status_code == status.HTTP_404_NOT_FOUND: return # Request done to a server not owning this user's data if response.status_code == status.HTTP_200_OK: if server_response["action"] == SYNC: job_id = startpeerfacilitysync(server, user) logger.info("Enqueuing a sync task for user {} in job {}".format( user, job_id)) elif server_response["action"] == QUEUED: pk = server_response["id"] time_alive = server_response["keep_alive"] dt = datetime.timedelta(seconds=int(time_alive)) job = Job(request_soud_sync, server, user, pk) scheduler.enqueue_in(dt, job) logger.info( "Server busy, will try again in {} seconds with pk={}".format( time_alive, pk))
def setUp(self): self.job = Job(id) self.job.storage = mock.MagicMock()