コード例 #1
0
ファイル: rqueue.py プロジェクト: Masutangu/Elric
 def submit_job(self, func, job_key, args=None, kwargs=None, trigger=None, job_id=None,
                replace_exist=False, need_filter=False, **trigger_args):
     """
         submit job to master through redis queue
         :type func: str or callable obj or unicode
         :type job_key: str or unicode
         :type args: tuple or list
         :type kwargs: dict
         :type trigger: str or unicode
         :type job_id: str or unicode
         :type replace_exist: bool
         :type trigger_args: dict
     """
     # use worker's timezone if trigger don't provide specific `timezone` configuration
     trigger_args['timezone'] = self.timezone
     job_in_dict = {
         'id': "%s:%s" % (self.name, job_id) if job_id else None,
         'func': func,
         'args': args,
         'trigger': create_trigger(trigger, trigger_args) if trigger else None,
         'kwargs': kwargs,
         'job_key': '%s:%s' % (self.name, job_key),
         'need_filter': need_filter,
         'replace_exist': replace_exist
     }
     job = Job(**job_in_dict)
     job.check()
     self.jobqueue.enqueue('__elric_submit_channel__', job.serialize())
コード例 #2
0
ファイル: rqueue.py プロジェクト: Masutangu/Elric
 def finish_job(self, job_id, is_success, details, job_key, need_filter):
     job_in_dict = {
         'id': job_id,
         'job_key': job_key,
         'is_success': is_success,
         'details': details,
         'need_filter': need_filter
     }
     job = Job(**job_in_dict)
     self.jobqueue.enqueue('__elric_finish_channel__', job.serialize())
コード例 #3
0
ファイル: rqueue.py プロジェクト: Masutangu/Elric
 def remove_job(self, job_id):
     """
         send remove job request to master through redis queue
         :type job_id: str
     """
     job_in_dict = {
         'id': "%s:%s" % (self.name, job_id)
     }
     job = Job(**job_in_dict)
     self.jobqueue.enqueue('__elric_remove_channel__', job.serialize())
コード例 #4
0
ファイル: rqbase.py プロジェクト: Masutangu/Elric
 def subscribe_mq(self):
     while self.running:
         try:
             # grab job from job queue only if internal_job_queue has space
             key, serialized_job = self.jobqueue.dequeue_any(['__elric_submit_channel__',
                                                              '__elric_remove_channel__',
                                                              '__elric_finish_channel__'])
             Job.deserialize(serialized_job)
             self.func_map[key](Job.deserialize(serialized_job))
         except TypeError as e:
             self.log.error(e)
             time.sleep(60)
             continue
コード例 #5
0
 def subscribe_mq(self):
     while self.running:
         try:
             # grab job from job queue only if internal_job_queue has space
             key, serialized_job = self.jobqueue.dequeue_any([
                 '__elric_submit_channel__', '__elric_remove_channel__',
                 '__elric_finish_channel__'
             ])
             Job.deserialize(serialized_job)
             self.func_map[key](Job.deserialize(serialized_job))
         except TypeError as e:
             self.log.error(e)
             time.sleep(60)
             continue
コード例 #6
0
    def start(self):
        """
            Start elric master. Select all due jobs from jobstore and enqueue them into redis queue.
            Then update due jobs' information into jobstore.
        :return:
        """
        if self.running:
            raise AlreadyRunningException
        self._stopped = False
        self.log.debug('eric master start...')

        self.start_subscribe_thread()

        while True:
            now = datetime.now(self.timezone)
            wait_seconds = None
            with distributed_lock(**settings.DISTRIBUTED_LOCK_CONFIG):
                for job_id, job_key, serialized_job in self.jobstore.get_due_jobs(
                        now):
                    # enqueue due job into redis queue
                    self._enqueue_job(job_key, serialized_job)
                    # update job's information, such as next_run_time
                    job = Job.deserialize(serialized_job)
                    last_run_time = Job.get_serial_run_times(job, now)
                    if last_run_time:
                        next_run_time = Job.get_next_trigger_time(
                            job, last_run_time[-1])
                        if next_run_time:
                            job.next_run_time = next_run_time
                            self.update_job(job)
                        else:
                            # if job has no next run time, then remove it from jobstore
                            self.remove_job(job)

                # get next closet run time job from jobstore and set it to be wake up time
                closest_run_time = self.jobstore.get_closest_run_time()

            if closest_run_time is not None:
                wait_seconds = min(
                    max(timedelta_seconds(closest_run_time - now), 0),
                    self.MIN_WAIT_TIME)
                self.log.debug('Next wakeup is due at %s (in %f seconds)' %
                               (closest_run_time, wait_seconds))
            self._event.wait(wait_seconds if wait_seconds is not None else self
                             .MIN_WAIT_TIME)
            self._event.clear()
コード例 #7
0
ファイル: rqbase.py プロジェクト: Masutangu/Elric
    def start(self):
        """
            Start elric master. Select all due jobs from jobstore and enqueue them into redis queue.
            Then update due jobs' information into jobstore.
        :return:
        """
        if self.running:
            raise AlreadyRunningException
        self._stopped = False
        self.log.debug('eric master start...')

        self.start_subscribe_thread()

        while True:
            now = datetime.now(self.timezone)
            wait_seconds = None
            with distributed_lock(**settings.DISTRIBUTED_LOCK_CONFIG):
                for job_id, job_key, serialized_job in self.jobstore.get_due_jobs(now):
                    # enqueue due job into redis queue
                    self._enqueue_job(job_key, serialized_job)
                    # update job's information, such as next_run_time
                    job = Job.deserialize(serialized_job)
                    last_run_time = Job.get_serial_run_times(job, now)
                    if last_run_time:
                        next_run_time = Job.get_next_trigger_time(job, last_run_time[-1])
                        if next_run_time:
                            job.next_run_time = next_run_time
                            self.update_job(job)
                        else:
                            # if job has no next run time, then remove it from jobstore
                            self.remove_job(job)

                # get next closet run time job from jobstore and set it to be wake up time
                closest_run_time = self.jobstore.get_closest_run_time()

            if closest_run_time is not None:
                wait_seconds = min(max(timedelta_seconds(closest_run_time - now), 0), self.MIN_WAIT_TIME)
                self.log.debug('Next wakeup is due at %s (in %f seconds)' % (closest_run_time, wait_seconds))
            self._event.wait(wait_seconds if wait_seconds is not None else self.MIN_WAIT_TIME)
            self._event.clear()
コード例 #8
0
ファイル: rqueue.py プロジェクト: Masutangu/Elric
    def start(self):
        if self.running:
            raise AlreadyRunningException

        self._stopped = False
        self.log.debug('elric worker running..')
        while self.running:
            try:
                # grab job from job queue only if internal_job_queue has space
                self.internal_job_queue.put("#", True)
                job_key, serialized_job = self.jobqueue.dequeue_any(self.listen_keys)
                job = Job.deserialize(serialized_job)
                self.log.debug('get job id=[%s] func=[%s]from key %s' % (job.id, job.func, job.job_key))
                self.executor.execute_job(job)
            except TypeError as e:
                self.log.error(e)
                time.sleep(60)
                continue