コード例 #1
0
ファイル: tasks.py プロジェクト: makeittotop/py_queue
 def on_success(self, retval, task_id, args, kwargs):
     # SEND DONE MAIL
     mail_obj = Mail(task_owner=self.task_owner,
                     mail_type='DOWNLOAD_COMPLETE',
                     task_id=task_id,
                     task_uuid=self.task_uuid,
                     retval=retval,
                     count=args[2])
     mail_obj.send_()
     self.log.info("[{0}]: DOWNLOAD COMPLETE MAIL SENT".format(task_id))
コード例 #2
0
ファイル: tasks.py プロジェクト: makeittotop/py_queue
 def on_retry(self, exc, task_id, args, kwargs, einfo):
     retry_count = args[2] + 1
     #5. Retry send_ mail
     msg = "[{0}]: Download Failed: {1}! Retrying... attempt # {2}".format(
         task_id, exc.message, retry_count)
     self.log.info(msg)
     # SEND RETRY MAIL
     mail_obj = Mail(task_owner=self.task_owner,
                     mail_type='DOWNLOAD_RETRY',
                     exc=exc,
                     task_id=task_id,
                     task_uuid=self.task_uuid,
                     einfo=einfo,
                     retry=retry_count)
     mail_obj.send_()
     self.log.info("[{0}]: DOWNLOAD RETRY MAIL SENT".format(task_id))
コード例 #3
0
ファイル: tasks.py プロジェクト: makeittotop/py_queue
    def on_failure(self, exc, task_id, args, kwargs, einfo):
        msg = "[{0}]: Download failed: {1}".format(task_id, exc.message)
        self.log.info(msg)

        #db_update_task
        self.db_update_task(self.task_uuid,
                            'failed',
                            exc=exc.message,
                            count=args[2])

        # SEND FAIL MAIL
        mail_obj = Mail(task_owner=self.task_owner,
                        mail_type='DOWNLOAD_FAIL',
                        exc=exc,
                        task_id=task_id,
                        task_uuid=self.task_uuid,
                        einfo=einfo)
        mail_obj.send_()
        self.log.info("[{0}]: DOWNLOAD FAIL MAIL SENT".format(task_id))
コード例 #4
0
def submit(**kwargs):
    operation = kwargs.get('launch_type')
    dep_file = kwargs.get('dep_file')
    #ascp_cmd = submit_cmds.get_ascp_cmd(kwargs.get('sync_list'))
    ascp_cmd = submit_cmds.get_proxy_ascp_cmd(kwargs.get('sync_list'))
    task_uuid = kwargs.get('task_uuid')
    task_owner = kwargs.get('task_owner')
    alf_script = kwargs.get('alf_script')
    priority = kwargs.get('priority')
    tactic_file = kwargs.get('tactic_file')
    task_str = kwargs.get('task_str')

    #spool_cmd = submit_cmds.get_tractor_spool_cmd(kwargs.get('alf_script'), kwargs.get('spool_dry_run'))
    #engine='54.169.63.110:1503'
    engine = '119.81.131.43:1503'
    #priority=50

    #new_uuid = uuid4()
    file_base_name = os.path.basename(dep_file).split('.')[0]
    task_id = ("{0}__{1}").format(file_base_name, task_uuid)
    #print >>sys.stdout, "Task ID:  ", task_id
    '''
    # SEND SUBMIT MAIL
    mail_obj = Mail(task_owner=self.task_owner, mail_type='UPLOAD_START', task_id=task_id, task_uuid=self.task_uuid, dep_file_path=dep_file_path, cmd=cmd)
    mail_obj.send_()
    '''

    count = 0
    # Submit a `CHAIN` to the queue
    if operation == 'SYNC_RENDER':
        task_doc = db_insert_task('SYNC_RENDER', task_id, task_owner)
        counter = task_doc['_counter']

        sync_task_uuid = ("UP-{0}").format(counter)
        spool_task_uuid = ("SPOOL-{0}").format(counter)
        #sync_task_uuid = ("upload__{0}__{1}").format(file_base_name, task_uuid)
        #spool_task_uuid = ("spool__{0}__{1}").format(file_base_name, task_uuid)

        # Prepare the `SyncTask`
        #sync_task = SyncTask.subtask(args=(task_owner, dep_file, ascp_cmd, str(task_id), str(task_uuid), alf_script, operation), task_id=sync_task_uuid)
        sync_task = UploadTestTask.subtask(args=(task_owner, dep_file,
                                                 ascp_cmd, str(task_id),
                                                 str(task_uuid), alf_script,
                                                 operation, count),
                                           task_id=sync_task_uuid)

        # Prepare the `RenderTask`
        #spool_task = SpoolTask.subtask(args=(task_owner, engine, priority, alf_script, str(task_id), str(task_uuid), dep_file, operation), task_id=spool_task_uuid, immutable=True)
        spool_task = SpoolTestTask.subtask(
            args=(task_owner, engine, priority, alf_script, str(task_id),
                  str(task_uuid), dep_file, operation, tactic_file, task_str,
                  count),
            task_id=spool_task_uuid,
            immutable=True)

        # with both - sync and spool tasks
        chain(sync_task, spool_task)()

        # Send mail
        mail_obj = Mail(task_owner=task_owner,
                        mail_type='UPLOAD_SUBMIT',
                        unique_id=task_uuid,
                        task_id=task_id,
                        dep_file_path=dep_file,
                        operation=operation,
                        alf_script=alf_script,
                        upload_id=sync_task_uuid,
                        priority=priority,
                        tactic_file=tactic_file,
                        task_str=task_str)
        mail_obj.send_()

        #return (sync_task_uuid, task_id)
        print >> sys.stdout, sync_task_uuid, ':', task_id

    elif operation == 'SYNC':
        task_doc = db_insert_task('SYNC', task_id, task_owner)
        counter = task_doc['_counter']

        sync_task_uuid = ("UP-{0}").format(counter)
        #sync_task_uuid = ("upload__{0}__{1}").format(file_base_name, task_uuid)

        # Prepare the `SyncTask`
        #sync_task = SyncTask.subtask(args=(task_owner, dep_file, ascp_cmd, str(task_id), str(task_uuid), alf_script, operation), task_id=sync_task_uuid)
        sync_task = UploadTestTask.subtask(args=(task_owner, dep_file,
                                                 ascp_cmd, str(task_id),
                                                 str(task_uuid), alf_script,
                                                 operation, count),
                                           task_id=sync_task_uuid)

        # with just the sync task
        chain(sync_task)()

        # Send mail
        mail_obj = Mail(task_owner=task_owner,
                        mail_type='UPLOAD_SUBMIT',
                        unique_id=task_uuid,
                        task_id=task_id,
                        dep_file_path=dep_file,
                        operation=operation,
                        alf_script=alf_script,
                        upload_id=sync_task_uuid,
                        priority=priority,
                        tactic_file=tactic_file,
                        task_str=task_str)
        mail_obj.send_()

        #return (sync_task_uuid, task_id)
        print >> sys.stdout, sync_task_uuid, ':', task_id

    elif operation == 'RENDER':
        task_doc = db_insert_task('RENDER', task_id, task_owner)
        counter = task_doc['_counter']

        spool_task_uuid = ("SPOOL-{0}").format(counter)
        #spool_task_uuid = ("spool__{0}__{1}").format(file_base_name, task_uuid)

        # Prepare the `RenderTask`
        #spool_task = SpoolTask.subtask(args=(task_owner, engine, priority, alf_script, str(task_id), str(task_uuid), dep_file, operation), task_id=spool_task_uuid, immutable=True)
        spool_task = SpoolTestTask.subtask(
            args=(task_owner, engine, priority, alf_script, str(task_id),
                  str(task_uuid), dep_file, operation, tactic_file, task_str,
                  count),
            task_id=spool_task_uuid,
            immutable=True)

        # with just the spool task
        chain(spool_task)()

        # Send mail
        mail_obj = Mail(task_owner=task_owner,
                        mail_type='SPOOL_SUBMIT',
                        unique_id=task_uuid,
                        task_id=task_id,
                        dep_file_path=dep_file,
                        operation=operation,
                        alf_script=alf_script,
                        spool_id=spool_task_uuid,
                        priority=priority,
                        tactic_file=tactic_file,
                        task_str=task_str)
        mail_obj.send_()

        #return (spool_task_uuid, task_id
        print >> sys.stdout, spool_task_uuid, ':', task_id
コード例 #5
0
ファイル: tasks.py プロジェクト: makeittotop/py_queue
    def run(self, task_owner, dep_file_path, cmd, task_uuid, unique_id,
            alf_script, operation, count, **kwargs):
        task_id = self.request.id
        self.task_uuid = task_uuid

        self.error_type = None

        self.task_owner = task_owner
        self.dep_file_path = dep_file_path
        self.unique_id = unique_id
        self.alf_script = alf_script
        self.operation = operation

        # First try
        if count == 0:
            self.log.info("[{0}]: Inserting the db: upload".format(task_id))
            self.db_insert_task(self.task_uuid, task_owner)

            # SEND SUBMIT MAIL
            mail_obj = Mail(task_owner=self.task_owner,
                            mail_type='UPLOAD_START',
                            task_id=task_id,
                            task_uuid=self.task_uuid,
                            dep_file_path=dep_file_path,
                            cmd=cmd)
            mail_obj.send_()
            self.log.info("[{0}]: UPLOAD START MAIL SENT".format(task_id))

            self.log.info("[{0}]: Dep file: {1}".format(
                task_id, dep_file_path))
        # Retry
        else:
            #6. Retry log
            self.log.info("[{0}]: Retry # {1}, task_uuid {2}".format(
                task_id, count, self.task_uuid))

            #7. DB update
            #self.db_update_task(self.task_uuid, 'active', count=count)

        try:
            self.log.info("[{0}]: Start command: {1}".format(task_id, cmd))
            retval = self.exec_cmd(task_id, cmd)
            self.log.info("[{0}]: Finish command: {1}".format(task_id, cmd))

            #db_update_task
            self.log.info("[{0}]: Updating the db: upload".format(task_id))
            self.db_update_task(self.task_uuid, retval=retval, count=count)

            return "Upload Success!"
        except TaskException as e:
            #4. Retry if not user aborted
            if self.error_type is not None and self.error_type != 'user_abort':
                #8. DB update
                #self.db_update_task(self.task_uuid, 'failed', count=count, exc=e.message)
                self.db_update_task(self.task_uuid,
                                    'retry',
                                    count=count,
                                    exc=e.message)
                self.retry(args=[
                    task_owner, dep_file_path, cmd, task_uuid, unique_id,
                    alf_script, operation, count + 1
                ],
                           exc=e,
                           kwargs=kwargs)
            elif self.error_type == 'user_abort':
                raise TaskException(e.message)
        """
コード例 #6
0
ファイル: tasks.py プロジェクト: makeittotop/py_queue
 def send_mail(**kwargs):
     mail_obj = Mail(**kwargs)
     mail_obj.send_()
     self.log.info("[{0}]: DOWNLOAD FAIL MAIL SENT".format(task_id))
コード例 #7
0
ファイル: tasks.py プロジェクト: makeittotop/py_queue
    def run(self, task_owner, task_uuid, count, **kwargs):
        task_id = self.request.id
        self.task_uuid = task_uuid
        self.error_type = None
        self.task_owner = task_owner

        seq_str, _ = self.task_uuid.split('__')
        items = seq_str.split('_')
        seq = items[0]
        scn = items[1]
        shot = items[2]
        ver = items[5]

        # First try
        if count == 0:
            self.log.info("[{0}]: Received task_id: {1}".format(
                task_id, self.task_uuid))
            self.log.info("[{0}]: Received task_uuid: {1}".format(
                self.task_uuid, self.task_uuid))

            #1.  DB insert
            self.db_insert_task(self.task_uuid)

            #2. SEND SUBMIT MAIL
            mail_obj = Mail(task_owner=self.task_owner,
                            mail_type='DOWNLOAD_START',
                            task_id=task_id,
                            task_uuid=self.task_uuid)
            mail_obj.send_()
            self.log.info("[{0}]: DOWNLOAD START MAIL SENT".format(task_id))
        # Retry
        else:
            #6. Retry log
            self.log.info("[{0}]: Retry # {1}, task_uuid {2}".format(
                task_id, count, self.task_uuid))

            #7. DB update
            #self.db_update_task(self.task_uuid, 'active', count=count)

        #3. Command run
        # ascp -O 33001 -P 33001 -k 3 -p --overwrite=diff -d render@fox:/Tactic/bilal/render/$1/$2/$3/cg/$4 /nas/projects/Tactic/bilal/render/$1/$2/$3/cg/ #
        #cmd = "ascp -O 33001 -P 33001 -k 3 -p --overwrite=diff -d render@fox:/Tactic/bilal/render/{0}/{1}/{2}/cg/{3} /nas/projects/Tactic/bilal/render/{0}/{1}/{2}/cg/".format(seq, scn, shot, ver)
        cmd = "ascp -O 33001 -P 33001 -k 3 -p --overwrite=diff -d [email protected]:/Tactic/bilal/render/{0}/{1}/{2}/cg/{3} /nas/projects/Tactic/bilal/render/{0}/{1}/{2}/cg/".format(
            seq, scn, shot, ver)
        #self.log.info("[{0}]: Running cmd: {1}".format(task_id, cmd))

        try:
            retval = self.exec_cmd(task_id, cmd)
            #self.log.info("[{0}]: Finish command: {1}".format(task_id, cmd))
            #db_update_task
            self.db_update_task(self.task_uuid, retval=retval, count=count)
            return "Download Success!"
        except TaskException as e:
            #4. Retry if not user aborted
            if self.error_type is not None and self.error_type != 'user_abort':
                #8. DB update
                self.db_update_task(self.task_uuid,
                                    'retry',
                                    count=count,
                                    exc=e.message)
                self.retry(args=[task_owner, task_uuid, count + 1],
                           exc=e,
                           kwargs=kwargs)
            elif self.error_type == 'user_abort':
                raise TaskException(e.message)
        """