def on_failure(self, exc, task_id, args, kwargs, einfo): #self.log.info("[{0}]: FAILURE : {1}".format(task_id, exc)) # SEND FAIL MAIL mail_obj = Mail(mail_type='SYNC_FAIL', exc=exc, task_id=task_id, einfo=einfo) mail_obj.send() self.log.info("[{0}]: SYNC FAIL MAIL SENT".format(task_id))
def on_success(self, retval, task_id, args, kwargs): # SEND SPOOL COMPLETE MAIL mail_obj = Mail(mail_type='SPOOL_COMPLETE', task_id=task_id, retval=retval) mail_obj.send() self.log.info("[{0}]: SPOOL COMPLETE MAIL SENT".format(task_id)) pass
def on_success(self, retval, task_id, args, kwargs): #self.log.info("[{0}]: SUCCESS : {1}".format(task_id, retval)) # SEND DONE MAIL mail_obj = Mail(mail_type='SYNC_COMPLETE', task_id=task_id, retval=retval) mail_obj.send() self.log.info("[{0}]: SYNC COMPLETE MAIL SENT".format(task_id))
def on_retry(self, exc, task_id, args, kwargs, einfo): retry_count = args[2] + 1 #5. Retry send_ mail msg = "[{0}]: Download Failed: {1}! Retrying... attempt # {2}".format(task_id, exc.message, retry_count) self.log.info(msg) # SEND RETRY MAIL mail_obj = Mail(task_owner=self.task_owner, mail_type='DOWNLOAD_RETRY', exc=exc, task_id=task_id, task_uuid=self.task_uuid, einfo=einfo, retry=retry_count) mail_obj.send_() self.log.info("[{0}]: DOWNLOAD RETRY MAIL SENT".format(task_id))
def on_success(self, retval, task_id, args, kwargs): # SEND DONE MAIL mail_obj = Mail(task_owner=self.task_owner, mail_type='DOWNLOAD_COMPLETE', task_id=task_id, task_uuid=self.task_uuid, retval=retval, count=args[2]) mail_obj.send_() self.log.info("[{0}]: DOWNLOAD COMPLETE MAIL SENT".format(task_id))
def run(self, task_owner, task_uuid, count, **kwargs): task_id = self.request.id self.task_uuid = task_uuid self.error_type = None self.task_owner = task_owner seq_str, _ = self.task_uuid.split('__') items = seq_str.split('_') seq = items[0] scn = items[1] shot = items[2] ver = items[5] # First try if count == 0: self.log.info("[{0}]: Received task_id: {1}".format(task_id, self.task_uuid)) self.log.info("[{0}]: Received task_uuid: {1}".format(self.task_uuid, self.task_uuid)) #1. DB insert self.db_insert_task(self.task_uuid) #2. SEND SUBMIT MAIL mail_obj = Mail(task_owner=self.task_owner, mail_type='DOWNLOAD_START', task_id=task_id, task_uuid=self.task_uuid) mail_obj.send_() self.log.info("[{0}]: DOWNLOAD START MAIL SENT".format(task_id)) # Retry else: #6. Retry log self.log.info("[{0}]: Retry # {1}, task_uuid {2}".format(task_id, count, self.task_uuid)) #7. DB update #self.db_update_task(self.task_uuid, 'active', count=count) #3. Command run # ascp -O 33001 -P 33001 -k 3 -p --overwrite=diff -d render@fox:/Tactic/bilal/render/$1/$2/$3/cg/$4 /nas/projects/Tactic/bilal/render/$1/$2/$3/cg/ # #cmd = "ascp -O 33001 -P 33001 -k 3 -p --overwrite=diff -d render@fox:/Tactic/bilal/render/{0}/{1}/{2}/cg/{3} /nas/projects/Tactic/bilal/render/{0}/{1}/{2}/cg/".format(seq, scn, shot, ver) cmd = "ascp -O 33001 -P 33001 -k 3 -p --overwrite=diff -d [email protected]:/Tactic/bilal/render/{0}/{1}/{2}/cg/{3} /nas/projects/Tactic/bilal/render/{0}/{1}/{2}/cg/".format(seq, scn, shot, ver) #self.log.info("[{0}]: Running cmd: {1}".format(task_id, cmd)) try: retval = self.exec_cmd(task_id, cmd) #self.log.info("[{0}]: Finish command: {1}".format(task_id, cmd)) #db_update_task self.db_update_task(self.task_uuid, retval=retval, count=count) return "Download Success!" except TaskException as e: #4. Retry if not user aborted if self.error_type is not None and self.error_type != 'user_abort': #8. DB update self.db_update_task(self.task_uuid, 'retry', count=count, exc=e.message) self.retry(args=[task_owner, task_uuid, count+1], exc=e, kwargs=kwargs) elif self.error_type == 'user_abort': raise TaskException(e.message) """
def on_failure(self, exc, task_id, args, kwargs, einfo): msg = "[{0}]: Download failed: {1}".format(task_id, exc.message) self.log.info(msg) #db_update_task self.db_update_task(self.task_uuid, 'failed', exc=exc.message, count=args[2]) # SEND FAIL MAIL mail_obj = Mail(task_owner=self.task_owner, mail_type='DOWNLOAD_FAIL', exc=exc, task_id=task_id, task_uuid=self.task_uuid, einfo=einfo) mail_obj.send_() self.log.info("[{0}]: DOWNLOAD FAIL MAIL SENT".format(task_id))
def run(self, task_owner, dep_file_path, cmd, task_uuid, unique_id, alf_script, operation, count, **kwargs): task_id = self.request.id self.task_uuid = task_uuid self.error_type = None self.task_owner = task_owner self.dep_file_path = dep_file_path self.unique_id = unique_id self.alf_script = alf_script self.operation = operation # First try if count == 0: self.log.info("[{0}]: Inserting the db: upload".format(task_id)) self.db_insert_task(self.task_uuid, task_owner) # SEND SUBMIT MAIL mail_obj = Mail(task_owner=self.task_owner, mail_type='UPLOAD_START', task_id=task_id, task_uuid=self.task_uuid, dep_file_path=dep_file_path, cmd=cmd) mail_obj.send_() self.log.info("[{0}]: UPLOAD START MAIL SENT".format(task_id)) self.log.info("[{0}]: Dep file: {1}".format(task_id, dep_file_path)) # Retry else: #6. Retry log self.log.info("[{0}]: Retry # {1}, task_uuid {2}".format(task_id, count, self.task_uuid)) #7. DB update #self.db_update_task(self.task_uuid, 'active', count=count) try: self.log.info("[{0}]: Start command: {1}".format(task_id, cmd)) retval = self.exec_cmd(task_id, cmd) self.log.info("[{0}]: Finish command: {1}".format(task_id, cmd)) #db_update_task self.log.info("[{0}]: Updating the db: upload".format(task_id)) self.db_update_task(self.task_uuid, retval=retval, count=count) return "Upload Success!" except TaskException as e: #4. Retry if not user aborted if self.error_type is not None and self.error_type != 'user_abort': #8. DB update #self.db_update_task(self.task_uuid, 'failed', count=count, exc=e.message) self.db_update_task(self.task_uuid, 'retry', count=count, exc=e.message) self.retry(args=[task_owner, dep_file_path, cmd, task_uuid, unique_id, alf_script, operation, count+1], exc=e, kwargs=kwargs) elif self.error_type == 'user_abort': raise TaskException(e.message) """
def on_retry(self, exc, task_id, args, kwargs, einfo): retry_count = args[2] + 1 #5. Retry send_ mail msg = "[{0}]: Download Failed: {1}! Retrying... attempt # {2}".format( task_id, exc.message, retry_count) self.log.info(msg) # SEND RETRY MAIL mail_obj = Mail(task_owner=self.task_owner, mail_type='DOWNLOAD_RETRY', exc=exc, task_id=task_id, task_uuid=self.task_uuid, einfo=einfo, retry=retry_count) mail_obj.send_() self.log.info("[{0}]: DOWNLOAD RETRY MAIL SENT".format(task_id))
def run(self, dep_file_path, cmd): task_id = self.request.id #logger = get_or_create_task_logger(func=sync) # SEND SUBMIT MAIL mail_obj = Mail(mail_type='SYNC_SUBMIT', task_id=task_id) mail_obj.send() self.log.info("[{0}]: SYNC SUBMIT MAIL SENT".format(task_id)) self.log.info("[{0}]: Dep file: {1}".format(task_id, dep_file_path)) self.log.info("[{0}]: Start command: {1}".format(task_id, cmd)) retval = self.exec_cmd(task_id, cmd) self.log.info("[{0}]: Finish command: {1}".format(task_id, cmd)) if retval == 0: #msg = "[{0}]: Sync successful".format(task_id) #self.log.info(msg) return task_id else: # raise an error msg = "[{0}]: Sync failed: {0}".format(task_id) self.log.info(msg) raise TaskException(msg) '''
def send_mail(**kwargs): mail_obj = Mail(**kwargs) mail_obj.send_() self.log.info("[{0}]: DOWNLOAD FAIL MAIL SENT".format(task_id))
def run(self, task_owner, task_uuid, count, **kwargs): task_id = self.request.id self.task_uuid = task_uuid self.error_type = None self.task_owner = task_owner seq_str, _ = self.task_uuid.split('__') items = seq_str.split('_') seq = items[0] scn = items[1] shot = items[2] ver = items[5] # First try if count == 0: self.log.info("[{0}]: Received task_id: {1}".format( task_id, self.task_uuid)) self.log.info("[{0}]: Received task_uuid: {1}".format( self.task_uuid, self.task_uuid)) #1. DB insert self.db_insert_task(self.task_uuid) #2. SEND SUBMIT MAIL mail_obj = Mail(task_owner=self.task_owner, mail_type='DOWNLOAD_START', task_id=task_id, task_uuid=self.task_uuid) mail_obj.send_() self.log.info("[{0}]: DOWNLOAD START MAIL SENT".format(task_id)) # Retry else: #6. Retry log self.log.info("[{0}]: Retry # {1}, task_uuid {2}".format( task_id, count, self.task_uuid)) #7. DB update #self.db_update_task(self.task_uuid, 'active', count=count) #3. Command run # ascp -O 33001 -P 33001 -k 3 -p --overwrite=diff -d render@fox:/Tactic/bilal/render/$1/$2/$3/cg/$4 /nas/projects/Tactic/bilal/render/$1/$2/$3/cg/ # #cmd = "ascp -O 33001 -P 33001 -k 3 -p --overwrite=diff -d render@fox:/Tactic/bilal/render/{0}/{1}/{2}/cg/{3} /nas/projects/Tactic/bilal/render/{0}/{1}/{2}/cg/".format(seq, scn, shot, ver) cmd = "ascp -O 33001 -P 33001 -k 3 -p --overwrite=diff -d [email protected]:/Tactic/bilal/render/{0}/{1}/{2}/cg/{3} /nas/projects/Tactic/bilal/render/{0}/{1}/{2}/cg/".format( seq, scn, shot, ver) #self.log.info("[{0}]: Running cmd: {1}".format(task_id, cmd)) try: retval = self.exec_cmd(task_id, cmd) #self.log.info("[{0}]: Finish command: {1}".format(task_id, cmd)) #db_update_task self.db_update_task(self.task_uuid, retval=retval, count=count) return "Download Success!" except TaskException as e: #4. Retry if not user aborted if self.error_type is not None and self.error_type != 'user_abort': #8. DB update self.db_update_task(self.task_uuid, 'retry', count=count, exc=e.message) self.retry(args=[task_owner, task_uuid, count + 1], exc=e, kwargs=kwargs) elif self.error_type == 'user_abort': raise TaskException(e.message) """
def run(self, task_owner, dep_file_path, cmd, task_uuid, unique_id, alf_script, operation, count, **kwargs): task_id = self.request.id self.task_uuid = task_uuid self.error_type = None self.task_owner = task_owner self.dep_file_path = dep_file_path self.unique_id = unique_id self.alf_script = alf_script self.operation = operation # First try if count == 0: self.log.info("[{0}]: Inserting the db: upload".format(task_id)) self.db_insert_task(self.task_uuid, task_owner) # SEND SUBMIT MAIL mail_obj = Mail(task_owner=self.task_owner, mail_type='UPLOAD_START', task_id=task_id, task_uuid=self.task_uuid, dep_file_path=dep_file_path, cmd=cmd) mail_obj.send_() self.log.info("[{0}]: UPLOAD START MAIL SENT".format(task_id)) self.log.info("[{0}]: Dep file: {1}".format( task_id, dep_file_path)) # Retry else: #6. Retry log self.log.info("[{0}]: Retry # {1}, task_uuid {2}".format( task_id, count, self.task_uuid)) #7. DB update #self.db_update_task(self.task_uuid, 'active', count=count) try: self.log.info("[{0}]: Start command: {1}".format(task_id, cmd)) retval = self.exec_cmd(task_id, cmd) self.log.info("[{0}]: Finish command: {1}".format(task_id, cmd)) #db_update_task self.log.info("[{0}]: Updating the db: upload".format(task_id)) self.db_update_task(self.task_uuid, retval=retval, count=count) return "Upload Success!" except TaskException as e: #4. Retry if not user aborted if self.error_type is not None and self.error_type != 'user_abort': #8. DB update #self.db_update_task(self.task_uuid, 'failed', count=count, exc=e.message) self.db_update_task(self.task_uuid, 'retry', count=count, exc=e.message) self.retry(args=[ task_owner, dep_file_path, cmd, task_uuid, unique_id, alf_script, operation, count + 1 ], exc=e, kwargs=kwargs) elif self.error_type == 'user_abort': raise TaskException(e.message) """
def submit(**kwargs): operation = kwargs.get('launch_type') dep_file = kwargs.get('dep_file') #ascp_cmd = submit_cmds.get_ascp_cmd(kwargs.get('sync_list')) ascp_cmd = submit_cmds.get_proxy_ascp_cmd(kwargs.get('sync_list')) task_uuid = kwargs.get('task_uuid') task_owner = kwargs.get('task_owner') alf_script = kwargs.get('alf_script') priority = kwargs.get('priority') tactic_file = kwargs.get('tactic_file') task_str = kwargs.get('task_str') #spool_cmd = submit_cmds.get_tractor_spool_cmd(kwargs.get('alf_script'), kwargs.get('spool_dry_run')) #engine='54.169.63.110:1503' engine = '119.81.131.43:1503' #priority=50 #new_uuid = uuid4() file_base_name = os.path.basename(dep_file).split('.')[0] task_id = ("{0}__{1}").format(file_base_name, task_uuid) #print >>sys.stdout, "Task ID: ", task_id ''' # SEND SUBMIT MAIL mail_obj = Mail(task_owner=self.task_owner, mail_type='UPLOAD_START', task_id=task_id, task_uuid=self.task_uuid, dep_file_path=dep_file_path, cmd=cmd) mail_obj.send_() ''' count = 0 # Submit a `CHAIN` to the queue if operation == 'SYNC_RENDER': task_doc = db_insert_task('SYNC_RENDER', task_id, task_owner) counter = task_doc['_counter'] sync_task_uuid = ("UP-{0}").format(counter) spool_task_uuid = ("SPOOL-{0}").format(counter) #sync_task_uuid = ("upload__{0}__{1}").format(file_base_name, task_uuid) #spool_task_uuid = ("spool__{0}__{1}").format(file_base_name, task_uuid) # Prepare the `SyncTask` #sync_task = SyncTask.subtask(args=(task_owner, dep_file, ascp_cmd, str(task_id), str(task_uuid), alf_script, operation), task_id=sync_task_uuid) sync_task = UploadTestTask.subtask(args=(task_owner, dep_file, ascp_cmd, str(task_id), str(task_uuid), alf_script, operation, count), task_id=sync_task_uuid) # Prepare the `RenderTask` #spool_task = SpoolTask.subtask(args=(task_owner, engine, priority, alf_script, str(task_id), str(task_uuid), dep_file, operation), task_id=spool_task_uuid, immutable=True) spool_task = SpoolTestTask.subtask( args=(task_owner, engine, priority, alf_script, str(task_id), str(task_uuid), dep_file, operation, tactic_file, task_str, count), task_id=spool_task_uuid, immutable=True) # with both - sync and spool tasks chain(sync_task, spool_task)() # Send mail mail_obj = Mail(task_owner=task_owner, mail_type='UPLOAD_SUBMIT', unique_id=task_uuid, task_id=task_id, dep_file_path=dep_file, operation=operation, alf_script=alf_script, upload_id=sync_task_uuid, priority=priority, tactic_file=tactic_file, task_str=task_str) mail_obj.send_() #return (sync_task_uuid, task_id) print >> sys.stdout, sync_task_uuid, ':', task_id elif operation == 'SYNC': task_doc = db_insert_task('SYNC', task_id, task_owner) counter = task_doc['_counter'] sync_task_uuid = ("UP-{0}").format(counter) #sync_task_uuid = ("upload__{0}__{1}").format(file_base_name, task_uuid) # Prepare the `SyncTask` #sync_task = SyncTask.subtask(args=(task_owner, dep_file, ascp_cmd, str(task_id), str(task_uuid), alf_script, operation), task_id=sync_task_uuid) sync_task = UploadTestTask.subtask(args=(task_owner, dep_file, ascp_cmd, str(task_id), str(task_uuid), alf_script, operation, count), task_id=sync_task_uuid) # with just the sync task chain(sync_task)() # Send mail mail_obj = Mail(task_owner=task_owner, mail_type='UPLOAD_SUBMIT', unique_id=task_uuid, task_id=task_id, dep_file_path=dep_file, operation=operation, alf_script=alf_script, upload_id=sync_task_uuid, priority=priority, tactic_file=tactic_file, task_str=task_str) mail_obj.send_() #return (sync_task_uuid, task_id) print >> sys.stdout, sync_task_uuid, ':', task_id elif operation == 'RENDER': task_doc = db_insert_task('RENDER', task_id, task_owner) counter = task_doc['_counter'] spool_task_uuid = ("SPOOL-{0}").format(counter) #spool_task_uuid = ("spool__{0}__{1}").format(file_base_name, task_uuid) # Prepare the `RenderTask` #spool_task = SpoolTask.subtask(args=(task_owner, engine, priority, alf_script, str(task_id), str(task_uuid), dep_file, operation), task_id=spool_task_uuid, immutable=True) spool_task = SpoolTestTask.subtask( args=(task_owner, engine, priority, alf_script, str(task_id), str(task_uuid), dep_file, operation, tactic_file, task_str, count), task_id=spool_task_uuid, immutable=True) # with just the spool task chain(spool_task)() # Send mail mail_obj = Mail(task_owner=task_owner, mail_type='SPOOL_SUBMIT', unique_id=task_uuid, task_id=task_id, dep_file_path=dep_file, operation=operation, alf_script=alf_script, spool_id=spool_task_uuid, priority=priority, tactic_file=tactic_file, task_str=task_str) mail_obj.send_() #return (spool_task_uuid, task_id print >> sys.stdout, spool_task_uuid, ':', task_id
def submit(**kwargs): operation = kwargs.get('launch_type') dep_file = kwargs.get('dep_file') #ascp_cmd = submit_cmds.get_ascp_cmd(kwargs.get('sync_list')) ascp_cmd = submit_cmds.get_proxy_ascp_cmd(kwargs.get('sync_list')) task_uuid = kwargs.get('task_uuid') task_owner = kwargs.get('task_owner') alf_script=kwargs.get('alf_script') priority=kwargs.get('priority') tactic_file=kwargs.get('tactic_file') task_str=kwargs.get('task_str') #spool_cmd = submit_cmds.get_tractor_spool_cmd(kwargs.get('alf_script'), kwargs.get('spool_dry_run')) #engine='54.169.63.110:1503' engine='119.81.131.43:1503' #priority=50 #new_uuid = uuid4() file_base_name = os.path.basename(dep_file).split('.')[0] task_id = ("{0}__{1}").format(file_base_name, task_uuid) #print >>sys.stdout, "Task ID: ", task_id ''' # SEND SUBMIT MAIL mail_obj = Mail(task_owner=self.task_owner, mail_type='UPLOAD_START', task_id=task_id, task_uuid=self.task_uuid, dep_file_path=dep_file_path, cmd=cmd) mail_obj.send_() ''' count = 0 # Submit a `CHAIN` to the queue if operation == 'SYNC_RENDER': task_doc = db_insert_task('SYNC_RENDER', task_id, task_owner) counter = task_doc['_counter'] sync_task_uuid = ("UP-{0}").format(counter) spool_task_uuid = ("SPOOL-{0}").format(counter) #sync_task_uuid = ("upload__{0}__{1}").format(file_base_name, task_uuid) #spool_task_uuid = ("spool__{0}__{1}").format(file_base_name, task_uuid) # Prepare the `SyncTask` #sync_task = SyncTask.subtask(args=(task_owner, dep_file, ascp_cmd, str(task_id), str(task_uuid), alf_script, operation), task_id=sync_task_uuid) sync_task = UploadTestTask.subtask(args=(task_owner, dep_file, ascp_cmd, str(task_id), str(task_uuid), alf_script, operation, count), task_id=sync_task_uuid) # Prepare the `RenderTask` #spool_task = SpoolTask.subtask(args=(task_owner, engine, priority, alf_script, str(task_id), str(task_uuid), dep_file, operation), task_id=spool_task_uuid, immutable=True) spool_task = SpoolTestTask.subtask(args=(task_owner, engine, priority, alf_script, str(task_id), str(task_uuid), dep_file, operation, tactic_file, task_str, count), task_id=spool_task_uuid, immutable=True) # with both - sync and spool tasks chain(sync_task, spool_task)() # Send mail mail_obj = Mail(task_owner=task_owner, mail_type='UPLOAD_SUBMIT', unique_id=task_uuid, task_id=task_id, dep_file_path=dep_file, operation=operation, alf_script=alf_script, upload_id=sync_task_uuid, priority=priority, tactic_file=tactic_file, task_str=task_str) mail_obj.send_() #return (sync_task_uuid, task_id) print >>sys.stdout, sync_task_uuid, ':', task_id elif operation == 'SYNC': task_doc = db_insert_task('SYNC', task_id, task_owner) counter = task_doc['_counter'] sync_task_uuid = ("UP-{0}").format(counter) #sync_task_uuid = ("upload__{0}__{1}").format(file_base_name, task_uuid) # Prepare the `SyncTask` #sync_task = SyncTask.subtask(args=(task_owner, dep_file, ascp_cmd, str(task_id), str(task_uuid), alf_script, operation), task_id=sync_task_uuid) sync_task = UploadTestTask.subtask(args=(task_owner, dep_file, ascp_cmd, str(task_id), str(task_uuid), alf_script, operation, count), task_id=sync_task_uuid) # with just the sync task chain(sync_task)() # Send mail mail_obj = Mail(task_owner=task_owner, mail_type='UPLOAD_SUBMIT', unique_id=task_uuid, task_id=task_id, dep_file_path=dep_file, operation=operation, alf_script=alf_script, upload_id=sync_task_uuid, priority=priority, tactic_file=tactic_file, task_str=task_str) mail_obj.send_() #return (sync_task_uuid, task_id) print >>sys.stdout, sync_task_uuid, ':', task_id elif operation == 'RENDER': task_doc = db_insert_task('RENDER', task_id, task_owner) counter = task_doc['_counter'] spool_task_uuid = ("SPOOL-{0}").format(counter) #spool_task_uuid = ("spool__{0}__{1}").format(file_base_name, task_uuid) # Prepare the `RenderTask` #spool_task = SpoolTask.subtask(args=(task_owner, engine, priority, alf_script, str(task_id), str(task_uuid), dep_file, operation), task_id=spool_task_uuid, immutable=True) spool_task = SpoolTestTask.subtask(args=(task_owner, engine, priority, alf_script, str(task_id), str(task_uuid), dep_file, operation, tactic_file, task_str, count), task_id=spool_task_uuid, immutable=True) # with just the spool task chain(spool_task)() # Send mail mail_obj = Mail(task_owner=task_owner, mail_type='SPOOL_SUBMIT', unique_id=task_uuid, task_id=task_id, dep_file_path=dep_file, operation=operation, alf_script=alf_script, spool_id=spool_task_uuid, priority=priority, tactic_file=tactic_file, task_str=task_str) mail_obj.send_() #return (spool_task_uuid, task_id print >>sys.stdout, spool_task_uuid, ':', task_id