def execute(self): """ Task manager calls this function to execute the task. Return: MRLocalTaskResult """ #TODO - this cmd should be a member of the RTSJobParam class # which is read from the configuration file cmd = 'bash /scratch/astronomy556/MWA/ngas_rt/src/ngamsPlugIns/ngamsJob_MWA_RTS_Task.sh ' cmd += self.paramsToArgs() args = shlex.split(cmd) try: self._subproc = subprocess.Popen(args, stdout=subprocess.PIPE, preexec_fn=os.setsid) output = self._subproc.communicate()[ 0] # this will block until either task done or task killed retval = self._subproc.returncode ret = MRLocalTaskResult(self._taskId, retval, output, True) return ret except Exception, err: #logger.error('Fail to launch RTS Task %s: %s' % (self._taskId, str(err))) ret = MRLocalTaskResult(self._taskId, -128, str(err), True) return ret
def handleCmd(srvObj, reqPropsObj, httpRef): """ Handle the RUN TASK (RUNTASK) Command. srvObj: Reference to NG/AMS server class object (ngamsServer). reqPropsObj: Request Property object to keep track of actions done during the request handling (ngamsReqProps). httpRef: Reference to the HTTP request handler object (ngamsHttpRequestHandler). Returns: Void. """ T = TRACE() global queScanThread httpMethod = reqPropsObj.getHttpMethod() if (httpMethod != 'POST'): errMsg = 'OK' if (reqPropsObj.hasHttpPar('action')): action_req = reqPropsObj.getHttpPar('action') if ('cancel' == action_req): if (reqPropsObj.hasHttpPar('task_id')): taskId = reqPropsObj.getHttpPar('task_id') cancelDict[taskId] = 1 if (g_mrLocalTask and taskId == g_mrLocalTask._taskId): res = g_mrLocalTask.stop() if (res[0]): errMsg = 'Cancel result failed: %s\n' % str(res[1]) else: errMsg = 'task_id is missing' else: errMsg = 'Unknown RUNTASK command action %s' % action_req else: errMsg = 'RUNTASK command needs action for GET request\n' httpRef.send_data(errMsg, NGAMS_TEXT_MT) else: postContent = _getPostContent(srvObj, reqPropsObj, httpRef) mrLocalTask = pickle.loads(postContent) if (not mrLocalTask): errMsg = 'Cannot instantiate local task from POST' mrr = MRLocalTaskResult(None, -2, errMsg) httpRef.send_data(pickle.dumps(mrr), NGAMS_TEXT_MT) else: logger.debug('Local task %s is submitted', mrLocalTask._taskId) mrr = MRLocalTaskResult(mrLocalTask._taskId, 0, '') httpRef.send_data(pickle.dumps(mrr), NGAMS_TEXT_MT) args = (srvObj, mrLocalTask) scheduleThread = threading.Thread(None, _scheduleQScanThread, 'SCHEDULE_THRD', args) scheduleThread.setDaemon(0) scheduleThread.start()
def execute(self): """ Task manager calls this function to execute the task. Return: MRLocalTaskResult """ # create the working directory ids = self._taskId.split('__') job_id = ids[0] obs_num = int(ids[1]) work_dir = '%s/%s/%s/%d' % (self._params.ngas_processing_path, self._taskId, job_id, obs_num) if (os.path.exists(work_dir)): cmd = 'rm -rf %s' % work_dir ret = self._runBashCmd(cmd) if (ret): return ret cmd = 'mkdir -p %s' % work_dir ret = self._runBashCmd(cmd) if (ret): return ret # cd working directory and download images if (not os.path.exists(work_dir)): ret = MRLocalTaskResult(self._taskId, ERROR_LT_FAILCREATEDIR, 'Fail to create directory %s' % work_dir, True) return ret os.chdir(work_dir) for imgurl in self._imgurl_list: cmd = 'wget -O tmp.tar %s' % imgurl self._runBashCmd(cmd, False) cmd = 'tar -xf tmp.tar' self._runBashCmd(cmd, False) cmd = 'rm tmp.tar' self._runBashCmd(cmd, False) # go to the upper level directory, and pack all image files together upp_dir = '%s/%s' % (self._params.ngas_processing_path, self._taskId) os.chdir(upp_dir) cmd = 'tar -cf %s.tar %s/' % (self._taskId, job_id) ret = self._runBashCmd(cmd) if (ret): return ret # record the image path so that cmd_RUNTASK can use it to archive it on the # same host complete_img_path = '%s/%s.tar' % (upp_dir, self._taskId) ret = MRLocalTaskResult(self._taskId, 0, complete_img_path, True) return ret
def _queScanThread(jobManHost, ngas_hostId, ngas_client): svrUrl = 'http://%s/localtask/result' % jobManHost dqUrl = 'http://%s/localtask/dequeue?task_id=' % jobManHost global g_mrLocalTask while (1): g_mrLocalTask = queTasks.get() # skip tasks that have been cancelled if (cancelDict.has_key(g_mrLocalTask._taskId)): logger.debug('Task %s has been cancelled, skip it', g_mrLocalTask._taskId) continue # before executing the task, inform JobMAN that # this task is just dequeued and about to start... try: strRes = urllib2.urlopen( dqUrl + urllib2.quote(g_mrLocalTask._taskId), timeout=15).read() #HTTP Get (need to encode url) if (strRes.find('Error response') > -1): logger.error('Error when sending dequeue event to JobMAN: %s', strRes) except urllib2.URLError, urlerr: logger.error('Fail to send dequeue event to JobMAN: %s', str(urlerr)) # execute the task localTaskResult = None try: localTaskResult = g_mrLocalTask.execute() except Exception, execErr: logger.exception('Fail to execute task %s: Unexpected exception', g_mrLocalTask._taskId) localTaskResult = MRLocalTaskResult(g_mrLocalTask._taskId, ERROR_LT_UNEXPECTED, str(execErr), True)
def _runBashCmd(self, cmd, failonerr=True): re = commands.getstatusoutput(cmd) if (failonerr and re[0]): ret = MRLocalTaskResult(self._taskId, re[0], 'Fail to %s due to %s' % (cmd, re[1])) return ret else: return None
def execute(self): ret = MRLocalTaskResult(self._taskId, 0, 'everything is fine') time.sleep(10) #simulate work return ret