def updateCatalog (self, masterCat, other): newResources = [] master = GraysonUtil.readFile (masterCat) masterLines = master.split ('\n') for line in masterLines: logger.debug (" __ _ _ _ _ _ _____: %s", line) sub = GraysonUtil.readFile (other) subLines = sub.split ('\n') masterMap = {} for line in masterLines: parts = line.split (' ') if len (parts) > 0: resource = parts [0] if resource: masterMap [resource] = resource logger.debug (" resource : %s", resource) for line in subLines: parts = line.split (' ') if len (parts) > 0: resource = parts [0] if resource and not resource in masterMap: newResources.append (line) logger.debug (" new resource: %s", resource) GraysonUtil.writeFile (masterCat, "%s\n%s" % (master, '\n'.join (newResources)))
def readFlows (self, root): fileName = self.flowPath (root) text = GraysonUtil.readFile (fileName) flows = [] if text: flows = json.loads (text) return flows
def examineWorkflow (self): jobstatelog = os.path.join (self.workdir, 'jobstate.log') sched_id = 0 def process (line): finishedTag = 'DAGMAN_FINISHED' schedIdTag = 'DAGMAN STARTED' index = line.find (schedIdTag) if index > -1: sched_id = line.split (' ')[-2] index = line.find (finishedTag) if index > -1: self.isComplete = True text = GraysonUtil.readFile (jobstatelog, process) output = [] executor = Executor ({ 'condorHome' : os.environ ['CONDOR_HOME'], 'sched_id' : sched_id }) executor.execute (command = "${condorHome}/bin/condor_q ${sched_id} -format '%s' JobStatus", pipe = True, processor = lambda n : output.append (n)) self.isRunning = ''.join (output) == WorkflowStatus.CONDOR_JOB_STATUS__RUNNING logger.debug ("WorkflowMonitor - isRunning=%s, isComplete=%s", self.isRunning, self.isComplete)
def get_job_status(self, path): value = "" status = os.path.join(path, "jobstate.log") try: text = GraysonUtil.readFile(status) text = text.split("\n") if len(text) > 2: for line in text[len(text) - 3 :]: logger.debug("line: %s", line) if "DAGMAN_FINISHED" in line and "0 ***" in line: value = "0" break elif "DAGMAN_FINISHED" in line and "1 ***" in line: value = "1" break except IOError as e: pass return value
def get_job_output (request): user = ViewUtil.get_user (request) workdir = request.REQUEST ['workdir'] workflow_id = request.REQUEST ['workflowid'] job_id = request.REQUEST ['jobid'] run_id = request.REQUEST ['runid'] if not run_id: run_id = "" if not workflow_id: workflow_id = "" logger.debug ("getting job output: workdir=%s, workflowid: %s, runid: %s, jobid: %s", workdir, workflow_id, run_id, job_id) process_username = ViewUtil.get_os_username () workdirPath = GraysonUtil.form_workdir_path (workdir, process_username, workflow_id, run_id) workdirPath = ViewUtil.form_workflow_path (user, workdirPath) logger.debug ("workdirPath: %s", workdirPath) text = "" if job_id.startswith ('/'): job_id = job_id [1:] concrete = os.path.join (workdirPath, job_id) logger.debug ('concrete: --- %s', concrete) if os.path.exists (concrete): logger.debug ("concrete --- : %s", concrete) text = GraysonUtil.readFile (concrete) else: logger.debug ("regex: --- : %s", concrete) workflow = GridWorkflow (workdirPath) outputs = workflow.getOutputFiles (subworkflows = [ workdirPath ], item = job_id) jobOutput = None if outputs and len (outputs) > 0: jobOutput = outputs [0] logger.debug ("got job output: %s \n for job_id: %s", jobOutput, job_id) if jobOutput: text = GraysonUtil.readFileAsString (jobOutput) return ViewUtil.get_text_response (text)
def getfile (request): # TODO: construct path dynamically and selectively for security purposes. return ViewUtil.get_text_response (GraysonUtil.readFile (request.REQUEST ['file']))
def detectEventDetails (self, event, logdir, status): jobId = event.name aux = { 'sched_id' : event.sched_id } if not logdir: return aux if jobId.startswith ('stage_in') or jobId.startswith ('stage_out'): path = os.path.join (logdir, "%s.in" % jobId) #logger.debug ("opening path: %s", path) text = GraysonUtil.readFile (path) lines = text.split ('\n') if lines: transfers = [] pair = 0 while len (lines) > ((pair * 4) + 4): ''' for line in lines: logger.debug ("line: %s", line) ''' offset = pair * 4 transfer = { "sourceSite" : lines [offset + 0], "sourceFile" : lines [offset + 1], "destSite" : lines [offset + 2], "destFile" : lines [offset + 3], } pair += 1 execution = self.getExecutionData (logdir, jobId) if execution: stdout = execution ['stdout'] transferred = GraysonUtil.getPrecompiledPattern (self.transferBytesPattern, stdout) duration = GraysonUtil.getPrecompiledPattern (self.transferDurationPattern, stdout) rateUp = GraysonUtil.getPrecompiledPattern (self.transferRateUpPattern, stdout) rateDown = GraysonUtil.getPrecompiledPattern (self.transferRateDownPattern, stdout) transfer ["bytes"] = transferred transfer ["time"] = duration transfer ["up"] = rateUp transfer ["down"] = rateDown ''' logger.debug ("kickstart stdout/err: %s" + json.dumps (execution, indent=4)) logger.debug (" transferred: %s duration: %s rateUp: %s rateDown: %s", transferred, duration, rateUp, rateDown) ''' transfers.append (transfer) aux ['transfer'] = transfers if status == WorkflowStatus.STATUS_FAILED: execution = self.getExecutionData (logdir, jobId) if execution: aux ["detail"] = { "stdout" : GraysonUtil.ceilingString (execution ["stdout"], maxLength=500, fromEnd=True), "stderr" : GraysonUtil.ceilingString (execution ["stderr"], maxLength=500, fromEnd=True) } dagLog = glob.glob (os.path.join (logdir, '*.dag.dagman.out')) dax = glob.glob (os.path.join (logdir, 'dax', '*.dax')) if len(dagLog) > 0 or len (dax) > 0: log = {} aux ['log'] = log if len (dagLog) > 0: log ['daglog'] = os.path.basename (dagLog [0]) if len (dax) > 0: log ['dax'] = os.path.basename (dax [0]) return aux
def updateCatalog (self, master, other): masterText = GraysonUtil.readFile (master) text = GraysonUtil.readFile (other) GraysonUtil.writeFile (master, "%s\n%s" % (masterText, text))
def getFileLines (self, fileName): return GraysonUtil.readFile (fileName).split ('\n')