def updateCatalog (self, masterCat, other):
newResources = []
master = GraysonUtil.readFile (masterCat)
masterLines = master.split ('\n')
for line in masterLines:
logger.debug (" __ _ _ _ _ _ _____: %s", line)
sub = GraysonUtil.readFile (other)
subLines = sub.split ('\n')
masterMap = {}
for line in masterLines:
parts = line.split (' ')
if len (parts) > 0:
resource = parts [0]
if resource:
masterMap [resource] = resource
logger.debug (" resource : %s", resource)
for line in subLines:
parts = line.split (' ')
if len (parts) > 0:
resource = parts [0]
if resource and not resource in masterMap:
newResources.append (line)
logger.debug (" new resource: %s", resource)
GraysonUtil.writeFile (masterCat, "%s\n%s" % (master, '\n'.join (newResources)))
def readFlows (self, root):
fileName = self.flowPath (root)
text = GraysonUtil.readFile (fileName)
flows = []
if text:
flows = json.loads (text)
return flows
def examineWorkflow (self):
jobstatelog = os.path.join (self.workdir, 'jobstate.log')
sched_id = 0
def process (line):
finishedTag = 'DAGMAN_FINISHED'
schedIdTag = 'DAGMAN STARTED'
index = line.find (schedIdTag)
if index > -1:
sched_id = line.split (' ')[-2]
index = line.find (finishedTag)
if index > -1:
self.isComplete = True
text = GraysonUtil.readFile (jobstatelog, process)
output = []
executor = Executor ({
'condorHome' : os.environ ['CONDOR_HOME'],
'sched_id' : sched_id
})
executor.execute (command = "${condorHome}/bin/condor_q ${sched_id} -format '%s' JobStatus",
pipe = True,
processor = lambda n : output.append (n))
self.isRunning = ''.join (output) == WorkflowStatus.CONDOR_JOB_STATUS__RUNNING
logger.debug ("WorkflowMonitor - isRunning=%s, isComplete=%s", self.isRunning, self.isComplete)
def get_job_status(self, path):
value = ""
status = os.path.join(path, "jobstate.log")
try:
text = GraysonUtil.readFile(status)
text = text.split("\n")
if len(text) > 2:
for line in text[len(text) - 3 :]:
logger.debug("line: %s", line)
if "DAGMAN_FINISHED" in line and "0 ***" in line:
value = "0"
break
elif "DAGMAN_FINISHED" in line and "1 ***" in line:
value = "1"
break
except IOError as e:
pass
return value
def get_job_output (request):
user = ViewUtil.get_user (request)
workdir = request.REQUEST ['workdir']
workflow_id = request.REQUEST ['workflowid']
job_id = request.REQUEST ['jobid']
run_id = request.REQUEST ['runid']
if not run_id:
run_id = ""
if not workflow_id:
workflow_id = ""
logger.debug ("getting job output: workdir=%s, workflowid: %s, runid: %s, jobid: %s", workdir, workflow_id, run_id, job_id)
process_username = ViewUtil.get_os_username ()
workdirPath = GraysonUtil.form_workdir_path (workdir, process_username, workflow_id, run_id)
workdirPath = ViewUtil.form_workflow_path (user, workdirPath)
logger.debug ("workdirPath: %s", workdirPath)
text = ""
if job_id.startswith ('/'):
job_id = job_id [1:]
concrete = os.path.join (workdirPath, job_id)
logger.debug ('concrete: --- %s', concrete)
if os.path.exists (concrete):
logger.debug ("concrete --- : %s", concrete)
text = GraysonUtil.readFile (concrete)
else:
logger.debug ("regex: --- : %s", concrete)
workflow = GridWorkflow (workdirPath)
outputs = workflow.getOutputFiles (subworkflows = [ workdirPath ], item = job_id)
jobOutput = None
if outputs and len (outputs) > 0:
jobOutput = outputs [0]
logger.debug ("got job output: %s \n for job_id: %s", jobOutput, job_id)
if jobOutput:
text = GraysonUtil.readFileAsString (jobOutput)
return ViewUtil.get_text_response (text)
def getfile (request):
# TODO: construct path dynamically and selectively for security purposes.
return ViewUtil.get_text_response (GraysonUtil.readFile (request.REQUEST ['file']))
def detectEventDetails (self, event, logdir, status):
jobId = event.name
aux = { 'sched_id' : event.sched_id }
if not logdir:
return aux
if jobId.startswith ('stage_in') or jobId.startswith ('stage_out'):
path = os.path.join (logdir, "%s.in" % jobId)
#logger.debug ("opening path: %s", path)
text = GraysonUtil.readFile (path)
lines = text.split ('\n')
if lines:
transfers = []
pair = 0
while len (lines) > ((pair * 4) + 4):
'''
for line in lines:
logger.debug ("line: %s", line)
'''
offset = pair * 4
transfer = {
"sourceSite" : lines [offset + 0],
"sourceFile" : lines [offset + 1],
"destSite" : lines [offset + 2],
"destFile" : lines [offset + 3],
}
pair += 1
execution = self.getExecutionData (logdir, jobId)
if execution:
stdout = execution ['stdout']
transferred = GraysonUtil.getPrecompiledPattern (self.transferBytesPattern, stdout)
duration = GraysonUtil.getPrecompiledPattern (self.transferDurationPattern, stdout)
rateUp = GraysonUtil.getPrecompiledPattern (self.transferRateUpPattern, stdout)
rateDown = GraysonUtil.getPrecompiledPattern (self.transferRateDownPattern, stdout)
transfer ["bytes"] = transferred
transfer ["time"] = duration
transfer ["up"] = rateUp
transfer ["down"] = rateDown
'''
logger.debug ("kickstart stdout/err: %s" + json.dumps (execution, indent=4))
logger.debug (" transferred: %s duration: %s rateUp: %s rateDown: %s", transferred, duration, rateUp, rateDown)
'''
transfers.append (transfer)
aux ['transfer'] = transfers
if status == WorkflowStatus.STATUS_FAILED:
execution = self.getExecutionData (logdir, jobId)
if execution:
aux ["detail"] = {
"stdout" : GraysonUtil.ceilingString (execution ["stdout"], maxLength=500, fromEnd=True),
"stderr" : GraysonUtil.ceilingString (execution ["stderr"], maxLength=500, fromEnd=True)
}
dagLog = glob.glob (os.path.join (logdir, '*.dag.dagman.out'))
dax = glob.glob (os.path.join (logdir, 'dax', '*.dax'))
if len(dagLog) > 0 or len (dax) > 0:
log = {}
aux ['log'] = log
if len (dagLog) > 0:
log ['daglog'] = os.path.basename (dagLog [0])
if len (dax) > 0:
log ['dax'] = os.path.basename (dax [0])
return aux
def updateCatalog (self, master, other):
masterText = GraysonUtil.readFile (master)
text = GraysonUtil.readFile (other)
GraysonUtil.writeFile (master, "%s\n%s" % (masterText, text))
def getFileLines (self, fileName):
return GraysonUtil.readFile (fileName).split ('\n')
