else:
outputDir = group
if outputDir:
if not os.path.exists( outputDir ):
os.makedirs( outputDir )
else:
outputDir = os.getcwd()
jobs = [ str( job ) for job in jobs ]
doneJobs = os.listdir( outputDir )
todoJobs = [ job for job in jobs if not job in doneJobs ]
for job in todoJobs:
result = dirac.getOutputSandbox( job, outputDir = outputDir )
jobDir = str( job )
if outputDir:
jobDir = os.path.join( outputDir, job )
if result['OK']:
if os.path.exists( jobDir ):
print 'Job output sandbox retrieved in %s/' % ( jobDir )
else:
if os.path.exists( '%s' % jobDir ):
shutil.rmtree( jobDir )
errorList.append( ( job, result['Message'] ) )
exitCode = 2
for error in errorList:
print "ERROR %s: %s" % error
jobName = sys.argv[1]
finalStatus = ['Done', 'Failed']
dirac = Dirac()
idstr = open("%s/jobIdList.txt"%jobName, 'r').readlines()
ids = map(int, idstr)
print "found %s jobs"%(len(ids))
res = dirac.getJobSummary(ids)
if not res['OK']:
print res['Message']
sys.exit(1)
metadata = res['Value']
for jid in ids:
jobMeta = metadata.get( jid, None )
if not jobMeta :
print "No metadata for job ", jid
continue
status = jobMeta['Status']
print "%s %s" % ( jid, status )
if status in finalStatus:
outputDir = '%s/%s'%(jobName,status)
if not os.path.exists( "%s/%s" % ( outputDir, jid ) ):
print "Retrieving sandbox"
res = dirac.getOutputSandbox( jid, outputDir = outputDir )
# check if 'jobmeta.inf' is present (if not it's not a PIPELINE job )
if not os.path.isfile(dir_temp + "/InputSandbox" + j + "/jobmeta.inf"):
print "WARNING : not a pipeline task"
# notify the job as "already handled"
jobid_handled.append(j)
else:
# Get the working dir of the task from 'jobmeta.inf'
file_jobmeta = open(dir_temp + "/InputSandbox" + j + "/jobmeta.inf", "r")
workdir = file_jobmeta.readline().splitlines()[0]
# TEST workdir = dir_temp+"/"+file_jobmeta.readline().splitlines()[0]
file_jobmeta.close()
# retrieve the OUTPUT sandbox
res = d.getOutputSandbox(j, dir_temp)
if not res["OK"]:
print res["Message"]
sys.stderr.write(
time.strftime("%d/%m/%y %H:%M", time.localtime())
+ " => "
+ j
+ " => "
+ res["Message"]
+ "\n"
)
elif not os.path.isfile(dir_temp + "/" + j + "/jobmeta.inf"):
print "ERROR : no jobmeta.inf file in the outpusandbox"
sys.stderr.write(
time.strftime("%d/%m/%y %H:%M", time.localtime())
+ " => "
dirac = Dirac()
exitCode = 0
errorList = []
resultDict = {}
result = dirac.selectJobs( status = status, minorStatus = minorStatus, applicationStatus = appStatus,
site = site, owner = owner, jobGroup = jobGroup, date = date )
if result['OK']:
jobs = result['Value']
else:
print("Error in selectJob", result['Message'])
DIRAC.exit( 2 )
for job in jobs:
result = dirac.getOutputSandbox( job )
if result['OK']:
if os.path.exists( '%s' % job ):
lines = []
try:
lines = open( os.path.join( job, filename ) ).readlines()
except Exception as x:
errorList.append( ( job, x ) )
for line in lines:
if line.count( searchstring ):
resultDict[job] = line
rmtree( "%s" % ( job ) )
else:
errorList.append( ( job, result['Message'] ) )
exitCode = 2
jobName = sys.argv[1]
finalStatus = ['Done', 'Failed']
dirac = Dirac()
idstr = open("%s/jobIdList.txt" % jobName, 'r').readlines()
ids = map(int, idstr)
print("found %s jobs" % (len(ids)))
res = dirac.getJobSummary(ids)
if not res['OK']:
print(res['Message'])
sys.exit(1)
metadata = res['Value']
for jid in ids:
jobMeta = metadata.get(jid, None)
if not jobMeta:
print("No metadata for job ", jid)
continue
status = jobMeta['Status']
print("%s %s" % (jid, status))
if status in finalStatus:
outputDir = '%s/%s' % (jobName, status)
if not os.path.exists("%s/%s" % (outputDir, jid)):
print("Retrieving sandbox")
res = dirac.getOutputSandbox(jid, outputDir=outputDir)
class CEBaseTest( TestBase ):
"""
CEBaseTest is base class for all the CE test classes. Real CE test should
implement its _judge method.
"""
def __init__( self, args = None, apis = None ):
super( CEBaseTest, self ).__init__( args, apis )
self.timeout = self.args.get( 'timeout', 1800 )
self.vo = self.args.get( 'VO' )
self.testType = self.args[ 'TestType' ]
self.executable = self.args[ 'executable' ]
self.__logPath = '/opt/dirac/pro/BESDIRAC/ResourceStatusSystem/SAM/log'
self.__scriptPath = '/opt/dirac/pro/BESDIRAC/ResourceStatusSystem/SAM/sam_script'
if 'WMSAdministrator' in self.apis:
self.wmsAdmin = self.apis[ 'WMSAdministrator' ]
else:
self.wmsAdmin = RPCClient( 'WorkloadManagement/WMSAdministrator' )
if 'Dirac' in self.apis:
self.dirac = self.apis[ 'Dirac' ]
else:
self.dirac = Dirac()
def doTest( self, elementDict ):
"""
submit test job to the specified ce or cloud..
"""
elementName = elementDict[ 'ElementName' ]
elementType = elementDict[ 'ElementType' ]
vos = elementDict[ 'VO' ]
site = None; ce = None
if elementType == 'ComputingElement':
ce = elementName
if elementType == 'CLOUD':
site = elementName
if self.vo:
submitVO = self.vo
elif vos:
submitVO = vos[ 0 ]
else:
submitVO = 'bes'
submissionTime = datetime.utcnow().replace( microsecond = 0 )
sendRes = self.__submit( site, ce, submitVO )
if not sendRes[ 'OK' ]:
return sendRes
jobID = sendRes[ 'Value' ]
result = { 'Result' : { 'JobID' : jobID,
'VO' : submitVO,
'SubmissionTime' : submissionTime },
'Finish' : False }
return S_OK( result )
def __submit( self, site, CE, vo ):
"""
set the job and submit.
"""
job = Job()
job.setName( self.testType )
job.setJobGroup( 'CE-Test' )
job.setExecutable( self.executable )
job.setInputSandbox( '%s/%s' % ( self.__scriptPath, self.executable ) )
if site and not CE:
job.setDestination( site )
if CE:
job.setDestinationCE( CE )
LOCK.acquire()
proxyPath = BESUtils.getProxyByVO( 'zhangxm', vo )
if not proxyPath[ 'OK' ]:
LOCK.release()
return proxyPath
proxyPath = proxyPath[ 'Value' ]
oldProxy = os.environ.get( 'X509_USER_PROXY' )
os.environ[ 'X509_USER_PROXY' ] = proxyPath
result = self.dirac.submit( job )
if oldProxy is None:
del os.environ[ 'X509_USER_PROXY' ]
else:
os.environ[ 'X509_USER_PROXY' ] = oldProxy
LOCK.release()
return result
def getTestResult( self, elementName, vo, jobID, submissionTime ):
"""
download output sandbox and judge the test status from the log file.
"""
isFinish = False
res = self.__getJobOutput( jobID, vo )
if not res[ 'OK' ]:
return res
output = res[ 'Value' ]
status = res[ 'Status' ]
resDict = { 'CompletionTime' : None, 'Status' : None, 'Log' : None, 'ApplicationTime' : None }
utcNow = datetime.utcnow().replace( microsecond = 0 )
if output:
isFinish = True
resDict[ 'CompletionTime' ] = utcNow
log = output[ 'Log' ]
if not output[ 'Download' ]:
resDict[ 'Status' ] = 'Unknown'
resDict[ 'Log' ] = 'Fail to download log file for job %s: %s' % ( jobID, log )
else:
resDict[ 'Log' ] = log
resDict[ 'Status' ] = self._judge( log )
resDict[ 'AppliactionTime' ] = self.__getAppRunningTime( log )
else:
if utcNow - submissionTime >= timedelta( seconds = self.timeout ):
isFinish = True
if elementName.split( '.' )[ 0 ] == 'CLOUD':
site = elementName
else:
site = BESUtils.getSiteForCE( elementName )
jobCount = self.wmsAdmin.getSiteSummaryWeb( { 'Site' : site }, [], 0, 0 )
if not jobCount[ 'OK' ]:
return jobCount
params = jobCount[ 'Value' ][ 'ParameterNames' ]
records = jobCount[ 'Value' ][ 'Records' ][ 0 ]
run = records[ params.index( 'Running' ) ]
done = records[ params.index( 'Done' ) ]
if status == 'Waiting' and run == 0 and done == 0:
resDict[ 'Status' ] = 'Bad'
resDict[ 'Log' ] = 'The test job is waiting for %d seconds, but no running and done jobs at this site.' % self.timeout
else:
if run != 0:
resDict[ 'Status' ] = 'Busy'
resDict[ 'Log' ] = 'Site %s is too busy to execute this test job, job status is %s' % ( site, status )
else:
resDict[ 'Status' ] = 'Unknown'
resDict[ 'Log' ] = 'Test did not complete within the timeout of %d seconds, job status is %s' % ( self.timeout, status )
self.dirac.kill( jobID )
if not isFinish:
return S_OK()
else:
return S_OK( resDict )
def __getJobOutput( self, jobID, vo ):
status = self.dirac.status( jobID )
if not status[ 'OK' ]:
return status
status = status[ 'Value' ][ jobID ][ 'Status' ]
if status in ( 'Done', 'Failed' ):
LOCK.acquire()
proxyPath = BESUtils.getProxyByVO( 'zhangxm', vo )
if not proxyPath[ 'OK' ]:
LOCK.release()
return proxyPath
proxyPath = proxyPath[ 'Value' ]
oldProxy = os.environ.get( 'X509_USER_PROXY' )
os.environ[ 'X509_USER_PROXY' ] = proxyPath
outputRes = self.dirac.getOutputSandbox( jobID, self.__logPath )
if oldProxy is None:
del os.environ[ 'X509_USER_PROXY' ]
else:
os.environ[ 'X509_USER_PROXY' ] = oldProxy
LOCK.release()
if not outputRes[ 'OK' ]:
ret = S_OK( { 'Download' : False, 'Log' : outputRes[ 'Message' ] } )
else:
try:
logfile = open( '%s/%d/Script1_CodeOutput.log' % ( self.__logPath, jobID ), 'r' )
log = logfile.read()
logfile.close()
except IOError, e:
raise IOError
os.system( 'rm -rf %s/%d' % ( self.__logPath, jobID ) )
ret = S_OK( { 'Download' : True, 'Log' : log } )
else:
def main():
Script.registerSwitch("D:", "Dir=", "Store the output in this directory")
Script.registerSwitch("f:", "File=", "Get output for jobs with IDs from the file")
Script.registerSwitch("g:", "JobGroup=", "Get output for jobs in the given group")
# Registering arguments will automatically add their description to the help menu
Script.registerArgument(["JobID: DIRAC Job ID or a name of the file with JobID per line"], mandatory=False)
sws, args = Script.parseCommandLine(ignoreErrors=True)
from DIRAC.Interfaces.API.Dirac import Dirac, parseArguments
from DIRAC.Core.Utilities.Time import toString, date, day
from DIRAC.Core.Utilities.File import mkDir
dirac = Dirac()
exitCode = 0
errorList = []
outputDir = None
group = None
jobs = []
for sw, value in sws:
if sw in ("D", "Dir"):
outputDir = value
elif sw.lower() in ("f", "file"):
if os.path.exists(value):
jFile = open(value)
jobs += jFile.read().split()
jFile.close()
elif sw.lower() in ("g", "jobgroup"):
group = value
jobDate = toString(date() - 30 * day)
# Choose jobs in final state, no more than 30 days old
result = dirac.selectJobs(jobGroup=value, date=jobDate, status="Done")
if not result["OK"]:
if "No jobs selected" not in result["Message"]:
print("Error:", result["Message"])
DIRAC.exit(-1)
else:
jobs += result["Value"]
result = dirac.selectJobs(jobGroup=value, date=jobDate, status="Failed")
if not result["OK"]:
if "No jobs selected" not in result["Message"]:
print("Error:", result["Message"])
DIRAC.exit(-1)
else:
jobs += result["Value"]
for arg in parseArguments(args):
if os.path.isdir(arg):
print("Output for job %s already retrieved, remove the output directory to redownload" % arg)
else:
jobs.append(arg)
if not jobs:
print("No jobs selected")
DIRAC.exit(0)
if group:
if outputDir:
outputDir = os.path.join(outputDir, group)
else:
outputDir = group
if outputDir:
mkDir(outputDir)
else:
outputDir = os.getcwd()
jobs = [str(job) for job in jobs]
doneJobs = os.listdir(outputDir)
todoJobs = [job for job in jobs if job not in doneJobs]
for job in todoJobs:
result = dirac.getOutputSandbox(job, outputDir=outputDir)
jobDir = str(job)
if outputDir:
jobDir = os.path.join(outputDir, job)
if result["OK"]:
if os.path.exists(jobDir):
print("Job output sandbox retrieved in %s/" % (jobDir))
else:
if os.path.exists("%s" % jobDir):
shutil.rmtree(jobDir)
errorList.append((job, result["Message"]))
exitCode = 2
for error in errorList:
print("ERROR %s: %s" % error)
DIRAC.exit(exitCode)
dirac = Dirac()
exitCode = 0
errorList = []
resultDict = {}
result = dirac.selectJobs( status = status, minorStatus = minorStatus, applicationStatus = appStatus,
site = site, owner = owner, jobGroup = jobGroup, date = date )
if result['OK']:
jobs = result['Value']
else:
print "Error in selectJob", result['Message']
DIRAC.exit( 2 )
for job in jobs:
result = dirac.getOutputSandbox( job )
if result['OK']:
if os.path.exists( '%s' % job ):
lines = []
try:
lines = open( os.path.join( job, filename ) ).readlines()
except Exception, x:
errorList.append( ( job, x ) )
for line in lines:
if line.count( searchstring ):
resultDict[job] = line
rmtree( "%s" % ( job ) )
else:
errorList.append( ( job, result['Message'] ) )
exitCode = 2
from DIRAC.Interfaces.API.Job import Job
from DIRAC.Interfaces.API.Dirac import Dirac
if (len(sys.argv) < 2) :
print 'the input file with ids should be specified'
print 'if the 3rd argument is <get_output> the job output sandbox will be downloaded'
sys.exit(os.EX_USAGE)
list = sys.argv[1]
get_output = False
if (len(sys.argv) > 2):
if (sys.argv[2] == 'get_output'): get_output = True
id_list_file = open(list, 'r')
for line in id_list_file:
## line = line.strip().decode("utf-8").replace("True","true").replace("False","false")
line = line.replace("True","true").replace("False","false")
line = line.replace("'","\"")
j = json.loads(line)
dirac = Dirac()
print dirac.status(j['Value'])
if get_output: print dirac.getOutputSandbox(j['Value'])
id_list_file.close()
def main():
Script.registerSwitch("D:", "Dir=", "Store the output in this directory")
Script.registerSwitch("f:", "File=", "Get output for jobs with IDs from the file")
Script.registerSwitch("g:", "JobGroup=", "Get output for jobs in the given group")
Script.parseCommandLine(ignoreErrors=True)
args = Script.getPositionalArgs()
from DIRAC.Interfaces.API.Dirac import Dirac, parseArguments
from DIRAC.Core.Utilities.Time import toString, date, day
from DIRAC.Core.Utilities.File import mkDir
dirac = Dirac()
exitCode = 0
errorList = []
outputDir = None
group = None
jobs = []
for sw, value in Script.getUnprocessedSwitches():
if sw in ('D', 'Dir'):
outputDir = value
elif sw.lower() in ('f', 'file'):
if os.path.exists(value):
jFile = open(value)
jobs += jFile.read().split()
jFile.close()
elif sw.lower() in ('g', 'jobgroup'):
group = value
jobDate = toString(date() - 30 * day)
# Choose jobs in final state, no more than 30 days old
result = dirac.selectJobs(jobGroup=value, date=jobDate, status='Done')
if not result['OK']:
if "No jobs selected" not in result['Message']:
print("Error:", result['Message'])
DIRAC.exit(-1)
else:
jobs += result['Value']
result = dirac.selectJobs(jobGroup=value, date=jobDate, status='Failed')
if not result['OK']:
if "No jobs selected" not in result['Message']:
print("Error:", result['Message'])
DIRAC.exit(-1)
else:
jobs += result['Value']
for arg in parseArguments(args):
if os.path.isdir(arg):
print("Output for job %s already retrieved, remove the output directory to redownload" % arg)
else:
jobs.append(arg)
if not jobs:
print("No jobs selected")
DIRAC.exit(0)
if group:
if outputDir:
outputDir = os.path.join(outputDir, group)
else:
outputDir = group
if outputDir:
mkDir(outputDir)
else:
outputDir = os.getcwd()
jobs = [str(job) for job in jobs]
doneJobs = os.listdir(outputDir)
todoJobs = [job for job in jobs if job not in doneJobs]
for job in todoJobs:
result = dirac.getOutputSandbox(job, outputDir=outputDir)
jobDir = str(job)
if outputDir:
jobDir = os.path.join(outputDir, job)
if result['OK']:
if os.path.exists(jobDir):
print('Job output sandbox retrieved in %s/' % (jobDir))
else:
if os.path.exists('%s' % jobDir):
shutil.rmtree(jobDir)
errorList.append((job, result['Message']))
exitCode = 2
for error in errorList:
print("ERROR %s: %s" % error)
DIRAC.exit(exitCode)
outputDir = os.path.join(outputDir, group)
else:
outputDir = group
if outputDir:
mkDir(outputDir)
else:
outputDir = os.getcwd()
jobs = [str(job) for job in jobs]
doneJobs = os.listdir(outputDir)
todoJobs = [job for job in jobs if not job in doneJobs]
for job in todoJobs:
result = dirac.getOutputSandbox(job, outputDir=outputDir)
jobDir = str(job)
if outputDir:
jobDir = os.path.join(outputDir, job)
if result['OK']:
if os.path.exists(jobDir):
print 'Job output sandbox retrieved in %s/' % (jobDir)
else:
if os.path.exists('%s' % jobDir):
shutil.rmtree(jobDir)
errorList.append((job, result['Message']))
exitCode = 2
for error in errorList:
print "ERROR %s: %s" % error
class CEBaseTest(TestBase):
"""
CEBaseTest is base class for all the CE test classes. Real CE test should
implement its _judge method.
"""
def __init__(self, args=None, apis=None):
super(CEBaseTest, self).__init__(args, apis)
self.timeout = self.args.get('timeout', 1800)
self.vo = self.args.get('VO')
self.testType = self.args['TestType']
self.executable = self.args['executable']
self.__logPath = '/opt/dirac/work/ResourceStatus/SAMTestAgent/SAM/log'
self.__scriptPath = '/opt/dirac/pro/IHEPDIRAC/ResourceStatusSystem/SAM/sam_script'
if 'WMSAdministrator' in self.apis:
self.wmsAdmin = self.apis['WMSAdministrator']
else:
self.wmsAdmin = RPCClient('WorkloadManagement/WMSAdministrator')
if 'Dirac' in self.apis:
self.dirac = self.apis['Dirac']
else:
self.dirac = Dirac()
def doTest(self, elementDict):
"""
submit test job to the specified ce or cloud..
"""
elementName = elementDict['ElementName']
elementType = elementDict['ElementType']
vos = elementDict['VO']
site = None
ce = None
if elementType == 'ComputingElement':
ce = elementName
if elementType == 'CLOUD':
site = elementName
if self.vo:
submitVO = self.vo
elif vos:
submitVO = vos[0]
else:
submitVO = 'bes'
submissionTime = datetime.utcnow().replace(microsecond=0)
sendRes = self.__submit(site, ce, submitVO)
if not sendRes['OK']:
return sendRes
jobID = sendRes['Value']
result = {
'Result': {
'JobID': jobID,
'VO': submitVO,
'SubmissionTime': submissionTime
},
'Finish': False
}
return S_OK(result)
def __submit(self, site, CE, vo):
"""
set the job and submit.
"""
job = Job()
job.setName(self.testType)
job.setJobGroup('CE-Test')
job.setExecutable(self.executable)
job.setInputSandbox('%s/%s' % (self.__scriptPath, self.executable))
if site and not CE:
job.setDestination(site)
if CE:
job.setDestinationCE(CE)
LOCK.acquire()
proxyPath = BESUtils.getProxyByVO('zhangxm', vo)
if not proxyPath['OK']:
LOCK.release()
return proxyPath
proxyPath = proxyPath['Value']
oldProxy = os.environ.get('X509_USER_PROXY')
os.environ['X509_USER_PROXY'] = proxyPath
result = self.dirac.submitJob(job)
if oldProxy is None:
del os.environ['X509_USER_PROXY']
else:
os.environ['X509_USER_PROXY'] = oldProxy
LOCK.release()
return result
def getTestResult(self, elementName, vo, jobID, submissionTime):
"""
download output sandbox and judge the test status from the log file.
"""
isFinish = False
res = self.__getJobOutput(jobID, vo)
if not res['OK']:
return res
output = res['Value']
status = res['Status']
resDict = {
'CompletionTime': None,
'Status': None,
'Log': None,
'ApplicationTime': None
}
utcNow = datetime.utcnow().replace(microsecond=0)
if output:
isFinish = True
resDict['CompletionTime'] = utcNow
log = output['Log']
if not output['Download']:
resDict['Status'] = 'Unknown'
resDict['Log'] = 'Fail to download log file for job %s: %s' % (
jobID, log)
else:
resDict['Log'] = log
resDict['Status'] = self._judge(log)
resDict['AppliactionTime'] = self.__getAppRunningTime(log)
else:
if utcNow - submissionTime >= timedelta(seconds=self.timeout):
isFinish = True
if elementName.split('.')[0] == 'CLOUD':
site = elementName
else:
site = BESUtils.getSiteForCE(elementName)
jobCount = self.wmsAdmin.getSiteSummaryWeb({'Site': site}, [],
0, 0)
if not jobCount['OK']:
return jobCount
params = jobCount['Value']['ParameterNames']
records = jobCount['Value']['Records'][0]
run = records[params.index('Running')]
done = records[params.index('Done')]
if status == 'Waiting' and run == 0 and done == 0:
resDict['Status'] = 'Bad'
resDict[
'Log'] = 'The test job is waiting for %d seconds, but no running and done jobs at this site.' % self.timeout
else:
if run != 0:
resDict['Status'] = 'Busy'
resDict[
'Log'] = 'Site %s is too busy to execute this test job, job status is %s' % (
site, status)
else:
resDict['Status'] = 'Unknown'
resDict[
'Log'] = 'Test did not complete within the timeout of %d seconds, job status is %s' % (
self.timeout, status)
self.dirac.killJob(jobID)
if not isFinish:
return S_OK()
else:
return S_OK(resDict)
def __getJobOutput(self, jobID, vo):
status = self.dirac.getJobStatus(jobID)
if not status['OK']:
return status
status = status['Value'][jobID]['Status']
if status in ('Done', 'Failed'):
LOCK.acquire()
proxyPath = BESUtils.getProxyByVO('zhangxm', vo)
if not proxyPath['OK']:
LOCK.release()
return proxyPath
proxyPath = proxyPath['Value']
oldProxy = os.environ.get('X509_USER_PROXY')
os.environ['X509_USER_PROXY'] = proxyPath
outputRes = self.dirac.getOutputSandbox(jobID, self.__logPath)
if oldProxy is None:
del os.environ['X509_USER_PROXY']
else:
os.environ['X509_USER_PROXY'] = oldProxy
LOCK.release()
if not outputRes['OK']:
ret = S_OK({'Download': False, 'Log': outputRes['Message']})
else:
try:
logfile = open(
'%s/%d/Script1_CodeOutput.log' %
(self.__logPath, jobID), 'r')
log = logfile.read()
logfile.close()
except IOError, e:
raise IOError
os.system('rm -rf %s/%d' % (self.__logPath, jobID))
ret = S_OK({'Download': True, 'Log': log})
else:
"/jobmeta.inf"):
print "WARNING : not a pipeline task"
# notify the job as "already handled"
jobid_handled.append(j)
else:
# Get the working dir of the task from 'jobmeta.inf'
file_jobmeta = open(
dir_temp + "/InputSandbox" + j + "/jobmeta.inf",
"r")
workdir = file_jobmeta.readline().splitlines()[0]
# TEST workdir = dir_temp+"/"+file_jobmeta.readline().splitlines()[0]
file_jobmeta.close()
# retrieve the OUTPUT sandbox
res = d.getOutputSandbox(j, dir_temp)
if not res['OK']:
print res['Message']
sys.stderr.write(
time.strftime('%d/%m/%y %H:%M',
time.localtime()) + " => " + j +
" => " + res['Message'] + "\n")
elif not os.path.isfile(dir_temp + "/" + j +
"/jobmeta.inf"):
print "ERROR : no jobmeta.inf file in the outpusandbox"
sys.stderr.write(
time.strftime('%d/%m/%y %H:%M',
time.localtime()) + " => " + j +
" => " +
"ERROR : no jobmeta.inf file in the outpusandbox\n"
)
def main():
Script.registerSwitch("", "Status=", "Primary status")
Script.registerSwitch("", "MinorStatus=", "Secondary status")
Script.registerSwitch("", "ApplicationStatus=", "Application status")
Script.registerSwitch("", "Site=", "Execution site")
Script.registerSwitch("", "Owner=", "Owner (DIRAC nickname)")
Script.registerSwitch("", "JobGroup=",
"Select jobs for specified job group")
Script.registerSwitch(
"", "Date=",
"Date in YYYY-MM-DD format, if not specified default is today")
Script.registerSwitch("", "File=",
"File name,if not specified default is std.out ")
Script.parseCommandLine(ignoreErrors=True)
args = Script.getPositionalArgs()
# Default values
status = None
minorStatus = None
appStatus = None
site = None
owner = None
jobGroup = None
date = None
filename = 'std.out'
if len(args) != 1:
Script.showHelp()
searchstring = str(args[0])
for switch in Script.getUnprocessedSwitches():
if switch[0].lower() == "status":
status = switch[1]
elif switch[0].lower() == "minorstatus":
minorStatus = switch[1]
elif switch[0].lower() == "applicationstatus":
appStatus = switch[1]
elif switch[0].lower() == "site":
site = switch[1]
elif switch[0].lower() == "owner":
owner = switch[1]
elif switch[0].lower() == "jobgroup":
jobGroup = switch[1]
elif switch[0].lower() == "date":
date = switch[1]
elif switch[0].lower() == "file":
filename = switch[1]
selDate = date
if not date:
selDate = 'Today'
from DIRAC.Interfaces.API.Dirac import Dirac
dirac = Dirac()
exitCode = 0
errorList = []
resultDict = {}
result = dirac.selectJobs(status=status,
minorStatus=minorStatus,
applicationStatus=appStatus,
site=site,
owner=owner,
jobGroup=jobGroup,
date=date)
if result['OK']:
jobs = result['Value']
else:
print("Error in selectJob", result['Message'])
DIRAC.exit(2)
for job in jobs:
result = dirac.getOutputSandbox(job)
if result['OK']:
if os.path.exists('%s' % job):
lines = []
try:
lines = open(os.path.join(job, filename)).readlines()
except Exception as x:
errorList.append((job, x))
for line in lines:
if line.count(searchstring):
resultDict[job] = line
rmtree("%s" % (job))
else:
errorList.append((job, result['Message']))
exitCode = 2
for result in resultDict.items():
print(result)
DIRAC.exit(exitCode)
#!/bin/env python import sys from DIRAC.Core.Base import Script Script.parseCommandLine() from DIRAC.Interfaces.API.Job import Job from DIRAC.Interfaces.API.Dirac import Dirac dirac = Dirac() jobid = sys.argv[1] print dirac.getOutputSandbox(jobid)