def submitProbeJobs(self, ce):
""" Submit some jobs to the CEs
"""
#need credentials, should be there since the initialize
from DIRAC.Interfaces.API.Dirac import Dirac
d = Dirac()
from DIRAC.Interfaces.API.Job import Job
from DIRAC.ConfigurationSystem.Client.Helpers.Operations import Operations
import DIRAC
ops = Operations()
scriptname = ops.getValue("ResourceStatus/SofwareManagementScript", self.script)
j = Job()
j.setDestinationCE(ce)
j.setCPUTime(1000)
j.setName("Probe %s" % ce)
j.setJobGroup("SoftwareProbe")
j.setExecutable("%s/GlastDIRAC/ResourceStatusSystem/Client/%s" % (DIRAC.rootPath, scriptname),
logFile='SoftwareProbe.log')
j.setOutputSandbox('*.log')
res = d.submit(j)
if not res['OK']:
return res
return S_OK()
class DiracTestCases(unittest.TestCase):
""" Dirac API test cases
"""
def setUp(self):
self.dirac = Dirac()
def tearDown(self):
pass
def test_basicJob(self):
jdl = "Parameter=Value;Parameter2=Value2"
ret = self.dirac._Dirac__getJDLParameters(jdl)
self.assertTrue(ret['OK'])
self.assertIn('Parameter', ret['Value'])
self.assertEqual('Value', ret['Value']['Parameter'])
self.assertIn('Parameter2', ret['Value'])
self.assertEqual('Value2', ret['Value']['Parameter2'])
def test_JobJob(self):
from DIRAC.Interfaces.API.Job import Job
job = Job(stdout='printer', stderr='/dev/null')
ret = self.dirac._Dirac__getJDLParameters(job)
self.assertTrue(ret['OK'])
self.assertEqual('printer', ret['Value']['StdOutput'])
self.assertEqual('/dev/null', ret['Value']['StdError'])
def _getOutputs():
repoLocation = ''
clip = _Params()
clip.registerSwitches()
Script.parseCommandLine( ignoreErrors = False )
repoLocation = clip.repo
if not repoLocation:
Script.showHelp()
dexit(1)
from DIRAC import gLogger
from DIRAC.Interfaces.API.Dirac import Dirac
dirac = Dirac(True, repoLocation)
exitCode = 0
res = dirac.monitorRepository(False)
if not res['OK']:
gLogger.error("Failed because %s" % res['Message'])
dexit(1)
res = dirac.retrieveRepositorySandboxes()
if not res['OK']:
gLogger.error("Failed because %s" % res['Message'])
dexit(1)
if clip.outputdata:
res = dirac.retrieveRepositoryData()
if not res['OK']:
gLogger.error("Failed because %s" % res['Message'])
exit(1)
dexit(exitCode)
def do_get(self,args):
""" Download file from grid and store in a local directory
usage:
get <lfn> [<local_directory>]
"""
argss = args.split()
lfn = argss[0]
lfn = self.getPath(lfn)
dir = ''
if len(argss)>1:
dir = argss[1]
dirac = Dirac()
localCWD = ''
if dir:
localCWD = os.getcwd()
os.chdir(dir)
result = dirac.getFile(lfn)
if localCWD:
os.chdir(localCWD)
if not result['OK']:
print 'Error: %s' %(result['Message'])
else:
print "File %s successfully downloaded" % lfn
def do_add(self,args):
""" Upload a new file to a SE and register in the File Catalog
usage:
add <lfn> <pfn> <SE> [<guid>]
"""
# ToDo - adding directories
argss = args.split()
if len(argss) < 3:
print "Error: unsufficient number of arguments"
lfn = argss[0]
lfn = self.getPath(lfn)
pfn = argss[1]
se = argss[2]
guid = None
if len(argss)>3:
guid = argss[3]
dirac = Dirac()
result = dirac.addFile(lfn,pfn,se,guid,printOutput=False)
if not result['OK']:
print 'Error: %s' %(result['Message'])
else:
print "File %s successfully uploaded to the %s SE" % (lfn,se)
def do_replicate(self,args):
""" Replicate a given file to a given SE
usage:
replicate <LFN> <SE> [<SourceSE>]
"""
argss = args.split()
if len(args) < 2:
print "Error: unsufficient number of arguments"
lfn = argss[0]
lfn = self.getPath(lfn)
se = argss[1]
sourceSE = ''
if len(argss)>2:
sourceSE=argss[2]
if len(argss)>3:
localCache=argss[3]
try:
dirac = Dirac()
result = dirac.replicate(lfn,se,sourceSE,printOutput=True)
if not result['OK']:
print 'Error: %s' %(result['Message'])
elif not result['Value']:
print "Replica is already present at the target SE"
else:
print "File %s successfully replicated to the %s SE" % (lfn,se)
except Exception, x:
print "Error: replicate failed with exception: ", x
def runLocal( self, dirac = None ):
""" The dirac (API) object is for local submission.
"""
if dirac is None:
dirac = Dirac()
return dirac.submit( self, mode = 'local' )
def __downloadJobDescriptionXML(jobID, basepath): """ Downloads the jobDescription.xml file into the temporary directory created. """ from DIRAC.Interfaces.API.Dirac import Dirac jdXML = Dirac() jdXML.getInputSandbox(jobID, basepath)
def uploadAndRegisterFiles(self, fileList, SE="IHEPD-USER", guid=None, ePoint=""):
"""upload a set of files to SE and register it in DFC.
user input the directory of localfile.
argument:
ePoint is the energy point,for scan data
we can treat localDir as a kind of datasetName.
"""
result_OK = 1
errorList = []
# fileList = self.getFilenamesByLocaldir(localDir)
for fullpath in fileList:
# get the attributes of the file
fileAttr = self.__getFileAttributes(fullpath)
if len(fileAttr) == 0:
print "failed to get file %s attributes" % fullpath
return S_ERROR("failed to get file attributes")
# create dir and set dirMetadata to associated dir
lastDir = self.registerHierarchicalDir(fileAttr, rootDir="/bes")
dirMeta = self.getDirMetaVal(lastDir)
if not (dirMeta.has_key("jobOptions") or dirMeta.has_key("description")):
lastDirMetaDict = {}
lastDirMetaDict["jobOptions"] = fileAttr["jobOptions"]
lastDirMetaDict["description"] = fileAttr["description"]
try:
self.__registerDirMetadata(lastDir, lastDirMetaDict)
except:
pass
if len(ePoint):
lastDir = lastDir + os.sep + ePoint
lfn = lastDir + os.sep + fileAttr["LFN"]
# upload and register file.
dirac = Dirac()
result = dirac.addFile(lfn, fullpath, SE, guid, printOutput=True)
# register file metadata
if not result["OK"]:
print "ERROR %s" % (result["Message"])
# return S_ERROR(result['Message'])
errorList.append(fullpath)
result_OK = 0
else:
result = self.__registerFileMetadata(lfn, fileAttr)
if not result["OK"]:
result_OK = 0
print "failed to register file metadata"
if result_OK:
return S_OK()
else:
return S_ERROR(errorList)
def _getOutputData():
cliParams = _Params()
cliParams.registerSwitches()
Script.parseCommandLine( ignoreErrors = False )
if not cliParams.repo:
Script.showHelp()
dexit(2)
from DIRAC.Interfaces.API.Dirac import Dirac
dirac = Dirac(True, cliParams.repo)
exitCode = 0
dirac.monitorRepository(False)
dirac.retrieveRepositoryData()
dexit(exitCode)
def do_installonsite(self,argss):
""" Install a release on a grid site :
installonsite tag site
"""
args = argss.split()
if len(args)<2:
print self.do_installonsite.__doc__
return
tag = args[0]
site = args[1]
#print "Check if the software with the tag '"+tag+"' exists on the rsync server..."
#res = self.client.getSitesForTag(tag)
#if not res['OK']:
#print res['Message']
#return
#print "tag found !"
from DIRAC.Interfaces.API.Dirac import Dirac
d = Dirac()
from DIRAC.Interfaces.API.Job import Job
from DIRAC.ConfigurationSystem.Client.Helpers.Operations import Operations
import os
ops = Operations()
scriptname = "InstallSoftware.py"
j = Job()
j.setDestination(site)
j.setCPUTime(1000)
j.setName("Installation "+tag)
j.setExecutable(os.environ['DIRAC']+"/GlastDIRAC/ResourceStatusSystem/Client/"+scriptname , logFile='SoftwareInstallation.log')
j.setOutputSandbox('*.log')
res = d.submit(j)
if not res['OK']:
print "Could not submit the installation at site %s, message %s"%(site,res['Message'])
return
print "Job submitted, id = "+str(res['Value'])
print "Add tag :"
res = self.client.addTagAtSite(tag,site)
if not res['OK']:
print "Could not register tag %s at site %s, message %s"%(tag,site,res['Message'])
return
print "Added %s to %i CEs"%(tag,len(res['Value'][tag]))
def submitJob(jobPara):
dirac = Dirac()
j = Job()
j.setName(jobPara['jobName'])
j.setJobGroup(jobPara['jobGroup'])
j.setExecutable(jobPara['jobScript'], logFile = jobPara['jobScriptLog'])
j.setInputSandbox(jobPara['inputSandbox'])
j.setOutputSandbox(jobPara['outputSandbox'])
j.setOutputData(jobPara['outputData'], jobPara['SE'])
j.setDestination(jobPara['sites'])
j.setCPUTime(jobPara['CPUTime'])
result = dirac.submit(j)
if result['OK']:
print 'Job %s submitted successfully. ID = %d' %(jobPara['jobName'],result['Value'])
else:
print 'Job %s submitted failed' %jobPara['jobName']
return result
def main():
""" Main program entry point. """
if len(sys.argv) < 2 or len(sys.argv) > 3:
usage()
uname = sys.argv[1]
site = None
if len(sys.argv) >= 3:
site = sys.argv[2]
print "Fetching job list for user '%s'..." % uname
jlist = find_jobs(uname, site)
jlist.append('1')
print "Found %u jobs, killing..." % len(jlist)
dirac = Dirac()
for loc in xrange(0, len(jlist), BATCH_SIZE):
print "%u/%u complete." % (loc, len(jlist))
dirac.killJob(jlist[loc:loc+BATCH_SIZE])
print "%u/%u complete." % (len(jlist), len(jlist))
print "Exiting."
def submitNewBigJob( self ):
result = jobDB.getJobJDL( str( self.__jobID ) , True )
classAdJob = ClassAd( result['Value'] )
executableFile = ""
if classAdJob.lookupAttribute( 'Executable' ):
executableFile = classAdJob.getAttributeString( 'Executable' )
tempPath = self.__tmpSandBoxDir
dirac = Dirac()
if not os.path.exists( tempPath ):
os.makedirs( tempPath )
settingJobSandBoxDir = dirac.getInputSandbox( self.__jobID, tempPath )
self.log.info( 'Writting temporal SandboxDir in Server', settingJobSandBoxDir )
moveData = self.__tmpSandBoxDir + "/InputSandbox" + str( self.__jobID )
HiveV1Cli = HiveV1Client( self.__User , self.__publicIP )
returned = HiveV1Cli.dataCopy( moveData, self.__tmpSandBoxDir )
self.log.info( 'Copy the job contain to the Hadoop Master with HIVE: ', returned )
jobInfo = jobDB.getJobAttributes( self.__jobID )
if not jobInfo['OK']:
return S_ERROR( jobInfo['Value'] )
proxy = ""
jobInfo = jobInfo['Value']
if gProxyManager.userHasProxy( jobInfo["OwnerDN"], jobInfo["OwnerGroup"] ):
proxy = gProxyManager.downloadProxyToFile( jobInfo["OwnerDN"], jobInfo["OwnerGroup"] )
else:
proxy = self.__requestProxyFromProxyManager( jobInfo["OwnerDN"], jobInfo["OwnerGroup"] )
HiveJob = "InputSandbox" + str( self.__jobID ) + "/" + executableFile
HiveJobOutput = str( self.__jobID ) + "_" + executableFile + "_out"
returned = HiveV1Cli.jobSubmit( tempPath, HiveJob, proxy['chain'], HiveJobOutput )
self.log.info( 'Launch Hadoop-Hive job to the Master: ', returned )
if not returned['OK']:
return S_ERROR( returned['Message'] )
else:
self.log.info( 'Hadoop-Hive Job ID: ', returned['Value'] )
return S_OK( returned['Value'] )
def submit(self, param):
j = Job()
j.setName(param['jobName'])
j.setExecutable(param['jobScript'],logFile = param['jobScriptLog'])
if self.site:
j.setDestination(self.site)
if self.jobGroup:
j.setJobGroup(self.jobGroup)
j.setInputSandbox(param['inputSandbox'])
j.setOutputSandbox(param['outputSandbox'])
j.setOutputData(param['outputData'], outputSE = self.outputSE, outputPath = self.outputPath)
dirac = GridDirac()
result = dirac.submit(j)
status = {}
status['submit'] = result['OK']
if status['submit']:
status['job_id'] = result['Value']
return status
def downloadFilesByFilelist(self, fileList, destDir=""):
"""downLoad a set of files form SE.
use getFilesByFilelist() get a list of lfns and download these files.
fileList get from function getFilesByDatesetName()
Example usage:
>>>badger.downloadFilesByFilelist(fileList)
"""
errorDict = {}
dirac = Dirac()
# fileList = self.getFilesByDatasetName(dataset_name)
for lfn in fileList:
result = dirac.getFile(lfn, destDir, printOutput=False)
if not result["OK"]:
errorDict[lfn] = result["Message"]
if errorDict:
serr = S_ERROR()
serr["errorDict"] = errorDict
return serr
else:
return S_OK("File download successfully.")
def submitNewBigPilot(self):
tempPath = self.__tmpSandBoxDir + str(self.__jobID)
dirac = Dirac()
if not os.path.exists(tempPath):
os.makedirs(tempPath)
settingJobSandBoxDir = dirac.getInputSandbox(self.__jobID, tempPath)
self.log.info("Writting temporal SandboxDir in Server", settingJobSandBoxDir)
jobXMLName = "job:" + str(self.__jobID) + ".xml"
with open(os.path.join(tempPath, jobXMLName), "wb") as temp_file:
temp_file.write(self.jobWrapper())
self.log.info("Writting temporal Hadoop Job.xml")
HadoopV1cli = HadoopV1Client(self.__User, self.__publicIP, self.__Port)
# returned = HadoopV1cli.dataCopy( tempPath, self.__tmpSandBoxDir )
# self.log.info( 'Copy the job contain to the Hadoop Master: ', returned )
jobInfo = jobDB.getJobAttributes(self.__jobID)
if not jobInfo["OK"]:
return S_ERROR(jobInfo["Value"])
proxy = ""
jobInfo = jobInfo["Value"]
if gProxyManager.userHasProxy(jobInfo["OwnerDN"], jobInfo["OwnerGroup"]):
proxy = gProxyManager.downloadProxyToFile(jobInfo["OwnerDN"], jobInfo["OwnerGroup"])
else:
proxy = self.__requestProxyFromProxyManager(jobInfo["OwnerDN"], jobInfo["OwnerGroup"])
returned = HadoopV1cli.submitPilotJob(tempPath, jobXMLName, proxy["chain"])
self.log.info("Launch Hadoop pilot to the Hadoop Master: ", returned)
if not returned["OK"]:
return S_ERROR(returned["Value"])
else:
self.log.info("Hadoop Job ID: ", returned["Value"])
return S_OK(returned["Value"])
def submitNewBigJob( self ):
tempPath = self.__tmpSandBoxDir + str( self.__jobID )
dirac = Dirac()
if not os.path.exists( tempPath ):
os.makedirs( tempPath )
settingJobSandBoxDir = dirac.getInputSandbox( self.__jobID, tempPath )
self.log.info( 'Writting temporal SandboxDir in Server', settingJobSandBoxDir )
jobXMLName = "job:" + str( self.__jobID ) + '.xml'
with open( os.path.join( tempPath, jobXMLName ), 'wb' ) as temp_file:
temp_file.write( self.jobWrapper() )
self.log.info( 'Writting temporal Hadoop Job.xml' )
HadoopV1cli = HadoopV2Client( self.__User , self.__publicIP )
returned = HadoopV1cli.dataCopy( tempPath, self.__tmpSandBoxDir )
self.log.info( 'Copy the job contain to the Hadoop Master: ', returned )
jobInfo = jobDB.getJobAttributes( self.__jobID )
if not jobInfo['OK']:
return S_ERROR( jobInfo['Value'] )
proxy = ""
jobInfo = jobInfo['Value']
if gProxyManager.userHasProxy( jobInfo["OwnerDN"], jobInfo["OwnerGroup"] ):
proxy = gProxyManager.downloadProxyToFile( jobInfo["OwnerDN"], jobInfo["OwnerGroup"] )
else:
proxy = self.__requestProxyFromProxyManager( jobInfo["OwnerDN"], jobInfo["OwnerGroup"] )
returned = HadoopV1cli.jobSubmit( tempPath, jobXMLName, proxy['chain'] )
self.log.info( 'Launch Hadoop job to the Hadoop Master: ', returned )
if not returned['OK']:
return S_ERROR( returned['Message'] )
else:
self.log.info( 'Hadoop Job ID: ', returned['Value'] )
return S_OK( returned['Value'] )
def __init__( self, args = None, apis = None ):
super( CEBaseTest, self ).__init__( args, apis )
self.timeout = self.args.get( 'timeout', 1800 )
self.vo = self.args.get( 'VO' )
self.testType = self.args[ 'TestType' ]
self.executable = self.args[ 'executable' ]
self.__logPath = '/opt/dirac/pro/BESDIRAC/ResourceStatusSystem/SAM/log'
self.__scriptPath = '/opt/dirac/pro/BESDIRAC/ResourceStatusSystem/SAM/sam_script'
if 'WMSAdministrator' in self.apis:
self.wmsAdmin = self.apis[ 'WMSAdministrator' ]
else:
self.wmsAdmin = RPCClient( 'WorkloadManagement/WMSAdministrator' )
if 'Dirac' in self.apis:
self.dirac = self.apis[ 'Dirac' ]
else:
self.dirac = Dirac()
Script.parseCommandLine(ignoreErrors=True)
files = []
for oFile in Script.getPositionalArgs():
files += oFile.split(',')
import DIRAC
from DIRAC.Interfaces.API.Dirac import Dirac
from LHCbDIRAC.Core.Utilities.File import getRootFileGUIDs
from LHCbDIRAC.DataManagementSystem.Client.DMScript import printDMResult
if not files:
Script.showHelp()
DIRAC.exit(0)
existFiles = {}
nonExisting = []
dirac = Dirac()
for localFile in files:
if os.path.exists(localFile):
existFiles[os.path.realpath(localFile)] = localFile
elif localFile.startswith('/lhcb'):
res = dirac.getReplicas(localFile, active=True, preferDisk=True)
if res['OK'] and localFile in res['Value']['Successful']:
ses = res['Value']['Successful'][localFile].keys()
for se in ses:
res = dirac.getAccessURL(localFile, se, protocol=['root', 'xroot'])
if res['OK'] and localFile in res['Value']['Successful']:
existFiles[res['Value']['Successful'][localFile]] = "%s @ %s" % (localFile, se)
else:
nonExisting.append(localFile)
elif localFile.startswith('root:'):
from DIRAC.Core.Base import Script
Script.setUsageMessage('\n'.join([
__doc__.split('\n')[1], 'Usage:',
' %s [option|cfgfile] ... JobID ...' % Script.scriptName, 'Arguments:',
' JobID: DIRAC Job ID'
]))
Script.registerSwitch("D:", "Dir=", "Store the output in this directory")
Script.parseCommandLine(ignoreErrors=True)
args = Script.getPositionalArgs()
if len(args) < 1:
Script.showHelp()
from DIRAC.Interfaces.API.Dirac import Dirac, parseArguments
dirac = Dirac()
exitCode = 0
errorList = []
outputDir = ''
for sw, v in Script.getUnprocessedSwitches():
if sw in ('D', 'Dir'):
outputDir = v
for job in parseArguments(args):
result = dirac.getJobOutputData(job, destinationDir=outputDir)
if result['OK']:
print 'Job %s output data retrieved' % (job)
else:
errorList.append((job, result['Message']))
__doc__.split('\n')[1], 'Usage:',
' %s [option|cfgfile] ... PFN SE' % Script.scriptName, 'Arguments:',
' PFN: Physical File Name or file containing PFNs (mandatory)',
' SE: Valid DIRAC SE (mandatory)'
]))
Script.parseCommandLine(ignoreErrors=True)
args = Script.getPositionalArgs()
if len(args) < 2:
Script.showHelp(exitCode=1)
if len(args) > 2:
print('Only one PFN SE pair will be considered')
from DIRAC.Interfaces.API.Dirac import Dirac
dirac = Dirac()
exitCode = 0
pfn = args[0]
seName = args[1]
try:
f = open(pfn, 'r')
pfns = f.read().splitlines()
f.close()
except BaseException:
pfns = [pfn]
for pfn in pfns:
result = dirac.getPhysicalFileMetadata(pfn, seName, printOutput=True)
if not result['OK']:
print('ERROR: ', result['Message'])
import os
Script.setUsageMessage('\n'.join([
__doc__.split('\n')[1], 'Usage:',
' %s [option|cfgfile] ... JobID ...' % Script.scriptName, 'Arguments:',
' JobID: DIRAC Job ID'
]))
Script.registerSwitch("D:", "Dir=", "Store the output in this directory")
Script.parseCommandLine(ignoreErrors=True)
args = Script.getPositionalArgs()
if len(args) < 1:
Script.showHelp()
from DIRAC.Interfaces.API.Dirac import Dirac, parseArguments
dirac = Dirac()
exitCode = 0
errorList = []
outputDir = None
for sw, v in Script.getUnprocessedSwitches():
if sw in ('D', 'Dir'):
outputDir = v
for job in parseArguments(args):
result = dirac.getInputSandbox(job, outputDir=outputDir)
if result['OK']:
if os.path.exists('InputSandbox%s' % job):
print 'Job input sandbox retrieved in InputSandbox%s/' % (job)
else:
Usage:
%s [option] ... JobID ...
""" % Script.scriptName)
fcType = 'FileCatalog'
Script.registerSwitch("f:", "file-catalog=",
"Catalog client type to use (default %s)" % fcType)
Script.registerSwitch("l:", "lfn=", "File lfn")
Script.parseCommandLine(ignoreErrors=False)
options = Script.getUnprocessedSwitches()
args = Script.getPositionalArgs()
from DIRAC.Interfaces.API.Dirac import Dirac
dirac = Dirac()
from DIRAC.Resources.Catalog.FileCatalogClient import FileCatalogClient
fccType = 'DataManagement/FileCatalog'
fcc = FileCatalogClient(fccType)
import os
import sys
import tarfile
def findFiles(jobIds):
'''Find specified log files from the job ids'''
lfns = []
for jobId in jobIds:
Script.parseCommandLine(ignoreErrors=True)
args = Script.getPositionalArgs()
for switch in Script.getUnprocessedSwitches():
if switch[0] == 'Original' or switch[0] == 'O':
original = True
for switch in Script.getUnprocessedSwitches():
if switch[0] == 'Original':
original = True
if len(args) < 1:
Script.showHelp()
from DIRAC.Interfaces.API.Dirac import Dirac, parseArguments
dirac = Dirac()
exitCode = 0
errorList = []
for job in parseArguments(args):
result = dirac.getJobJDL(job, original=original, printOutput=True)
if not result['OK']:
errorList.append((job, result['Message']))
exitCode = 2
for error in errorList:
print("ERROR %s: %s" % error)
DIRAC.exit(exitCode)
"""
Issue a kill signal to a running DIRAC job
"""
from __future__ import print_function
__RCSID__ = "$Id$"
import DIRAC
from DIRAC.Core.Base import Script
Script.setUsageMessage( '\n'.join( [ __doc__.split( '\n' )[1],
'Usage:',
' %s [option|cfgfile] ... JobID ...' % Script.scriptName,
'Arguments:',
' JobID: DIRAC Job ID' ] ) )
Script.parseCommandLine( ignoreErrors = True )
args = Script.getPositionalArgs()
if len( args ) < 1:
Script.showHelp()
from DIRAC.Interfaces.API.Dirac import Dirac, parseArguments
result = Dirac().killJob( parseArguments( args ) )
if result['OK']:
print('Killed jobs %s' % ','.join([str(j) for j in result['Value']]))
exitCode = 0
else:
print('ERROR', result['Message'])
exitCode = 2
DIRAC.exit( exitCode )
except:
pass
elif switch[0] == 'Job':
try:
job = int(switch[1])
except:
print "Invalid jobID", switch[1]
if not job:
args = Script.getPositionalArgs()
if len(args) == 1:
requestName = args[0]
else:
from DIRAC.Interfaces.API.Dirac import Dirac
dirac = Dirac()
res = dirac.attributes(job)
if not res['OK']:
print "Error getting job parameters", res['Message']
else:
jobName = res['Value'].get('JobName')
if not jobName:
print 'Job %d not found' % job
else:
requestName = jobname + '_job_%d' % job
requests = []
if requestName:
requests = [requestName]
elif resetFailed:
res = reqClient.getRequestNamesList(['Failed'], maxReset)
'Arguments:', ' LFN: Logical File Name or file containing LFNs',
' SE: Valid DIRAC SE', ' PROTO: Optional protocol for accessURL'
]))
Script.parseCommandLine(ignoreErrors=True)
args = Script.getPositionalArgs()
# pylint: disable=wrong-import-position
from DIRAC.Interfaces.API.Dirac import Dirac
if len(args) < 2:
Script.showHelp()
if len(args) > 3:
print 'Only one LFN SE pair will be considered'
dirac = Dirac()
exitCode = 0
lfn = args[0]
seName = args[1]
proto = False
if len(args) > 2:
proto = args[2]
try:
f = open(lfn, 'r')
lfns = f.read().splitlines()
f.close()
except IOError:
lfns = [lfn]
" SE: Valid DIRAC SE",
]
)
)
Script.parseCommandLine(ignoreErrors=True)
args = Script.getPositionalArgs()
if len(args) < 2:
Script.showHelp()
if len(args) > 2:
print "Only one LFN SE pair will be considered"
from DIRAC.Interfaces.API.Dirac import Dirac
dirac = Dirac()
exitCode = 0
lfn = args[0]
seName = args[1]
try:
f = open(lfn, "r")
lfns = f.read().splitlines()
f.close()
except:
lfns = [lfn]
for lfn in lfns:
result = dirac.getAccessURL(lfn, seName, printOutput=True)
if not result["OK"]:
import sys
from DIRAC.Interfaces.API.Dirac import Dirac
import os
if len(sys.argv)< 2 :
print "Usage %s <jobName>"%sys.argv[0]
sys.exit(1)
jobName = sys.argv[1]
finalStatus = ['Done', 'Failed']
dirac = Dirac()
idstr = open("%s/jobIdList.txt"%jobName, 'r').readlines()
ids = map(int, idstr)
print "found %s jobs"%(len(ids))
res = dirac.getJobSummary(ids)
if not res['OK']:
print res['Message']
sys.exit(1)
metadata = res['Value']
for jid in ids:
jobMeta = metadata.get( jid, None )
if not jobMeta :
import DIRAC
from DIRAC.Core.Base import Script
Script.setUsageMessage( '\n'.join( [ __doc__.split( '\n' )[1],
'Usage:',
' %s [option|cfgfile] ... JobID ...' % Script.scriptName,
'Arguments:',
' JobID: DIRAC Job ID' ] ) )
Script.parseCommandLine( ignoreErrors = True )
args = Script.getPositionalArgs()
if len( args ) < 1:
Script.showHelp()
from DIRAC.Interfaces.API.Dirac import Dirac
dirac = Dirac()
exitCode = 0
errorList = []
for job in args:
result = dirac.attributes( int(job), printOutput = True )
if not result['OK']:
errorList.append( ( job, result['Message'] ) )
exitCode = 2
for error in errorList:
print "ERROR %s: %s" % error
DIRAC.exit( exitCode )
def main():
Script.registerSwitch("", "Status=", "Primary status")
Script.registerSwitch("", "MinorStatus=", "Secondary status")
Script.registerSwitch("", "ApplicationStatus=", "Application status")
Script.registerSwitch("", "Site=", "Execution site")
Script.registerSwitch("", "Owner=", "Owner (DIRAC nickname)")
Script.registerSwitch("", "JobGroup=",
"Select jobs for specified job group")
Script.registerSwitch(
"", "Date=",
"Date in YYYY-MM-DD format, if not specified default is today")
Script.registerSwitch("", "File=",
"File name,if not specified default is std.out ")
# Registering arguments will automatically add their description to the help menu
Script.registerArgument("String: string to search for")
_, args = Script.parseCommandLine(ignoreErrors=True)
# Default values
status = None
minorStatus = None
appStatus = None
site = None
owner = None
jobGroup = None
date = None
filename = "std.out"
if len(args) != 1:
Script.showHelp()
searchstring = str(args[0])
for switch in Script.getUnprocessedSwitches():
if switch[0].lower() == "status":
status = switch[1]
elif switch[0].lower() == "minorstatus":
minorStatus = switch[1]
elif switch[0].lower() == "applicationstatus":
appStatus = switch[1]
elif switch[0].lower() == "site":
site = switch[1]
elif switch[0].lower() == "owner":
owner = switch[1]
elif switch[0].lower() == "jobgroup":
jobGroup = switch[1]
elif switch[0].lower() == "date":
date = switch[1]
elif switch[0].lower() == "file":
filename = switch[1]
selDate = date
if not date:
selDate = "Today"
from DIRAC.Interfaces.API.Dirac import Dirac
dirac = Dirac()
exitCode = 0
errorList = []
resultDict = {}
result = dirac.selectJobs(
status=status,
minorStatus=minorStatus,
applicationStatus=appStatus,
site=site,
owner=owner,
jobGroup=jobGroup,
date=date,
)
if result["OK"]:
jobs = result["Value"]
else:
print("Error in selectJob", result["Message"])
DIRAC.exit(2)
for job in jobs:
result = dirac.getOutputSandbox(job)
if result["OK"]:
if os.path.exists("%s" % job):
lines = []
try:
lines = open(os.path.join(job, filename)).readlines()
except Exception as x:
errorList.append((job, x))
for line in lines:
if line.count(searchstring):
resultDict[job] = line
rmtree("%s" % (job))
else:
errorList.append((job, result["Message"]))
exitCode = 2
for result in resultDict.items():
print(result)
DIRAC.exit(exitCode)
def Flux(args=None):
from DIRAC.Interfaces.API.Dirac import Dirac
from DIRAC.Interfaces.API.Job import Job
import time
import os.path
user_script = './flux.sh'
modmacro = './CTAflux_speeed.C'
site = "PARANAL"
if (len(args) != 5):
Script.showHelp()
typeofdata = "test"
particle = args[0]
direction = args[1]
MOD = args[2]
exten = args[3]
zenName = args[4]
# List of files over which flux should be run
LFN_file = "./stereofiles/lfn_%s_%s_%s_%s.lfns" % (particle, exten,
zenName, direction)
fileLength = sum(1 for line in open(LFN_file))
f = open(LFN_file, 'r')
if particle == "proton":
filesPerJob = 10
else:
filesPerJob = 20
fileList = []
text_file_name = "lfnStereoFiles_%s_%s_%s_%s.txt" % (particle, exten,
typeofdata, direction)
text_file = open(text_file_name, "w")
# File containing the id number of files already produced. The relaunch of these jobs will be skipped
done_file_name = "./stereofiles/done/done_%s_%s_%s_%s.lfns" % (
particle, exten, zenName, direction)
if os.path.exists(done_file_name):
done_content = [
int(line.strip()) for line in open(done_file_name, 'r')
]
else:
done_content = []
loop = 0
iJob = 0
for line in f:
loop = loop + 1
infileLFN = line.strip()
fileList.append(infileLFN)
text_file.write("%s\n" % infileLFN)
remain = loop % filesPerJob
if remain == 0 or loop == fileLength:
iJob = iJob + 1
# Skipping of already finished jobs
if iJob in done_content:
text_file.close()
fileList = []
text_file = open(text_file_name, "w")
continue
else:
j = Job()
text_file.close()
j.setInputSandbox([
user_script, "setupPackageMARS.sh", "CheckFileZombie.C",
text_file_name, modmacro
])
jobName = "%s_%s_%s_%s_%s_%s_%s" % (user_script, site,
particle, direction, iJob,
exten, zenName)
jobOut = "%s_%s_%s_%s_%s.out" % (user_script, site, particle,
direction, iJob)
script_args = "%s %s %s %s %s %s %s" % (
particle, site, iJob, direction, MOD, exten, zenName)
j.setExecutable(user_script, script_args)
j.setOutputSandbox([jobOut, "applicationLog.txt"])
j.setName(jobName)
j.setBannedSites([
'LCG.MSFG.fr', 'LCG.M3PEC.fr', 'LCG.OBSPM.fr',
'LCG.UNI-DORTMUND.de', 'LCG.UNIV-LILLE.fr',
'LCG.Prague.cz', 'LCG.GRIF.fr'
])
Script.gLogger.info(j._toJDL())
print "Submitting job %s %s %s %s %s %s" % (
user_script, zenName, particle, direction, site, iJob)
time.sleep(3)
Dirac().submit(j)
fileList = []
text_file = open(text_file_name, "w")
def main():
from DIRAC.Core.Base import Script
Script.registerSwitch("p:", "run_number=", "Run Number", setRunNumber)
Script.registerSwitch("T:", "template=", "Template", setCorsikaTemplate)
Script.registerSwitch("E:", "executable=", "Executable", setExecutable)
Script.registerSwitch("S:", "simtelConfig=", "SimtelConfig", setConfig)
Script.registerSwitch("V:", "version=", "Version", setVersion)
Script.registerSwitch("M:", "mode=", "Mode", setMode)
Script.registerSwitch("C:", "savecorsika=", "Save Corsika", setSaveCorsika)
from DIRAC.Resources.Catalog.FileCatalogClient import FileCatalogClient
from DIRAC.Resources.Catalog.FileCatalog import FileCatalog
Script.parseCommandLine()
global fcc, fcL, storage_element
from CTADIRAC.Core.Utilities.SoftwareInstallation import getSoftwareEnviron
from CTADIRAC.Core.Utilities.SoftwareInstallation import installSoftwareEnviron
from CTADIRAC.Core.Utilities.SoftwareInstallation import workingArea
from CTADIRAC.Core.Workflow.Modules.CorsikaApp import CorsikaApp
from CTADIRAC.Core.Workflow.Modules.Read_CtaApp import Read_CtaApp
from DIRAC.Core.Utilities.Subprocess import systemCall
jobID = os.environ['JOBID']
jobID = int(jobID)
global jobReport
jobReport = JobReport(jobID)
###########
## Checking MD coherence
fc = FileCatalog('LcgFileCatalog')
res = fc._getCatalogConfigDetails('DIRACFileCatalog')
print 'DFC CatalogConfigDetails:', res
res = fc._getCatalogConfigDetails('LcgFileCatalog')
print 'LCG CatalogConfigDetails:', res
fcc = FileCatalogClient()
fcL = FileCatalog('LcgFileCatalog')
from DIRAC.Interfaces.API.Dirac import Dirac
dirac = Dirac()
#############
simtelConfigFilesPath = 'sim_telarray/multi'
simtelConfigFile = simtelConfigFilesPath + '/multi_cta-ultra5.cfg'
#simtelConfigFile = simtelConfigFilesPath + '/multi_cta-prod1s.cfg'
createGlobalsFromConfigFiles('prodConfigFile', corsikaTemplate, version)
######################Building prod Directory Metadata #######################
resultCreateProdDirMD = createProdFileSystAndMD()
if not resultCreateProdDirMD['OK']:
DIRAC.gLogger.error('Failed to create prod Directory MD')
jobReport.setApplicationStatus('Failed to create prod Directory MD')
DIRAC.gLogger.error('Metadata coherence problem, no file produced')
DIRAC.exit(-1)
else:
print 'prod Directory MD successfully created'
######################Building corsika Directory Metadata #######################
resultCreateCorsikaDirMD = createCorsikaFileSystAndMD()
if not resultCreateCorsikaDirMD['OK']:
DIRAC.gLogger.error('Failed to create corsika Directory MD')
jobReport.setApplicationStatus('Failed to create corsika Directory MD')
DIRAC.gLogger.error(
'Metadata coherence problem, no corsikaFile produced')
DIRAC.exit(-1)
else:
print 'corsika Directory MD successfully created'
############ Producing Corsika File
global CorsikaSimtelPack
CorsikaSimtelPack = os.path.join('corsika_simhessarray', version,
'corsika_simhessarray')
install_CorsikaSimtelPack(version, 'sim')
cs = CorsikaApp()
cs.setSoftwarePackage(CorsikaSimtelPack)
cs.csExe = executable
cs.csArguments = [
'--run-number', run_number, '--run', 'corsika', corsikaTemplate
]
#############################
corsikaReturnCode = cs.execute()
if corsikaReturnCode != 0:
DIRAC.gLogger.error('Corsika Application: Failed')
jobReport.setApplicationStatus('Corsika Application: Failed')
DIRAC.exit(-1)
if (os.system('sleep 60')):
DIRAC.exit(-1)
if (os.system('grep Broken corsika_autoinputs.log') == 0):
DIRAC.gLogger.error('Broken string found in corsika_autoinputs.log')
jobReport.setApplicationStatus('Broken pipe')
DIRAC.exit(-1)
###################### rename of corsika output file #######################
rundir = 'run' + run_number
# filein = rundir + '/' + corsikaOutputFileName
corsikaFileName = particle + '_' + thetaP + '_' + phiP + '_alt' + obslev + '_' + 'run' + run_number + '.corsika.gz'
# cmd = 'mv ' + filein + ' ' + corsikaFileName
# if(os.system(cmd)):
# DIRAC.exit( -1 )
########################
########################
## files spread in 1000-runs subDirectories
runNum = int(run_number)
subRunNumber = '%03d' % runNum
runNumModMille = runNum % 1000
runNumTrunc = (runNum - runNumModMille) / 1000
runNumSeriesDir = '%03dxxx' % runNumTrunc
print 'runNumSeriesDir=', runNumSeriesDir
### create corsika tar luisa ####################
corsikaTarName = particle + '_' + thetaP + '_' + phiP + '_alt' + obslev + '_' + 'run' + run_number + '.corsika.tar.gz'
filetar1 = rundir + '/' + 'input'
filetar2 = rundir + '/' + 'DAT' + run_number + '.dbase'
filetar3 = rundir + '/run' + str(int(run_number)) + '.log'
cmdTuple = [
'/bin/tar', 'zcf', corsikaTarName, filetar1, filetar2, filetar3
]
DIRAC.gLogger.notice('Executing command tuple:', cmdTuple)
ret = systemCall(0, cmdTuple, sendOutput)
if not ret['OK']:
DIRAC.gLogger.error('Failed to execute tar')
DIRAC.exit(-1)
######################################################
corsikaOutFileDir = os.path.join(corsikaDirPath, particle, 'Data',
runNumSeriesDir)
corsikaOutFileLFN = os.path.join(corsikaOutFileDir, corsikaFileName)
corsikaRunNumberSeriesDirExist = fcc.isDirectory(
corsikaOutFileDir)['Value']['Successful'][corsikaOutFileDir]
newCorsikaRunNumberSeriesDir = (
corsikaRunNumberSeriesDirExist != True
) # if new runFileSeries, will need to add new MD
#### create a file to DISABLE_WATCHDOG_CPU_WALLCLOCK_CHECK ################
f = open('DISABLE_WATCHDOG_CPU_WALLCLOCK_CHECK', 'w')
f.close()
if savecorsika == 'True':
DIRAC.gLogger.notice('Put and register corsika File in LFC and DFC:',
corsikaOutFileLFN)
ret = dirac.addFile(corsikaOutFileLFN, corsikaFileName,
storage_element) # obsolete
res = CheckCatalogCoherence(corsikaOutFileLFN)
if res != DIRAC.S_OK:
DIRAC.gLogger.error('Job failed: Catalog Coherence problem found')
jobReport.setApplicationStatus('OutputData Upload Error')
DIRAC.exit(-1)
if not ret['OK']:
DIRAC.gLogger.error('Error during addFile call:', ret['Message'])
jobReport.setApplicationStatus('OutputData Upload Error')
DIRAC.exit(-1)
# put and register corsikaTarFile:
corsikaTarFileDir = os.path.join(corsikaDirPath, particle, 'Log',
runNumSeriesDir)
corsikaTarFileLFN = os.path.join(corsikaTarFileDir, corsikaTarName)
DIRAC.gLogger.notice(
'Put and register corsikaTar File in LFC and DFC:',
corsikaTarFileLFN)
ret = dirac.addFile(corsikaTarFileLFN, corsikaTarName,
storage_element) # obsolete
####Checking and restablishing catalog coherence #####################
res = CheckCatalogCoherence(corsikaTarFileLFN)
if res != DIRAC.S_OK:
DIRAC.gLogger.error('Job failed: Catalog Coherence problem found')
jobReport.setApplicationStatus('OutputData Upload Error')
DIRAC.exit(-1)
if not ret['OK']:
DIRAC.gLogger.error('Error during addFile call:', ret['Message'])
jobReport.setApplicationStatus('OutputData Upload Error')
DIRAC.exit(-1)
######################################################################
if newCorsikaRunNumberSeriesDir:
insertRunFileSeriesMD(corsikaOutFileDir, runNumTrunc)
insertRunFileSeriesMD(corsikaTarFileDir, runNumTrunc)
###### insert corsika File Level metadata ############################################
corsikaFileMD = {}
corsikaFileMD['runNumber'] = int(run_number)
corsikaFileMD['jobID'] = jobID
corsikaFileMD['corsikaReturnCode'] = corsikaReturnCode
corsikaFileMD['nbShowers'] = nbShowers
result = fcc.setMetadata(corsikaOutFileLFN, corsikaFileMD)
print "result setMetadata=", result
if not result['OK']:
print 'ResultSetMetadata:', result['Message']
result = fcc.setMetadata(corsikaTarFileLFN, corsikaFileMD)
print "result setMetadata=", result
if not result['OK']:
print 'ResultSetMetadata:', result['Message']
##### Exit now if only corsika simulation required
if (mode == 'corsika_standalone'):
DIRAC.exit()
############ Producing SimTel File
######################Building simtel Directory Metadata #######################
cfg_dict = {
"4MSST": 'cta-prod2-4m-dc',
"SCSST": 'cta-prod2-sc-sst',
"STD": 'cta-prod2',
"NSBX3": 'cta-prod2',
"ASTRI": 'cta-prod2-astri',
"SCMST": 'cta-prod2-sc3',
"NORTH": 'cta-prod2n'
}
# if simtelConfig=="6INROW":
# all_configs=["4MSST","SCSST","ASTRI","NSBX3","STD","SCMST"]
# elif simtelConfig=="5INROW":
# all_configs=["4MSST","SCSST","ASTRI","NSBX3","STD"]
# elif simtelConfig=="3INROW":
# all_configs=["SCSST","STD","SCMST"]
# else:
# all_configs=[simtelConfig]
########## Added as default ###################
if simtelConfig == "3INROW":
all_configs = ["SCSST", "STD", "SCMST"]
else:
DIRAC.gLogger.error('Not valid simtelConfig: choose 3INROW')
DIRAC.exit(-1)
for current_conf in all_configs:
DIRAC.gLogger.notice('current conf is', current_conf)
if current_conf == "SCMST":
current_version = version + '_sc3'
DIRAC.gLogger.notice('current version is', current_version)
# if os.path.isdir('sim_telarray'):
# DIRAC.gLogger.notice('Package found in the local area. Removing package...')
# cmd = 'rm -R sim_telarray corsika-6990 hessioxxx corsika-run'
# if(os.system(cmd)):
# DIRAC.exit( -1 )
# install_CorsikaSimtelPack(current_version)
installSoftwareEnviron(CorsikaSimtelPack, workingArea(), 'sim-sc3')
else:
current_version = version
DIRAC.gLogger.notice('current version is', current_version)
########################################################
global simtelDirPath
global simtelProdVersion
simtelProdVersion = current_version + '_simtel'
simtelDirPath = os.path.join(corsikaParticleDirPath, simtelProdVersion)
resultCreateSimtelDirMD = createSimtelFileSystAndMD(current_conf)
if not resultCreateSimtelDirMD['OK']:
DIRAC.gLogger.error('Failed to create simtelArray Directory MD')
jobReport.setApplicationStatus(
'Failed to create simtelArray Directory MD')
DIRAC.gLogger.error(
'Metadata coherence problem, no simtelArray File produced')
DIRAC.exit(-1)
else:
DIRAC.gLogger.notice('simtel Directory MD successfully created')
############## check simtel data file LFN exists ########################
simtelFileName = particle + '_' + str(thetaP) + '_' + str(
phiP) + '_alt' + str(
obslev) + '_' + 'run' + run_number + '.simtel.gz'
simtelDirPath_conf = simtelDirPath + '_' + current_conf
simtelOutFileDir = os.path.join(simtelDirPath_conf, 'Data',
runNumSeriesDir)
simtelOutFileLFN = os.path.join(simtelOutFileDir, simtelFileName)
res = CheckCatalogCoherence(simtelOutFileLFN)
if res == DIRAC.S_OK:
DIRAC.gLogger.notice('Current conf already done', current_conf)
continue
#### execute simtelarray ################
####################################
## check simtel data/log/histo Output File exist
cfg = cfg_dict[current_conf]
cmd = 'mv sim/Data/sim_telarray/' + cfg + '/0.0deg/Data/*' + cfg + '_*.simtel.gz ' + simtelFileName
if (os.system(cmd)):
DIRAC.exit(-1)
############################################
simtelRunNumberSeriesDirExist = fcc.isDirectory(
simtelOutFileDir)['Value']['Successful'][simtelOutFileDir]
newSimtelRunFileSeriesDir = (
simtelRunNumberSeriesDirExist != True
) # if new runFileSeries, will need to add new MD
simtelLogFileName = particle + '_' + str(thetaP) + '_' + str(
phiP) + '_alt' + str(obslev) + '_' + 'run' + run_number + '.log.gz'
cmd = 'mv sim/Data/sim_telarray/' + cfg + '/0.0deg/Log/*' + cfg + '_*.log.gz ' + simtelLogFileName
if (os.system(cmd)):
DIRAC.exit(-1)
simtelOutLogFileDir = os.path.join(simtelDirPath_conf, 'Log',
runNumSeriesDir)
simtelOutLogFileLFN = os.path.join(simtelOutLogFileDir,
simtelLogFileName)
simtelHistFileName = particle + '_' + str(thetaP) + '_' + str(
phiP) + '_alt' + str(
obslev) + '_' + 'run' + run_number + '.hdata.gz'
cmd = 'mv sim/Data/sim_telarray/' + cfg + '/0.0deg/Histograms/*' + cfg + '_*.hdata.gz ' + simtelHistFileName
if (os.system(cmd)):
DIRAC.exit(-1)
simtelOutHistFileDir = os.path.join(simtelDirPath_conf, 'Histograms',
runNumSeriesDir)
simtelOutHistFileLFN = os.path.join(simtelOutHistFileDir,
simtelHistFileName)
########### quality check on Histo #############################################
fd = open('check_histo.sh', 'w')
fd.write("""#! /bin/sh
nsim=$(list_histograms %s|fgrep 'Histogram 6 '|sed 's/^.*contents: //'| sed 's:/.*$::')
nevents=%d
if [ $nsim -lt $(( $nevents - 20 )) ]; then
echo 'nsim found:' $nsim
echo 'nsim expected:' $nevents
exit 1
else
echo 'nsim found:' $nsim
echo 'nsim expected:' $nevents
fi
""" % (simtelHistFileName, int(nbShowers) * int(cscat)))
fd.close()
ret = getSoftwareEnviron(CorsikaSimtelPack)
if not ret['OK']:
error = ret['Message']
DIRAC.gLogger.error(error, CorsikaSimtelPack)
DIRAC.exit(-1)
corsikaEnviron = ret['Value']
os.system('chmod u+x check_histo.sh')
cmdTuple = ['./check_histo.sh']
DIRAC.gLogger.notice('Executing command tuple:', cmdTuple)
ret = systemCall(0, cmdTuple, sendOutput, env=corsikaEnviron)
checkHistoReturnCode, stdout, stderr = ret['Value']
if not ret['OK']:
DIRAC.gLogger.error('Failed to execute check_histo.sh')
DIRAC.gLogger.error('check_histo.sh status is:',
checkHistoReturnCode)
DIRAC.exit(-1)
if (checkHistoReturnCode != 0):
DIRAC.gLogger.error('Failure during check_histo.sh')
DIRAC.gLogger.error('check_histo.sh status is:',
checkHistoReturnCode)
jobReport.setApplicationStatus('Histo check Failed')
DIRAC.exit(-1)
########## quality check on Log #############################
cmd = 'zcat %s | grep Finished.' % simtelLogFileName
DIRAC.gLogger.notice('Executing system call:', cmd)
if (os.system(cmd)):
jobReport.setApplicationStatus('Log check Failed')
DIRAC.exit(-1)
################################################
from DIRAC.Core.Utilities import List
from DIRAC.ConfigurationSystem.Client.Helpers.Operations import Operations
opsHelper = Operations()
global seList
seList = opsHelper.getValue('ProductionOutputs/SimtelProd', [])
seList = List.randomize(seList)
DIRAC.gLogger.notice('SeList is:', seList)
######### Upload simtel data/log/histo ##############################################
res = upload_to_seList(simtelOutFileLFN, simtelFileName)
if res != DIRAC.S_OK:
DIRAC.gLogger.error('OutputData Upload Error', simtelOutFileLFN)
jobReport.setApplicationStatus('OutputData Upload Error')
DIRAC.exit(-1)
res = CheckCatalogCoherence(simtelOutLogFileLFN)
if res == DIRAC.S_OK:
DIRAC.gLogger.notice('Log file already exists. Removing:',
simtelOutLogFileLFN)
ret = dirac.removeFile(simtelOutLogFileLFN)
res = upload_to_seList(simtelOutLogFileLFN, simtelLogFileName)
if res != DIRAC.S_OK:
DIRAC.gLogger.error('Upload simtel Log Error', simtelOutLogFileLFN)
DIRAC.gLogger.notice('Removing simtel data file:',
simtelOutFileLFN)
ret = dirac.removeFile(simtelOutFileLFN)
jobReport.setApplicationStatus('OutputData Upload Error')
DIRAC.exit(-1)
res = CheckCatalogCoherence(simtelOutHistFileLFN)
if res == DIRAC.S_OK:
DIRAC.gLogger.notice('Histo file already exists. Removing:',
simtelOutHistFileLFN)
ret = dirac.removeFile(simtelOutHistFileLFN)
res = upload_to_seList(simtelOutHistFileLFN, simtelHistFileName)
if res != DIRAC.S_OK:
DIRAC.gLogger.error('Upload simtel Histo Error',
simtelOutHistFileLFN)
DIRAC.gLogger.notice('Removing simtel data file:',
simtelOutFileLFN)
ret = dirac.removeFile(simtelOutFileLFN)
DIRAC.gLogger.notice('Removing simtel log file:',
simtelOutLogFileLFN)
ret = dirac.removeFile(simtelOutLogFileLFN)
jobReport.setApplicationStatus('OutputData Upload Error')
DIRAC.exit(-1)
# simtelRunNumberSeriesDirExist = fcc.isDirectory(simtelOutFileDir)['Value']['Successful'][simtelOutFileDir]
# newSimtelRunFileSeriesDir = (simtelRunNumberSeriesDirExist != True) # if new runFileSeries, will need to add new MD
if newSimtelRunFileSeriesDir:
print 'insertRunFileSeriesMD'
insertRunFileSeriesMD(simtelOutFileDir, runNumTrunc)
insertRunFileSeriesMD(simtelOutLogFileDir, runNumTrunc)
insertRunFileSeriesMD(simtelOutHistFileDir, runNumTrunc)
else:
print 'NotinsertRunFileSeriesMD'
###### simtel File level metadata ############################################
simtelFileMD = {}
simtelFileMD['runNumber'] = int(run_number)
simtelFileMD['jobID'] = jobID
simtelReturnCode = corsikaReturnCode
simtelFileMD['simtelReturnCode'] = simtelReturnCode
result = fcc.setMetadata(simtelOutFileLFN, simtelFileMD)
print "result setMetadata=", result
if not result['OK']:
print 'ResultSetMetadata:', result['Message']
result = fcc.setMetadata(simtelOutLogFileLFN, simtelFileMD)
print "result setMetadata=", result
if not result['OK']:
print 'ResultSetMetadata:', result['Message']
result = fcc.setMetadata(simtelOutHistFileLFN, simtelFileMD)
print "result setMetadata=", result
if not result['OK']:
print 'ResultSetMetadata:', result['Message']
if savecorsika == 'True':
result = fcc.addFileAncestors(
{simtelOutFileLFN: {
'Ancestors': [corsikaOutFileLFN]
}})
print 'result addFileAncestor:', result
result = fcc.addFileAncestors(
{simtelOutLogFileLFN: {
'Ancestors': [corsikaOutFileLFN]
}})
print 'result addFileAncestor:', result
result = fcc.addFileAncestors(
{simtelOutHistFileLFN: {
'Ancestors': [corsikaOutFileLFN]
}})
print 'result addFileAncestor:', result
##### Exit now if only corsika simulation required
if (mode == 'corsika_simtel'):
continue
######### run read_cta #######################################
rcta = Read_CtaApp()
rcta.setSoftwarePackage(CorsikaSimtelPack)
rcta.rctaExe = 'read_cta'
powerlaw_dict = {
'gamma': '-2.57',
'gamma_ptsrc': '-2.57',
'proton': '-2.70',
'electron': '-3.21'
}
dstFileName = particle + '_' + str(thetaP) + '_' + str(
phiP) + '_alt' + str(
obslev) + '_' + 'run' + run_number + '.simtel-dst0.gz'
dstHistoFileName = particle + '_' + str(thetaP) + '_' + str(
phiP) + '_alt' + str(
obslev) + '_' + 'run' + run_number + '.hdata-dst0.gz'
# rcta.rctaArguments = ['-r', '4', '-u', '--integration-scheme', '4', '--integration-window', '7,3', '--tail-cuts', '6,8', '--min-pix', '2', '--min-amp', '20', '--type', '1,0,0,400', '--tail-cuts', '9,12', '--min-amp', '20', '--type', '2,0,0,100', '--tail-cuts', '8,11', '--min-amp', '19', '--type', '3,0,0,40', '--tail-cuts', '6,9', '--min-amp', '15', '--type', '4,0,0,15', '--tail-cuts', '3.7,5.5', '--min-amp', '8', '--dst-level', '0', '--dst-file', dstFileName, '--histogram-file', dstHistoFileName, '--powerlaw', powerlaw_dict[particle], simtelFileName]
## added some options starting from Armazones_2K prod.
rcta.rctaArguments = [
'-r', '4', '-u', '--integration-scheme', '4',
'--integration-window', '7,3', '--tail-cuts', '6,8', '--min-pix',
'2', '--min-amp', '20', '--type', '1,0,0,400', '--tail-cuts',
'9,12', '--min-amp', '20', '--type', '2,0,0,100', '--tail-cuts',
'8,11', '--min-amp', '19', '--type', '3,0,0,40', '--tail-cuts',
'6,9', '--min-amp', '15', '--type', '4,0,0,15', '--tail-cuts',
'3.7,5.5', '--min-amp', '8', '--type', '5,0,0,70,5.6',
'--tail-cuts', '2.4,3.2', '--min-amp', '5.6', '--dst-level', '0',
'--dst-file', dstFileName, '--histogram-file', dstHistoFileName,
'--powerlaw', powerlaw_dict[particle], simtelFileName
]
rctaReturnCode = rcta.execute()
if rctaReturnCode != 0:
DIRAC.gLogger.error('read_cta Application: Failed')
jobReport.setApplicationStatus('read_cta Application: Failed')
DIRAC.exit(-1)
######## run dst quality checks ######################################
fd = open('check_dst_histo.sh', 'w')
fd.write("""#! /bin/bash
dsthistfilename=%s
dstfile=%s
n6="$(list_histograms -h 6 ${dsthistfilename} | grep 'Histogram of type' | sed 's/.*bins, //' | sed 's/ entries.//')"
n12001="$(list_histograms -h 12001 ${dsthistfilename} | grep 'Histogram of type' | sed 's/.*bins, //' | sed 's/ entries.//')"
if [ $n6 -ne $n12001 ]; then
echo 'n6 found:' $n6
echo 'n12001 found:' $n12001
exit 1
else
echo 'n6 found:' $n6
echo 'n12001 found:' $n12001
fi
n12002="$(list_histograms -h 12002 ${dsthistfilename} | grep 'Histogram of type' | sed 's/.*bins, //' | sed 's/ entries.//')"
nev="$(statio ${dstfile} | egrep '^2010' | cut -f2)"
if [ -z "$nev" ]; then nev="0"; fi
if [ $nev -ne $n12002 ]; then
echo 'nev found:' $nev
echo 'n12002 found:' $n12002
exit 1
else
echo 'nev found:' $nev
echo 'n12002 found:' $n12002
fi
""" % (dstHistoFileName, dstFileName))
fd.close()
os.system('chmod u+x check_dst_histo.sh')
cmdTuple = ['./check_dst_histo.sh']
DIRAC.gLogger.notice('Executing command tuple:', cmdTuple)
ret = systemCall(0, cmdTuple, sendOutput, env=corsikaEnviron)
checkHistoReturnCode, stdout, stderr = ret['Value']
if not ret['OK']:
DIRAC.gLogger.error('Failed to execute check_dst_histo.sh')
DIRAC.gLogger.error('check_dst_histo.sh status is:',
checkHistoReturnCode)
DIRAC.exit(-1)
if (checkHistoReturnCode != 0):
DIRAC.gLogger.error('Failure during check_dst_histo.sh')
DIRAC.gLogger.error('check_dst_histo.sh status is:',
checkHistoReturnCode)
jobReport.setApplicationStatus('Histo check Failed')
DIRAC.exit(-1)
############create MD and upload dst data/histo ##########################################################
global dstDirPath
global dstProdVersion
dstProdVersion = current_version + '_dst'
dstDirPath = os.path.join(simtelDirPath_conf, dstProdVersion)
dstOutFileDir = os.path.join(dstDirPath, 'Data', runNumSeriesDir)
dstOutFileLFN = os.path.join(dstOutFileDir, dstFileName)
resultCreateDstDirMD = createDstFileSystAndMD()
if not resultCreateDstDirMD['OK']:
DIRAC.gLogger.error('Failed to create Dst Directory MD')
jobReport.setApplicationStatus('Failed to create Dst Directory MD')
DIRAC.gLogger.error(
'Metadata coherence problem, no Dst File produced')
DIRAC.exit(-1)
else:
DIRAC.gLogger.notice('Dst Directory MD successfully created')
############################################################
res = CheckCatalogCoherence(dstOutFileLFN)
if res == DIRAC.S_OK:
DIRAC.gLogger.notice('dst file already exists. Removing:',
dstOutFileLFN)
ret = dirac.removeFile(dstOutFileLFN)
res = upload_to_seList(dstOutFileLFN, dstFileName)
if res != DIRAC.S_OK:
DIRAC.gLogger.error('Upload dst Error', dstOutFileLFN)
jobReport.setApplicationStatus('OutputData Upload Error')
DIRAC.exit(-1)
##############################################################
dstHistoFileDir = os.path.join(dstDirPath, 'Histograms',
runNumSeriesDir)
dstHistoFileLFN = os.path.join(dstHistoFileDir, dstHistoFileName)
res = CheckCatalogCoherence(dstHistoFileLFN)
if res == DIRAC.S_OK:
DIRAC.gLogger.notice('dst histo file already exists. Removing:',
dstHistoFileLFN)
ret = dirac.removeFile(dstHistoFileLFN)
res = upload_to_seList(dstHistoFileLFN, dstHistoFileName)
if res != DIRAC.S_OK:
DIRAC.gLogger.error('Upload dst Error', dstHistoFileName)
jobReport.setApplicationStatus('OutputData Upload Error')
DIRAC.exit(-1)
########### Insert RunNumSeries MD ##########################
dstRunNumberSeriesDirExist = fcc.isDirectory(
dstOutFileDir)['Value']['Successful'][dstOutFileDir]
newDstRunFileSeriesDir = (
dstRunNumberSeriesDirExist != True
) # if new runFileSeries, will need to add new MD
if newDstRunFileSeriesDir:
insertRunFileSeriesMD(dstOutFileDir, runNumTrunc)
insertRunFileSeriesMD(dstHistoFileDir, runNumTrunc)
####### dst File level metadata ###############################################
dstFileMD = {}
dstFileMD['runNumber'] = int(run_number)
dstFileMD['jobID'] = jobID
dstFileMD['rctaReturnCode'] = rctaReturnCode
result = fcc.setMetadata(dstOutFileLFN, dstFileMD)
print "result setMetadata=", result
if not result['OK']:
print 'ResultSetMetadata:', result['Message']
result = fcc.setMetadata(dstHistoFileLFN, dstFileMD)
print "result setMetadata=", result
if not result['OK']:
print 'ResultSetMetadata:', result['Message']
########## set the ancestors for dst #####################################
result = fcc.addFileAncestors(
{dstOutFileLFN: {
'Ancestors': [simtelOutFileLFN]
}})
print 'result addFileAncestor:', result
result = fcc.addFileAncestors(
{dstHistoFileLFN: {
'Ancestors': [simtelOutFileLFN]
}})
print 'result addFileAncestor:', result
######################################################
DIRAC.exit()
jobGroup = switch[1]
elif switch[0].lower() == "date":
date = switch[1]
selDate = date
if not date:
selDate = 'Today'
conditions = { 'Status':status,
'MinorStatus':minorStatus,
'ApplicationStatus':appStatus,
'Owner':owner,
'JobGroup':jobGroup,
'Date':selDate }
from DIRAC.Interfaces.API.Dirac import Dirac
dirac = Dirac()
result = dirac.selectJobs( Status = status,
MinorStatus = minorStatus,
ApplicationStatus = appStatus,
Site = site,
Owner = owner,
JobGroup = jobGroup,
Date = date )
if not result['OK']:
print 'ERROR %s' % result['Message']
exitCode = 2
else:
conds = []
for n, v in conditions.items():
if v:
conds.append( '%s = %s' % ( n, v ) )
def submitJob(self):
dirac = Dirac()
res = dirac.submitJob(self.__job)
gLogger.notice('Job submitted: {0}'.format(res["Value"]))
return res
except: print 'Failed to get DIRAC username. No proxy set up?' sys.exit(1) jobGroup = diracUsername + '.rmsynthesis.000' jdl += 'JobGroup = "' + jobGroup + '";\n' print 'Will submit this DIRAC JDL:' print '=====' print jdl print '=====' print # Submit the job(s) print 'Attempting to submit job(s) in JobGroup ' + jobGroup print dirac = Dirac() result = dirac.submitJob(jdl) print print '=====' print print 'Submission Result: ',result print print '=====' print if result['OK']: print 'Retrieve output with dirac-wms-job-get-output --JobGroup ' + jobGroup else: print 'There was a problem submitting your job(s) - see above!!!' print
def HapLookupExample(destination=None):
from CTADIRAC.Interfaces.API.HapApplicationJob import HapApplicationJob
from DIRAC.Interfaces.API.Dirac import Dirac
HapVersion = 'v0.16'
####### DoCtaIrf option values for Table prod #######################
AnalysisType = 'ScaleParam'
EnergyMethod = 'NoEnergy'
CutsConfig = 'mva_40pe_Wm_E_MST_PSFA_cta0909'
RunList = 'dstrun'
Zenith = '20'
Offset = '0'
Array = 'array-E.lis'
ParticleType = 'gamma'
#### for Energy prod ##################
# AnalysisType = 'Energy'
# EnergyMethod = 'Oak'
infileLFNList = [
'/vo.cta.in2p3.fr/user/a/arrabito/HAP/mini-prod3/DST/gamma/dst_CTA_00283000.root',
'/vo.cta.in2p3.fr/user/a/arrabito/HAP/mini-prod3/DST/gamma/dst_CTA_00283001.root',
'/vo.cta.in2p3.fr/user/a/arrabito/HAP/mini-prod3/DST/gamma/dst_CTA_00283002.root',
'/vo.cta.in2p3.fr/user/a/arrabito/HAP/mini-prod3/DST/gamma/dst_CTA_00283003.root',
'/vo.cta.in2p3.fr/user/a/arrabito/HAP/mini-prod3/DST/gamma/dst_CTA_00283004.root',
'/vo.cta.in2p3.fr/user/a/arrabito/HAP/mini-prod3/DST/gamma/dst_CTA_00283005.root',
'/vo.cta.in2p3.fr/user/a/arrabito/HAP/mini-prod3/DST/gamma/dst_CTA_00283006.root',
'/vo.cta.in2p3.fr/user/a/arrabito/HAP/mini-prod3/DST/gamma/dst_CTA_00283007.root',
'/vo.cta.in2p3.fr/user/a/arrabito/HAP/mini-prod3/DST/gamma/dst_CTA_00283008.root',
'/vo.cta.in2p3.fr/user/a/arrabito/HAP/mini-prod3/DST/gamma/dst_CTA_00283009.root'
]
general_opts = ['-V', HapVersion]
DoCtaIrf_opts = [
'-A', AnalysisType, '-C', CutsConfig, '-R', RunList, '-Z', Zenith,
'-O', Offset, '-T', Array, '-M', EnergyMethod, '-P', ParticleType
]
opts = general_opts + DoCtaIrf_opts
j = HapApplicationJob(opts)
if destination:
j.setDestination(destination)
j.setInputSandbox([
'LFN:/vo.cta.in2p3.fr/user/a/arrabito/HAP/mini-prod3/conf/v0.1/AnalysisConfig.tar.gz',
'passphrase'
])
j.setOutputSandbox(['DoCtaIrf.log'])
j.setName(AnalysisType)
j.setInputData(infileLFNList)
j.setInputDataPolicy('Protocol')
if (AnalysisType == 'ScaleParam'):
outfile = 'ScaleInfo_' + RunList + '.root'
elif (AnalysisType == 'Energy'):
outfile = 'EnergyCalib_' + RunList + '.root'
j.setOutputData([outfile])
j.setCPUTime(100000)
Script.gLogger.info(j._toJDL())
Dirac().submit(j)
class DIRACBackend(GridBackend):
"""Grid backend using the GFAL command line tools `gfal-*`."""
def __init__(self, **kwargs):
GridBackend.__init__(self, catalogue_prefix='', **kwargs)
from DIRAC.Core.Base import Script
Script.initialize()
from DIRAC.FrameworkSystem.Client.ProxyManagerClient import ProxyManagerClient
self.pm = ProxyManagerClient()
proxy = self.pm.getUserProxiesInfo()
if not proxy['OK']:
raise BackendException("Proxy error.")
from DIRAC.Interfaces.API.Dirac import Dirac
self.dirac = Dirac()
from DIRAC.Resources.Catalog.FileCatalog import FileCatalog
self.fc = FileCatalog()
from DIRAC.DataManagementSystem.Client.DataManager import DataManager
self.dm = DataManager()
self._xattr_cmd = sh.Command('gfal-xattr').bake(_tty_out=False)
self._replica_checksum_cmd = sh.Command('gfal-sum').bake(_tty_out=False)
self._bringonline_cmd = sh.Command('gfal-legacy-bringonline').bake(_tty_out=False)
self._cp_cmd = sh.Command('gfal-copy').bake(_tty_out=False)
self._ls_se_cmd = sh.Command('gfal-ls').bake(color='never', _tty_out=False)
self._move_cmd = sh.Command('gfal-rename').bake(_tty_out=False)
self._mkdir_cmd = sh.Command('gfal-mkdir').bake(_tty_out=False)
self._replicate_cmd = sh.Command('dirac-dms-replicate-lfn').bake(_tty_out=False)
self._add_cmd = sh.Command('dirac-dms-add-file').bake(_tty_out=False)
@staticmethod
def _check_return_value(ret):
if not ret['OK']:
raise BackendException("Failed: %s", ret['Message'])
for path, error in ret['Value']['Failed'].items():
if ('No such' in error) or ('Directory does not' in error):
raise DoesNotExistException("No such file or directory.")
else:
raise BackendException(error)
def _is_dir(self, lurl):
isdir = self.fc.isDirectory(lurl)
self._check_return_value(isdir)
return isdir['Value']['Successful'][lurl]
def _is_file(self, lurl):
isfile = self.fc.isFile(lurl)
self._check_return_value(isfile)
return isfile['Value']['Successful'][lurl]
def _get_dir_entry(self, lurl, infodict=None):
"""Take a lurl and return a DirEntry."""
# If no dctionary with the information is specified, get it from the catalogue
try:
md = infodict['MetaData']
except TypeError:
md = self.fc.getFileMetadata(lurl)
if not md['OK']:
raise BackendException("Failed to list path '%s': %s", lurl, md['Message'])
for path, error in md['Value']['Failed'].items():
if 'No such file' in error:
# File does not exist, maybe a directory?
md = self.fc.getDirectoryMetadata(lurl)
for path, error in md['Value']['Failed'].items():
raise DoesNotExistException("No such file or directory.")
else:
raise BackendException(md['Value']['Failed'][lurl])
md = md['Value']['Successful'][lurl]
return DirEntry(posixpath.basename(lurl), mode=oct(md.get('Mode', -1)), links=md.get('links', -1), gid=md['OwnerGroup'], uid=md['Owner'], size=md.get('Size', -1), modified=str(md.get('ModificationDate', '?')))
def _iter_directory(self, lurl):
"""Iterate over entries in a directory."""
ret = self.fc.listDirectory(lurl)
if not ret['OK']:
raise BackendException("Failed to list path '%s': %s", lurl, ret['Message'])
for path, error in ret['Value']['Failed'].items():
if 'Directory does not' in error:
# Dir does not exist, maybe a File?
if self.fc.isFile(lurl):
lst = [(lurl, None)]
break
else:
raise DoesNotExistException("No such file or Directory.")
else:
raise BackendException(ret['Value']['Failed'][lurl])
else:
# Sort items by keys, i.e. paths
lst = sorted(ret['Value']['Successful'][lurl]['Files'].items() + ret['Value']['Successful'][lurl]['SubDirs'].items())
for item in lst:
yield item # = path, dict
def _ls(self, lurl, **kwargs):
# Translate keyword arguments
d = kwargs.pop('directory', False)
if d:
# Just the requested entry itself
yield self._get_dir_entry(lurl)
return
for path, info in self._iter_directory(lurl):
yield self._get_dir_entry(path, info)
def _ls_se(self, surl, **kwargs):
# Translate keyword arguments
d = kwargs.pop('directory', False)
args = []
if -d:
args.append('-d')
args.append('-l')
args.append(surl)
try:
output = self._ls_se_cmd(*args, **kwargs)
except sh.ErrorReturnCode as e:
if 'No such file' in e.stderr:
raise DoesNotExistException("No such file or Directory.")
else:
raise BackendException(e.stderr)
for line in output:
fields = line.split()
mode, links, gid, uid, size = fields[:5]
name = fields[-1]
modified = ' '.join(fields[5:-1])
yield DirEntry(name, mode=mode, links=int(links), gid=gid, uid=uid, size=int(size), modified=modified)
def _replicas(self, lurl, **kwargs):
# Check the lurl actually exists
self._ls(lurl, directory=True)
rep = self.dirac.getReplicas(lurl)
self._check_return_value(rep)
rep = rep['Value']['Successful'][lurl]
return rep.values()
def _exists(self, surl, **kwargs):
try:
ret = self._ls_se_cmd(surl, '-d', '-l', **kwargs).strip()
except sh.ErrorReturnCode as e:
if 'No such file' in e.stderr:
return False
else:
if len(e.stderr) == 0:
raise BackendException(e.stdout)
else:
raise BackendException(e.stderr)
else:
return ret[0] != 'd' # Return `False` for directories
def _register(self, surl, lurl, verbose=False, **kwargs):
# Register an existing physical copy in the file catalogue
se = storage.get_SE(surl).name
# See if file already exists in DFC
ret = self.fc.getFileMetadata(lurl)
try:
self._check_return_value(ret)
except DoesNotExistException:
# Add new file
size = next(self._ls_se(surl, directory=True)).size
checksum = self.checksum(surl)
guid = str(uuid.uuid4()) # The guid does not seem to be important. Make it unique if possible.
ret = self.dm.registerFile((lurl, surl, size, se, guid, checksum))
else:
# Add new replica
ret = self.dm.registerReplica((lurl, surl, se))
self._check_return_value(ret)
if verbose:
print_("Successfully registered replica %s of %s from %s."%(surl, lurl, se))
return True
def _deregister(self, surl, lurl, verbose=False, **kwargs):
# DIRAC only needs to know the SE name to deregister a replica
se = storage.get_SE(surl).name
ret = self.dm.removeReplicaFromCatalog(se, [lurl])
self._check_return_value(ret)
if verbose:
print_("Successfully deregistered replica of %s from %s."%(lurl, se))
return True
def _state(self, surl, **kwargs):
try:
state = self._xattr_cmd(surl, 'user.status', **kwargs).strip()
except sh.ErrorReturnCode as e:
if "No such file" in e.stderr:
raise DoesNotExistException("No such file or Directory.")
state = '?'
except sh.SignalException_SIGSEGV:
state = '?'
return state
def _checksum(self, surl, **kwargs):
try:
checksum = self._replica_checksum_cmd(surl, 'ADLER32', **kwargs).split()[1]
except sh.ErrorReturnCode:
checksum = '?'
except sh.SignalException_SIGSEGV:
checksum = '?'
except IndexError:
checksum = '?'
return checksum
def _bringonline(self, surl, timeout, verbose=False, **kwargs):
if verbose:
out = sys.stdout
else:
out = None
# gfal does not notice when files come online, it seems
# Just send a single short request, then check regularly
if verbose:
out = sys.stdout
else:
out = None
end = time.time() + timeout
try:
self._bringonline_cmd('-t', 10, surl, _out=out, **kwargs)
except sh.ErrorReturnCode as e:
# The command fails if the file is not online
# To be expected after 10 seconds
if "No such file" in e.stderr:
# Except when the file does not actually exist on the tape storage
raise DoesNotExistException("No such file or Directory.")
wait = 5
while(True):
if verbose:
print_("Checking replica state...")
if self.is_online(surl):
if verbose:
print_("Replica brought online.")
return True
time_left = end - time.time()
if time_left <= 0:
if verbose:
print_("Could not bring replica online.")
return False
wait *= 2
if time_left < wait:
wait = time_left
if verbose:
print_("Timeout remaining: %d s"%(time_left))
print_("Checking again in: %d s"%(wait))
time.sleep(wait)
def _replicate(self, source_surl, destination_surl, lurl, verbose=False, **kwargs):
if verbose:
out = sys.stdout
else:
out = None
source = storage.get_SE(source_surl).name
destination = storage.get_SE(destination_surl).name
try:
self._replicate_cmd(lurl, destination, source, _out=out, **kwargs)
except sh.ErrorReturnCode as e:
if 'No such file' in e.stderr:
raise DoesNotExistException("No such file or directory.")
else:
if len(e.stderr) == 0:
raise BackendException(e.stdout)
else:
raise BackendException(e.stderr)
return True
def _get(self, surl, localpath, verbose=False, **kwargs):
if verbose:
out = sys.stdout
else:
out = None
try:
self._cp_cmd('-f', '--checksum', 'ADLER32', surl, localpath, _out=out, **kwargs)
except sh.ErrorReturnCode as e:
if 'No such file' in e.stderr:
raise DoesNotExistException("No such file or directory.")
else:
if len(e.stderr) == 0:
raise BackendException(e.stdout)
else:
raise BackendException(e.stderr)
return os.path.isfile(localpath)
def _put(self, localpath, surl, lurl, verbose=False, **kwargs):
if verbose:
out = sys.stdout
else:
out = None
se = storage.get_SE(surl).name
try:
self._add_cmd(lurl, localpath, se, _out=out, **kwargs)
except sh.ErrorReturnCode as e:
if 'No such file' in e.stderr:
raise DoesNotExistException("No such file or directory.")
else:
if len(e.stderr) == 0:
raise BackendException(e.stdout)
else:
raise BackendException(e.stderr)
return True
def _remove(self, surl, lurl, last=False, verbose=False, **kwargs):
se = storage.get_SE(surl).name
if last:
# Delete lfn
if verbose:
print_("Removing all replicas of %s."%(lurl,))
ret = self.dm.removeFile([lurl])
else:
if verbose:
print_("Removing replica of %s from %s."%(lurl, se))
ret = self.dm.removeReplica(se, [lurl])
if not ret['OK']:
raise BackendException('Failed: %s'%(ret['Message']))
for lurl, error in ret['Value']['Failed'].items():
if 'No such file' in error:
raise DoesNotExistException("No such file or directory.")
else:
raise BackendException(error)
return True
def _rmdir(self, lurl, verbose=False):
"""Remove the an empty directory from the catalogue."""
rep = self.fc.removeDirectory(lurl)
self._check_return_value(rep)
return True
def _move_replica(self, surl, new_surl, verbose=False, **kwargs):
if verbose:
out = sys.stdout
else:
out = None
try:
folder = posixpath.dirname(new_surl)
self._mkdir_cmd(folder, '-p', _out=out, **kwargs)
self._move_cmd(surl, new_surl, _out=out, **kwargs)
except sh.ErrorReturnCode as e:
if 'No such file' in e.stderr:
raise DoesNotExistException("No such file or directory.")
else:
if len(e.stderr) == 0:
raise BackendException(e.stdout)
else:
raise BackendException(e.stderr)
return True
'Usage:',
' %s [option|cfgfile] ... JobID ...' % Script.scriptName,
'Arguments:',
' JobID: DIRAC Job ID or a name of the file with JobID per line' ] ) )
Script.registerSwitch( "D:", "Dir=", "Store the output in this directory" )
Script.registerSwitch( "f:", "File=", "Get output for jobs with IDs from the file" )
Script.registerSwitch( "g:", "JobGroup=", "Get output for jobs in the given group" )
Script.parseCommandLine( ignoreErrors = True )
args = Script.getPositionalArgs()
from DIRAC.Interfaces.API.Dirac import Dirac, parseArguments
from DIRAC.Core.Utilities.Time import toString, date, day
dirac = Dirac()
exitCode = 0
errorList = []
outputDir = None
group = None
jobs = []
for sw, value in Script.getUnprocessedSwitches():
if sw in ( 'D', 'Dir' ):
outputDir = value
elif sw.lower() in ( 'f', 'file' ):
if os.path.exists( value ):
jFile = open( value )
jobs += jFile.read().split()
jFile.close()
elif sw.lower() in ( 'g', 'jobgroup' ):
Script.setUsageMessage('\n'.join([
__doc__.split('\n')[1], '\nUsage:\n',
' %s [option|cfgfile] ... JobID ...' % Script.scriptName,
'\nArguments:\n', ' JobID: DIRAC Job ID'
]))
Script.registerSwitch("f:", "File=",
"Get status for jobs with IDs from the file")
Script.registerSwitch("g:", "JobGroup=",
"Get status for jobs in the given group")
Script.parseCommandLine(ignoreErrors=True)
args = Script.getPositionalArgs()
from DIRAC.Interfaces.API.Dirac import Dirac
dirac = Dirac()
exitCode = 0
jobs = []
for key, value in Script.getUnprocessedSwitches():
if key.lower() in ('f', 'file'):
if os.path.exists(value):
jFile = open(value)
jobs += jFile.read().split()
jFile.close()
elif key.lower() in ('g', 'jobgroup'):
jobDate = toString(date() - 30 * day)
# Choose jobs no more than 30 days old
result = dirac.selectJobs(jobGroup=value, date=jobDate)
if not result['OK']:
print "Error:", result['Message']
def main():
site = "BOINC.World.org"
status = ["Running"]
minorStatus = None
workerNodes = None
since = None
date = "today"
full = False
until = None
batchIDs = None
Script.registerSwitch("", "Site=", " Select site (default: %s)" % site)
Script.registerSwitch("", "Status=",
" Select status (default: %s)" % status)
Script.registerSwitch("", "MinorStatus=", " Select minor status")
Script.registerSwitch("", "WorkerNode=", " Select WN")
Script.registerSwitch("", "BatchID=", " Select batch jobID")
Script.registerSwitch(
"", "Since=",
" Date since when to select jobs, or number of days (default: today)"
)
Script.registerSwitch("", "Date=",
" Specify the date (check for a full day)")
Script.registerSwitch(
"", "Full",
" Printout full list of job (default: False except if --WorkerNode)")
Script.parseCommandLine()
from DIRAC import gLogger
from DIRAC.Interfaces.API.Dirac import Dirac
from DIRAC.WorkloadManagementSystem.Client.JobMonitoringClient import JobMonitoringClient
switches = Script.getUnprocessedSwitches()
for switch in switches:
if switch[0] == "Site":
site = switch[1]
elif switch[0] == "MinorStatus":
minorStatus = switch[1]
elif switch[0] == "Status":
if switch[1].lower() == "all":
status = [None]
else:
status = switch[1].split(",")
elif switch[0] == "WorkerNode":
workerNodes = switch[1].split(",")
elif switch[0] == "BatchID":
try:
batchIDs = [int(id) for id in switch[1].split(",")]
except Exception:
gLogger.error("Invalid jobID", switch[1])
DIRAC.exit(1)
elif switch[0] == "Full":
full = True
elif switch[0] == "Date":
since = switch[1].split()[0]
until = str(
datetime.datetime.strptime(since, "%Y-%m-%d") +
datetime.timedelta(days=1)).split()[0]
elif switch[0] == "Since":
date = switch[1].lower()
if date == "today":
since = None
elif date == "yesterday":
since = 1
elif date == "ever":
since = 2 * 365
elif date.isdigit():
since = int(date)
date += " days"
else:
since = date
if isinstance(since, int):
since = str(datetime.datetime.now() -
datetime.timedelta(days=since)).split()[0]
if workerNodes or batchIDs:
# status = [None]
full = True
monitoring = JobMonitoringClient()
dirac = Dirac()
# Get jobs according to selection
jobs = set()
for stat in status:
res = dirac.selectJobs(site=site,
date=since,
status=stat,
minorStatus=minorStatus)
if not res["OK"]:
gLogger.error("Error selecting jobs", res["Message"])
DIRAC.exit(1)
allJobs = set(int(job) for job in res["Value"])
if until:
res = dirac.selectJobs(site=site, date=until, status=stat)
if not res["OK"]:
gLogger.error("Error selecting jobs", res["Message"])
DIRAC.exit(1)
allJobs -= set(int(job) for job in res["Value"])
jobs.update(allJobs)
if not jobs:
gLogger.always("No jobs found...")
DIRAC.exit(0)
# res = monitoring.getJobsSummary( jobs )
# print eval( res['Value'] )[jobs[0]]
allJobs = set()
result = {}
wnJobs = {}
gLogger.always("%d jobs found" % len(jobs))
# Get host name
for job in jobs:
res = monitoring.getJobParameter(job, "HostName")
node = res.get("Value", {}).get("HostName", "Unknown")
res = monitoring.getJobParameter(job, "LocalJobID")
batchID = res.get("Value", {}).get("LocalJobID", "Unknown")
if workerNodes:
if not [wn for wn in workerNodes if node.startswith(wn)]:
continue
allJobs.add(job)
if batchIDs:
if batchID not in batchIDs:
continue
allJobs.add(job)
if full or status == [None]:
allJobs.add(job)
result.setdefault(job, {})["Status"] = status
result[job]["Node"] = node
result[job]["LocalJobID"] = batchID
wnJobs[node] = wnJobs.setdefault(node, 0) + 1
# If necessary get jobs' status
statusCounters = {}
if allJobs:
allJobs = sorted(allJobs, reverse=True)
res = monitoring.getJobsStates(allJobs)
if not res["OK"]:
gLogger.error("Error getting job parameter", res["Message"])
else:
jobStates = res["Value"]
for job in allJobs:
stat = (
jobStates.get(job, {}).get("Status", "Unknown") + "; " +
jobStates.get(job, {}).get("MinorStatus", "Unknown") +
"; " +
jobStates.get(job, {}).get("ApplicationStatus", "Unknown"))
result[job]["Status"] = stat
statusCounters[stat] = statusCounters.setdefault(stat, 0) + 1
elif not workerNodes and not batchIDs:
allJobs = sorted(jobs, reverse=True)
# Print out result
if workerNodes or batchIDs:
gLogger.always("Found %d jobs at %s, WN %s (since %s):" %
(len(allJobs), site, workerNodes, date))
if allJobs:
gLogger.always("List of jobs:",
",".join([str(job) for job in allJobs]))
else:
if status == [None]:
gLogger.always("Found %d jobs at %s (since %s):" %
(len(allJobs), site, date))
for stat in sorted(statusCounters):
gLogger.always("%d jobs %s" % (statusCounters[stat], stat))
else:
gLogger.always("Found %d jobs %s at %s (since %s):" %
(len(allJobs), status, site, date))
gLogger.always(
"List of WNs:",
",".join([
"%s (%d)" % (node, wnJobs[node]) for node in sorted(
wnJobs,
key=cmp_to_key(lambda n1, n2: (wnJobs[n2] - wnJobs[n1])))
]),
)
if full:
if workerNodes or batchIDs:
nodeJobs = {}
for job in allJobs:
status = result[job]["Status"]
node = result[job]["Node"].split(".")[0]
jobID = result[job].get("LocalJobID")
nodeJobs.setdefault(node, []).append((jobID, job, status))
if not workerNodes:
workerNodes = sorted(nodeJobs)
for node in workerNodes:
for job in nodeJobs.get(node.split(".")[0], []):
gLogger.always("%s " % node + "(%s): %s - %s" % job)
else:
for job in allJobs:
status = result[job]["Status"]
node = result[job]["Node"]
jobID = result[job].get("LocalJobID")
gLogger.always("%s (%s): %s - %s" % (node, jobID, job, status))
import DIRAC
from DIRAC.Core.Base import Script
Script.setUsageMessage('\n'.join([
__doc__.split('\n')[1], 'Usage:',
' %s [option|cfgfile] ... JobID ...' % Script.scriptName, 'Arguments:',
' JobID: DIRAC Job ID'
]))
Script.parseCommandLine(ignoreErrors=True)
args = Script.getPositionalArgs()
if len(args) < 1:
Script.showHelp()
from DIRAC.Interfaces.API.Dirac import Dirac
dirac = Dirac()
exitCode = 0
errorList = []
for job in args:
result = dirac.peek(job, printout=True)
if not result['OK']:
errorList.append((job, result['Message']))
exitCode = 2
for error in errorList:
print "ERROR %s: %s" % error
DIRAC.exit(exitCode)
result = gProxyManager.downloadProxyToFile(shifter, shifter_group, requiredTimeLeft=10000)
print result
if not result["OK"]:
sys.stderr.write(result["Message"] + "\n")
sys.stderr.write("No valid proxy found.\n")
exit(1)
proxy = result["Value"]
os.environ["X509_USER_PROXY"] = proxy
print ("*INFO* using proxy %s" % proxy)
print "*********************************************************************************************************"
print "Execution at : '" + time.strftime("%d/%m/%y %H:%M", time.localtime()) + "'"
try:
d = Dirac()
except AttributeError:
sys.stderr.write(time.strftime("%d/%m/%y %H:%M", time.localtime()) + " => Error loading Dirac monitor\n")
raise Exception("Error loading Dirac monitor")
w = RPCClient("WorkloadManagement/JobMonitoring")
delTime = str(Time.dateTime() - delay_job_handled * Time.day)
jobid_handled = []
file_jobhandled = open(filename_jobhandled, "r")
for line in file_jobhandled:
try:
jobid_handled.append(str(int(line)))
except ValueError:
def main():
global extendedPrint
Script.registerSwitch('e', 'extended', 'Get extended printout',
setExtendedPrint)
Script.parseCommandLine(ignoreErrors=True)
from DIRAC import exit as DIRACExit
from DIRAC.Interfaces.API.Dirac import Dirac
from DIRAC.Interfaces.API.DiracAdmin import DiracAdmin
args = Script.getPositionalArgs()
if len(args) < 1:
Script.showHelp()
diracAdmin = DiracAdmin()
dirac = Dirac()
exitCode = 0
errorList = []
for gridID in args:
result = diracAdmin.getPilotInfo(gridID)
if not result['OK']:
errorList.append((gridID, result['Message']))
exitCode = 2
else:
res = result['Value'][gridID]
if extendedPrint:
tab = ''
for key in [
'PilotJobReference',
'Status',
'OwnerDN',
'OwnerGroup',
'SubmissionTime',
'DestinationSite',
'GridSite',
]:
if key in res:
diracAdmin.log.notice('%s%s: %s' %
(tab, key, res[key]))
if not tab:
tab = ' '
diracAdmin.log.notice('')
for jobID in res['Jobs']:
tab = ' '
result = dirac.getJobAttributes(int(jobID))
if not result['OK']:
errorList.append((gridID, result['Message']))
exitCode = 2
else:
job = result['Value']
diracAdmin.log.notice('%sJob ID: %s' % (tab, jobID))
tab += ' '
for key in [
'OwnerDN', 'OwnerGroup', 'JobName', 'Status',
'StartExecTime', 'LastUpdateTime',
'EndExecTime'
]:
if key in job:
diracAdmin.log.notice('%s%s:' % (tab, key),
job[key])
diracAdmin.log.notice('')
else:
print(diracAdmin.pPrint.pformat({gridID: res}))
for error in errorList:
print("ERROR %s: %s" % error)
DIRACExit(exitCode)
from DIRAC.Core.Base import Script
Script.setUsageMessage( '\n'.join( [ __doc__.split( '\n' )[1],
'Usage:',
' %s [option|cfgfile] ... JobID ...' % Script.scriptName,
'Arguments:',
' JobID: DIRAC Job ID' ] ) )
Script.registerSwitch( "D:", "Dir=", "Store the output in this directory" )
Script.parseCommandLine( ignoreErrors = True )
args = Script.getPositionalArgs()
if len( args ) < 1:
Script.showHelp()
from DIRAC.Interfaces.API.Dirac import Dirac, parseArguments
dirac = Dirac()
exitCode = 0
errorList = []
outputDir = ''
for sw, v in Script.getUnprocessedSwitches():
if sw in ( 'D', 'Dir' ):
outputDir = v
for job in parseArguments( args ):
result = dirac.getJobOutputData( job, destinationDir = outputDir )
if result['OK']:
print 'Job %s output data retrieved' % ( job )
else:
errorList.append( ( job, result['Message'] ) )
"../tools/StorageElement.py"
]
# these files are created by the job
outputSandbox =\
[
stdout,
stderr,
logfile
]
# the executible here '' is later set, so don't confuse users later on
diracJob = Job('', stdout, stderr)
# give a descriptive name
diracJob.setName('ND280Custom')
# set the program/executable, arguments, logFile, ...
diracJob.setExecutable(exe, arguments=args, logFile=logfile)
# set the job length
diracJob.setCPUTime(3600)
diracJob.setExecutionEnv(environmentDict)
diracJob.setInputSandbox(inputSandbox)
diracJob.setOutputSandbox(outputSandbox)
print 'job being submitted...'
dirac = Dirac()
result = dirac.submit(diracJob)
print 'Submission Result: ', result
args = Script.getPositionalArgs()
if len( args ) < 2 or len( args ) > 4:
Script.showHelp()
lfn = args[0]
seName = args[1]
sourceSE = ''
localCache = ''
if len( args ) > 2:
sourceSE = args[2]
if len( args ) == 4:
localCache = args[3]
from DIRAC.Interfaces.API.Dirac import Dirac
dirac = Dirac()
exitCode = 0
try:
f = open( lfn, 'r' )
lfns = f.read().splitlines()
f.close()
except:
lfns = [lfn]
finalResult = {"Failed":[], "Successful":[]}
for lfn in lfns:
result = dirac.replicateFile( lfn, seName, sourceSE, localCache, printOutput = True )
if not result['OK']:
finalResult["Failed"].append( lfn )
print 'ERROR %s' % ( result['Message'] )
""" simple hello world job
"""
from __future__ import print_function
from DIRAC.Interfaces.API.Job import Job
from DIRAC.Interfaces.API.Dirac import Dirac
from DIRAC.DataManagementSystem.Utilities.DMSHelpers import DMSHelpers
j = Job()
j.setName("helloWorld-test")
j.setExecutable("exe-script.py", "", "Executable.log")
# <-- user settings
j.setCPUTime(172800)
tier1s = DMSHelpers().getTiers(tier=(0, 1))
j.setBannedSites(tier1s)
# user settings -->
# print j.workflow
# submit the job to dirac
result = Dirac().submitJob(j)
print(result)
from DIRAC.Interfaces.API.Dirac import Dirac
from DIRAC.FrameworkSystem.Client.ProxyManagerClient import gProxyManager
from DIRAC.ConfigurationSystem.Client.Helpers.Operations import Operations
# get necessary credentials
op = Operations("glast.org")
shifter = op.getValue("Pipeline/Shifter","/DC=org/DC=doegrids/OU=People/CN=Stephan Zimmer 799865")
shifter_group = op.getValue("Pipeline/ShifterGroup","glast_user")
result = gProxyManager.downloadProxyToFile(shifter,shifter_group,requiredTimeLeft=10000)
if not result['OK']:
gLogger.error("ERROR: No valid proxy found; ",result['Message'])
dexit(1)
proxy = result[ 'Value' ]
environ['X509_USER_PROXY'] = proxy
gLogger.info("using proxy %s"%proxy)
dirac = Dirac(True,"myRepo.rep")
exitCode = 0
errorList = []
if len( args ) < 1:
Script.showHelp()
for job in args:
result = dirac.kill( job )
if result['OK']:
gLogger.info('Killed job %s' % ( job ))
else:
errorList.append( ( job, result['Message'] ) )
exitCode = 2
for error in errorList:
gLogger.error("ERROR %s: %s" % error)
dexit.exit( exitCode )
' %s [option|cfgfile] ... JobID ...' % Script.scriptName, 'Arguments:',
' JobID: DIRAC Job ID'
]))
Script.parseCommandLine(ignoreErrors=True)
args = Script.getPositionalArgs()
if len(args) < 1:
Script.showHelp()
parameters = None
for switch in Script.getUnprocessedSwitches():
if switch[0] == 'Parameters':
parameters = switch[1].split(',')
from DIRAC.Interfaces.API.Dirac import Dirac, parseArguments
dirac = Dirac()
results = {'OK': True, 'Value': {'Successful': {}, 'Failed': {}}}
success = results['Value']['Successful']
failed = results['Value']['Failed']
for job in parseArguments(args):
jobStr = 'Job %s' % job
result = dirac.getJobParameters(job, printOutput=False)
if not result['OK']:
failed.update({jobStr: result['Message']})
elif not result['Value']:
failed.update({jobStr: 'Job not found'})
elif parameters:
params = dict((key, val) for key, val in result['Value'].iteritems()
if key in parameters)
success.update({jobStr: params})
jobGroup = switch[1]
elif switch[0].lower() == "date":
date = switch[1]
selDate = date
if not date:
selDate = 'Today'
conditions = { 'Status':status,
'MinorStatus':minorStatus,
'ApplicationStatus':appStatus,
'Owner':owner,
'JobGroup':jobGroup,
'Date':selDate }
from DIRAC.Interfaces.API.Dirac import Dirac
dirac = Dirac()
result = dirac.selectJobs( status = status,
minorStatus = minorStatus,
applicationStatus = appStatus,
site = site,
owner = owner,
jobGroup = jobGroup,
date = date )
if not result['OK']:
print 'ERROR %s' % result['Message']
exitCode = 2
else:
conds = []
for n, v in conditions.items():
if v:
conds.append( '%s = %s' % ( n, v ) )
def __init__(self, *args, **kwargs):
AgentModule.__init__(self, *args, **kwargs)
self.name = "DataRecoveryAgent"
self.enabled = False
self.getJobInfoFromJDLOnly = False
self.__getCSOptions()
# This needs to be both otherwise we cannot account for all cases
self.jobStatus = [JobStatus.DONE, JobStatus.FAILED]
self.jobMon = JobMonitoringClient()
self.fcClient = FileCatalogClient()
self.tClient = TransformationClient()
self.reqClient = ReqClient()
self.diracAPI = Dirac()
self.inputFilesProcessed = set()
self.todo = {
"NoInputFiles": [
dict(
Message="NoInputFiles: OutputExists: Job 'Done'",
ShortMessage="NoInputFiles: job 'Done' ",
Counter=0,
Check=lambda job: job.allFilesExist() and job.status == JobStatus.FAILED,
Actions=lambda job, tInfo: [job.setJobDone(tInfo)],
),
dict(
Message="NoInputFiles: OutputMissing: Job 'Failed'",
ShortMessage="NoInputFiles: job 'Failed' ",
Counter=0,
Check=lambda job: job.allFilesMissing() and job.status == JobStatus.DONE,
Actions=lambda job, tInfo: [job.setJobFailed(tInfo)],
),
],
"InputFiles": [ # must always be first!
dict(
Message="One of many Successful: clean others",
ShortMessage="Other Tasks --> Keep",
Counter=0,
Check=lambda job: job.allFilesExist()
and job.otherTasks
and not set(job.inputFiles).issubset(self.inputFilesProcessed),
Actions=lambda job, tInfo: [
self.inputFilesProcessed.update(job.inputFiles),
job.setJobDone(tInfo),
job.setInputProcessed(tInfo),
],
),
dict(
Message="Other Task processed Input, no Output: Fail",
ShortMessage="Other Tasks --> Fail",
Counter=0,
Check=lambda job: set(job.inputFiles).issubset(self.inputFilesProcessed)
and job.allFilesMissing()
and job.status != JobStatus.FAILED,
Actions=lambda job, tInfo: [job.setJobFailed(tInfo)],
),
dict(
Message="Other Task processed Input: Fail and clean",
ShortMessage="Other Tasks --> Cleanup",
Counter=0,
Check=lambda job: set(job.inputFiles).issubset(self.inputFilesProcessed)
and not job.allFilesMissing(),
Actions=lambda job, tInfo: [job.setJobFailed(tInfo), job.cleanOutputs(tInfo)],
),
dict(
Message="InputFile(s) missing: mark job 'Failed', mark input 'Deleted', clean",
ShortMessage="Input Missing --> Job 'Failed, Input 'Deleted', Cleanup",
Counter=0,
Check=lambda job: job.inputFiles and job.allInputFilesMissing() and not job.allTransFilesDeleted(),
Actions=lambda job, tInfo: [
job.cleanOutputs(tInfo),
job.setJobFailed(tInfo),
job.setInputDeleted(tInfo),
],
),
dict(
Message="InputFile(s) Deleted, output Exists: mark job 'Failed', clean",
ShortMessage="Input Deleted --> Job 'Failed, Cleanup",
Counter=0,
Check=lambda job: job.inputFiles
and job.allInputFilesMissing()
and job.allTransFilesDeleted()
and not job.allFilesMissing(),
Actions=lambda job, tInfo: [job.cleanOutputs(tInfo), job.setJobFailed(tInfo)],
),
# All Output Exists
dict(
Message="Output Exists, job Failed, input not Processed --> Job Done, Input Processed",
ShortMessage="Output Exists --> Job Done, Input Processed",
Counter=0,
Check=lambda job: job.allFilesExist()
and not job.otherTasks
and job.status == JobStatus.FAILED
and not job.allFilesProcessed()
and job.allInputFilesExist(),
Actions=lambda job, tInfo: [job.setJobDone(tInfo), job.setInputProcessed(tInfo)],
),
dict(
Message="Output Exists, job Failed, input Processed --> Job Done",
ShortMessage="Output Exists --> Job Done",
Counter=0,
Check=lambda job: job.allFilesExist()
and not job.otherTasks
and job.status == JobStatus.FAILED
and job.allFilesProcessed()
and job.allInputFilesExist(),
Actions=lambda job, tInfo: [job.setJobDone(tInfo)],
),
dict(
Message="Output Exists, job Done, input not Processed --> Input Processed",
ShortMessage="Output Exists --> Input Processed",
Counter=0,
Check=lambda job: job.allFilesExist()
and not job.otherTasks
and job.status == JobStatus.DONE
and not job.allFilesProcessed()
and job.allInputFilesExist(),
Actions=lambda job, tInfo: [job.setInputProcessed(tInfo)],
),
# outputmissing
dict(
Message="Output Missing, job Failed, input Assigned, MaxError --> Input MaxReset",
ShortMessage="Max ErrorCount --> Input MaxReset",
Counter=0,
Check=lambda job: job.allFilesMissing()
and not job.otherTasks
and job.status == JobStatus.FAILED
and job.allFilesAssigned()
and not set(job.inputFiles).issubset(self.inputFilesProcessed)
and job.allInputFilesExist()
and job.checkErrorCount(),
Actions=lambda job, tInfo: [job.setInputMaxReset(tInfo)],
),
dict(
Message="Output Missing, job Failed, input Assigned --> Input Unused",
ShortMessage="Output Missing --> Input Unused",
Counter=0,
Check=lambda job: job.allFilesMissing()
and not job.otherTasks
and job.status == JobStatus.FAILED
and job.allFilesAssigned()
and not set(job.inputFiles).issubset(self.inputFilesProcessed)
and job.allInputFilesExist(),
Actions=lambda job, tInfo: [job.setInputUnused(tInfo)],
),
dict(
Message="Output Missing, job Done, input Assigned --> Job Failed, Input Unused",
ShortMessage="Output Missing --> Job Failed, Input Unused",
Counter=0,
Check=lambda job: job.allFilesMissing()
and not job.otherTasks
and job.status == JobStatus.DONE
and job.allFilesAssigned()
and not set(job.inputFiles).issubset(self.inputFilesProcessed)
and job.allInputFilesExist(),
Actions=lambda job, tInfo: [job.setInputUnused(tInfo), job.setJobFailed(tInfo)],
),
# some files missing, needing cleanup. Only checking for
# assigned, because processed could mean an earlier job was
# succesful and this one is just the duplicate that needed
# to be removed! But we check for other tasks earlier, so
# this should not happen
dict(
Message="Some missing, job Failed, input Assigned --> cleanup, Input 'Unused'",
ShortMessage="Output Missing --> Cleanup, Input Unused",
Counter=0,
Check=lambda job: job.someFilesMissing()
and not job.otherTasks
and job.status == JobStatus.FAILED
and job.allFilesAssigned()
and job.allInputFilesExist(),
Actions=lambda job, tInfo: [job.cleanOutputs(tInfo), job.setInputUnused(tInfo)],
),
dict(
Message="Some missing, job Done, input Assigned --> cleanup, job Failed, Input 'Unused'",
ShortMessage="Output Missing --> Cleanup, Job Failed, Input Unused",
Counter=0,
Check=lambda job: job.someFilesMissing()
and not job.otherTasks
and job.status == JobStatus.DONE
and job.allFilesAssigned()
and job.allInputFilesExist(),
Actions=lambda job, tInfo: [
job.cleanOutputs(tInfo),
job.setInputUnused(tInfo),
job.setJobFailed(tInfo),
],
),
dict(
Message="Some missing, job Done --> job Failed",
ShortMessage="Output Missing, Done --> Job Failed",
Counter=0,
Check=lambda job: not job.allFilesExist() and job.status == "Done",
Actions=lambda job, tInfo: [job.setJobFailed(tInfo)],
),
dict(
Message="Something Strange",
ShortMessage="Strange",
Counter=0,
Check=lambda job: job.status not in (JobStatus.FAILED, JobStatus.DONE),
Actions=lambda job, tInfo: [],
),
# should always be the last one!
dict(
Message="Failed Hard",
ShortMessage="Failed Hard",
Counter=0,
Check=lambda job: False, # never
Actions=lambda job, tInfo: [],
),
],
}
self.jobCache = defaultdict(lambda: (0, 0))
# Notification options
self.notesToSend = ""
self.subject = "DataRecoveryAgent"
self.startTime = time.time()
import os
Script.setUsageMessage( '\n'.join( [ __doc__.split( '\n' )[1],
'Usage:',
' %s [option|cfgfile] ... JobID ...' % Script.scriptName,
'Arguments:',
' JobID: DIRAC Job ID' ] ) )
Script.registerSwitch( "D:", "Dir=", "Store the output in this directory" )
Script.parseCommandLine( ignoreErrors = True )
args = Script.getPositionalArgs()
if len( args ) < 1:
Script.showHelp()
from DIRAC.Interfaces.API.Dirac import Dirac
dirac = Dirac()
exitCode = 0
errorList = []
outputDir = None
for sw, v in Script.getUnprocessedSwitches():
if sw in ( 'D', 'Dir' ):
outputDir = v
for job in args:
result = dirac.getInputSandbox( job, outputDir = outputDir )
if result['OK']:
if os.path.exists( 'InputSandbox%s' % job ):
print 'Job input sandbox retrieved in InputSandbox%s/' % ( job )
else:
Script.registerSwitch("f:", "File=",
"Get output for jobs with IDs from the file")
Script.registerSwitch("g:", "JobGroup=",
"Get output for jobs in the given group")
Script.parseCommandLine(ignoreErrors=True)
args = Script.getPositionalArgs()
import os.path
if __name__ == "__main__":
from DIRAC.Interfaces.API.Dirac import Dirac, parseArguments
from DIRAC.Core.Utilities.Time import toString, date, day
dirac = Dirac()
jobs = []
for sw, value in Script.getUnprocessedSwitches():
if sw.lower() in ('f', 'file'):
if os.path.exists(value):
jFile = open(value)
jobs += jFile.read().split()
jFile.close()
elif sw.lower() in ('g', 'jobgroup'):
group = value
jobDate = toString(date() - 30 * day)
result = dirac.selectJobs(jobGroup=value, date=jobDate)
if not result['OK']:
if not "No jobs selected" in result['Message']:
print "Error:", result['Message']
'Usage:',
' %s [option|cfgfile] ... LFN SE' % Script.scriptName,
'Arguments:',
' LFN: Logical File Name or file containing LFNs',
' SE: Valid DIRAC SE' ] ) )
Script.parseCommandLine( ignoreErrors = True )
args = Script.getPositionalArgs()
if len( args ) < 2:
Script.showHelp()
if len( args ) > 2:
print 'Only one LFN SE pair will be considered'
from DIRAC.Interfaces.API.Dirac import Dirac
dirac = Dirac()
exitCode = 0
lfn = args[0]
seName = args[1]
try:
f = open( lfn, 'r' )
lfns = f.read().splitlines()
f.close()
except:
lfns = [lfn]
for lfn in lfns:
result = dirac.removeReplica( lfn, seName, printOutput = True )
if not result['OK']:
elif switch[0].lower() == "owner":
owner = switch[1]
elif switch[0].lower() == "jobgroup":
jobGroup = switch[1]
elif switch[0].lower() == "date":
date = switch[1]
elif switch[0].lower() == "file":
filename = switch[1]
selDate = date
if not date:
selDate = 'Today'
from DIRAC.Interfaces.API.Dirac import Dirac
dirac = Dirac()
exitCode = 0
errorList = []
resultDict = {}
result = dirac.selectJobs(status=status,
minorStatus=minorStatus,
applicationStatus=appStatus,
site=site,
owner=owner,
jobGroup=jobGroup,
date=date)
if result['OK']:
jobs = result['Value']
else:
print "Error in selectJob", result['Message']
