def setUp(self):
self.dirac = Dirac()
csAPI = CSAPI()
self.lfn5 = os.path.join(DESTINATION_PATH, "test_file_10MB_v5.bin")
self.dir5 = os.path.dirname(self.lfn5)
# local file, for now:
self.fname = os.path.basename(self.lfn5)
random_dd(self.fname, 10)
self.diracSE = "SE-1"
try:
self.fc = FileCatalogClient("DataManagement/MultiVOFileCatalog")
except Exception:
self.fail(
" FileCatalog(['MultiVOFileCatalog']) raised Exception unexpectedly!\n"
+ traceback.format_exc())
return
# add a replica
self.fileadded = self.dirac.addFile(self.lfn5, self.fname,
self.diracSE)
self.assertTrue(self.fileadded["OK"])
def setUp(self):
super(RegressionTestCase, self).setUp()
gLogger.setLevel('DEBUG')
self.dirac = Dirac()
exeScriptLoc = find_all('exe-script.py', '..',
'/DIRAC/tests/Workflow/Regression')[0]
helloWorldLoc = find_all('helloWorld.py', '..',
'/DIRAC/tests/Workflow/Regression')[0]
shutil.copyfile(exeScriptLoc, './exe-script.py')
shutil.copyfile(helloWorldLoc, './helloWorld.py')
helloWorldXMLLocation = find_all('helloWorld.xml', '..',
'/DIRAC/tests/Workflow/Regression')[0]
self.j_u_hello = Job(helloWorldXMLLocation)
helloWorldXMLFewMoreLocation = find_all(
'helloWorld.xml', '..', '/DIRAC/tests/Workflow/Regression')[0]
self.j_u_helloPlus = Job(helloWorldXMLFewMoreLocation)
def get_job_list(owner, job_group, n_hours):
''' get a list of jobs for a selection
'''
from DIRAC.Interfaces.API.Dirac import Dirac
dirac = Dirac()
now = datetime.datetime.now()
onehour = datetime.timedelta(hours=1)
results = dirac.selectJobs(
jobGroup=job_group,
owner=owner,
date=now - n_hours * onehour)
if 'Value' not in results:
DIRAC.gLogger.error(
"No job found for group \"%s\" and owner \"%s\" in the past %s hours" %
(job_group, owner, n_hours))
DIRAC.exit(-1)
# Found some jobs, print information)
jobs_list = results['Value']
return jobs_list
def SimtelProdExample(args=None):
from CTADIRAC.Interfaces.API.SimtelProdJob import SimtelProdJob
from DIRAC.Interfaces.API.Dirac import Dirac
if (len(args) < 1 or len(args) > 2):
Script.showHelp()
if (len(args) == 2):
storage_element = args[1]
else:
storage_element = 'LUPM-Disk'
LFN_file = args[0]
f = open(LFN_file, 'r')
infileLFNList = []
for line in f:
infileLFN = line.strip()
infileLFNList.append(infileLFN)
j = SimtelProdJob()
j.setVersion('prod-2_21122012')
j.setExecutable('cta-ultra5')
j.setParametricInputData(infileLFNList)
j.setName('repro')
j.setInputSandbox(['fileCatalog.cfg'])
j.setParameters(['fileCatalog.cfg', '-D', storage_element])
j.setOutputSandbox(['simtel.log'])
j.setCPUTime(100000)
Script.gLogger.info(j._toJDL())
Dirac().submit(j)
def HapConverterExample( destination = None ) :
from CTADIRAC.Interfaces.API.HapConverterJob import HapConverterJob
from DIRAC.Interfaces.API.Dirac import Dirac
### general options ###############
HapVersion = 'v0.18'
infileLFNList = [
'LFN:/vo.cta.in2p3.fr/Simulation/sim_telarray/Prod1S_PS/2000/gamma/20/90/spectrum_-2.0/0.003_300/pointlike/cta-prod1/0.0deg/Data/run283xxx/gamma_20deg_90deg_run283000___cta-prod1_desert.simhess.gz',
'LFN:/vo.cta.in2p3.fr/Simulation/sim_telarray/Prod1S_PS/2000/gamma/20/90/spectrum_-2.0/0.003_300/pointlike/cta-prod1/0.0deg/Data/run283xxx/gamma_20deg_90deg_run283001___cta-prod1_desert.simhess.gz']
for infileLFN in infileLFNList:
infile = os.path.basename(infileLFN)
#### build the output file name for rawdata #############################
PartType = infile.split( '_' )[0]
RunNum = infile.split( 'run' )[1].split('___cta-prod1_desert.simhess.gz')[0]
raw_fileout = 'raw_' + PartType + '_run' + RunNum + '.root'
tellist = 'array-E.lis'
general_opts = ['-V', HapVersion]
### eventio_cta options ###############
eventio_cta_opts = ['--infile',infile,'--outfile',raw_fileout,'--tellist',tellist,'--pixelslices','true']
opts = general_opts + eventio_cta_opts
j = HapConverterJob(opts)
if destination:
j.setDestination( destination )
j.setInputSandbox( [ 'passphrase'] )
j.setOutputSandbox( ['eventio_cta.log','Open_Raw.log'])
jobName = 'eventio_' + RunNum
j.setName(jobName)
j.setInputData([infileLFN])
j.setOutputData([raw_fileout])
j.setCPUTime(100000)
Script.gLogger.info( j._toJDL() )
Dirac().submit( j )
def main():
# Registering arguments will automatically add their description to the help menu
Script.registerArgument(["JobID: DIRAC Job ID"])
_, args = Script.parseCommandLine(ignoreErrors=True)
from DIRAC.Interfaces.API.Dirac import Dirac, parseArguments
dirac = Dirac()
exitCode = 0
errorList = []
for job in parseArguments(args):
result = dirac.getJobParameters(job, printOutput=True)
if not result["OK"]:
errorList.append((job, result["Message"]))
exitCode = 2
for error in errorList:
print("ERROR %s: %s" % error)
DIRAC.exit(exitCode)
def main():
input_file_jobvar_name = 'JSUB_' + os.environ.get(
'JSUB_input_file_jobvar_name', 'input_file'
) # if input file is local, and put to DFC through jsub register.
input_lfn_jobvar_name = 'JSUB_' + os.environ.get(
'JSUB_input_lfn_jobvar_name',
'input_lfn') # if input file is directly from DFC
source_lfn_prefix = os.environ.get('JSUB_source_lfn_prefix')
input_path = os.environ.get(input_file_jobvar_name)
input_lfn = os.environ.get(input_lfn_jobvar_name)
destination = os.environ.get('JSUB_destination', './')
if input_lfn is None:
if source_lfn_prefix:
lfn = os.path.join(source_lfn_prefix, input_path)
else:
lfn = input_path_to_lfn(input_path)
else:
lfn = input_lfn
fname = os.path.basename(lfn)
# gfal_prefix = 'srm://storm.ihep.ac.cn:8444'
# gfal_path = gfal_prefix + input_path
# dest_dir = os.environ.get('destination_dir','./')
# dest_filename = dest_dir + os.path.basename(input_path)
# os.system('gfal-copy {} {}'.format(gfal_path,dest_filename)) #replace the input file
# download the file to the action folder
dirac = Dirac()
result = dirac.getFile(lfn)
if not result['OK']:
gLogger.error('Download file error: %s' % result['Message'])
return 2
# mv to destination
os.system('mv %s %s' % (fname, destination))
return 0
def HapDSTExample(destination=None):
from CTADIRAC.Interfaces.API.HapDSTJob import HapDSTJob
from DIRAC.Interfaces.API.Dirac import Dirac
### general options ###############
HapVersion = 'v0.18'
infileLFNList = [
'LFN:/vo.cta.in2p3.fr/user/a/arrabito/HAP/mini-prod3/Rawdata/gamma/raw_gamma_run283000.root',
'LFN:/vo.cta.in2p3.fr/user/a/arrabito/HAP/mini-prod3/Rawdata/gamma/raw_gamma_run283001.root'
]
for infileLFN in infileLFNList:
infile = os.path.basename(infileLFN)
RunNum = infile.split('run')[1].split('.root')[0]
tellist = 'array-E.lis'
general_opts = ['-V', HapVersion]
### make_CTA_DST.C options ###############
make_CTA_DST_opts = ['-R', RunNum, '-I', infile, '-T', tellist]
opts = general_opts + make_CTA_DST_opts
j = HapDSTJob(opts)
if destination:
j.setDestination(destination)
j.setInputSandbox(['passphrase'])
j.setOutputSandbox(['make_CTA_DST.log', 'CheckDST.log'])
jobName = 'DSTrun_' + RunNum
j.setName(jobName)
j.setInputData([infileLFN])
j.setOutputData(['dst*.root'])
j.setCPUTime(100000)
Script.gLogger.info(j._toJDL())
Dirac().submit(j)
def main():
Script.parseCommandLine(ignoreErrors=True)
args = Script.getPositionalArgs()
# pylint: disable=wrong-import-position
from DIRAC.Interfaces.API.Dirac import Dirac
if len(args) < 2:
Script.showHelp(exitCode=1)
if len(args) > 3:
print('Only one LFN SE pair will be considered')
dirac = Dirac()
exitCode = 0
lfn = args[0]
seName = args[1]
proto = False
if len(args) > 2:
proto = args[2]
try:
with open(lfn, 'r') as f:
lfns = f.read().splitlines()
except IOError:
lfns = [lfn]
for lfn in lfns:
result = dirac.getAccessURL(lfn,
seName,
protocol=proto,
printOutput=True)
if not result['OK']:
print('ERROR: ', result['Message'])
exitCode = 2
DIRAC.exit(exitCode)
def main():
# Read command line options
parser = argparse.ArgumentParser(description="Download collection files from Dirac")
parser.add_argument("--indir", default=None, help="Dirac repository")
parser.add_argument("--outdir", default="", help="Output file directory")
args = parser.parse_args()
print("Download file from: {}".format(args.indir))
print("Output directory: {}".format(args.outdir))
# Create output directory
if not os.path.exists(args.outdir):
os.makedirs(args.outdir)
# Get list of files
batcmd = "dirac-dms-user-lfns --BaseDir {}".format(args.indir)
result = subprocess.check_output(batcmd, shell=True)
file_list = result.split()[-1]
# try reading the lfns file
try:
grid_file_list = open(file_list)
except IOError:
raise IOError("cannot read lfns file list...")
dirac = Dirac()
file_collection = []
for line in grid_file_list.read().splitlines():
if len(file_collection) < 100:
file_collection.append(line) # files will be downloaded later
else:
dirac.getFile(file_collection, destDir=args.outdir)
file_collection = [] # there won't be any loop at the end
if file_collection:
dirac.getFile(file_collection, destDir=args.outdir)
def getSub2Sub5():
""" Download subarray-2 and 5 files corresponding to a list of subarray-1 files
Keyword arguments:
none -- none
"""
DIRAC.gLogger.info('Get Subarray-2 and 5 files')
# get JDL
dirac = Dirac()
resJDL = dirac.getJobJDL(os.environ['JOBID'] )
# get list of output files
idata=resJDL['Value']['InputData']
# dowload files
for sub1 in idata:
DIRAC.gLogger.debug("Input %s "%sub1)
sub2=sub1.strip('\n').replace('subarray-1-nosct', 'subarray-2-nosct')
downloadFile(sub2)
sub5=sub1.strip('\n').replace('subarray-1-nosct', 'subarray-5-nosct')
downloadFile(sub5)
return DIRAC.S_OK()
def setUp(self):
super(UserJobTestCase, self).setUp()
self.d = Dirac()
try:
self.exeScriptLocation = find_all('exe-script.py', rootPath,
'/DIRAC/tests/Workflow')[0]
self.helloWorld = find_all("helloWorld.py", rootPath,
'/DIRAC/tests/Workflow')[0]
self.mpExe = find_all('testMpJob.sh', rootPath,
'/DIRAC/tests/Utilities')[0]
except IndexError: # we are in Jenkins
self.exeScriptLocation = find_all('exe-script.py',
os.environ['WORKSPACE'],
'/DIRAC/tests/Workflow')[0]
self.helloWorld = find_all("helloWorld.py",
os.environ['WORKSPACE'],
'/DIRAC/tests/Workflow')[0]
self.mpExe = find_all('testMpJob.sh', os.environ['WORKSPACE'],
'/DIRAC/tests/Utilities')[0]
gLogger.setLevel('DEBUG')
def main():
Script.parseCommandLine(ignoreErrors=True)
args = Script.getPositionalArgs()
if len(args) < 1:
Script.showHelp(exitCode=1)
from DIRAC.Interfaces.API.Dirac import Dirac, parseArguments
dirac = Dirac()
exitCode = 0
errorList = []
for job in parseArguments(args):
result = dirac.getJobLoggingInfo(job, printOutput=True)
if not result['OK']:
errorList.append((job, result['Message']))
exitCode = 2
for error in errorList:
print("ERROR %s: %s" % error)
DIRAC.exit(exitCode)
def main():
Script.parseCommandLine(ignoreErrors=True)
args = Script.getPositionalArgs()
system = None
service = None
url = None
if len(args) == 1:
# it is a URL
if args[0].startswith('dips://'):
url = args[0]
# It is System/Service
else:
sys_serv = args[0].split('/')
if len(sys_serv) != 2:
Script.showHelp(exitCode=1)
else:
system, service = sys_serv
elif len(args) == 2:
system, service = args[0], args[1]
else:
Script.showHelp(exitCode=1)
from DIRAC.Interfaces.API.Dirac import Dirac
dirac = Dirac()
exitCode = 0
result = dirac.pingService(system, service, printOutput=True, url=url)
if not result:
print('ERROR: Null result from ping()')
exitCode = 2
elif not result['OK']:
print('ERROR: ', result['Message'])
exitCode = 2
DIRAC.exit(exitCode)
def main():
Script.parseCommandLine(ignoreErrors=True)
lfns = Script.getPositionalArgs()
if len(lfns) < 1:
Script.showHelp()
from DIRAC.Interfaces.API.Dirac import Dirac
dirac = Dirac()
exitCode = 0
if len(lfns) == 1:
try:
with open(lfns[0], 'r') as f:
lfns = f.read().splitlines()
except Exception:
pass
result = dirac.getFile(lfns, printOutput=True)
if not result['OK']:
print('ERROR %s' % (result['Message']))
exitCode = 2
DIRAC.exit(exitCode)
def main():
# Registering arguments will automatically add their description to the help menu
Script.registerArgument(
"LFN: Logical File Name or file containing LFNs")
Script.registerArgument("SE: Valid DIRAC SE")
Script.registerArgument("PROTO: Optional protocol for accessURL",
default=False,
mandatory=False)
_, args = Script.parseCommandLine(ignoreErrors=True)
lfn, seName, proto = Script.getPositionalArgs(group=True)
# pylint: disable=wrong-import-position
from DIRAC.Interfaces.API.Dirac import Dirac
if len(args) > 3:
print("Only one LFN SE pair will be considered")
dirac = Dirac()
exitCode = 0
try:
with open(lfn, "r") as f:
lfns = f.read().splitlines()
except IOError:
lfns = [lfn]
for lfn in lfns:
result = dirac.getAccessURL(lfn,
seName,
protocol=proto,
printOutput=True)
if not result["OK"]:
print("ERROR: ", result["Message"])
exitCode = 2
DIRAC.exit(exitCode)
def submitWMS(job, infileList):
""" Submit the job to the WMS """
dirac = Dirac()
job.setParameterSequence('InputData',
infileList,
addToWorkflow='ParametricInputData')
job.setOutputData(
['./*evndisp.tar.gz'], outputPath='evndisp_data'
) # to be used if DataManagement step in EvnDisp3UserJob is commented
#job.setJobGroup( 'EvnDisp-proton' )
job.setName('evndispjob')
job.setOutputSandbox(['*Log.txt'])
#job.setInputSandbox( ['myconf'] )
# To allow jobs run at other sites than the site where the InputData are located
#job.setType( 'DataReprocessing' )
res = dirac.submit(job)
if res['OK']:
Script.gLogger.info('Submission Result: ', res['Value'])
return res
def setUp(self):
super(RegressionTestCase, self).setUp()
gLogger.setLevel('DEBUG')
self.dirac = Dirac()
try:
exeScriptLoc = find_all('exe-script.py', rootPath, '/DIRAC/tests/Workflow')[0]
helloWorldLoc = find_all('helloWorld.py', rootPath, '/DIRAC/tests/Workflow')[0]
except IndexError: # we are in Jenkins
exeScriptLoc = find_all('exe-script.py', os.environ['WORKSPACE'], '/DIRAC/tests/Workflow')[0]
helloWorldLoc = find_all('helloWorld.py', os.environ['WORKSPACE'], '/DIRAC/tests/Workflow')[0]
shutil.copyfile(exeScriptLoc, './exe-script.py')
shutil.copyfile(helloWorldLoc, './helloWorld.py')
try:
helloWorldXMLLocation = find_all('helloWorld.xml', rootPath, '/DIRAC/tests/Workflow/Regression')[0]
except IndexError: # we are in Jenkins
helloWorldXMLLocation = find_all('helloWorld.xml',
os.environ['WORKSPACE'],
'/DIRAC/tests/Workflow/Regression')[0]
self.j_u_hello = Job(helloWorldXMLLocation)
self.j_u_hello.setConfigArgs('pilot.cfg')
try:
helloWorldXMLFewMoreLocation = find_all('helloWorld.xml', rootPath, '/DIRAC/tests/Workflow/Regression')[0]
except IndexError: # we are in Jenkins
helloWorldXMLFewMoreLocation = find_all(
'helloWorld.xml',
os.environ['WORKSPACE'],
'/DIRAC/tests/Workflow/Regression')[0]
self.j_u_helloPlus = Job(helloWorldXMLFewMoreLocation)
self.j_u_helloPlus.setConfigArgs('pilot.cfg')
def __init__(self, widget, parent):
"""initialize the main controller"""
ControlerAbstract.__init__(self, widget, parent)
self.__bkClient = LHCB_BKKDBClient()
self.__diracAPI = Dirac()
self.__fileName = ''
self.__pathfilename = ''
self.__SelectionDict = {}
self.__SortDict = {}
self.__StartItem = 0
self.__Maxitems = 0
self.__qualityFlags = {}
retVal = self.__bkClient.getAvailableDataQuality()
if not retVal['OK']:
gLogger.error(retVal['Message'])
else:
for i in retVal['Value']:
if i == "OK":
self.__qualityFlags[i] = True
else:
self.__qualityFlags[i] = False
#self.__progressBar = ProgressThread(False, 'Query on database...',self.getWidget())
self.__bkClient.setDataQualities(self.__qualityFlags)
def main():
# Registering arguments will automatically add their description to the help menu
Script.registerArgument(["JobID: DIRAC Job ID"])
_, args = Script.parseCommandLine(ignoreErrors=True)
from DIRAC.Interfaces.API.Dirac import Dirac, parseArguments
dirac = Dirac()
exitCode = 0
errorList = []
result = dirac.rescheduleJob(parseArguments(args))
if result["OK"]:
print("Rescheduled job %s" %
",".join([str(j) for j in result["Value"]]))
else:
errorList.append((result["Value"][-1], result["Message"]))
print(result["Message"])
exitCode = 2
for error in errorList:
print("ERROR %s: %s" % error)
DIRAC.exit(exitCode)
def HapDSTwfParamExample(destination=None):
from CTADIRAC.Interfaces.API.HapDSTwfParamJob import HapDSTwfParamJob
from DIRAC.Interfaces.API.Dirac import Dirac
### general options ###############
HapVersion = 'v0.18'
infileLFNList = [
'LFN:/vo.cta.in2p3.fr/Simulation/sim_telarray/Prod1S_PS/2000/gamma/20/90/spectrum_-2.0/0.003_300/pointlike/cta-prod1/0.0deg/Data/run283xxx/gamma_20deg_90deg_run283000___cta-prod1_desert.simhess.gz',
'LFN:/vo.cta.in2p3.fr/Simulation/sim_telarray/Prod1S_PS/2000/gamma/20/90/spectrum_-2.0/0.003_300/pointlike/cta-prod1/0.0deg/Data/run283xxx/gamma_20deg_90deg_run283001___cta-prod1_desert.simhess.gz'
]
tellist = 'array-E.lis'
general_opts = ['-V', HapVersion]
### options for raw and dst production ###############
eventio_cta_opts = ['--tellist', tellist, '--pixelslices', 'true']
opts = general_opts + eventio_cta_opts
j = HapDSTwfParamJob(opts)
j.setParametricInputData(infileLFNList)
j.setInputSandbox(['passphrase'])
j.setOutputSandbox([
'eventio_cta.log', 'Open_Raw.log', 'make_CTA_DST.log', 'CheckDST.log'
])
j.setOutputData(['raw_*.root', 'dst*.root'])
j.setName('DSTwf')
if destination:
j.setDestination(destination)
j.setCPUTime(100000)
Script.gLogger.info(j._toJDL())
Dirac().submit(j)
def main():
from DIRAC.Core.Base import Script
Script.registerSwitch("p:", "inputfile=", "Input File", setInputFile)
Script.registerSwitch("E:", "simtelExecName=", "SimtelExecName",
setExecutable)
Script.registerSwitch("S:", "simtelConfig=", "SimtelConfig", setConfig)
Script.registerSwitch("V:", "version=", "Version", setVersion)
Script.registerSwitch("D:", "storage_element=", "Storage Element",
setStorageElement)
from DIRAC.Resources.Catalog.FileCatalogClient import FileCatalogClient
from DIRAC.Resources.Catalog.FileCatalog import FileCatalog
Script.parseCommandLine()
DIRAC.gLogger.setLevel('INFO')
global fcc, fcL, storage_element
from CTADIRAC.Core.Utilities.SoftwareInstallation import checkSoftwarePackage
from CTADIRAC.Core.Utilities.SoftwareInstallation import installSoftwarePackage
from CTADIRAC.Core.Utilities.SoftwareInstallation import installSoftwareEnviron
from CTADIRAC.Core.Utilities.SoftwareInstallation import localArea
from CTADIRAC.Core.Utilities.SoftwareInstallation import sharedArea
from CTADIRAC.Core.Utilities.SoftwareInstallation import workingArea
from DIRAC.Core.Utilities.Subprocess import systemCall
from DIRAC.WorkloadManagementSystem.Client.JobReport import JobReport
jobID = os.environ['JOBID']
jobID = int(jobID)
jobReport = JobReport(jobID)
CorsikaSimtelPack = 'corsika_simhessarray/' + version + '/corsika_simhessarray'
packs = [CorsikaSimtelPack]
for package in packs:
DIRAC.gLogger.notice('Checking:', package)
if sharedArea:
if checkSoftwarePackage(package, sharedArea())['OK']:
DIRAC.gLogger.notice('Package found in Shared Area:', package)
installSoftwareEnviron(package, workingArea())
packageTuple = package.split('/')
corsika_subdir = sharedArea(
) + '/' + packageTuple[0] + '/' + version
cmd = 'cp -u -r ' + corsika_subdir + '/* .'
os.system(cmd)
continue
DIRAC.gLogger.error('Check Failed for software package:', package)
DIRAC.gLogger.error('Software package not available')
DIRAC.exit(-1)
###########
## Checking MD coherence
fc = FileCatalog('LcgFileCatalog')
res = fc._getCatalogConfigDetails('DIRACFileCatalog')
print 'DFC CatalogConfigDetails:', res
res = fc._getCatalogConfigDetails('LcgFileCatalog')
print 'LCG CatalogConfigDetails:', res
fcc = FileCatalogClient()
fcL = FileCatalog('LcgFileCatalog')
from DIRAC.Interfaces.API.Dirac import Dirac
dirac = Dirac()
############################
#############
# CLAUDIA: simtelConfigFile should be built from ???
#simtelConfigFilesPath = 'sim_telarray/multi'
#simtelConfigFile = simtelConfigFilesPath + '/multi_cta-ultra5.cfg'
#createGlobalsFromConfigFiles(simtelConfigFile)
createGlobalsFromConfigFiles()
#######################
## files spread in 1000-runs subDirectories
corsikaFileName = os.path.basename(corsikaFileLFN)
run_number = corsikaFileName.split('run')[1].split('.corsika.gz')[
0] # run001412.corsika.gz
runNum = int(run_number)
subRunNumber = '%03d' % runNum
runNumModMille = runNum % 1000
runNumTrunc = (runNum - runNumModMille) / 1000
runNumSeriesDir = '%03dxxx' % runNumTrunc
print 'runNumSeriesDir=', runNumSeriesDir
f = open('DISABLE_WATCHDOG_CPU_WALLCLOCK_CHECK', 'w')
f.close()
##### If storage element is IN2P3-tape save simtel file on disk ###############
if storage_element == 'CC-IN2P3-Tape':
storage_element = 'CC-IN2P3-Disk'
############ Producing SimTel File
######################Building simtel Directory Metadata #######################
resultCreateSimtelDirMD = createSimtelFileSystAndMD()
if not resultCreateSimtelDirMD['OK']:
DIRAC.gLogger.error('Failed to create simtelArray Directory MD')
jobReport.setApplicationStatus(
'Failed to create simtelArray Directory MD')
DIRAC.gLogger.error(
'Metadata coherence problem, no simtelArray File produced')
DIRAC.exit(-1)
else:
print 'simtel Directory MD successfully created'
#### execute simtelarray ################
# fd = open('run_sim.sh', 'w' )
# fd.write( """#! /bin/sh
#echo "go for sim_telarray"
#. ./examples_common.sh
#export CORSIKA_IO_BUFFER=800MB
#zcat %s | $SIM_TELARRAY_PATH/run_sim_%s""" % (corsikaFileName, simtelExecName))
# fd.close()
#### execute simtelarray ################
fd = open('run_sim.sh', 'w')
fd.write("""#! /bin/sh
export SVNPROD2=$PWD
export SVNTAG=SVN-PROD2
export CORSIKA_IO_BUFFER=800MB
./grid_prod2-repro.sh %s %s""" % (corsikaFileName, simtelConfig))
fd.close()
os.system('chmod u+x run_sim.sh')
cmdTuple = ['./run_sim.sh']
ret = systemCall(0, cmdTuple, sendOutputSimTel)
simtelReturnCode, stdout, stderr = ret['Value']
if (os.system('grep Broken simtel.log')):
print 'not broken'
else:
print 'broken'
# Tag corsika File if Broken Pipe
corsikaTagMD = {}
corsikaTagMD['CorsikaToReprocess'] = 'CorsikaToReprocess'
result = fcc.setMetadata(corsikaFileLFN, corsikaTagMD)
print "result setMetadata=", result
if not result['OK']:
print 'ResultSetMetadata:', result['Message']
jobReport.setApplicationStatus('Broken pipe')
DIRAC.exit(-1)
if not ret['OK']:
DIRAC.gLogger.error('Failed to execute run_sim.sh')
DIRAC.gLogger.error('run_sim.sh status is:', simtelReturnCode)
DIRAC.exit(-1)
## putAndRegister simtel data/log/histo Output File:
simtelFileName = particle + '_' + str(thetaP) + '_' + str(
phiP) + '_alt' + str(obslev) + '_' + 'run' + run_number + '.simtel.gz'
cfg = simtelExecName
if simtelExecName == "cta-prod2-nsbx3":
cfg = "cta-prod2"
cmd = 'mv Data/sim_telarray/' + cfg + '/0.0deg/Data/*.simtel.gz ' + simtelFileName
if (os.system(cmd)):
DIRAC.exit(-1)
simtelOutFileDir = os.path.join(simtelDirPath, 'Data', runNumSeriesDir)
simtelOutFileLFN = os.path.join(simtelOutFileDir, simtelFileName)
simtelRunNumberSeriesDirExist = fcc.isDirectory(
simtelOutFileDir)['Value']['Successful'][simtelOutFileDir]
newSimtelRunFileSeriesDir = (
simtelRunNumberSeriesDirExist != True
) # if new runFileSeries, will need to add new MD
simtelLogFileName = particle + '_' + str(thetaP) + '_' + str(
phiP) + '_alt' + str(obslev) + '_' + 'run' + run_number + '.log.gz'
cmd = 'mv Data/sim_telarray/' + cfg + '/0.0deg/Log/*.log.gz ' + simtelLogFileName
if (os.system(cmd)):
DIRAC.exit(-1)
simtelOutLogFileDir = os.path.join(simtelDirPath, 'Log', runNumSeriesDir)
simtelOutLogFileLFN = os.path.join(simtelOutLogFileDir, simtelLogFileName)
simtelHistFileName = particle + '_' + str(thetaP) + '_' + str(
phiP) + '_alt' + str(obslev) + '_' + 'run' + run_number + '.hdata.gz'
cmd = 'mv Data/sim_telarray/' + cfg + '/0.0deg/Histograms/*.hdata.gz ' + simtelHistFileName
if (os.system(cmd)):
DIRAC.exit(-1)
simtelOutHistFileDir = os.path.join(simtelDirPath, 'Histograms',
runNumSeriesDir)
simtelOutHistFileLFN = os.path.join(simtelOutHistFileDir,
simtelHistFileName)
################################################
DIRAC.gLogger.notice('Put and register simtel File in LFC and DFC:',
simtelOutFileLFN)
ret = dirac.addFile(simtelOutFileLFN, simtelFileName, storage_element)
res = CheckCatalogCoherence(simtelOutFileLFN)
if res != DIRAC.S_OK:
DIRAC.gLogger.error('Job failed: Catalog Coherence problem found')
jobReport.setApplicationStatus('OutputData Upload Error')
DIRAC.exit(-1)
if not ret['OK']:
DIRAC.gLogger.error('Error during addFile call:', ret['Message'])
jobReport.setApplicationStatus('OutputData Upload Error')
DIRAC.exit(-1)
######################################################################
DIRAC.gLogger.notice('Put and register simtel Log File in LFC and DFC:',
simtelOutLogFileLFN)
ret = dirac.addFile(simtelOutLogFileLFN, simtelLogFileName,
storage_element)
res = CheckCatalogCoherence(simtelOutLogFileLFN)
if res != DIRAC.S_OK:
DIRAC.gLogger.error('Job failed: Catalog Coherence problem found')
jobReport.setApplicationStatus('OutputData Upload Error')
DIRAC.exit(-1)
if not ret['OK']:
DIRAC.gLogger.error('Error during addFile call:', ret['Message'])
jobReport.setApplicationStatus('OutputData Upload Error')
DIRAC.exit(-1)
######################################################################
DIRAC.gLogger.notice('Put and register simtel Histo File in LFC and DFC:',
simtelOutHistFileLFN)
ret = dirac.addFile(simtelOutHistFileLFN, simtelHistFileName,
storage_element)
res = CheckCatalogCoherence(simtelOutHistFileLFN)
if res != DIRAC.S_OK:
DIRAC.gLogger.error('Job failed: Catalog Coherence problem found')
jobReport.setApplicationStatus('OutputData Upload Error')
DIRAC.exit(-1)
if not ret['OK']:
DIRAC.gLogger.error('Error during addFile call:', ret['Message'])
jobReport.setApplicationStatus('OutputData Upload Error')
DIRAC.exit(-1)
######################################################################
if newSimtelRunFileSeriesDir:
insertRunFileSeriesMD(simtelOutFileDir, runNumTrunc)
insertRunFileSeriesMD(simtelOutLogFileDir, runNumTrunc)
insertRunFileSeriesMD(simtelOutHistFileDir, runNumTrunc)
###### simtel File level metadata ############################################
simtelFileMD = {}
simtelFileMD['runNumber'] = int(run_number)
simtelFileMD['jobID'] = jobID
simtelFileMD['simtelReturnCode'] = simtelReturnCode
result = fcc.setMetadata(simtelOutFileLFN, simtelFileMD)
print "result setMetadata=", result
if not result['OK']:
print 'ResultSetMetadata:', result['Message']
result = fcc.setMetadata(simtelOutLogFileLFN, simtelFileMD)
print "result setMetadata=", result
if not result['OK']:
print 'ResultSetMetadata:', result['Message']
result = fcc.setMetadata(simtelOutHistFileLFN, simtelFileMD)
print "result setMetadata=", result
if not result['OK']:
print 'ResultSetMetadata:', result['Message']
result = fcc.addFileAncestors(
{simtelOutFileLFN: {
'Ancestors': [corsikaFileLFN]
}})
print 'result addFileAncestor:', result
result = fcc.addFileAncestors(
{simtelOutLogFileLFN: {
'Ancestors': [corsikaFileLFN]
}})
print 'result addFileAncestor:', result
result = fcc.addFileAncestors(
{simtelOutHistFileLFN: {
'Ancestors': [corsikaFileLFN]
}})
print 'result addFileAncestor:', result
result = fcc.setMetadata(simtelOutFileLFN, simtelFileMD)
if not result['OK']:
print 'ResultSetMetadata:', result['Message']
DIRAC.exit()
def SimtelProdExample( args = None ) :
from CTADIRAC.Interfaces.API.SimtelProdJob import SimtelProdJob
from DIRAC.Interfaces.API.Dirac import Dirac
if (len(args)<1 or len(args)>3):
Script.showHelp()
if (len(args)>=2):
storage_element = args[1]
else :
storage_element = 'LUPM-Disk'
LFN_file = args[0]
f = open(LFN_file,'r')
infileLFNList = []
for line in f:
infileLFN = line.strip()
if line!="\n":
infileLFNList.append(infileLFN)
j = SimtelProdJob()
j.setVersion('prod-2_06052013')
cfg = 'cta-prod2' #this is hardcoded in generic_run
#simtelArrayConfig, passed as argument to this script or defaulted to STD, refers to the MD in the catalog and the actual storage path.
simtelArrayConfig = "STD"
if (len(args)>=3):
simtelArrayConfig = args[2]
if simtelArrayConfig == "SCMST":
cfg=='cta-prod2-sc3'
j.setVersion('prod-2_06052013_sc3')
elif simtelArrayConfig == "4MSST":
cfg="cta-prod2-4m-dc"
elif simtelArrayConfig == "SCSST":
cfg="cta-prod2-sc-sst"
elif simtelArrayConfig == "ASTRI":
cfg="cta-prod2-astri"
elif simtelArrayConfig == "NSBX3":
cfg="cta-prod2"
elif simtelArrayConfig == "STD":
cfg="cta-prod2"
else :
print "arrayConfig argument %s incorrect"%simtelArrayConfig
exit(1)
j.setExecutable(cfg)
j.setParametricInputData(infileLFNList)
#From Ricardo, to manage smooth loading
j.setType( 'CorsikaRepro' )
name = 'repro_' + simtelArrayConfig
j.setName(name)
j.setJobGroup(name)
j.setInputSandbox( [ 'fileCatalog.cfg','CTADIRAC','cta-simtelprodold.py','grid_prod2-repro.sh','SVN-PROD2_rev1868'] )
j.setParameters(['fileCatalog.cfg','-D',storage_element,'-S',simtelArrayConfig])
j.setOutputSandbox( ['simtel.log'])
j.setCPUTime(100000)
Script.gLogger.info( j._toJDL() )
Dirac().submit( j )
"""
from __future__ import print_function
__RCSID__ = "$Id$"
import DIRAC
from DIRAC.Core.Base import Script
Script.setUsageMessage('\n'.join([
__doc__.split('\n')[1], 'Usage:',
' %s [option|cfgfile] ... JobID ...' % Script.scriptName, 'Arguments:',
' JobID: DIRAC Job ID'
]))
Script.parseCommandLine(ignoreErrors=True)
args = Script.getPositionalArgs()
if len(args) < 1:
Script.showHelp(exitCode=1)
from DIRAC.Interfaces.API.Dirac import Dirac, parseArguments
result = Dirac().killJob(parseArguments(args))
if result['OK']:
print('Killed jobs %s' % ','.join([str(j) for j in result['Value']]))
exitCode = 0
else:
print('ERROR', result['Message'])
exitCode = 2
DIRAC.exit(exitCode)
def main():
parameters = ["OwnerDN", "StartExecTime", "EndExecTime"]
Script.registerSwitch(
"", "Parameters=",
" List of strings to be matched by job parameters or attributes")
# Registering arguments will automatically add their description to the help menu
Script.registerArgument(["PilotID: Grid ID of the pilot"])
switches, args = Script.parseCommandLine(ignoreErrors=True)
for switch in switches:
if switch[0] == "Parameters":
parameters += [par for par in switch[1].split(",")]
parameters = [(i, par.lower()) for i, par in enumerate(parameters) if par]
from DIRAC.Interfaces.API.DiracAdmin import DiracAdmin
from DIRAC.Interfaces.API.Dirac import Dirac
diracAdmin = DiracAdmin()
dirac = Dirac()
errorList = []
for gridID in args:
result = {}
res = diracAdmin.getPilotInfo(gridID)
if not res["OK"]:
errorList.append((gridID, res["Message"]))
else:
jobIDs = set(
[int(jobID) for jobID in res["Value"][gridID]["Jobs"]])
totCPU = 0
totWall = 0
effRequested = False
for jobID in sorted(jobIDs):
result.setdefault(jobID, {})
for func in (dirac.getJobParameters, dirac.getJobAttributes):
res = func(jobID)
if not res["OK"]:
errorList.append(("Job %d" % jobID, res["Message"]))
else:
params = res["Value"]
if "TotalCPUTime(s)" in params:
totCPU += float(params["TotalCPUTime(s)"])
totWall += float(params["WallClockTime(s)"])
params["CPUEfficiency"] = "%s %%" % (
100.0 * float(params["TotalCPUTime(s)"]) /
float(params["WallClockTime(s)"]))
for i, par in parameters:
for param in [
p for p in _stringInList(
str(par), str(params))
if not _stringInList(
str(p), str(result[jobID]))
]:
if param == "CPUEfficiency":
effRequested = True
result[jobID]["%d.%s" %
(i, param)] = params[param]
if effRequested:
result["CPUEfficiency"] = "%s %%" % (100.0 * totCPU /
totWall)
print(diracAdmin.pPrint.pformat({gridID: result}))
for error in errorList:
print("ERROR %s: %s" % error)
"""
__RCSID__ = "$Id$"
import DIRAC
from DIRAC.Core.Base import Script
import os, sys
Script.setUsageMessage('\n'.join([
__doc__.split('\n')[1], 'Usage:',
' %s [option|cfgfile] ... RepoDir' % Script.scriptName, 'Arguments:',
' RepoDir: Location of Job Repository'
]))
Script.parseCommandLine(ignoreErrors=False)
args = Script.getPositionalArgs()
if len(args) != 1:
Script.showHelp()
repoLocation = args[0]
from DIRAC.Interfaces.API.Dirac import Dirac
dirac = Dirac(WithRepo=True, RepoLocation=repoLocation)
exitCode = 0
result = dirac.monitorRepository(printOutput=True)
if not result['OK']:
print 'ERROR: ', result['Message']
exitCode = 2
DIRAC.exit(exitCode)
def createMDStructure( self, metadata, metadatafield, basepath, program_category ):
""" create meta data structure
"""
# ## Add metadata fields to the DFC
mdfield = json.loads( metadatafield )
for key, value in mdfield.items():
res = self.fc.addMetadataField( key, value )
if not res['OK']:
return res
# ## Create the directory structure
md = json.loads( metadata , object_pairs_hook = collections.OrderedDict )
path = basepath
process_program = program_category + '_prog'
for key, value in collections.OrderedDict( ( k, md[k] ) for k in ( 'site', 'particle', process_program ) if k in md ).items():
path = os.path.join( path, self._formatPath( value ) )
res = self.fc.createDirectory( path )
if not res['OK']:
return res
# Set directory metadata for each subdir: 'site', 'particle', 'process_program'
res = self.fcc.setMetadata( path, {key:value} )
if not res['OK']:
return res
# Create the TransformationID subdir and set MD
# ## Get the TransformationID
if os.environ.has_key( 'JOBID' ):
jobID = os.environ['JOBID']
dirac = Dirac()
res = dirac.getJobJDL( jobID )
TransformationID = '0000'
if res['Value'].has_key( 'TransformationID' ):
TransformationID = res['Value']['TransformationID']
path = os.path.join( path, TransformationID )
res = self.fc.createDirectory( path )
if not res['OK']:
return res
process_program_version = process_program + '_version'
res = self.fcc.setMetadata( path, dict( ( k, md[k] ) for k in ( 'phiP', 'thetaP', 'array_layout', process_program_version ) if k in md ) )
if not res['OK']:
return res
# Create the Data and Log subdirs and set MD
Transformation_path = path
for subdir in ['Data', 'Log']:
path = os.path.join( Transformation_path, subdir )
res = self.fc.createDirectory( path )
if not res['OK']:
return res
# Set metadata if not already defined
res = self.fcc.getDirectoryUserMetadata( path )
if not res['OK']:
return res
if 'outputType' not in res['Value']:
res = self.fcc.setMetadata( path, {'outputType':subdir} )
if not res['OK']:
return res
# MD for the Data directory - data_level and configuration_id
path = os.path.join(Transformation_path, 'Data')
# Set metadata if not already defined
res = self.fcc.getDirectoryUserMetadata( path )
if not res['OK']:
return res
if 'data_level' and 'configuration_id' not in res['Value']:
res = self.fcc.setMetadata(path, {'data_level': md['data_level'],
'configuration_id': md['configuration_id']
})
if not res['OK']:
return res
return DIRAC.S_OK( Transformation_path )
input_stage_files = []
if pipeline:
xrd_keytab = op.getValue("Pipeline/XrdKey", None)
if not xrd_keytab:
gLogger.notice("*DEBUG* adding XrdKey file %s to input" %
xrd_keytab)
input_stage_files.append(xrd_keytab)
if not opts.stagein is None:
# we do add. input staging
files = opts.stagein.split(",")
for f in files:
input_stage_files.append(f)
if len(input_stage_files):
if len(input_stage_files) == 1:
j.setInputData(input_stage_files[-1])
else:
j.setInputData(input_stage_files)
if opts.debug:
gLogger.notice(
'*DEBUG* just showing the JDL of the job to be submitted')
gLogger.notice(j._toJDL())
d = Dirac(True, "myRepo.rep")
res = d.submit(j)
if not res['OK']:
gLogger.error("Error during Job Submission ", res['Message'])
dexit(1)
JobID = res['Value']
gLogger.notice("Your job %s (\"%s\") has been submitted." %
(str(JobID), executable))
import DIRAC
from DIRAC.Core.Base import Script
Script.setUsageMessage('\n'.join([
__doc__.split('\n')[1], 'Usage:',
' %s [option|cfgfile] ... JobID ...' % Script.scriptName, 'Arguments:',
' JobID: DIRAC Job ID'
]))
Script.parseCommandLine(ignoreErrors=True)
args = Script.getPositionalArgs()
if len(args) < 1:
Script.showHelp()
from DIRAC.Interfaces.API.Dirac import Dirac, parseArguments
dirac = Dirac()
exitCode = 0
errorList = []
for job in parseArguments(args):
result = dirac.peekJob(job, printOutput=True)
if not result['OK']:
errorList.append((job, result['Message']))
exitCode = 2
for error in errorList:
print "ERROR %s: %s" % error
DIRAC.exit(exitCode)
def main():
Script.registerSwitch("e", "extended", "Get extended printout",
setExtendedPrint)
_, args = Script.parseCommandLine(ignoreErrors=True)
from DIRAC import exit as DIRACExit
from DIRAC.Interfaces.API.Dirac import Dirac
from DIRAC.Interfaces.API.DiracAdmin import DiracAdmin
diracAdmin = DiracAdmin()
dirac = Dirac()
exitCode = 0
errorList = []
for gridID in args:
result = diracAdmin.getPilotInfo(gridID)
if not result["OK"]:
errorList.append((gridID, result["Message"]))
exitCode = 2
else:
res = result["Value"][gridID]
if extendedPrint:
tab = ""
for key in [
"PilotJobReference",
"Status",
"OwnerDN",
"OwnerGroup",
"SubmissionTime",
"DestinationSite",
"GridSite",
]:
if key in res:
diracAdmin.log.notice("%s%s: %s" %
(tab, key, res[key]))
if not tab:
tab = " "
diracAdmin.log.notice("")
for jobID in res["Jobs"]:
tab = " "
result = dirac.getJobAttributes(int(jobID))
if not result["OK"]:
errorList.append((gridID, result["Message"]))
exitCode = 2
else:
job = result["Value"]
diracAdmin.log.notice("%sJob ID: %s" % (tab, jobID))
tab += " "
for key in [
"OwnerDN",
"OwnerGroup",
"JobName",
"Status",
"StartExecTime",
"LastUpdateTime",
"EndExecTime",
]:
if key in job:
diracAdmin.log.notice("%s%s:" % (tab, key),
job[key])
diracAdmin.log.notice("")
else:
print(diracAdmin.pPrint.pformat({gridID: res}))
for error in errorList:
print("ERROR %s: %s" % error)
DIRACExit(exitCode)
