def setUp(self): self.dirac = Dirac() csAPI = CSAPI() self.lfn5 = os.path.join(DESTINATION_PATH, "test_file_10MB_v5.bin") self.dir5 = os.path.dirname(self.lfn5) # local file, for now: self.fname = os.path.basename(self.lfn5) random_dd(self.fname, 10) self.diracSE = "SE-1" try: self.fc = FileCatalogClient("DataManagement/MultiVOFileCatalog") except Exception: self.fail( " FileCatalog(['MultiVOFileCatalog']) raised Exception unexpectedly!\n" + traceback.format_exc()) return # add a replica self.fileadded = self.dirac.addFile(self.lfn5, self.fname, self.diracSE) self.assertTrue(self.fileadded["OK"])
def setUp(self): super(RegressionTestCase, self).setUp() gLogger.setLevel('DEBUG') self.dirac = Dirac() exeScriptLoc = find_all('exe-script.py', '..', '/DIRAC/tests/Workflow/Regression')[0] helloWorldLoc = find_all('helloWorld.py', '..', '/DIRAC/tests/Workflow/Regression')[0] shutil.copyfile(exeScriptLoc, './exe-script.py') shutil.copyfile(helloWorldLoc, './helloWorld.py') helloWorldXMLLocation = find_all('helloWorld.xml', '..', '/DIRAC/tests/Workflow/Regression')[0] self.j_u_hello = Job(helloWorldXMLLocation) helloWorldXMLFewMoreLocation = find_all( 'helloWorld.xml', '..', '/DIRAC/tests/Workflow/Regression')[0] self.j_u_helloPlus = Job(helloWorldXMLFewMoreLocation)
def get_job_list(owner, job_group, n_hours): ''' get a list of jobs for a selection ''' from DIRAC.Interfaces.API.Dirac import Dirac dirac = Dirac() now = datetime.datetime.now() onehour = datetime.timedelta(hours=1) results = dirac.selectJobs( jobGroup=job_group, owner=owner, date=now - n_hours * onehour) if 'Value' not in results: DIRAC.gLogger.error( "No job found for group \"%s\" and owner \"%s\" in the past %s hours" % (job_group, owner, n_hours)) DIRAC.exit(-1) # Found some jobs, print information) jobs_list = results['Value'] return jobs_list
def SimtelProdExample(args=None): from CTADIRAC.Interfaces.API.SimtelProdJob import SimtelProdJob from DIRAC.Interfaces.API.Dirac import Dirac if (len(args) < 1 or len(args) > 2): Script.showHelp() if (len(args) == 2): storage_element = args[1] else: storage_element = 'LUPM-Disk' LFN_file = args[0] f = open(LFN_file, 'r') infileLFNList = [] for line in f: infileLFN = line.strip() infileLFNList.append(infileLFN) j = SimtelProdJob() j.setVersion('prod-2_21122012') j.setExecutable('cta-ultra5') j.setParametricInputData(infileLFNList) j.setName('repro') j.setInputSandbox(['fileCatalog.cfg']) j.setParameters(['fileCatalog.cfg', '-D', storage_element]) j.setOutputSandbox(['simtel.log']) j.setCPUTime(100000) Script.gLogger.info(j._toJDL()) Dirac().submit(j)
def HapConverterExample( destination = None ) : from CTADIRAC.Interfaces.API.HapConverterJob import HapConverterJob from DIRAC.Interfaces.API.Dirac import Dirac ### general options ############### HapVersion = 'v0.18' infileLFNList = [ 'LFN:/vo.cta.in2p3.fr/Simulation/sim_telarray/Prod1S_PS/2000/gamma/20/90/spectrum_-2.0/0.003_300/pointlike/cta-prod1/0.0deg/Data/run283xxx/gamma_20deg_90deg_run283000___cta-prod1_desert.simhess.gz', 'LFN:/vo.cta.in2p3.fr/Simulation/sim_telarray/Prod1S_PS/2000/gamma/20/90/spectrum_-2.0/0.003_300/pointlike/cta-prod1/0.0deg/Data/run283xxx/gamma_20deg_90deg_run283001___cta-prod1_desert.simhess.gz'] for infileLFN in infileLFNList: infile = os.path.basename(infileLFN) #### build the output file name for rawdata ############################# PartType = infile.split( '_' )[0] RunNum = infile.split( 'run' )[1].split('___cta-prod1_desert.simhess.gz')[0] raw_fileout = 'raw_' + PartType + '_run' + RunNum + '.root' tellist = 'array-E.lis' general_opts = ['-V', HapVersion] ### eventio_cta options ############### eventio_cta_opts = ['--infile',infile,'--outfile',raw_fileout,'--tellist',tellist,'--pixelslices','true'] opts = general_opts + eventio_cta_opts j = HapConverterJob(opts) if destination: j.setDestination( destination ) j.setInputSandbox( [ 'passphrase'] ) j.setOutputSandbox( ['eventio_cta.log','Open_Raw.log']) jobName = 'eventio_' + RunNum j.setName(jobName) j.setInputData([infileLFN]) j.setOutputData([raw_fileout]) j.setCPUTime(100000) Script.gLogger.info( j._toJDL() ) Dirac().submit( j )
def main(): # Registering arguments will automatically add their description to the help menu Script.registerArgument(["JobID: DIRAC Job ID"]) _, args = Script.parseCommandLine(ignoreErrors=True) from DIRAC.Interfaces.API.Dirac import Dirac, parseArguments dirac = Dirac() exitCode = 0 errorList = [] for job in parseArguments(args): result = dirac.getJobParameters(job, printOutput=True) if not result["OK"]: errorList.append((job, result["Message"])) exitCode = 2 for error in errorList: print("ERROR %s: %s" % error) DIRAC.exit(exitCode)
def main(): input_file_jobvar_name = 'JSUB_' + os.environ.get( 'JSUB_input_file_jobvar_name', 'input_file' ) # if input file is local, and put to DFC through jsub register. input_lfn_jobvar_name = 'JSUB_' + os.environ.get( 'JSUB_input_lfn_jobvar_name', 'input_lfn') # if input file is directly from DFC source_lfn_prefix = os.environ.get('JSUB_source_lfn_prefix') input_path = os.environ.get(input_file_jobvar_name) input_lfn = os.environ.get(input_lfn_jobvar_name) destination = os.environ.get('JSUB_destination', './') if input_lfn is None: if source_lfn_prefix: lfn = os.path.join(source_lfn_prefix, input_path) else: lfn = input_path_to_lfn(input_path) else: lfn = input_lfn fname = os.path.basename(lfn) # gfal_prefix = 'srm://storm.ihep.ac.cn:8444' # gfal_path = gfal_prefix + input_path # dest_dir = os.environ.get('destination_dir','./') # dest_filename = dest_dir + os.path.basename(input_path) # os.system('gfal-copy {} {}'.format(gfal_path,dest_filename)) #replace the input file # download the file to the action folder dirac = Dirac() result = dirac.getFile(lfn) if not result['OK']: gLogger.error('Download file error: %s' % result['Message']) return 2 # mv to destination os.system('mv %s %s' % (fname, destination)) return 0
def HapDSTExample(destination=None): from CTADIRAC.Interfaces.API.HapDSTJob import HapDSTJob from DIRAC.Interfaces.API.Dirac import Dirac ### general options ############### HapVersion = 'v0.18' infileLFNList = [ 'LFN:/vo.cta.in2p3.fr/user/a/arrabito/HAP/mini-prod3/Rawdata/gamma/raw_gamma_run283000.root', 'LFN:/vo.cta.in2p3.fr/user/a/arrabito/HAP/mini-prod3/Rawdata/gamma/raw_gamma_run283001.root' ] for infileLFN in infileLFNList: infile = os.path.basename(infileLFN) RunNum = infile.split('run')[1].split('.root')[0] tellist = 'array-E.lis' general_opts = ['-V', HapVersion] ### make_CTA_DST.C options ############### make_CTA_DST_opts = ['-R', RunNum, '-I', infile, '-T', tellist] opts = general_opts + make_CTA_DST_opts j = HapDSTJob(opts) if destination: j.setDestination(destination) j.setInputSandbox(['passphrase']) j.setOutputSandbox(['make_CTA_DST.log', 'CheckDST.log']) jobName = 'DSTrun_' + RunNum j.setName(jobName) j.setInputData([infileLFN]) j.setOutputData(['dst*.root']) j.setCPUTime(100000) Script.gLogger.info(j._toJDL()) Dirac().submit(j)
def main(): Script.parseCommandLine(ignoreErrors=True) args = Script.getPositionalArgs() # pylint: disable=wrong-import-position from DIRAC.Interfaces.API.Dirac import Dirac if len(args) < 2: Script.showHelp(exitCode=1) if len(args) > 3: print('Only one LFN SE pair will be considered') dirac = Dirac() exitCode = 0 lfn = args[0] seName = args[1] proto = False if len(args) > 2: proto = args[2] try: with open(lfn, 'r') as f: lfns = f.read().splitlines() except IOError: lfns = [lfn] for lfn in lfns: result = dirac.getAccessURL(lfn, seName, protocol=proto, printOutput=True) if not result['OK']: print('ERROR: ', result['Message']) exitCode = 2 DIRAC.exit(exitCode)
def main(): # Read command line options parser = argparse.ArgumentParser(description="Download collection files from Dirac") parser.add_argument("--indir", default=None, help="Dirac repository") parser.add_argument("--outdir", default="", help="Output file directory") args = parser.parse_args() print("Download file from: {}".format(args.indir)) print("Output directory: {}".format(args.outdir)) # Create output directory if not os.path.exists(args.outdir): os.makedirs(args.outdir) # Get list of files batcmd = "dirac-dms-user-lfns --BaseDir {}".format(args.indir) result = subprocess.check_output(batcmd, shell=True) file_list = result.split()[-1] # try reading the lfns file try: grid_file_list = open(file_list) except IOError: raise IOError("cannot read lfns file list...") dirac = Dirac() file_collection = [] for line in grid_file_list.read().splitlines(): if len(file_collection) < 100: file_collection.append(line) # files will be downloaded later else: dirac.getFile(file_collection, destDir=args.outdir) file_collection = [] # there won't be any loop at the end if file_collection: dirac.getFile(file_collection, destDir=args.outdir)
def getSub2Sub5(): """ Download subarray-2 and 5 files corresponding to a list of subarray-1 files Keyword arguments: none -- none """ DIRAC.gLogger.info('Get Subarray-2 and 5 files') # get JDL dirac = Dirac() resJDL = dirac.getJobJDL(os.environ['JOBID'] ) # get list of output files idata=resJDL['Value']['InputData'] # dowload files for sub1 in idata: DIRAC.gLogger.debug("Input %s "%sub1) sub2=sub1.strip('\n').replace('subarray-1-nosct', 'subarray-2-nosct') downloadFile(sub2) sub5=sub1.strip('\n').replace('subarray-1-nosct', 'subarray-5-nosct') downloadFile(sub5) return DIRAC.S_OK()
def setUp(self): super(UserJobTestCase, self).setUp() self.d = Dirac() try: self.exeScriptLocation = find_all('exe-script.py', rootPath, '/DIRAC/tests/Workflow')[0] self.helloWorld = find_all("helloWorld.py", rootPath, '/DIRAC/tests/Workflow')[0] self.mpExe = find_all('testMpJob.sh', rootPath, '/DIRAC/tests/Utilities')[0] except IndexError: # we are in Jenkins self.exeScriptLocation = find_all('exe-script.py', os.environ['WORKSPACE'], '/DIRAC/tests/Workflow')[0] self.helloWorld = find_all("helloWorld.py", os.environ['WORKSPACE'], '/DIRAC/tests/Workflow')[0] self.mpExe = find_all('testMpJob.sh', os.environ['WORKSPACE'], '/DIRAC/tests/Utilities')[0] gLogger.setLevel('DEBUG')
def main(): Script.parseCommandLine(ignoreErrors=True) args = Script.getPositionalArgs() if len(args) < 1: Script.showHelp(exitCode=1) from DIRAC.Interfaces.API.Dirac import Dirac, parseArguments dirac = Dirac() exitCode = 0 errorList = [] for job in parseArguments(args): result = dirac.getJobLoggingInfo(job, printOutput=True) if not result['OK']: errorList.append((job, result['Message'])) exitCode = 2 for error in errorList: print("ERROR %s: %s" % error) DIRAC.exit(exitCode)
def main(): Script.parseCommandLine(ignoreErrors=True) args = Script.getPositionalArgs() system = None service = None url = None if len(args) == 1: # it is a URL if args[0].startswith('dips://'): url = args[0] # It is System/Service else: sys_serv = args[0].split('/') if len(sys_serv) != 2: Script.showHelp(exitCode=1) else: system, service = sys_serv elif len(args) == 2: system, service = args[0], args[1] else: Script.showHelp(exitCode=1) from DIRAC.Interfaces.API.Dirac import Dirac dirac = Dirac() exitCode = 0 result = dirac.pingService(system, service, printOutput=True, url=url) if not result: print('ERROR: Null result from ping()') exitCode = 2 elif not result['OK']: print('ERROR: ', result['Message']) exitCode = 2 DIRAC.exit(exitCode)
def main(): Script.parseCommandLine(ignoreErrors=True) lfns = Script.getPositionalArgs() if len(lfns) < 1: Script.showHelp() from DIRAC.Interfaces.API.Dirac import Dirac dirac = Dirac() exitCode = 0 if len(lfns) == 1: try: with open(lfns[0], 'r') as f: lfns = f.read().splitlines() except Exception: pass result = dirac.getFile(lfns, printOutput=True) if not result['OK']: print('ERROR %s' % (result['Message'])) exitCode = 2 DIRAC.exit(exitCode)
def main(): # Registering arguments will automatically add their description to the help menu Script.registerArgument( "LFN: Logical File Name or file containing LFNs") Script.registerArgument("SE: Valid DIRAC SE") Script.registerArgument("PROTO: Optional protocol for accessURL", default=False, mandatory=False) _, args = Script.parseCommandLine(ignoreErrors=True) lfn, seName, proto = Script.getPositionalArgs(group=True) # pylint: disable=wrong-import-position from DIRAC.Interfaces.API.Dirac import Dirac if len(args) > 3: print("Only one LFN SE pair will be considered") dirac = Dirac() exitCode = 0 try: with open(lfn, "r") as f: lfns = f.read().splitlines() except IOError: lfns = [lfn] for lfn in lfns: result = dirac.getAccessURL(lfn, seName, protocol=proto, printOutput=True) if not result["OK"]: print("ERROR: ", result["Message"]) exitCode = 2 DIRAC.exit(exitCode)
def submitWMS(job, infileList): """ Submit the job to the WMS """ dirac = Dirac() job.setParameterSequence('InputData', infileList, addToWorkflow='ParametricInputData') job.setOutputData( ['./*evndisp.tar.gz'], outputPath='evndisp_data' ) # to be used if DataManagement step in EvnDisp3UserJob is commented #job.setJobGroup( 'EvnDisp-proton' ) job.setName('evndispjob') job.setOutputSandbox(['*Log.txt']) #job.setInputSandbox( ['myconf'] ) # To allow jobs run at other sites than the site where the InputData are located #job.setType( 'DataReprocessing' ) res = dirac.submit(job) if res['OK']: Script.gLogger.info('Submission Result: ', res['Value']) return res
def setUp(self): super(RegressionTestCase, self).setUp() gLogger.setLevel('DEBUG') self.dirac = Dirac() try: exeScriptLoc = find_all('exe-script.py', rootPath, '/DIRAC/tests/Workflow')[0] helloWorldLoc = find_all('helloWorld.py', rootPath, '/DIRAC/tests/Workflow')[0] except IndexError: # we are in Jenkins exeScriptLoc = find_all('exe-script.py', os.environ['WORKSPACE'], '/DIRAC/tests/Workflow')[0] helloWorldLoc = find_all('helloWorld.py', os.environ['WORKSPACE'], '/DIRAC/tests/Workflow')[0] shutil.copyfile(exeScriptLoc, './exe-script.py') shutil.copyfile(helloWorldLoc, './helloWorld.py') try: helloWorldXMLLocation = find_all('helloWorld.xml', rootPath, '/DIRAC/tests/Workflow/Regression')[0] except IndexError: # we are in Jenkins helloWorldXMLLocation = find_all('helloWorld.xml', os.environ['WORKSPACE'], '/DIRAC/tests/Workflow/Regression')[0] self.j_u_hello = Job(helloWorldXMLLocation) self.j_u_hello.setConfigArgs('pilot.cfg') try: helloWorldXMLFewMoreLocation = find_all('helloWorld.xml', rootPath, '/DIRAC/tests/Workflow/Regression')[0] except IndexError: # we are in Jenkins helloWorldXMLFewMoreLocation = find_all( 'helloWorld.xml', os.environ['WORKSPACE'], '/DIRAC/tests/Workflow/Regression')[0] self.j_u_helloPlus = Job(helloWorldXMLFewMoreLocation) self.j_u_helloPlus.setConfigArgs('pilot.cfg')
def __init__(self, widget, parent): """initialize the main controller""" ControlerAbstract.__init__(self, widget, parent) self.__bkClient = LHCB_BKKDBClient() self.__diracAPI = Dirac() self.__fileName = '' self.__pathfilename = '' self.__SelectionDict = {} self.__SortDict = {} self.__StartItem = 0 self.__Maxitems = 0 self.__qualityFlags = {} retVal = self.__bkClient.getAvailableDataQuality() if not retVal['OK']: gLogger.error(retVal['Message']) else: for i in retVal['Value']: if i == "OK": self.__qualityFlags[i] = True else: self.__qualityFlags[i] = False #self.__progressBar = ProgressThread(False, 'Query on database...',self.getWidget()) self.__bkClient.setDataQualities(self.__qualityFlags)
def main(): # Registering arguments will automatically add their description to the help menu Script.registerArgument(["JobID: DIRAC Job ID"]) _, args = Script.parseCommandLine(ignoreErrors=True) from DIRAC.Interfaces.API.Dirac import Dirac, parseArguments dirac = Dirac() exitCode = 0 errorList = [] result = dirac.rescheduleJob(parseArguments(args)) if result["OK"]: print("Rescheduled job %s" % ",".join([str(j) for j in result["Value"]])) else: errorList.append((result["Value"][-1], result["Message"])) print(result["Message"]) exitCode = 2 for error in errorList: print("ERROR %s: %s" % error) DIRAC.exit(exitCode)
def HapDSTwfParamExample(destination=None): from CTADIRAC.Interfaces.API.HapDSTwfParamJob import HapDSTwfParamJob from DIRAC.Interfaces.API.Dirac import Dirac ### general options ############### HapVersion = 'v0.18' infileLFNList = [ 'LFN:/vo.cta.in2p3.fr/Simulation/sim_telarray/Prod1S_PS/2000/gamma/20/90/spectrum_-2.0/0.003_300/pointlike/cta-prod1/0.0deg/Data/run283xxx/gamma_20deg_90deg_run283000___cta-prod1_desert.simhess.gz', 'LFN:/vo.cta.in2p3.fr/Simulation/sim_telarray/Prod1S_PS/2000/gamma/20/90/spectrum_-2.0/0.003_300/pointlike/cta-prod1/0.0deg/Data/run283xxx/gamma_20deg_90deg_run283001___cta-prod1_desert.simhess.gz' ] tellist = 'array-E.lis' general_opts = ['-V', HapVersion] ### options for raw and dst production ############### eventio_cta_opts = ['--tellist', tellist, '--pixelslices', 'true'] opts = general_opts + eventio_cta_opts j = HapDSTwfParamJob(opts) j.setParametricInputData(infileLFNList) j.setInputSandbox(['passphrase']) j.setOutputSandbox([ 'eventio_cta.log', 'Open_Raw.log', 'make_CTA_DST.log', 'CheckDST.log' ]) j.setOutputData(['raw_*.root', 'dst*.root']) j.setName('DSTwf') if destination: j.setDestination(destination) j.setCPUTime(100000) Script.gLogger.info(j._toJDL()) Dirac().submit(j)
def main(): from DIRAC.Core.Base import Script Script.registerSwitch("p:", "inputfile=", "Input File", setInputFile) Script.registerSwitch("E:", "simtelExecName=", "SimtelExecName", setExecutable) Script.registerSwitch("S:", "simtelConfig=", "SimtelConfig", setConfig) Script.registerSwitch("V:", "version=", "Version", setVersion) Script.registerSwitch("D:", "storage_element=", "Storage Element", setStorageElement) from DIRAC.Resources.Catalog.FileCatalogClient import FileCatalogClient from DIRAC.Resources.Catalog.FileCatalog import FileCatalog Script.parseCommandLine() DIRAC.gLogger.setLevel('INFO') global fcc, fcL, storage_element from CTADIRAC.Core.Utilities.SoftwareInstallation import checkSoftwarePackage from CTADIRAC.Core.Utilities.SoftwareInstallation import installSoftwarePackage from CTADIRAC.Core.Utilities.SoftwareInstallation import installSoftwareEnviron from CTADIRAC.Core.Utilities.SoftwareInstallation import localArea from CTADIRAC.Core.Utilities.SoftwareInstallation import sharedArea from CTADIRAC.Core.Utilities.SoftwareInstallation import workingArea from DIRAC.Core.Utilities.Subprocess import systemCall from DIRAC.WorkloadManagementSystem.Client.JobReport import JobReport jobID = os.environ['JOBID'] jobID = int(jobID) jobReport = JobReport(jobID) CorsikaSimtelPack = 'corsika_simhessarray/' + version + '/corsika_simhessarray' packs = [CorsikaSimtelPack] for package in packs: DIRAC.gLogger.notice('Checking:', package) if sharedArea: if checkSoftwarePackage(package, sharedArea())['OK']: DIRAC.gLogger.notice('Package found in Shared Area:', package) installSoftwareEnviron(package, workingArea()) packageTuple = package.split('/') corsika_subdir = sharedArea( ) + '/' + packageTuple[0] + '/' + version cmd = 'cp -u -r ' + corsika_subdir + '/* .' os.system(cmd) continue DIRAC.gLogger.error('Check Failed for software package:', package) DIRAC.gLogger.error('Software package not available') DIRAC.exit(-1) ########### ## Checking MD coherence fc = FileCatalog('LcgFileCatalog') res = fc._getCatalogConfigDetails('DIRACFileCatalog') print 'DFC CatalogConfigDetails:', res res = fc._getCatalogConfigDetails('LcgFileCatalog') print 'LCG CatalogConfigDetails:', res fcc = FileCatalogClient() fcL = FileCatalog('LcgFileCatalog') from DIRAC.Interfaces.API.Dirac import Dirac dirac = Dirac() ############################ ############# # CLAUDIA: simtelConfigFile should be built from ??? #simtelConfigFilesPath = 'sim_telarray/multi' #simtelConfigFile = simtelConfigFilesPath + '/multi_cta-ultra5.cfg' #createGlobalsFromConfigFiles(simtelConfigFile) createGlobalsFromConfigFiles() ####################### ## files spread in 1000-runs subDirectories corsikaFileName = os.path.basename(corsikaFileLFN) run_number = corsikaFileName.split('run')[1].split('.corsika.gz')[ 0] # run001412.corsika.gz runNum = int(run_number) subRunNumber = '%03d' % runNum runNumModMille = runNum % 1000 runNumTrunc = (runNum - runNumModMille) / 1000 runNumSeriesDir = '%03dxxx' % runNumTrunc print 'runNumSeriesDir=', runNumSeriesDir f = open('DISABLE_WATCHDOG_CPU_WALLCLOCK_CHECK', 'w') f.close() ##### If storage element is IN2P3-tape save simtel file on disk ############### if storage_element == 'CC-IN2P3-Tape': storage_element = 'CC-IN2P3-Disk' ############ Producing SimTel File ######################Building simtel Directory Metadata ####################### resultCreateSimtelDirMD = createSimtelFileSystAndMD() if not resultCreateSimtelDirMD['OK']: DIRAC.gLogger.error('Failed to create simtelArray Directory MD') jobReport.setApplicationStatus( 'Failed to create simtelArray Directory MD') DIRAC.gLogger.error( 'Metadata coherence problem, no simtelArray File produced') DIRAC.exit(-1) else: print 'simtel Directory MD successfully created' #### execute simtelarray ################ # fd = open('run_sim.sh', 'w' ) # fd.write( """#! /bin/sh #echo "go for sim_telarray" #. ./examples_common.sh #export CORSIKA_IO_BUFFER=800MB #zcat %s | $SIM_TELARRAY_PATH/run_sim_%s""" % (corsikaFileName, simtelExecName)) # fd.close() #### execute simtelarray ################ fd = open('run_sim.sh', 'w') fd.write("""#! /bin/sh export SVNPROD2=$PWD export SVNTAG=SVN-PROD2 export CORSIKA_IO_BUFFER=800MB ./grid_prod2-repro.sh %s %s""" % (corsikaFileName, simtelConfig)) fd.close() os.system('chmod u+x run_sim.sh') cmdTuple = ['./run_sim.sh'] ret = systemCall(0, cmdTuple, sendOutputSimTel) simtelReturnCode, stdout, stderr = ret['Value'] if (os.system('grep Broken simtel.log')): print 'not broken' else: print 'broken' # Tag corsika File if Broken Pipe corsikaTagMD = {} corsikaTagMD['CorsikaToReprocess'] = 'CorsikaToReprocess' result = fcc.setMetadata(corsikaFileLFN, corsikaTagMD) print "result setMetadata=", result if not result['OK']: print 'ResultSetMetadata:', result['Message'] jobReport.setApplicationStatus('Broken pipe') DIRAC.exit(-1) if not ret['OK']: DIRAC.gLogger.error('Failed to execute run_sim.sh') DIRAC.gLogger.error('run_sim.sh status is:', simtelReturnCode) DIRAC.exit(-1) ## putAndRegister simtel data/log/histo Output File: simtelFileName = particle + '_' + str(thetaP) + '_' + str( phiP) + '_alt' + str(obslev) + '_' + 'run' + run_number + '.simtel.gz' cfg = simtelExecName if simtelExecName == "cta-prod2-nsbx3": cfg = "cta-prod2" cmd = 'mv Data/sim_telarray/' + cfg + '/0.0deg/Data/*.simtel.gz ' + simtelFileName if (os.system(cmd)): DIRAC.exit(-1) simtelOutFileDir = os.path.join(simtelDirPath, 'Data', runNumSeriesDir) simtelOutFileLFN = os.path.join(simtelOutFileDir, simtelFileName) simtelRunNumberSeriesDirExist = fcc.isDirectory( simtelOutFileDir)['Value']['Successful'][simtelOutFileDir] newSimtelRunFileSeriesDir = ( simtelRunNumberSeriesDirExist != True ) # if new runFileSeries, will need to add new MD simtelLogFileName = particle + '_' + str(thetaP) + '_' + str( phiP) + '_alt' + str(obslev) + '_' + 'run' + run_number + '.log.gz' cmd = 'mv Data/sim_telarray/' + cfg + '/0.0deg/Log/*.log.gz ' + simtelLogFileName if (os.system(cmd)): DIRAC.exit(-1) simtelOutLogFileDir = os.path.join(simtelDirPath, 'Log', runNumSeriesDir) simtelOutLogFileLFN = os.path.join(simtelOutLogFileDir, simtelLogFileName) simtelHistFileName = particle + '_' + str(thetaP) + '_' + str( phiP) + '_alt' + str(obslev) + '_' + 'run' + run_number + '.hdata.gz' cmd = 'mv Data/sim_telarray/' + cfg + '/0.0deg/Histograms/*.hdata.gz ' + simtelHistFileName if (os.system(cmd)): DIRAC.exit(-1) simtelOutHistFileDir = os.path.join(simtelDirPath, 'Histograms', runNumSeriesDir) simtelOutHistFileLFN = os.path.join(simtelOutHistFileDir, simtelHistFileName) ################################################ DIRAC.gLogger.notice('Put and register simtel File in LFC and DFC:', simtelOutFileLFN) ret = dirac.addFile(simtelOutFileLFN, simtelFileName, storage_element) res = CheckCatalogCoherence(simtelOutFileLFN) if res != DIRAC.S_OK: DIRAC.gLogger.error('Job failed: Catalog Coherence problem found') jobReport.setApplicationStatus('OutputData Upload Error') DIRAC.exit(-1) if not ret['OK']: DIRAC.gLogger.error('Error during addFile call:', ret['Message']) jobReport.setApplicationStatus('OutputData Upload Error') DIRAC.exit(-1) ###################################################################### DIRAC.gLogger.notice('Put and register simtel Log File in LFC and DFC:', simtelOutLogFileLFN) ret = dirac.addFile(simtelOutLogFileLFN, simtelLogFileName, storage_element) res = CheckCatalogCoherence(simtelOutLogFileLFN) if res != DIRAC.S_OK: DIRAC.gLogger.error('Job failed: Catalog Coherence problem found') jobReport.setApplicationStatus('OutputData Upload Error') DIRAC.exit(-1) if not ret['OK']: DIRAC.gLogger.error('Error during addFile call:', ret['Message']) jobReport.setApplicationStatus('OutputData Upload Error') DIRAC.exit(-1) ###################################################################### DIRAC.gLogger.notice('Put and register simtel Histo File in LFC and DFC:', simtelOutHistFileLFN) ret = dirac.addFile(simtelOutHistFileLFN, simtelHistFileName, storage_element) res = CheckCatalogCoherence(simtelOutHistFileLFN) if res != DIRAC.S_OK: DIRAC.gLogger.error('Job failed: Catalog Coherence problem found') jobReport.setApplicationStatus('OutputData Upload Error') DIRAC.exit(-1) if not ret['OK']: DIRAC.gLogger.error('Error during addFile call:', ret['Message']) jobReport.setApplicationStatus('OutputData Upload Error') DIRAC.exit(-1) ###################################################################### if newSimtelRunFileSeriesDir: insertRunFileSeriesMD(simtelOutFileDir, runNumTrunc) insertRunFileSeriesMD(simtelOutLogFileDir, runNumTrunc) insertRunFileSeriesMD(simtelOutHistFileDir, runNumTrunc) ###### simtel File level metadata ############################################ simtelFileMD = {} simtelFileMD['runNumber'] = int(run_number) simtelFileMD['jobID'] = jobID simtelFileMD['simtelReturnCode'] = simtelReturnCode result = fcc.setMetadata(simtelOutFileLFN, simtelFileMD) print "result setMetadata=", result if not result['OK']: print 'ResultSetMetadata:', result['Message'] result = fcc.setMetadata(simtelOutLogFileLFN, simtelFileMD) print "result setMetadata=", result if not result['OK']: print 'ResultSetMetadata:', result['Message'] result = fcc.setMetadata(simtelOutHistFileLFN, simtelFileMD) print "result setMetadata=", result if not result['OK']: print 'ResultSetMetadata:', result['Message'] result = fcc.addFileAncestors( {simtelOutFileLFN: { 'Ancestors': [corsikaFileLFN] }}) print 'result addFileAncestor:', result result = fcc.addFileAncestors( {simtelOutLogFileLFN: { 'Ancestors': [corsikaFileLFN] }}) print 'result addFileAncestor:', result result = fcc.addFileAncestors( {simtelOutHistFileLFN: { 'Ancestors': [corsikaFileLFN] }}) print 'result addFileAncestor:', result result = fcc.setMetadata(simtelOutFileLFN, simtelFileMD) if not result['OK']: print 'ResultSetMetadata:', result['Message'] DIRAC.exit()
def SimtelProdExample( args = None ) : from CTADIRAC.Interfaces.API.SimtelProdJob import SimtelProdJob from DIRAC.Interfaces.API.Dirac import Dirac if (len(args)<1 or len(args)>3): Script.showHelp() if (len(args)>=2): storage_element = args[1] else : storage_element = 'LUPM-Disk' LFN_file = args[0] f = open(LFN_file,'r') infileLFNList = [] for line in f: infileLFN = line.strip() if line!="\n": infileLFNList.append(infileLFN) j = SimtelProdJob() j.setVersion('prod-2_06052013') cfg = 'cta-prod2' #this is hardcoded in generic_run #simtelArrayConfig, passed as argument to this script or defaulted to STD, refers to the MD in the catalog and the actual storage path. simtelArrayConfig = "STD" if (len(args)>=3): simtelArrayConfig = args[2] if simtelArrayConfig == "SCMST": cfg=='cta-prod2-sc3' j.setVersion('prod-2_06052013_sc3') elif simtelArrayConfig == "4MSST": cfg="cta-prod2-4m-dc" elif simtelArrayConfig == "SCSST": cfg="cta-prod2-sc-sst" elif simtelArrayConfig == "ASTRI": cfg="cta-prod2-astri" elif simtelArrayConfig == "NSBX3": cfg="cta-prod2" elif simtelArrayConfig == "STD": cfg="cta-prod2" else : print "arrayConfig argument %s incorrect"%simtelArrayConfig exit(1) j.setExecutable(cfg) j.setParametricInputData(infileLFNList) #From Ricardo, to manage smooth loading j.setType( 'CorsikaRepro' ) name = 'repro_' + simtelArrayConfig j.setName(name) j.setJobGroup(name) j.setInputSandbox( [ 'fileCatalog.cfg','CTADIRAC','cta-simtelprodold.py','grid_prod2-repro.sh','SVN-PROD2_rev1868'] ) j.setParameters(['fileCatalog.cfg','-D',storage_element,'-S',simtelArrayConfig]) j.setOutputSandbox( ['simtel.log']) j.setCPUTime(100000) Script.gLogger.info( j._toJDL() ) Dirac().submit( j )
""" from __future__ import print_function __RCSID__ = "$Id$" import DIRAC from DIRAC.Core.Base import Script Script.setUsageMessage('\n'.join([ __doc__.split('\n')[1], 'Usage:', ' %s [option|cfgfile] ... JobID ...' % Script.scriptName, 'Arguments:', ' JobID: DIRAC Job ID' ])) Script.parseCommandLine(ignoreErrors=True) args = Script.getPositionalArgs() if len(args) < 1: Script.showHelp(exitCode=1) from DIRAC.Interfaces.API.Dirac import Dirac, parseArguments result = Dirac().killJob(parseArguments(args)) if result['OK']: print('Killed jobs %s' % ','.join([str(j) for j in result['Value']])) exitCode = 0 else: print('ERROR', result['Message']) exitCode = 2 DIRAC.exit(exitCode)
def main(): parameters = ["OwnerDN", "StartExecTime", "EndExecTime"] Script.registerSwitch( "", "Parameters=", " List of strings to be matched by job parameters or attributes") # Registering arguments will automatically add their description to the help menu Script.registerArgument(["PilotID: Grid ID of the pilot"]) switches, args = Script.parseCommandLine(ignoreErrors=True) for switch in switches: if switch[0] == "Parameters": parameters += [par for par in switch[1].split(",")] parameters = [(i, par.lower()) for i, par in enumerate(parameters) if par] from DIRAC.Interfaces.API.DiracAdmin import DiracAdmin from DIRAC.Interfaces.API.Dirac import Dirac diracAdmin = DiracAdmin() dirac = Dirac() errorList = [] for gridID in args: result = {} res = diracAdmin.getPilotInfo(gridID) if not res["OK"]: errorList.append((gridID, res["Message"])) else: jobIDs = set( [int(jobID) for jobID in res["Value"][gridID]["Jobs"]]) totCPU = 0 totWall = 0 effRequested = False for jobID in sorted(jobIDs): result.setdefault(jobID, {}) for func in (dirac.getJobParameters, dirac.getJobAttributes): res = func(jobID) if not res["OK"]: errorList.append(("Job %d" % jobID, res["Message"])) else: params = res["Value"] if "TotalCPUTime(s)" in params: totCPU += float(params["TotalCPUTime(s)"]) totWall += float(params["WallClockTime(s)"]) params["CPUEfficiency"] = "%s %%" % ( 100.0 * float(params["TotalCPUTime(s)"]) / float(params["WallClockTime(s)"])) for i, par in parameters: for param in [ p for p in _stringInList( str(par), str(params)) if not _stringInList( str(p), str(result[jobID])) ]: if param == "CPUEfficiency": effRequested = True result[jobID]["%d.%s" % (i, param)] = params[param] if effRequested: result["CPUEfficiency"] = "%s %%" % (100.0 * totCPU / totWall) print(diracAdmin.pPrint.pformat({gridID: result})) for error in errorList: print("ERROR %s: %s" % error)
""" __RCSID__ = "$Id$" import DIRAC from DIRAC.Core.Base import Script import os, sys Script.setUsageMessage('\n'.join([ __doc__.split('\n')[1], 'Usage:', ' %s [option|cfgfile] ... RepoDir' % Script.scriptName, 'Arguments:', ' RepoDir: Location of Job Repository' ])) Script.parseCommandLine(ignoreErrors=False) args = Script.getPositionalArgs() if len(args) != 1: Script.showHelp() repoLocation = args[0] from DIRAC.Interfaces.API.Dirac import Dirac dirac = Dirac(WithRepo=True, RepoLocation=repoLocation) exitCode = 0 result = dirac.monitorRepository(printOutput=True) if not result['OK']: print 'ERROR: ', result['Message'] exitCode = 2 DIRAC.exit(exitCode)
def createMDStructure( self, metadata, metadatafield, basepath, program_category ): """ create meta data structure """ # ## Add metadata fields to the DFC mdfield = json.loads( metadatafield ) for key, value in mdfield.items(): res = self.fc.addMetadataField( key, value ) if not res['OK']: return res # ## Create the directory structure md = json.loads( metadata , object_pairs_hook = collections.OrderedDict ) path = basepath process_program = program_category + '_prog' for key, value in collections.OrderedDict( ( k, md[k] ) for k in ( 'site', 'particle', process_program ) if k in md ).items(): path = os.path.join( path, self._formatPath( value ) ) res = self.fc.createDirectory( path ) if not res['OK']: return res # Set directory metadata for each subdir: 'site', 'particle', 'process_program' res = self.fcc.setMetadata( path, {key:value} ) if not res['OK']: return res # Create the TransformationID subdir and set MD # ## Get the TransformationID if os.environ.has_key( 'JOBID' ): jobID = os.environ['JOBID'] dirac = Dirac() res = dirac.getJobJDL( jobID ) TransformationID = '0000' if res['Value'].has_key( 'TransformationID' ): TransformationID = res['Value']['TransformationID'] path = os.path.join( path, TransformationID ) res = self.fc.createDirectory( path ) if not res['OK']: return res process_program_version = process_program + '_version' res = self.fcc.setMetadata( path, dict( ( k, md[k] ) for k in ( 'phiP', 'thetaP', 'array_layout', process_program_version ) if k in md ) ) if not res['OK']: return res # Create the Data and Log subdirs and set MD Transformation_path = path for subdir in ['Data', 'Log']: path = os.path.join( Transformation_path, subdir ) res = self.fc.createDirectory( path ) if not res['OK']: return res # Set metadata if not already defined res = self.fcc.getDirectoryUserMetadata( path ) if not res['OK']: return res if 'outputType' not in res['Value']: res = self.fcc.setMetadata( path, {'outputType':subdir} ) if not res['OK']: return res # MD for the Data directory - data_level and configuration_id path = os.path.join(Transformation_path, 'Data') # Set metadata if not already defined res = self.fcc.getDirectoryUserMetadata( path ) if not res['OK']: return res if 'data_level' and 'configuration_id' not in res['Value']: res = self.fcc.setMetadata(path, {'data_level': md['data_level'], 'configuration_id': md['configuration_id'] }) if not res['OK']: return res return DIRAC.S_OK( Transformation_path )
input_stage_files = [] if pipeline: xrd_keytab = op.getValue("Pipeline/XrdKey", None) if not xrd_keytab: gLogger.notice("*DEBUG* adding XrdKey file %s to input" % xrd_keytab) input_stage_files.append(xrd_keytab) if not opts.stagein is None: # we do add. input staging files = opts.stagein.split(",") for f in files: input_stage_files.append(f) if len(input_stage_files): if len(input_stage_files) == 1: j.setInputData(input_stage_files[-1]) else: j.setInputData(input_stage_files) if opts.debug: gLogger.notice( '*DEBUG* just showing the JDL of the job to be submitted') gLogger.notice(j._toJDL()) d = Dirac(True, "myRepo.rep") res = d.submit(j) if not res['OK']: gLogger.error("Error during Job Submission ", res['Message']) dexit(1) JobID = res['Value'] gLogger.notice("Your job %s (\"%s\") has been submitted." % (str(JobID), executable))
import DIRAC from DIRAC.Core.Base import Script Script.setUsageMessage('\n'.join([ __doc__.split('\n')[1], 'Usage:', ' %s [option|cfgfile] ... JobID ...' % Script.scriptName, 'Arguments:', ' JobID: DIRAC Job ID' ])) Script.parseCommandLine(ignoreErrors=True) args = Script.getPositionalArgs() if len(args) < 1: Script.showHelp() from DIRAC.Interfaces.API.Dirac import Dirac, parseArguments dirac = Dirac() exitCode = 0 errorList = [] for job in parseArguments(args): result = dirac.peekJob(job, printOutput=True) if not result['OK']: errorList.append((job, result['Message'])) exitCode = 2 for error in errorList: print "ERROR %s: %s" % error DIRAC.exit(exitCode)
def main(): Script.registerSwitch("e", "extended", "Get extended printout", setExtendedPrint) _, args = Script.parseCommandLine(ignoreErrors=True) from DIRAC import exit as DIRACExit from DIRAC.Interfaces.API.Dirac import Dirac from DIRAC.Interfaces.API.DiracAdmin import DiracAdmin diracAdmin = DiracAdmin() dirac = Dirac() exitCode = 0 errorList = [] for gridID in args: result = diracAdmin.getPilotInfo(gridID) if not result["OK"]: errorList.append((gridID, result["Message"])) exitCode = 2 else: res = result["Value"][gridID] if extendedPrint: tab = "" for key in [ "PilotJobReference", "Status", "OwnerDN", "OwnerGroup", "SubmissionTime", "DestinationSite", "GridSite", ]: if key in res: diracAdmin.log.notice("%s%s: %s" % (tab, key, res[key])) if not tab: tab = " " diracAdmin.log.notice("") for jobID in res["Jobs"]: tab = " " result = dirac.getJobAttributes(int(jobID)) if not result["OK"]: errorList.append((gridID, result["Message"])) exitCode = 2 else: job = result["Value"] diracAdmin.log.notice("%sJob ID: %s" % (tab, jobID)) tab += " " for key in [ "OwnerDN", "OwnerGroup", "JobName", "Status", "StartExecTime", "LastUpdateTime", "EndExecTime", ]: if key in job: diracAdmin.log.notice("%s%s:" % (tab, key), job[key]) diracAdmin.log.notice("") else: print(diracAdmin.pPrint.pformat({gridID: res})) for error in errorList: print("ERROR %s: %s" % error) DIRACExit(exitCode)