Exemple #1
0
def softinstall( args = None ) :

  from DIRAC.Interfaces.API.Dirac import Dirac
  from DIRAC.Interfaces.API.Job import Job

  if (len(args)!=2):
    Script.showHelp()

  version = args[0]
  site = args[1]

  if version not in ['prod-2_21122012','prod-2_08032013']:
    Script.gLogger.error('Version not valid')
    Script.showHelp()

  j = Job()

  j.setInputSandbox( ['cta-swinstall.py','SoftwareInstallation.py','CODE'] )   

  j.setExecutable('./cta-swinstall.py', version)

  j.setDestination([site])

  j.setName('SoftInstall')

  j.setCPUTime(100000)

  Script.gLogger.info( j._toJDL() )

  Dirac().submit( j )
Exemple #2
0
def cleanProd3sw(args=None):
    """ Simple wrapper to remove Software package
  """

    # get arguments
    package = args[0]
    version = args[1]
    site = args[2]

    # ##
    job = Job()
    dirac = Dirac()

    Step = job.setExecutable( './cta-cleansw.py',
                              arguments = '%s %s' % ( package, version ), \
                              logFile = 'cleanSoftware_Log.txt' )
    Step['Value']['name'] = 'Step_cleanSoftware'
    Step['Value']['descr_short'] = 'clean Software'

    # override for testing
    job.setName('CleanProd3Sw')

    # send job at Lyon CC
    job.setDestination([site])

    # run job

    # res = dirac.submit( job, "local" )
    res = dirac.submit(job)

    Script.gLogger.notice('Submission Result: ', res)

    return DIRAC.S_OK('Done')
    def createJob(self):
        job = Job()
        job.setName(self.__stepName)
        job.setOutputSandbox(['*log'])

        job.setExecutable('/usr/bin/wget',
                          arguments='"{0}/{1}"'.format(URL_ROOT,
                                                       self.__executable))
        job.setExecutable('/bin/chmod',
                          arguments='+x "{0}"'.format(self.__executable))

        arguments = '"{0}" "{1}" "{2}" "{3}" "{4}" "{5}" @{{JOB_ID}}'.format(
            self.__softwareVersion, self.__application, self.__outputPath,
            self.__outputPattern, self.__outputSE, self.__outputMode)
        if self.__extraArgs:
            arguments += ' ' + self.__extraArgs
        job.setExecutable(self.__executable, arguments=arguments)

        # failover for failed jobs
        job.setExecutable('/bin/ls -l',
                          modulesList=['Script', 'FailoverRequest'])

        if self.__inputData:
            job.setInputData(self.__inputData)

        if self.__site:
            job.setDestination(self.__site)

        if self.__bannedsite:
            job.setBannedSites(self.__bannedsite)

        job.setOutputSandbox(['app.out', 'app.err', 'Script3_CodeOutput.log'])

        self.__job = job
Exemple #4
0
def test_basicJob():
    job = Job()

    job.setOwner('ownerName')
    job.setOwnerGroup('ownerGroup')
    job.setName('jobName')
    job.setJobGroup('jobGroup')
    job.setExecutable('someExe')
    job.setType('jobType')
    job.setDestination('ANY')

    xml = job._toXML()

    try:
        with open('./DIRAC/Interfaces/API/test/testWF.xml') as fd:
            expected = fd.read()
    except IOError:
        with open('./Interfaces/API/test/testWF.xml') as fd:
            expected = fd.read()

    assert xml == expected

    try:
        with open('./DIRAC/Interfaces/API/test/testWFSIO.jdl') as fd:
            expected = fd.read()
    except IOError:
        with open('./Interfaces/API/test/testWFSIO.jdl') as fd:
            expected = fd.read()

    jdlSIO = job._toJDL(jobDescriptionObject=StringIO.StringIO(job._toXML()))
    assert jdlSIO == expected
Exemple #5
0
def test_basicJob():
  job = Job()

  job.setOwner('ownerName')
  job.setOwnerGroup('ownerGroup')
  job.setName('jobName')
  job.setJobGroup('jobGroup')
  job.setExecutable('someExe')
  job.setType('jobType')
  job.setDestination('ANY')

  xml = job._toXML()

  try:
    with open('./DIRAC/Interfaces/API/test/testWF.xml') as fd:
      expected = fd.read()
  except IOError:
    with open('./Interfaces/API/test/testWF.xml') as fd:
      expected = fd.read()

  assert xml == expected

  try:
    with open('./DIRAC/Interfaces/API/test/testWFSIO.jdl') as fd:
      expected = fd.read()
  except IOError:
    with open('./Interfaces/API/test/testWFSIO.jdl') as fd:
      expected = fd.read()

  jdlSIO = job._toJDL(jobDescriptionObject=StringIO.StringIO(job._toXML()))
  assert jdlSIO == expected
Exemple #6
0
    def __submit(self, site, CE, vo):
        """
      set the job and submit.
    """

        job = Job()
        job.setName(self.testType)
        job.setJobGroup('CE-Test')
        job.setExecutable(self.executable)
        job.setInputSandbox('%s/%s' % (self.__scriptPath, self.executable))
        if site and not CE:
            job.setDestination(site)
        if CE:
            job.setDestinationCE(CE)

        LOCK.acquire()
        proxyPath = BESUtils.getProxyByVO('zhangxm', vo)
        if not proxyPath['OK']:
            LOCK.release()
            return proxyPath
        proxyPath = proxyPath['Value']
        oldProxy = os.environ.get('X509_USER_PROXY')
        os.environ['X509_USER_PROXY'] = proxyPath
        result = self.dirac.submitJob(job)
        if oldProxy is None:
            del os.environ['X509_USER_PROXY']
        else:
            os.environ['X509_USER_PROXY'] = oldProxy
        LOCK.release()

        return result
Exemple #7
0
def SoftClean(args=None):

    from DIRAC.Interfaces.API.Dirac import Dirac
    from DIRAC.Interfaces.API.Job import Job

    if (len(args) != 3):
        Script.showHelp()

    package = args[0]
    version = args[1]
    site = args[2]

    j = Job()

    j.setInputSandbox(['cta-swclean.py'])

    arguments = package + ' ' + version
    j.setExecutable('./cta-swclean.py', arguments)

    j.setDestination([site])

    name = 'SoftClean_' + package + '_' + version
    j.setName('SoftClean')
    j.setJobGroup('SoftClean')

    j.setCPUTime(100000)

    Script.gLogger.info(j._toJDL())

    res = Dirac().submit(j)

    print res['Value']
Exemple #8
0
    def test_execute(self):

        job = Job()

        job.setName("helloWorld-test")
        job.setExecutable(find_all("helloWorld.py", '.', 'Integration')[0],
                          arguments="This is an argument",
                          logFile="aLogFileForTest.txt",
                          parameters=[('executable', 'string', '',
                                       "Executable Script"),
                                      ('arguments', 'string', '',
                                       'Arguments for executable Script'),
                                      ('applicationLog', 'string', '',
                                       "Log file name"),
                                      ('someCustomOne', 'string', '', "boh")],
                          paramValues=[('someCustomOne', 'aCustomValue')])
        job.setBannedSites(['LCG.SiteA.com', 'DIRAC.SiteB.org'])
        job.setOwner('ownerName')
        job.setOwnerGroup('ownerGroup')
        job.setName('jobName')
        job.setJobGroup('jobGroup')
        job.setType('jobType')
        job.setDestination('DIRAC.someSite.ch')
        job.setCPUTime(12345)
        job.setLogLevel('DEBUG')

        res = job.runLocal(self.d)
        self.assertTrue(res['OK'])
  def test_execute( self ):

    job = Job()

    job.setName( "helloWorld-test" )
    job.setExecutable( find_all( "helloWorld.py", '.', 'Integration' )[0],
                       arguments = "This is an argument",
                       logFile = "aLogFileForTest.txt" ,
                       parameters=[('executable', 'string', '', "Executable Script"),
                                   ('arguments', 'string', '', 'Arguments for executable Script'),
                                   ( 'applicationLog', 'string', '', "Log file name" ),
                                   ( 'someCustomOne', 'string', '', "boh" )],
                       paramValues = [( 'someCustomOne', 'aCustomValue' )] )
    job.setBannedSites( ['LCG.SiteA.com', 'DIRAC.SiteB.org'] )
    job.setOwner( 'ownerName' )
    job.setOwnerGroup( 'ownerGroup' )
    job.setName( 'jobName' )
    job.setJobGroup( 'jobGroup' )
    job.setType( 'jobType' )
    job.setDestination( 'DIRAC.someSite.ch' )
    job.setCPUTime( 12345 )
    job.setLogLevel( 'DEBUG' )

    res = job.runLocal( self.d )
    self.assertTrue( res['OK'] )
Exemple #10
0
  def __submit( self, site, CE, vo ):
    """
      set the job and submit.
    """

    job = Job()
    job.setName( self.testType )
    job.setJobGroup( 'CE-Test' )
    job.setExecutable( self.executable )
    job.setInputSandbox( '%s/%s' % ( self.__scriptPath, self.executable ) )
    if site and not CE:
      job.setDestination( site )
    if CE:
      job.setDestinationCE( CE )

    LOCK.acquire()
    proxyPath = BESUtils.getProxyByVO( 'zhangxm', vo )
    if not proxyPath[ 'OK' ]:
      LOCK.release()
      return proxyPath
    proxyPath = proxyPath[ 'Value' ]
    oldProxy = os.environ.get( 'X509_USER_PROXY' )
    os.environ[ 'X509_USER_PROXY' ] = proxyPath
    result = self.dirac.submit( job )
    if oldProxy is None:
      del os.environ[ 'X509_USER_PROXY' ]
    else:
      os.environ[ 'X509_USER_PROXY' ] = oldProxy
    LOCK.release()

    return result
  def prepareTransformationTasks(self,transBody,taskDict,owner='',ownerGroup=''):
    if (not owner) or (not ownerGroup):
      res = getProxyInfo(False,False)
      if not res['OK']:
        return res
      proxyInfo = res['Value']
      owner = proxyInfo['username']
      ownerGroup = proxyInfo['group']

    oJob = Job(transBody)
    for taskNumber in sortList(taskDict.keys()):
      paramsDict = taskDict[taskNumber]
      transID = paramsDict['TransformationID']
      self.log.verbose('Setting job owner:group to %s:%s' % (owner,ownerGroup))
      oJob.setOwner(owner)
      oJob.setOwnerGroup(ownerGroup)
      transGroup = str(transID).zfill(8)
      self.log.verbose('Adding default transformation group of %s' % (transGroup))
      oJob.setJobGroup(transGroup)
      constructedName = str(transID).zfill(8)+'_'+str(taskNumber).zfill(8)
      self.log.verbose('Setting task name to %s' % constructedName)
      oJob.setName(constructedName)
      oJob._setParamValue('PRODUCTION_ID',str(transID).zfill(8))
      oJob._setParamValue('JOB_ID',str(taskNumber).zfill(8))
      inputData = None
      for paramName,paramValue in paramsDict.items():
        self.log.verbose('TransID: %s, TaskID: %s, ParamName: %s, ParamValue: %s' %(transID,taskNumber,paramName,paramValue))
        if paramName=='InputData':
          if paramValue:
            self.log.verbose('Setting input data to %s' %paramValue)
            oJob.setInputData(paramValue)
        elif paramName=='Site':
          if paramValue:
            self.log.verbose('Setting allocated site to: %s' %(paramValue))
            oJob.setDestination(paramValue)
        elif paramValue:
          self.log.verbose('Setting %s to %s' % (paramName,paramValue))
          oJob._addJDLParameter(paramName,paramValue)

      hospitalTrans = [int(x) for x in gConfig.getValue("/Operations/Hospital/Transformations",[])]
      if int(transID) in hospitalTrans:
        hospitalSite = gConfig.getValue("/Operations/Hospital/HospitalSite",'DIRAC.JobDebugger.ch')
        hospitalCEs = gConfig.getValue("/Operations/Hospital/HospitalCEs",[])
        oJob.setType('Hospital')
        oJob.setDestination(hospitalSite)
        oJob.setInputDataPolicy('download',dataScheduling=False)
        if hospitalCEs:
          oJob._addJDLParameter('GridRequiredCEs',hospitalCEs)        
      taskDict[taskNumber]['TaskObject'] = '' 
      res = self.getOutputData({'Job':oJob._toXML(),'TransformationID':transID,'TaskID':taskNumber,'InputData':inputData})
      if not res ['OK']:
        self.log.error("Failed to generate output data",res['Message'])
        continue
      for name,output in res['Value'].items():
        oJob._addJDLParameter(name,string.join(output,';'))
      taskDict[taskNumber]['TaskObject'] = Job(oJob._toXML())
    return S_OK(taskDict)
Exemple #12
0
def base():
  job = Job()
  job.setName("helloWorld-TEST-TO-Jenkins")
  job.setInputSandbox([find_all('exe-script.py', '..', '/DIRAC/tests/Workflow/')[0]])
  job.setExecutable("exe-script.py", "", "helloWorld.log")
  job.setCPUTime(1780)
  job.setDestination('DIRAC.Jenkins.ch')
  job.setLogLevel('DEBUG')
  return job
Exemple #13
0
  def prepareTransformationTasks(self,transBody,taskDict,owner='',ownerGroup=''):
    if (not owner) or (not ownerGroup):
      res = getProxyInfo(False,False)
      if not res['OK']:
        return res
      proxyInfo = res['Value']
      owner = proxyInfo['username']
      ownerGroup = proxyInfo['group']

    oJob = Job(transBody)
    for taskNumber in sortList(taskDict.keys()):
      paramsDict = taskDict[taskNumber]
      transID = paramsDict['TransformationID']
      self.log.verbose('Setting job owner:group to %s:%s' % (owner,ownerGroup))
      oJob.setOwner(owner)
      oJob.setOwnerGroup(ownerGroup)
      transGroup = str(transID).zfill(8)
      self.log.verbose('Adding default transformation group of %s' % (transGroup))
      oJob.setJobGroup(transGroup)
      constructedName = str(transID).zfill(8)+'_'+str(taskNumber).zfill(8)
      self.log.verbose('Setting task name to %s' % constructedName)
      oJob.setName(constructedName)
      oJob._setParamValue('PRODUCTION_ID',str(transID).zfill(8))
      oJob._setParamValue('JOB_ID',str(taskNumber).zfill(8))
      inputData = None
      for paramName,paramValue in paramsDict.items():
        self.log.verbose('TransID: %s, TaskID: %s, ParamName: %s, ParamValue: %s' %(transID,taskNumber,paramName,paramValue))
        if paramName=='InputData':
          if paramValue:
            self.log.verbose('Setting input data to %s' %paramValue)
            oJob.setInputData(paramValue)
        elif paramName=='Site':
          if paramValue:
            self.log.verbose('Setting allocated site to: %s' %(paramValue))
            oJob.setDestination(paramValue)
        elif paramValue:
          self.log.verbose('Setting %s to %s' % (paramName,paramValue))
          oJob._addJDLParameter(paramName,paramValue)

      hospitalTrans = [int(x) for x in gConfig.getValue("/Operations/Hospital/Transformations",[])]
      if int(transID) in hospitalTrans:
        hospitalSite = gConfig.getValue("/Operations/Hospital/HospitalSite",'DIRAC.JobDebugger.ch')
        hospitalCEs = gConfig.getValue("/Operations/Hospital/HospitalCEs",[])
        oJob.setType('Hospital')
        oJob.setDestination(hospitalSite)
        oJob.setInputDataPolicy('download',dataScheduling=False)
        if hospitalCEs:
          oJob._addJDLParameter('GridRequiredCEs',hospitalCEs)        
      taskDict[taskNumber]['TaskObject'] = '' 
      res = self.getOutputData({'Job':oJob._toXML(),'TransformationID':transID,'TaskID':taskNumber,'InputData':inputData})
      if not res ['OK']:
        self.log.error("Failed to generate output data",res['Message'])
        continue
      for name,output in res['Value'].items():
        oJob._addJDLParameter(name,string.join(output,';'))
      taskDict[taskNumber]['TaskObject'] = Job(oJob._toXML())
    return S_OK(taskDict)
Exemple #14
0
def submit(name,
           job_group,
           task_id,
           input_sandbox,
           output_sandbox,
           executable,
           site=None,
           banned_site=None,
           sub_ids=[]):
    dirac = Dirac()

    submit_result = {'backend_job_ids': {}}
    jobInfos = {}

    for run in range(int((len(sub_ids) + 99) / 100)):
        ids_this_run = [x for x in sub_ids[run * 100:(run + 1) * 100]]
        job_names = ['%s.%s' % (name, sub_id) for sub_id in ids_this_run]
        j = Job()
        j.setName(name)
        j.setExecutable(executable)

        j.setParameterSequence('JobName', job_names, addToWorkflow=True)
        j.setParameterSequence('arguments', ids_this_run, addToWorkflow=True)

        if input_sandbox:
            j.setInputSandbox(input_sandbox)
        if output_sandbox:
            j.setOutputSandbox(output_sandbox)

        if job_group:
            j.setJobGroup(job_group)
        if site:  # set destination to a certain site; list not allowed
            j.setDestination(site)

        if banned_site:
            j.setBannedSites(banned_site)

        result = dirac.submitJob(j)

        if not result['OK']:
            sys.stdout.write('DIRAC job submit error: %s\n' %
                             result['Message'])
            sys.exit(1)

        for sub_id, dirac_id in zip(ids_this_run, result['Value']):
            submit_result['backend_job_ids'][sub_id] = dirac_id
            jobInfos[dirac_id] = {'SubID': sub_id}

    #Register on Task-manager Webapp of IHEPDIRAC
    task = RPCClient('WorkloadManagement/TaskManager')
    taskInfo = {'TaskName': name, 'JobGroup': job_group, 'JSUB-ID': task_id}
    task_result = task.createTask(name, taskInfo, jobInfos)
    task_web_id = task_result['Value']
    submit_result['backend_task_id'] = task_web_id

    return submit_result
Exemple #15
0
def base():
    job = Job()
    job.setName("helloWorld-TEST-TO-Jenkins")
    executablePath = find_all("exe-script.py", "..",
                              "/DIRAC/tests/Workflow/")[0]
    job.setInputSandbox([executablePath])
    job.setExecutable(executablePath, "", "helloWorld.log")
    job.setCPUTime(1780)
    job.setDestination("DIRAC.Jenkins.ch")
    job.setLogLevel("DEBUG")
    return job
Exemple #16
0
def softinstall(args=None):

    from DIRAC.Interfaces.API.Dirac import Dirac
    from DIRAC.Interfaces.API.Job import Job

    if (len(args) < 2):
        Script.showHelp()

    version = args[0]
    site = args[1]

    if version not in ['18122013', 'v0r7p0']:
        Script.gLogger.error('Version not valid')
        Script.showHelp()

    j = Job()

    j.setInputSandbox(['cta-ctools-install.py', 'SoftwareInstallation.py'])

    j.setExecutable('./cta-ctools-install.py', version)

    j.setDestination([site])

    j.setName('ctoolsInstall')

    j.setCPUTime(100000)

    Script.gLogger.info(j._toJDL())

    if site in ['LCG.GRIF.fr', 'LCG.M3PEC.fr']:
        if site == 'LCG.GRIF.fr':
            ceList = [
                'apcce02.in2p3.fr', 'grid36.lal.in2p3.fr',
                'lpnhe-cream.in2p3.fr', 'llrcream.in2p3.fr',
                'node74.datagrid.cea.fr'
            ]
        elif site == 'LCG.M3PEC.fr':
            #      ceList = ['ce0.bordeaux.inra.fr','ce0.m3pec.u-bordeaux1.fr']
            ceList = ['ce0.bordeaux.inra.fr']
        for ce in ceList:
            j.setDestinationCE(ce)
            name = 'ctoolsInstall' + '_' + ce
            j.setName(name)
            res = Dirac().submit(j)
            print res
        DIRAC.exit()
    else:
        name = 'ctoolsInstall'

    j.setName(name)
    res = Dirac().submit(j)
    print res
 def _submitJob(self, result_id, executable, test_name, site_name):
     executable = executable.split('&')
     j = Job()
     j.setExecutable('python', arguments=executable[0] + " " + str(result_id))
     sandBox = []
     for file_name in executable:
         sandBox.append(SAM_TEST_DIR + file_name)
     j.setInputSandbox(sandBox)
     j.setName(test_name)
     j.setJobGroup('sam_test')
     j.setDestination(site_name)
     result = self.dirac.submit(j)
     return result
    def do_installonsite(self, argss):
        """ Install a release on a grid site : 
            installonsite tag site
        """
        args = argss.split()
        if len(args) < 2:
            print self.do_installonsite.__doc__
            return
        tag = args[0]
        site = args[1]

        #print "Check if the software with the tag '"+tag+"' exists on the rsync server..."
        #res = self.client.getSitesForTag(tag)
        #if not res['OK']:
        #print res['Message']
        #return
        #print "tag found !"

        from DIRAC.Interfaces.API.Dirac import Dirac
        d = Dirac()
        from DIRAC.Interfaces.API.Job import Job

        from DIRAC.ConfigurationSystem.Client.Helpers.Operations import Operations
        import os

        ops = Operations()
        scriptname = "InstallSoftware.py"
        j = Job()
        j.setDestination(site)
        j.setCPUTime(1000)
        j.setName("Installation " + tag)
        j.setExecutable(os.environ['DIRAC'] +
                        "/GlastDIRAC/ResourceStatusSystem/Client/externals/" +
                        scriptname,
                        logFile='SoftwareInstallation.log')
        j.setOutputSandbox('*.log')
        res = d.submit(j)
        if not res['OK']:
            print "Could not submit the installation at site %s, message %s" % (
                site, res['Message'])
            return

        print "Job submitted, id = " + str(res['Value'])

        print "Add tag :"
        res = self.client.addTagAtSite(tag, site)
        if not res['OK']:
            print "Could not register tag %s at site %s, message %s" % (
                tag, site, res['Message'])
            return
        print "Added %s to %i CEs" % (tag, len(res['Value'][tag]))
    def do_installonsite(self,argss):
        """ Install a release on a grid site : 
            installonsite tag site
        """
        args = argss.split()
        if len(args)<2:
            print self.do_installonsite.__doc__
            return
        tag = args[0]
        site = args[1]
        
        #print "Check if the software with the tag '"+tag+"' exists on the rsync server..." 
        #res = self.client.getSitesForTag(tag)
        #if not res['OK']:
          #print res['Message']
          #return 
        #print "tag found !"  

        from DIRAC.Interfaces.API.Dirac import Dirac
        d = Dirac()
        from DIRAC.Interfaces.API.Job import Job
        
        from DIRAC.ConfigurationSystem.Client.Helpers.Operations import Operations
        import os
        
        ops = Operations()
        scriptname = "InstallSoftware.py"
        j = Job()
        j.setDestination(site)
        j.setCPUTime(1000)
        j.setName("Installation "+tag)
        j.setExecutable(os.environ['DIRAC']+"/GlastDIRAC/ResourceStatusSystem/Client/"+scriptname , logFile='SoftwareInstallation.log')
        j.setOutputSandbox('*.log')
        res = d.submit(j)
        if not res['OK']:
          print "Could not submit the installation at site %s, message %s"%(site,res['Message'])
          return
        
        print "Job submitted, id = "+str(res['Value'])
        
        print "Add tag :"
        res = self.client.addTagAtSite(tag,site)
        if not res['OK']:
            print "Could not register tag %s at site %s, message %s"%(tag,site,res['Message'])
            return
        print "Added %s to %i CEs"%(tag,len(res['Value'][tag]))
Exemple #20
0
def CorsikaSimtelInstall(args=None):

    from DIRAC.Interfaces.API.Dirac import Dirac
    from DIRAC.Interfaces.API.Job import Job

    if (len(args) != 2):
        Script.showHelp()

    version = args[0]
    site = args[1]

    if version not in ['prod-2_13112014']:
        Script.gLogger.error('Version not valid')
        Script.showHelp()

    j = Job()
    CorsikaSimtelPack = os.path.join('corsika_simhessarray', version,
                                     'corsika_simhessarray')
    CorsikaSimtelLFN = 'LFN:' + os.path.join('/vo.cta.in2p3.fr/software',
                                             CorsikaSimtelPack) + '.tar.gz'
    j.setInputSandbox(['cta-corsikasimtel-install.py', CorsikaSimtelLFN])
    j.setExecutable('./cta-corsikasimtel-install.py', version)
    j.setDestination([site])
    j.setJobGroup('SoftInstall')
    j.setCPUTime(100000)

    if site in ['LCG.GRIF.fr', 'LCG.M3PEC.fr']:
        if site == 'LCG.GRIF.fr':
            ceList = [
                'apcce02.in2p3.fr', 'grid36.lal.in2p3.fr',
                'lpnhe-cream.in2p3.fr', 'llrcream.in2p3.fr',
                'node74.datagrid.cea.fr'
            ]
        if site == 'LCG.M3PEC.fr':
            ceList = ['ce0.bordeaux.inra.fr', 'ce0.m3pec.u-bordeaux1.fr']

        for ce in ceList:
            j.setDestinationCE(ce)
            name = 'corsikasimtelInstall' + '_' + ce
            j.setName(name)
            Dirac().submit(j)
        DIRAC.exit()

    j.setName('corsikasimtelInstall')
    Script.gLogger.info(j._toJDL())
    Dirac().submit(j)
Exemple #21
0
def submitJob(jobPara):
    dirac = Dirac()
    j = Job()
    j.setName(jobPara['jobName'])
    j.setJobGroup(jobPara['jobGroup'])
    j.setExecutable(jobPara['jobScript'], logFile = jobPara['jobScriptLog'])
    j.setInputSandbox(jobPara['inputSandbox'])
    j.setOutputSandbox(jobPara['outputSandbox'])
    j.setOutputData(jobPara['outputData'], jobPara['SE'])
    j.setDestination(jobPara['sites'])
    j.setCPUTime(jobPara['CPUTime'])
    result = dirac.submit(j)
    if result['OK']:
        print 'Job %s submitted successfully. ID = %d' %(jobPara['jobName'],result['Value'])
    else:
        print 'Job %s submitted failed' %jobPara['jobName']
    return result
Exemple #22
0
def submitJob(jobPara):
    dirac = Dirac()
    j = Job()
    j.setName(jobPara['jobName'])
    j.setJobGroup(jobPara['jobGroup'])
    j.setExecutable(jobPara['jobScript'], logFile=jobPara['jobScriptLog'])
    j.setInputSandbox(jobPara['inputSandbox'])
    j.setOutputSandbox(jobPara['outputSandbox'])
    j.setOutputData(jobPara['outputData'], jobPara['SE'])
    j.setDestination(jobPara['sites'])
    j.setCPUTime(jobPara['CPUTime'])
    result = dirac.submit(j)
    if result['OK']:
        print 'Job %s submitted successfully. ID = %d' % (jobPara['jobName'],
                                                          result['Value'])
    else:
        print 'Job %s submitted failed' % jobPara['jobName']
    return result
Exemple #23
0
 def basicTest(self):
     j = Job()
     j.setCPUTime(50000)
     j.setExecutable(
         '/Users/stuart/dirac/workspace/DIRAC3/DIRAC/Interfaces/API/test/myPythonScript.py'
     )
     # j.setExecutable('/bin/echo hello')
     j.setOwner('paterson')
     j.setType('test')
     j.setName('MyJobName')
     #j.setAncestorDepth(1)
     j.setInputSandbox([
         '/Users/stuart/dirac/workspace/DIRAC3/DIRAC/Interfaces/API/test/DV.opts',
         '/Users/stuart/dirac/workspace/DIRAC3/DIRAC/Interfaces/API/test/DV2.opts'
     ])
     j.setOutputSandbox(['firstfile.txt', 'anotherfile.root'])
     j.setInputData([
         '/lhcb/production/DC04/v2/DST/00000742_00003493_11.dst',
         '/lhcb/production/DC04/v2/DST/00000742_00003493_10.dst'
     ])
     j.setOutputData(['my.dst', 'myfile.log'])
     j.setDestination('LCG.CERN.ch')
     j.setPlatform('LCG')
     j.setSystemConfig('x86_64-slc5-gcc43-opt')
     j.setSoftwareTags(['VO-lhcb-Brunel-v30r17', 'VO-lhcb-Boole-v12r10'])
     #print j._toJDL()
     #print j.printObj()
     xml = j._toXML()
     testFile = 'jobDescription.xml'
     if os.path.exists(testFile):
         os.remove(testFile)
     xmlfile = open(testFile, 'w')
     xmlfile.write(xml)
     xmlfile.close()
     print '>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>Creating code for the workflow'
     print j.createCode()
     print '>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>Executing the workflow'
     j.execute()
     print '>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>Trying to run the same workflow from generated XML file'
     workflow = fromXMLFile(testFile)
     code = workflow.createCode()
     print code
     workflow.execute()
Exemple #24
0
    def submit(self, param):        
        j = Job()
        j.setName(param['jobName'])
        j.setExecutable(param['jobScript'],logFile = param['jobScriptLog'])
        if self.site:
            j.setDestination(self.site)
        if self.jobGroup:
            j.setJobGroup(self.jobGroup)            
        j.setInputSandbox(param['inputSandbox'])
        j.setOutputSandbox(param['outputSandbox'])
        j.setOutputData(param['outputData'], outputSE = self.outputSE, outputPath = self.outputPath)

        dirac = GridDirac()
        result = dirac.submit(j)

        status = {}
        status['submit'] = result['OK']
        if status['submit']:
            status['job_id'] = result['Value']

        return status
Exemple #25
0
    def test_execute(self):

        job = Job()
        job._siteSet = {"DIRAC.someSite.ch"}

        job.setName("helloWorld-test")
        job.setExecutable(
            self.helloWorld,
            arguments="This is an argument",
            logFile="aLogFileForTest.txt",
            parameters=[
                ("executable", "string", "", "Executable Script"),
                ("arguments", "string", "", "Arguments for executable Script"),
                ("applicationLog", "string", "", "Log file name"),
                ("someCustomOne", "string", "", "boh"),
            ],
            paramValues=[("someCustomOne", "aCustomValue")],
        )
        job.setBannedSites(["LCG.SiteA.com", "DIRAC.SiteB.org"])
        job.setOwner("ownerName")
        job.setOwnerGroup("ownerGroup")
        job.setName("jobName")
        job.setJobGroup("jobGroup")
        job.setType("jobType")
        job.setDestination("DIRAC.someSite.ch")
        job.setCPUTime(12345)
        job.setLogLevel("DEBUG")
        try:
            # This is the standard location in Jenkins
            job.setInputSandbox(
                find_all("pilot.cfg",
                         os.environ["WORKSPACE"] + "/PilotInstallDIR")[0])
        except (IndexError, KeyError):
            job.setInputSandbox(find_all("pilot.cfg", rootPath)[0])
        job.setConfigArgs("pilot.cfg")

        res = job.runLocal(self.d)
        self.assertTrue(res["OK"])
Exemple #26
0
def test_basicJob():
    job = Job()

    job.setOwner("ownerName")
    job.setOwnerGroup("ownerGroup")
    job.setName("jobName")
    job.setJobGroup("jobGroup")
    job.setExecutable("someExe")
    job.setType("jobType")
    job.setDestination("ANY")

    xml = job._toXML()

    with open(join(dirname(__file__), "testWF.xml")) as fd:
        expected = fd.read()

    assert xml == expected

    with open(join(dirname(__file__), "testWFSIO.jdl")) as fd:
        expected = fd.read()

    jdlSIO = job._toJDL(jobDescriptionObject=StringIO(job._toXML()))
    assert jdlSIO == expected
Exemple #27
0
    def submit(self, param):
        j = Job()
        j.setName(param['jobName'])
        j.setExecutable(param['jobScript'], logFile=param['jobScriptLog'])
        if self.site:
            j.setDestination(self.site)
        if self.jobGroup:
            j.setJobGroup(self.jobGroup)
        j.setInputSandbox(param['inputSandbox'])
        j.setOutputSandbox(param['outputSandbox'])
        j.setOutputData(param['outputData'],
                        outputSE=self.outputSE,
                        outputPath=self.outputPath)

        dirac = GridDirac()
        result = dirac.submit(j)

        status = {}
        status['submit'] = result['OK']
        if status['submit']:
            status['job_id'] = result['Value']

        return status
Exemple #28
0
    def test_execute(self):

        job = Job()
        job._siteSet = {'DIRAC.someSite.ch'}

        job.setName("helloWorld-test")
        job.setExecutable(self.helloWorld,
                          arguments="This is an argument",
                          logFile="aLogFileForTest.txt",
                          parameters=[('executable', 'string', '',
                                       "Executable Script"),
                                      ('arguments', 'string', '',
                                       'Arguments for executable Script'),
                                      ('applicationLog', 'string', '',
                                       "Log file name"),
                                      ('someCustomOne', 'string', '', "boh")],
                          paramValues=[('someCustomOne', 'aCustomValue')])
        job.setBannedSites(['LCG.SiteA.com', 'DIRAC.SiteB.org'])
        job.setOwner('ownerName')
        job.setOwnerGroup('ownerGroup')
        job.setName('jobName')
        job.setJobGroup('jobGroup')
        job.setType('jobType')
        job.setDestination('DIRAC.someSite.ch')
        job.setCPUTime(12345)
        job.setLogLevel('DEBUG')
        try:
            # This is the standard location in Jenkins
            job.setInputSandbox(
                find_all('pilot.cfg',
                         os.environ['WORKSPACE'] + '/PilotInstallDIR')[0])
        except (IndexError, KeyError):
            job.setInputSandbox(find_all('pilot.cfg', rootPath)[0])
        job.setConfigArgs('pilot.cfg')

        res = job.runLocal(self.d)
        self.assertTrue(res['OK'])
Exemple #29
0
 def basicTest(self):
   j = Job()
   j.setCPUTime(50000)
   j.setExecutable('/Users/stuart/dirac/workspace/DIRAC3/DIRAC/Interfaces/API/test/myPythonScript.py')
  # j.setExecutable('/bin/echo hello')
   j.setOwner('paterson')
   j.setType('test')
   j.setName('MyJobName')
   #j.setAncestorDepth(1)
   j.setInputSandbox(['/Users/stuart/dirac/workspace/DIRAC3/DIRAC/Interfaces/API/test/DV.opts','/Users/stuart/dirac/workspace/DIRAC3/DIRAC/Interfaces/API/test/DV2.opts'])
   j.setOutputSandbox(['firstfile.txt','anotherfile.root'])
   j.setInputData(['/lhcb/production/DC04/v2/DST/00000742_00003493_11.dst',
                   '/lhcb/production/DC04/v2/DST/00000742_00003493_10.dst'])
   j.setOutputData(['my.dst','myfile.log'])
   j.setDestination('LCG.CERN.ch')
   j.setPlatform('LCG')
   j.setSystemConfig('x86_64-slc5-gcc43-opt')
   j.setSoftwareTags(['VO-lhcb-Brunel-v30r17','VO-lhcb-Boole-v12r10'])
   #print j._toJDL()
   #print j.printObj()
   xml = j._toXML()
   testFile = 'jobDescription.xml'
   if os.path.exists(testFile):
     os.remove(testFile)
   xmlfile = open(testFile,'w')
   xmlfile.write(xml)
   xmlfile.close()
   print '>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>Creating code for the workflow'
   print j.createCode()
   print '>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>Executing the workflow'
   j.execute()
   print '>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>Trying to run the same workflow from generated XML file'
   workflow = fromXMLFile(testFile)
   code = workflow.createCode()
   print code
   workflow.execute()
            input_sandbox_files.append(executable)
        j.setExecutable(str(executable))
    else:
        gLogger.error("No executable defined.")
        dexit(1)
        
    j.setName("MC job")
    if not opts.name is None:
        j.setName(opts.name)

    j.setInputSandbox(input_sandbox_files) # all input files in the sandbox
    j.setOutputSandbox(output_sandbox_files)

    j.setCPUTime(opts.cpu)
    if not opts.site is None:
        j.setDestination(opts.site.split(","))#can also be a list
        
    if not opts.bannedSites is None:
        j.setBannedSites(opts.bannedSites.split(","))

    if not opts.release is None:
        tag = opts.release
        cl = SoftwareTagClient()
        result = cl.getSitesForTag(tag,'Valid') # keyword doesn't work there.
        if not result['OK']:
            gLogger.error("*ERROR* Could not get sites for Tag %s"%tag,result['Message'])
            dexit(1)
        sites = result[ 'Value' ]
        j.setDestination(sites)

    if not opts.stagein is None:
    j.setInputSandbox(file_dict.keys())

    #...and added to the desired storage element with the corresponding
    # LFN via the job's OutputData. You may wish to change:
    # * The Storage Element - by changing the outputSE parameter;
    # * The LFN base name   - by changing the outputPath parameter.
    j.setOutputData(file_dict.keys(), \
                    outputSE='%s' % (se), \
                    outputPath='/%s/' % (gridoutdir)\
                   )

    # These are the files retrieved with the local job output.
    j.setOutputSandbox(['StdOut', 'StdErr'])

    # You can set your preferred site here.
    j.setDestination(sitename)

    ## The DIRAC instance.
    dirac = Dirac()

#    # Submit the job and print the result.
#    result = dirac.submit(j)
#    print 'Submission result: ', result

    # Delete the (temporary) data files.
    for fn in file_dict.keys():
        os.remove(fn)

    ## The dataset name (chip ID + start time).
    dn = sorted(file_dict.keys())[0][:-4]
def main(dataset, chunksize, test):
    '''
    The DATASET argument is a list of paths to MC files on the grid. Like the output of
    cta-prod3-dump-dataset for example. See also
    https://forge.in2p3.fr/projects/cta_dirac/wiki/CTA-DIRAC_MC_PROD3_Status

    Keep in mind that for some effing reason this needs to be executed within this weird 'dirac'
    environment which comes with its own glibc, python and pip. I guess the real Mr. Dirac would turn in his grave.

    '''
    dirac = Dirac()

    with open(dataset) as f:
        simtel_files = f.readlines()
        print('Analysing {}'.format(len(simtel_files)))

    server_list = [
        "TORINO-USER", "CYF-STORM-USER", "CYF-STORM-Disk", "M3PEC-Disk",
        "OBSPM-Disk", "POLGRID-Disk", "FRASCATI-USER", "LAL-Disk",
        "CIEMAT-Disk", "CIEMAT-USER", "CPPM-Disk", "LAL-USER", "CYFRONET-Disk",
        "DESY-ZN-USER", "M3PEC-USER", "LPNHE-Disk", "LPNHE-USER", "LAPP-USER",
        "LAPP-Disk"
    ]
    desy_server = 'DESY-ZN-USER'

    servers_with_miniconda = [
        'LCG.IN2P3-CC.fr', 'LCG.DESY-ZEUTHEN.de', 'LCG.CNAF.it', 'LCG.GRIF.fr',
        'LCG.CYFRONET.pl', 'LCG.Prague.cz', 'LCG.CIEMAT.es'
    ]

    chunks = np.array_split(sorted(simtel_files),
                            int(len(simtel_files) / chunksize))

    print('Got a total of {} chunks'.format(len(chunks)))
    for c, simtel_filenames in tqdm(enumerate(
            chunks[0:2])):  # send just 2 jobs for now.
        # convert chunk to a list of strings. becasue this dirac thing cant take numpy arrays
        simtel_filenames = [
            str(s).strip() for s in simtel_filenames if 'SCT' not in s
        ]
        print('Starting processing for chunk {}'.format(c))
        print(simtel_filenames)
        j = Job()
        # set runtime to 0.5h
        j.setCPUTime(30 * 60)
        j.setName('cta_preprocessing_{}'.format(c))
        j.setInputData(simtel_filenames)
        j.setOutputData(['./processing_output/*.hdf5'],
                        outputSE=None,
                        outputPath='cta_preprocessing/')

        j.setInputSandbox(
            ['../process_simtel.py', './install_dependencies.py'])
        j.setOutputSandbox(['cta_preprocessing.log'])
        j.setExecutable('./job_script.sh')
        # These servers seem to  have mini conda installed
        # destination = np.random.choice(servers_with_miniconda)
        j.setDestination(servers_with_miniconda)

        value = dirac.submit(j)
        print('Number {} Submission Result: {}'.format(c, value))
Exemple #33
0
        inputdata_list.append(lfn + 'second/HPC_data.tar.gz.' + str(i))
#SitesList = ['LCG.UKI-NORTHGRID-MAN-HEP.uk', 'LCG.UKI-LT2-IC-HEP.uk', 'LCG.UKI-LT2-QMUL.uk', 'LCG.UKI-NORTHGRID-LANCS-HEP.uk']
#SEList = ['UKI-NORTHGRID-MAN-HEP-disk', 'UKI-LT2-IC-HEP-disk', 'UKI-LT2-QMUL2-disk', 'UKI-NORTHGRID-LANCS-HEP-disk']
SitesList = ['LCG.UKI-NORTHGRID-MAN-HEP.uk']
SEList = ['UKI-NORTHGRID-MAN-HEP-disk']
print("Input data list:\n")
print(inputdata_list)
for i in range(0, total_pixels, chunks):
    id_start = i
    id_end = i + chunks
    dirac = Dirac()
    j.setName('CS Faraday Rotation Measurement Reconstruction - Pixels from ' +
              str(id_start) + ' to ' + str(id_end - 1))
    j.setPlatform('EL7')
    j.setTag([str(nprocs) + 'Processors'])
    j.setDestination(SitesList)
    j.setExecutable('RMSynthesis2.sh',
                    arguments=str(nprocs) + ' ' + str(id_start) + ' ' +
                    str(id_end) + ' ' + str(expmnt))
    # Input data
    j.setInputData(inputdata_list)
    j.setInputSandbox([
        'RMSynthesis2.sh', 'run2.sh',
        'prmon_1.0.1_x86_64-static-gnu72-opt.tar.gz'
    ])
    # Output data
    j.setOutputSandbox([
        'StdOut', 'StdErr',
        'outputtxt_' + str(id_start) + '_' + str(id_end - 1) + '.txt',
        'prmon' + str(id_start) + '_' + str(id_end - 1) + '.txt'
    ])
Exemple #34
0
#from tests.Workflow.Integration.Test_UserJobs import createJob

gLogger.setLevel( 'DEBUG' )

cwd = os.path.realpath( '.' )

dirac = Dirac()

# Simple Hello Word job to DIRAC.Jenkins.ch
gLogger.info( "\n Submitting hello world job targeting DIRAC.Jenkins.ch" )
helloJ = Job()
helloJ.setName( "helloWorld-TEST-TO-Jenkins" )
helloJ.setInputSandbox( [find_all( 'exe-script.py', '..', '/DIRAC/tests/Workflow/' )[0]] )
helloJ.setExecutable( "exe-script.py", "", "helloWorld.log" )
helloJ.setCPUTime( 17800 )
helloJ.setDestination( 'DIRAC.Jenkins.ch' )
result = dirac.submit( helloJ )
gLogger.info( "Hello world job: ", result )
if not result['OK']:
  gLogger.error( "Problem submitting job", result['Message'] )
  exit( 1 )

# Simple Hello Word job to DIRAC.Jenkins.ch, that needs to be matched by a MP WN
gLogger.info( "\n Submitting hello world job targeting DIRAC.Jenkins.ch and a MP WN" )
helloJMP = Job()
helloJMP.setName( "helloWorld-TEST-TO-Jenkins-MP" )
helloJMP.setInputSandbox( [find_all( 'exe-script.py', '..', '/DIRAC/tests/Workflow/' )[0]] )
helloJMP.setExecutable( "exe-script.py", "", "helloWorld.log" )
helloJMP.setCPUTime( 17800 )
helloJMP.setDestination( 'DIRAC.Jenkins.ch' )
helloJMP.setTag('MultiProcessor')
Exemple #35
0
#from tests.Workflow.Integration.Test_UserJobs import createJob

gLogger.setLevel('DEBUG')

cwd = os.path.realpath('.')

dirac = Dirac()

# Simple Hello Word job to DIRAC.Jenkins.ch
gLogger.info("\n Submitting hello world job targeting DIRAC.Jenkins.ch")
helloJ = Job()
helloJ.setName("helloWorld-TEST-TO-Jenkins")
helloJ.setInputSandbox([find_all('exe-script.py', '..', '/DIRAC/tests/Workflow/')[0]])
helloJ.setExecutable("exe-script.py", "", "helloWorld.log")
helloJ.setCPUTime(1780)
helloJ.setDestination('DIRAC.Jenkins.ch')
helloJ.setLogLevel('DEBUG')
result = dirac.submitJob(helloJ)
gLogger.info("Hello world job: ", result)
if not result['OK']:
  gLogger.error("Problem submitting job", result['Message'])
  exit(1)

# Simple Hello Word job to DIRAC.Jenkins.ch, that needs to be matched by a MP WN
gLogger.info("\n Submitting hello world job targeting DIRAC.Jenkins.ch and a MP WN")
helloJMP = Job()
helloJMP.setName("helloWorld-TEST-TO-Jenkins-MP")
helloJMP.setInputSandbox([find_all('exe-script.py', '..', '/DIRAC/tests/Workflow/')[0]])
helloJMP.setExecutable("exe-script.py", "", "helloWorld.log")
helloJMP.setCPUTime(1780)
helloJMP.setDestination('DIRAC.Jenkins.ch')
    else:
##        base_output_path = PWD
        j.setOutputData(output_files, outputSE=se, outputPath=outdir)
        print 'Output files = ', output_files
        print 'outputPath = ', outdir

#####################
##   PREPARE JOB   ##
#####################
    if (DO_NOT_SUBMIT):
        sys.exit(os.EX_USAGE)

    ### ALWAYS, INFO, VERBOSE, WARN, DEBUG
    j.setLogLevel('debug')

    j.setDestination(site_dirac)

    JOB_IDX = first_job + 1 + idx
    JOB_NAME = PROD_NAME + " IDX_" + str(JOB_IDX)
    print '\nJOB NAME is : ', JOB_NAME

    j.setName(JOB_NAME)
    j.setCPUTime(JOB_CPUTIME) ## 4 days

    run_corsika_sim_args = input_file_base + " " + corsika_version + " " + corsika_bin
    j.setExecutable( './run_corsika_sim', arguments = run_corsika_sim_args, logFile='run_sim.log')

    if (TEST_JOB) :
        jobID = dirac.submit(j,mode='local')
    else :
        jobID = dirac.submit(j)
Exemple #37
0
            print("test run -- only submitting one job")
            exit()


try:
    os.remove("datapipe.tar.gz")
    os.remove("tino_cta.tar.gz")
except:
    pass

print("\nall done -- exiting now")
exit()

# ########    ###     #######
# ##         ## ##   ##     ##
# ##        ##   ##  ##     ##
# ######   ##     ## ##     ##
# ##       ######### ##  ## ##
# ##       ##     ## ##    ##
# ##       ##     ##  ##### ##

# specify allowed sites to send the job to
j.setDestination(['LCG.IN2P3-CC.fr', 'LCG.DESY-ZEUTHEN.de', 'LCG.CNAF.it',
                  'LCG.GRIF.fr', 'LCG.CYFRONET.pl', 'LCG.PRAGUE-CESNET.cz',
                  'LCG.Prague.cz', 'LCG.LAPP.fr', 'LCG.CIEMAT.es', 'LCG.PIC.es'])

# to specify input GRID files independent of the site the job is send to
file1 = 'LFN:/vo.cta.in2p3.fr/user/c/ciro.bigongiari/MiniArray9/Simtel/'\
        'gamma/run1011.simtel.gz'
j.setInputSandbox([file1])
    args = visit + ' ' + insidename + ' ' + str(startsensor) + ' ' + str(
        numsensors) + ' ' + str(idx)
    outputname = 'fits_' + visit + '_' + str(idx) + '.tar'

    j.setCPUTime(1209600)
    j.setExecutable('runimsim2.1.sh', arguments=args)
    j.stderr = "std.err"
    j.stdout = "std.out"
    #!!! May need the 2.1i directory here depending on visit number !!!
    j.setInputSandbox([
        "runimsim2.1.sh", "run_imsim_nersc.py",
        "LFN:/lsst/user/j/james.perry/instcats/2.1i/" + instcatname
    ])
    j.setOutputSandbox(["std.out", "std.err"])
    j.setTag(["8Processors"])
    #j.setOutputData([visit + "/" + outputname], outputPath="", outputSE=["IN2P3-CC-disk"])
    j.setOutputData([visit + "/" + outputname],
                    outputPath="",
                    outputSE=["UKI-NORTHGRID-LANCS-HEP-disk"])
    j.setPlatform("AnyPlatform")

    j.setDestination(site)

    jobID = dirac.submitJob(j)
    print("Submitted job to " + site + " as ID " + str(jobID))
    print "Status is:", dirac.status(jobID['JobID'])

    joblistfile.write(str(jobID['JobID']) + '\n')

joblistfile.close()
Exemple #39
0
    j.stdout = "std.out"
    #!!! May need the 2.1i directory here depending on visit number !!!
    j.setInputSandbox([
        "launch_container_userns.sh", "docker_run.sh", "run_imsim_nersc.py",
        "parsl_imsim_configs", "finals2000A.all",
        "LFN:/lsst/user/j/james.perry/instcats/2.2i/y04/" + instcatname
    ])
    j.setOutputSandbox(["std.out", "std.err"])
    #j.setTag(["4Processors"])
    j.setOutputData([outputname],
                    outputPath="",
                    outputSE=["UKI-NORTHGRID-LANCS-HEP-disk"])
    #j.setOutputData([outputname], outputPath="", outputSE=["IN2P3-CC-disk"])
    j.setPlatform("EL7")

    # FIXME: remove these once those sites are working again
    #j.setBannedSites(["VAC.UKI-NORTHGRID-MAN-HEP.uk", "LCG.IN2P3-CC.fr"])
    #j.setDestination("LCG.UKI-LT2-IC-HEP.uk")
    #j.setDestination("LCG.UKI-SOUTHGRID-OX-HEP.uk")
    j.setDestination("CLOUD.UK-CAM-CUMULUS-backfill.uk")
    #j.setDestination("LCG.RAL-LCG2.uk")
    #j.setDestination("LCG.IN2P3-CC.fr")

    jobID = dirac.submitJob(j)
    print("Submitted job as ID " + str(jobID))
    print "Status is:", dirac.status(jobID['JobID'])

    joblistfile.write(str(jobID['JobID']) + '\n')

joblistfile.close()
    # Here we add the list of LFNs we have obtained from the metadata
    # query.
    j.setInputSandbox(lfns)

    #...and added to the desried storage element with the corresponding
    # LFN via the job's OutputData. You may wish to change:
    # * The Storage Element - by changing the outputSE parameter;
    # * The LFN base name   - by changing the outputPath parameter.
    #j.setOutputData(file_dict.keys(), \
    #                outputSE='GLASGOW-disk', \
    #                outputPath='/diractest%03d/' % (jobnum)\
    #               )

    # These are the files retrieved with the local job output.
    j.setOutputSandbox(['StdOut', 'StdErr'])

    # You can set your preferred site here.
    #j.setDestination('LCG.Liverpool.uk')
    #j.setDestination('LCG.Glasgow.uk')
    j.setDestination('LCG.UKI-LT2-QMUL.uk')

    ## The DIRAC instance.
    dirac = Dirac()

    # Submit the job and print the result.
    #result = dirac.submit(j)
    print
    print 'Submission result: ', result
    print
Exemple #41
0
        break

try:
    os.remove("datapipe.tar.gz")
    os.remove("modules.tar.gz")
except:
    pass

print("\nall done -- exiting now")
exit()

# ########    ###     #######
# ##         ## ##   ##     ##
# ##        ##   ##  ##     ##
# ######   ##     ## ##     ##
# ##       ######### ##  ## ##
# ##       ##     ## ##    ##
# ##       ##     ##  ##### ##

# specify allowed sites to send the job to
j.setDestination([
    'LCG.IN2P3-CC.fr', 'LCG.DESY-ZEUTHEN.de', 'LCG.CNAF.it', 'LCG.GRIF.fr',
    'LCG.CYFRONET.pl', 'LCG.PRAGUE-CESNET.cz', 'LCG.Prague.cz', 'LCG.LAPP.fr',
    'LCG.CIEMAT.es', 'LCG.PIC.es'
])

# to specify input GRID files independent of the site the job is send to
file1 = 'LFN:/vo.cta.in2p3.fr/user/c/ciro.bigongiari/MiniArray9/Simtel/'\
        'gamma/run1011.simtel.gz'
j.setInputSandbox([file1])