Ejemplo n.º 1
0
def test_SimpleParametricJob():

    job = Job()
    job.setExecutable("myExec")
    job.setLogLevel("DEBUG")
    parList = [1, 2, 3]
    job.setParameterSequence("JOB_ID", parList, addToWorkflow=True)
    inputDataList = [
        ["/lhcb/data/data1", "/lhcb/data/data2"],
        ["/lhcb/data/data3", "/lhcb/data/data4"],
        ["/lhcb/data/data5", "/lhcb/data/data6"],
    ]
    job.setParameterSequence("InputData", inputDataList, addToWorkflow=True)

    jdl = job._toJDL()

    with open(join(dirname(__file__), "testWF.jdl")) as fd:
        expected = fd.read()

    assert jdl == expected

    clad = ClassAd("[" + jdl + "]")

    arguments = clad.getAttributeString("Arguments")
    job_id = clad.getAttributeString("JOB_ID")
    inputData = clad.getAttributeString("InputData")

    assert job_id == "%(JOB_ID)s"
    assert inputData == "%(InputData)s"
    assert "jobDescription.xml" in arguments
    assert "-o LogLevel=DEBUG" in arguments
    assert "-p JOB_ID=%(JOB_ID)s" in arguments
    assert "-p InputData=%(InputData)s" in arguments
Ejemplo n.º 2
0
    def test_SimpleParametricJob(self):

        job = Job()
        job.setExecutable('myExec')
        job.setLogLevel('DEBUG')
        parList = [1, 2, 3]
        job.setParameterSequence('JOB_ID', parList, addToWorkflow=True)
        inputDataList = [['/lhcb/data/data1', '/lhcb/data/data2'],
                         ['/lhcb/data/data3', '/lhcb/data/data4'],
                         ['/lhcb/data/data5', '/lhcb/data/data6']]
        job.setParameterSequence('InputData',
                                 inputDataList,
                                 addToWorkflow=True)

        jdl = job._toJDL()

        print jdl

        clad = ClassAd('[' + jdl + ']')

        arguments = clad.getAttributeString('Arguments')
        job_id = clad.getAttributeString('JOB_ID')
        inputData = clad.getAttributeString('InputData')

        print "arguments", arguments

        self.assertEqual(job_id, '%(JOB_ID)s')
        self.assertEqual(inputData, '%(InputData)s')
        self.assertIn('jobDescription.xml', arguments)
        self.assertIn('-o LogLevel=DEBUG', arguments)
        self.assertIn('-p JOB_ID=%(JOB_ID)s', arguments)
        self.assertIn('-p InputData=%(InputData)s', arguments)
Ejemplo n.º 3
0
def test_SimpleParametricJob():

    job = Job()
    job.setExecutable('myExec')
    job.setLogLevel('DEBUG')
    parList = [1, 2, 3]
    job.setParameterSequence('JOB_ID', parList, addToWorkflow=True)
    inputDataList = [['/lhcb/data/data1', '/lhcb/data/data2'],
                     ['/lhcb/data/data3', '/lhcb/data/data4'],
                     ['/lhcb/data/data5', '/lhcb/data/data6']]
    job.setParameterSequence('InputData', inputDataList, addToWorkflow=True)

    jdl = job._toJDL()

    try:
        with open('./DIRAC/Interfaces/API/test/testWF.jdl') as fd:
            expected = fd.read()
    except IOError:
        with open('./Interfaces/API/test/testWF.jdl') as fd:
            expected = fd.read()

    assert jdl == expected

    clad = ClassAd('[' + jdl + ']')

    arguments = clad.getAttributeString('Arguments')
    job_id = clad.getAttributeString('JOB_ID')
    inputData = clad.getAttributeString('InputData')

    assert job_id == '%(JOB_ID)s'
    assert inputData == '%(InputData)s'
    assert 'jobDescription.xml' in arguments
    assert '-o LogLevel=DEBUG' in arguments
    assert '-p JOB_ID=%(JOB_ID)s' in arguments
    assert '-p InputData=%(InputData)s' in arguments
Ejemplo n.º 4
0
def parametricJob():
  job = Job()
  job.setName("parametric_helloWorld_%n")
  exeScriptLocation = find_all('exe-script.py', '..', '/DIRAC/tests/Integration')[0]
  job.setInputSandbox(exeScriptLocation)
  job.setParameterSequence("args", ['one', 'two', 'three'])
  job.setParameterSequence("iargs", [1, 2, 3])
  job.setExecutable(exeScriptLocation, arguments=": testing %(args)s %(iargs)s", logFile='helloWorld_%n.log')
  return job
Ejemplo n.º 5
0
def parametricJob():
  job = Job()
  job.setName("parametric_helloWorld_%n")
  exeScriptLocation = find_all('exe-script.py', '..', '/DIRAC/tests/Integration')[0]
  job.setInputSandbox(exeScriptLocation)
  job.setParameterSequence("args", ['one', 'two', 'three'])
  job.setParameterSequence("iargs", [1, 2, 3])
  job.setExecutable(exeScriptLocation, arguments=": testing %(args)s %(iargs)s", logFile='helloWorld_%n.log')
  return job
Ejemplo n.º 6
0
def parametricJob():
    job = Job()
    job.setName("parametric_helloWorld_%n")
    exeScriptLocation = find_all("exe-script.py", "..", "/DIRAC/tests/Integration")[0]
    job.setInputSandbox(exeScriptLocation)
    job.setParameterSequence("args", ["one", "two", "three"])
    job.setParameterSequence("iargs", [1, 2, 3])
    job.setExecutable(exeScriptLocation, arguments=": testing %(args)s %(iargs)s", logFile="helloWorld_%n.log")
    return job
Ejemplo n.º 7
0
def submit(name,
           job_group,
           task_id,
           input_sandbox,
           output_sandbox,
           executable,
           site=None,
           banned_site=None,
           sub_ids=[]):
    dirac = Dirac()

    submit_result = {'backend_job_ids': {}}
    jobInfos = {}

    for run in range(int((len(sub_ids) + 99) / 100)):
        ids_this_run = [x for x in sub_ids[run * 100:(run + 1) * 100]]
        job_names = ['%s.%s' % (name, sub_id) for sub_id in ids_this_run]
        j = Job()
        j.setName(name)
        j.setExecutable(executable)

        j.setParameterSequence('JobName', job_names, addToWorkflow=True)
        j.setParameterSequence('arguments', ids_this_run, addToWorkflow=True)

        if input_sandbox:
            j.setInputSandbox(input_sandbox)
        if output_sandbox:
            j.setOutputSandbox(output_sandbox)

        if job_group:
            j.setJobGroup(job_group)
        if site:  # set destination to a certain site; list not allowed
            j.setDestination(site)

        if banned_site:
            j.setBannedSites(banned_site)

        result = dirac.submitJob(j)

        if not result['OK']:
            sys.stdout.write('DIRAC job submit error: %s\n' %
                             result['Message'])
            sys.exit(1)

        for sub_id, dirac_id in zip(ids_this_run, result['Value']):
            submit_result['backend_job_ids'][sub_id] = dirac_id
            jobInfos[dirac_id] = {'SubID': sub_id}

    #Register on Task-manager Webapp of IHEPDIRAC
    task = RPCClient('WorkloadManagement/TaskManager')
    taskInfo = {'TaskName': name, 'JobGroup': job_group, 'JSUB-ID': task_id}
    task_result = task.createTask(name, taskInfo, jobInfos)
    task_web_id = task_result['Value']
    submit_result['backend_task_id'] = task_web_id

    return submit_result
Ejemplo n.º 8
0
def test_SimpleParametricJob():

  job = Job()
  job.setExecutable('myExec')
  job.setLogLevel('DEBUG')
  parList = [1, 2, 3]
  job.setParameterSequence('JOB_ID', parList, addToWorkflow=True)
  inputDataList = [
      [
          '/lhcb/data/data1',
          '/lhcb/data/data2'
      ],
      [
          '/lhcb/data/data3',
          '/lhcb/data/data4'
      ],
      [
          '/lhcb/data/data5',
          '/lhcb/data/data6'
      ]
  ]
  job.setParameterSequence('InputData', inputDataList, addToWorkflow=True)

  jdl = job._toJDL()

  try:
    with open('./DIRAC/Interfaces/API/test/testWF.jdl') as fd:
      expected = fd.read()
  except IOError:
    with open('./Interfaces/API/test/testWF.jdl') as fd:
      expected = fd.read()

  assert jdl == expected

  clad = ClassAd('[' + jdl + ']')

  arguments = clad.getAttributeString('Arguments')
  job_id = clad.getAttributeString('JOB_ID')
  inputData = clad.getAttributeString('InputData')

  assert job_id == '%(JOB_ID)s'
  assert inputData == '%(InputData)s'
  assert 'jobDescription.xml' in arguments
  assert '-o LogLevel=DEBUG' in arguments
  assert'-p JOB_ID=%(JOB_ID)s' in arguments
  assert'-p InputData=%(InputData)s' in arguments
Ejemplo n.º 9
0
  def test_SimpleParametricJob( self ):

    job = Job()
    job.setExecutable( 'myExec' )
    job.setLogLevel( 'DEBUG' )
    parList = [1,2,3]
    job.setParameterSequence( 'JOB_ID', parList, addToWorkflow=True )
    inputDataList = [
      [
        '/lhcb/data/data1',
        '/lhcb/data/data2'
      ],
      [
        '/lhcb/data/data3',
        '/lhcb/data/data4'
      ],
      [
        '/lhcb/data/data5',
        '/lhcb/data/data6'
      ]
    ]
    job.setParameterSequence( 'InputData', inputDataList, addToWorkflow=True )

    jdl = job._toJDL()

    print jdl

    clad = ClassAd( '[' + jdl + ']' )

    arguments = clad.getAttributeString( 'Arguments' )
    job_id = clad.getAttributeString( 'JOB_ID' )
    inputData = clad.getAttributeString( 'InputData' )

    print "arguments", arguments

    self.assertEqual( job_id, '%(JOB_ID)s' )
    self.assertEqual( inputData, '%(InputData)s' )
    self.assertIn( 'jobDescription.xml', arguments )
    self.assertIn( '-o LogLevel=DEBUG', arguments )
    self.assertIn( '-p JOB_ID=%(JOB_ID)s', arguments )
    self.assertIn( '-p InputData=%(InputData)s', arguments )
Ejemplo n.º 10
0
    def submit_job(self,
                   request_id,
                   executable,
                   macro,
                   starting_seed=8000000,
                   njobs=10,
                   platform='ANY',
                   output_data_site='UKI-LT2-IC-HEP-disk',
                   output_log='lzproduction_output.log'):
        """
        Submit LZProduction job to DIRAC.

        Args:
            request_id (int): The id number of the associated request
            executable (str): The full path to the executable job script
            macro (str): The full path to the macro for this job
            starting_seed (int): The random seed for the first of the parametric jobs
            njobs (int): The number of parametric jobs to create
            platform (str): The required platform
            output_data_site (str): The name of the grid site to store the output data at
            output_log (str): The file name for the output log file

        Returns:
           list: The list of created parametric job DIRAC ids
        """
        j = Job()
        j.setName(os.path.splitext(os.path.basename(macro))[0] + '%(args)s')
        j.setExecutable(os.path.basename(executable),
                        os.path.basename(macro) + ' %(args)s', output_log)
        j.setInputSandbox([executable, macro])
        j.setOutputData('*.root', output_data_site, str(request_id))
        j.setParameterSequence(
            "args",
            [str(i) for i in xrange(starting_seed, starting_seed + njobs)],
            addToWorkflow=True)
        j.setPlatform(platform)

        return self.status(self._dirac_api.submit(j).get("Value", []))