Esempio n. 1
0
  def execute(self):
    """ Execute it.
    """
    jobDescription = self.paramDict['Job']
    prodID = self.paramDict['TransformationID']
    jobID = self.paramDict['TaskID']
    inputData = self.paramDict['InputData']
    
    job = Job(jobDescription)
    commons = job._getParameters() #pylint: disable=protected-access
    code = job.workflow.createCode()
    outputList = []
    for line in code.split("\n"):
      if line.count("listoutput"):
        outputList += eval(line.split("#")[0].split("=")[-1]) #pylint: disable=eval-used

    commons['outputList'] = outputList
    commons['PRODUCTION_ID'] = prodID
    commons['JOB_ID'] = jobID
    if inputData:
      commons['InputData'] = inputData

    result = constructProductionLFNs(commons)
    if not result['OK']:
      LOG.error(result['Message'])
      return result

    if commons['JobType'] in ILDJOBTYPES and commons['InputData']:
      for index, outputFile in enumerate( result['Value']['ProductionOutputData'] ):
        outputFileILD = getProdFilenameFromInput( commons['InputData'], outputFile, prodID, jobID )
        result['Value']['ProductionOutputData'][index] = outputFileILD
        LOG.debug("Changed output file name from '%s' to '%s' " % (outputFile, outputFileILD))


    return result
Esempio n. 2
0
  def execute(self):
    """ Execute it.
    """
    jobDescription = self.paramDict['Job']
    prodID = self.paramDict['TransformationID']
    jobID = self.paramDict['TaskID']
    inputData = self.paramDict['InputData']
    
    job = Job(jobDescription)
    commons = job._getParameters()
    code = job.workflow.createCode()
    outputList = []
    for line in code.split("\n"):
      if line.count("listoutput"):
        outputList += eval(line.split("#")[0].split("=")[-1])

    commons['outputList'] = outputList
    commons['PRODUCTION_ID'] = prodID
    commons['JOB_ID'] = jobID
    if inputData:
      commons['InputData'] = inputData

    gLogger.debug(commons)
    result = constructProductionLFNs(commons)
    if not result['OK']:
      gLogger.error(result['Message'])
    return result
 def test_contructProductionLFNsim(self):
   """test ProductionOutputData construct sim LFN.................................................."""
   commons = {}
   commons['PRODUCTION_ID'] = 12345
   commons['JOB_ID'] = 1234
   commons['outputList'] = [{'outputFile':"something_sim.slcio",
                             'outputPath':'/ilc/prod/clic/test/SIM'}]
   result = constructProductionLFNs(commons)
   self.assertEqual( result['OK'], True)
 def test_contructProductionLFNsim(self):
     """test ProductionOutputData construct sim LFN.................................................."""
     commons = {}
     commons['PRODUCTION_ID'] = 12345
     commons['JOB_ID'] = 1234
     commons['outputList'] = [{
         'outputFile': "something_sim.slcio",
         'outputPath': '/ilc/prod/clic/test/SIM'
     }]
     result = constructProductionLFNs(commons)
     self.assertEqual(result['OK'], True)
 def test_contructProductionLFNoutput(self):
   """test ProductionOutputData construct out LFN.................................................."""
   commons = {}
   commons['PRODUCTION_ID'] = 12345
   commons['JOB_ID'] = 1234
   commons['outputList'] = [{'outputFile':"something_gen.stdhep",
                             'outputPath':'/ilc/prod/clic/test/gen'}]
   result = constructProductionLFNs(commons)
   res = {'ProductionOutputData' : ["/ilc/prod/clic/test/gen/00012345/001/something_gen_12345_1234.stdhep"], 
          'LogFilePath' : ["/ilc/prod/clic/test/gen/00012345/LOG/001"],
          'LogTargetPath' : ["/ilc/prod/clic/test/gen/LOG/00012345/00012345_1234.tar"]}
   for key in res.keys():
     self.assertEqual( result['Value'][key], res[key])
 def test_contructProductionLFNoutput(self):
     """test ProductionOutputData construct out LFN.................................................."""
     commons = {}
     commons['PRODUCTION_ID'] = 12345
     commons['JOB_ID'] = 1234
     commons['outputList'] = [{
         'outputFile': "something_gen.stdhep",
         'outputPath': '/ilc/prod/clic/test/gen'
     }]
     result = constructProductionLFNs(commons)
     res = {
         'ProductionOutputData': [
             "/ilc/prod/clic/test/gen/00012345/001/something_gen_12345_1234.stdhep"
         ],
         'LogFilePath': ["/ilc/prod/clic/test/gen/00012345/LOG/001"],
         'LogTargetPath':
         ["/ilc/prod/clic/test/gen/LOG/00012345/00012345_1234.tar"]
     }
     for key in res.keys():
         self.assertEqual(result['Value'][key], res[key])
Esempio n. 7
0
    def execute(self):
        """ Execute it.
    """
        jobDescription = self.paramDict['Job']
        prodID = self.paramDict['TransformationID']
        jobID = self.paramDict['TaskID']
        inputData = self.paramDict['InputData']

        job = Job(jobDescription)
        commons = job._getParameters()  #pylint: disable=protected-access
        code = job.workflow.createCode()
        outputList = []
        for line in code.split("\n"):
            if line.count("listoutput"):
                outputList += eval(line.split("#")[0].split("=")[-1])  #pylint: disable=eval-used

        commons['outputList'] = outputList
        commons['PRODUCTION_ID'] = prodID
        commons['JOB_ID'] = jobID
        if inputData:
            commons['InputData'] = inputData

        result = constructProductionLFNs(commons)
        if not result['OK']:
            gLogger.error(result['Message'])
            return result

        if commons['JobType'] in ILDJOBTYPES and commons['InputData']:
            for index, outputFile in enumerate(
                    result['Value']['ProductionOutputData']):
                outputFileILD = getProdFilenameFromInput(
                    commons['InputData'], outputFile, prodID, jobID)
                result['Value']['ProductionOutputData'][index] = outputFileILD
                gLogger.debug("Changed output file name from '%s' to '%s' " %
                              (outputFile, outputFileILD))

        return result
 def test_contructProductionLFNBad(self):
   """test ProductionOutputData construct Bad LFN.................................................."""
   commons = {}
   result = constructProductionLFNs(commons)
   self.assertEqual( result['OK'], False)
 def test_contructProductionLFNBad(self):
     """test ProductionOutputData construct Bad LFN.................................................."""
     commons = {}
     result = constructProductionLFNs(commons)
     self.assertEqual(result['OK'], False)
Esempio n. 10
0
  def checkDescendents(self, transformation, filedict, jobFileDict):
    """ look that all jobs produced, or not output
    """
    res = self.prodDB.getTransformationParameters(transformation, ['Body'])
    if not res['OK']:
      self.log.error('Could not get Body from TransformationDB')
      return res
    body = res['Value']
    workflow = fromXMLString(body)
    workflow.resolveGlobalVars()

    olist = []
    jtype = workflow.findParameter('JobType')
    if not jtype:
      self.log.error('Type for transformation %d was not defined' % transformation)
      return S_ERROR('Type for transformation %d was not defined' % transformation)
    for step in workflow.step_instances:
      param = step.findParameter('listoutput')
      if not param:
        continue
      olist.extend(param.value)
    expectedlfns = []
    contactfailed = []
    fileprocessed = []
    files = []
    tasks_to_be_checked = {}
    for files in jobFileDict.values():
      for f in files:
        if f in filedict:
          tasks_to_be_checked[f] = filedict[f] #get the tasks that need to be checked
    for filep, task in tasks_to_be_checked.items():
      commons = {}
      commons['outputList'] = olist
      commons['PRODUCTION_ID'] = transformation
      commons['JOB_ID'] = task
      commons['JobType'] = jtype
      out = constructProductionLFNs(commons)
      expectedlfns = out['Value']['ProductionOutputData']
      res = self.replicaManager.getCatalogFileMetadata(expectedlfns)
      if not res['OK']:
        self.log.error('Getting metadata failed')
        contactfailed.append(filep)
        continue
      if not filep in files:
        files.append(filep)      
      success = res['Value']['Successful'].keys()
      failed = res['Value']['Failed'].keys()
      if len(success) and not len(failed):
        fileprocessed.append(filep)
        
    final_list_unused = files
    for file_all in files:
      if file_all in fileprocessed:
        try:
          final_list_unused.remove(filep)
        except:
          self.log.warn("Item not in list anymore")

        
    result = {'filesprocessed' : fileprocessed, 'filesToMarkUnused' : final_list_unused}    
    return S_OK(result)