Пример #1
0
 def checkPyConfigs(self, deps = []):
   print "Going to check python configs in ", os.getcwd()
   cmd = scriptPath+'/checkPyConfigs.py > chkPyConf.log 2>&1'
   try:
     doCmd(cmd,self.dryRun,self.cmsswBuildDir)
     self.logger.updateLogFile("chkPyConf.log")
     self.logger.updateLogFile("chkPyConf.log",'testLogs')
   except:
     pass
   return None
Пример #2
0
 def checkPyConfigs(self, deps=[]):
     print("Going to check python configs in ", os.getcwd())
     cmd = scriptPath + '/checkPyConfigs.py > chkPyConf.log 2>&1'
     try:
         doCmd(cmd, self.dryRun, self.cmsswBuildDir)
         self.logger.updateLogFile("chkPyConf.log")
         self.logger.updateLogFile("chkPyConf.log", 'testLogs')
     except:
         pass
     return None
Пример #3
0
  def runDuplicateDictCheck(self, deps = []):
    print "runDuplicateDictTests> Going to run duplicateReflexLibrarySearch.py ... "
    script = 'duplicateReflexLibrarySearch.py'
    for opt in ['dup', 'lostDefs', 'edmPD']:
      cmd = script+' --'+opt+' 2>&1 >dupDict-'+opt+'.log'
      try:
        doCmd(cmd,self.dryRun,self.cmsswBuildDir)
      except Exception, e :
	print "ERROR during test duplicateDictCheck : caught exception: " + str(e)
      self.logger.updateDupDictTestLogs()
Пример #4
0
 def runProjectInit(self, deps = []):
   print "runProjectInit> Going regenerate scram caches ... "
   try:
     ver=os.environ["CMSSW_VERSION"]
     cmd= "cd "+self.cmsswBuildDir+"; rm -rf src;"
     cmd+="curl -k -L -s -o src.tar.gz https://github.com/cms-sw/cmssw/archive/"+ver+".tar.gz;"
     cmd+="tar -xzf src.tar.gz; mv cmssw-"+ver+" src; rm -rf src.tar.gz;"
     cmd+="scram build -r echo_CXX"
     doCmd(cmd)
   except Exception, e :
     print "ERROR during runProjectInit: caught exception: " + str(e)
Пример #5
0
 def copy2RemoteHost(self, src, des, host):
     cmd ="scp "+self.ssh_opt+" -r "+src+" "+host+":"+des
     try:
         if self.dryRun:
           print "CMD>>",cmd
         else:
           for i in range (10):
             err,out = doCmd(cmd)
             if not err: return (err,out)
             sleep(60)
           return doCmd(cmd)
     except Exception, e:
         print "Ignoring exception during copy2Remote:", str(e)
         return (1,str(e))
Пример #6
0
 def copy2RemoteHost(self, src, des, host):
     cmd = "scp " + self.ssh_opt + " -r " + src + " " + host + ":" + des
     try:
         if self.dryRun:
             print("CMD>>", cmd)
         else:
             for i in range(10):
                 err, out = doCmd(cmd)
                 if not err: return (err, out)
                 sleep(60)
             return doCmd(cmd)
     except Exception as e:
         print("Ignoring exception during copy2Remote:", str(e))
         return (1, str(e))
Пример #7
0
 def runRemoteHostCmd(self, cmd, host):
     cmd ="ssh -Y "+self.ssh_opt+" "+host+" 'echo CONNECTION=OK && "+cmd+"'"
     try:
         if self.dryRun:
           print "CMD>>",cmd
         else:
           for i in range (10):
             err,out = doCmd(cmd)
             if not err: return (err,out)
             for l in out.split("\n"):
               if "CONNECTION=OK" in l: return (err,out)
             sleep(60)
           return doCmd(cmd)
     except Exception, e:
         print "Ignoring exception during runRemoteCmd:", str(e)
         return (1,str(e))
Пример #8
0
 def runRemoteHostCmd(self, cmd, host):
     cmd = "ssh -Y " + self.ssh_opt + " " + host + " 'echo CONNECTION=OK && " + cmd + "'"
     try:
         if self.dryRun:
             print("CMD>>", cmd)
         else:
             for i in range(10):
                 err, out = doCmd(cmd)
                 if not err: return (err, out)
                 for l in out.split("\n"):
                     if "CONNECTION=OK" in l: return (err, out)
                 sleep(60)
             return doCmd(cmd)
     except Exception as e:
         print("Ignoring exception during runRemoteCmd:", str(e))
         return (1, str(e))
Пример #9
0
 def run_workflows(self,
                   workflows=[],
                   logger=None,
                   force=False,
                   known_errors={}):
     if not workflows: return
     workflows = workflows[::-1]
     threads = []
     while (len(workflows) > 0):
         threads = [t for t in threads if t.is_alive()]
         if (len(threads) < self.jobs):
             try:
                 wf = workflows.pop()
                 wf_err = {}
                 if wf in known_errors: wf_err = known_errors[wf]
                 t = threading.Thread(
                     target=runThreadMatrix,
                     args=(self.basedir, wf,
                           self.args['rest'] + " " + self.args['w'], logger,
                           force, wf_err))
                 t.start()
                 threads.append(t)
             except Exception as e:
                 print(
                     "runPyRelVal> ERROR threading matrix : caught exception: "
                     + str(e))
         else:
             time.sleep(5)
     for t in threads:
         t.join()
     ret, out = doCmd("touch " + self.basedir + "/done." + self.jobid)
     if logger:
         logger.updateRelValMatrixPartialLogs(self.basedir,
                                              "done." + self.jobid)
     return
Пример #10
0
 def getWorkFlows(self, args):
   self.setArgs(args)
   workflowsCmd = "runTheMatrix.py -n "+self.args['w']+" "+self.args['s']+" "+self.args['l']+" |  grep -v ' workflows with ' | grep -E '^[0-9][0-9]*(\.[0-9][0-9]*|)\s\s*' | sort -nr | awk '{print $1}'"
   print("RunTheMatrix>>",workflowsCmd)
   cmsstat, workflows = doCmd(workflowsCmd)
   if not cmsstat:
     return workflows.split("\n")
   print("runPyRelVal> ERROR during test PyReleaseValidation : could not get output of " + workflowsCmd)
   return []
Пример #11
0
 def getWorkFlows(self, args):
   self.setArgs(args)
   workflowsCmd = "runTheMatrix.py -n "+self.args['w']+" "+self.args['l']+" |  grep -v ' workflows with ' | grep -E '^[0-9][0-9]*(\.[0-9][0-9]*|)\s\s*' | sort -nr | awk '{print $1}'"
   print "RunTheMatrix>>",workflowsCmd
   cmsstat, workflows = doCmd(workflowsCmd)
   if not cmsstat:
     return workflows.split("\n")
   print "runPyRelVal> ERROR during test PyReleaseValidation : could not get output of " + workflowsCmd
   return []
Пример #12
0
 def runRemoteCmd(self, cmd):
     cmd ="ssh -Y "+self.ssh_opt+" "+self.remote+" "+cmd
     try:
         if self.dryRun:
           print "CMD>>",cmd
         else:
           return doCmd(cmd)
     except Exception, e:
         print "Ignoring exception during runRemoteCmd:", str(e)
         return (1,str(e))
Пример #13
0
 def copy2Remote(self, src, des):
     cmd ="scp "+self.ssh_opt+" -r "+src+" "+self.remote+":"+des
     try:
         if self.dryRun:
           print "CMD>>",cmd
         else:
           return doCmd(cmd)
     except Exception, e:
         print "Ignoring exception during copy2Remote:", str(e)
         return (1,str(e))
Пример #14
0
 def runRemoteCmd(self, cmd):
     cmd = "ssh -Y " + self.ssh_opt + " " + self.remote + " " + cmd
     try:
         if self.dryRun:
             print "CMD>>", cmd
         else:
             return doCmd(cmd)
     except Exception, e:
         print "Ignoring exception during runRemoteCmd:", str(e)
         return (1, str(e))
Пример #15
0
 def copy2Remote(self, src, des):
     cmd = "scp " + self.ssh_opt + " -r " + src + " " + self.remote + ":" + des
     try:
         if self.dryRun:
             print "CMD>>", cmd
         else:
             return doCmd(cmd)
     except Exception, e:
         print "Ignoring exception during copy2Remote:", str(e)
         return (1, str(e))
Пример #16
0
class PyRelValsThread(object):
    def __init__(self, jobs, basedir, jobid="1of1", outdir=None):
        if not outdir: outdir = basedir
        self.jobs = jobs
        self.basedir = basedir
        self.jobid = jobid
        self.outdir = outdir
        self.args = {}
        self.setArgs("")

    def setArgs(self, args):
        args = args.replace('\\"', '"')
        args, self.args['w'], tmp = find_argv(args, "-w|--what")
        args, self.args['l'], tmp = find_argv(args, "-l|--list")
        args, self.args['j'], tmp = find_argv(args, "-j|--nproc")
        self.args['rest'] = args

    def getWorkFlows(self, args):
        self.setArgs(args)
        workflowsCmd = "runTheMatrix.py -n " + self.args['w'] + " " + self.args[
            'l'] + " |  grep -v ' workflows with ' | grep -E '^[0-9][0-9]*(\.[0-9][0-9]*|)\s\s*' | sort -nr | awk '{print $1}'"
        print "RunTheMatrix>>", workflowsCmd
        cmsstat, workflows = doCmd(workflowsCmd)
        if not cmsstat:
            return workflows.split("\n")
        print "runPyRelVal> ERROR during test PyReleaseValidation : could not get output of " + workflowsCmd
        return []

    def run_workflows(self, workflows=[], logger=None, force=False):
        if not workflows: return
        workflows = workflows[::-1]
        threads = []
        while (len(workflows) > 0):
            threads = [t for t in threads if t.is_alive()]
            if (len(threads) < self.jobs):
                try:
                    t = threading.Thread(target=runThreadMatrix,
                                         args=(self.basedir, workflows.pop(),
                                               self.args['rest'] + " " +
                                               self.args['w'], logger, force))
                    t.start()
                    threads.append(t)
                except Exception, e:
                    print "runPyRelVal> ERROR threading matrix : caught exception: " + str(
                        e)
            else:
                time.sleep(5)
        for t in threads:
            t.join()
        ret, out = doCmd("touch " + self.basedir + "/done." + self.jobid)
        if logger:
            logger.updateRelValMatrixPartialLogs(self.basedir,
                                                 "done." + self.jobid)
        return
Пример #17
0
def runThreadMatrix(basedir,
                    workflow,
                    args='',
                    logger=None,
                    force=False,
                    wf_err={}):
    if (not force) and logger and logger.relvalAlreadyDone(workflow):
        print("Message>> Not runing workflow ", workflow,
              " as it is already ran")
        return
    args = FixWFArgs(os.environ["CMSSW_VERSION"], os.environ["SCRAM_ARCH"],
                     workflow, args)
    workdir = os.path.join(basedir, workflow)
    matrixCmd = 'runTheMatrix.py -l ' + workflow + ' ' + args
    try:
        if not os.path.isdir(workdir):
            os.makedirs(workdir)
    except Exception as e:
        print("runPyRelVal> ERROR during test PyReleaseValidation, workflow " +
              str(workflow) + " : can't create thread folder: " + str(e))
    wftime = time.time()
    try:
        ret = doCmd(matrixCmd, False, workdir)
    except Exception as e:
        print("runPyRelVal> ERROR during test PyReleaseValidation, workflow " +
              str(workflow) + " : caught exception: " + str(e))
    wftime = time.time() - wftime
    outfolders = [
        file for file in os.listdir(workdir)
        if re.match("^" + str(workflow) + "_", file)
    ]
    if len(outfolders) == 0: return
    outfolder = os.path.join(basedir, outfolders[0])
    wfdir = os.path.join(workdir, outfolders[0])
    ret = doCmd("rm -rf " + outfolder + "; mkdir -p " + outfolder)
    ret = doCmd(
        "find . -mindepth 1 -maxdepth 1 -name '*.xml' -o -name '*.log' -o -name '*.py' -o -name '*.json' -o -name 'cmdLog' -type f | xargs -i mv '{}' "
        + outfolder + "/", False, wfdir)
    logRE = re.compile('^(.*/[0-9]+(\.[0-9]+|)_([^/]+))/step1_dasquery.log$')
    for logFile in glob.glob(outfolder + "/step1_dasquery.log"):
        m = logRE.match(logFile)
        if not m: continue
        ret = doCmd("cp " + logFile + " " + m.group(1) + "/step1_" +
                    m.group(3) + ".log")
    ret = doCmd("mv " + os.path.join(workdir, "runall-report-step*.log") +
                " " + os.path.join(outfolder, "workflow.log"))
    ret = doCmd("echo " + str(wftime) + " > " +
                os.path.join(outfolder, "time.log"))
    ret = doCmd("hostname -s > " + os.path.join(outfolder, "hostname"))
    if wf_err: json.dump(wf_err, open("%s/known_error.json" % outfolder, "w"))
    if logger: logger.updateRelValMatrixPartialLogs(basedir, outfolders[0])
    shutil.rmtree(workdir)
    return
Пример #18
0
def runStep1Only(basedir, workflow, args=''):
  args = FixWFArgs (os.environ["CMSSW_VERSION"],os.environ["SCRAM_ARCH"],workflow,args)
  workdir = os.path.join(basedir, workflow)
  matrixCmd = 'runTheMatrix.py --maxSteps=0 -l ' + workflow +' '+args
  try:
    if not os.path.isdir(workdir):
      os.makedirs(workdir)
  except Exception as e:
    print("runPyRelVal> ERROR during test PyReleaseValidation steps, workflow "+str(workflow)+" : can't create thread folder: " + str(e))
  try:
    ret = doCmd(matrixCmd, False, workdir)
  except Exception as e:
    print("runPyRelVal> ERROR during test PyReleaseValidation steps, workflow "+str(workflow)+" : caught exception: " + str(e))
  return
Пример #19
0
from cmssw_known_errors import get_known_errors
from workflow_final import upload_logs

def runStep1Only(basedir, workflow, args=''):
  args = FixWFArgs (os.environ["CMSSW_VERSION"],os.environ["SCRAM_ARCH"],workflow,args)


  workdir = os.path.join(basedir, workflow)
  matrixCmd = 'runTheMatrix.py --maxSteps=0 -l ' + workflow +' '+args
  try:
    if not os.path.isdir(basedir):
      os.makedirs(basedir)
  except Exception, e:
    print "runPyRelVal> ERROR during test PyReleaseValidation steps, workflow "+str(workflow)+" : can't create thread folder: " + str(e)
  try:
    ret = doCmd(matrixCmd, False, basedir)
  except Exception, e:
    print "runPyRelVal> ERROR during test PyReleaseValidation steps, workflow "+str(workflow)+" : caught exception: " + str(e)
  return

if __name__ == "__main__":
  parser = OptionParser(usage="%prog -i|--id <jobid> -l|--list <list of workflows>")
  parser.add_option("-i", "--id",   dest="jobid", help="Job Id e.g. 1of3", default="1of1")
  parser.add_option("-l", "--list", dest="workflow", help="List of workflows to run e.g. 1.0,2.0,3.0", type=str, default=None)
  parser.add_option("-f", "--force",dest="force", help="Force running of workflows without checking the server for previous run", action="store_true", default=False)
  opts, args = parser.parse_args()

  if len(args) > 0: parser.error("Too many/few arguments")
  if not opts.workflow: parser.error("Missing -l|--list <workflows> argument.")
  if (not environ.has_key("CMSSW_VERSION")) or (not environ.has_key("CMSSW_BASE")) or (not environ.has_key("SCRAM_ARCH")):
    print "ERROR: Unable to file the release environment, please make sure you have set the cmssw environment before calling this script"
Пример #20
0
 def isNewRunTheMatrix(self):
   e, o = doCmd("runTheMatrix.py --help | grep 'maxSteps=MAXSTEPS' | wc -l")
   if e: return False
   return o=="1"
Пример #21
0
from es_relval_log import es_parse_log
from RelValArgs import FixWFArgs
import json
from logreaderUtils import write_config_file, add_exception_to_config

def runStep1Only(basedir, workflow, args=''):
  args = FixWFArgs (os.environ["CMSSW_VERSION"],os.environ["SCRAM_ARCH"],workflow,args)
  workdir = os.path.join(basedir, workflow)
  matrixCmd = 'runTheMatrix.py --maxSteps=0 -l ' + workflow +' '+args
  try:
    if not os.path.isdir(workdir):
      os.makedirs(workdir)
  except Exception, e:
    print "runPyRelVal> ERROR during test PyReleaseValidation steps, workflow "+str(workflow)+" : can't create thread folder: " + str(e)
  try:
    ret = doCmd(matrixCmd, False, workdir)
  except Exception, e:
    print "runPyRelVal> ERROR during test PyReleaseValidation steps, workflow "+str(workflow)+" : caught exception: " + str(e)
  return

def runThreadMatrix(basedir, workflow, args='', logger=None, force=False, wf_err={}):
  if (not force) and logger and logger.relvalAlreadyDone(workflow):
    print "Message>> Not runing workflow ",workflow," as it is already ran"
    return
  args = FixWFArgs (os.environ["CMSSW_VERSION"],os.environ["SCRAM_ARCH"],workflow,args)
  workdir = os.path.join(basedir, workflow)
  matrixCmd = 'runTheMatrix.py -l ' + workflow +' '+args
  try:
    if not os.path.isdir(workdir):
      os.makedirs(workdir)
  except Exception, e: 
Пример #22
0

def runStep1Only(basedir, workflow, args=''):
    args = FixWFArgs(os.environ["CMSSW_VERSION"], os.environ["SCRAM_ARCH"],
                     workflow, args)

    workdir = os.path.join(basedir, workflow)
    matrixCmd = 'runTheMatrix.py --maxSteps=0 -l ' + workflow + ' ' + args
    try:
        if not os.path.isdir(basedir):
            os.makedirs(basedir)
    except Exception, e:
        print "runPyRelVal> ERROR during test PyReleaseValidation steps, workflow " + str(
            workflow) + " : can't create thread folder: " + str(e)
    try:
        ret = doCmd(matrixCmd, False, basedir)
    except Exception, e:
        print "runPyRelVal> ERROR during test PyReleaseValidation steps, workflow " + str(
            workflow) + " : caught exception: " + str(e)
    return


if __name__ == "__main__":
    parser = OptionParser(
        usage="%prog -i|--id <jobid> -l|--list <list of workflows>")
    parser.add_option("-i",
                      "--id",
                      dest="jobid",
                      help="Job Id e.g. 1of3",
                      default="1of1")
    parser.add_option("-l",
Пример #23
0
from logreaderUtils import transform_and_write_config_file, add_exception_to_config


def runStep1Only(basedir, workflow, args=''):
    args = FixWFArgs(os.environ["CMSSW_VERSION"], os.environ["SCRAM_ARCH"],
                     workflow, args)
    workdir = os.path.join(basedir, workflow)
    matrixCmd = 'runTheMatrix.py --maxSteps=0 -l ' + workflow + ' ' + args
    try:
        if not os.path.isdir(workdir):
            os.makedirs(workdir)
    except Exception, e:
        print "runPyRelVal> ERROR during test PyReleaseValidation steps, workflow " + str(
            workflow) + " : can't create thread folder: " + str(e)
    try:
        ret = doCmd(matrixCmd, False, workdir)
    except Exception, e:
        print "runPyRelVal> ERROR during test PyReleaseValidation steps, workflow " + str(
            workflow) + " : caught exception: " + str(e)
    return


def runThreadMatrix(basedir,
                    workflow,
                    args='',
                    logger=None,
                    force=False,
                    wf_err={}):
    if (not force) and logger and logger.relvalAlreadyDone(workflow):
        print "Message>> Not runing workflow ", workflow, " as it is already ran"
        return
Пример #24
0
#! /usr/bin/env python
from sys import exit, argv
from os import environ
from cmsutils import cmsRunProcessCount, doCmd
from logUpdater import LogUpdater

if (not environ.has_key("CMSSW_BASE")) or (not environ.has_key("SCRAM_ARCH")):
  print "ERROR: Unable to file the release environment, please make sure you have set the cmssw environment before calling this script"
  exit(1)

timeout=7200
try: timeout=int(argv[1])
except: timeout=7200
logger = LogUpdater(environ["CMSSW_BASE"])
ret = doCmd('cd %s; rm -rf addOnTests; timeout %s addOnTests.py -j %s 2>&1 >addOnTests.log' % (environ["CMSSW_BASE"], timeout,cmsRunProcessCount))
doCmd('cd '+environ["CMSSW_BASE"]+'/addOnTests/logs; zip -r addOnTests.zip *.log')
logger.updateAddOnTestsLogs()

Пример #25
0
 def isNewRunTheMatrix(self):
   e, o = doCmd("runTheMatrix.py --help | grep 'maxSteps=MAXSTEPS' | wc -l")
   if e: return False
   return o=="1"
Пример #26
0
      json_data={"avg": []}
    for tm_str in sorted(json_data["avg"],key=int, reverse=True):
      tm=int(tm_str)
      if tm > max_tm : max_tm=tm
      for wf in json_data["avg"][tm_str]:
        if wf in workflows: owf.append([wf,tm])
  uwf = []
  owfs = [ x[0] for x in owf ]
  for wf in workflows:
    if not wf in owfs: uwf.append([wf,max_tm])
  workflows = uwf + owf
if workflows:
  workflows = splitWorkflows(workflows, max_wf)
  print(workflows)
  on_grid = 0
  #if '_DEVEL_X' in environ['CMSSW_VERSION']:
  #  on_grid = 2
  total = len(workflows)
  try:
    for i in range(1, total+1):
      wf=",".join(workflows[i-1])
      jobid   = str(i)+"of"+str(total)
      jobfile = workdir+"/ib-run-relval-"+jobid
      doCmd("echo WORKFLOWS="+wf+" >"+jobfile)
      doCmd("echo JOBID="+jobid+" >>"+jobfile)
      if on_grid>0:
        doCmd("echo 'SLAVE_LABELS=(condor&&cpu-8)' >>"+jobfile)
        on_grid=on_grid-1
  except Exception as e:
    print("Error " , e)
Пример #27
0
#! /usr/bin/env python
from sys import exit
from os import environ
from cmsutils import cmsRunProcessCount, doCmd
from logUpdater import LogUpdater

if (not environ.has_key("CMSSW_BASE")) or (not environ.has_key("SCRAM_ARCH")):
  print "ERROR: Unable to file the release environment, please make sure you have set the cmssw environment before calling this script"
  exit(1)

logger = LogUpdater(environ["CMSSW_BASE"])
ret = doCmd('cd '+environ["CMSSW_BASE"]+'; rm -rf addOnTests; timeout 5400 addOnTests.py -j '+str(cmsRunProcessCount)+' 2>&1 >addOnTests.log ')
doCmd('cd '+environ["CMSSW_BASE"]+'/addOnTests/logs; zip -r addOnTests.zip *.log')
logger.updateAddOnTestsLogs()

Пример #28
0
    max_tm = 0
    with open(RelValtimes) as json_file:
        try:
            json_data = json.load(json_file)
        except:
            print "Error reading RelVal Times"
            json_data = {"avg": []}
        for tm_str in sorted(json_data["avg"], key=int, reverse=True):
            tm = int(tm_str)
            if tm > max_tm: max_tm = tm
            for wf in json_data["avg"][tm_str]:
                if wf in workflows: owf.append([wf, tm])
    uwf = []
    owfs = [x[0] for x in owf]
    for wf in workflows:
        if not wf in owfs: uwf.append([wf, max_tm])
    workflows = uwf + owf
if workflows:
    workflows = splitWorkflows(workflows, max_wf)
    print workflows
    total = len(workflows)
    try:
        for i in range(1, total + 1):
            wf = ",".join(workflows[i - 1])
            jobid = str(i) + "of" + str(total)
            jobfile = workdir + "/ib-run-relval-" + jobid
            doCmd("echo WORKFLOWS=" + wf + " >" + jobfile)
            doCmd("echo JOBID=" + jobid + " >>" + jobfile)
    except Exception as e:
        print "Error ", e
Пример #29
0
""":"
python_cmd="python3"
python -V >/dev/null 2>&1 && python_cmd="python"
exec ${python_cmd} $0 ${1+"$@"}
"""

from __future__ import print_function
from sys import exit, argv
from os import environ
from cmsutils import cmsRunProcessCount, doCmd
from logUpdater import LogUpdater

if ("CMSSW_BASE" not in environ) or ("SCRAM_ARCH" not in environ):
    print(
        "ERROR: Unable to file the release environment, please make sure you have set the cmssw environment before calling this script"
    )
    exit(1)

timeout = 7200
try:
    timeout = int(argv[1])
except:
    timeout = 7200
logger = LogUpdater(environ["CMSSW_BASE"])
ret = doCmd(
    'cd %s; rm -rf addOnTests; timeout %s addOnTests.py -j %s 2>&1 >addOnTests.log'
    % (environ["CMSSW_BASE"], timeout, cmsRunProcessCount))
doCmd('cd ' + environ["CMSSW_BASE"] +
      '/addOnTests/logs; zip -r addOnTests.zip *.log')
logger.updateAddOnTestsLogs()
Пример #30
0
            json_data = {"avg": []}
        for tm_str in sorted(json_data["avg"], key=int, reverse=True):
            tm = int(tm_str)
            if tm > max_tm: max_tm = tm
            for wf in json_data["avg"][tm_str]:
                if wf in workflows: owf.append([wf, tm])
    uwf = []
    owfs = [x[0] for x in owf]
    for wf in workflows:
        if not wf in owfs: uwf.append([wf, max_tm])
    workflows = uwf + owf
if workflows:
    workflows = splitWorkflows(workflows, max_wf)
    print workflows
    on_grid = 0
    if '_DEVEL_X' in environ['CMSSW_VERSION']:
        on_grid = 2
    total = len(workflows)
    try:
        for i in range(1, total + 1):
            wf = ",".join(workflows[i - 1])
            jobid = str(i) + "of" + str(total)
            jobfile = workdir + "/ib-run-relval-" + jobid
            doCmd("echo WORKFLOWS=" + wf + " >" + jobfile)
            doCmd("echo JOBID=" + jobid + " >>" + jobfile)
            if on_grid > 0:
                doCmd("echo 'SLAVE_LABELS=(condor&&cpu-8)' >>" + jobfile)
                on_grid = on_grid - 1
    except Exception as e:
        print "Error ", e
Пример #31
0
except:
  max_wf=100
relval_args = GetMatrixOptions(environ["CMSSW_VERSION"], environ["SCRAM_ARCH"])
matrix =  PyRelValsThread(1,environ["CMSSW_BASE"])
workflows = matrix.getWorkFlows(relval_args)
if exists(RelValtimes):
  owf = []
  max_tm=0
  with open(RelValtimes) as json_file:
    json_data = json.load(json_file)
    for tm_str in sorted(json_data["avg"],key=int, reverse=True):
      tm=int(tm_str)
      if tm > max_tm : max_tm=tm
      for wf in json_data["avg"][tm_str]:
        if wf in workflows: owf.append([wf,tm])
  uwf = []
  owfs = [ x[0] for x in owf ]
  for wf in workflows:
    if not wf in owfs: uwf.append([wf,max_tm])
  workflows = uwf + owf
if workflows:
  workflows = splitWorkflows(workflows, max_wf)
  print workflows
  total = len(workflows)
  for i in range(1, total+1):
    wf=",".join(workflows[i-1])
    jobid   = str(i)+"of"+str(total)
    jobfile = workdir+"/ib-run-relval-"+jobid
    doCmd("echo WORKFLOWS="+wf+" >"+jobfile)
    doCmd("echo JOBID="+jobid+" >>"+jobfile)