def backport_pull (repo, pr, branch):
pr_branch = pr.base.label.split(":")[1]
print("Source Branch:",pr_branch)
if pr_branch == branch: return "Warning: Can not backport, same branch %s vs %s" % (pr_branch, branch),False
br = gh_repo.get_branch(branch)
commits = []
for c in pr.get_commits().reversed: commits.insert(0,"git cherry-pick %s" % c.sha)
if not commits: return "There are no commits to backport",False
print("Cherry-pick commands:")
print(" "+"\n ".join(commits))
if len(commits)>=250:
return "Error: Too many commits in PR %s\nBot can only handle max 250 commits." % len(commits),False
new_branch = "backport-%s-%s" % (branch.replace("/","_"), pr.number)
print("New Branch:",new_branch)
git_ref = ""
if repo.name == "cmssw": git_ref = "--reference "+CMSSW_GIT_REF
print("GIT REF:",git_ref)
e , o = run_cmd("rm -rf pr_backport; git clone --branch %s %s [email protected]:%s pr_backport && cd pr_backport && git checkout -b %s" % (branch, git_ref, repo.full_name, new_branch))
if e:
print(o)
exit(1)
e, o = run_cmd('cd pr_backport; %s' % ";".join(commits))
if e: return "Error: Failed to cherry-pick commits. Please backport this PR yourself.\n```"+o+"\n```",False
e, o = run_cmd("cd pr_backport; git push origin %s" % new_branch)
if e:
print(o)
exit(1)
run_cmd("rm -rf pr_backport")
newBody = "backport of #%s\n\n%s" %(pr.number, pr.body)
newPR = repo.create_pull(title = pr.title, body = newBody, base = branch, head = new_branch )
return "Successfully backported PR #%s as #%s for branch %s" % (pr.number, newPR.number, branch),True
def cleanup_exit(msg, tmpdirs=None, image_hash="", exit_code=1):
if not tmpdirs:
tmpdirs = []
if msg: print(msg)
for tdir in tmpdirs: run_cmd("rm -rf %s" % tdir)
if image_hash: run_cmd("docker rm -f %s" % image_hash)
exit(exit_code)
def write_json(outfile, cache):
outdir = dirname(outfile)
if not exists(outdir): run_cmd("mkdir -p %s" % outdir)
ofile = open(outfile, 'w')
if ofile:
ofile.write(json.dumps(cache, sort_keys=True, indent=2,separators=(',',': ')))
ofile.close()
def fix_lognames(workflow_dir):
workflow_id = os.path.basename(workflow_dir).split("_",1)[1]
for log in glob.glob(os.path.join(workflow_dir,"step*_*.log")):
logname = os.path.basename(log)
step = logname.split("_",1)[0]
deslog = step+".log"
if logname.endswith('_dasquery.log'): deslog = '%s_%s.log' % (step, workflow_id)
run_cmd("cp %s %s/%s" % (log, workflow_dir, deslog))
def cleanup_exit(msg, tmpdirs=None, image_hash="", exit_code=1):
if not tmpdirs:
tmpdirs = []
if msg: print(msg)
for tdir in tmpdirs:
run_cmd("rm -rf %s" % tdir)
if image_hash: run_cmd("docker rm -f %s" % image_hash)
exit(exit_code)
def fix_lognames(workflow_dir):
workflow_id = os.path.basename(workflow_dir).split("_",1)[1]
for log in glob.glob(os.path.join(workflow_dir,"step*_*.log")):
logname = os.path.basename(log)
step = logname.split("_",1)[0]
deslog = step+".log"
if logname.endswith('_dasquery.log'): deslog = '%s_%s.log' % (step, workflow_id)
run_cmd("cp %s %s/%s" % (log, workflow_dir, deslog))
def update_worklog(workflow_dir, jobs):
if not jobs["commands"]: return False
workflow_logfile=os.path.join(workflow_dir,"workflow.log")
if not os.path.exists(workflow_logfile): return False
workflow_time=0
exit_codes=""
test_passed=""
test_failed=""
steps_res=[]
failed=False
step_num = 0
for job in jobs["commands"]:
step_num+=1
try:
cmd_step = int(job['command'].split(" step",1)[-1].strip().split(" ")[0])
while cmd_step>step_num:
das_log = os.path.join(workflow_dir,"step%s_dasquery.log" % step_num)
step_num+=1
if os.path.exists(das_log):
e, o = run_cmd("grep ' tests passed,' %s" % workflow_logfile)
if o=="": return False
ecodes = o.split()
if ecodes[step_num-2]=="0":
exit_codes+=" 1"
test_passed+=" 0"
test_failed+=" 1"
failed=True
steps_res.append("FAILED")
continue
exit_codes+=" 0"
test_passed+=" 1"
test_failed+=" 0"
steps_res.append("PASSED")
except Exception as e:
pass
if job["exit_code"]==-1: failed=True
if job["exit_code"]>0:
exit_codes+=" "+str(job["exit_code"])
test_passed+=" 0"
test_failed+=" 1"
failed=True
steps_res.append("FAILED")
else:
exit_codes+=" 0"
test_failed+=" 0"
if failed: test_passed+=" 0"
else: test_passed+=" 1"
steps_res.append("NORUN" if failed else "PASSED")
step_str = ""
for step, res in enumerate(steps_res): step_str = "%s Step%s-%s" % (step_str, step, res)
e, o = run_cmd("grep ' exit: ' %s | sed 's|exit:.*$|exit: %s|'" % (workflow_logfile, exit_codes.strip()))
o = re.sub("\s+Step0-.+\s+-\s+time\s+",step_str+" - time ",o)
wfile = open(workflow_logfile,"w")
wfile.write(o+"\n")
wfile.write("%s tests passed, %s failed\n" % (test_passed.strip(), test_failed.strip()))
wfile.close()
return True
def send_comparison_ready_message(repo, pr_number, tests_results_url, comparison_errors_file, wfs_with_das_inconsistency_file, missing_map ):
pull_request = repo.get_pull(pr_number)
message = COMPARISON_READY_MSG +'\n' + tests_results_url
wfs_with_errors = ''
for line in open( comparison_errors_file ):
line = line.rstrip()
parts = line.split( ';' )
wf = parts[ 0 ]
step = parts[ 1 ]
wfs_with_errors += ( wf + ' step ' + step + '\n' )
if wfs_with_errors != '':
error_info = COMPARISON_INCOMPLETE_MSG.format( workflows=wfs_with_errors )
message += '\n\n' + error_info
wfs_das_inconsistency = open( wfs_with_das_inconsistency_file ).readline().rstrip().rstrip(',').split( ',' )
if '' in wfs_das_inconsistency:
wfs_das_inconsistency.remove( '' )
if wfs_das_inconsistency:
das_inconsistency_info = DAS_INCONSISTENCY_MSG.format( workflows=', '.join( wfs_das_inconsistency ) )
message += '\n\n' + das_inconsistency_info
if missing_map and exists (missing_map):
missing = []
for line in open(missing_map):
line = line.strip()
if line: missing.append(" * "+line)
if missing:
from categories import COMPARISON_MISSING_MAP
map_notify = ", ".join([ "@"+u for u in COMPARISON_MISSING_MAP] )
message += "\n\n"+map_notify+" comparisons for the following workflows were not done due to missing matrix map:\n"+"\n".join(missing)
alt_comp_dir = join(dirname(comparison_errors_file), "upload","alternative-comparisons")
print("Alt comparison directory: ",alt_comp_dir)
if exists(alt_comp_dir):
err, out = run_cmd("grep ' Compilation failed' %s/runDQMComp-*.log" % alt_comp_dir)
print(out)
if not err:
err_wfs = {}
for line in out.split("\n"):
wf = line.split(".log:",1)[0].split("runDQMComp-")[-1]
err_wfs [wf]=1
if err_wfs: message += "\n\nAlternative comparison was/were failed for workflow(s):\n"+"\n".join(list(err_wfs.keys()))
JRCompSummaryLog = join(dirname(comparison_errors_file), "upload/validateJR/qaResultsSummary.log")
print("JR comparison Summary: ",JRCompSummaryLog)
if exists(JRCompSummaryLog):
err, out = run_cmd("cat %s" % JRCompSummaryLog)
if (not err) and out:
message += "\n\nComparison Summary:\n"
for l in out.split("\n"):
if l.strip(): message += " - %s\n" % l.strip()
send_message_pr( pull_request, message )
def send_comparison_ready_message(repo, pr_number, tests_results_url, comparison_errors_file, wfs_with_das_inconsistency_file, missing_map ):
pull_request = repo.get_pull(pr_number)
message = COMPARISON_READY_MSG +'\n' + tests_results_url
wfs_with_errors = ''
for line in open( comparison_errors_file ):
line = line.rstrip()
parts = line.split( ';' )
wf = parts[ 0 ]
step = parts[ 1 ]
wfs_with_errors += ( wf + ' step ' + step + '\n' )
if wfs_with_errors != '':
error_info = COMPARISON_INCOMPLETE_MSG.format( workflows=wfs_with_errors )
message += '\n\n' + error_info
wfs_das_inconsistency = open( wfs_with_das_inconsistency_file ).readline().rstrip().rstrip(',').split( ',' )
if '' in wfs_das_inconsistency:
wfs_das_inconsistency.remove( '' )
if wfs_das_inconsistency:
das_inconsistency_info = DAS_INCONSISTENCY_MSG.format( workflows=', '.join( wfs_das_inconsistency ) )
message += '\n\n' + das_inconsistency_info
if missing_map and exists (missing_map):
missing = []
for line in open(missing_map):
line = line.strip()
if line: missing.append(" * "+line)
if missing:
from categories import COMPARISON_MISSING_MAP
map_notify = ", ".join([ "@"+u for u in COMPARISON_MISSING_MAP] )
message += "\n\n"+map_notify+" comparisons for the following workflows were not done due to missing matrix map:\n"+"\n".join(missing)
alt_comp_dir = join(dirname(comparison_errors_file), "upload","alternative-comparisons")
print("Alt comparison directory: ",alt_comp_dir)
if exists(alt_comp_dir):
err, out = run_cmd("grep ' Compilation failed' %s/runDQMComp-*.log" % alt_comp_dir)
print(out)
if not err:
err_wfs = {}
for line in out.split("\n"):
wf = line.split(".log:",1)[0].split("runDQMComp-")[-1]
err_wfs [wf]=1
if err_wfs: message += "\n\nAlternative comparison was/were failed for workflow(s):\n"+"\n".join(list(err_wfs.keys()))
JRCompSummaryLog = join(dirname(comparison_errors_file), "upload/validateJR/qaResultsSummary.log")
print("JR comparison Summary: ",JRCompSummaryLog)
if exists(JRCompSummaryLog):
err, out = run_cmd("cat %s" % JRCompSummaryLog)
if (not err) and out:
message += "\n\nComparison Summary:\n"
for l in out.split("\n"):
if l.strip(): message += " - %s\n" % l.strip()
send_message_pr( pull_request, message )
def createJob(workflow, cmssw_ver, arch):
workflow_args = FixWFArgs(cmssw_ver, arch, workflow, GetMatrixOptions(cmssw_ver, arch))
cmd = format("rm -rf %(workflow)s %(workflow)s_*; mkdir %(workflow)s; cd %(workflow)s; PATH=%(das_utils)s:$PATH runTheMatrix.py --maxSteps=0 -l %(workflow)s %(workflow_args)s",workflow=workflow,workflow_args=workflow_args, das_utils=CMS_BOT_DIR+"/das-utils")
print("Running ",cmd)
e, o = run_cmd(cmd)
if e: print("ERROR:%s:%s" % (workflow, o))
try:
workflow_dir = glob.glob(format("%(workflow)s/%(workflow)s_*", workflow=workflow))[0]
run_cmd(format("mv %(workflow)s/runall-report-step123-.log %(workflow_dir)s/workflow.log; touch %(workflow_dir)s/cmdLog; mv %(workflow_dir)s .; rm -rf %(workflow)s", workflow=workflow, workflow_dir=workflow_dir))
print("Commands for workflow %s generated" % workflow)
except Exception as e:
print("ERROR: Creating workflow job:",workflow,str(e))
run_cmd("rm -rf %s %s_*" % (workflow,workflow))
def createJob(workflow, cmssw_ver, arch):
workflow_args = FixWFArgs(cmssw_ver, arch, workflow, GetMatrixOptions(cmssw_ver, arch))
cmd = format("rm -rf %(workflow)s %(workflow)s_*; mkdir %(workflow)s; cd %(workflow)s; PATH=%(das_utils)s:$PATH runTheMatrix.py --maxSteps=0 -l %(workflow)s %(workflow_args)s",workflow=workflow,workflow_args=workflow_args, das_utils=CMS_BOT_DIR+"/das-utils")
print("Running ",cmd)
e, o = run_cmd(cmd)
if e: print("ERROR:%s:%s" % (workflow, o))
try:
workflow_dir = glob.glob(format("%(workflow)s/%(workflow)s_*", workflow=workflow))[0]
run_cmd(format("mv %(workflow)s/runall-report-step123-.log %(workflow_dir)s/workflow.log; touch %(workflow_dir)s/cmdLog; mv %(workflow_dir)s .; rm -rf %(workflow)s", workflow=workflow, workflow_dir=workflow_dir))
print("Commands for workflow %s generated" % workflow)
except Exception as e:
print("ERROR: Creating workflow job:",workflow,str(e))
run_cmd("rm -rf %s %s_*" % (workflow,workflow))
def run(self):
IBThreadBase.run(self)
if platform.system() == 'Darwin':
print('unitTest> Skipping unit tests for MacOS')
return
if (self.xType == 'GPU') or ("_GPU_X" in os.environ["CMSSW_VERSION"]):
cmd = "cd " + self.startDir + "; scram b -f echo_cuda_USED_BY | tr ' ' '\\n' | grep '^\\(self\\|cmssw\\)/' | cut -d/ -f2-3 > cuda_pkgs.txt; "
cmd = cmd + " cat cuda_pkgs.txt; mv src src.full;"
cmd = cmd + " for p in $(cat cuda_pkgs.txt); do mkdir -p src/${p} ; rsync -a src.full/${p}/ src/${p}/ ; done ; scram build -r echo_CXX"
ret = runCmd(cmd)
if ret != 0:
print("ERROR when getting GPU unit-tests sources: cmd returned " + str(ret))
skiptests = ""
if 'lxplus' in getHostName(): skiptests = 'SKIP_UNITTESTS=ExpressionEvaluatorUnitTest'
TEST_PATH = os.environ['CMSSW_RELEASE_BASE'] + "/test/" + os.environ['SCRAM_ARCH']
err, cmd = run_cmd(
"cd " + self.startDir + ";scram tool info cmssw 2>&1 | grep CMSSW_BASE= | sed 's|^CMSSW_BASE=||'")
if cmd: TEST_PATH = TEST_PATH + ":" + cmd + "/test/" + os.environ['SCRAM_ARCH']
print(TEST_PATH)
try:
cmd = "cd " + self.startDir + "; touch nodelete.root nodelete.txt nodelete.log; sed -i -e 's|testing.log; *$(CMD_rm) *-f *$($(1)_objdir)/testing.log;|testing.log;|;s|test $(1) had ERRORS\") *\&\&|test $(1) had ERRORS\" >> $($(1)_objdir)/testing.log) \&\&|' config/SCRAM/GMake/Makefile.rules; "
cmd += " if which timeout 2>/dev/null; then TIMEOUT=timeout; fi ; "
cmd += 'PATH=' + TEST_PATH + ':$PATH ${TIMEOUT+timeout 3h} scram b -f -k -j ' + str(MachineCPUCount) + ' unittests ' + skiptests + ' >unitTests1.log 2>&1 ; '
cmd += 'touch nodelete.done; ls -l nodelete.*'
print('unitTest> Going to run ' + cmd)
ret = runCmd(cmd)
if ret != 0:
print("ERROR when running unit-tests: cmd returned " + str(ret))
except Exception as e:
print("ERROR during runtests : caught exception: " + str(e))
pass
try:
testLog = self.startDir + '/tmp/' + os.environ['SCRAM_ARCH'] + '/src/'
logFile = self.startDir + '/unitTests.log'
runCmd('rm -f %s; touch %s' % (logFile, logFile))
for packDir in glob.glob(testLog + '*/*'):
pack = packDir.replace(testLog, '')
runCmd("echo '>> Entering Package %s' >> %s" % (pack, logFile))
packDir += '/test'
if os.path.exists(packDir):
err, testFiles = run_cmd('find ' + packDir + ' -maxdepth 2 -mindepth 2 -name testing.log -type f')
for lFile in testFiles.strip().split('\n'):
if lFile: runCmd("cat %s >> %s" % (lFile, logFile))
runCmd("echo '>> Leaving Package %s' >> %s" % (pack, logFile))
runCmd("echo '>> Tests for package %s ran.' >> %s" % (pack, logFile))
except Exception as e:
pass
self.checkTestLogs()
self.logger.updateUnitTestLogs(self.xType)
return
def upload_logs(workflow, workflow_dir,exit_code):
files_to_keep = [ ".txt", ".xml", ".log", ".py", ".json","/cmdLog", "/hostname",".done" ]
basedir = os.path.dirname(workflow_dir)
for wf_file in glob.glob("%s/*" % workflow_dir):
found=False
for ext in files_to_keep:
if wf_file.endswith(ext):
found=True
break
if not found:
print("Removing ",wf_file)
run_cmd("rm -rf %s" % wf_file)
logger=LogUpdater(dirIn=os.environ["CMSSW_BASE"])
logger.updateRelValMatrixPartialLogs(basedir, os.path.basename(workflow_dir))
def run(self):
IBThreadBase.run(self)
if platform.system() == 'Darwin':
print('unitTest> Skipping unit tests for MacOS')
return
if self.xType == 'GPU':
cmd = "cd " + self.startDir + "; scram b -f echo_cuda_USED_BY | tr ' ' '\\n' | grep '^self/' | cut -d/ -f2-3 > cuda_pkgs.txt; mv src src.full;"
cmd = cmd + " for p in $(cat cuda_pkgs.txt); do mkdir -p src/${p} ; rsync -a src.full/${p}/ src/${p}/ ; done ; scram build -r echo_CXX"
ret = runCmd(cmd)
if ret != 0:
print("ERROR when getting GPU unit-tests sources: cmd returned " + str(ret))
skiptests = ""
if 'lxplus' in getHostName(): skiptests = 'SKIP_UNITTESTS=ExpressionEvaluatorUnitTest'
TEST_PATH = os.environ['CMSSW_RELEASE_BASE'] + "/test/" + os.environ['SCRAM_ARCH']
err, cmd = run_cmd(
"cd " + self.startDir + ";scram tool info cmssw 2>&1 | grep CMSSW_BASE= | sed 's|^CMSSW_BASE=||'")
if cmd: TEST_PATH = TEST_PATH + ":" + cmd + "/test/" + os.environ['SCRAM_ARCH']
print(TEST_PATH)
try:
cmd = "cd " + self.startDir + "; touch nodelete.root nodelete.txt nodelete.log; sed -i -e 's|testing.log; *$(CMD_rm) *-f *$($(1)_objdir)/testing.log;|testing.log;|;s|test $(1) had ERRORS\") *\&\&|test $(1) had ERRORS\" >> $($(1)_objdir)/testing.log) \&\&|' config/SCRAM/GMake/Makefile.rules; "
cmd += " if which timeout 2>/dev/null; then TIMEOUT=timeout; fi ; "
cmd += 'PATH=' + TEST_PATH + ':$PATH ${TIMEOUT+timeout 3h} scram b -f -k -j ' + str(
MachineCPUCount) + ' unittests ' + skiptests + ' >unitTests1.log 2>&1 ; '
cmd += 'touch nodelete.done; ls -l nodelete.*'
print('unitTest> Going to run ' + cmd)
ret = runCmd(cmd)
if ret != 0:
print("ERROR when running unit-tests: cmd returned " + str(ret))
except Exception as e:
print("ERROR during runtests : caught exception: " + str(e))
pass
try:
testLog = self.startDir + '/tmp/' + os.environ['SCRAM_ARCH'] + '/src/'
logFile = self.startDir + '/unitTests.log'
runCmd('rm -f %s; touch %s' % (logFile, logFile))
for packDir in glob.glob(testLog + '*/*'):
pack = packDir.replace(testLog, '')
runCmd("echo '>> Entering Package %s' >> %s" % (pack, logFile))
packDir += '/test'
if os.path.exists(packDir):
err, testFiles = run_cmd('find ' + packDir + ' -maxdepth 2 -mindepth 2 -name testing.log -type f')
for lFile in testFiles.strip().split('\n'):
if lFile: runCmd("cat %s >> %s" % (lFile, logFile))
runCmd("echo '>> Leaving Package %s' >> %s" % (pack, logFile))
runCmd("echo '>> Tests for package %s ran.' >> %s" % (pack, logFile))
except Exception as e:
pass
self.checkTestLogs()
self.logger.updateUnitTestLogs(self.xType)
return
def doCmd(cmd, dryRun=False, inDir=None, debug=True):
if not inDir:
if debug:
print("--> " + asctime() + " in ", getcwd(), " executing ", cmd)
else:
if debug:
print("--> " + asctime() + " in " + inDir + " executing ", cmd)
cmd = "cd " + inDir + "; " + cmd
sys.stdout.flush()
sys.stderr.flush()
start = time()
ret = 0
outX = ""
while cmd.endswith(";"):
cmd = cmd[:-1]
if dryRun:
print("DryRun for: " + cmd)
else:
ret, outX = run_cmd(cmd)
if debug:
print(outX)
stop = time()
if debug:
print("--> " + asctime() + " cmd took", stop - start,
"sec. (" + strftime("%H:%M:%S", gmtime(stop - start)) + ")")
sys.stdout.flush()
sys.stderr.flush()
return (ret, outX)
def upload_logs(workflow, workflow_dir,exit_code):
files_to_keep = [ ".txt", ".xml", ".log", ".py", ".json","/cmdLog", "/hostname",".done" ]
if (exit_code in [34304, 35584, 22016]) and os.getenv("CMSSW_VERSION","").startswith("CMSSW_10_1_"):
files_to_keep.append(".root")
basedir = os.path.dirname(workflow_dir)
for wf_file in glob.glob("%s/*" % workflow_dir):
found=False
for ext in files_to_keep:
if wf_file.endswith(ext):
found=True
break
if not found:
print("Removing ",wf_file)
run_cmd("rm -rf %s" % wf_file)
logger=LogUpdater(dirIn=os.environ["CMSSW_BASE"])
logger.updateRelValMatrixPartialLogs(basedir, os.path.basename(workflow_dir))
def runCmd(cmd):
while cmd.endswith(";"):
cmd = cmd[:-1]
print("Running cmd> ", cmd)
ret, out = run_cmd(cmd)
if out: print(out)
return ret
def get_merge_prs(prev_tag,
this_tag,
git_dir,
cmsprs,
cache={},
repo_name=None):
print("Getting merged Pull Requests b/w", prev_tag, this_tag)
cmd = format(
"GIT_DIR=%(git_dir)s"
" git log --graph --merges --pretty='%%s: %%P' %(previous)s..%(release)s | "
" grep ' Merge pull request #[1-9][0-9]* from ' | "
" sed 's|^.* Merge pull request #||' | "
" sed 's|Dr15Jones:clangRecoParticleFlowPFProducer:|Dr15Jones/clangRecoParticleFlowPFProducer:|' | "
" sed 's|/[^:]*:||;s|from ||'",
git_dir=git_dir,
previous=prev_tag,
release=this_tag)
error, notes = run_cmd(cmd)
print("Getting Merged Commits:", cmd)
print(notes)
if error:
print("Error while getting release notes.")
print(notes)
exit(1)
if not repo_name:
repo_name = basename(git_dir[:-4])
return fill_notes_description(notes, "cms-sw/" + repo_name, cmsprs, cache)
def get_docker_token(repo):
print("Getting docker.io token ....")
e, o = run_cmd('curl --silent --request "GET" "https://auth.docker.io/token?service=registry.docker.io&scope=repository:%s:pull"' % repo)
if e:
print(o)
exit(1)
return loads(o)['token']
def load_graph(release_queue, maxNodes):
command = MAGIC_COMMAND_GRAPH.replace('RELEASE_QUEUE', release_queue)
error, out = run_cmd(command)
prev_node_lane = {}
previous_lane = 1
node_number = 0
all_nodes = {}
for line in out.splitlines():
if maxNodes != -1 and node_number > maxNodes:
identify_automated_merges(all_nodes)
return all_nodes
#check if the line contains a node
if INFO_SEPARATOR in line:
node_number += 1
line_parts = line.split(INFO_SEPARATOR)
lanes = line_parts[0].replace('"', '').replace(' ', '')
lane = lanes.index('*') + 1
node_info = line_parts[1]
node_info_parts = node_info.split(",")
#hash, description
new_node = Node(node_info_parts[0], node_info_parts[1], lane)
all_nodes[node_info_parts[0]] = new_node
# for the first node I just add it without any conection
if node_number == 1:
set_previous_node_lane(prev_node_lane, lane, new_node)
continue
#changed lane?
if previous_lane < lane:
#connect this node with the preivous one from the previous lane
previous_node = get_previous_node_lane(prev_node_lane,
previous_lane)
else:
#connect this node with the previous one from of the same lane
previous_node = get_previous_node_lane(prev_node_lane, lane)
if previous_node == None:
set_previous_node_lane(prev_node_lane, lane, new_node)
previous_lane = lane
continue
link_nodes(new_node, previous_node)
set_previous_node_lane(prev_node_lane, lane, new_node)
all_nodes[node_info_parts[0]] = new_node
previous_lane = lane
identify_automated_merges(all_nodes)
return all_nodes
def get_ref_commit(repo, ref):
for n in ["tags", "heads"]:
error, out = run_cmd("curl -s -L https://api.github.com/repos/%s/git/refs/%s/%s" % (repo, n, ref))
if not error:
info = json.loads(out)
if "object" in info: return info["object"]["sha"]
print("Error: Unable to get sha for %s" % ref)
return None
def doDu(what):
error, out = run_cmd('du -k -s %s' % what)
if error:
print("Error while getting directory size.")
sys.exit(1)
results = [l.split() for l in out.split("\n")]
return dict([(pkg.strip().replace("src/", ''), int(sz.strip()*1024))
for (sz, pkg) in results])
def get_unused_days(eosfile):
e, o = run_cmd(
"%s fileinfo %s | grep 'Modify:' | sed 's|.* Timestamp: ||'" %
(eos_cmd, eosfile))
if e or (o == ""):
print("Error: Getting timestamp for %s\n%s" % (eosfile, o))
return -1
return int((time() - float(o)) / 86400)
def cmd_to_addon_test(command, addon_dir):
commandbase = command.replace(' ','_').replace('/','_')
logfile='%s.log' % commandbase[:150].replace("'",'').replace('"','').replace('../','')
e, o = run_cmd("ls -d %s/*/%s 2>/dev/null | tail -1" % (addon_dir, logfile))
if e or (o==""):
print("ERROR: %s -> %s" % (command, o))
return ("", "")
return (o.split("/")[-2], get_wf_error_msg(o, False).strip())
def get_command(self):
if self.input: return self.input.readline().strip()
cFile = 'auto-load'
while not exists(cFile):
sleep(0.2)
sleep(0.5)
o, cmd = run_cmd("head -1 %s; rm -f %s" % (cFile, cFile))
return cmd.strip()
def doDu(what):
error, out = run_cmd('du -k -s %s' % what)
if error:
print("Error while getting directory size.")
sys.exit(1)
results = [l.split() for l in out.split("\n")]
return dict([(pkg.strip().replace("src/", ''), int(sz.strip() * 1024))
for (sz, pkg) in results])
def get_ref_commit(repo, ref):
for n in ["tags", "heads"]:
error, out = run_cmd("curl -s -L https://api.github.com/repos/%s/git/refs/%s/%s" % (repo, n, ref))
if not error:
info = json.loads(out)
if "object" in info: return info["object"]["sha"]
print("Error: Unable to get sha for %s" % ref)
return None
def get_git_mt(path, filename):
status, rez = run_cmd(
'cd %s; git log -1 --format="%%ad" --date=unix -- %s' %
(path, filename))
if status is not 0:
print("ERROR, " + rez)
sys.exit(1) # todo throws an exception
return rez
def get_docker_manifest(repo, tag):
token = get_docker_token(repo)
print("Getting manifest for %s/%s" % (repo, tag))
e, o = run_cmd('curl --silent --request "GET" --header "Authorization: Bearer %s" "https://registry-1.docker.io/v2/%s/manifests/%s"' % (get_docker_token(repo), repo, tag))
if e:
print(o)
exit(1)
return loads(o)
def get_commit_info(repo, commit):
error, out = run_cmd("curl -s -L https://api.github.com/repos/%s/git/commits/%s" % (repo, commit))
if error:
tag = 'X (tag is undefined)' # TODO tag is undefined
print("Error, unable to get sha for tag %s" % tag)
return {}
commit_info = json.loads(out)
if "sha" in commit_info: return commit_info
return {}
def get_docker_token(repo):
print("Getting docker.io token ....")
e, o = run_cmd(
'curl --silent --request "GET" "https://auth.docker.io/token?service=registry.docker.io&scope=repository:%s:pull"'
% repo)
if e:
print(o)
exit(1)
return loads(o)['token']
def update_timestamp(timestramps, timestramps_file, store):
e, o = run_cmd("find %s -name '*.timestamp'" % store)
for ts_file in o.split("\n"):
if not ts_file.endswith('.timestamp'): continue
sha = basename(ts_file).replace(".timestamp", "")
with open(ts_file) as f:
timestramps[sha] = int(float(f.readlines()[0].strip()))
write_json(timestramps_file, timestramps)
cleanup_timestamps(store)
def get_commit_info(repo, commit):
error, out = run_cmd("curl -s -L https://api.github.com/repos/%s/git/commits/%s" % (repo, commit))
if error:
tag = 'X (tag is undefined)' # TODO tag is undefined
print("Error, unable to get sha for tag %s" % tag)
return {}
commit_info = json.loads(out)
if "sha" in commit_info: return commit_info
return {}
def get_docker_manifest(repo, tag):
token = get_docker_token(repo)
print("Getting manifest for %s/%s" % (repo, tag))
e, o = run_cmd(
'curl --silent --request "GET" --header "Authorization: Bearer %s" "https://registry-1.docker.io/v2/%s/manifests/%s"'
% (get_docker_token(repo), repo, tag))
if e:
print(o)
exit(1)
return loads(o)
def backport_pull(repo, pr, branch):
pr_branch = pr.base.label.split(":")[1]
print("Source Branch:", pr_branch)
if pr_branch == branch:
return "Warning: Can not backport, same branch %s vs %s" % (
pr_branch, branch), False
br = gh_repo.get_branch(branch)
commits = []
for c in pr.get_commits().reversed:
commits.insert(0, "git cherry-pick %s" % c.sha)
if not commits: return "There are no commits to backport", False
print("Cherry-pick commands:")
print(" " + "\n ".join(commits))
if len(commits) >= 250:
return "Error: Too many commits in PR %s\nBot can only handle max 250 commits." % len(
commits), False
new_branch = "backport-%s-%s" % (branch.replace("/", "_"), pr.number)
print("New Branch:", new_branch)
git_ref = ""
if repo.name == "cmssw": git_ref = "--reference " + CMSSW_GIT_REF
print("GIT REF:", git_ref)
e, o = run_cmd(
"rm -rf pr_backport; git clone --branch %s %s [email protected]:%s pr_backport && cd pr_backport && git checkout -b %s"
% (branch, git_ref, repo.full_name, new_branch))
if e:
print(o)
exit(1)
e, o = run_cmd('cd pr_backport; %s' % ";".join(commits))
if e:
return "Error: Failed to cherry-pick commits. Please backport this PR yourself.\n```" + o + "\n```", False
e, o = run_cmd("cd pr_backport; git push origin %s" % new_branch)
if e:
print(o)
exit(1)
run_cmd("rm -rf pr_backport")
newBody = "backport of #%s\n\n%s" % (pr.number, pr.body)
newPR = repo.create_pull(title=pr.title,
body=newBody,
base=branch,
head=new_branch)
return "Successfully backported PR #%s as #%s for branch %s" % (
pr.number, newPR.number, branch), True
def update_runall(self):
self.update_known_errors()
runall = os.path.join(self.outdir, "runall-report-step123-.log")
outFile = open(runall + ".tmp", "w")
status_ok = []
status_err = []
len_ok = 0
len_err = 0
for logFile in glob.glob(self.basedir + '/*/workflow.log'):
inFile = open(logFile)
for line in inFile:
if re.match("^\s*(\d+\s+)+tests passed,\s+(\d+\s+)+failed\s*$",
line):
res = line.strip().split(" tests passed, ")
res[0] = res[0].split()
res[1] = res[1].replace(" failed", "").split()
len_res = len(res[0])
if len_res > len_ok:
for i in range(len_ok, len_res):
status_ok.append(0)
len_ok = len_res
for i in range(0, len_res):
status_ok[i] = status_ok[i] + int(res[0][i])
len_res = len(res[1])
if len_res > len_err:
for i in range(len_err, len_res):
status_err.append(0)
len_err = len_res
for i in range(0, len_res):
status_err[i] = status_err[i] + int(res[1][i])
else:
outFile.write(line)
inFile.close()
outFile.write(" ".join(str(x) for x in status_ok) + " tests passed, " +
" ".join(str(x) for x in status_err) + " failed\n")
outFile.close()
save = True
if os.path.exists(runall):
e, o = run_cmd("diff %s.tmp %s | wc -l" % (runall, runall))
if o == "0": save = False
if save: run_cmd("mv %s.tmp %s" % (runall, runall))
return
def _getCPUCount():
cmd = "nproc"
if platform == "darwin":
cmd = "sysctl -n hw.ncpu"
error, count = run_cmd(cmd)
if error:
print("Warning: unable to detect cpu count. Using 4 as default value")
out = "4"
if not count.isdigit():
return 4
return int(count)
def get_repos(user, cache):
if user not in cache:
cache[user] = []
url = 'https://hub.docker.com/v2/repositories/%s?page_size=100' % user
while True:
e, o = run_cmd('curl -s -L %s' % url)
repo_data = json.loads(o)
if "results" in repo_data:
for r in repo_data["results"]:
cache[user].append(r["name"])
if "next" in repo_data and repo_data["next"]:
url = repo_data["next"]
else:
break
return cache[user]
def get_repos(user, cache):
if user not in cache:
cache[user] = []
url = 'https://hub.docker.com/v2/repositories/%s?page_size=100' % user
while True:
e , o = run_cmd('curl -s -L %s' % url)
repo_data = json.loads(o)
if "results" in repo_data:
for r in repo_data["results"]:
cache[user].append(r["name"])
if "next" in repo_data and repo_data["next"]:
url = repo_data["next"]
else:
break
return cache[user]
def _memorySizeGB():
cmd = ""
if platform == "darwin":
cmd = "sysctl -n hw.memsize"
elif platform.startswith("linux"):
cmd = "free -t -m | grep '^Mem: *' | awk '{print $2}'"
error, out = run_cmd(cmd)
if error:
print("Warning: unable to detect memory info. Using 8GB as default value")
return 8
if not out.isdigit():
return 8
from math import ceil
count = int(ceil(float(out)/1024))
if count == 0: count =1
return count
def auto_node_schedule(auto_jobs):
count=0
for job in auto_jobs:
jid = auto_jobs[job]
err, out = run_cmd("cat %s/jenkins/find-jenkins-job.groovy | %s groovy = '%s' 'JENKINS_DYNAMIC_JOB_ID=%s'" % (CMS_BOT_DIR, environ['JENKINS_CLI_CMD'],job,jid))
if err:
count+=1
prop_file = "jenkins-trigger-dynamic-job-%s.txt" % count
jpram = join(SCRIPT_DIR, 'auto-nodes', job)
run_cmd("echo 'JENKINS_DYNAMIC_JOB_NAME=%s' > %s" % (job, prop_file))
run_cmd("echo 'JENKINS_DYNAMIC_JOB_ID=%s' >> %s" % (jid, prop_file))
if exists (jpram):
run_cmd("cat %s >> %s" % (jpram, prop_file))
else:
print(out)
return
def get_merge_prs(prev_tag, this_tag, git_dir, cmsprs, cache={}, repo_name=None):
print("Getting merged Pull Requests b/w", prev_tag, this_tag)
cmd = format("GIT_DIR=%(git_dir)s"
" git log --graph --merges --pretty='%%s: %%P' %(previous)s..%(release)s | "
" grep ' Merge pull request #[1-9][0-9]* from ' | "
" sed 's|^.* Merge pull request #||' | "
" sed 's|Dr15Jones:clangRecoParticleFlowPFProducer:|Dr15Jones/clangRecoParticleFlowPFProducer:|' | "
" sed 's|/[^:]*:||;s|from ||'",
git_dir=git_dir,
previous=prev_tag,
release=this_tag)
error, notes = run_cmd(cmd)
print("Getting Merged Commits:", cmd)
print(notes)
if error:
print("Error while getting release notes.")
print(notes)
exit(1)
if not repo_name:
repo_name = basename(git_dir[:-4])
return fill_notes_description(notes, "cms-sw/" + repo_name, cmsprs, cache)
def parse_workflows(workflow_file):
err, out = run_cmd("cat %s" % workflow_file)
if err:
print(out)
exit(1)
wf = ""
wfs = {}
steps = 0
for line in out.split("\n"):
line =line.strip()
m = re.match("^.*\[(\d+)\] *: *(.+)$",line)
if not m: continue
step = m.group(1)
cmd = m.group(2).strip()
prefix, rest = line.split(":",1)
items = prefix.split(" ")
if re.match("^\d+(\.\d+|)$",items[0]): wf = items[0]
if not wf in wfs: wfs[wf]={}
wfs[wf][step]=re.sub(" +"," ",cmd)
steps += 1
print("%s: %s workflows, %s steps" % (workflow_file, len(wfs), steps))
return wfs
str_out = str_in.split(' ')
return str_out[-4].lstrip('/').rstrip('"'),int(str_out[-2])
def map_int_val(pair):
key , val = pair
return key , int(val)
#get relevant info
match_hn = re.compile('.*\|\/.*emails')
match_tmp = re.compile('.*\|\/.*Time\(s\)')
temp_fails = []
egrps = []
payload = {}
err , cmd_out = run_cmd('logwatch --range yesterday --detail 10 --service sendmail')
if err:
sys.exit(1)
for line in cmd_out.split('\n'):
if re.match(match_tmp,line): temp_fails.append(line)
elif 'Messages To Recipients:' in line: msgs_num = line
elif 'Addressed Recipients:' in line : adrpts = line
elif 'Bytes Transferred:' in line: byttr = line
elif re.match(match_hn,line): egrps.append(line)
#process info
yesterday = datetime.date.today() - datetime.timedelta(1)
timestp = int(yesterday.strftime("%s")) * 1000
temp_fails = dict(list(map(map_int_val,list((dict([list(map(cust_strip,x.split(' ')[-3:-1])) for x in temp_fails])).items()))))
msgs_num= list(map(str.strip,msgs_num.split(':')))
adrpts = list(map(str.strip, adrpts.split(':')))
from _py2with3compatibility import run_cmd
WF_PATH=sys.argv[1]
WF_DIR=basename(WF_PATH)
DES_DIR=join(sys.argv[2],WF_DIR)
htmls=[basename(h)[:-5] for h in glob.glob(WF_PATH+"/*.html")]
all_ok=[]
DES_DIR_CREATED=False
for h in sorted(htmls):
for s in all_ok[::-1]:
if h.startswith(s):
h = None
break
if not h: continue
e, o = run_cmd("grep 'Skipped:\|Null:\|Fail:' '%s/%s.html' | wc -l" % (WF_PATH,h))
if not e:
if int(o)>0:
if not DES_DIR_CREATED:
run_cmd("mkdir -p %s" % DES_DIR)
DES_DIR_CREATED=True
run_cmd("mv '%s/%s.html' '%s/%s.html'" % (WF_PATH,h,DES_DIR,h))
else:
all_ok.append("%s_" % h)
else:
print("ERROR: %s/%s\n%s" %(WF_DIR,h,o))
if DES_DIR_CREATED:
run_cmd("mv %s/*.png %s/" % (WF_PATH, DES_DIR))
run_cmd("echo ErrorDocument 404 /SDT/html/pr_comparison_ok.html > %s/.htaccess" % DES_DIR)
print("DONE:", WF_DIR)
errs[isrc][inc]=includes[isrc][inc]
#Free memory
checked = {}
includes = {}
uses = {}
usedby = {}
pkg_errs = {}
for e in errs:
pkg = '/'.join(e.split('/')[:2])
if pkg not in pkg_errs: pkg_errs[pkg] = {}
pkg_errs[pkg][e]=errs[e]
outdir = 'invalid-includes'
run_cmd('rm -f %s; mkdir %s' % (outdir, outdir))
all_count = {}
for p in sorted(pkg_errs):
all_count[p]=len(pkg_errs[p])
pdir = join(outdir, p)
run_cmd('mkdir -p %s' % pdir)
with open(join(pdir, 'index.html'),'w') as ref:
ref.write("<html><head></head><body>\n")
for e in sorted(pkg_errs[p]):
ref.write("<h3>%s:</h3>\n" % e)
for inc in sorted(errs[e].keys()):
url = 'https://github.com/cms-sw/cmssw/blob/%s/%s#L%s' % (environ['CMSSW_VERSION'],e,errs[e][inc])
ref.write('<l1><a href="%s">%s</a></l1></br>\n' % (url, inc))
ref.write("</ul></br>\n")
ref.write("</body></html>\n")
def port_pr(repo, pr_num, des_branch, dryRun=False):
pr = repo.get_pull(pr_num)
if pr.base.ref == des_branch:
print("Warning: Requested to make a PR to same branch", pr.base.ref)
return False
done_prs_id = get_ported_PRs(repo, pr.base.ref, des_branch)
if pr_num in done_prs_id:
print("Already ported as #", done_prs_id[pr.number])
return True
branch = repo.get_branch(des_branch)
print("Preparing checkout area:", pr_num, repo.full_name, pr.head.user.login, pr.head.ref, des_branch)
prepare_cmd = format(
"%(cmsbot)s/prepare-repo-clone-for-port.sh %(pr)s %(pr_user)s/%(pr_branch)s %(repo)s %(des_branch)s",
cmsbot=scriptPath,
pr=pr_num,
repo=repo.full_name,
pr_user=pr.head.user.login,
pr_branch=pr.head.ref,
des_branch=des_branch)
err, out = run_cmd(prepare_cmd)
print(out)
if err: return False
all_commits = set([])
for c in pr.get_commits():
all_commits.add(c.sha)
git_cmd = format("cd %(clone_dir)s; git cherry-pick -x %(commit)s",
clone_dir=pr.base.repo.name,
commit=c.sha)
err, out = run_cmd(git_cmd)
print(out)
if err: return False
git_cmd = format("cd %(clone_dir)s; git log %(des_branch)s..",
clone_dir=pr.base.repo.name,
des_branch=des_branch)
err, out = run_cmd(git_cmd)
print(out)
if err: return False
last_commit = None
new_commit = None
new_commits = {}
for line in out.split("\n"):
m = re.match('^commit\s+([0-9a-f]+)$', line)
if m:
print("New commit:", m.group(1), last_commit)
if last_commit:
new_commits[new_commit] = last_commit
new_commit = m.group(1)
new_commits[new_commit] = None
continue
m = re.match('^\s*\(cherry\s+picked\s+from\s+commit\s([0-9a-f]+)\)$', line)
if m:
print("found commit", m.group(1))
last_commit = m.group(1)
if last_commit: new_commits[new_commit] = last_commit
if pr.commits != len(new_commits):
print("Error: PR has ", pr.commits, " commits while we only found ", len(new_commits), ":", new_commits)
for c in new_commits:
all_commits.remove(new_commits[c])
if all_commits:
print("Something went wrong: Following commists not cherry-picked", all_commits)
return False
git_cmd = format("cd %(clone_dir)s; git rev-parse --abbrev-ref HEAD", clone_dir=pr.base.repo.name)
err, out = run_cmd(git_cmd)
print(out)
if err or not out.startswith("port-" + str(pr_num) + "-"): return False
new_branch = out
git_cmd = format("cd %(clone_dir)s; git push origin %(new_branch)s",
clone_dir=pr.base.repo.name,
new_branch=new_branch)
if not dryRun:
err, out = run_cmd(git_cmd)
print(out)
if err: return False
else:
print("DryRun: should have push %s branch" % new_branch)
from cms_static import GH_CMSSW_ORGANIZATION
newHead = "%s:%s" % (GH_CMSSW_ORGANIZATION, new_branch)
newBody = pr.body + "\nAutomatically ported from " + pr.base.ref + " #%s (original by @%s)." % (
pr_num, str(pr.head.user.login))
print(newHead)
print(newBody)
if not dryRun:
newPR = repo.create_pull(title=pr.title, body=newBody, base=des_branch, head=newHead)
else:
print("DryRun: should have created Pull Request for %s using %s" % (des_branch, newHead))
print("Every thing looks good")
git_cmd = format("cd %(clone_dir)s; git branch -d %(new_branch)s",
clone_dir=pr.base.repo.name,
new_branch=new_branch)
err, out = run_cmd(git_cmd)
print("Local branch %s deleted" % new_branch)
return True
sys.path.append(dirname(dirname(abspath(__file__)))) # in order to import top level modules
from _py2with3compatibility import run_cmd
script_path = abspath(dirname(argv[0]))
eos_cmd = "EOS_MGM_URL=root://eoscms.cern.ch /usr/bin/eos"
eos_base = "/eos/cms/store/user/cmsbuild"
unused_days_threshold = 180
try:
days=int(argv[1])
except:
days=30
if days<30:
days=30
if (unused_days_threshold-days)<30: unused_days_threshold=days+30
e , o = run_cmd("PYTHONPATH=%s/.. %s/ib-datasets.py --days %s" % (script_path, script_path, days))
if e:
print(o)
exit(1)
jdata = json.loads(o)
used = {}
for o in jdata['hits']['hits']:
used[o['_source']['lfn'].strip()]=1
e, o = run_cmd("%s find -f %s" % (eos_cmd, eos_base))
if e:
print(o)
exit(1)
total = 0
except Exception as e: print(e)
return
partial_log_dirpath=argv[1]
jobs=6
try: jobs=int(argv[2])
except: jobs=6
items = partial_log_dirpath.split("/")
if items[-1]!="pyRelValPartialLogs": exit(1)
release=items[-2]
arch=items[-6]
week, rel_sec = cmsswIB2Week(release)
rel_msec = rel_sec*1000
ex_fields=["rss", "vms", "pss", "uss", "shared", "data", "cpu"]
if not exists("%s/threads.txt" % partial_log_dirpath):
e, o = run_cmd("grep ' --nThreads ' %s/*/cmdLog | tail -1 | sed 's|.* *--nThreads *||;s| .*||'" % partial_log_dirpath)
if e:
print(o)
exit(1)
if not o: o="1"
run_cmd("echo %s > %s/threads.txt" % (o, partial_log_dirpath))
e, o = run_cmd("head -1 %s/threads.txt" % partial_log_dirpath)
if e:
print(o)
exit(1)
cmsThreads = o.strip('\n')
e, o = run_cmd("ls -d %s/*" % partial_log_dirpath)
threads = []
for wf in o.split("\n"):
if not isdir(wf): continue
def createJob(workflow, cmssw_ver, arch):
workflow_args = FixWFArgs(cmssw_ver, arch, workflow, GetMatrixOptions(cmssw_ver, arch))
cmd = format("rm -rf %(workflow)s %(workflow)s_*; mkdir %(workflow)s; cd %(workflow)s; PATH=%(das_utils)s:$PATH runTheMatrix.py --maxSteps=0 -l %(workflow)s %(workflow_args)s",workflow=workflow,workflow_args=workflow_args, das_utils=CMS_BOT_DIR+"/das-utils")
print("Running ",cmd)
e, o = run_cmd(cmd)
if e: print("ERROR:%s:%s" % (workflow, o))
try:
workflow_dir = glob.glob(format("%(workflow)s/%(workflow)s_*", workflow=workflow))[0]
run_cmd(format("mv %(workflow)s/runall-report-step123-.log %(workflow_dir)s/workflow.log; touch %(workflow_dir)s/cmdLog; mv %(workflow_dir)s .; rm -rf %(workflow)s", workflow=workflow, workflow_dir=workflow_dir))
print("Commands for workflow %s generated" % workflow)
except Exception as e:
print("ERROR: Creating workflow job:",workflow,str(e))
run_cmd("rm -rf %s %s_*" % (workflow,workflow))
pyRunDir=os.path.join(os.environ["CMSSW_BASE"],"pyRelval")
run_cmd("rm -rf %s; mkdir -p %s" % (pyRunDir, pyRunDir))
os.chdir(pyRunDir)
cmssw_ver = os.environ["CMSSW_VERSION"]
arch = os.environ["SCRAM_ARCH"]
#Run runTheMatrix with maxStep=0
thrds=[]
jobs=MachineCPUCount
wf_query=""
print("Creating jobs (%s) ...." % jobs)
for wf in sys.argv[1].split(","):
wf_query+=" OR workflow:"+wf
while len(thrds)>=jobs:
sleep(1)
thrds = [ t for t in thrds if t.is_alive() ]
t = threading.Thread(target=createJob, args=(wf, cmssw_ver, arch))
#!/bin/env python
from __future__ import print_function
from os.path import basename
from sys import argv, exit
from _py2with3compatibility import run_cmd
from json import loads, dumps
try:
authors_info = {}
repo = argv[1]
err, output = run_cmd("curl -s https://api.github.com/repos/" + repo + "/stats/contributors")
if err:
print(output)
exit(1)
data = loads(output)
for item in data:
authors_info[item['author']['login']] = item['total']
if not authors_info:
print(output)
exit(1)
print(basename(repo).upper().replace('-','_') + "_AUTHORS="+dumps(authors_info,sort_keys=True, indent=2))
except IndexError:
print("Repo Name Required ... Arugement missing !!!!")
exit (1)
for data in repos:
for u in data:
existing_repos = get_repos(u, cache)
for r in data[u]:
if r not in existing_repos:
print("%s/%s NEW" % (u, r))
if __name__ == "__main__":
parser = OptionParser(usage="%prog <pull-request-id>")
parser.add_option("-n", "--dry-run", dest="dryRun", action="store_true", help="Do not modify Github", default=False)
parser.add_option("-r", "--repo-list", dest="repo_list", help="Yaml file with list of repositories to create under docker hun", type=str, default=None)
opts, args = parser.parse_args()
repos = {}
if not opts.repo_list:
parser.error("Missing repository list file, please use -r|--repo-list option")
if opts.repo_list.startswith('https://'):
e, o = run_cmd('curl -s -L %s' % opts.repo_list)
if e:
print (o)
exit(1)
repos = yaml.load_all(o)
elif exists(opts.repo_list):
repos = yaml.load(open(opts.repo_list))
else:
print ("Error: No such file: %s" % opts.repo_list)
exit (1)
repo_cache = {}
process(repos, opts.dryRun, repo_cache)
def get_git_mt(path, filename):
status, rez = run_cmd('cd %s; git log -1 --format="%%ad" --date=unix -- %s' % (path, filename))
if status is not 0:
print("ERROR, " + rez)
sys.exit(1) # todo throws an exception
return rez
payload["status"] = 1
else:
payload["status"] = 0
utest= line.split(' ')[0]
payload["package"] = pkg
payload["name"] = utest
id = sha1(release + architecture + utest).hexdigest()
send_payload(index,document,id,json.dumps(payload))
line = it.next().strip()
except Exception as e:
print("File processed:", e)
else:
print("Invalid File Path")
#get log files
logs = run_cmd("find /data/sdt/buildlogs -mindepth 6 -maxdepth 6 -name 'unitTests-summary.log'")
logs = logs[1].split('\n')
#process log files
for logFile in logs:
flagFile = logFile + '.checked'
if not exists(flagFile):
print("Working on ",logFile)
process_ib_utests(logFile)
os.system('touch "' + flagFile + '"')
logs = run_cmd("find /data/sdt/buildlogs -mindepth 6 -maxdepth 6 -name 'unitTestLogs.zip'")
logs = logs[1].split('\n')
#process zip log files
for logFile in logs:
flagFile = logFile + '.checked'
if not exists(flagFile):
parser.add_argument("-m", "--message", dest="msg", help="Message to be posted s body of the GH issue",type=str, default='')
parser.add_argument("-R", "--report_file", dest="report_file", help="File name contaning the issue message",type=str, default='')
args = parser.parse_args()
mgs=""
if not args.repo: parser.error("Missing Repo")
if not args.title: parser.error("Missing PR title")
if args.msg: msg = re.sub("@N@","\n",args.msg)
elif args.report_file: msg = open(args.report_file).read()
else: parser.error("Missing issue message: -m|--message <message> OR -R|--report-file <file-path>")
print("Authenticating to Github and connecting to repo")
repo_dir = join(SCRIPT_DIR,'repos',args.repo.replace("-","_"))
if exists(join(repo_dir,"repo_config.py")): sys.path.insert(0,repo_dir)
import repo_config
gh = Github(login_or_token=open(expanduser(repo_config.GH_TOKEN)).read().strip())
print("Authentication succeeeded")
gh_repo = gh.get_repo(args.repo)
e, o = run_cmd("curl -s 'https://api.github.com/search/issues?q=%s+repo:%s+in:title+type:issue' | grep '\"number\"' | sed -e 's|.*: ||;s|,.*||'" % (quote(args.title),args.repo))
issue = None
if not e:
try:issue = gh_repo.get_issue(int(o))
except: pass
if issue:
print("Updaing comment")
issue.create_comment(msg)
else:
print("Creating issue request")
gh_repo.create_issue(args.title, msg)
def update_hostname(workflow_dir): run_cmd("hostname > %s/hostname" % workflow_dir)
def update_known_error(worflow, workflow_dir):
for wf_file in glob.glob("%s/*" % workflow_dir):
found=False
for ext in files_to_keep:
if wf_file.endswith(ext):
found=True
break
if not found:
print("Removing ",wf_file)
run_cmd("rm -rf %s" % wf_file)
logger=LogUpdater(dirIn=os.environ["CMSSW_BASE"])
logger.updateRelValMatrixPartialLogs(basedir, os.path.basename(workflow_dir))
if __name__ == "__main__":
jobs=json.load(open(sys.argv[1]))
exit_code = 0
for cmd in jobs["commands"]:
if cmd["exit_code"]>0:
exit_code=cmd["exit_code"]
break
workflow = jobs["name"]
workflow_dir=os.path.abspath(glob.glob("%s_*" % workflow)[0])
run_cmd("mv %s %s/job.json" % (sys.argv[1], workflow_dir))
fix_lognames(workflow_dir)
if update_worklog(workflow_dir, jobs):
update_cmdlog(workflow_dir, jobs)
update_timelog(workflow_dir, jobs)
update_hostname(workflow_dir)
update_known_error(workflow, workflow_dir)
if not 'CMSSW_DRY_RUN' in os.environ: upload_logs(workflow, workflow_dir, exit_code)
