def copy_lfns_to_eos(eos_lfns): threads = [] all_logs = {} logdir = "logs" run_cmd("rm -rf %s && mkdir -p %s" % (logdir , logdir)) job_monitor = {} already_done =0 total_lfns = len(eos_lfns) eos_lfns_to_copy = [] for lfn in eos_lfns: eos_file = "%s%s" % (eos_base, lfn) if eos_exists(eos_file) or (eos_exists(eos_file+".unused") and eos_rename(eos_file+".unused", eos_file)): already_done += 1 print("OK (%s/%s): %s" % (already_done, total_lfns, lfn)) elif opts.dryRun: print("DryRun: Copy %s -> %s" % (lfn, eos_file)) continue else: eos_lfns_to_copy.append(lfn) for lfn in eos_lfns_to_copy: eos_file = "%s%s" % (eos_base, lfn) while True: threads = get_alive_threads(threads) if(len(threads) < opts.jobs): log_file=logdir+"/"+sha256(lfn).hexdigest()+".log" all_logs[log_file]=lfn print("Copy (%s/%s): %s" % (already_done+len(all_logs), total_lfns, lfn)) t = Thread(name=lfn,target=copy_to_eos, args=(lfn, log_file)) job_monitor[lfn]=[int(time()), 0, 0,log_file] t.start() threads.append(t) break elif not check_dead_transfers(threads, job_monitor): sleep(10) while len(threads)>0: sleep(10) threads = get_alive_threads(threads) check_dead_transfers(threads, job_monitor) total_failed = 0 total_copied = 0 for log in all_logs: lfn = all_logs[log] err, out = run_cmd("cat %s" % log,debug=False) err, out = getstatusoutput("grep '^ALL_OK$' %s | wc -l" % log) if out=="0": total_failed+=1 print("FAIL (%s/%s): %s" % (already_done+total_copied+total_failed, total_lfns, lfn)) err, out = getstatusoutput("cat %s" % log) print(out) print("###################################") else: total_copied += 1 print("OK (%s/%s): %s" % (already_done+total_copied+total_failed, total_lfns, lfn)) run_cmd("rm -rf %s" % logdir) print("Total LFNs: %s" % total_lfns) print("Already available: %s" % already_done) print("Newly fetched: %s" % total_copied) print("Error: %s" % total_failed) return total_failed==0
def run_cmd(cmd, exit_on_error=True, debug=True): if debug: print(">> %s" % cmd) err, out = getstatusoutput(cmd) if err: if exit_on_error: print(out) exit(1) return err, out
def run_cmd(cmd, exit_on_error=True,debug=True): if debug: print(">> %s" % cmd) err, out = getstatusoutput(cmd) if err: if exit_on_error: print(out) exit(1) return err, out
def run(self, ignominyDir): self.initArea() if not os.path.exists(ignominyDir + '/igDone'): err = self.appType + 'BuildSet> Skipping test for release, ignominy test did not run or failed' self.setStatus('skip', err) return if not self.getRefFiles(): err = self.appType + 'BuildSet> Skipping test for release, There are errors in getting ref files.' self.setStatus('skip', err) return cmd = 'cd ' + self.startDir + '; eval `scramv1 run -sh`; cd ' + self.appDir for xtype in ['packages', 'tools']: for xsec in ['binary', 'source']: cmd += ' ; MakeBuildSet -f ./RefAppSet -o ' + xtype + ' -D ' + xsec + ' -d ' + ignominyDir + ' | sort > ' + xtype + '_' + xsec print(self.appType + 'BuildSet> Going to run ' + cmd) ret, outX = getstatusoutput(cmd) if outX: print(outX) if ret != 0: err = "ERROR when running MakeBuildSet: cmd returned " + str(ret) self.setStatus('error', err) return
def copy_lfns_to_eos(eos_lfns): threads = [] all_logs = {} logdir = "logs" run_cmd("rm -rf %s && mkdir -p %s" % (logdir, logdir)) job_monitor = {} already_done = 0 total_lfns = len(eos_lfns) eos_lfns_to_copy = [] for lfn in eos_lfns: eos_file = "%s%s" % (eos_base, lfn) if eos_exists(eos_file) or (eos_exists(eos_file + ".unused") and eos_rename(eos_file + ".unused", eos_file)): already_done += 1 print("OK (%s/%s): %s" % (already_done, total_lfns, lfn)) elif opts.dryRun: print("DryRun: Copy %s -> %s" % (lfn, eos_file)) continue else: eos_lfns_to_copy.append(lfn) for lfn in eos_lfns_to_copy: eos_file = "%s%s" % (eos_base, lfn) while True: threads = get_alive_threads(threads) if (len(threads) < opts.jobs): log_file = logdir + "/" + sha256(lfn).hexdigest() + ".log" all_logs[log_file] = lfn print("Copy (%s/%s): %s" % (already_done + len(all_logs), total_lfns, lfn)) t = Thread(name=lfn, target=copy_to_eos, args=(lfn, log_file)) job_monitor[lfn] = [int(time()), 0, 0, log_file] t.start() threads.append(t) break elif not check_dead_transfers(threads, job_monitor): sleep(10) while len(threads) > 0: sleep(10) threads = get_alive_threads(threads) check_dead_transfers(threads, job_monitor) total_failed = 0 total_copied = 0 for log in all_logs: lfn = all_logs[log] err, out = run_cmd("cat %s" % log, debug=False) err, out = getstatusoutput("grep '^ALL_OK$' %s | wc -l" % log) if out == "0": total_failed += 1 print( "FAIL (%s/%s): %s" % (already_done + total_copied + total_failed, total_lfns, lfn)) err, out = getstatusoutput("cat %s" % log) print(out) print("###################################") else: total_copied += 1 print( "OK (%s/%s): %s" % (already_done + total_copied + total_failed, total_lfns, lfn)) run_cmd("rm -rf %s" % logdir) print("Total LFNs: %s" % total_lfns) print("Already available: %s" % already_done) print("Newly fetched: %s" % total_copied) print("Error: %s" % total_failed) return total_failed == 0
def initArea(self): cmd = 'rm -rf ' + self.appDir + '; mkdir -p ' + self.appDir ret, outX = getstatusoutput(cmd) return
def run_cmd(cmd, exit_on_error=True): err, out = getstatusoutput(cmd) if err and exit_on_error: print(out) exit(1) return out