def main(): stub = new_client() TOTAL_JOBS_TODO = 0 jobs = [] outputlist = [] for i in range(16, 17): dsc = descriptor dsc['container'][ 'cmd'] = "sh -lc '/input/images/makegcc/run.sh {} > /output/test.txt 2>&1'".format( i) job = Job( input=json.dumps(dsc), kind="docker", ) job = stub.CreateJob(job) jobs.append(job) print("Job", job) TOTAL_JOBS_TODO += 1 while TOTAL_JOBS_TODO > 0: time.sleep(3) for idx, job in enumerate(jobs): if job.status in STATUS_FINAL: continue jobs[idx] = stub.GetJob(RequestWithId(id=job.id)) print("[{}] Job :\n {}\n".format(time.time(), jobs[idx])) if jobs[idx].status in STATUS_FINAL: TOTAL_JOBS_TODO -= 1 if jobs[idx].status == Job.FAILED: print("Job failed!") else: x = json.loads(jobs[idx].output) sxtemp = x[0] d = json.loads(jobs[idx].input) cmd = d['container']["cmd"] temp = cmd.find(" ", 36) cores = int(cmd[temp:cmd.find(">") - 1]) tempvar = sxtemp[sxtemp.find("run { date "):] substr = "elapsed = " tempvar = tempvar[tempvar.find(substr):] tempvar = tempvar[len(substr):tempvar. find(" ", len(substr))] timevar = float(tempvar) paramsvector = "--enable-languages=c,c++,fortran --disable-multilib --disable-bootstrap --build=x86_64-linux-gnu" outputlist.append({ "cores": cores, "program": { "image": d['container']["name"], "cmd": cmd }, "time": timevar, "paramsvector": paramsvector }) print("result:", x) dump_data(outputlist, path) print("All done!")
def main(): stub = new_client() TOTAL_JOBS_TODO = 0 jobs = [] outputlist = [] for i in range(1, 17): dsc = descriptor dsc['container'][ 'cmd'] = "sh -lc 'python3 run.py 7207200 {} > /output/test.txt'".format( i) job = Job( input=json.dumps(dsc), kind="docker", ) job = stub.CreateJob(job) jobs.append(job) print("Job", job) TOTAL_JOBS_TODO += 1 while TOTAL_JOBS_TODO > 0: time.sleep(3) for idx, job in enumerate(jobs): if job.status in STATUS_FINAL: continue jobs[idx] = stub.GetJob(RequestWithId(id=job.id)) print("[{}] Job :\n {}\n".format(time.time(), jobs[idx])) if jobs[idx].status in STATUS_FINAL: TOTAL_JOBS_TODO -= 1 if jobs[idx].status == Job.FAILED: print("Job failed!") else: x = json.loads(jobs[idx].output) timevar = float(x[0][13:]) d = json.loads(jobs[idx].input) cmd = d['container']["cmd"] param = int(cmd[23:cmd.find(" ", 24)]) temp = cmd.find(" ", 24) cores = int(cmd[temp:cmd.find(">") - 1]) outputlist.append({ "cores": cores, "program": { "image": d['container']["name"], "cmd": cmd }, "time": timevar, "paramsvector": [param] }) print("result:", x) dump_data(outputlist, path) print("All done!")
def do_docker_job(job, completion_event): logger.debug("Got descriptor: {}".format(job.input)) try: job.status = Job.RUNNING stub = new_client() stub.ModifyJob(job, timeout=5) del stub process(job) stub = new_client() job.status = Job.COMPLETED stub.ModifyJob(job, timeout=5) completion_event.set() logger.debug("Finished") except BaseException as e: capture_exception() if job.status != Job.COMPLETED: job.status = Job.FAILED debug_info = { "hostname": socket.gethostname(), "exception": str(e), "traceback": traceback.format_exc() } job.metadata = json.dumps(debug_info) stub = new_client() stub.ModifyJob(job, timeout=5) logger.error(str(e)) logger.error(traceback.format_exc()) completion_event.set() raise e
def main(): break_lock() worker = Worker( new_client(), "docker", do_docker_job, threads_num=config.THREADS_NUM, sleep_time=config.SLEEP_TIME, ) signal.signal(signal.SIGQUIT, lambda n, f: sigquit_handler(n, f, worker)) logger.debug("Starting worker...") worker.start()
def main(): stub = new_client() job = Job( input=json.dumps(descriptor), kind="docker", ) job = stub.CreateJob(job) print("Job", job) while True: time.sleep(3) job = stub.GetJob(RequestWithId(id=job.id)) print("[{}] Job :\n {}\n".format(time.time(), job)) if job.status in STATUS_FINAL: break if job.status == Job.FAILED: print("Job failed!") print("result:", json.loads(job.output))
default=0 ) parser.add_argument( '--jobsNum', type=int, help='Number of jobs', default=1 ) args = parser.parse_args() # jobsNum = 100 jobsNum = args.jobsNum nEvents_in = fileLen[args.fileN*1000] #nEvents_in = 100 n = nEvents_in k = jobsNum startPoints = [i * (n // k) + min(i, n % k) for i in range(k)] chunkLength = [(n // k) + (1 if i < (n % k) else 0) for i in range(k)] chunkLength[-1] = chunkLength[-1] - 1 my_options = {'startPoints':startPoints, 'chunkLength':chunkLength, 'fileN':args.fileN, 'jobsNum':jobsNum} tag = f'{RUN}_oneshot' print("result:", makeRun(my_options, tag=tag, verbose=True)) if __name__ == '__main__': stub = new_client() main()