def main():
stub = new_client()
TOTAL_JOBS_TODO = 0
jobs = []
outputlist = []
for i in range(16, 17):
dsc = descriptor
dsc['container'][
'cmd'] = "sh -lc '/input/images/makegcc/run.sh {} > /output/test.txt 2>&1'".format(
i)
job = Job(
input=json.dumps(dsc),
kind="docker",
)
job = stub.CreateJob(job)
jobs.append(job)
print("Job", job)
TOTAL_JOBS_TODO += 1
while TOTAL_JOBS_TODO > 0:
time.sleep(3)
for idx, job in enumerate(jobs):
if job.status in STATUS_FINAL:
continue
jobs[idx] = stub.GetJob(RequestWithId(id=job.id))
print("[{}] Job :\n {}\n".format(time.time(), jobs[idx]))
if jobs[idx].status in STATUS_FINAL:
TOTAL_JOBS_TODO -= 1
if jobs[idx].status == Job.FAILED:
print("Job failed!")
else:
x = json.loads(jobs[idx].output)
sxtemp = x[0]
d = json.loads(jobs[idx].input)
cmd = d['container']["cmd"]
temp = cmd.find(" ", 36)
cores = int(cmd[temp:cmd.find(">") - 1])
tempvar = sxtemp[sxtemp.find("run { date "):]
substr = "elapsed = "
tempvar = tempvar[tempvar.find(substr):]
tempvar = tempvar[len(substr):tempvar.
find(" ", len(substr))]
timevar = float(tempvar)
paramsvector = "--enable-languages=c,c++,fortran --disable-multilib --disable-bootstrap --build=x86_64-linux-gnu"
outputlist.append({
"cores": cores,
"program": {
"image": d['container']["name"],
"cmd": cmd
},
"time": timevar,
"paramsvector": paramsvector
})
print("result:", x)
dump_data(outputlist, path)
print("All done!")
def main():
stub = new_client()
TOTAL_JOBS_TODO = 0
jobs = []
outputlist = []
for i in range(1, 17):
dsc = descriptor
dsc['container'][
'cmd'] = "sh -lc 'python3 run.py 7207200 {} > /output/test.txt'".format(
i)
job = Job(
input=json.dumps(dsc),
kind="docker",
)
job = stub.CreateJob(job)
jobs.append(job)
print("Job", job)
TOTAL_JOBS_TODO += 1
while TOTAL_JOBS_TODO > 0:
time.sleep(3)
for idx, job in enumerate(jobs):
if job.status in STATUS_FINAL:
continue
jobs[idx] = stub.GetJob(RequestWithId(id=job.id))
print("[{}] Job :\n {}\n".format(time.time(), jobs[idx]))
if jobs[idx].status in STATUS_FINAL:
TOTAL_JOBS_TODO -= 1
if jobs[idx].status == Job.FAILED:
print("Job failed!")
else:
x = json.loads(jobs[idx].output)
timevar = float(x[0][13:])
d = json.loads(jobs[idx].input)
cmd = d['container']["cmd"]
param = int(cmd[23:cmd.find(" ", 24)])
temp = cmd.find(" ", 24)
cores = int(cmd[temp:cmd.find(">") - 1])
outputlist.append({
"cores": cores,
"program": {
"image": d['container']["name"],
"cmd": cmd
},
"time": timevar,
"paramsvector": [param]
})
print("result:", x)
dump_data(outputlist, path)
print("All done!")
def do_docker_job(job, completion_event):
logger.debug("Got descriptor: {}".format(job.input))
try:
job.status = Job.RUNNING
stub = new_client()
stub.ModifyJob(job, timeout=5)
del stub
process(job)
stub = new_client()
job.status = Job.COMPLETED
stub.ModifyJob(job, timeout=5)
completion_event.set()
logger.debug("Finished")
except BaseException as e:
capture_exception()
if job.status != Job.COMPLETED:
job.status = Job.FAILED
debug_info = {
"hostname": socket.gethostname(),
"exception": str(e),
"traceback": traceback.format_exc()
}
job.metadata = json.dumps(debug_info)
stub = new_client()
stub.ModifyJob(job, timeout=5)
logger.error(str(e))
logger.error(traceback.format_exc())
completion_event.set()
raise e
def main():
break_lock()
worker = Worker(
new_client(),
"docker",
do_docker_job,
threads_num=config.THREADS_NUM,
sleep_time=config.SLEEP_TIME,
)
signal.signal(signal.SIGQUIT, lambda n, f: sigquit_handler(n, f, worker))
logger.debug("Starting worker...")
worker.start()
def main():
stub = new_client()
job = Job(
input=json.dumps(descriptor),
kind="docker",
)
job = stub.CreateJob(job)
print("Job", job)
while True:
time.sleep(3)
job = stub.GetJob(RequestWithId(id=job.id))
print("[{}] Job :\n {}\n".format(time.time(), job))
if job.status in STATUS_FINAL:
break
if job.status == Job.FAILED:
print("Job failed!")
print("result:", json.loads(job.output))
default=0
)
parser.add_argument(
'--jobsNum',
type=int,
help='Number of jobs',
default=1
)
args = parser.parse_args()
# jobsNum = 100
jobsNum = args.jobsNum
nEvents_in = fileLen[args.fileN*1000]
#nEvents_in = 100
n = nEvents_in
k = jobsNum
startPoints = [i * (n // k) + min(i, n % k) for i in range(k)]
chunkLength = [(n // k) + (1 if i < (n % k) else 0) for i in range(k)]
chunkLength[-1] = chunkLength[-1] - 1
my_options = {'startPoints':startPoints, 'chunkLength':chunkLength, 'fileN':args.fileN, 'jobsNum':jobsNum}
tag = f'{RUN}_oneshot'
print("result:", makeRun(my_options, tag=tag, verbose=True))
if __name__ == '__main__':
stub = new_client()
main()