def main(**kwds):
R = dispatch_script.main(**kwds)
pid = R['pid']
fname = R['exelog_fname_json']
dry = kwds.get('dry',False)
# dispatch JSON compiler
if 'fname' in kwds:
row_fname = kwds['fname']
col_fname = kwds['fname']
elif 'fname1' in kwds and 'fname2' in kwds:
row_fname = kwds['fname1']
col_fname = kwds['fname2']
else:
raise Exception, "No matrix filenames fname or (fname1, fname2)"
cmd = "time python %(script)s fname=%(fname)s row_fname=%(row_fname)s col_fname=%(col_fname)s outdir=%(outdir)s" % {'script':WRAPPER_FNAME, 'fname':fname, 'row_fname':row_fname, 'col_fname':col_fname, 'outdir':kwds['outdir']}
Q = qsub.Qsub(hours=12, n_ppn="max", jobname="toR_%s"%pid)
Q.add(cmd)
pid = Q.submit(dry)
print Q.script()
print "Submitted R-compile script, job ID: %s" % pid
return pid
def main(n_permutes=1, fname=None, fname1=None, fname2=None, outdir=None, dry=False, **kwds):
print outdir
n_permutes = int(n_permutes)
assert n_permutes >= 1
if not os.path.exists(outdir):
make_dir(outdir)
print "Created outdir %s" % outdir
os.chdir(outdir)
# We don't compile an all-permutations report, so don't compute these variables.
## now_timestamp = datetime.datetime.now().isoformat('_')
## out_fname = os.path.join(outdir, "permute%d_report.%s.txt" % (n_permutes, now_timestamp))
if fname:
mtype = "self"
assert os.path.exists(fname)
M, fname_work = load_cp_and_make_bin(fname, outdir)
print "Loading single matrix for permutation testing."
else:
mtype = "dual"
assert os.path.exists(fname1) and os.path.exists(fname2)
M1, fname_work1 = load_cp_and_make_bin(fname1, outdir)
M2, fname_work2 = load_cp_and_make_bin(fname2, outdir)
print "Loading two matrices for permutation testing."
# Dispatch dependency matrices, collect completion pids and execution logs.
PIDs = []
JSONs = []
for i in range(1, n_permutes + 1):
perm_dir = os.path.join(outdir, "permuted_dep_matrix_%d" % i)
if not os.path.exists(perm_dir):
make_dir(perm_dir)
print "Created permutation results directory in outdir: %s" % perm_dir
## Dispatch dependency computation
if mtype == "self":
print "Permuting %s, iteration %d..." % (fname, i)
permute_rows(M)
perm_fname = bname(fname).rpartition(".")[0] + ".perm%d.pkl" % i
perm_path = os.path.join(perm_dir, perm_fname)
print "Saving permutated copy as %s..." % perm_path
print save(M, perm_path, ftype="pkl")
print "Calling dispatcher......\n................"
R = dispatch_script.main(fname=perm_path, outdir=perm_dir, dry=dry, **kwds)
else: # mtype == "dual"
print "Permuting %s, iteration %d..." % (fname1, i)
permute_rows(M1)
perm_fname = bname(fname1).rpartition(".")[0] + ".perm%d.pkl" % i
perm_path = os.path.join(perm_dir, perm_fname)
print "Saving permutated copy as %s..." % perm_path
save(M1, perm_path, ftype="pkl")
print "Calling dispatcher..."
R = dispatch_script.main(fname1=perm_path, fname2=fname_work2, outdir=perm_dir, dry=dry, **kwds)
PIDs.append(R["pid"])
JSONs.append(R["exelog_fname_json"])
## Given expected results from dispatcher, dispatch permutation report generator
## to be executed after all dispatched permutations complete.
if mtype == "self":
jobname = "END_PERM_COMP_" + bname(fname)
else:
jobname = "END_PERM_COMP_" + bname(fname1) + ":" + bname(fname2)
# Generate redundant report after all permutations complete.
## TODO: report should combine reports for all permutations rather than simply repeating
# ----------------------------------------
## reports for each matrix
# Q = Qsub(jobname=jobname, n_nodes=1, n_ppn=12, hours=1, work_dir=outdir, email=True, after_jobids=PIDs)
# cmd = shell_permutation_compile(json_fnames=JSONs, out_fname=out_fname)
# Q.add(cmd)
# pid = Q.submit(dry)
# print Q.script()
# print "Final permutation PID: %s" % pid
# return pid
# ----------------------------------------
## In liu of the this function, simply create a capping dummy job to report status completion.
Q = Qsub(jobname=jobname, n_nodes=1, n_ppn=1, hours=1, work_dir=outdir, email=True, after_jobids=PIDs)
Q.echo("Jobs complete for %s" % jobname)
pid = Q.submit(dry)
print Q.script()
print "Final permutation PID: %s" % pid
return pid
def main(n_permutes=1,
fname=None,
fname1=None,
fname2=None,
outdir=None,
dry=False,
**kwds):
print outdir
n_permutes = int(n_permutes)
assert n_permutes >= 1
if not os.path.exists(outdir):
make_dir(outdir)
print "Created outdir %s" % outdir
os.chdir(outdir)
# We don't compile an all-permutations report, so don't compute these variables.
## now_timestamp = datetime.datetime.now().isoformat('_')
## out_fname = os.path.join(outdir, "permute%d_report.%s.txt" % (n_permutes, now_timestamp))
if fname:
mtype = "self"
assert os.path.exists(fname)
M, fname_work = load_cp_and_make_bin(fname, outdir)
print "Loading single matrix for permutation testing."
else:
mtype = "dual"
assert os.path.exists(fname1) and os.path.exists(fname2)
M1, fname_work1 = load_cp_and_make_bin(fname1, outdir)
M2, fname_work2 = load_cp_and_make_bin(fname2, outdir)
print "Loading two matrices for permutation testing."
# Dispatch dependency matrices, collect completion pids and execution logs.
PIDs = []
JSONs = []
for i in range(1, n_permutes + 1):
perm_dir = os.path.join(outdir, "permuted_dep_matrix_%d" % i)
if not os.path.exists(perm_dir):
make_dir(perm_dir)
print "Created permutation results directory in outdir: %s" % perm_dir
## Dispatch dependency computation
if mtype == "self":
print "Permuting %s, iteration %d..." % (fname, i)
permute_rows(M)
perm_fname = bname(fname).rpartition(".")[0] + ".perm%d.pkl" % i
perm_path = os.path.join(perm_dir, perm_fname)
print "Saving permutated copy as %s..." % perm_path
print save(M, perm_path, ftype="pkl")
print "Calling dispatcher......\n................"
R = dispatch_script.main(fname=perm_path,
outdir=perm_dir,
dry=dry,
**kwds)
else: # mtype == "dual"
print "Permuting %s, iteration %d..." % (fname1, i)
permute_rows(M1)
perm_fname = bname(fname1).rpartition(".")[0] + ".perm%d.pkl" % i
perm_path = os.path.join(perm_dir, perm_fname)
print "Saving permutated copy as %s..." % perm_path
save(M1, perm_path, ftype="pkl")
print "Calling dispatcher..."
R = dispatch_script.main(fname1=perm_path,
fname2=fname_work2,
outdir=perm_dir,
dry=dry,
**kwds)
PIDs.append(R['pid'])
JSONs.append(R['exelog_fname_json'])
## Given expected results from dispatcher, dispatch permutation report generator
## to be executed after all dispatched permutations complete.
if mtype == "self":
jobname = "END_PERM_COMP_" + bname(fname)
else:
jobname = "END_PERM_COMP_" + bname(fname1) + ":" + bname(fname2)
# Generate redundant report after all permutations complete.
## TODO: report should combine reports for all permutations rather than simply repeating
# ----------------------------------------
## reports for each matrix
# Q = Qsub(jobname=jobname, n_nodes=1, n_ppn=12, hours=1, work_dir=outdir, email=True, after_jobids=PIDs)
# cmd = shell_permutation_compile(json_fnames=JSONs, out_fname=out_fname)
# Q.add(cmd)
# pid = Q.submit(dry)
# print Q.script()
# print "Final permutation PID: %s" % pid
# return pid
# ----------------------------------------
## In liu of the this function, simply create a capping dummy job to report status completion.
Q = Qsub(jobname=jobname,
n_nodes=1,
n_ppn=1,
hours=1,
work_dir=outdir,
email=True,
after_jobids=PIDs)
Q.echo("Jobs complete for %s" % jobname)
pid = Q.submit(dry)
print Q.script()
print "Final permutation PID: %s" % pid
return pid
