def main(**kwds): R = dispatch_script.main(**kwds) pid = R['pid'] fname = R['exelog_fname_json'] dry = kwds.get('dry',False) # dispatch JSON compiler if 'fname' in kwds: row_fname = kwds['fname'] col_fname = kwds['fname'] elif 'fname1' in kwds and 'fname2' in kwds: row_fname = kwds['fname1'] col_fname = kwds['fname2'] else: raise Exception, "No matrix filenames fname or (fname1, fname2)" cmd = "time python %(script)s fname=%(fname)s row_fname=%(row_fname)s col_fname=%(col_fname)s outdir=%(outdir)s" % {'script':WRAPPER_FNAME, 'fname':fname, 'row_fname':row_fname, 'col_fname':col_fname, 'outdir':kwds['outdir']} Q = qsub.Qsub(hours=12, n_ppn="max", jobname="toR_%s"%pid) Q.add(cmd) pid = Q.submit(dry) print Q.script() print "Submitted R-compile script, job ID: %s" % pid return pid
def main(n_permutes=1, fname=None, fname1=None, fname2=None, outdir=None, dry=False, **kwds): print outdir n_permutes = int(n_permutes) assert n_permutes >= 1 if not os.path.exists(outdir): make_dir(outdir) print "Created outdir %s" % outdir os.chdir(outdir) # We don't compile an all-permutations report, so don't compute these variables. ## now_timestamp = datetime.datetime.now().isoformat('_') ## out_fname = os.path.join(outdir, "permute%d_report.%s.txt" % (n_permutes, now_timestamp)) if fname: mtype = "self" assert os.path.exists(fname) M, fname_work = load_cp_and_make_bin(fname, outdir) print "Loading single matrix for permutation testing." else: mtype = "dual" assert os.path.exists(fname1) and os.path.exists(fname2) M1, fname_work1 = load_cp_and_make_bin(fname1, outdir) M2, fname_work2 = load_cp_and_make_bin(fname2, outdir) print "Loading two matrices for permutation testing." # Dispatch dependency matrices, collect completion pids and execution logs. PIDs = [] JSONs = [] for i in range(1, n_permutes + 1): perm_dir = os.path.join(outdir, "permuted_dep_matrix_%d" % i) if not os.path.exists(perm_dir): make_dir(perm_dir) print "Created permutation results directory in outdir: %s" % perm_dir ## Dispatch dependency computation if mtype == "self": print "Permuting %s, iteration %d..." % (fname, i) permute_rows(M) perm_fname = bname(fname).rpartition(".")[0] + ".perm%d.pkl" % i perm_path = os.path.join(perm_dir, perm_fname) print "Saving permutated copy as %s..." % perm_path print save(M, perm_path, ftype="pkl") print "Calling dispatcher......\n................" R = dispatch_script.main(fname=perm_path, outdir=perm_dir, dry=dry, **kwds) else: # mtype == "dual" print "Permuting %s, iteration %d..." % (fname1, i) permute_rows(M1) perm_fname = bname(fname1).rpartition(".")[0] + ".perm%d.pkl" % i perm_path = os.path.join(perm_dir, perm_fname) print "Saving permutated copy as %s..." % perm_path save(M1, perm_path, ftype="pkl") print "Calling dispatcher..." R = dispatch_script.main(fname1=perm_path, fname2=fname_work2, outdir=perm_dir, dry=dry, **kwds) PIDs.append(R["pid"]) JSONs.append(R["exelog_fname_json"]) ## Given expected results from dispatcher, dispatch permutation report generator ## to be executed after all dispatched permutations complete. if mtype == "self": jobname = "END_PERM_COMP_" + bname(fname) else: jobname = "END_PERM_COMP_" + bname(fname1) + ":" + bname(fname2) # Generate redundant report after all permutations complete. ## TODO: report should combine reports for all permutations rather than simply repeating # ---------------------------------------- ## reports for each matrix # Q = Qsub(jobname=jobname, n_nodes=1, n_ppn=12, hours=1, work_dir=outdir, email=True, after_jobids=PIDs) # cmd = shell_permutation_compile(json_fnames=JSONs, out_fname=out_fname) # Q.add(cmd) # pid = Q.submit(dry) # print Q.script() # print "Final permutation PID: %s" % pid # return pid # ---------------------------------------- ## In liu of the this function, simply create a capping dummy job to report status completion. Q = Qsub(jobname=jobname, n_nodes=1, n_ppn=1, hours=1, work_dir=outdir, email=True, after_jobids=PIDs) Q.echo("Jobs complete for %s" % jobname) pid = Q.submit(dry) print Q.script() print "Final permutation PID: %s" % pid return pid
def main(n_permutes=1, fname=None, fname1=None, fname2=None, outdir=None, dry=False, **kwds): print outdir n_permutes = int(n_permutes) assert n_permutes >= 1 if not os.path.exists(outdir): make_dir(outdir) print "Created outdir %s" % outdir os.chdir(outdir) # We don't compile an all-permutations report, so don't compute these variables. ## now_timestamp = datetime.datetime.now().isoformat('_') ## out_fname = os.path.join(outdir, "permute%d_report.%s.txt" % (n_permutes, now_timestamp)) if fname: mtype = "self" assert os.path.exists(fname) M, fname_work = load_cp_and_make_bin(fname, outdir) print "Loading single matrix for permutation testing." else: mtype = "dual" assert os.path.exists(fname1) and os.path.exists(fname2) M1, fname_work1 = load_cp_and_make_bin(fname1, outdir) M2, fname_work2 = load_cp_and_make_bin(fname2, outdir) print "Loading two matrices for permutation testing." # Dispatch dependency matrices, collect completion pids and execution logs. PIDs = [] JSONs = [] for i in range(1, n_permutes + 1): perm_dir = os.path.join(outdir, "permuted_dep_matrix_%d" % i) if not os.path.exists(perm_dir): make_dir(perm_dir) print "Created permutation results directory in outdir: %s" % perm_dir ## Dispatch dependency computation if mtype == "self": print "Permuting %s, iteration %d..." % (fname, i) permute_rows(M) perm_fname = bname(fname).rpartition(".")[0] + ".perm%d.pkl" % i perm_path = os.path.join(perm_dir, perm_fname) print "Saving permutated copy as %s..." % perm_path print save(M, perm_path, ftype="pkl") print "Calling dispatcher......\n................" R = dispatch_script.main(fname=perm_path, outdir=perm_dir, dry=dry, **kwds) else: # mtype == "dual" print "Permuting %s, iteration %d..." % (fname1, i) permute_rows(M1) perm_fname = bname(fname1).rpartition(".")[0] + ".perm%d.pkl" % i perm_path = os.path.join(perm_dir, perm_fname) print "Saving permutated copy as %s..." % perm_path save(M1, perm_path, ftype="pkl") print "Calling dispatcher..." R = dispatch_script.main(fname1=perm_path, fname2=fname_work2, outdir=perm_dir, dry=dry, **kwds) PIDs.append(R['pid']) JSONs.append(R['exelog_fname_json']) ## Given expected results from dispatcher, dispatch permutation report generator ## to be executed after all dispatched permutations complete. if mtype == "self": jobname = "END_PERM_COMP_" + bname(fname) else: jobname = "END_PERM_COMP_" + bname(fname1) + ":" + bname(fname2) # Generate redundant report after all permutations complete. ## TODO: report should combine reports for all permutations rather than simply repeating # ---------------------------------------- ## reports for each matrix # Q = Qsub(jobname=jobname, n_nodes=1, n_ppn=12, hours=1, work_dir=outdir, email=True, after_jobids=PIDs) # cmd = shell_permutation_compile(json_fnames=JSONs, out_fname=out_fname) # Q.add(cmd) # pid = Q.submit(dry) # print Q.script() # print "Final permutation PID: %s" % pid # return pid # ---------------------------------------- ## In liu of the this function, simply create a capping dummy job to report status completion. Q = Qsub(jobname=jobname, n_nodes=1, n_ppn=1, hours=1, work_dir=outdir, email=True, after_jobids=PIDs) Q.echo("Jobs complete for %s" % jobname) pid = Q.submit(dry) print Q.script() print "Final permutation PID: %s" % pid return pid