def main(): options,args = parser.parse_args(sys.argv[1:]) if len(args) < 1: parser.print_help() sys.exit(45) run=args[0] c = shapesim.read_config(run) run = c['run'] sim_config = c['sim_config'] mcmc_config = c['mcmc_config'] s2n_vals = c['s2n_vals'] ns2n = len(s2n_vals) d = shapesim.get_wq_dir(run, bytrial=True, fs='local') if not os.path.exists(d): os.makedirs(d) d = shapesim.get_output_dir(run, sub='bytrial') if not os.path.exists(d): os.makedirs(d) groups='' if options.groups is not None: groups = 'group: [%s]' % options.groups smax=numpy.iinfo('i8').max for is2n in xrange(ns2n): s2n = s2n_vals[is2n] npair, nsplit = get_npair_nsplit(c, is2n) for isplit in xrange(nsplit): job_name='%s-%03i-%03i' % (run,is2n,isplit) # note the wq logs are local wqurl = shapesim.get_wq_url(run,0,is2n,itrial=isplit,fs='local') output = shapesim.get_output_url(run, 0, is2n, itrial=isplit) seed = numpy.random.randint(smax) wlog("writing wq script:",wqurl) with open(wqurl,'w') as fobj: d={'job_name':job_name, 'version':options.version, 'groups':groups, 'pri':options.priority, 'sim_config':sim_config, 'mcmc_config':mcmc_config, 's2n':s2n, 'npair':npair, 'seed':seed, 'output':output} wqscript=_wqtemplate % d fobj.write(wqscript)
def make_some_dirs(run): d = shapesim.get_condor_dir(run) outd = shapesim.get_output_dir(run, sub='bytrial') if not os.path.exists(d): os.makedirs(d) if not os.path.exists(outd): os.makedirs(outd) tmpdir='/data/esheldon/tmp' if not os.path.exists(tmpdir): os.makedirs(tmpdir)
def make_some_dirs(run): d = shapesim.get_condor_dir(run) # logdir=os.path.join(d,'logs') outd = shapesim.get_output_dir(run, sub="bytrial") if not os.path.exists(d): os.makedirs(d) # if not os.path.exists(logdir): # os.makedirs(logdir) if not os.path.exists(outd): os.makedirs(outd) tmpdir = "/data/esheldon/tmp" if not os.path.exists(tmpdir): os.makedirs(tmpdir)
def main(): options,args = parser.parse_args(sys.argv[1:]) if len(args) < 3: parser.print_help() sys.exit(45) run=args[0] nodes=int(args[1]) ppn=int(args[2]) np=nodes*ppn # seconds per ellip ring pair. this is for exp... seconds_per=get_seconds_per_pair() c = shapesim.read_config(run) sim_config = c['sim_config'] mcmc_config = c['mcmc_config'] nsplit = c['nsplit'] s2n_vals = c['s2n_vals'] s2n_fac = c['s2n_fac'] min_npair = c['min_npair'] ns2n = len(s2n_vals) pbsd = shapesim.get_pbs_dir(run) logdir=os.path.join(pbsd,'logs') outd = shapesim.get_output_dir(run, sub='bytrial') if not os.path.exists(pbsd): os.makedirs(pbsd) if not os.path.exists(logdir): os.makedirs(logdir) if not os.path.exists(outd): os.makedirs(outd) rstr=run job_name = '-'.join( (rstr.split('-'))[1:] ) script_url=shapesim.get_minions_script_url(run) script_base=os.path.basename(script_url) print script_url with open(script_url,'w') as fobj: c['version'] = options.version fobj.write(_script_template % c) print pbsf=shapesim.get_minions_url(run,0) cmdf=shapesim.get_commands_url(run,0) print cmdf ntot=0 with open(cmdf,'w') as fobj: for is2n in xrange(ns2n): s2n = s2n_vals[is2n] npair = shapesim.get_s2n_nrepeat(s2n, fac=s2n_fac) if npair < min_npair: npair = min_npair ntot += npair*nsplit for isplit in xrange(nsplit): output = shapesim.get_output_url(run, 0, is2n, itrial=isplit) outbname=os.path.basename(output) logf=outbname.replace('.rec','.log') cmd=_cmd_template.format(script_file=script_base, s2n=s2n, npair=npair, output=output, logfile=logf) fobj.write(cmd) fobj.write('\n') time_seconds = ntot*seconds_per/(np-1) print pbsf hours_raw = time_seconds/3600. hours = int(round(hours_raw)) + 1 print ' raw hours:',hours_raw,'hours used:',hours if options.version=='hopper': minions_cmd='minions-hopper' else: minions_cmd='minions' with open(pbsf,'w') as fobj: cmd_bname=os.path.basename(cmdf) pbslog=os.path.basename(pbsf)+'.pbslog' pbs_text=_pbs_template.format(minions_cmd=minions_cmd, job_name=job_name, pbslog=pbslog, nodes=nodes, ppn=ppn, np=np, hours=hours, commands_file=cmd_bname) fobj.write(pbs_text)