def main(): options,args = parser.parse_args(sys.argv[1:]) if len(args) < 1: parser.print_help() sys.exit(45) run=args[0] c = shapesim.read_config(run) run = c['run'] sim_config = c['sim_config'] mcmc_config = c['mcmc_config'] s2n_vals = c['s2n_vals'] ns2n = len(s2n_vals) d = shapesim.get_wq_dir(run, bytrial=True, fs='local') if not os.path.exists(d): os.makedirs(d) d = shapesim.get_output_dir(run, sub='bytrial') if not os.path.exists(d): os.makedirs(d) groups='' if options.groups is not None: groups = 'group: [%s]' % options.groups smax=numpy.iinfo('i8').max for is2n in xrange(ns2n): s2n = s2n_vals[is2n] npair, nsplit = get_npair_nsplit(c, is2n) for isplit in xrange(nsplit): job_name='%s-%03i-%03i' % (run,is2n,isplit) # note the wq logs are local wqurl = shapesim.get_wq_url(run,0,is2n,itrial=isplit,fs='local') output = shapesim.get_output_url(run, 0, is2n, itrial=isplit) seed = numpy.random.randint(smax) wlog("writing wq script:",wqurl) with open(wqurl,'w') as fobj: d={'job_name':job_name, 'version':options.version, 'groups':groups, 'pri':options.priority, 'sim_config':sim_config, 'mcmc_config':mcmc_config, 's2n':s2n, 'npair':npair, 'seed':seed, 'output':output} wqscript=_wqtemplate % d fobj.write(wqscript)
def write_condor_file(c, master_script, equal_time=False): run = c["run"] job_name = "-".join((run.split("-"))[1:]) condor_job_url = shapesim.get_condor_job_url(run) ns2n = len(c["s2n_vals"]) seconds_per = get_seconds_per_pair() if "desired_err" in c: do_by_noise = True else: do_by_noise = False print condor_job_url njobs = 0 smax = numpy.iinfo("i8").max with open(condor_job_url, "w") as fobj: text = _condor_template_head.format(master_script=master_script, overall_name=job_name) fobj.write(text) # for is2n in reversed(xrange(ns2n)): for is2n in xrange(ns2n): s2n = c["s2n_vals"][is2n] npair, nsplit = get_npair_nsplit(c, is2n) time_hours = npair * seconds_per / 3600.0 if time_hours > MAXTIME_HOURS: raise ValueError( "time is greater than %.2f " "hours: %d*%.2f/3600.0 = %s" % (MAXTIME_HOURS, npair, seconds_per, time_hours) ) print "nsplit:", nsplit, "npair:", npair, "time (hours):", time_hours njobs += nsplit for isplit in xrange(nsplit): output = shapesim.get_output_url(run, 0, is2n, itrial=isplit) logfile = output.replace(".rec", ".log") this_job_name = "%s-%03d-%03d" % (job_name, is2n, isplit) # seed = numpy.random.randint(smax) qdata = _queue_template.format( job_name=this_job_name, s2n=s2n, npair=npair, # seed=seed, output=output, logfile=logfile, ) fobj.write(qdata) print "total jobs:", njobs
def check_is2n(run, is2n, flist, full=False): fs=get_default_fs() c = read_config(run) npair,nsplit = get_npair_nsplit(c, is2n) ntot=npair*2 for isplit in xrange(nsplit): f=get_output_url(run, 0, is2n, itrial=isplit, fs=fs) if f not in flist: print 'missing:',f continue if full: t=None try: t=eu.io.read(f) except: print 'failed to read:',f if t is not None: if t.size != ntot: print "expected %d, got %d" % (npair,t.size)
def write_condor_file(c, master_script, equal_time=False, missing=False, max_jobs=10000): run=c['run'] overall_name = '-'.join( (run.split('-'))[1:] ) if missing: flist=get_flist(run) overall_name += 'missing' else: flist=None ns2n=len(c['s2n_vals']) seconds_per=get_seconds_per_pair(c) print 'seconds per:',seconds_per if 'desired_err' in c: do_by_noise=True else: do_by_noise=False njobs_thisfile=0 njobs=0 old_filenum=-1 filenum=0 smax=numpy.iinfo('i8').max fobj=start_new_file(run, filenum, master_script, overall_name, missing=missing) for is2n in xrange(ns2n): s2n=c['s2n_vals'][is2n] npair, nsplit = get_npair_nsplit(c, is2n) time_hours = npair*seconds_per/3600.0 if time_hours > MAXTIME_HOURS: raise ValueError("time is greater than %.2f " "hours: %d*%.2f/3600.0 = %s" % (MAXTIME_HOURS,npair,seconds_per,time_hours)) print ' nsplit:',nsplit,'npair:',npair,'time (hours):',time_hours for isplit in xrange(nsplit): output = shapesim.get_output_url(run, 0, is2n, itrial=isplit) logfile = output.replace('.fits','.log') this_job_name='%s-%03d-%05d' % (overall_name,is2n,isplit) qdata=_queue_template.format(job_name=this_job_name, s2n=s2n, npair=npair, output=output, logfile=logfile) do_write=True if missing and output in flist: do_write=False if do_write: njobs += 1 njobs_thisfile += 1 fobj.write(qdata) if njobs_thisfile >= max_jobs: filenum += 1 njobs_thisfile=0 fobj.close() fobj=start_new_file(run, filenum, master_script, overall_name, missing=missing) print 'total jobs:',njobs