Exemplo n.º 1
0
def main():
    options,args = parser.parse_args(sys.argv[1:])

    if len(args) < 1:
        parser.print_help()
        sys.exit(45)

    run=args[0]

    c = shapesim.read_config(run)

    run         = c['run']
    sim_config  = c['sim_config']
    mcmc_config = c['mcmc_config']
    s2n_vals    = c['s2n_vals']

    ns2n = len(s2n_vals)

    d = shapesim.get_wq_dir(run, bytrial=True, fs='local')
    if not os.path.exists(d):
        os.makedirs(d)
    d = shapesim.get_output_dir(run, sub='bytrial')
    if not os.path.exists(d):
        os.makedirs(d)


    groups=''
    if options.groups is not None:
        groups = 'group: [%s]' % options.groups

    smax=numpy.iinfo('i8').max
    for is2n in xrange(ns2n):

        s2n = s2n_vals[is2n]

        npair, nsplit = get_npair_nsplit(c, is2n)

        for isplit in xrange(nsplit):
            job_name='%s-%03i-%03i' % (run,is2n,isplit)

            # note the wq logs are local
            wqurl = shapesim.get_wq_url(run,0,is2n,itrial=isplit,fs='local')
            output = shapesim.get_output_url(run, 0, is2n, itrial=isplit)

            seed = numpy.random.randint(smax)

            wlog("writing wq script:",wqurl)
            with open(wqurl,'w') as fobj:
                d={'job_name':job_name,
                   'version':options.version,
                   'groups':groups,
                   'pri':options.priority,
                   'sim_config':sim_config,
                   'mcmc_config':mcmc_config,
                   's2n':s2n,
                   'npair':npair,
                   'seed':seed,
                   'output':output}
                wqscript=_wqtemplate % d
                fobj.write(wqscript)
Exemplo n.º 2
0
def write_condor_file(c, master_script, equal_time=False):
    run = c["run"]
    job_name = "-".join((run.split("-"))[1:])

    condor_job_url = shapesim.get_condor_job_url(run)
    ns2n = len(c["s2n_vals"])
    seconds_per = get_seconds_per_pair()

    if "desired_err" in c:
        do_by_noise = True
    else:
        do_by_noise = False

    print condor_job_url
    njobs = 0

    smax = numpy.iinfo("i8").max
    with open(condor_job_url, "w") as fobj:

        text = _condor_template_head.format(master_script=master_script, overall_name=job_name)
        fobj.write(text)

        # for is2n in reversed(xrange(ns2n)):

        for is2n in xrange(ns2n):
            s2n = c["s2n_vals"][is2n]

            npair, nsplit = get_npair_nsplit(c, is2n)

            time_hours = npair * seconds_per / 3600.0
            if time_hours > MAXTIME_HOURS:
                raise ValueError(
                    "time is greater than %.2f "
                    "hours: %d*%.2f/3600.0 = %s" % (MAXTIME_HOURS, npair, seconds_per, time_hours)
                )

            print "nsplit:", nsplit, "npair:", npair, "time (hours):", time_hours
            njobs += nsplit

            for isplit in xrange(nsplit):
                output = shapesim.get_output_url(run, 0, is2n, itrial=isplit)
                logfile = output.replace(".rec", ".log")

                this_job_name = "%s-%03d-%03d" % (job_name, is2n, isplit)
                # seed = numpy.random.randint(smax)
                qdata = _queue_template.format(
                    job_name=this_job_name,
                    s2n=s2n,
                    npair=npair,
                    # seed=seed,
                    output=output,
                    logfile=logfile,
                )
                fobj.write(qdata)

    print "total jobs:", njobs
Exemplo n.º 3
0
def check_is2n(run, is2n, flist, full=False):
    fs=get_default_fs()
    c = read_config(run)

    npair,nsplit = get_npair_nsplit(c, is2n)
    ntot=npair*2

    for isplit in xrange(nsplit):
        f=get_output_url(run, 0, is2n, itrial=isplit, fs=fs)

        if f not in flist:
            print 'missing:',f
            continue

        if full:
            t=None
            try:
                t=eu.io.read(f)
            except:
                print 'failed to read:',f

            if t is not None:
                if t.size != ntot:
                    print "expected %d, got %d" % (npair,t.size)
Exemplo n.º 4
0
def write_condor_file(c, master_script, equal_time=False, missing=False, max_jobs=10000):
    run=c['run']
    overall_name = '-'.join( (run.split('-'))[1:] )

    if missing:
        flist=get_flist(run)
        overall_name += 'missing'
    else:
        flist=None


    ns2n=len(c['s2n_vals'])
    seconds_per=get_seconds_per_pair(c)
    print 'seconds per:',seconds_per

    if 'desired_err' in c:
        do_by_noise=True
    else:
        do_by_noise=False

    njobs_thisfile=0
    njobs=0

    old_filenum=-1
    filenum=0

    smax=numpy.iinfo('i8').max

    fobj=start_new_file(run, filenum, master_script, 
                        overall_name, missing=missing)
    for is2n in xrange(ns2n):

        s2n=c['s2n_vals'][is2n]

        npair, nsplit = get_npair_nsplit(c, is2n)

        time_hours = npair*seconds_per/3600.0
        if time_hours > MAXTIME_HOURS:
            raise ValueError("time is greater than %.2f "
                             "hours: %d*%.2f/3600.0 = %s" % (MAXTIME_HOURS,npair,seconds_per,time_hours))

        print '    nsplit:',nsplit,'npair:',npair,'time (hours):',time_hours

        for isplit in xrange(nsplit):
            output = shapesim.get_output_url(run, 0, is2n, itrial=isplit)
            logfile = output.replace('.fits','.log')

            this_job_name='%s-%03d-%05d' % (overall_name,is2n,isplit)
            qdata=_queue_template.format(job_name=this_job_name,
                                         s2n=s2n,
                                         npair=npair,
                                         output=output,
                                         logfile=logfile)
            do_write=True
            if missing and output in flist:
                do_write=False
            if do_write:
                njobs += 1
                njobs_thisfile += 1
                fobj.write(qdata)

            if njobs_thisfile >= max_jobs:
                filenum += 1
                njobs_thisfile=0
                fobj.close()
                fobj=start_new_file(run, filenum, master_script, 
                                    overall_name, missing=missing)


    print 'total jobs:',njobs