Пример #1
0
def main():
    options,args = parser.parse_args(sys.argv[1:])

    if len(args) < 1:
        parser.print_help()
        sys.exit(45)

    run=args[0]

    c = shapesim.read_config(run)
    cs = shapesim.read_config(c['sim'])

    group=options.group
    if group is not None:
        group = 'group: ['+group+']'
    else:
        group=''

    mode=''
    if options.bynode:
        mode='mode: bynode'

    extra=''
    if options.extra is not None:
        extra='\n'.join( options.extra.split(';') )

    wqd = shapesim.get_wq_dir(run, combine=True)
    if not os.path.exists(wqd):
        os.makedirs(wqd)

    if run[0:8] == 'gmix-fit':
        rstr=run.replace('gmix-fit','gmix')
    else:
        rstr=run
 
    n1 = cs['nums2']

    runtype = c['runtype']
    if runtype == 'byellip':
        n2 = cs['nume']
    else:
        n2 = shapesim.get_nums2n(c)

    for i1 in xrange(n1):
        for i2 in xrange(n2):
            job_name='%s-combine-%03i-%03i' % (rstr,i1,i2)

            wqurl = shapesim.get_wq_url(run,i1,i2,combine=True)

            wlog("writing wq script:",wqurl)
            with open(wqurl,'w') as fobj:
                wqscript=_wqtemplate % {'job_name':job_name,
                                        'run':run, 
                                        'i1':i1,
                                        'i2':i2,
                                        'group':group,
                                        'mode':mode,
                                        'extra':extra,
                                        'pri':options.priority}
                fobj.write(wqscript)
Пример #2
0
def main():
    options, args = parser.parse_args(sys.argv[1:])

    if len(args) < 1:
        parser.print_help()
        sys.exit(45)

    simname = args[0]
    ss = shapesim.ShapeSim(simname)

    orient = ss.get("orient", "rand")
    if orient == "ring":
        numper_str = ""
    else:
        if len(args) < 2:
            parser.print_help()
            sys.exit(45)
        numper_str = args[1]

    if ss.fs != "hdfs":
        raise ValueError("This only works for HDFS right now " "would need to worry about making dirs")

    c = shapesim.read_config(simname)

    groups = options.groups
    if groups is None:
        groups = ""
    else:
        groups = "group: [%s]" % groups

    wqd = shapesim.get_cache_wq_dir(simname)
    if not os.path.exists(wqd):
        os.makedirs(wqd)

    extra = ""
    if options.bynode:
        extra = "mode: bynode\nN: 1"
    elif options.ncores is not None:
        ncores = int(options.ncores)
        extra = "mode: bycore1\nN: %d" % ncores

    for is2 in xrange(c["nums2"]):
        for ie in xrange(c["nume"]):
            job_name = "%s-%i-%i" % (simname, is2, ie)

            wqurl = shapesim.get_cache_wq_url(simname, is2, ie)

            wlog("writing wq script:", wqurl)
            with open(wqurl, "w") as fobj:
                wqscript = _wqtemplate % {
                    "job_name": job_name,
                    "simname": simname,
                    "is2": is2,
                    "ie": ie,
                    "numper": numper_str,
                    "groups": groups,
                    "extra": extra,
                    "pri": options.priority,
                }
                fobj.write(wqscript)
Пример #3
0
def main():
    options,args = parser.parse_args(sys.argv[1:])

    if len(args) < 1:
        parser.print_help()
        sys.exit(45)

    run=args[0]

    c = shapesim.read_config(run)

    run         = c['run']
    sim_config  = c['sim_config']
    mcmc_config = c['mcmc_config']
    s2n_vals    = c['s2n_vals']

    ns2n = len(s2n_vals)

    d = shapesim.get_wq_dir(run, bytrial=True, fs='local')
    if not os.path.exists(d):
        os.makedirs(d)
    d = shapesim.get_output_dir(run, sub='bytrial')
    if not os.path.exists(d):
        os.makedirs(d)


    groups=''
    if options.groups is not None:
        groups = 'group: [%s]' % options.groups

    smax=numpy.iinfo('i8').max
    for is2n in xrange(ns2n):

        s2n = s2n_vals[is2n]

        npair, nsplit = get_npair_nsplit(c, is2n)

        for isplit in xrange(nsplit):
            job_name='%s-%03i-%03i' % (run,is2n,isplit)

            # note the wq logs are local
            wqurl = shapesim.get_wq_url(run,0,is2n,itrial=isplit,fs='local')
            output = shapesim.get_output_url(run, 0, is2n, itrial=isplit)

            seed = numpy.random.randint(smax)

            wlog("writing wq script:",wqurl)
            with open(wqurl,'w') as fobj:
                d={'job_name':job_name,
                   'version':options.version,
                   'groups':groups,
                   'pri':options.priority,
                   'sim_config':sim_config,
                   'mcmc_config':mcmc_config,
                   's2n':s2n,
                   'npair':npair,
                   'seed':seed,
                   'output':output}
                wqscript=_wqtemplate % d
                fobj.write(wqscript)
Пример #4
0
def main():
    options,args = parser.parse_args(sys.argv[1:])

    if len(args) < 1:
        parser.print_help()
        sys.exit(45)

    run=args[0]

    if options.skip is None:
        skip=[]
    else:
        skip = [int(v) for v in options.skip.split(',')]

    c = shapesim.read_config(run)
    s2n_vals    = c['s2n_vals']

    n_s2n = len(s2n_vals)

    dlist=[]
    #for is2n in reversed(xrange(n_s2n)):

    for is2n in xrange(n_s2n):
        if is2n in skip:
            continue

        s2n_matched = s2n_vals[is2n]
        fname=shapesim.get_output_url(run, 0, is2n)
        print >>stderr,fname
        data=eu.io.read(fname)

        if options.gerror:
            d = get_averaged_gerror(data, s2n_matched, verbose=options.verbose)
        else:
            plot_dir=shapesim.get_plot_dir(run)
            plot_name=get_image_file(plot_dir, fname)
            d = get_averaged_jackknife(data, s2n_matched,
                                       verbose=options.verbose,
                                       show=options.show,
                                       fname=plot_name)
        dlist.append(d)

    output = eu.numpy_util.combine_arrlist(dlist)
    out_fname=shapesim.get_averaged_url(run, 0)
    print >>stderr,'writing:',out_fname
    eu.io.write(out_fname, output, clobber=True)
Пример #5
0
def main():
    options,args = parser.parse_args(sys.argv[1:])

    if len(args) < 1:
        parser.print_help()
        sys.exit(45)

    run=args[0]

    max_jobs=int(options.max_jobs)


    c = shapesim.read_config(run)
    c['version'] = options.version

    make_some_dirs(run)
    master_script=write_master(c)

    write_condor_file(c, master_script, missing=options.missing,
                      max_jobs=max_jobs)
Пример #6
0
def main():
    options, args = parser.parse_args(sys.argv[1:])

    if len(args) < 1:
        parser.print_help()
        sys.exit(45)

    run = args[0]

    c = shapesim.read_config(run)
    c["version"] = options.version
    if options.wshear:
        c["exec_name"] = "gsim-ring-mcmc-wshear"
    else:
        c["exec_name"] = "gsim-ring-mcmc"

    ns2n = len(c["s2n_vals"])

    make_some_dirs(run)
    master_script = write_master(c)

    write_condor_file(c, master_script)
Пример #7
0
def main():
    options,args = parser.parse_args(sys.argv[1:])

    if len(args) < 1:
        parser.print_help()
        sys.exit(45)

    run=args[0]


    c = shapesim.read_config(run)

    ns2n=len(c['s2n_vals'])

    flist=get_flist(run)

    if options.is2n is None:
        is2n_vals = range(len(c['s2n_vals']))
    else:
        is2n_vals = options.is2n.split(',')
        is2n_vals = [int(ii) for ii in is2n_vals]

    for is2n in is2n_vals:
        check_is2n(run, is2n, flist, full=options.full)
Пример #8
0
def check_is2n(run, is2n, flist, full=False):
    fs=get_default_fs()
    c = read_config(run)

    npair,nsplit = get_npair_nsplit(c, is2n)
    ntot=npair*2

    for isplit in xrange(nsplit):
        f=get_output_url(run, 0, is2n, itrial=isplit, fs=fs)

        if f not in flist:
            print 'missing:',f
            continue

        if full:
            t=None
            try:
                t=eu.io.read(f)
            except:
                print 'failed to read:',f

            if t is not None:
                if t.size != ntot:
                    print "expected %d, got %d" % (npair,t.size)
Пример #9
0
def main():
    options,args = parser.parse_args(sys.argv[1:])

    if len(args) < 1:
        parser.print_help()
        sys.exit(45)

    run=args[0]

    c = shapesim.read_config(run)
    cs = shapesim.read_config(c['sim'])


    if options.bytrial:
        if 'stack-' in run:
            ntrial=c['nsplit']
        else:
            orient=cs.get('orient','rand')
            if orient == 'ring':
                ntrial = cs['nsplit']
            else:
                ntrial = cs['ntrial']


    wqd = shapesim.get_wq_dir(run, bytrial=options.bytrial)
    if not os.path.exists(wqd):
        os.makedirs(wqd)

    extra=''
    if options.bynode:
        extra='mode: bynode\nN: 1'
    elif options.ncores is not None:
        ncores=int(options.ncores)
        extra='mode: bycore1\nN: %d' % ncores

    if run[0:8] == 'gmix-fit':
        rstr=run.replace('gmix-fit','gmix')
    else:
        rstr=run

    n1 = shapesim.get_numT(cs)

    runtype = c['runtype']
    if runtype == 'byellip':
        n2 = cs['nume']
    else:
        n2 = shapesim.get_nums2n(c)



    for i1 in xrange(n1):
        for i2 in xrange(n2):
            groups=''
            if options.i2new is not None and i2 <= int(options.i2new):
                groups='group: [new,new2]'
            elif options.i2new1 is not None and i2 <= int(options.i2new1):
                groups='group: [new]'
            elif options.i2new2 is not None and i2 <= int(options.i2new2):
                groups='group: [new2]'
            elif options.groups is not None:
                groups = 'group: [%s]' % options.groups

            if options.bytrial:
                for itrial in xrange(ntrial):
                    job_name='%s-%03i-%03i-%02i' % (rstr,i1,i2,itrial)
                    wqurl = shapesim.get_wq_url(run,i1,i2,itrial=itrial)
                    wlog("writing wq script:",wqurl)
                    with open(wqurl,'w') as fobj:
                        d={'job_name':job_name,
                           'run':run, 
                           'i1':i1,
                           'i2':i2,
                           'itrial':itrial,
                           'groups':groups,
                           'extra':extra,
                           'pri':options.priority}
                        wqscript=_wqtemplate_bytrial % d
                        fobj.write(wqscript)


            else:
                job_name='%s-%03i-%03i' % (rstr,i1,i2)
                wqurl = shapesim.get_wq_url(run,i1,i2)

                wlog("writing wq script:",wqurl)
                with open(wqurl,'w') as fobj:
                    wqscript=_wqtemplate % {'job_name':job_name,
                                            'run':run, 
                                            'i1':i1,
                                            'i2':i2,
                                            'groups':groups,
                                            'extra':extra,
                                            'pri':options.priority}
                    fobj.write(wqscript)
Пример #10
0
def main():
    options,args = parser.parse_args(sys.argv[1:])

    if len(args) < 2:
        parser.print_help()
        sys.exit(1)

    sim_name=args[0]
    run_type=args[1]

    cs = shapesim.read_config(sim_name)
    objmodel = cs['objmodel']
    psfmodel = cs['psfmodel']
    simtype = shortnames[objmodel]+shortnames[psfmodel]

    if options.run_name is not None:
        run_name=options.run_name
        fname = shapesim.get_config_file(run_name)
        if os.path.exists(fname) and not options.overwrite:
            raise ValueError("file already exists: %s" % fname)
    else:
        run_name, fname = get_run_name_and_file(sim_name)

    s2n_fac=options.s2n_fac
    if s2n_fac is None:
        raise ValueError("send --s2n-fac")
    if run_type == 'bys2n':
        ie=options.ie
        if ie is None:
            raise ValueError("send --ie for bys2n")

        tmps2,e_val = shapesim.get_s2_e(cs, 0, int(ie))

        #s2n_vals_def='5,10,15,20,25,30,40,50,60,70,80,90,100'
        s2n_vals_def='10,15,20,25,30,40,50,60,70,80,90,100'
        s2n_vals=options.s2n_vals
        if s2n_vals is None:
            s2n_vals=s2n_vals_def
        if simtype == 'edg':
            text=edg_bys2n_template % {'run_name':run_name,
                                       'sim_name':sim_name,
                                       's2n_fac':s2n_fac,
                                       's2n_vals':s2n_vals,
                                       'ie':ie,
                                       'e_val':e_val}
        elif simtype == 'et':
            text=et_bys2n_template % {'run_name':run_name,
                                      'sim_name':sim_name,
                                      's2n_fac':s2n_fac,
                                      's2n_vals':s2n_vals,
                                      'ie':ie,
                                      'e_val':e_val}
        elif simtype == 'dt':
            text=dt_bys2n_template % {'run_name':run_name,
                                      'sim_name':sim_name,
                                      's2n_fac':s2n_fac,
                                      's2n_vals':s2n_vals,
                                      'ie':ie,
                                      'e_val':e_val}
        elif simtype == 'gg':
            text=gg_bys2n_template % {'run_name':run_name,
                                      'sim_name':sim_name,
                                      's2n_fac':s2n_fac,
                                      's2n_vals':s2n_vals,
                                      'ie':ie,
                                      'e_val':e_val}

        else:
            raise ValueError("support other sim types")
    elif run_type == 'byellip':
        s2n=options.s2n
        if s2n is None:
            raise ValueError("send s2n for byellip")
        if simtype == 'edg':
            text=edg_byellip_template
        elif simtype == 'gg':
            text=gg_byellip_template

        else:
            raise ValueError("support other sim types")

        text=text % {'run_name':run_name,
                     'sim_name':sim_name,
                     's2n_fac':s2n_fac,
                     's2n':s2n}

    else:
        raise ValueError("support others?")

    print text
    print 'fname:',fname
    if not options.dryrun:
        with open(fname,'w') as fobj:
            fobj.write(text)
    else:
        print 'dryrun'
Пример #11
0
def main():
    options,args = parser.parse_args(sys.argv[1:])

    if len(args) < 3:
        parser.print_help()
        sys.exit(45)

    run=args[0]
    nodes=int(args[1])
    ppn=int(args[2])

    np=nodes*ppn
    # seconds per ellip ring pair. this is for exp...

    seconds_per=get_seconds_per_pair()

    c = shapesim.read_config(run)

    sim_config  = c['sim_config']
    mcmc_config = c['mcmc_config']
    nsplit      = c['nsplit']
    s2n_vals    = c['s2n_vals']
    s2n_fac     = c['s2n_fac']
    min_npair   = c['min_npair']

    ns2n = len(s2n_vals)

    pbsd = shapesim.get_pbs_dir(run)
    logdir=os.path.join(pbsd,'logs')
    outd = shapesim.get_output_dir(run, sub='bytrial')

    if not os.path.exists(pbsd):
        os.makedirs(pbsd)
    if not os.path.exists(logdir):
        os.makedirs(logdir)
    if not os.path.exists(outd):
        os.makedirs(outd)

    rstr=run

    job_name = '-'.join( (rstr.split('-'))[1:] )

    script_url=shapesim.get_minions_script_url(run)
    script_base=os.path.basename(script_url)

    print script_url
    with open(script_url,'w') as fobj:
        c['version'] = options.version
        fobj.write(_script_template % c)
    print

    pbsf=shapesim.get_minions_url(run,0)
    cmdf=shapesim.get_commands_url(run,0)

    print cmdf
    ntot=0
    with open(cmdf,'w') as fobj:

        for is2n in xrange(ns2n):

            s2n = s2n_vals[is2n]
            npair = shapesim.get_s2n_nrepeat(s2n, fac=s2n_fac)
            if npair < min_npair:
                npair = min_npair

            ntot += npair*nsplit

            for isplit in xrange(nsplit):
                output = shapesim.get_output_url(run, 0, is2n, itrial=isplit)
                outbname=os.path.basename(output)

                logf=outbname.replace('.rec','.log')
                cmd=_cmd_template.format(script_file=script_base,
                                         s2n=s2n,
                                         npair=npair,
                                         output=output,
                                         logfile=logf)
                fobj.write(cmd)
                fobj.write('\n')


    time_seconds = ntot*seconds_per/(np-1)

    print pbsf
    hours_raw = time_seconds/3600.
    hours = int(round(hours_raw)) + 1
    print '    raw hours:',hours_raw,'hours used:',hours

    if options.version=='hopper':
        minions_cmd='minions-hopper'
    else:
        minions_cmd='minions'

    with open(pbsf,'w') as fobj:
        cmd_bname=os.path.basename(cmdf)
        pbslog=os.path.basename(pbsf)+'.pbslog'
        pbs_text=_pbs_template.format(minions_cmd=minions_cmd,
                                      job_name=job_name,
                                      pbslog=pbslog,
                                      nodes=nodes,
                                      ppn=ppn,
                                      np=np,
                                      hours=hours,
                                      commands_file=cmd_bname)
        fobj.write(pbs_text)
Пример #12
0
def main():
    options,args = parser.parse_args(sys.argv[1:])

    if len(args) < 2:
        parser.print_help()
        sys.exit(45)

    run=args[0]
    nodes=int(args[1])
    ppn=int(options.ppn)

    np=nodes*ppn
    # seconds per ellip ring pair. this is for exp...

    c = shapesim.read_config(run)
    cs = shapesim.read_config(c['sim'])

    seconds_per=get_seconds_per(run,c)

    nsplit = cs['nsplit']

    pbsd = shapesim.get_pbs_dir(run)
    logdir=os.path.join(pbsd,'logs')
    if not os.path.exists(pbsd):
        os.makedirs(pbsd)
    if not os.path.exists(logdir):
        os.makedirs(logdir)

    if run[0:8] == 'gmix-fit':
        rstr=run.replace('gmix-fit','gmix')
    else:
        rstr=run

    job_name = '-'.join( (rstr.split('-'))[1:] )

    if 'nums2' in cs:
        n1 = cs['nums2']
    else:
        n1 = len(cs['Tobj'])

    n2 = shapesim.get_nums2n(c)

    script_url=shapesim.get_minions_script_url(run)
    script_base=os.path.basename(script_url)

    print script_url
    with open(script_url,'w') as fobj:
        fobj.write(_script_template)
    print

    for i1 in xrange(n1):
        pbsf=shapesim.get_minions_url(run,i1)
        cmdf=shapesim.get_commands_url(run,i1)

        print cmdf
        ntot=0
        with open(cmdf,'w') as fobj:
            for i2 in xrange(n2):
                nellip=get_nellip(c,i2)

                for itrial in xrange(nsplit):
                    outf=shapesim.get_output_url(run, i1, i2, itrial=itrial)
                    outbname=os.path.basename(outf)
                    logf=outbname.replace('.rec','.log')
                    cmd=_cmd_template.format(script_file=script_base,
                                             run=run,
                                             i1=i1,
                                             i2=i2,
                                             itrial=itrial,
                                             logfile=logf)

                    fobj.write(cmd)
                    fobj.write('\n')

                    ntot += nellip

        time_seconds = ntot*seconds_per/(np-1)

        print pbsf
        hours_raw = time_seconds/3600.
        hours = int(round(hours_raw)) + 1
        print '    raw hours:',hours_raw,'hours used:',hours

        with open(pbsf,'w') as fobj:
            cmd_bname=os.path.basename(cmdf)
            pbslog=os.path.basename(pbsf)+'.pbslog'
            pbs_text=_pbs_template.format(job_name=job_name,
                                          pbslog=pbslog,
                                          nodes=nodes,
                                          ppn=ppn,
                                          np=np,
                                          hours=hours,
                                          commands_file=cmd_bname)
            fobj.write(pbs_text)