def read(poetpcf, edpcf): """Reads in data and generates an event object (runs p1). Parameters ---------- poetpcf : reader3.pcf, string String giving location of poet.pcf, or pcf object read from the POET section of poet.pcf. This indirectly specifies the run directory and the event pcf to use. edpcf : reader3.pcf, string String giving location of poet.pcf, or pcf object read from the EDGAR section of poet.pcf. Returns ------- string or tuple If read completes successfully, returns a tuple of (poetpcf, edpcf) reader3.pcf objects. If read is unsuccessful, returns the string "quit". Notes ----- If you want to pass in your own pcf object, you should generate them like so: poetpcf = rd.read_pcf("poet.pcf", "POET", simple=True) edpcf = rd.read_pcf("poet.pcf", "EDGAR", simple=True, d1lists=['zenmodels']) """ if isinstance(poetpcf, str): poetpcf = rd.read_pcf("poet.pcf", "POET", simple=True) if isinstance(edpcf, str): edpcf = rd.read_pcf("poet.pcf", "EDGAR", simple=True, d1lists=['zenmodels']) edPrint("starting initial read in.") returnPrint("read") p1size = guessp1size(poetpcf) if p1size > edpcf.memory: edPrint( "You asked me to start a POET run. However, creating an Event" " object requires approximately", p1size, "MB of RAM and you " "only allotted me", edpcf.memory, "MB of RAM. Either increase 'mem" "ory' in poet.pcf or choose a smaller set of data.") return "quit" #run p1 poet.p(1, poetpcf=poetpcf) return (poetpcf, edpcf)
def mask(poetpcf, edpcf): """Generates a bad pixel mask for the data in your event (runs p2). Parameters ---------- poetpcf : reader3.pcf, string String giving location of poet.pcf, or pcf object read from the POET section of poet.pcf. This indirectly specifies the run directory and the event pcf to use. edpcf : reader3.pcf, string String giving location of poet.pcf, or pcf object read from the EDGAR section of poet.pcf. Returns ------- string or tuple If mask completes successfully, returns a tuple of (poetpcf, edpcf) reader3.pcf objects. If read is unsuccessful, returns the string "quit". Notes ----- If you want to pass in your own pcf object, you should generate them like so: poetpcf = rd.read_pcf("poet.pcf", "POET", simple=True) edpcf = rd.read_pcf("poet.pcf", "EDGAR", simple=True, d1lists=['zenmodels']) """ edPrint("starting to mask bad pixels.") returnPrint("mask") # file size in bytes datsize = os.path.getsize('/'.join( [poetpcf.rundir, poetpcf.eventname + "_ini.dat"])) h5size = os.path.getsize('/'.join( [poetpcf.rundir, poetpcf.eventname + "_ini.h5"])) # total size in MB p2size = 200 + (datsize + h5size) / 1024**2 if p2size > edpcf.memory: edPrint( "Generating bad pixel masks will require", p2size, "MB of RAM" " but you only allotted me", edpcf.memory, "MB of RAM. Either incr" "ease 'memory' in poet.pcf or reduce the memory required by p" "2.") return "quit" # run p2 poet.p(2, poetpcf=poetpcf) return (poetpcf, edpcf)
def zen(poetpcf, edpcf): """Runs ZEN. For each zenmodel specified in edpcf, this function runs ZEN on every centering and photometry (call these full runs). Then, for each model, ZEN is run again, this time only on the best centering and photometry as determined by the corresponding full run with binning set to 1 (call these BIC runs). The BIC run with the lowest BIC is selected and the full run with the same model is the "best". A report string indicating the "best" result directory, as well as all directories is printed. Parameters ---------- poetpcf : reader3.pcf, string String giving location of poet.pcf, or pcf object read from the POET section of poet.pcf. This specifies the run directory and the event pcf to use. edpcf : reader3.pcf, string String giving location of poet.pcf, or pcf object read from the EDGAR section of poet.pcf. This specifies the the models to run Returns ------- string If read is unsuccessful, returns the string "quit". Otherwise, returns "Done". Notes ----- The configuration for each model, [model], is found by searching the appropriate run directory for a file named zen_[model].cfg. If that file is not found, a new config is generated from a template. The following settings in zen_*.cfg will be overwritten by this function: eventname, nchains, cent, phot, maxbinsize If you want to pass in your own pcf object, you should generate them like so: poetpcf = rd.read_pcf("poet.pcf", "POET", simple=True) edpcf = rd.read_pcf("poet.pcf", "EDGAR", simple=True, d1lists=['zenmodels']) """ edPrint("Running ZEN on specified models") returnPrint("zen") if not edpcf.runzen: edPrint("`runzen` is false in poet.pcf, returning...") return 'quit' eventpcf = poetpcf.rundir + '/' + poetpcf.eventname + '.pcf' # find example centering and photometry dir cpcf = rd.read_pcf(eventpcf, "centering")[0] ppcf = rd.read_pcf(eventpcf, "photometry")[0] centdir = cpcf.method if cpcf.pcfname is None else cpcf.method + '_' + cpcf.pcfname photdir = photname(ppcf) # check memory usage filename = poetpcf.rundir + '/' + poetpcf.eventname + '-zen.cfg' zenmem = guesszensize(poetpcf, edpcf, centdir, photdir, filename) if zenmem > edpcf.memory: edPrint( "EDGAR does not have enough memory allotted to run the with t" "he specified number of cores. Either increase \"memory\" or " "decrease \"zenchains\" in \"EDGAR\" portion of poet.pcf and rer" "un.") returnPrint("zen") return 'quit' # Run zen with each model to determine the best cent/ap combo for each model bestdir = {} compareBIC = {} bicdir = {} for i, model_set in enumerate(edpcf.zenmodels): model_set = ' '.join(model_set) # move config files around config = cfg.ConfigParser() config.read([filename]) config['EVENT']['models'] = model_set date = datetime.datetime.today().strftime("%Y-%m-%d_%H_%M") config['EVENT']['outdir'] = date + '_zen_model%d' % i + "_full" config['EVENT']['cent'] = 'all' config['EVENT']['phot'] = 'all' # run ZEN outdir, centdir, photdir, chiout = poet.p("zen", config, poetpcf=poetpcf, cfilename='zen.cfg') bestdir[model_set] = outdir # Rerun each model on their respective best combo with bintry=1 to # compare across models. config = cfg.ConfigParser() config.read([filename]) config['EVENT']['models'] = model_set config['EVENT']['cent'] = centdir config['EVENT']['phot'] = photdir config['EVENT']['bintry'] = '1' date = datetime.datetime.today().strftime("%Y-%m-%d_%H_%M") config['EVENT']['outdir'] = date + '_zen_model%d' % i + "_BIC" config['MCMC']['chisqscale'] = "False" # run zen bintry=1 outdir, centdir, photdir, chiout = poet.p("zen", config, poetpcf=poetpcf, cfilename='zen.cfg') compareBIC[model_set] = chiout[3] bicdir[model_set] = outdir edPrint("Ran the following ZEN models:", edpcf.zenmodels) # find best BIC bestmodel = None bestBIC = None for model_set in compareBIC: if bestmodel is None or compareBIC[model_set] < bestBIC: bestmodel = model_set bestBIC = compareBIC[model_set] edPrint( "After rerunning models on their best cent/ap combinations with bi" "ntry=1, BICs were compared indicating that", bestmodel, "was the " "best. The output can be found at:\n", bestdir[bestmodel], "\nHowev" "er, you should still check to make sure that the ZEN runs all con" "verged. Here are the directories of the full runs' can be found a" "t:\n" + "\n".join(model_set + "\n" + bestdir[model_set] for model_set in bestdir), "\nand here are the directories of the mini runs that were conducted" " to compare BIC values:\n" + "\n".join(model_set + "\n" + bicdir[model_set] for model_set in bicdir)) edPrint("All Done!") return 'Done'
def finish(poetpcf, edpcf, p6outdir): """Runs p7 through 10 on the specified output directory Parameters ---------- poetpcf : reader3.pcf, string String giving location of poet.pcf, or pcf object read from the POET section of poet.pcf. This specifies the run directory and the event pcf to use. edpcf : reader3.pcf, string String giving location of poet.pcf, or pcf object read from the EDGAR section of poet.pcf. This specifies the the models to run p6outdir : string relative path to p6 output directory to be run on. Returns ------- string If read is unsuccessful, returns the string "quit". tuple If finish completes successfully, returns a tuple of (poetpcf, edpcf) which are the pcf objects passed in. Notes ---- If you want to pass in your own pcf object, you should generate them like so: poetpcf = rd.read_pcf("poet.pcf", "POET", simple=True) edpcf = rd.read_pcf("poet.pcf", "EDGAR", simple=True, d1lists=['zenmodels']) """ returnPrint("finish", p6outdir) if not edpcf.runp7_10: edPrint("`runp7_10` is false in poet.pcf, skipping p7 through p10...") return (poetpcf, edpcf) p710size = guessp710size(p6outdir, poetpcf) if p710size > edpcf.memory: edPrint("You did not give me enough memory in poet.pcf to finish runn" "ing p7 through p10.") return 'quit' # run the rest of POET edPrint("p6 ran to completion, running analysis (p7)...") poet.p(7, p6outdir, poetpcf=poetpcf) edPrint("p7 ran to completion, generating tables (p8)...") poet.p(8, p6outdir, poetpcf=poetpcf) edPrint("p8 ran to completion, generating figures (p9)...") poet.p(9, p6outdir, poetpcf=poetpcf) edPrint("p9 ran to completion, generating IRSA tables (p10)...") poet.p(10, p6outdir, poetpcf=poetpcf) edPrint("p10 ran to completion, check", p6outdir, "in your specified run" " directory for the results.") return (poetpcf, edpcf)
def p6_final(poetpcf, edpcf, centering, photometry, model): """Runs p6 one last time on the chosen centering, photometry, and model with a higher mcmc iteration. Parameters ---------- poetpcf : reader3.pcf, string String giving location of poet.pcf, or pcf object read from the POET section of poet.pcf. This specifies the run directory and the event pcf to use. edpcf : reader3.pcf, string String giving location of poet.pcf, or pcf object read from the EDGAR section of poet.pcf. This specifies the the models to run centering : string The name of the centering directory. photometry : string The name of the photometry directory. model : string Space separated model to be run. Returns ------- string If read is unsuccessful, returns the string "quit". tuple If p6_final completes successfully, returns a tuple of (poetpcf, edpcf) which are the pcf objects passed in. Notes ----- This function will edit eventname.pcf and replace the [p6] and [params] sections. After p6 is run, it will return eventname.pcf to its original state, but will put the sections that were used in the files params_final.pcf and p6_final.pcf as a record of what was done If you want to pass in your own pcf object, you should generate them like so: poetpcf = rd.read_pcf("poet.pcf", "POET", simple=True) edpcf = rd.read_pcf("poet.pcf", "EDGAR", simple=True, d1lists=['zenmodels']) The following setting in the params section of poet.pcf will be overwritten: model, mcmc, chi2flag (mcmc set to True, chi2flag set to 1) """ edPrint("I am rerunning with more iterations on the centering/photometry" "with the lowest bsigchi using the model selected from the init p" "hase (and confirmed in the check phase).") returnPrint("p6_final", centering, photometry, model) # grab params section config = poetpcf.eventname + '.pcf' params = rd.read_pcf(poetpcf.rundir + '/' + config, 'params', simple=True, d1lists=('numit', )) # find the number of threads for p6 threads = edpcf.cores // params.nchains if threads < 1: edPrint("I am unable to run", params.nchains, "chains when you" "only allotted me", edpcf.cores, "cores for this run.\n") return 'quit' # check memory usage p6mem = guessp6size(poetpcf.rundir, params.nchains, poetpcf.eventname, [model], centering, photometry, params.numit[1]) # adjust cpu count based on memory if 200 + p6mem * threads > edpcf.memory: if 200 + p6mem > edpcf.memory: edPrint("Cannot run p6 with only", edpcf.memory, "MB of RAM allocated " "when running p6 requires", 200 + p6mem, "MB of RAM.") return "quit" threads = int((edpcf.memory - 200) / p6mem) edPrint("I am reducing number of cores used to compensate for a lack " "of allotted RAM.") # moving params file to rundir with chosen model p6_override = { 'header': 'p6', 'centering': [centering], 'photometry': [photometry], 'modeldir': 'final', 'threads': threads } params_override = {'model': [model.split()], 'mcmc': True, 'chi2flag': 1} moddir = poet.p(6, nodate=False, retmoddir=True, poetpcf=poetpcf, control=p6_override, params_override=params_override)[0] edPrint("Everything seems to have worked out fine. Check out", '/'.join([poetpcf.rundir, centering, photometry, moddir]), "to see the results.") p6outdir = '/'.join([centering, photometry, moddir]) return poetpcf, edpcf, p6outdir,
def p6_check(poetpcf, edpcf, centering, photometry, model): """Runs p6 on the centering and photometry passed in but on every model in edpcf and compares to ensure that a different model is not chosen. This is done in the rare situation that a different model is chosen, in which case a human should continue the analysis as this should not happen in theory Parameters ---------- poetpcf : reader3.pcf, string String giving location of poet.pcf, or pcf object read from the POET section of poet.pcf. This specifies the run directory and the event pcf to use. edpcf : reader3.pcf, string String giving location of poet.pcf, or pcf object read from the EDGAR section of poet.pcf. This specifies the the models to run centering : string The name of the centering directory. photometry : string The name of the photometry directory. model : string Space separated model. This model is not the only one that is run, but an error will occurred if this model does not yield the lowest BIC. Returns ------- string If read is unsuccessful, returns the string "quit". tuple If p6_check completes successfully, returns a tuple of (poetpcf, edpcf, centering, photometry, model) where the first two are the pcf objects passed in, centering is the string giving the centering directory, photometry a string giving the photometry directory, and model a space separated list of p6 models. This is EDGAR's best guess of the best centering, photometry, and model. Notes ----- This function will edit eventname.pcf and replace the [p6] and [params] sections. After p6 is run, it will return eventname.pcf to its original state, but will put the sections that were used in the files params_check.pcf and p6_check.pcf as a record of what was done If you want to pass in your own pcf object, you should generate them like so: poetpcf = rd.read_pcf("poet.pcf", "POET", simple=True) edpcf = rd.read_pcf("poet.pcf", "EDGAR", simple=True, d1lists=['zenmodels']) The following setting in the params section of poet.pcf will be overwritten: model, mcmc, chi2flag (mcmc set to False, chi2flag set to 0) """ edPrint("I am rerunning all models on the centering and photometry select" "ed in the full phase to make sure that the optimal model did not" " change from using a different centering and photometry. (You wi" "ll get a lengthy error message if this happens.)") returnPrint("p6_check", centering, photometry, model) # find the number of threads for p6 threads = edpcf.cores if threads < 1: edPrint("I am unable to run when you" "only allotted me", edpcf.cores, "cores for this run.\n") return 'quit' # check memory usage numit = 0 nchains = 0 p6mem = guessp6size(poetpcf.rundir, nchains, poetpcf.eventname, edpcf.p6model, centering, photometry, numit) # adjust cpu count based on memory if 200 + p6mem * threads > edpcf.memory: if 200 + p6mem > edpcf.memory: edPrint("Cannot run p6 with only", edpcf.memory, "MB of RAM allocated " "when running p6 requires", 200 + p6mem, "MB of RAM.") return "quit" threads = int((edpcf.memory - 200) / p6mem) edPrint("I am reducing number of cores used to compensate for a lack " "of allotted RAM.") config = poetpcf.eventname + '.pcf' # use custom config overrides p6_override = { 'header': 'p6', 'centering': [centering], 'photometry': [photometry], 'modeldir': 'check', 'threads': threads } params_override = {'mcmc': False, 'model': edpcf.p6model, 'chi2flag': 0} # run p6 lines, info = poet.p(6, nodate=False, poetpcf=poetpcf, control=p6_override, params_override=params_override) edPrint("I am finding the best model based on BIC values." "This should be the same as calculated before.") # Find the best model bestline = None bestoff = None bestval = None for line in lines.keys(): for aper in lines[line]: if bestval is None or aper[5] < bestval: bestline = line bestoff = float(aper[3]) bestval = aper[5] bestmodel = info[bestline][bestoff]['model'] # find model that was best for mod in edpcf.p6model: modeljazz = mc.setupmodel(mod) if modeljazz[-1] == bestmodel: newmodel = ' '.join(mod) break if newmodel != model: edPrint( "After running p6 on the selected centering and photometry, " "a different best model was obtained. This is irregular and " "beyond the scope of EDGAR's functionality. You will have to r" "erun and finish the analysis by hand. The model obtained in" " the init phase was:", model, ". This was done with centeri" "ng:", edpcf.test_centering, "; and photometry:", edpcf.test_photometry, ". The model obtained in the check p" "hase (this phase) is:", newmodel, ". This was run on center" "ing:", centering, "; and photometry:", photometry, ". Good " "Luck!\n\nWhen you get a good result, you can run p7-10 with" " by passing the arguments 'ed', 'finish', '<p6ouputdir>' to " "poet in that order where 'p6outputdir' a full path. Also, Z" "EN can be run through EDGAR with arguments 'ed', 'zen'") return 'quit' edPrint("models agree") return poetpcf, edpcf, centering, photometry, model
def p6_full(poetpcf, edpcf, model): """Runs p6 on every centering and photometry using the model combination passed in to determine the best centering and photometry combination. Parameters ---------- poetpcf : reader3.pcf, string String giving location of poet.pcf, or pcf object read from the POET section of poet.pcf. This specifies the run directory and the event pcf to use. edpcf : reader3.pcf, string String giving location of poet.pcf, or pcf object read from the EDGAR section of poet.pcf. This specifies the amount of computer resources to use model : string Space separated list of models to combine. This will be inserted into the p6 section of eventname. See Notes below Returns ------- string If read is unsuccessful, returns the string "quit". tuple If p6_full completes successfully, returns a tuple of (poetpcf, edpcf, centering, photometry, model) where the first two are the pcf objects passed in, centering is the string giving the centering directory, photometry a string giving the photometry directory, and model a space separated list of p6 models. This is EDGAR's best guess of the best centering, photometry, and model Notes ----- This function will edit eventname.pcf and replace the [p6] and [params] sections. After p6 is run, it will return eventname.pcf to its original state, but will put the sections that were used in the files params_full.pcf and p6_full.pcf as a record of what was done If you want to pass in your own pcf object, you should generate them like so: poetpcf = rd.read_pcf("poet.pcf", "POET", simple=True) edpcf = rd.read_pcf("poet.pcf", "EDGAR", simple=True, d1lists=['zenmodels']) The following setting in the params section of poet.pcf will be overwritten: model, mcmc, chi2flag (mcmc set to False, chi2flag set to 0) """ returnPrint("p6_full", model) edPrint( "I am running p6 on all centering/photometry with the decided" " model:", model, ". This will Identify the optimal centering" " and photometry methods.") # find the number of threads for p6 threads = edpcf.cores if threads < 1: edPrint("I am unable to run when you" "only allotted me", edpcf.cores, "cores for this run.\n") return 'quit' # check memory usage numit = 0 nchains = 0 p6mem = guessp6size(poetpcf.rundir, nchains, poetpcf.eventname, [model], edpcf.test_centering, edpcf.test_photometry, numit) # adjust cpu count based on memory if 200 + p6mem * threads > edpcf.memory: if 200 + p6mem > edpcf.memory: edPrint("Cannot run p6 with only", edpcf.memory, "MB of RAM allocated " "when running p6 requires", 200 + p6mem, "MB of RAM.") return "quit" threads = int((edpcf.memory - 200) / p6mem) edPrint("I am reducing number of cores used to compensate for a lack " "of allotted RAM.") # use custom config overrides p6_override = { 'centering': ['all'], 'photometry': ['all'], 'modeldir': None, 'threads': threads } params_override = {'mcmc': False, 'model': [model.split()], 'chi2flag': 1} # run p6 lines, info = poet.p(6, nodate=False, poetpcf=poetpcf, control=p6_override, params_override=params_override) edPrint("I am finding the best centering/photometry based on binned-sigma" " chi-squared") # find best centering/photometry bestline = None bestoff = None bestval = None for line in lines.keys(): for aper in lines[line]: if bestline is None or aper[8] < bestval: bestline = line bestoff = float(aper[3]) bestval = aper[8] centering = info[bestline][bestoff]['centdir'] photometry = info[bestline][bestoff]['photdir'] edPrint("I have found a centering method of", centering, "with a" "photometry of", photometry, "using '", model, "' as the model to" "yield the most accurate results.") return poetpcf, edpcf, centering, photometry, model
def p6_init(poetpcf, edpcf): """Runs p6 on the test centering and photometry using every model combination to determine the best model combination. Parameters ---------- poetpcf : reader3.pcf, string String giving location of poet.pcf, or pcf object read from the POET section of poet.pcf. This specifies the run directory and the event pcf to use. edpcf : reader3.pcf, string String giving location of poet.pcf, or pcf object read from the EDGAR section of poet.pcf. This specifies the test centering and photometry as well as the models to run Returns ------- string If read is unsuccessful, returns the string "quit". tuple If p6_init completes successfully, returns a tuple of (poetpcf, edpcf, model) where the first two are the pcf objects passed in, and model is a space separated list of p6 models. Notes ----- This function will edit eventname.pcf and replace the [p6] and [params] sections. After p6 is run, it will return eventname.pcf to its original state, but will put the sections that were used in the files params_init.pcf and p6_init.pcf as a record of what was done If you want to pass in your own pcf object, you should generate them like so: poetpcf = rd.read_pcf("poet.pcf", "POET", simple=True) edpcf = rd.read_pcf("poet.pcf", "EDGAR", simple=True, d1lists=['zenmodels']) The following setting in the params section of poet.pcf will be overwritten: model, mcmc, chi2flag (mcmc set to False, chi2flag set to 0) """ returnPrint("p6_init") edPrint("I am running p6 for the first time to Identify the optimal set o" "f models.") if not edpcf.runp6: edPrint( "`runp6` is false in poet.pcf, skipping p6...\nIf you would like to run ZEN though EDGAR, run \"poet.py ed zen\" from the command line or \"poet.p('ed', 'zen')\" from a python session." ) return 'quit' # find the number of threads for p6 threads = edpcf.cores if threads < 1: edPrint("I am unable to run when you" " only allotted me", edpcf.cores, "cores for this run.\n") return "quit" numit = 0 nchains = 0 p6mem = guessp6size(poetpcf.rundir, nchains, poetpcf.eventname, edpcf.p6model, edpcf.test_centering, edpcf.test_photometry, numit) # adjust cpu count based on memory if 200 + p6mem * threads > edpcf.memory: if 200 + p6mem > edpcf.memory: edPrint("Cannot run p6 with only", edpcf.memory, "MB of RAM allocated " "when running p6 requires", 200 + p6mem, "MB of RAM.") return "quit" threads = int((edpcf.memory - 200) / p6mem) edPrint("I am reducing number of cores used to compensate for a lack " "of allotted RAM.") config = poetpcf.eventname + '.pcf' # use custom config overrides params_override = {"mcmc": False, "model": edpcf.p6model, "chi2flag": 0} p6_override = { "centering": [edpcf.test_centering], "photometry": [edpcf.test_photometry], "modeldir": 'dry_run', "threads": threads } # run p6 lines, info = poet.p(6, nodate=False, poetpcf=poetpcf, control=p6_override, params_override=params_override) edPrint("I am finding the best model based on BIC values.") # Find the best model bestline = None bestoff = None bestval = None for line in lines.keys(): for aper in lines[line]: if bestval is None or aper[5] < bestval: bestline = line bestoff = float(aper[3]) bestval = aper[5] bestmodel = info[bestline][bestoff]['model'] # find model that was best for mod in edpcf.p6model: modeljazz = mc.setupmodel(mod) if modeljazz[-1] == bestmodel: model = ' '.join(mod) break edPrint("I have found '" + model + "' to be the best model.") return poetpcf, edpcf, model,