def read(poetpcf, edpcf): """Reads in data and generates an event object (runs p1). Parameters ---------- poetpcf : reader3.pcf, string String giving location of poet.pcf, or pcf object read from the POET section of poet.pcf. This indirectly specifies the run directory and the event pcf to use. edpcf : reader3.pcf, string String giving location of poet.pcf, or pcf object read from the EDGAR section of poet.pcf. Returns ------- string or tuple If read completes successfully, returns a tuple of (poetpcf, edpcf) reader3.pcf objects. If read is unsuccessful, returns the string "quit". Notes ----- If you want to pass in your own pcf object, you should generate them like so: poetpcf = rd.read_pcf("poet.pcf", "POET", simple=True) edpcf = rd.read_pcf("poet.pcf", "EDGAR", simple=True, d1lists=['zenmodels']) """ if isinstance(poetpcf, str): poetpcf = rd.read_pcf("poet.pcf", "POET", simple=True) if isinstance(edpcf, str): edpcf = rd.read_pcf("poet.pcf", "EDGAR", simple=True, d1lists=['zenmodels']) edPrint("starting initial read in.") returnPrint("read") p1size = guessp1size(poetpcf) if p1size > edpcf.memory: edPrint( "You asked me to start a POET run. However, creating an Event" " object requires approximately", p1size, "MB of RAM and you " "only allotted me", edpcf.memory, "MB of RAM. Either increase 'mem" "ory' in poet.pcf or choose a smaller set of data.") return "quit" #run p1 poet.p(1, poetpcf=poetpcf) return (poetpcf, edpcf)
def __init__(self, eventpcf): tini = time.time() # Open new log based on the file name logname = eventpcf[:-4] + "ini.log" global log log = le.Logedit(logname) self.logname = logname # initialize Univ Univ.__init__(self) pcf = rd.read_pcf(eventpcf) self.initpars(pcf) self.calc(pcf) self.read() self.check() self.save() # Print time elapsed and close log: cwd = os.getcwd() + "/" log.writelog("\nOutput files:") log.writelog("Data:") log.writelog(" " + cwd + self.eventname + "_ini.dat") log.writelog(" " + cwd + self.eventname + "_ini.h5") log.writelog("Log:") log.writelog(" " + cwd + logname) log.writelog("Figures:") log.writelog(" " + cwd + self.eventname + "-fig101.png") dt = t.hms_time(time.time() - tini) log.writeclose('\nEnd init and read. Time (h:m:s): %s' % dt)
def guessp710size(p6dir, poetpcf): """guesses the amount of memory that will be used to run p7 through p10 Parameters ---------- p6dir : str relative path including rundir to the p6 output directory poetpcf : reader3.Pcf pcf object containing the POET section of poet.pcf Returns ------- float amount of memory that will (in theory) be used to run p7-p10 """ p6dir = poetpcf.rundir + '/' + p6dir params = rd.read_pcf('/'.join([p6dir, poetpcf.eventname + '.pcf']), "params", simple=True, d1lists=('numit', ), d2lists=('model', ), nparrs=('numit', )) modeljazz = mc.setupmodel(params.model[0]) allparams = os.stat(p6dir + '/d-' + poetpcf.eventname + '-allparams-' + modeljazz[-1] + '.npy').st_size event = os.stat(p6dir + '/d-' + poetpcf.eventname + '-6model.dat').st_size return 200 + (allparams + event) / 1024**2
def run_centering(eventname, cwd): """ Read the control file. Load the event. Launch a thread for each centering run. """ owd = os.getcwd() os.chdir(cwd) config = os.path.basename(eventname)[:-4] + '.pcf' pcfs = rd.read_pcf(config, 'centering') if len(pcfs) == 1: #, I may be in the center dir, to re-run: # Get name of centering dir: pcf = pcfs[0] centerdir = pcf.method if pcf.pcfname is not None: centerdir += "_" + str(pcf.pcfname) if cwd[-len(centerdir):] == centerdir: # Go to dir where poet2 files were saved. cwd = cwd[:-len(centerdir)] os.chdir(cwd) # Load the event: try: event = me.loadevent(eventname, load=['dendata', 'data', 'uncd', 'mask']) print("Performing centering on denoised data") except: event = me.loadevent(eventname, load=['data', 'uncd', 'mask']) # Loop over each run: for pcf in pcfs: # Make a copy of the event: this_event = copy.copy(event) # Name of the directory to put the results: centerdir = pcf.method if pcf.pcfname is not None: centerdir += "_" + str(pcf.pcfname) this_event.centerdir = centerdir # Create the centering directory if it doesn't exist: if not os.path.exists(centerdir): os.mkdir(centerdir) # copy the photometry and centering configuration into the # centering directory filename = centerdir + '/' + event.eventname + '.pcf' pcf.make_file(filename, 'centering') rd.copy_config(config, ['photometry'], filename) # Launch the thread: p = Process(target=centering, args=(this_event, pcf, centerdir, owd)) p.start() os.chdir(owd)
def run_denoising(eventname, control): """ Load the event. Read the control file. Launch a thread for each centering run. """ global event pcf = rd.read_pcf(control) nruns = len(pcf) cwd = os.getcwd() if nruns == 1: #, I may be in the denoise dir, to re-run: # Get name of denoising dir: denoisedir = pcf[0].wavelet.get() + '_' + pcf[0].threshold.get( ) + '_L' + str(pcf[0].numlvls.get()) if pcf[0].pcfname.get() != "": denoisedir += "_" + pcf[0].pcfname.get() if cwd[-len(denoisedir):] == denoisedir: # Go to dir where poet2 files were saved. cwd = cwd[:-len(denoisedir)] os.chdir(cwd) # Loop over each run: for run in np.arange(nruns): os.chdir(cwd) # Load a fresh event: print("Loading " + eventname) event = me.loadevent(eventname, load=['data', 'uncd', 'mask']) # Name of the directory to put the results: denoisedir = pcf[run].wavelet.get() + '_' + pcf[run].threshold.get( ) + '_L' + str(pcf[run].numlvls.get()) if pcf[run].pcfname.get() != "": denoisedir += "_" + pcf[run].pcfname.get() event.denoisedir = denoisedir # Create the denoising directory if it doesn't exist: if not os.path.exists(denoisedir): os.mkdir(denoisedir) # copy the centering and photometry control files to denoisedir shutil.copy('center.pcf', denoisedir + '/center.pcf') shutil.copy('photom.pcf', denoisedir + '/photom.pcf') # Modify source estimate if hasattr(pcf[run], 'srcest'): if nruns == event.npos: event.srcest[:, run] = pcf[run].srcest.get() else: for pos in range(event.npos): event.srcest[:, pos] = pcf[0].srcest.get() #Call denoising on each wavelet: denoise(pcf[run], denoisedir) return
def run_centering(eventname, control, cwd=None): """ Read the control file. Load the event. Launch a thread for each centering run. """ if cwd is None: cwd = os.getcwd() os.chdir(cwd) pcf = rd.read_pcf(control) nruns = len(pcf) if nruns == 1: #, I may be in the center dir, to re-run: # Get name of centering dir: centerdir = pcf[0].method.get() if pcf[0].pcfname.get() is not "": centerdir += "_" + pcf[0].pcfname.get() if cwd[-len(centerdir):] == centerdir: # Go to dir where poet2 files were saved. cwd = cwd[:-len(centerdir)] os.chdir(cwd) # Load the event: try: event = me.loadevent(eventname, load=['dendata', 'data', 'uncd', 'mask']) print("Performing centering on denoised data") except: event = me.loadevent(eventname, load=['data', 'uncd', 'mask']) event.denoised = False # Loop over each run: for run in np.arange(nruns): os.chdir(cwd) # Make a copy of the event: this_event = copy.copy(event) # Name of the directory to put the results: centerdir = pcf[run].method.get() if pcf[run].pcfname.get() is not "": centerdir += "_" + pcf[run].pcfname.get() this_event.centerdir = centerdir # Create the centering directory if it doesn't exist: if not os.path.exists(centerdir): os.mkdir(centerdir) # copy the photometry control file to centerdir shutil.copy('photom.pcf', centerdir + '/photom.pcf') # Launch the thread: p = Process(target=centering, args=(this_event, pcf[run], centerdir)) p.start()
def __init__(self, eventpcf, cwd): owd = os.getcwd() os.chdir(cwd) tini = time.time() # Open new log based on the file name logname = eventpcf[:-4] + "_ini.log" log = le.Logedit(logname) self.logname = logname # initialize Univ Univ.__init__(self) pcf, = rd.read_pcf(eventpcf, 'event', expand=False) self.initpars(pcf, log) self.calc(pcf, log) self.read(log) self.check(log) self.save() # Print time elapsed and close log: log.writelog("\nOutput files:") log.writelog("Data:") log.writelog(" " + cwd + '/' + self.eventname + "_ini.dat") log.writelog(" " + cwd + '/' + self.eventname + "_ini.h5") log.writelog("Log:") log.writelog(" " + logname) log.writelog("Figures:") log.writelog(" " + cwd + '/' + self.eventname + "-fig101.png") dt = t.hms_time(time.time() - tini) log.writeclose('\nEnd init and read. Time (h:m:s): %s' % dt) os.chdir(owd) if self.runp2: os.system("python3 poet.py p2")
def poet(func, *args, **kwargs): # read config if "poetpcf" in kwargs.keys(): poetpcf = kwargs.pop("poetpcf") else: poetpcf = rd.read_pcf("poet.pcf", 'POET', simple=True) rundir = os.path.abspath(poetpcf.rundir) eventname = poetpcf.eventname # extract key-word arguments from args args = list(args) kwargs.update(getkwargs(args)) # args = [formt(arg) for arg in args] # If the user is using EDGAR, switch to that immediately. if func == "ed" or func == "edgar" or func == "poe": return edgar.edgar(*args, **kwargs) # reformat func if given as int or with p omitted if func in range(1, 12): func = str(func) if func in [ '1', '2', 'd', '3', '4', '5', '6', '7', '8', '9', '10', '11', '12' ]: func = 'p' + func # find proper directory to run in if len(args) > 0 and func != "zen": directory = '/'.join([rundir, args.pop(0)]) else: directory = rundir directory = os.path.abspath(directory) # run POET if func == "p1": return p1.Event(eventname + '.pcf', directory, **kwargs) elif func == "p2": return p2.badpix(eventname + '_ini', directory, **kwargs) elif func == "pd": pd.run_denoising(eventname + '_bpm', directory, **kwargs) elif func == "p3": if os.path.isfile(directory + '/' + eventname + '_den.dat'): p3.run_centering(directory + '/' + eventname + '_den', directory, **kwargs) elif os.path.isfile(directory + '/' + eventname + '_bpm.dat'): p3.run_centering(directory + '/' + eventname + '_bpm', directory, **kwargs) else: rootdir = os.path.dirname(directory) if os.path.isfile(rootdir + '/' + eventname + '_den.dat'): p3.run_centering(rootdir + '/' + eventname + '_den', directory, **kwargs) else: p3.run_centering(rootdir + '/' + eventname + '_bpm', directory, **kwargs) elif func == "p4": if os.path.isfile(directory + '/' + eventname + '_ctr.dat'): p4.run_photometry(directory + '/' + eventname + '_ctr', directory, **kwargs) else: rootdir = os.path.dirname(directory) p4.run_photometry(rootdir + '/' + eventname + '_ctr', directory, **kwargs) elif func == "p5": p5.checks1(directory + '/' + eventname + '_pht', directory, *args, **kwargs) elif func == "zen": if len(args) > 0: return zen.main(rundir, *args, **kwargs) else: return zen.main(rundir, None, **kwargs) elif func == "p6": if directory != rundir: return run.p6model(None, directory, *args, rundir=rundir, **kwargs) else: if "control" not in kwargs.keys(): kwargs["control"] = directory + '/' + eventname + '.pcf' return run.p6model(None, directory, rundir=rundir, **kwargs) elif func == "p7": run.p7anal(None, directory, *args, **kwargs) elif func == "p8": run.p8tables(None, directory, *args, **kwargs) elif func == "p9": run.p9figs(None, directory, *args, **kwargs) elif func == "p10": # The config is not in the p6output directory cfgdir = directory + '/../../../' p10.make_irsa(None, directory, config=cfgdir + '/p10.pcf', *args, **kwargs) else: print("Unrecognized function:", func) return
def run_photometry(eventname, cwd): """ Load the event. Read the control file. Launch a thread for each centering run. """ owd = os.getcwd() os.chdir(cwd) config = os.path.basename(eventname)[:-4] + '.pcf' pcfs = rd.read_pcf(config, 'photometry') if len(pcfs) == 1: #, I may be in photdir to re-run: pcf = pcfs[0] if pcf.offset < 0: sign = '-' else: sign = '+' # Get name of photometry dir: if pcf.phottype == "psffit": photdir = 'psffit' elif pcf.phottype == "optimal": photdir = 'optimal' elif pcf.phottype == "var": photdir = ('va%03d' % (pcf.photap * 100) + sign + '%03d' % (np.abs(pcf.offset * 100))) elif pcf.phottype == "ell": photdir = ('el%03d' % (pcf.photap * 100) + sign + '%03d' % (np.abs(pcf.offset * 100))) else: # pcf[0].phottype == "aper": photdir = ('ap%03d' % (pcf.photap * 100) + sign + '%03d' % (np.abs(pcf.offset * 100))) # Append suffix to folder if suplied: if pcf.pcfname is not None: photdir += "_" + str(pcf.pcfname) # If I am in the photometry dir already: if cwd[-len(photdir):] == photdir: # Go to dir where poet3 files were saved. cwd = cwd[:-len(photdir)] os.chdir(cwd) mute = False # print to screen else: mute = True # do not print to screen # Load the event data: if pcfs[0].denphot: # Use denoised data if requested: readdata = 'dendata' else: readdata = 'data' event = me.loadevent(eventname, load=[readdata, 'uncd', 'mask']) # Loop over each run: for pcf in pcfs: # Make a copy of the event: this_event = copy.copy(event) if pcf.offset < 0: sign = '-' else: sign = '+' # Get name of photometry dir: if pcf.phottype == "psffit": photdir = 'psffit' elif pcf.phottype == "optimal": photdir = 'optimal' elif pcf.phottype == "var": photdir = ('va%03d' % (pcf.photap * 100) + sign + '%03d' % (np.abs(pcf.offset * 100))) elif pcf.phottype == "ell": photdir = ('el%03d' % (pcf.photap * 100) + sign + '%03d' % (np.abs(pcf.offset * 100))) else: photdir = ('ap%03d' % (pcf.photap * 100) + sign + '%03d' % (np.abs(pcf.offset * 100))) # Append suffix to folder if suplied: if pcf.pcfname is not None: photdir += "_" + str(pcf.pcfname) this_event.photdir = photdir # Create the photometry directory if it doesn't exist: if not os.path.exists(photdir): os.mkdir(photdir) # copy photom.pcf in photdir pcf.make_file(photdir + '/' + config, 'photometry') # Launch the thread: p = Process(target=photometry, args=(this_event, pcf, photdir, mute, owd)) p.start() os.chdir(owd)
def p6model(events=None, directory='./', modeldir=None, clip=None, idl=False, mode='full', numit=None, nodate=True, rundir=None, justplot=False, isinteractive=isinteractive, control=None, retmoddir=False, params_override=None): # adjust modeldir if modeldir is None or not nodate: date = datetime.datetime.today().strftime("%Y-%m-%d_%H_%M") # if modeldir is not None: # modeldir = date + modeldir + '/' # If a control is given, recursively # call on all paths specified in control. if control is not None: if isinstance(control, dict): p6pcf = rd.Pcf(control) else: p6pcf = rd.read_pcf(control, 'p6', simple=True, d1lists=["centering", "photometry"], d2lists=["path"]) paths = getp6filedirs(p6pcf, directory) if modeldir is None: modeldir = p6pcf.modeldir if modeldir is None: modeldir = date elif not nodate: modeldir = date + modeldir + '/' if p6pcf.threads < 1: print("ERROR: less than 1 threads specified in p6.pcf." "\nExiting...") return procs = [] for path in paths: while len(mp.active_children()) >= p6pcf.threads: time.sleep(10) p = mp.Process(target=p6model, kwargs={"events":events, "directory":path, "modeldir":modeldir, "clip":clip, "idl":idl, "mode":mode, "rundir":rundir, "numit":numit, "nodate":True, "isinteractive":isinteractive, "control":None, "params_override":params_override}) p.start() procs.append(p) for proc in procs: proc.join() print("Making p6 plots.") if retmoddir: return modeldir, plots.p6plots(modeldir, paths, directory) return plots.p6plots(modeldir, paths, directory) if modeldir is None: modeldir = date # Create new directory for model run with custom name: if os.path.isdir(directory + '/' + modeldir): print("path :", directory + '/' + modeldir, "already exists.\nSkipping this p6 run...") return "skip" try: os.mkdir(directory + '/' + modeldir) except: print("WARNING: Specified model directory already exists.") if events == None: if idl: events = p5idlRestore(directory) else: events = poetRestore(directory, clip, rundir, modeldir, params_override=params_override) numevents = len(events) if numevents == 0: print("Event object is empty.") return # Reload model parameters: nummodels = np.zeros(numevents, dtype=int) # attributes to be converted to 1d-lists d1lists = ['modelfile', 'numit', 'preclip', 'postclip', 'priorvars', 'xstep', 'ystep', 'minnumpts', 'sssteps', 'nx', 'ny', 'sx', 'sy'] # attributes to be converted to 2d-lists d2lists = ['model', 'ortholist', 'interclip', 'priorvals', 'ipclip'] # attributes to be converted to 2d-lists nparrs = ['numit', 'priorvals'] for i, event in enumerate(events): event.params = rd.read_pcf(event.paramsfile, 'params', simple=True, d1lists=d1lists, d2lists=d2lists, nparrs=nparrs) if params_override: for key in params_override: setattr(event.params, key, params_override[key]) event.fit = [] event.modeldir = directory + '/' + modeldir nummodels[i] = len(event.params.model) if i > 0 and nummodels[i] != nummodels[i-1]: print("WARNING: Number of models in each event does not" + "match.") # Initialize output type: stdout, a file object or a file printout = po.init(events[0].params.printout, events) # Execute p6 setup for i in range(nummodels.min()): p6.setup(events, i, printout, mode=mode, numit=numit) printout = po.init(events[0].params.printout, events) dataout = open(directory + '/plotpoint.txt', 'a') for i in range(nummodels.min()): p6.finalrun(events, i, printout, numit=numit) for event in events: p6Save(event, directory + '/' + modeldir) # Print parameters used for comparison: print("\nBest-fit eclipse depths or transit radius ratios with" + " errors:", file=printout) for event in events: event.minbic = np.inf # Min BIC value of all fits for one # event print(event.eventname, file=printout) print(event.eventname, file=dataout) for fit in event.fit: if hasattr(fit.i,'depth'): print(fit.bestp[fit.i.depth], fit.std [fit.i.depth], fit.saveext, fit.bsigchi, file=printout) print(fit.bestp[fit.i.depth], fit.std [fit.i.depth], fit.saveext, fit.bsigchi, file=dataout) if hasattr(fit.i,'depth2'): print(fit.bestp[fit.i.depth2], fit.std [fit.i.depth2], fit.saveext, fit.bsigchi, file=printout) print(fit.bestp[fit.i.depth2], fit.std [fit.i.depth2], fit.saveext, fit.bsigchi, file=dataout) if hasattr(fit.i,'depth3'): print(fit.bestp[fit.i.depth3], fit.std [fit.i.depth3], fit.saveext, fit.bsigchi, file=printout) print(fit.bestp[fit.i.depth3], fit.std [fit.i.depth3], fit.saveext, fit.bsigchi, file=dataout) if hasattr(fit.i,'trrprs'): print(fit.bestp[fit.i.trrprs], fit.std [fit.i.trrprs], fit.saveext, fit.bsigchi, file=printout) print(fit.bestp[fit.i.trrprs], fit.std [fit.i.trrprs], fit.saveext, fit.bsigchi, file=dataout) if hasattr(fit.i,'trrprs2'): print(fit.bestp[fit.i.trrprs2], fit.std [fit.i.trrprs2], fit.saveext, fit.bsigchi, file=printout) print(fit.bestp[fit.i.trrprs2], fit.std [fit.i.trrprs2], fit.saveext, fit.bsigchi, file=dataout) if hasattr(fit.i,'rprs'): print(fit.bestp[fit.i.rprs], fit.std [fit.i.rprs], fit.saveext, fit.bsigchi, file=printout) print(fit.bestp[fit.i.rprs], fit.std [fit.i.rprs], fit.saveext, fit.bsigchi, file=dataout) if hasattr(fit.i,'rprs2'): print(fit.bestp[fit.i.rprs2], fit.std [fit.i.rprs2], fit.saveext, fit.bsigchi, file=printout) print(fit.bestp[fit.i.rprs2], fit.std [fit.i.rprs2], fit.saveext, fit.bsigchi, file=dataout) if hasattr(fit.i,'trdepth'): print(fit.bestp[fit.i.trdepth], fit.std [fit.i.trdepth], fit.saveext, fit.bsigchi, file=printout) print(fit.bestp[fit.i.trdepth], fit.std [fit.i.trdepth], fit.saveext, fit.bsigchi, file=dataout) if hasattr(fit.i,'gdepth'): print(fit.bestp[fit.i.gdepth], fit.std [fit.i.gdepth], fit.saveext, fit.bsigchi, file=printout) print(fit.bestp[fit.i.gdepth], fit.std [fit.i.gdepth], fit.saveext, fit.bsigchi, file=dataout) event.minbic = np.min((event.minbic,fit.bic)) # Print SDNR for joint fit if more than one event: if len(events) > 1: # Joint-fit normalized residuals: jnres = np.array([]) for event in events: # Force the normalized residuals to 1D shape: nres = event.fit[0].normresiduals.flatten() # Concatenate values: jnres = np.concatenate((jnres,nres)) # Joint SDNR: jsdnr = np.std(jnres) print("\nJoint-fit SDNR: %9.7f"%jsdnr, file=printout) print("\n S/N SDNR \xce\x94BIC MODEL NUMIT " + "BIN_SZ(y,x) MinNumPts", file=printout) # Delta = '\xce\x94' in utf-8 for event in events: print(event.eventname, file=printout) print(event.eventname, file=dataout) minbic = event.minbic for i, fit in enumerate(event.fit): try: sdnr = fit.sdnr bic = fit.bic - minbic model = fit.saveext numit = event.params.numit[1] if len(event.params.ystep) == \ len(event.params.model): ystep, xstep = event.params.ystep[i],\ event.params.xstep[i] else: ystep, xstep = event.params.ystep[0],\ event.params.xstep[0] if len(event.params.minnumpts) == \ len(event.params.model): minnumpts = event.params.minnumpts[i] else: minnumpts = event.params.minnumpts[0] # extract string code ec ='%8.4f %9.7f %8.1f %11s %7.1e %6.3f,%5.3f %4.0f' print(sdnr, file=dataout) if hasattr(fit.i,'depth'): snr = fit.bestp[fit.i.depth] /\ fit.std [fit.i.depth] print(ec%(snr, sdnr, bic, model, numit, ystep, xstep, minnumpts), file=printout) if hasattr(fit.i,'depth2'): snr = fit.bestp[fit.i.depth2] /\ fit.std [fit.i.depth2] print(ec%(snr, sdnr, bic, model, numit, ystep, xstep, minnumpts), file=printout) if hasattr(fit.i,'depth3'): snr = fit.bestp[fit.i.depth3] /\ fit.std [fit.i.depth3] print(ec%(snr, sdnr, bic, model, numit, ystep, xstep, minnumpts), file=printout) if hasattr(fit.i,'trrprs'): snr = fit.bestp[fit.i.trrprs] /\ fit.std [fit.i.trrprs] print(ec%(snr, sdnr, bic, model, numit, ystep, xstep, minnumpts), file=printout) if hasattr(fit.i,'trrprs2'): snr = fit.bestp[fit.i.trrprs2] /\ fit.std [fit.i.trrprs2] print(ec%(snr, sdnr, bic, model, numit, ystep, xstep, minnumpts), file=printout) if hasattr(fit.i,'rprs'): snr = fit.bestp[fit.i.rprs] /\ fit.std [fit.i.rprs] print(ec%(snr, sdnr, bic, model, numit, ystep, xstep, minnumpts), file=printout) if hasattr(fit.i,'rprs2'): snr = fit.bestp[fit.i.rprs2] /\ fit.std [fit.i.rprs2] print(ec%(snr, sdnr, bic, model, numit, ystep, xstep, minnumpts), file=printout) if hasattr(fit.i,'trdepth'): snr = fit.bestp[fit.i.trdepth] /\ fit.std [fit.i.trdepth] print(ec%(snr, sdnr, bic, model, numit, ystep, xstep, minnumpts), file=printout) if hasattr(fit.i,'gdepth'): snr = fit.bestp[fit.i.gdepth] /\ fit.std [fit.i.gdepth] print(ec%(snr, sdnr, bic, model, numit, ystep, xstep, minnumpts), file=printout) except: print("Error calculating values. %13s" % event.fit[i].saveext, file=printout) po.close(printout) if isinteractive == False: plt.close('all') else: plt.show() return event.modeldir
def poetRestore(directory='../', clip=None, rundir=None, modeldir=None, params_override=None): files = [] events = [] filename = '' for fname in os.listdir(directory): if (fname.endswith("_p5c.dat")): files.append(fname[:-4]) files.sort() if len(files) == 0: print('Cannot find any files to restore.') return [] for f in files: # Load event event = me.loadevent(directory + '/' + f) events.append(event) print('Finished loading: ' + event.eventname) filename = filename + event.eventname event.ancildir = directory + '/' + modeldir + '/' # Clip data set to model and plot only a portion of the entire # light curve. Good for modeling a transit or eclipse in an # around-the-orbit data set if clip is not None and clip != 'None': if type(clip) == str: #Convert from string to 2 ints start, end = clip.split(':',1) try: start = int(start) except: print("Error with format of optional variable clip.") return [] try: end = int(end) except: end = None else: if len(clip) == 2: start, end = clip else: start = clip[0] end = None # Use only data points from 'start' to 'end' event.phase = event.phase [:,start:end] event.aplev = event.aplev [:,start:end] event.aperr = event.aperr [:,start:end] event.good = event.good [:,start:end] event.time = event.time [:,start:end] event.y = event.y [:,start:end] event.x = event.x [:,start:end] event.juldat = event.juldat[:,start:end] event.bjdutc = event.bjdutc[:,start:end] event.bjdtdb = event.bjdtdb[:,start:end] for event in events: # Copy params file into output dir. paramsfile = event.eventname + '.pcf' event.paramsfile = directory + '/' + modeldir + '/' + paramsfile if not os.path.isfile(event.paramsfile): if params_override: mod = {"params" : params_override} else: mod = {} rd.copy_config(rundir + '/' + paramsfile, ['params'], event.paramsfile, 'w', mod) # Copy initial values file into output dir initvfile = rd.read_pcf(event.paramsfile, "params", simple=True).modelfile event.initvalsfile = directory + '/' + modeldir + '/' + initvfile if not os.path.isfile(event.initvalsfile): if os.path.isfile(rundir + '/' + initvfile): shutil.copy(rundir + '/' + initvfile, event.initvalsfile) else: shutil.copy(rundir + '/initvals.txt', event.initvalsfile) return events
def p6_final(poetpcf, edpcf, centering, photometry, model): """Runs p6 one last time on the chosen centering, photometry, and model with a higher mcmc iteration. Parameters ---------- poetpcf : reader3.pcf, string String giving location of poet.pcf, or pcf object read from the POET section of poet.pcf. This specifies the run directory and the event pcf to use. edpcf : reader3.pcf, string String giving location of poet.pcf, or pcf object read from the EDGAR section of poet.pcf. This specifies the the models to run centering : string The name of the centering directory. photometry : string The name of the photometry directory. model : string Space separated model to be run. Returns ------- string If read is unsuccessful, returns the string "quit". tuple If p6_final completes successfully, returns a tuple of (poetpcf, edpcf) which are the pcf objects passed in. Notes ----- This function will edit eventname.pcf and replace the [p6] and [params] sections. After p6 is run, it will return eventname.pcf to its original state, but will put the sections that were used in the files params_final.pcf and p6_final.pcf as a record of what was done If you want to pass in your own pcf object, you should generate them like so: poetpcf = rd.read_pcf("poet.pcf", "POET", simple=True) edpcf = rd.read_pcf("poet.pcf", "EDGAR", simple=True, d1lists=['zenmodels']) The following setting in the params section of poet.pcf will be overwritten: model, mcmc, chi2flag (mcmc set to True, chi2flag set to 1) """ edPrint("I am rerunning with more iterations on the centering/photometry" "with the lowest bsigchi using the model selected from the init p" "hase (and confirmed in the check phase).") returnPrint("p6_final", centering, photometry, model) # grab params section config = poetpcf.eventname + '.pcf' params = rd.read_pcf(poetpcf.rundir + '/' + config, 'params', simple=True, d1lists=('numit', )) # find the number of threads for p6 threads = edpcf.cores // params.nchains if threads < 1: edPrint("I am unable to run", params.nchains, "chains when you" "only allotted me", edpcf.cores, "cores for this run.\n") return 'quit' # check memory usage p6mem = guessp6size(poetpcf.rundir, params.nchains, poetpcf.eventname, [model], centering, photometry, params.numit[1]) # adjust cpu count based on memory if 200 + p6mem * threads > edpcf.memory: if 200 + p6mem > edpcf.memory: edPrint("Cannot run p6 with only", edpcf.memory, "MB of RAM allocated " "when running p6 requires", 200 + p6mem, "MB of RAM.") return "quit" threads = int((edpcf.memory - 200) / p6mem) edPrint("I am reducing number of cores used to compensate for a lack " "of allotted RAM.") # moving params file to rundir with chosen model p6_override = { 'header': 'p6', 'centering': [centering], 'photometry': [photometry], 'modeldir': 'final', 'threads': threads } params_override = {'model': [model.split()], 'mcmc': True, 'chi2flag': 1} moddir = poet.p(6, nodate=False, retmoddir=True, poetpcf=poetpcf, control=p6_override, params_override=params_override)[0] edPrint("Everything seems to have worked out fine. Check out", '/'.join([poetpcf.rundir, centering, photometry, moddir]), "to see the results.") p6outdir = '/'.join([centering, photometry, moddir]) return poetpcf, edpcf, p6outdir,
def edgar(*args, **kwargs): """This function reads the "poet.pcf" file in the current working directory, and runs the various parts of POET on the specified data. EDGAR keeps track of memory and CPU usage to partially serialize the highly parallelized code so large amount of computation can be performed with minimal interaction (It doesn't run everything at once and crash your computer). Parameters ---------- task : string, optional Specifies which task to begin with. Can take on values in {"read", "mask", "p6_init", "p6_full", "p6_check", "p6_final", "finish", "zen"}. If not specified, EDGAR will start in "read". After finishing one task, EDGAR will proceed to the next task in the list unless otherwise specified in the "poet.pcf" control file. *args : strings, optional optional sequence of arguments passed to the starting task. In general, you will only use this after running EDGAR, during which EDGAR will produce print statements giving what arguments to use here to resume from a given spot. See individual task functions if you would like to abuse this functionality (not recommended). Returns ------- string string returned by the last task attempted. Will be "quit" if the task did not fully run. Will be "Done" if everything completed successfully. Notes ----- Running and part of EDGAR will set all "runp#" parameters in your event pcf to false. The functionality of each task is documented in that task's function's docstring. The function name is the same as the string corresponding to it {read, mask, p6_init, p6_full, p6_check, p6_final, finish, zen}. Each function receives the edpcf and poetpcf objects instead of rereading it so EDGAR can be run multiple times concurrently from the same working directory (start EDGAR, edit configs, start another EDGAR) """ edpcf = rd.read_pcf("poet.pcf", "EDGAR", simple=True, d2lists=['p6model', 'zenmodels']) poetpcf = rd.read_pcf("poet.pcf", "POET", simple=True) args = [poetpcf, edpcf] + list(args) if "task" in kwargs.keys(): task = kwargs["task"] elif len(args) > 2: task = args.pop(2) else: task = "read" edPrint("\n" + (79 * '=') + '\nNew EDGAR run starting with task:', task, "\nRundir is:", poetpcf.rundir, "\nTime is:", datetime.datetime.now(), "\n" + (79 * '=')) if edpcf.cores < 1: edPrint("Cannot use less than 1 core. Exiting...") return () # change config to not run next portions on their own pcfname = poetpcf.rundir + '/' + poetpcf.eventname + '.pcf' eventpcf = open(pcfname, 'r').readlines() for i, line in enumerate(eventpcf): for num in ('2', '3', '4', '5'): if line.startswith("runp" + num): eventpcf[i] = "runp" + num + " False\n" open(pcfname, 'w').writelines(eventpcf) # Run parts of EDGAR starting from where is specified resume = [ read, mask, centphot, p6_init, p6_full, p6_check, p6_final, finish, zen ] start = { "read": 0, "mask": 1, "centphot": 2, "p6_init": 3, "p6_full": 4, "p6_check": 5, "p6_final": 6, "finish": 7, "zen": 8 } for func in resume[start[task]:]: args = func(*args) if isinstance(args, str): return args
def centphot(poetpcf, edpcf): """Runs centering, photometry, and checks (p3, p4, & p5). Parameters ---------- poetpcf : reader3.pcf, string String giving location of poet.pcf, or pcf object read from the POET section of poet.pcf. This indirectly specifies the run directory and the event pcf to use. edpcf : reader3.pcf, string String giving location of poet.pcf, or pcf object read from the EDGAR section of poet.pcf. Returns ------- string or tuple If centphot completes successfully, returns a tuple of (poetpcf, edpcf) reader3.pcf objects. If read is unsuccessful, returns the string "quit". Notes ----- If you want to pass in your own pcf object, you should generate them like so: poetpcf = rd.read_pcf("poet.pcf", "POET", simple=True) edpcf = rd.read_pcf("poet.pcf", "EDGAR", simple=True, d1lists=['zenmodels']) """ edPrint("I am queuing up centering and photometry tasks...") returnPrint("centphot") config = poetpcf.eventname + '.pcf' # get pcfs and queue up tasks cpcfs = rd.read_pcf(poetpcf.rundir + "/" + config, 'centering') ppcfs = rd.read_pcf(poetpcf.rundir + "/" + config, 'photometry') p3tasks = [] p4tasks = [] p5tasks = [] for cpcf in cpcfs: # make directory and place pcfs center = cpcf.method if cpcf.pcfname is None else cpcf.method + '_' + cpcf.pcfname cdir = poetpcf.rundir + '/' + center os.makedirs(cdir, exist_ok=True) cpcf.make_file(cdir + '/' + config, "centering") rd.copy_config(poetpcf.rundir + '/' + config, 'photometry', cdir + '/' + config) # add task p3task = Task(poetpcf.rundir, poetpcf.eventname, cpcf.ccores, poet.p, args=(3, center), kwargs={"poetpcf": poetpcf}) p3tasks.append(p3task) for ppcf in ppcfs: # check that elliptical photometry is only used on fgc or rfgc if ppcf.phottype == 'ell': if cpcf.method not in ['fgc', 'rfgc']: continue # make photometry directory and place pcfs photom = photname(ppcf) pdir = cdir + '/' + photom os.makedirs(pdir, exist_ok=True) ppcf.make_file(pdir + '/' + config, "photometry") # add photometry tasks to queue p4task = Task(poetpcf.rundir, poetpcf.eventname, ppcf.ncores, poet.p, args=(4, center + '/' + photom), prereqs=[p3task], kwargs={"poetpcf": poetpcf}) p4tasks.append(p4task) p5task = Task(poetpcf.rundir, poetpcf.eventname, 1, poet.p, args=(5, center + '/' + photom), prereqs=[p4task], kwargs={"poetpcf": poetpcf}) p5tasks.append(p5task) edPrint("I am completing p3-5 in a convenient order") curr_core = 0 curr_mem = 200 curr_tasks = 'p3' tasks = p3tasks + p4tasks + p5tasks while True: for task in tasks: # if task is finished, remove its mem and core # contributions to counters if task.started and not task.thread.is_alive(): curr_core -= task.cores curr_mem -= task.mem tasks.remove(task) elif task.started: continue # otherwise try to start the task else: cores, mem = task.check(curr_core, edpcf.cores, curr_mem, edpcf.memory) curr_core += cores curr_mem += mem if hasattr(task, "mem"): if task.mem > 200 + edpcf.memory: edPrint("Woops, it seems I don't have enough memory a" "located to finish these centering and photo" "metry runs. Increase 'memory' in poet.pcf?") return 'quit' if task.cores > edpcf.cores: edPrint("Woops, it seems I don't have enough cores a" "located to finish these centering and photo" "metry runs. Increase 'cores' in poet.pcf?") return 'quit' # change task set once you are ready if tasks == []: return (poetpcf, edpcf) # don't use up an entire cpu time.sleep(1)
def guessp1size(poetpcf): """From and event pcf, guesses how much memory running p1 will take.""" # the size of a p1 object is roughly 4.047 (rounding to 4.1) times # the size of one of its four arrays (data, uncertainty, bdmask, # and brmask). To guess the size of an event generated in p1, we # need to find the size of one of these arrays which at the time # of this writing, follows the formula: # # arrsize = 80 + 16*dim + 8*#ofvals [bytes] (assuming 64 bit values) # # Changes to this formula in future versions should be # insignificant (unless 128 or higher bit values are # used). Additionally, running python and importing everything # uses about 116 MB (res) of RAM. I round this up to 200 MB for # safesies and because more memory will be used to make other # things. Hence the final equation I will use is: # # size = 4.1 * (arrsize / 1024**2) + 200 # get values from pcf pcf, = rd.read_pcf(poetpcf.rundir + '/' + poetpcf.eventname + '.pcf', "event", expand=False) npos = pcf.npos[0] nnod = pcf.nnod[0] aornames = np.array(pcf.aorname) aortypes = np.array(pcf.aortype) # find dimensions of array which are ny, nx, npos, maxnimpos instru = inst.Instrument(pcf.photchan[0]) naor = np.size(aornames[np.where(aortypes == 0)]) nexpid = np.empty(naor, np.long) for i, aorname in enumerate(aornames[:naor]): bcddir = '/'.join([ pcf.topdir[0], pcf.datadir[0], pcf.sscver[0], 'r' + str(aorname), instru.bcddir ]) files = sorted( [f for f in os.listdir(bcddir) if f.endswith(instru.bcdsuf)]) first, last = files[0], files[-1] fexpid = first.split('_')[-4] expid, dcenum = last.split('_')[-4:-2] nexpid[i] = int(expid) - int(fexpid) + 1 ndcenum = int(dcenum) + 1 header = fits.getheader(bcddir + '/' + first) # number of dimension is hard coded to be 4 in # pdataread.poet_dataread() and throughout the code dim = 4 if header['NAXIS'] >= 3: nz = header['NAXIS1'] ny = header['NAXIS2'] nx = header['NAXIS3'] else: nz = 1 ny = header['NAXIS1'] nx = header['NAXIS2'] # equations taken from p1event if instru.name == 'mips': maxnimpos = int((np.sum(nexpid) / nnod) * (((ndcenum - 1) / 7) + 1)) else: maxnimpos = int(np.sum(nexpid) * ndcenum * nz / nnod) # in bytes arrsize = 80 + 16 * dim + 8 * (nx * ny * npos * maxnimpos) # in MB return 4.1 * (arrsize / 1024**2) + 200
def run_photometry(eventname, control, cwd=None): """ Load the event. Read the control file. Launch a thread for each centering run. """ if cwd == None: cwd = os.getcwd() os.chdir(cwd) pcf = rd.read_pcf(control) nruns = len(pcf) if nruns == 1: #, I may be in photdir to re-run: # Get name of photometry dir: if pcf[0].dooptimal.get(): photdir = 'optimal' + '%02d' % pcf[0].oresize.get() else: photdir = ('ap%03d' % (pcf[0].photap.get() * 100) + '%02d' % pcf[0].skyin.get() + '%02d' % pcf[0].skyout.get()) if pcf[0].pcfname.get() != "": photdir += "_" + pcf[0].pcfname.get() # If I am in the photometry dir already: if cwd[-len(photdir):] == photdir: # Go to dir where poet3 files were saved. cwd = cwd[:-len(photdir)] os.chdir(cwd) mute = False else: mute = True # Load the event: event = me.loadevent(eventname, load=['data', 'uncd', 'mask']) # Loop over each run: for run in np.arange(nruns): os.chdir(cwd) # Make a copy of the event: this_event = copy.copy(event) # Get name of photometry dir: if pcf[run].dooptimal.get(): photdir = 'optimal' + '%02d' % pcf[run].oresize.get() else: photdir = ('ap%03d' % (pcf[run].photap.get() * 100) + '%02d' % pcf[run].skyin.get() + '%02d' % pcf[run].skyout.get()) if pcf[run].pcfname.get() != "": photdir += "_" + pcf[run].pcfname.get() this_event.photdir = photdir # Create the photometry directory if it doesn't exist: if not os.path.exists(photdir): os.mkdir(photdir) # Launch the thread: p = Process(target=photometry, args=(this_event, pcf[run], photdir, mute)) p.start()
def zen(poetpcf, edpcf): """Runs ZEN. For each zenmodel specified in edpcf, this function runs ZEN on every centering and photometry (call these full runs). Then, for each model, ZEN is run again, this time only on the best centering and photometry as determined by the corresponding full run with binning set to 1 (call these BIC runs). The BIC run with the lowest BIC is selected and the full run with the same model is the "best". A report string indicating the "best" result directory, as well as all directories is printed. Parameters ---------- poetpcf : reader3.pcf, string String giving location of poet.pcf, or pcf object read from the POET section of poet.pcf. This specifies the run directory and the event pcf to use. edpcf : reader3.pcf, string String giving location of poet.pcf, or pcf object read from the EDGAR section of poet.pcf. This specifies the the models to run Returns ------- string If read is unsuccessful, returns the string "quit". Otherwise, returns "Done". Notes ----- The configuration for each model, [model], is found by searching the appropriate run directory for a file named zen_[model].cfg. If that file is not found, a new config is generated from a template. The following settings in zen_*.cfg will be overwritten by this function: eventname, nchains, cent, phot, maxbinsize If you want to pass in your own pcf object, you should generate them like so: poetpcf = rd.read_pcf("poet.pcf", "POET", simple=True) edpcf = rd.read_pcf("poet.pcf", "EDGAR", simple=True, d1lists=['zenmodels']) """ edPrint("Running ZEN on specified models") returnPrint("zen") if not edpcf.runzen: edPrint("`runzen` is false in poet.pcf, returning...") return 'quit' eventpcf = poetpcf.rundir + '/' + poetpcf.eventname + '.pcf' # find example centering and photometry dir cpcf = rd.read_pcf(eventpcf, "centering")[0] ppcf = rd.read_pcf(eventpcf, "photometry")[0] centdir = cpcf.method if cpcf.pcfname is None else cpcf.method + '_' + cpcf.pcfname photdir = photname(ppcf) # check memory usage filename = poetpcf.rundir + '/' + poetpcf.eventname + '-zen.cfg' zenmem = guesszensize(poetpcf, edpcf, centdir, photdir, filename) if zenmem > edpcf.memory: edPrint( "EDGAR does not have enough memory allotted to run the with t" "he specified number of cores. Either increase \"memory\" or " "decrease \"zenchains\" in \"EDGAR\" portion of poet.pcf and rer" "un.") returnPrint("zen") return 'quit' # Run zen with each model to determine the best cent/ap combo for each model bestdir = {} compareBIC = {} bicdir = {} for i, model_set in enumerate(edpcf.zenmodels): model_set = ' '.join(model_set) # move config files around config = cfg.ConfigParser() config.read([filename]) config['EVENT']['models'] = model_set date = datetime.datetime.today().strftime("%Y-%m-%d_%H_%M") config['EVENT']['outdir'] = date + '_zen_model%d' % i + "_full" config['EVENT']['cent'] = 'all' config['EVENT']['phot'] = 'all' # run ZEN outdir, centdir, photdir, chiout = poet.p("zen", config, poetpcf=poetpcf, cfilename='zen.cfg') bestdir[model_set] = outdir # Rerun each model on their respective best combo with bintry=1 to # compare across models. config = cfg.ConfigParser() config.read([filename]) config['EVENT']['models'] = model_set config['EVENT']['cent'] = centdir config['EVENT']['phot'] = photdir config['EVENT']['bintry'] = '1' date = datetime.datetime.today().strftime("%Y-%m-%d_%H_%M") config['EVENT']['outdir'] = date + '_zen_model%d' % i + "_BIC" config['MCMC']['chisqscale'] = "False" # run zen bintry=1 outdir, centdir, photdir, chiout = poet.p("zen", config, poetpcf=poetpcf, cfilename='zen.cfg') compareBIC[model_set] = chiout[3] bicdir[model_set] = outdir edPrint("Ran the following ZEN models:", edpcf.zenmodels) # find best BIC bestmodel = None bestBIC = None for model_set in compareBIC: if bestmodel is None or compareBIC[model_set] < bestBIC: bestmodel = model_set bestBIC = compareBIC[model_set] edPrint( "After rerunning models on their best cent/ap combinations with bi" "ntry=1, BICs were compared indicating that", bestmodel, "was the " "best. The output can be found at:\n", bestdir[bestmodel], "\nHowev" "er, you should still check to make sure that the ZEN runs all con" "verged. Here are the directories of the full runs' can be found a" "t:\n" + "\n".join(model_set + "\n" + bestdir[model_set] for model_set in bestdir), "\nand here are the directories of the mini runs that were conducted" " to compare BIC values:\n" + "\n".join(model_set + "\n" + bicdir[model_set] for model_set in bicdir)) edPrint("All Done!") return 'Done'
def run_denoising(eventname, cwd): """ Load the event. Read the control file. Launch a thread for each centering run. """ owd = os.getcwd() os.chdir(cwd) config = os.path.basename(eventname)[:-4] + '.pcf' pcfs = rd.read_pcf(config, 'denoise') if len(pcfs) == 1: #, I may be in the denoise dir, to re-run: # Get name of denoising dir: pcf = pcfs[0] denoisedir = pcf.wavelet +'_'+ pcf.threshold +'_L'+ \ str(pcf.numlvls) if pcf.pcfname is not None: denoisedir += "_" + str(pcf.pcfname) if cwd[-len(denoisedir):] == denoisedir: # Go to dir where poet2 files were saved. cwd = cwd[:-len(denoisedir)] os.chdir(cwd) # Loop over each run: # for run in np.arange(nruns): for run, pcf in enumerate(pcfs): # Load a fresh event: print("Loading " + eventname) event = me.loadevent(eventname, load=['data','uncd','mask']) # Name of the directory to put the results: denoisedir = pcf.wavelet +'_'+ pcf.threshold +'_L'+ str(pcf.numlvls) if pcf.pcfname is not None: denoisedir += "_" + str(pcf.pcfname) event.denoisedir = denoisedir # Create the denoising directory if it doesn't exist: if not os.path.exists(denoisedir): os.mkdir(denoisedir) # copy the centering and photometry configs and this pcf into # denoise directory filename = denoisedir + '/' + event.eventname + '.pcf' pcf.make_file(filename, 'denoise') rd.copy_config(config, ['centering', 'photometry'], filename) # Modify source estimate if hasattr(pcf, 'srcest'): if nruns == event.npos: event.srcest[:,run] = pcf.srcest else: for pos in range(event.npos): event.srcest[:,pos] = pcf.srcest # Set denoised flag to True: event.denoised = True # Call denoising on each wavelet: denoise(pcf, denoisedir, owd) os.chdir(owd) return