def resampleObjs(parser): (options, args) = parser.parse_args() if len(args) == 0: parser.print_help() return if os.path.exists(args[0]): print filename + " exists" print "Remove this file before running ..." print "Returning ..." return None if options.fitsfile is None: print "-f or --fitsfile must be set to the file containing the fits ..." print "Returning ..." return None #Load location of the data if options.sample == 'nuvx': dir = '../data/nuvx/' elif options.sample == 'qso': dir = '../data/s82qsos/' objs = QSOfilenames(dir=dir) #Load the fits if os.path.exists(options.fitsfile): fitsfile = open(options.fitsfile, 'rb') params = pickle.load(fitsfile) type = pickle.load(fitsfile) band = pickle.load(fitsfile) fitsfile.close() else: print options.fitsfile + " does not exist ..." print "Returning ..." return None #Load sampling if options.sampling == 'PS1': sampling = panstarrs_sampling(3, startmjd=2. * 365.25) elif options.sampling == 'SDSS-PS1': sampling = sdss_sampling(startmjd=-2. * 365.25) sampling.extend(panstarrs_sampling(1, startmjd=2. * 365.25)) elif options.sampling == 'SDSS': pass else: print "Input to --sampling not understood ..." print "Returning ..." return None #Re-sample each source out = [] savecount, count = 0, 0 for obj in objs: key = os.path.basename(obj) print "Working on " + str(count) + " (" + str(savecount) + "): " + key savecount += 1 v = VarQso(obj) #Find fit try: thisfit = params[key] except KeyError: print "Fit not found, skipping this object ..." nepochs = v.nepochs(band) if nepochs < 20: print "Because #epochs < 20 ..." continue #Set LC model if options.nocolorvar: thisfit['gammagr'] = 0. if 'logAgr' in thisfit.keys(): thisfit['logAgr'] = -7. else: thisfit['logAri'] = -7. if options.sampling == 'SDSS': sampling = v.mjd['g'] sampling = [(s, 'g') for s in sampling] v.setLCmodel(thisfit, band, type) indx = v.mjd_overlap(band=band) refband = 'r' try: o = v.resample([(mjd, refband) for mjd in v.mjd[refband][indx[refband]]], band=refband, errors=False) except nu.linalg.LinAlgError: print thisfit continue xs = [] ys = [] errs = [] for b in band: #Add errors to the underlying xs.extend([(mjd, b) for mjd in v.mjd[b][indx[b]]]) for ii in range(len(v.mjd[b][indx[b]])): ys.append(o.m[refband][ii] + nu.random.randn() * v.err_m[b][indx[b]][ii]) errs.append(v.err_m[b][indx[b]][ii]) out.append([key, VarQso(xs, ys, errs, band=band, medianize=False)]) else: v.setLCmodel(thisfit, band, type) #Resample out.append([key, v.resample(sampling, band=band)]) count += 1 #Save outfile = open(args[0], 'wb') pickle.dump(out, outfile) pickle.dump(band, outfile) outfile.close() return None
def fitQSO(parser): (options,args)= parser.parse_args() if len(args) == 0: parser.print_help() return if len(args) == 2: othersavefilename= args[1] othersavefile= open(othersavefilename,'rb') otherparams= pickle.load(othersavefile) othersavefile.close() else: otherparams= {} savefilename= args[0] if os.path.exists(savefilename): savefile= open(savefilename,'rb') params= pickle.load(savefile) type= pickle.load(savefile) band= pickle.load(savefile) mean= pickle.load(savefile) savefile.close() if params.has_key('.fit'): params.pop('.fit') for key in params.keys(): if (params[key].has_key('gamma') and \ (params[key]['gamma'] < 0. or params[key]['gamma'] > 2.)) \ or (params[key].has_key('gammagr') \ and (params[key]['gammagr'] < 0. \ or params[key]['gammagr'] > 2.)): print "Popping bad gamma ..." params.pop(key) else: params= {} type= options.type mean= options.mean band= options.band if options.star: dir= '../data/star/' elif options.nuvx: dir= '../data/nuvx/' elif options.nuvxall: dir= '../data/nuvx_all/' elif options.uvx: dir= '../data/uvx/' elif options.rrlyrae: dir= '../data/rrlyrae/' else: dir= '../data/s82qsos/' if options.resampled: if os.path.exists(options.infile): samplefile= open(options.infile,'rb') qsos= pickle.load(samplefile) samplefile.close() else: print "'--resampled' is set, but -i filename does not exist ..." print "Returning ..." return None else: qsos= QSOfilenames(dir=dir) savecount= 0 count= len(params) for qso in qsos: if options.resampled: key= qso[0] else: key= os.path.basename(qso) if params.has_key(key) or otherparams.has_key(key): continue try: if int(key[5:7]) != options.rah and options.rah != -1: continue except ValueError: if options.rah == -2 or options.rah == -1: pass else: print "Skipping ValueError "+key continue print "Working on "+str(count)+": "+key if options.resampled: v= qso[1] else: v= VarQso(qso,flux=options.fitflux) if options.lownepochs: if v.nepochs(band) >= 20: print "This object has too many epochs ..." continue elif v.nepochs(band) < 3: print "This object does not have enough epochs ..." continue elif not options.lownepochs and v.nepochs(band) < 20: print "This object does not have enough epochs ..." continue params[key]= v.fit(band=band,type=type,loglike=True,mean=mean) if _DEBUG: print params[key] if params[key]['loglike'] == -numpy.finfo(numpy.dtype(numpy.float64)).max: print "Popping bad fit ..." params.pop(key) if savecount == options.saveevery: print "Saving ..." save_pickles(params,type,band,mean,savefilename) savecount= 0 savecount+= 1 count+= 1 save_pickles(params,type,band,mean,savefilename) print "All done"
def classQSO(parser): (options,args)= parser.parse_args() if len(args) == 0: parser.print_help() return if os.path.exists(args[0]): print filename+" exists" print "Remove this file before running ..." print "Returning ..." return None #Load fit params: Quasars if options.qsomodel == 'test': if len(options.band) == 1: qsoparams= [{'gamma':0.3,'logA':-2.}] else:#Multi-band qsoparams= [{'gamma':0.2,'logA':-2., 'gammagr':0.0001,'logAgr':-2.}] qsoweights= [1.] elif options.qsomodel == 'zero': qsoparams= [{}] qsoweights= [1.] elif os.path.exists(options.qsomodel): qsofile= open(options.qsomodel,'rb') qsoparams= pickle.load(qsofile) qsoweights= nu.array(pickle.load(qsofile),dtype='float64') qsofile.close() else: print "Input to 'qsomodel' not recognized ..." print "Returning ..." return #Stars if options.starmodel == 'test': if len(options.band) == 1: starparams= [{'gamma':0.0001,'logA':-3.5}] else:#Multi-band starparams= [{'gamma':0.1,'logA':-3., 'gammagr':0.0001,'logAgr':-2.}] starweights= [1.] elif options.starmodel == 'zero': starparams= [{}] starweights= [1.] elif os.path.exists(options.starmodel): starfile= open(options.starmodel,'rb') starparams= pickle.load(starfile) starweights= nu.array(pickle.load(starfile),dtype='float64') starfile.close() else: print "Input to 'starmodel' not recognized ..." print "Returning ..." return #RR Lyrae if options.rrlyraemodel == 'test': if len(options.band) == 1: rrlyraeparams= [{'gamma':0.0001,'logA':-2.}] else:#Multi-band rrlyraeparams= [{'gamma':0.1,'logA':-2., 'gammagr':0.0001,'logAgr':-2.}] rrlyraeweights= [1.] elif options.rrlyraemodel == 'zero': rrlyraeparams= [{}] rrlyraeweights= [1.] elif os.path.exists(options.rrlyraemodel): rrlyraefile= open(options.rrlyraemodel,'rb') rrlyraeparams= pickle.load(rrlyraefile) rrlyraeweights= nu.array(pickle.load(rrlyraefile),dtype='float64') rrlyraefile.close() else: print "Input to 'rrlyraemodel' not recognized ..." print "Returning ..." return #normalize weights qsoweights/= nu.sum(qsoweights) starweights/= nu.sum(starweights) rrlyraeweights/= nu.sum(rrlyraeweights) #Load location of the data if options.resampled: if os.path.exists(options.sample): samplefile= open(options.sample,'rb') objs= pickle.load(samplefile) samplefile.close() else: print "'--resampled' is set, but --sample= filename does not exist ..." print "Returning ..." return None else: if options.sample == 'nuvx': dir= '../data/nuvx/' if options.sample == 'nuvxall': dir= '../data/nuvx_all/' if options.sample == 'uvx': dir= '../data/uvx/' objs= QSOfilenames(dir=dir) #Classify each source out= [] allcount, count= 0, 0 for obj in objs: allcount+= 1 if options.resampled: key= obj[0] else: key= os.path.basename(obj) #if key != 'SDSSJ013306.18-004523.8.fit': # continue print "Working on "+str(count)+"(%i/%i): " % (allcount,len(objs))+key if options.resampled: v= obj[1] else: v= VarQso(obj) if v.nepochs(options.band) < options.minepochs: print "This object does not have enough epochs ..." continue varout= VarClass() varout.key= key #quasar likelihoods qsolike= [] for ii in range(len(qsoparams)): qsolike.append(v.loglike(band=options.band,type=options.type, params=qsoparams[ii]) +nu.log(qsoweights[ii])) qsolike= logsum(qsolike) varout.qsologlike= qsolike #star likelihoods starlike= [] for ii in range(len(starparams)): starlike.append(v.loglike(band=options.band,type=options.type, params=starparams[ii]) +nu.log(starweights[ii])) starlike= logsum(starlike) varout.starloglike= starlike #RR Lyrae likelihoods rrlyraelike= [] for ii in range(len(rrlyraeparams)): rrlyraelike.append(v.loglike(band=options.band,type=options.type, params=rrlyraeparams[ii]) +nu.log(rrlyraeweights[ii])) rrlyraelike= logsum(rrlyraelike) varout.rrlyraeloglike= rrlyraelike #print qsolike, starlike if qsolike > starlike and qsolike > rrlyraelike: print qsolike, starlike, rrlyraelike out.append(varout) count+= 1 #if count > 500: break #Save for jj in range(len(out)): if out[jj].qsologlike > out[jj].starloglike and out[jj].qsologlike > out[jj].rrlyraeloglike: print out[jj].key saveClass(out,args[0]) return None
def compareMagFluxFits(parser): (options, args) = parser.parse_args() if len(args) == 0: parser.print_help() return params = [] for filename in args: if os.path.exists(filename): savefile = open(filename, 'rb') params.append(pickle.load(savefile)) savefile.close() else: print filename + " does not exist ..." print "Returning ..." return if options.plottype == 'AA': ys = [] xs = [] for key in params[1].keys(): try: ys.append(params[0][key]['logA'] / 2.) xs.append(params[1][key]['logA'] / 2.) except KeyError: continue xs = nu.array(xs).reshape(len(xs)) ys = nu.array(ys).reshape(len(xs)) ys = xs - ys - nu.log(nu.log(10.) / 2.5) xrange = [-9.21 / 2., 0.] yrange = [-.25, .25] xlabel = r'$\log A^{\mathrm{flux}}_' + options.band + r'\ \mathrm{(amplitude\ at\ 1\ yr)}$' ylabel = r'$\log A^{\mathrm{flux}}_' + options.band + r'-\log A^{\mathrm{mag}}_' + options.band + r'- \log\left(\frac{\log 10}{2.5}\right)$' elif options.plottype == 'gg': ys = [] xs = [] for key in params[1].keys(): try: ys.append(params[0][key]['gamma']) xs.append(params[1][key]['gamma']) except KeyError: continue xs = nu.array(xs).reshape(len(xs)) ys = nu.array(ys).reshape(len(xs)) print len(xs) ys = xs - ys xrange = [0., 1.2] yrange = [-.25, .25] xlabel = r'$\gamma^{\mathrm{flux}}_' + options.band + r'\ \mathrm{(power-law\ exponent)}$' ylabel = r'$\gamma^{\mathrm{flux}}_' + options.band + r'- \gamma^{\mathrm{mag}}_' + options.band + '$' elif options.plottype == 'loglike2': ys = [] xs = [] nepochs = [] cnt = 0 for key in params[1].keys(): #cnt+= 1 #print cnt #if cnt > 10: break #Get the number of epochs v = VarQso(os.path.join('../data/s82qsos/', key)) try: ys.append(params[0][key]['loglike'] - v.nepochs(options.band) * nu.log(nu.log(10.) / 2.5)) nepochs.append(v.nepochs(options.band)) xs.append(params[1][key]['loglike']) except KeyError: continue xs = -nu.array(xs).reshape(len(xs)) ys = -nu.array(ys).reshape(len(xs)) nepochs = nu.array(nepochs).reshape(len(nepochs)) ys /= nepochs xs /= nepochs ys = xs - ys xrange = [-3., 0.] yrange = [-.1, .1] xlabel = r'$\log \mathcal{L}^{\mathrm{flux}}_{' + options.band + r',\mathrm{red}}\ \mathrm{(amplitude\ at\ 1\ yr)}$' ylabel = r'$\log \mathcal{L}^{\mathrm{flux}}_{' + options.band + r',\mathrm{red}}- \log \mathcal{L}^{\mathrm{mag}}_{' + options.band + ',\mathrm{red}}' + r'- \log\left(\frac{\log 10}{2.5}\right)$' bovy_plot.bovy_print() bovy_plot.scatterplot(xs, ys, 'k,', onedhists=True, yrange=yrange, xrange=xrange, bins=31, xlabel=xlabel, ylabel=ylabel) bovy_plot.bovy_plot(nu.array(xrange), [0., 0.], '0.5', overplot=True) bovy_plot.bovy_end_print(options.plotfilename) return None
def sampleQSO(parser): (options, args) = parser.parse_args() if len(args) == 0: parser.print_help() return if len(args) == 2: othersavefilename = args[1] othersavefile = open(othersavefilename, 'rb') othersamples = pickle.load(othersavefile) othersavefile.close() else: othersamples = {} savefilename = args[0] if os.path.exists(savefilename): savefile = open(savefilename, 'rb') samples = pickle.load(savefile) type = pickle.load(savefile) band = pickle.load(savefile) mean = pickle.load(savefile) savefile.close() else: samples = {} type = options.type mean = options.mean band = options.band if os.path.exists(options.fitsfile): fitsfile = open(options.fitsfile, 'rb') params = pickle.load(fitsfile) fitsfile.close() else: raise IOError( "--fitsfile (or -f) has to be set to the file holding the best-fits" ) if options.star: dir = '../data/star/' elif options.nuvx: dir = '../data/nuvx/' elif options.nuvxall: dir = '../data/nuvx_all/' elif options.uvx: dir = '../data/uvx/' elif options.rrlyrae: dir = '../data/rrlyrae/' else: dir = '../data/s82qsos/' if options.resampled: raise NotImplementedError("resampled not implemented yet") if os.path.exists(options.infile): samplefile = open(options.infile, 'rb') qsos = pickle.load(samplefile) samplefile.close() else: print "'--resampled' is set, but -i filename does not exist ..." print "Returning ..." return None else: qsos = QSOfilenames(dir=dir) #Register time-out handler signal.signal(signal.SIGALRM, handler) savecount = 0 count = len(samples) for qso in qsos: if options.resampled: key = qso[0] else: key = os.path.basename(qso) if samples.has_key(key) or othersamples.has_key(key): continue try: if int(key[5:7]) != options.rah and options.rah != -1: continue except ValueError: if options.rah == -2 or options.rah == -1: pass else: print "Skipping ValueError " + key continue print "Working on " + str(count) + ": " + key if options.resampled: v = qso[1] else: v = VarQso(qso) if v.nepochs(band) < 20: print "This object does not have enough epochs ..." continue #Set best-fit v.LCparams = params[key] v.LC = LCmodel(trainSet=v._build_trainset(band), type=type, mean=mean) v.LCtype = type v.LCmean = mean v.fitband = band #Now sample signal.alarm(options.timeout) try: v.sampleGP(nsamples=options.nsamples, metropolis=options.metropolis, markovpy=options.markovpy, burnin=int(nu.floor(0.2 * options.nsamples))) except Exception, exc: if str(exc) == "Sampling timed out": print exc continue else: raise signal.alarm(0) samples[key] = v.get_sampleGP() if _DEBUG: _print_diagnostics(samples[key]) #print samples[key][options.nsamples] savecount += 1 if savecount == options.saveevery: print "Saving ..." save_pickles(samples, type, band, mean, savefilename) savecount = 0 count += 1
def resampleObjs(parser): (options,args)= parser.parse_args() if len(args) == 0: parser.print_help() return if os.path.exists(args[0]): print filename+" exists" print "Remove this file before running ..." print "Returning ..." return None if options.fitsfile is None: print "-f or --fitsfile must be set to the file containing the fits ..." print "Returning ..." return None #Load location of the data if options.sample == 'nuvx': dir= '../data/nuvx/' elif options.sample == 'qso': dir= '../data/s82qsos/' objs= QSOfilenames(dir=dir) #Load the fits if os.path.exists(options.fitsfile): fitsfile= open(options.fitsfile,'rb') params= pickle.load(fitsfile) type= pickle.load(fitsfile) band= pickle.load(fitsfile) fitsfile.close() else: print options.fitsfile+" does not exist ..." print "Returning ..." return None #Load sampling if options.sampling == 'PS1': sampling= panstarrs_sampling(3,startmjd=2.*365.25) elif options.sampling == 'SDSS-PS1': sampling= sdss_sampling(startmjd=-2.*365.25) sampling.extend(panstarrs_sampling(1,startmjd=2.*365.25)) elif options.sampling == 'SDSS': pass else: print "Input to --sampling not understood ..." print "Returning ..." return None #Re-sample each source out= [] savecount, count= 0, 0 for obj in objs: key= os.path.basename(obj) print "Working on "+str(count)+" ("+str(savecount)+"): "+key savecount+= 1 v= VarQso(obj) #Find fit try: thisfit= params[key] except KeyError: print "Fit not found, skipping this object ..." nepochs= v.nepochs(band) if nepochs < 20: print "Because #epochs < 20 ..." continue #Set LC model if options.nocolorvar: thisfit['gammagr']= 0. if 'logAgr' in thisfit.keys(): thisfit['logAgr']= -7. else: thisfit['logAri']= -7. if options.sampling == 'SDSS': sampling= v.mjd['g'] sampling= [(s,'g') for s in sampling] v.setLCmodel(thisfit,band,type) indx= v.mjd_overlap(band=band) refband= 'r' try: o= v.resample([(mjd,refband) for mjd in v.mjd[refband][indx[refband]]], band=refband,errors=False) except nu.linalg.LinAlgError: print thisfit continue xs= [] ys= [] errs= [] for b in band: #Add errors to the underlying xs.extend([(mjd,b) for mjd in v.mjd[b][indx[b]]]) for ii in range(len(v.mjd[b][indx[b]])): ys.append(o.m[refband][ii]+ nu.random.randn()*v.err_m[b][indx[b]][ii]) errs.append(v.err_m[b][indx[b]][ii]) out.append([key,VarQso(xs,ys,errs,band=band,medianize=False)]) else: v.setLCmodel(thisfit,band,type) #Resample out.append([key,v.resample(sampling,band=band)]) count+= 1 #Save outfile= open(args[0],'wb') pickle.dump(out,outfile) pickle.dump(band,outfile) outfile.close() return None
def skewQSO(parser): (options,args)= parser.parse_args() if len(args) == 0: parser.print_help() return savefilename= args[0] if os.path.exists(savefilename): savefile= open(savefilename,'rb') skews= pickle.load(savefile) gaussskews= pickle.load(savefile) fittype= pickle.load(savefile) band= pickle.load(savefile) mean= pickle.load(savefile) taus= pickle.load(savefile) savefile.close() else: skews= {} gaussskews= {} fittype= options.type mean= options.mean band= options.band taus= nu.arange(options.dtau,options.taumax,options.dtau)/365.25 if os.path.exists(options.fitsfile): fitsfile= open(options.fitsfile,'rb') params= pickle.load(fitsfile) fitsfile.close() else: raise IOError("--fitsfile (or -f) has to be set to the file holding the best-fits") if options.star: dir= '../data/star/' elif options.nuvx: dir= '../data/nuvx/' elif options.nuvxall: dir= '../data/nuvx_all/' elif options.uvx: dir= '../data/uvx/' elif options.rrlyrae: dir= '../data/rrlyrae/' else: dir= '../data/s82qsos/' qsos= QSOfilenames(dir=dir) if not options.split is None: splitarr= nu.arange(len(qsos)) / int(nu.ceil(len(qsos)/float(options.split))) splitDict= {} for ii, qso in enumerate(qsos): key= os.path.basename(qso) splitDict[key]= splitarr[ii] print "Running bin %i ..." % (options.rah-1) savecount= 0 count= len(skews) if not options.star and not options.rrlyrae: #Read master file for redshifts dataqsos= open_qsos() qsoDict= {} ii=0 for qso in dataqsos: qsoDict[qso.oname.strip().replace(' ', '')+'.fit']= ii ii+= 1 for qso in qsos: key= os.path.basename(qso) if skews.has_key(key): continue if not options.split is None: if splitDict[key] != (options.rah-1): continue else: try: if int(key[5:7]) != options.rah and options.rah != -1: continue except ValueError: if options.rah == -2 or options.rah == -1: pass else: print "Skipping ValueError "+key continue sys.stdout.write('\r'+_ERASESTR+'\r') sys.stdout.flush() sys.stdout.write('\rWorking on %s: %s\r' % (str(count),key)) sys.stdout.flush() v= VarQso(qso,flux=options.flux) if v.nepochs(band) < 20: #print "This object does not have enough epochs ..." continue #Set best-fit if options.flux: params[key]['logA']+= 2.*nu.log(nu.log(10.)/2.5) if options.mean == 'const': params[key]['m']= -nu.log(10.)/2.5*params[key]['m'] v.LCparams= params[key] v.LC= LCmodel(trainSet=v._build_trainset(band),type=fittype,mean=mean, init_params=params[key]) v.LCtype= fittype v.LCmean= mean v.fitband= band #Now compute skew and Gaussian samples try: thisskew= v.skew(taus,band) except RuntimeError: continue if options.flux: thisskew*= -1. thisgaussskews= nu.zeros((options.nsamples,len(taus))) for ii in range(options.nsamples): #First re-sample if options.star or options.rrlyrae: redshift= 0. else: redshift= dataqsos[qsoDict[key]].z try: o= v.resample(v.mjd[band],band=band,noconstraints=True, wedge=options.wedge, wedgerate=options.wedgerate*365.25/(1.+redshift), wedgetau=(1.+redshift)) #1yr except nu.linalg.linalg.LinAlgError: if params[key]['gamma'] > 1.5 \ or params[key]['logA'] < -10.: continue #re-sampling fails because of bad gamma/logA else: print key, params[key] o.LCparams= v.LCparams o.LC= v.LC o.fitband= v.fitband o.LCtype= v.LCtype o.LCmean= v.LCmean if options.wedge: o.LCparams['gamma']= 1. o.LCparams['logA']= o.LCparams['logA']\ +nu.log(0.025**v.LCparams['gamma']/0.025) o.LCmean= 'zero' #bc we remove the mean when resampling wedge #Set up LC with correct params o.LC= LCmodel(trainSet=o._build_trainset(band), type=o.LCtype,mean=o.LCmean, init_params=o.LCparams) thisgaussskews[ii,:]= o.skew(taus,band) if options.flux: thisgaussskews[ii,:]*= -1. skews[key]= thisskew gaussskews[key]= thisgaussskews savecount+= 1 if savecount == options.saveevery: sys.stdout.write('\r'+_ERASESTR+'\r') sys.stdout.flush() sys.stdout.write('\rSaving ...\r') sys.stdout.flush() save_pickles(savefilename,skews,gaussskews,fittype,band,mean,taus) savecount= 0 count+= 1 sys.stdout.write('\r'+_ERASESTR+'\r') sys.stdout.flush() save_pickles(savefilename,skews,gaussskews,fittype,band,mean,taus) print "All done"
def sampleQSO(parser): (options,args)= parser.parse_args() if len(args) == 0: parser.print_help() return if len(args) == 2: othersavefilename= args[1] othersavefile= open(othersavefilename,'rb') othersamples= pickle.load(othersavefile) othersavefile.close() else: othersamples= {} savefilename= args[0] if os.path.exists(savefilename): savefile= open(savefilename,'rb') samples= pickle.load(savefile) type= pickle.load(savefile) band= pickle.load(savefile) mean= pickle.load(savefile) savefile.close() else: samples= {} type= options.type mean= options.mean band= options.band if os.path.exists(options.fitsfile): fitsfile= open(options.fitsfile,'rb') params= pickle.load(fitsfile) fitsfile.close() else: raise IOError("--fitsfile (or -f) has to be set to the file holding the best-fits") if options.star: dir= '../data/star/' elif options.nuvx: dir= '../data/nuvx/' elif options.nuvxall: dir= '../data/nuvx_all/' elif options.uvx: dir= '../data/uvx/' elif options.rrlyrae: dir= '../data/rrlyrae/' else: dir= '../data/s82qsos/' if options.resampled: raise NotImplementedError("resampled not implemented yet") if os.path.exists(options.infile): samplefile= open(options.infile,'rb') qsos= pickle.load(samplefile) samplefile.close() else: print "'--resampled' is set, but -i filename does not exist ..." print "Returning ..." return None else: qsos= QSOfilenames(dir=dir) #Register time-out handler signal.signal(signal.SIGALRM, handler) savecount= 0 count= len(samples) for qso in qsos: if options.resampled: key= qso[0] else: key= os.path.basename(qso) if samples.has_key(key) or othersamples.has_key(key): continue try: if int(key[5:7]) != options.rah and options.rah != -1: continue except ValueError: if options.rah == -2 or options.rah == -1: pass else: print "Skipping ValueError "+key continue print "Working on "+str(count)+": "+key if options.resampled: v= qso[1] else: v= VarQso(qso) if options.lownepochs: if v.nepochs(band) >= 20: print "This object has too many epochs ..." continue elif v.nepochs(band) < 3: print "This object has < 3 epochs ..." continue elif not options.lownepochs and v.nepochs(band) < 20: print "This object does not have enough epochs ..." continue #Set best-fit v.LCparams= params[key] v.LC= LCmodel(trainSet=v._build_trainset(band),type=type,mean=mean, init_params=params[key]) v.LCtype= type v.LCmean= mean v.fitband= band #Now sample signal.alarm(options.timeout) try: v.sampleGP(nsamples=options.nsamples,metropolis=options.metropolis, markovpy=options.markovpy, burnin=int(nu.floor(0.2*options.nsamples))) except Exception, exc: if str(exc) == "Sampling timed out": print exc continue else: raise signal.alarm(0) samples[key]= v.get_sampleGP() if _DEBUG: _print_diagnostics(samples[key]) #print samples[key][options.nsamples] savecount+= 1 if savecount == options.saveevery: print "Saving ..." save_pickles(samples,type,band,mean,savefilename) savecount= 0 count+= 1
def fitQSO(parser): (options, args) = parser.parse_args() if len(args) == 0: parser.print_help() return if len(args) == 2: othersavefilename = args[1] othersavefile = open(othersavefilename, 'rb') otherparams = pickle.load(othersavefile) othersavefile.close() else: otherparams = {} savefilename = args[0] if os.path.exists(savefilename): savefile = open(savefilename, 'rb') params = pickle.load(savefile) type = pickle.load(savefile) band = pickle.load(savefile) mean = pickle.load(savefile) savefile.close() if params.has_key('.fit'): params.pop('.fit') for key in params.keys(): if (params[key].has_key('gamma') and \ (params[key]['gamma'] < 0. or params[key]['gamma'] > 2.)) \ or (params[key].has_key('gammagr') \ and (params[key]['gammagr'] < 0. \ or params[key]['gammagr'] > 2.)): print "Popping bad gamma ..." params.pop(key) else: params = {} type = options.type mean = options.mean band = options.band if options.star: dir = '../data/star/' elif options.nuvx: dir = '../data/nuvx/' elif options.nuvxall: dir = '../data/nuvx_all/' elif options.uvx: dir = '../data/uvx/' elif options.rrlyrae: dir = '../data/rrlyrae/' else: dir = '../data/s82qsos/' if options.resampled: if os.path.exists(options.infile): samplefile = open(options.infile, 'rb') qsos = pickle.load(samplefile) samplefile.close() else: print "'--resampled' is set, but -i filename does not exist ..." print "Returning ..." return None else: qsos = QSOfilenames(dir=dir) savecount = 0 count = len(params) for qso in qsos: if options.resampled: key = qso[0] else: key = os.path.basename(qso) if params.has_key(key) or otherparams.has_key(key): continue try: if int(key[5:7]) != options.rah and options.rah != -1: continue except ValueError: if options.rah == -2 or options.rah == -1: pass else: print "Skipping ValueError " + key continue print "Working on " + str(count) + ": " + key if options.resampled: v = qso[1] else: v = VarQso(qso, flux=options.fitflux) if v.nepochs(band) < 20: print "This object does not have enough epochs ..." continue params[key] = v.fit(band=band, type=type, loglike=True, mean=mean) if _DEBUG: print params[key] if params[key]['loglike'] == -numpy.finfo(numpy.dtype( numpy.float64)).max: print "Popping bad fit ..." params.pop(key) if savecount == options.saveevery: print "Saving ..." save_pickles(params, type, band, mean, savefilename) savecount = 0 savecount += 1 count += 1 save_pickles(params, type, band, mean, savefilename) print "All done"
def skew0957(parser): (options,args)= parser.parse_args() if len(args) == 0: parser.print_help() return savefilename= args[0] #Read data if options.vanderriest: vA= VarQso('../data/0957-A.fits',band=options.band) vB= VarQso('../data/0957-B.fits',band=options.band) else: vA= VarQso('../data/L0957-A_%s.fits' % options.band,band=options.band) vB= VarQso('../data/L0957-B_%s.fits' % options.band,band=options.band) if not options.fitsfile is None and os.path.exists(options.fitsfile): fitsfile= open(options.fitsfile,'rb') paramsA= pickle.load(fitsfile) paramsB= pickle.load(fitsfile) paramsAB= pickle.load(fitsfile) fitsfile.close() vA.LCparams= paramsA vB.LCparams= paramsB vA.LCtype='powerlawSF' vB.LCtype='powerlawSF' vA.LCmean= 'const' vB.LCmean= 'const' vA.fitband= options.band vB.fitband= options.band else: #Fit for means print "Fitting SF for both images ..." vA.fit(options.band,mean='const') vB.fit(options.band,mean='const') #Load into single new VarQso newm= list(vA.m[options.band]-vA.LCparams['m']) newerrm= list(vA.err_m[options.band]) newmjd= list(vA.mjd[options.band]) newm.extend(list(vB.m[options.band]+nu.mean(vA.m[options.band]) -nu.mean(vB.m[options.band])-vB.LCparams['m'])) newerrm.extend(list(vB.err_m[options.band])) newmjd.extend(list(vB.mjd[options.band]-417./365.25))#shift lagged B v= VarQso(newmjd,newm,newerrm,band=options.band,medianize=False) if not options.fitsfile is None and os.path.exists(options.fitsfile): v.LCparams= paramsAB v.LCtype='powerlawSF' v.LCmean= 'zero' v.fitband= options.band v.LC= LCmodel(trainSet=v._build_trainset(options.band), type=v.LCtype,mean=v.LCmean, init_params=paramsAB) else: v.fit(options.band) if not options.fitsfile is None and not os.path.exists(options.fitsfile): save_pickles(options.fitsfile,vA.LCparams,vB.LCparams,v.LCparams) taus= nu.arange(1.,201.,1.)/365.25 thisskew= v.skew(taus,options.band,duration=0.7) thisgaussskews= nu.zeros((options.nsamples,len(taus))) print "Calculating simulated skews ..." for ii in range(options.nsamples): #First re-sample redshift= 1.41 o= v.resample(v.mjd[options.band],band=options.band,noconstraints=True, wedge=options.wedge, wedgerate=options.wedgerate*365.25/(1.+redshift), wedgetau=(1.+redshift)) #1yr o.LCparams= v.LCparams o.LC= v.LC o.fitband= v.fitband o.LCtype= v.LCtype o.LCmean= v.LCmean if options.wedge: o.LCparams['gamma']= 1. o.LCparams['logA']= o.LCparams['logA']\ +nu.log(0.05**v.LCparams['gamma']/0.05) o.LCmean= 'zero' #bc we remove the mean when resampling wedge #Set up LC with correct params o.LC= LCmodel(trainSet=o._build_trainset(options.band), type=o.LCtype,mean=o.LCmean, init_params=o.LCparams) thisgaussskews[ii,:]= o.skew(taus,options.band,duration=0.7) skews= {} gaussskews= {} skews['0957']= thisskew gaussskews['0957']= thisgaussskews save_pickles(savefilename,skews,gaussskews, None,options.band,None,taus) return None
def classQSO(parser): (options, args) = parser.parse_args() if len(args) == 0: parser.print_help() return if os.path.exists(args[0]): print filename + " exists" print "Remove this file before running ..." print "Returning ..." return None #Load fit params: Quasars if options.qsomodel == 'test': if len(options.band) == 1: qsoparams = [{'gamma': 0.3, 'logA': -2.}] else: #Multi-band qsoparams = [{ 'gamma': 0.2, 'logA': -2., 'gammagr': 0.0001, 'logAgr': -2. }] qsoweights = [1.] elif options.qsomodel == 'zero': qsoparams = [{}] qsoweights = [1.] elif os.path.exists(options.qsomodel): qsofile = open(options.qsomodel, 'rb') qsoparams = pickle.load(qsofile) qsoweights = nu.array(pickle.load(qsofile), dtype='float64') qsofile.close() else: print "Input to 'qsomodel' not recognized ..." print "Returning ..." return #Stars if options.starmodel == 'test': if len(options.band) == 1: starparams = [{'gamma': 0.0001, 'logA': -3.5}] else: #Multi-band starparams = [{ 'gamma': 0.1, 'logA': -3., 'gammagr': 0.0001, 'logAgr': -2. }] starweights = [1.] elif options.starmodel == 'zero': starparams = [{}] starweights = [1.] elif os.path.exists(options.starmodel): starfile = open(options.starmodel, 'rb') starparams = pickle.load(starfile) starweights = nu.array(pickle.load(starfile), dtype='float64') starfile.close() else: print "Input to 'starmodel' not recognized ..." print "Returning ..." return #RR Lyrae if options.rrlyraemodel == 'test': if len(options.band) == 1: rrlyraeparams = [{'gamma': 0.0001, 'logA': -2.}] else: #Multi-band rrlyraeparams = [{ 'gamma': 0.1, 'logA': -2., 'gammagr': 0.0001, 'logAgr': -2. }] rrlyraeweights = [1.] elif options.rrlyraemodel == 'zero': rrlyraeparams = [{}] rrlyraeweights = [1.] elif os.path.exists(options.rrlyraemodel): rrlyraefile = open(options.rrlyraemodel, 'rb') rrlyraeparams = pickle.load(rrlyraefile) rrlyraeweights = nu.array(pickle.load(rrlyraefile), dtype='float64') rrlyraefile.close() else: print "Input to 'rrlyraemodel' not recognized ..." print "Returning ..." return #normalize weights qsoweights /= nu.sum(qsoweights) starweights /= nu.sum(starweights) rrlyraeweights /= nu.sum(rrlyraeweights) #Load location of the data if options.resampled: if os.path.exists(options.sample): samplefile = open(options.sample, 'rb') objs = pickle.load(samplefile) samplefile.close() else: print "'--resampled' is set, but --sample= filename does not exist ..." print "Returning ..." return None else: if options.sample == 'nuvx': dir = '../data/nuvx/' if options.sample == 'nuvxall': dir = '../data/nuvx_all/' if options.sample == 'uvx': dir = '../data/uvx/' objs = QSOfilenames(dir=dir) #Classify each source out = [] allcount, count = 0, 0 for obj in objs: allcount += 1 if options.resampled: key = obj[0] else: key = os.path.basename(obj) #if key != 'SDSSJ013306.18-004523.8.fit': # continue print "Working on " + str(count) + "(%i/%i): " % (allcount, len(objs)) + key if options.resampled: v = obj[1] else: v = VarQso(obj) if v.nepochs(options.band) < options.minepochs: print "This object does not have enough epochs ..." continue varout = VarClass() varout.key = key #quasar likelihoods qsolike = [] for ii in range(len(qsoparams)): qsolike.append( v.loglike(band=options.band, type=options.type, params=qsoparams[ii]) + nu.log(qsoweights[ii])) qsolike = logsum(qsolike) varout.qsologlike = qsolike #star likelihoods starlike = [] for ii in range(len(starparams)): starlike.append( v.loglike(band=options.band, type=options.type, params=starparams[ii]) + nu.log(starweights[ii])) starlike = logsum(starlike) varout.starloglike = starlike #RR Lyrae likelihoods rrlyraelike = [] for ii in range(len(rrlyraeparams)): rrlyraelike.append( v.loglike(band=options.band, type=options.type, params=rrlyraeparams[ii]) + nu.log(rrlyraeweights[ii])) rrlyraelike = logsum(rrlyraelike) varout.rrlyraeloglike = rrlyraelike #print qsolike, starlike if qsolike > starlike and qsolike > rrlyraelike: print qsolike, starlike, rrlyraelike out.append(varout) count += 1 #if count > 500: break #Save for jj in range(len(out)): if out[jj].qsologlike > out[jj].starloglike and out[ jj].qsologlike > out[jj].rrlyraeloglike: print out[jj].key saveClass(out, args[0]) return None
def compareMagFluxFits(parser): (options,args)= parser.parse_args() if len(args) == 0: parser.print_help() return params= [] for filename in args: if os.path.exists(filename): savefile= open(filename,'rb') params.append(pickle.load(savefile)) savefile.close() else: print filename+" does not exist ..." print "Returning ..." return if options.plottype == 'AA': ys= [] xs= [] for key in params[1].keys(): try: ys.append(params[0][key]['logA']/2.) xs.append(params[1][key]['logA']/2.) except KeyError: continue xs= nu.array(xs).reshape(len(xs)) ys= nu.array(ys).reshape(len(xs)) ys= xs-ys-nu.log(nu.log(10.)/2.5) xrange=[-9.21/2.,0.] yrange=[-.25,.25] xlabel= r'$\log A^{\mathrm{flux}}_'+options.band+r'\ \mathrm{(amplitude\ at\ 1\ yr)}$' ylabel= r'$\log A^{\mathrm{flux}}_'+options.band+r'-\log A^{\mathrm{mag}}_'+options.band+r'- \log\left(\frac{\log 10}{2.5}\right)$' elif options.plottype == 'gg': ys= [] xs= [] for key in params[1].keys(): try: ys.append(params[0][key]['gamma']) xs.append(params[1][key]['gamma']) except KeyError: continue xs= nu.array(xs).reshape(len(xs)) ys= nu.array(ys).reshape(len(xs)) print len(xs) ys= xs-ys xrange=[0.,1.2] yrange=[-.25,.25] xlabel= r'$\gamma^{\mathrm{flux}}_'+options.band+r'\ \mathrm{(power-law\ exponent)}$' ylabel= r'$\gamma^{\mathrm{flux}}_'+options.band+r'- \gamma^{\mathrm{mag}}_'+options.band+'$' elif options.plottype == 'loglike2': ys= [] xs= [] nepochs= [] cnt= 0 for key in params[1].keys(): #cnt+= 1 #print cnt #if cnt > 10: break #Get the number of epochs v= VarQso(os.path.join('../data/s82qsos/',key)) try: ys.append(params[0][key]['loglike'] -v.nepochs(options.band)*nu.log(nu.log(10.)/2.5)) nepochs.append(v.nepochs(options.band)) xs.append(params[1][key]['loglike']) except KeyError: continue xs= -nu.array(xs).reshape(len(xs)) ys= -nu.array(ys).reshape(len(xs)) nepochs= nu.array(nepochs).reshape(len(nepochs)) ys/= nepochs xs/= nepochs ys= xs-ys xrange=[-3.,0.] yrange=[-.1,.1] xlabel= r'$\log \mathcal{L}^{\mathrm{flux}}_{'+options.band+r',\mathrm{red}}\ \mathrm{(amplitude\ at\ 1\ yr)}$' ylabel= r'$\log \mathcal{L}^{\mathrm{flux}}_{'+options.band+r',\mathrm{red}}- \log \mathcal{L}^{\mathrm{mag}}_{'+options.band+',\mathrm{red}}'+r'- \log\left(\frac{\log 10}{2.5}\right)$' bovy_plot.bovy_print() bovy_plot.scatterplot(xs,ys,'k,',onedhists=True, yrange=yrange, xrange=xrange,bins=31, xlabel=xlabel, ylabel=ylabel) bovy_plot.bovy_plot(nu.array(xrange),[0.,0.],'0.5', overplot=True) bovy_plot.bovy_end_print(options.plotfilename) return None