def resampleObjs(parser): (options, args) = parser.parse_args() if len(args) == 0: parser.print_help() return if os.path.exists(args[0]): print filename + " exists" print "Remove this file before running ..." print "Returning ..." return None if options.fitsfile is None: print "-f or --fitsfile must be set to the file containing the fits ..." print "Returning ..." return None #Load location of the data if options.sample == 'nuvx': dir = '../data/nuvx/' elif options.sample == 'qso': dir = '../data/s82qsos/' objs = QSOfilenames(dir=dir) #Load the fits if os.path.exists(options.fitsfile): fitsfile = open(options.fitsfile, 'rb') params = pickle.load(fitsfile) type = pickle.load(fitsfile) band = pickle.load(fitsfile) fitsfile.close() else: print options.fitsfile + " does not exist ..." print "Returning ..." return None #Load sampling if options.sampling == 'PS1': sampling = panstarrs_sampling(3, startmjd=2. * 365.25) elif options.sampling == 'SDSS-PS1': sampling = sdss_sampling(startmjd=-2. * 365.25) sampling.extend(panstarrs_sampling(1, startmjd=2. * 365.25)) elif options.sampling == 'SDSS': pass else: print "Input to --sampling not understood ..." print "Returning ..." return None #Re-sample each source out = [] savecount, count = 0, 0 for obj in objs: key = os.path.basename(obj) print "Working on " + str(count) + " (" + str(savecount) + "): " + key savecount += 1 v = VarQso(obj) #Find fit try: thisfit = params[key] except KeyError: print "Fit not found, skipping this object ..." nepochs = v.nepochs(band) if nepochs < 20: print "Because #epochs < 20 ..." continue #Set LC model if options.nocolorvar: thisfit['gammagr'] = 0. if 'logAgr' in thisfit.keys(): thisfit['logAgr'] = -7. else: thisfit['logAri'] = -7. if options.sampling == 'SDSS': sampling = v.mjd['g'] sampling = [(s, 'g') for s in sampling] v.setLCmodel(thisfit, band, type) indx = v.mjd_overlap(band=band) refband = 'r' try: o = v.resample([(mjd, refband) for mjd in v.mjd[refband][indx[refband]]], band=refband, errors=False) except nu.linalg.LinAlgError: print thisfit continue xs = [] ys = [] errs = [] for b in band: #Add errors to the underlying xs.extend([(mjd, b) for mjd in v.mjd[b][indx[b]]]) for ii in range(len(v.mjd[b][indx[b]])): ys.append(o.m[refband][ii] + nu.random.randn() * v.err_m[b][indx[b]][ii]) errs.append(v.err_m[b][indx[b]][ii]) out.append([key, VarQso(xs, ys, errs, band=band, medianize=False)]) else: v.setLCmodel(thisfit, band, type) #Resample out.append([key, v.resample(sampling, band=band)]) count += 1 #Save outfile = open(args[0], 'wb') pickle.dump(out, outfile) pickle.dump(band, outfile) outfile.close() return None
def sampleQSO(parser): (options, args) = parser.parse_args() if len(args) == 0: parser.print_help() return if len(args) == 2: othersavefilename = args[1] othersavefile = open(othersavefilename, 'rb') othersamples = pickle.load(othersavefile) othersavefile.close() else: othersamples = {} savefilename = args[0] if os.path.exists(savefilename): savefile = open(savefilename, 'rb') samples = pickle.load(savefile) type = pickle.load(savefile) band = pickle.load(savefile) mean = pickle.load(savefile) savefile.close() else: samples = {} type = options.type mean = options.mean band = options.band if os.path.exists(options.fitsfile): fitsfile = open(options.fitsfile, 'rb') params = pickle.load(fitsfile) fitsfile.close() else: raise IOError( "--fitsfile (or -f) has to be set to the file holding the best-fits" ) if options.star: dir = '../data/star/' elif options.nuvx: dir = '../data/nuvx/' elif options.nuvxall: dir = '../data/nuvx_all/' elif options.uvx: dir = '../data/uvx/' elif options.rrlyrae: dir = '../data/rrlyrae/' else: dir = '../data/s82qsos/' if options.resampled: raise NotImplementedError("resampled not implemented yet") if os.path.exists(options.infile): samplefile = open(options.infile, 'rb') qsos = pickle.load(samplefile) samplefile.close() else: print "'--resampled' is set, but -i filename does not exist ..." print "Returning ..." return None else: qsos = QSOfilenames(dir=dir) #Register time-out handler signal.signal(signal.SIGALRM, handler) savecount = 0 count = len(samples) for qso in qsos: if options.resampled: key = qso[0] else: key = os.path.basename(qso) if samples.has_key(key) or othersamples.has_key(key): continue try: if int(key[5:7]) != options.rah and options.rah != -1: continue except ValueError: if options.rah == -2 or options.rah == -1: pass else: print "Skipping ValueError " + key continue print "Working on " + str(count) + ": " + key if options.resampled: v = qso[1] else: v = VarQso(qso) if v.nepochs(band) < 20: print "This object does not have enough epochs ..." continue #Set best-fit v.LCparams = params[key] v.LC = LCmodel(trainSet=v._build_trainset(band), type=type, mean=mean) v.LCtype = type v.LCmean = mean v.fitband = band #Now sample signal.alarm(options.timeout) try: v.sampleGP(nsamples=options.nsamples, metropolis=options.metropolis, markovpy=options.markovpy, burnin=int(nu.floor(0.2 * options.nsamples))) except Exception, exc: if str(exc) == "Sampling timed out": print exc continue else: raise signal.alarm(0) samples[key] = v.get_sampleGP() if _DEBUG: _print_diagnostics(samples[key]) #print samples[key][options.nsamples] savecount += 1 if savecount == options.saveevery: print "Saving ..." save_pickles(samples, type, band, mean, savefilename) savecount = 0 count += 1
def fitQSO(parser): (options, args) = parser.parse_args() if len(args) == 0: parser.print_help() return if len(args) == 2: othersavefilename = args[1] othersavefile = open(othersavefilename, 'rb') otherparams = pickle.load(othersavefile) othersavefile.close() else: otherparams = {} savefilename = args[0] if os.path.exists(savefilename): savefile = open(savefilename, 'rb') params = pickle.load(savefile) type = pickle.load(savefile) band = pickle.load(savefile) mean = pickle.load(savefile) savefile.close() if params.has_key('.fit'): params.pop('.fit') for key in params.keys(): if (params[key].has_key('gamma') and \ (params[key]['gamma'] < 0. or params[key]['gamma'] > 2.)) \ or (params[key].has_key('gammagr') \ and (params[key]['gammagr'] < 0. \ or params[key]['gammagr'] > 2.)): print "Popping bad gamma ..." params.pop(key) else: params = {} type = options.type mean = options.mean band = options.band if options.star: dir = '../data/star/' elif options.nuvx: dir = '../data/nuvx/' elif options.nuvxall: dir = '../data/nuvx_all/' elif options.uvx: dir = '../data/uvx/' elif options.rrlyrae: dir = '../data/rrlyrae/' else: dir = '../data/s82qsos/' if options.resampled: if os.path.exists(options.infile): samplefile = open(options.infile, 'rb') qsos = pickle.load(samplefile) samplefile.close() else: print "'--resampled' is set, but -i filename does not exist ..." print "Returning ..." return None else: qsos = QSOfilenames(dir=dir) savecount = 0 count = len(params) for qso in qsos: if options.resampled: key = qso[0] else: key = os.path.basename(qso) if params.has_key(key) or otherparams.has_key(key): continue try: if int(key[5:7]) != options.rah and options.rah != -1: continue except ValueError: if options.rah == -2 or options.rah == -1: pass else: print "Skipping ValueError " + key continue print "Working on " + str(count) + ": " + key if options.resampled: v = qso[1] else: v = VarQso(qso, flux=options.fitflux) if v.nepochs(band) < 20: print "This object does not have enough epochs ..." continue params[key] = v.fit(band=band, type=type, loglike=True, mean=mean) if _DEBUG: print params[key] if params[key]['loglike'] == -numpy.finfo(numpy.dtype( numpy.float64)).max: print "Popping bad fit ..." params.pop(key) if savecount == options.saveevery: print "Saving ..." save_pickles(params, type, band, mean, savefilename) savecount = 0 savecount += 1 count += 1 save_pickles(params, type, band, mean, savefilename) print "All done"
def compareMagFluxFits(parser): (options, args) = parser.parse_args() if len(args) == 0: parser.print_help() return params = [] for filename in args: if os.path.exists(filename): savefile = open(filename, 'rb') params.append(pickle.load(savefile)) savefile.close() else: print filename + " does not exist ..." print "Returning ..." return if options.plottype == 'AA': ys = [] xs = [] for key in params[1].keys(): try: ys.append(params[0][key]['logA'] / 2.) xs.append(params[1][key]['logA'] / 2.) except KeyError: continue xs = nu.array(xs).reshape(len(xs)) ys = nu.array(ys).reshape(len(xs)) ys = xs - ys - nu.log(nu.log(10.) / 2.5) xrange = [-9.21 / 2., 0.] yrange = [-.25, .25] xlabel = r'$\log A^{\mathrm{flux}}_' + options.band + r'\ \mathrm{(amplitude\ at\ 1\ yr)}$' ylabel = r'$\log A^{\mathrm{flux}}_' + options.band + r'-\log A^{\mathrm{mag}}_' + options.band + r'- \log\left(\frac{\log 10}{2.5}\right)$' elif options.plottype == 'gg': ys = [] xs = [] for key in params[1].keys(): try: ys.append(params[0][key]['gamma']) xs.append(params[1][key]['gamma']) except KeyError: continue xs = nu.array(xs).reshape(len(xs)) ys = nu.array(ys).reshape(len(xs)) print len(xs) ys = xs - ys xrange = [0., 1.2] yrange = [-.25, .25] xlabel = r'$\gamma^{\mathrm{flux}}_' + options.band + r'\ \mathrm{(power-law\ exponent)}$' ylabel = r'$\gamma^{\mathrm{flux}}_' + options.band + r'- \gamma^{\mathrm{mag}}_' + options.band + '$' elif options.plottype == 'loglike2': ys = [] xs = [] nepochs = [] cnt = 0 for key in params[1].keys(): #cnt+= 1 #print cnt #if cnt > 10: break #Get the number of epochs v = VarQso(os.path.join('../data/s82qsos/', key)) try: ys.append(params[0][key]['loglike'] - v.nepochs(options.band) * nu.log(nu.log(10.) / 2.5)) nepochs.append(v.nepochs(options.band)) xs.append(params[1][key]['loglike']) except KeyError: continue xs = -nu.array(xs).reshape(len(xs)) ys = -nu.array(ys).reshape(len(xs)) nepochs = nu.array(nepochs).reshape(len(nepochs)) ys /= nepochs xs /= nepochs ys = xs - ys xrange = [-3., 0.] yrange = [-.1, .1] xlabel = r'$\log \mathcal{L}^{\mathrm{flux}}_{' + options.band + r',\mathrm{red}}\ \mathrm{(amplitude\ at\ 1\ yr)}$' ylabel = r'$\log \mathcal{L}^{\mathrm{flux}}_{' + options.band + r',\mathrm{red}}- \log \mathcal{L}^{\mathrm{mag}}_{' + options.band + ',\mathrm{red}}' + r'- \log\left(\frac{\log 10}{2.5}\right)$' bovy_plot.bovy_print() bovy_plot.scatterplot(xs, ys, 'k,', onedhists=True, yrange=yrange, xrange=xrange, bins=31, xlabel=xlabel, ylabel=ylabel) bovy_plot.bovy_plot(nu.array(xrange), [0., 0.], '0.5', overplot=True) bovy_plot.bovy_end_print(options.plotfilename) return None
def classQSO(parser): (options, args) = parser.parse_args() if len(args) == 0: parser.print_help() return if os.path.exists(args[0]): print filename + " exists" print "Remove this file before running ..." print "Returning ..." return None #Load fit params: Quasars if options.qsomodel == 'test': if len(options.band) == 1: qsoparams = [{'gamma': 0.3, 'logA': -2.}] else: #Multi-band qsoparams = [{ 'gamma': 0.2, 'logA': -2., 'gammagr': 0.0001, 'logAgr': -2. }] qsoweights = [1.] elif options.qsomodel == 'zero': qsoparams = [{}] qsoweights = [1.] elif os.path.exists(options.qsomodel): qsofile = open(options.qsomodel, 'rb') qsoparams = pickle.load(qsofile) qsoweights = nu.array(pickle.load(qsofile), dtype='float64') qsofile.close() else: print "Input to 'qsomodel' not recognized ..." print "Returning ..." return #Stars if options.starmodel == 'test': if len(options.band) == 1: starparams = [{'gamma': 0.0001, 'logA': -3.5}] else: #Multi-band starparams = [{ 'gamma': 0.1, 'logA': -3., 'gammagr': 0.0001, 'logAgr': -2. }] starweights = [1.] elif options.starmodel == 'zero': starparams = [{}] starweights = [1.] elif os.path.exists(options.starmodel): starfile = open(options.starmodel, 'rb') starparams = pickle.load(starfile) starweights = nu.array(pickle.load(starfile), dtype='float64') starfile.close() else: print "Input to 'starmodel' not recognized ..." print "Returning ..." return #RR Lyrae if options.rrlyraemodel == 'test': if len(options.band) == 1: rrlyraeparams = [{'gamma': 0.0001, 'logA': -2.}] else: #Multi-band rrlyraeparams = [{ 'gamma': 0.1, 'logA': -2., 'gammagr': 0.0001, 'logAgr': -2. }] rrlyraeweights = [1.] elif options.rrlyraemodel == 'zero': rrlyraeparams = [{}] rrlyraeweights = [1.] elif os.path.exists(options.rrlyraemodel): rrlyraefile = open(options.rrlyraemodel, 'rb') rrlyraeparams = pickle.load(rrlyraefile) rrlyraeweights = nu.array(pickle.load(rrlyraefile), dtype='float64') rrlyraefile.close() else: print "Input to 'rrlyraemodel' not recognized ..." print "Returning ..." return #normalize weights qsoweights /= nu.sum(qsoweights) starweights /= nu.sum(starweights) rrlyraeweights /= nu.sum(rrlyraeweights) #Load location of the data if options.resampled: if os.path.exists(options.sample): samplefile = open(options.sample, 'rb') objs = pickle.load(samplefile) samplefile.close() else: print "'--resampled' is set, but --sample= filename does not exist ..." print "Returning ..." return None else: if options.sample == 'nuvx': dir = '../data/nuvx/' if options.sample == 'nuvxall': dir = '../data/nuvx_all/' if options.sample == 'uvx': dir = '../data/uvx/' objs = QSOfilenames(dir=dir) #Classify each source out = [] allcount, count = 0, 0 for obj in objs: allcount += 1 if options.resampled: key = obj[0] else: key = os.path.basename(obj) #if key != 'SDSSJ013306.18-004523.8.fit': # continue print "Working on " + str(count) + "(%i/%i): " % (allcount, len(objs)) + key if options.resampled: v = obj[1] else: v = VarQso(obj) if v.nepochs(options.band) < options.minepochs: print "This object does not have enough epochs ..." continue varout = VarClass() varout.key = key #quasar likelihoods qsolike = [] for ii in range(len(qsoparams)): qsolike.append( v.loglike(band=options.band, type=options.type, params=qsoparams[ii]) + nu.log(qsoweights[ii])) qsolike = logsum(qsolike) varout.qsologlike = qsolike #star likelihoods starlike = [] for ii in range(len(starparams)): starlike.append( v.loglike(band=options.band, type=options.type, params=starparams[ii]) + nu.log(starweights[ii])) starlike = logsum(starlike) varout.starloglike = starlike #RR Lyrae likelihoods rrlyraelike = [] for ii in range(len(rrlyraeparams)): rrlyraelike.append( v.loglike(band=options.band, type=options.type, params=rrlyraeparams[ii]) + nu.log(rrlyraeweights[ii])) rrlyraelike = logsum(rrlyraelike) varout.rrlyraeloglike = rrlyraelike #print qsolike, starlike if qsolike > starlike and qsolike > rrlyraelike: print qsolike, starlike, rrlyraelike out.append(varout) count += 1 #if count > 500: break #Save for jj in range(len(out)): if out[jj].qsologlike > out[jj].starloglike and out[ jj].qsologlike > out[jj].rrlyraeloglike: print out[jj].key saveClass(out, args[0]) return None