def resampleObjs(parser):
(options, args) = parser.parse_args()
if len(args) == 0:
parser.print_help()
return
if os.path.exists(args[0]):
print filename + " exists"
print "Remove this file before running ..."
print "Returning ..."
return None
if options.fitsfile is None:
print "-f or --fitsfile must be set to the file containing the fits ..."
print "Returning ..."
return None
#Load location of the data
if options.sample == 'nuvx':
dir = '../data/nuvx/'
elif options.sample == 'qso':
dir = '../data/s82qsos/'
objs = QSOfilenames(dir=dir)
#Load the fits
if os.path.exists(options.fitsfile):
fitsfile = open(options.fitsfile, 'rb')
params = pickle.load(fitsfile)
type = pickle.load(fitsfile)
band = pickle.load(fitsfile)
fitsfile.close()
else:
print options.fitsfile + " does not exist ..."
print "Returning ..."
return None
#Load sampling
if options.sampling == 'PS1':
sampling = panstarrs_sampling(3, startmjd=2. * 365.25)
elif options.sampling == 'SDSS-PS1':
sampling = sdss_sampling(startmjd=-2. * 365.25)
sampling.extend(panstarrs_sampling(1, startmjd=2. * 365.25))
elif options.sampling == 'SDSS':
pass
else:
print "Input to --sampling not understood ..."
print "Returning ..."
return None
#Re-sample each source
out = []
savecount, count = 0, 0
for obj in objs:
key = os.path.basename(obj)
print "Working on " + str(count) + " (" + str(savecount) + "): " + key
savecount += 1
v = VarQso(obj)
#Find fit
try:
thisfit = params[key]
except KeyError:
print "Fit not found, skipping this object ..."
nepochs = v.nepochs(band)
if nepochs < 20:
print "Because #epochs < 20 ..."
continue
#Set LC model
if options.nocolorvar:
thisfit['gammagr'] = 0.
if 'logAgr' in thisfit.keys():
thisfit['logAgr'] = -7.
else:
thisfit['logAri'] = -7.
if options.sampling == 'SDSS':
sampling = v.mjd['g']
sampling = [(s, 'g') for s in sampling]
v.setLCmodel(thisfit, band, type)
indx = v.mjd_overlap(band=band)
refband = 'r'
try:
o = v.resample([(mjd, refband)
for mjd in v.mjd[refband][indx[refband]]],
band=refband,
errors=False)
except nu.linalg.LinAlgError:
print thisfit
continue
xs = []
ys = []
errs = []
for b in band:
#Add errors to the underlying
xs.extend([(mjd, b) for mjd in v.mjd[b][indx[b]]])
for ii in range(len(v.mjd[b][indx[b]])):
ys.append(o.m[refband][ii] +
nu.random.randn() * v.err_m[b][indx[b]][ii])
errs.append(v.err_m[b][indx[b]][ii])
out.append([key, VarQso(xs, ys, errs, band=band, medianize=False)])
else:
v.setLCmodel(thisfit, band, type)
#Resample
out.append([key, v.resample(sampling, band=band)])
count += 1
#Save
outfile = open(args[0], 'wb')
pickle.dump(out, outfile)
pickle.dump(band, outfile)
outfile.close()
return None
def fitQSO(parser):
(options,args)= parser.parse_args()
if len(args) == 0:
parser.print_help()
return
if len(args) == 2:
othersavefilename= args[1]
othersavefile= open(othersavefilename,'rb')
otherparams= pickle.load(othersavefile)
othersavefile.close()
else:
otherparams= {}
savefilename= args[0]
if os.path.exists(savefilename):
savefile= open(savefilename,'rb')
params= pickle.load(savefile)
type= pickle.load(savefile)
band= pickle.load(savefile)
mean= pickle.load(savefile)
savefile.close()
if params.has_key('.fit'): params.pop('.fit')
for key in params.keys():
if (params[key].has_key('gamma') and \
(params[key]['gamma'] < 0. or params[key]['gamma'] > 2.)) \
or (params[key].has_key('gammagr') \
and (params[key]['gammagr'] < 0. \
or params[key]['gammagr'] > 2.)):
print "Popping bad gamma ..."
params.pop(key)
else:
params= {}
type= options.type
mean= options.mean
band= options.band
if options.star:
dir= '../data/star/'
elif options.nuvx:
dir= '../data/nuvx/'
elif options.nuvxall:
dir= '../data/nuvx_all/'
elif options.uvx:
dir= '../data/uvx/'
elif options.rrlyrae:
dir= '../data/rrlyrae/'
else:
dir= '../data/s82qsos/'
if options.resampled:
if os.path.exists(options.infile):
samplefile= open(options.infile,'rb')
qsos= pickle.load(samplefile)
samplefile.close()
else:
print "'--resampled' is set, but -i filename does not exist ..."
print "Returning ..."
return None
else:
qsos= QSOfilenames(dir=dir)
savecount= 0
count= len(params)
for qso in qsos:
if options.resampled:
key= qso[0]
else:
key= os.path.basename(qso)
if params.has_key(key) or otherparams.has_key(key):
continue
try:
if int(key[5:7]) != options.rah and options.rah != -1:
continue
except ValueError:
if options.rah == -2 or options.rah == -1:
pass
else:
print "Skipping ValueError "+key
continue
print "Working on "+str(count)+": "+key
if options.resampled:
v= qso[1]
else:
v= VarQso(qso,flux=options.fitflux)
if options.lownepochs:
if v.nepochs(band) >= 20:
print "This object has too many epochs ..."
continue
elif v.nepochs(band) < 3:
print "This object does not have enough epochs ..."
continue
elif not options.lownepochs and v.nepochs(band) < 20:
print "This object does not have enough epochs ..."
continue
params[key]= v.fit(band=band,type=type,loglike=True,mean=mean)
if _DEBUG:
print params[key]
if params[key]['loglike'] == -numpy.finfo(numpy.dtype(numpy.float64)).max:
print "Popping bad fit ..."
params.pop(key)
if savecount == options.saveevery:
print "Saving ..."
save_pickles(params,type,band,mean,savefilename)
savecount= 0
savecount+= 1
count+= 1
save_pickles(params,type,band,mean,savefilename)
print "All done"
def classQSO(parser):
(options,args)= parser.parse_args()
if len(args) == 0:
parser.print_help()
return
if os.path.exists(args[0]):
print filename+" exists"
print "Remove this file before running ..."
print "Returning ..."
return None
#Load fit params: Quasars
if options.qsomodel == 'test':
if len(options.band) == 1:
qsoparams= [{'gamma':0.3,'logA':-2.}]
else:#Multi-band
qsoparams= [{'gamma':0.2,'logA':-2.,
'gammagr':0.0001,'logAgr':-2.}]
qsoweights= [1.]
elif options.qsomodel == 'zero':
qsoparams= [{}]
qsoweights= [1.]
elif os.path.exists(options.qsomodel):
qsofile= open(options.qsomodel,'rb')
qsoparams= pickle.load(qsofile)
qsoweights= nu.array(pickle.load(qsofile),dtype='float64')
qsofile.close()
else:
print "Input to 'qsomodel' not recognized ..."
print "Returning ..."
return
#Stars
if options.starmodel == 'test':
if len(options.band) == 1:
starparams= [{'gamma':0.0001,'logA':-3.5}]
else:#Multi-band
starparams= [{'gamma':0.1,'logA':-3.,
'gammagr':0.0001,'logAgr':-2.}]
starweights= [1.]
elif options.starmodel == 'zero':
starparams= [{}]
starweights= [1.]
elif os.path.exists(options.starmodel):
starfile= open(options.starmodel,'rb')
starparams= pickle.load(starfile)
starweights= nu.array(pickle.load(starfile),dtype='float64')
starfile.close()
else:
print "Input to 'starmodel' not recognized ..."
print "Returning ..."
return
#RR Lyrae
if options.rrlyraemodel == 'test':
if len(options.band) == 1:
rrlyraeparams= [{'gamma':0.0001,'logA':-2.}]
else:#Multi-band
rrlyraeparams= [{'gamma':0.1,'logA':-2.,
'gammagr':0.0001,'logAgr':-2.}]
rrlyraeweights= [1.]
elif options.rrlyraemodel == 'zero':
rrlyraeparams= [{}]
rrlyraeweights= [1.]
elif os.path.exists(options.rrlyraemodel):
rrlyraefile= open(options.rrlyraemodel,'rb')
rrlyraeparams= pickle.load(rrlyraefile)
rrlyraeweights= nu.array(pickle.load(rrlyraefile),dtype='float64')
rrlyraefile.close()
else:
print "Input to 'rrlyraemodel' not recognized ..."
print "Returning ..."
return
#normalize weights
qsoweights/= nu.sum(qsoweights)
starweights/= nu.sum(starweights)
rrlyraeweights/= nu.sum(rrlyraeweights)
#Load location of the data
if options.resampled:
if os.path.exists(options.sample):
samplefile= open(options.sample,'rb')
objs= pickle.load(samplefile)
samplefile.close()
else:
print "'--resampled' is set, but --sample= filename does not exist ..."
print "Returning ..."
return None
else:
if options.sample == 'nuvx':
dir= '../data/nuvx/'
if options.sample == 'nuvxall':
dir= '../data/nuvx_all/'
if options.sample == 'uvx':
dir= '../data/uvx/'
objs= QSOfilenames(dir=dir)
#Classify each source
out= []
allcount, count= 0, 0
for obj in objs:
allcount+= 1
if options.resampled:
key= obj[0]
else:
key= os.path.basename(obj)
#if key != 'SDSSJ013306.18-004523.8.fit':
# continue
print "Working on "+str(count)+"(%i/%i): " % (allcount,len(objs))+key
if options.resampled:
v= obj[1]
else:
v= VarQso(obj)
if v.nepochs(options.band) < options.minepochs:
print "This object does not have enough epochs ..."
continue
varout= VarClass()
varout.key= key
#quasar likelihoods
qsolike= []
for ii in range(len(qsoparams)):
qsolike.append(v.loglike(band=options.band,type=options.type,
params=qsoparams[ii])
+nu.log(qsoweights[ii]))
qsolike= logsum(qsolike)
varout.qsologlike= qsolike
#star likelihoods
starlike= []
for ii in range(len(starparams)):
starlike.append(v.loglike(band=options.band,type=options.type,
params=starparams[ii])
+nu.log(starweights[ii]))
starlike= logsum(starlike)
varout.starloglike= starlike
#RR Lyrae likelihoods
rrlyraelike= []
for ii in range(len(rrlyraeparams)):
rrlyraelike.append(v.loglike(band=options.band,type=options.type,
params=rrlyraeparams[ii])
+nu.log(rrlyraeweights[ii]))
rrlyraelike= logsum(rrlyraelike)
varout.rrlyraeloglike= rrlyraelike
#print qsolike, starlike
if qsolike > starlike and qsolike > rrlyraelike:
print qsolike, starlike, rrlyraelike
out.append(varout)
count+= 1
#if count > 500: break
#Save
for jj in range(len(out)):
if out[jj].qsologlike > out[jj].starloglike and out[jj].qsologlike > out[jj].rrlyraeloglike:
print out[jj].key
saveClass(out,args[0])
return None
def compareMagFluxFits(parser):
(options, args) = parser.parse_args()
if len(args) == 0:
parser.print_help()
return
params = []
for filename in args:
if os.path.exists(filename):
savefile = open(filename, 'rb')
params.append(pickle.load(savefile))
savefile.close()
else:
print filename + " does not exist ..."
print "Returning ..."
return
if options.plottype == 'AA':
ys = []
xs = []
for key in params[1].keys():
try:
ys.append(params[0][key]['logA'] / 2.)
xs.append(params[1][key]['logA'] / 2.)
except KeyError:
continue
xs = nu.array(xs).reshape(len(xs))
ys = nu.array(ys).reshape(len(xs))
ys = xs - ys - nu.log(nu.log(10.) / 2.5)
xrange = [-9.21 / 2., 0.]
yrange = [-.25, .25]
xlabel = r'$\log A^{\mathrm{flux}}_' + options.band + r'\ \mathrm{(amplitude\ at\ 1\ yr)}$'
ylabel = r'$\log A^{\mathrm{flux}}_' + options.band + r'-\log A^{\mathrm{mag}}_' + options.band + r'- \log\left(\frac{\log 10}{2.5}\right)$'
elif options.plottype == 'gg':
ys = []
xs = []
for key in params[1].keys():
try:
ys.append(params[0][key]['gamma'])
xs.append(params[1][key]['gamma'])
except KeyError:
continue
xs = nu.array(xs).reshape(len(xs))
ys = nu.array(ys).reshape(len(xs))
print len(xs)
ys = xs - ys
xrange = [0., 1.2]
yrange = [-.25, .25]
xlabel = r'$\gamma^{\mathrm{flux}}_' + options.band + r'\ \mathrm{(power-law\ exponent)}$'
ylabel = r'$\gamma^{\mathrm{flux}}_' + options.band + r'- \gamma^{\mathrm{mag}}_' + options.band + '$'
elif options.plottype == 'loglike2':
ys = []
xs = []
nepochs = []
cnt = 0
for key in params[1].keys():
#cnt+= 1
#print cnt
#if cnt > 10: break
#Get the number of epochs
v = VarQso(os.path.join('../data/s82qsos/', key))
try:
ys.append(params[0][key]['loglike'] -
v.nepochs(options.band) * nu.log(nu.log(10.) / 2.5))
nepochs.append(v.nepochs(options.band))
xs.append(params[1][key]['loglike'])
except KeyError:
continue
xs = -nu.array(xs).reshape(len(xs))
ys = -nu.array(ys).reshape(len(xs))
nepochs = nu.array(nepochs).reshape(len(nepochs))
ys /= nepochs
xs /= nepochs
ys = xs - ys
xrange = [-3., 0.]
yrange = [-.1, .1]
xlabel = r'$\log \mathcal{L}^{\mathrm{flux}}_{' + options.band + r',\mathrm{red}}\ \mathrm{(amplitude\ at\ 1\ yr)}$'
ylabel = r'$\log \mathcal{L}^{\mathrm{flux}}_{' + options.band + r',\mathrm{red}}- \log \mathcal{L}^{\mathrm{mag}}_{' + options.band + ',\mathrm{red}}' + r'- \log\left(\frac{\log 10}{2.5}\right)$'
bovy_plot.bovy_print()
bovy_plot.scatterplot(xs,
ys,
'k,',
onedhists=True,
yrange=yrange,
xrange=xrange,
bins=31,
xlabel=xlabel,
ylabel=ylabel)
bovy_plot.bovy_plot(nu.array(xrange), [0., 0.], '0.5', overplot=True)
bovy_plot.bovy_end_print(options.plotfilename)
return None
def sampleQSO(parser):
(options, args) = parser.parse_args()
if len(args) == 0:
parser.print_help()
return
if len(args) == 2:
othersavefilename = args[1]
othersavefile = open(othersavefilename, 'rb')
othersamples = pickle.load(othersavefile)
othersavefile.close()
else:
othersamples = {}
savefilename = args[0]
if os.path.exists(savefilename):
savefile = open(savefilename, 'rb')
samples = pickle.load(savefile)
type = pickle.load(savefile)
band = pickle.load(savefile)
mean = pickle.load(savefile)
savefile.close()
else:
samples = {}
type = options.type
mean = options.mean
band = options.band
if os.path.exists(options.fitsfile):
fitsfile = open(options.fitsfile, 'rb')
params = pickle.load(fitsfile)
fitsfile.close()
else:
raise IOError(
"--fitsfile (or -f) has to be set to the file holding the best-fits"
)
if options.star:
dir = '../data/star/'
elif options.nuvx:
dir = '../data/nuvx/'
elif options.nuvxall:
dir = '../data/nuvx_all/'
elif options.uvx:
dir = '../data/uvx/'
elif options.rrlyrae:
dir = '../data/rrlyrae/'
else:
dir = '../data/s82qsos/'
if options.resampled:
raise NotImplementedError("resampled not implemented yet")
if os.path.exists(options.infile):
samplefile = open(options.infile, 'rb')
qsos = pickle.load(samplefile)
samplefile.close()
else:
print "'--resampled' is set, but -i filename does not exist ..."
print "Returning ..."
return None
else:
qsos = QSOfilenames(dir=dir)
#Register time-out handler
signal.signal(signal.SIGALRM, handler)
savecount = 0
count = len(samples)
for qso in qsos:
if options.resampled:
key = qso[0]
else:
key = os.path.basename(qso)
if samples.has_key(key) or othersamples.has_key(key):
continue
try:
if int(key[5:7]) != options.rah and options.rah != -1:
continue
except ValueError:
if options.rah == -2 or options.rah == -1:
pass
else:
print "Skipping ValueError " + key
continue
print "Working on " + str(count) + ": " + key
if options.resampled:
v = qso[1]
else:
v = VarQso(qso)
if v.nepochs(band) < 20:
print "This object does not have enough epochs ..."
continue
#Set best-fit
v.LCparams = params[key]
v.LC = LCmodel(trainSet=v._build_trainset(band), type=type, mean=mean)
v.LCtype = type
v.LCmean = mean
v.fitband = band
#Now sample
signal.alarm(options.timeout)
try:
v.sampleGP(nsamples=options.nsamples,
metropolis=options.metropolis,
markovpy=options.markovpy,
burnin=int(nu.floor(0.2 * options.nsamples)))
except Exception, exc:
if str(exc) == "Sampling timed out":
print exc
continue
else:
raise
signal.alarm(0)
samples[key] = v.get_sampleGP()
if _DEBUG:
_print_diagnostics(samples[key])
#print samples[key][options.nsamples]
savecount += 1
if savecount == options.saveevery:
print "Saving ..."
save_pickles(samples, type, band, mean, savefilename)
savecount = 0
count += 1
def resampleObjs(parser):
(options,args)= parser.parse_args()
if len(args) == 0:
parser.print_help()
return
if os.path.exists(args[0]):
print filename+" exists"
print "Remove this file before running ..."
print "Returning ..."
return None
if options.fitsfile is None:
print "-f or --fitsfile must be set to the file containing the fits ..."
print "Returning ..."
return None
#Load location of the data
if options.sample == 'nuvx':
dir= '../data/nuvx/'
elif options.sample == 'qso':
dir= '../data/s82qsos/'
objs= QSOfilenames(dir=dir)
#Load the fits
if os.path.exists(options.fitsfile):
fitsfile= open(options.fitsfile,'rb')
params= pickle.load(fitsfile)
type= pickle.load(fitsfile)
band= pickle.load(fitsfile)
fitsfile.close()
else:
print options.fitsfile+" does not exist ..."
print "Returning ..."
return None
#Load sampling
if options.sampling == 'PS1':
sampling= panstarrs_sampling(3,startmjd=2.*365.25)
elif options.sampling == 'SDSS-PS1':
sampling= sdss_sampling(startmjd=-2.*365.25)
sampling.extend(panstarrs_sampling(1,startmjd=2.*365.25))
elif options.sampling == 'SDSS': pass
else:
print "Input to --sampling not understood ..."
print "Returning ..."
return None
#Re-sample each source
out= []
savecount, count= 0, 0
for obj in objs:
key= os.path.basename(obj)
print "Working on "+str(count)+" ("+str(savecount)+"): "+key
savecount+= 1
v= VarQso(obj)
#Find fit
try:
thisfit= params[key]
except KeyError:
print "Fit not found, skipping this object ..."
nepochs= v.nepochs(band)
if nepochs < 20:
print "Because #epochs < 20 ..."
continue
#Set LC model
if options.nocolorvar:
thisfit['gammagr']= 0.
if 'logAgr' in thisfit.keys():
thisfit['logAgr']= -7.
else:
thisfit['logAri']= -7.
if options.sampling == 'SDSS':
sampling= v.mjd['g']
sampling= [(s,'g') for s in sampling]
v.setLCmodel(thisfit,band,type)
indx= v.mjd_overlap(band=band)
refband= 'r'
try:
o= v.resample([(mjd,refband) for mjd in v.mjd[refband][indx[refband]]],
band=refband,errors=False)
except nu.linalg.LinAlgError:
print thisfit
continue
xs= []
ys= []
errs= []
for b in band:
#Add errors to the underlying
xs.extend([(mjd,b) for mjd in v.mjd[b][indx[b]]])
for ii in range(len(v.mjd[b][indx[b]])):
ys.append(o.m[refband][ii]+
nu.random.randn()*v.err_m[b][indx[b]][ii])
errs.append(v.err_m[b][indx[b]][ii])
out.append([key,VarQso(xs,ys,errs,band=band,medianize=False)])
else:
v.setLCmodel(thisfit,band,type)
#Resample
out.append([key,v.resample(sampling,band=band)])
count+= 1
#Save
outfile= open(args[0],'wb')
pickle.dump(out,outfile)
pickle.dump(band,outfile)
outfile.close()
return None
def skewQSO(parser):
(options,args)= parser.parse_args()
if len(args) == 0:
parser.print_help()
return
savefilename= args[0]
if os.path.exists(savefilename):
savefile= open(savefilename,'rb')
skews= pickle.load(savefile)
gaussskews= pickle.load(savefile)
fittype= pickle.load(savefile)
band= pickle.load(savefile)
mean= pickle.load(savefile)
taus= pickle.load(savefile)
savefile.close()
else:
skews= {}
gaussskews= {}
fittype= options.type
mean= options.mean
band= options.band
taus= nu.arange(options.dtau,options.taumax,options.dtau)/365.25
if os.path.exists(options.fitsfile):
fitsfile= open(options.fitsfile,'rb')
params= pickle.load(fitsfile)
fitsfile.close()
else:
raise IOError("--fitsfile (or -f) has to be set to the file holding the best-fits")
if options.star:
dir= '../data/star/'
elif options.nuvx:
dir= '../data/nuvx/'
elif options.nuvxall:
dir= '../data/nuvx_all/'
elif options.uvx:
dir= '../data/uvx/'
elif options.rrlyrae:
dir= '../data/rrlyrae/'
else:
dir= '../data/s82qsos/'
qsos= QSOfilenames(dir=dir)
if not options.split is None:
splitarr= nu.arange(len(qsos)) / int(nu.ceil(len(qsos)/float(options.split)))
splitDict= {}
for ii, qso in enumerate(qsos):
key= os.path.basename(qso)
splitDict[key]= splitarr[ii]
print "Running bin %i ..." % (options.rah-1)
savecount= 0
count= len(skews)
if not options.star and not options.rrlyrae:
#Read master file for redshifts
dataqsos= open_qsos()
qsoDict= {}
ii=0
for qso in dataqsos:
qsoDict[qso.oname.strip().replace(' ', '')+'.fit']= ii
ii+= 1
for qso in qsos:
key= os.path.basename(qso)
if skews.has_key(key):
continue
if not options.split is None:
if splitDict[key] != (options.rah-1):
continue
else:
try:
if int(key[5:7]) != options.rah and options.rah != -1:
continue
except ValueError:
if options.rah == -2 or options.rah == -1:
pass
else:
print "Skipping ValueError "+key
continue
sys.stdout.write('\r'+_ERASESTR+'\r')
sys.stdout.flush()
sys.stdout.write('\rWorking on %s: %s\r' % (str(count),key))
sys.stdout.flush()
v= VarQso(qso,flux=options.flux)
if v.nepochs(band) < 20:
#print "This object does not have enough epochs ..."
continue
#Set best-fit
if options.flux:
params[key]['logA']+= 2.*nu.log(nu.log(10.)/2.5)
if options.mean == 'const':
params[key]['m']= -nu.log(10.)/2.5*params[key]['m']
v.LCparams= params[key]
v.LC= LCmodel(trainSet=v._build_trainset(band),type=fittype,mean=mean,
init_params=params[key])
v.LCtype= fittype
v.LCmean= mean
v.fitband= band
#Now compute skew and Gaussian samples
try:
thisskew= v.skew(taus,band)
except RuntimeError:
continue
if options.flux:
thisskew*= -1.
thisgaussskews= nu.zeros((options.nsamples,len(taus)))
for ii in range(options.nsamples):
#First re-sample
if options.star or options.rrlyrae:
redshift= 0.
else:
redshift= dataqsos[qsoDict[key]].z
try:
o= v.resample(v.mjd[band],band=band,noconstraints=True,
wedge=options.wedge,
wedgerate=options.wedgerate*365.25/(1.+redshift),
wedgetau=(1.+redshift)) #1yr
except nu.linalg.linalg.LinAlgError:
if params[key]['gamma'] > 1.5 \
or params[key]['logA'] < -10.:
continue #re-sampling fails because of bad gamma/logA
else:
print key, params[key]
o.LCparams= v.LCparams
o.LC= v.LC
o.fitband= v.fitband
o.LCtype= v.LCtype
o.LCmean= v.LCmean
if options.wedge:
o.LCparams['gamma']= 1.
o.LCparams['logA']= o.LCparams['logA']\
+nu.log(0.025**v.LCparams['gamma']/0.025)
o.LCmean= 'zero' #bc we remove the mean when resampling wedge
#Set up LC with correct params
o.LC= LCmodel(trainSet=o._build_trainset(band),
type=o.LCtype,mean=o.LCmean,
init_params=o.LCparams)
thisgaussskews[ii,:]= o.skew(taus,band)
if options.flux:
thisgaussskews[ii,:]*= -1.
skews[key]= thisskew
gaussskews[key]= thisgaussskews
savecount+= 1
if savecount == options.saveevery:
sys.stdout.write('\r'+_ERASESTR+'\r')
sys.stdout.flush()
sys.stdout.write('\rSaving ...\r')
sys.stdout.flush()
save_pickles(savefilename,skews,gaussskews,fittype,band,mean,taus)
savecount= 0
count+= 1
sys.stdout.write('\r'+_ERASESTR+'\r')
sys.stdout.flush()
save_pickles(savefilename,skews,gaussskews,fittype,band,mean,taus)
print "All done"
def sampleQSO(parser):
(options,args)= parser.parse_args()
if len(args) == 0:
parser.print_help()
return
if len(args) == 2:
othersavefilename= args[1]
othersavefile= open(othersavefilename,'rb')
othersamples= pickle.load(othersavefile)
othersavefile.close()
else:
othersamples= {}
savefilename= args[0]
if os.path.exists(savefilename):
savefile= open(savefilename,'rb')
samples= pickle.load(savefile)
type= pickle.load(savefile)
band= pickle.load(savefile)
mean= pickle.load(savefile)
savefile.close()
else:
samples= {}
type= options.type
mean= options.mean
band= options.band
if os.path.exists(options.fitsfile):
fitsfile= open(options.fitsfile,'rb')
params= pickle.load(fitsfile)
fitsfile.close()
else:
raise IOError("--fitsfile (or -f) has to be set to the file holding the best-fits")
if options.star:
dir= '../data/star/'
elif options.nuvx:
dir= '../data/nuvx/'
elif options.nuvxall:
dir= '../data/nuvx_all/'
elif options.uvx:
dir= '../data/uvx/'
elif options.rrlyrae:
dir= '../data/rrlyrae/'
else:
dir= '../data/s82qsos/'
if options.resampled:
raise NotImplementedError("resampled not implemented yet")
if os.path.exists(options.infile):
samplefile= open(options.infile,'rb')
qsos= pickle.load(samplefile)
samplefile.close()
else:
print "'--resampled' is set, but -i filename does not exist ..."
print "Returning ..."
return None
else:
qsos= QSOfilenames(dir=dir)
#Register time-out handler
signal.signal(signal.SIGALRM, handler)
savecount= 0
count= len(samples)
for qso in qsos:
if options.resampled:
key= qso[0]
else:
key= os.path.basename(qso)
if samples.has_key(key) or othersamples.has_key(key):
continue
try:
if int(key[5:7]) != options.rah and options.rah != -1:
continue
except ValueError:
if options.rah == -2 or options.rah == -1:
pass
else:
print "Skipping ValueError "+key
continue
print "Working on "+str(count)+": "+key
if options.resampled:
v= qso[1]
else:
v= VarQso(qso)
if options.lownepochs:
if v.nepochs(band) >= 20:
print "This object has too many epochs ..."
continue
elif v.nepochs(band) < 3:
print "This object has < 3 epochs ..."
continue
elif not options.lownepochs and v.nepochs(band) < 20:
print "This object does not have enough epochs ..."
continue
#Set best-fit
v.LCparams= params[key]
v.LC= LCmodel(trainSet=v._build_trainset(band),type=type,mean=mean,
init_params=params[key])
v.LCtype= type
v.LCmean= mean
v.fitband= band
#Now sample
signal.alarm(options.timeout)
try:
v.sampleGP(nsamples=options.nsamples,metropolis=options.metropolis,
markovpy=options.markovpy,
burnin=int(nu.floor(0.2*options.nsamples)))
except Exception, exc:
if str(exc) == "Sampling timed out":
print exc
continue
else:
raise
signal.alarm(0)
samples[key]= v.get_sampleGP()
if _DEBUG:
_print_diagnostics(samples[key])
#print samples[key][options.nsamples]
savecount+= 1
if savecount == options.saveevery:
print "Saving ..."
save_pickles(samples,type,band,mean,savefilename)
savecount= 0
count+= 1
def fitQSO(parser):
(options, args) = parser.parse_args()
if len(args) == 0:
parser.print_help()
return
if len(args) == 2:
othersavefilename = args[1]
othersavefile = open(othersavefilename, 'rb')
otherparams = pickle.load(othersavefile)
othersavefile.close()
else:
otherparams = {}
savefilename = args[0]
if os.path.exists(savefilename):
savefile = open(savefilename, 'rb')
params = pickle.load(savefile)
type = pickle.load(savefile)
band = pickle.load(savefile)
mean = pickle.load(savefile)
savefile.close()
if params.has_key('.fit'): params.pop('.fit')
for key in params.keys():
if (params[key].has_key('gamma') and \
(params[key]['gamma'] < 0. or params[key]['gamma'] > 2.)) \
or (params[key].has_key('gammagr') \
and (params[key]['gammagr'] < 0. \
or params[key]['gammagr'] > 2.)):
print "Popping bad gamma ..."
params.pop(key)
else:
params = {}
type = options.type
mean = options.mean
band = options.band
if options.star:
dir = '../data/star/'
elif options.nuvx:
dir = '../data/nuvx/'
elif options.nuvxall:
dir = '../data/nuvx_all/'
elif options.uvx:
dir = '../data/uvx/'
elif options.rrlyrae:
dir = '../data/rrlyrae/'
else:
dir = '../data/s82qsos/'
if options.resampled:
if os.path.exists(options.infile):
samplefile = open(options.infile, 'rb')
qsos = pickle.load(samplefile)
samplefile.close()
else:
print "'--resampled' is set, but -i filename does not exist ..."
print "Returning ..."
return None
else:
qsos = QSOfilenames(dir=dir)
savecount = 0
count = len(params)
for qso in qsos:
if options.resampled:
key = qso[0]
else:
key = os.path.basename(qso)
if params.has_key(key) or otherparams.has_key(key):
continue
try:
if int(key[5:7]) != options.rah and options.rah != -1:
continue
except ValueError:
if options.rah == -2 or options.rah == -1:
pass
else:
print "Skipping ValueError " + key
continue
print "Working on " + str(count) + ": " + key
if options.resampled:
v = qso[1]
else:
v = VarQso(qso, flux=options.fitflux)
if v.nepochs(band) < 20:
print "This object does not have enough epochs ..."
continue
params[key] = v.fit(band=band, type=type, loglike=True, mean=mean)
if _DEBUG:
print params[key]
if params[key]['loglike'] == -numpy.finfo(numpy.dtype(
numpy.float64)).max:
print "Popping bad fit ..."
params.pop(key)
if savecount == options.saveevery:
print "Saving ..."
save_pickles(params, type, band, mean, savefilename)
savecount = 0
savecount += 1
count += 1
save_pickles(params, type, band, mean, savefilename)
print "All done"
def skew0957(parser):
(options,args)= parser.parse_args()
if len(args) == 0:
parser.print_help()
return
savefilename= args[0]
#Read data
if options.vanderriest:
vA= VarQso('../data/0957-A.fits',band=options.band)
vB= VarQso('../data/0957-B.fits',band=options.band)
else:
vA= VarQso('../data/L0957-A_%s.fits' % options.band,band=options.band)
vB= VarQso('../data/L0957-B_%s.fits' % options.band,band=options.band)
if not options.fitsfile is None and os.path.exists(options.fitsfile):
fitsfile= open(options.fitsfile,'rb')
paramsA= pickle.load(fitsfile)
paramsB= pickle.load(fitsfile)
paramsAB= pickle.load(fitsfile)
fitsfile.close()
vA.LCparams= paramsA
vB.LCparams= paramsB
vA.LCtype='powerlawSF'
vB.LCtype='powerlawSF'
vA.LCmean= 'const'
vB.LCmean= 'const'
vA.fitband= options.band
vB.fitband= options.band
else:
#Fit for means
print "Fitting SF for both images ..."
vA.fit(options.band,mean='const')
vB.fit(options.band,mean='const')
#Load into single new VarQso
newm= list(vA.m[options.band]-vA.LCparams['m'])
newerrm= list(vA.err_m[options.band])
newmjd= list(vA.mjd[options.band])
newm.extend(list(vB.m[options.band]+nu.mean(vA.m[options.band])
-nu.mean(vB.m[options.band])-vB.LCparams['m']))
newerrm.extend(list(vB.err_m[options.band]))
newmjd.extend(list(vB.mjd[options.band]-417./365.25))#shift lagged B
v= VarQso(newmjd,newm,newerrm,band=options.band,medianize=False)
if not options.fitsfile is None and os.path.exists(options.fitsfile):
v.LCparams= paramsAB
v.LCtype='powerlawSF'
v.LCmean= 'zero'
v.fitband= options.band
v.LC= LCmodel(trainSet=v._build_trainset(options.band),
type=v.LCtype,mean=v.LCmean,
init_params=paramsAB)
else:
v.fit(options.band)
if not options.fitsfile is None and not os.path.exists(options.fitsfile):
save_pickles(options.fitsfile,vA.LCparams,vB.LCparams,v.LCparams)
taus= nu.arange(1.,201.,1.)/365.25
thisskew= v.skew(taus,options.band,duration=0.7)
thisgaussskews= nu.zeros((options.nsamples,len(taus)))
print "Calculating simulated skews ..."
for ii in range(options.nsamples):
#First re-sample
redshift= 1.41
o= v.resample(v.mjd[options.band],band=options.band,noconstraints=True,
wedge=options.wedge,
wedgerate=options.wedgerate*365.25/(1.+redshift),
wedgetau=(1.+redshift)) #1yr
o.LCparams= v.LCparams
o.LC= v.LC
o.fitband= v.fitband
o.LCtype= v.LCtype
o.LCmean= v.LCmean
if options.wedge:
o.LCparams['gamma']= 1.
o.LCparams['logA']= o.LCparams['logA']\
+nu.log(0.05**v.LCparams['gamma']/0.05)
o.LCmean= 'zero' #bc we remove the mean when resampling wedge
#Set up LC with correct params
o.LC= LCmodel(trainSet=o._build_trainset(options.band),
type=o.LCtype,mean=o.LCmean,
init_params=o.LCparams)
thisgaussskews[ii,:]= o.skew(taus,options.band,duration=0.7)
skews= {}
gaussskews= {}
skews['0957']= thisskew
gaussskews['0957']= thisgaussskews
save_pickles(savefilename,skews,gaussskews,
None,options.band,None,taus)
return None
def classQSO(parser):
(options, args) = parser.parse_args()
if len(args) == 0:
parser.print_help()
return
if os.path.exists(args[0]):
print filename + " exists"
print "Remove this file before running ..."
print "Returning ..."
return None
#Load fit params: Quasars
if options.qsomodel == 'test':
if len(options.band) == 1:
qsoparams = [{'gamma': 0.3, 'logA': -2.}]
else: #Multi-band
qsoparams = [{
'gamma': 0.2,
'logA': -2.,
'gammagr': 0.0001,
'logAgr': -2.
}]
qsoweights = [1.]
elif options.qsomodel == 'zero':
qsoparams = [{}]
qsoweights = [1.]
elif os.path.exists(options.qsomodel):
qsofile = open(options.qsomodel, 'rb')
qsoparams = pickle.load(qsofile)
qsoweights = nu.array(pickle.load(qsofile), dtype='float64')
qsofile.close()
else:
print "Input to 'qsomodel' not recognized ..."
print "Returning ..."
return
#Stars
if options.starmodel == 'test':
if len(options.band) == 1:
starparams = [{'gamma': 0.0001, 'logA': -3.5}]
else: #Multi-band
starparams = [{
'gamma': 0.1,
'logA': -3.,
'gammagr': 0.0001,
'logAgr': -2.
}]
starweights = [1.]
elif options.starmodel == 'zero':
starparams = [{}]
starweights = [1.]
elif os.path.exists(options.starmodel):
starfile = open(options.starmodel, 'rb')
starparams = pickle.load(starfile)
starweights = nu.array(pickle.load(starfile), dtype='float64')
starfile.close()
else:
print "Input to 'starmodel' not recognized ..."
print "Returning ..."
return
#RR Lyrae
if options.rrlyraemodel == 'test':
if len(options.band) == 1:
rrlyraeparams = [{'gamma': 0.0001, 'logA': -2.}]
else: #Multi-band
rrlyraeparams = [{
'gamma': 0.1,
'logA': -2.,
'gammagr': 0.0001,
'logAgr': -2.
}]
rrlyraeweights = [1.]
elif options.rrlyraemodel == 'zero':
rrlyraeparams = [{}]
rrlyraeweights = [1.]
elif os.path.exists(options.rrlyraemodel):
rrlyraefile = open(options.rrlyraemodel, 'rb')
rrlyraeparams = pickle.load(rrlyraefile)
rrlyraeweights = nu.array(pickle.load(rrlyraefile), dtype='float64')
rrlyraefile.close()
else:
print "Input to 'rrlyraemodel' not recognized ..."
print "Returning ..."
return
#normalize weights
qsoweights /= nu.sum(qsoweights)
starweights /= nu.sum(starweights)
rrlyraeweights /= nu.sum(rrlyraeweights)
#Load location of the data
if options.resampled:
if os.path.exists(options.sample):
samplefile = open(options.sample, 'rb')
objs = pickle.load(samplefile)
samplefile.close()
else:
print "'--resampled' is set, but --sample= filename does not exist ..."
print "Returning ..."
return None
else:
if options.sample == 'nuvx':
dir = '../data/nuvx/'
if options.sample == 'nuvxall':
dir = '../data/nuvx_all/'
if options.sample == 'uvx':
dir = '../data/uvx/'
objs = QSOfilenames(dir=dir)
#Classify each source
out = []
allcount, count = 0, 0
for obj in objs:
allcount += 1
if options.resampled:
key = obj[0]
else:
key = os.path.basename(obj)
#if key != 'SDSSJ013306.18-004523.8.fit':
# continue
print "Working on " + str(count) + "(%i/%i): " % (allcount,
len(objs)) + key
if options.resampled:
v = obj[1]
else:
v = VarQso(obj)
if v.nepochs(options.band) < options.minepochs:
print "This object does not have enough epochs ..."
continue
varout = VarClass()
varout.key = key
#quasar likelihoods
qsolike = []
for ii in range(len(qsoparams)):
qsolike.append(
v.loglike(band=options.band,
type=options.type,
params=qsoparams[ii]) + nu.log(qsoweights[ii]))
qsolike = logsum(qsolike)
varout.qsologlike = qsolike
#star likelihoods
starlike = []
for ii in range(len(starparams)):
starlike.append(
v.loglike(band=options.band,
type=options.type,
params=starparams[ii]) + nu.log(starweights[ii]))
starlike = logsum(starlike)
varout.starloglike = starlike
#RR Lyrae likelihoods
rrlyraelike = []
for ii in range(len(rrlyraeparams)):
rrlyraelike.append(
v.loglike(band=options.band,
type=options.type,
params=rrlyraeparams[ii]) +
nu.log(rrlyraeweights[ii]))
rrlyraelike = logsum(rrlyraelike)
varout.rrlyraeloglike = rrlyraelike
#print qsolike, starlike
if qsolike > starlike and qsolike > rrlyraelike:
print qsolike, starlike, rrlyraelike
out.append(varout)
count += 1
#if count > 500: break
#Save
for jj in range(len(out)):
if out[jj].qsologlike > out[jj].starloglike and out[
jj].qsologlike > out[jj].rrlyraeloglike:
print out[jj].key
saveClass(out, args[0])
return None
def compareMagFluxFits(parser):
(options,args)= parser.parse_args()
if len(args) == 0:
parser.print_help()
return
params= []
for filename in args:
if os.path.exists(filename):
savefile= open(filename,'rb')
params.append(pickle.load(savefile))
savefile.close()
else:
print filename+" does not exist ..."
print "Returning ..."
return
if options.plottype == 'AA':
ys= []
xs= []
for key in params[1].keys():
try:
ys.append(params[0][key]['logA']/2.)
xs.append(params[1][key]['logA']/2.)
except KeyError:
continue
xs= nu.array(xs).reshape(len(xs))
ys= nu.array(ys).reshape(len(xs))
ys= xs-ys-nu.log(nu.log(10.)/2.5)
xrange=[-9.21/2.,0.]
yrange=[-.25,.25]
xlabel= r'$\log A^{\mathrm{flux}}_'+options.band+r'\ \mathrm{(amplitude\ at\ 1\ yr)}$'
ylabel= r'$\log A^{\mathrm{flux}}_'+options.band+r'-\log A^{\mathrm{mag}}_'+options.band+r'- \log\left(\frac{\log 10}{2.5}\right)$'
elif options.plottype == 'gg':
ys= []
xs= []
for key in params[1].keys():
try:
ys.append(params[0][key]['gamma'])
xs.append(params[1][key]['gamma'])
except KeyError:
continue
xs= nu.array(xs).reshape(len(xs))
ys= nu.array(ys).reshape(len(xs))
print len(xs)
ys= xs-ys
xrange=[0.,1.2]
yrange=[-.25,.25]
xlabel= r'$\gamma^{\mathrm{flux}}_'+options.band+r'\ \mathrm{(power-law\ exponent)}$'
ylabel= r'$\gamma^{\mathrm{flux}}_'+options.band+r'- \gamma^{\mathrm{mag}}_'+options.band+'$'
elif options.plottype == 'loglike2':
ys= []
xs= []
nepochs= []
cnt= 0
for key in params[1].keys():
#cnt+= 1
#print cnt
#if cnt > 10: break
#Get the number of epochs
v= VarQso(os.path.join('../data/s82qsos/',key))
try:
ys.append(params[0][key]['loglike']
-v.nepochs(options.band)*nu.log(nu.log(10.)/2.5))
nepochs.append(v.nepochs(options.band))
xs.append(params[1][key]['loglike'])
except KeyError:
continue
xs= -nu.array(xs).reshape(len(xs))
ys= -nu.array(ys).reshape(len(xs))
nepochs= nu.array(nepochs).reshape(len(nepochs))
ys/= nepochs
xs/= nepochs
ys= xs-ys
xrange=[-3.,0.]
yrange=[-.1,.1]
xlabel= r'$\log \mathcal{L}^{\mathrm{flux}}_{'+options.band+r',\mathrm{red}}\ \mathrm{(amplitude\ at\ 1\ yr)}$'
ylabel= r'$\log \mathcal{L}^{\mathrm{flux}}_{'+options.band+r',\mathrm{red}}- \log \mathcal{L}^{\mathrm{mag}}_{'+options.band+',\mathrm{red}}'+r'- \log\left(\frac{\log 10}{2.5}\right)$'
bovy_plot.bovy_print()
bovy_plot.scatterplot(xs,ys,'k,',onedhists=True,
yrange=yrange,
xrange=xrange,bins=31,
xlabel=xlabel,
ylabel=ylabel)
bovy_plot.bovy_plot(nu.array(xrange),[0.,0.],'0.5',
overplot=True)
bovy_plot.bovy_end_print(options.plotfilename)
return None
