def update(algorithm,spec1,spec2): """ Checks database for consecutive occurrences of the same beam and flags them in rfi_found. Inputs: algorithm: an integer between 0 and 7, inclusive spec1: the first spectrum to be checked spec2: the last spectrum to be checked """ import MySQLFunction, numpy, sys #Compute RFI value num = 2**algorithm #Create query and grab beamnums cmd = 'select beamnum from config where specid>=%s and specid<=%s;'%(spec1,spec2) data = MySQLFunction.mysqlcommand(cmd) data = [y[0] for y in data] length = len(data) #Create list of values to loop through vals = list(numpy.arange(1,length,10000)) vals.append(length+1) #Loop through data, checking and updating for x in range(len(vals)-1): #Create shorter list of data to work with tempdata = data[vals[x]-1:vals[x+1]-1] #Grab indices where next two beamnums are equivalent indices = [a for a in range(len(tempdata)-2) if (tempdata[a+2]==tempdata[a] and tempdata[a+1]==tempdata[a])] #Manually flag two spectra after final flagged spectrum if len(indices)>0: final = max(indices) indices.append(final+1) indices.append(final+2) #Compute offset for this particular interval indices = [b+(vals[x]-1)+spec1 for b in indices] #Create where string where_string = ', '.join([str(c) for c in indices]) #Create queries and update db if len(indices)>0: query1= 'update hit set rfi_found=rfi_found+%s where specid in (%s);'%(num,where_string) MySQLFunction.mysqlcommand(query1) query2= 'update hit set rfi_checked=rfi_checked+%s where specid>=%s and specid<%s;'%(num,vals[x]+spec1-1,vals[x+1]+spec1-1) MySQLFunction.mysqlcommand(query2) print 'done with interval %s to %s'%(vals[x]+spec1-1,vals[x+1]+spec1-2) return
def check(spec_init,spec_final,filename,algorithm):
"""Loads a text file of limits for RFI bands and checks
hit table, marking those hits where the frequency
lies inside an RFI band. Spec_init is the first specid
in the range to be checked, and spec_final is the
final specid in the range. Filename is the name of the file
with the RFI band bounds, with one tab separated upper
and lower bound per line. algorithm denotes which rfi check
is being performed, and it is an integer from 0 to 7,
inclusive. """
import numpy, MySQLFunction
#Load file
rfi = numpy.loadtxt(filename)
#Determine value to add to rfi_check and rfi_found
num = 2**algorithm
#Create list of intervals
vals = list(numpy.arange(spec_init,spec_final,10000))
vals.append(spec_final+1)
#Loop through rfi bands and intervals, updating db
for x in range(len(rfi)):
bounds = list(rfi[x])
print 'checking rfi band %s of %s' %(x+1,len(rfi))
for y in range(len(vals)-1):
print 'checking specid range %s to %s' %(vals[y],vals[y+1])
#Create query
query = 'select hitid from hit where specid>=%s and specid<%s and topocentric_freq>%s and topocentric_freq<%s;' %(vals[y],vals[y+1],bounds[0],bounds[1])
print query
#Get hitids where rfi_found
data = MySQLFunction.mysqlcommand(query)
data = [str(x[0]) for x in data]
#Update rfi_found
if len(data) > 0:
where_string = ', '.join([z for z in data])
query = "UPDATE hit SET rfi_found = +%s WHERE hitid in (%s)" %(num,where_string)
MySQLFunction.mysqlcommand(query)
print 'rfi_found updated'
else:
print 'no rfi found'
#Update rfi_checked
query = 'UPDATE hit SET rfi_checked = +%s WHERE specid>=%s and specid<%s;' %(num,vals[y],vals[y+1])
MySQLFunction.mysqlcommand(query)
print 'rfi_checked updated'
return
def fetchdata(where='', savedata=''):
"""Retrieves mysql data for RA and DEC.
where is a string specifying which data from mysql to grab.
ONLY include columns from table 'config'. do not include
the word 'where' at the beginning or the semicolon at the
end. all other common mysql syntax rules apply.
savedata allows the user to have the data saved by
inputting the file name. data is saved as an ASCII file. if
left empty, no file will be saved
returns three arrays, ra, dec, and specid"""
#Create command to send to mysql
if not where:
cmd = command.generate('ra,decl,specid','config')
else: cmd = command.generate('ra,decl,specid','config',where=where)
#Send command to mysql, return data
data = MySQLFunction.mysqlcommand(cmd)
#Separate data into two arrays
length = len(data)
ra = numpy.asarray([data[x][0] for x in range(length)])
dec = numpy.asarray([data[x][1] for x in range(length)])
specid = numpy.asarray([data[x][2] for x in range(length)])
#Save data?
if savedata!='':
numpy.savetxt('%s' %savedata,(ra,dec,specid))
return(ra,dec,specid)
def fetchdata(where='',freqtype='topo',savedata=''):
"""Fetches data to produce the dynamic spectrum plot for hits.
where is a string to include additional information to
narrow the results. typically it will be used to specify a
range of specids. each column name MUST be prefixed with
the first letter of the table name and a period, like
c.obstime. don't forget to include 'h.specid=c.specid if
referencing config and hit. do not include the word 'where'
at the beginning or the semicolon at the end. all other common
mysql syntax rules apply. Ex: 'h.specid>1 and h.specid<=20 and
c.beamnum!=8 and c.specid=h.specid'.
freqtype is the frequency type for the y-axis, either
'binnum', 'topo', or 'bary'.
returns three arrays: eventpower,frequency (in MHz),time (in
seconds since first spectra in data set)"""
import MySQLFunction, numpy, command
# create full column name from freq_type
if freqtype=='topo':
freqtype = 'h.topocentric_freq'
elif freqtype=='bary':
freqtype = 'h.barycentric_freq'
elif freqtype=='binnum':
freqtype = 'h.binnum'
# generate mysql command
if ('c.specid=h.specid' or 'h.specid=c.specid') not in where:
where = where + ' and c.specid=h.specid'
cmd = command.generate('h.eventpower,%s,c.obstime'%freqtype,'hit h, config c',where=where)
# send command to database and return results
data = MySQLFunction.mysqlcommand(cmd)
# separate data into individual arrays
length = len(data)
eventpower = numpy.asarray([data[x][0] for x in xrange(length)])
if freqtype != 'h.binnum':
freq = numpy.asarray([data[x][1]/1000000 for x in xrange(length)])
elif freqtype=='h.binnum':
freq=numpy.asarray([data[x][1] for x in xrange(length)])
time = numpy.array([data[x][2] for x in xrange(length)])
# create seconds array
time = time-min(time)
time = time*86400
# save data?
if savedata!='':
numpy.savetxt('%s' %savedata,(eventpower,freq,time))
return (eventpower,freq,time)
def fetchdata(where='',freqtype='binnum'):
"""Fetches data to produce the three panel dynamic spectrum
plot.
where is a string to include additional information to
narrow the results. typically it will be used to specify a
range of specids. each column name MUST be prefixed with
the first letter of the table name and a period, like
c.obstime. don't forget to include 's.specid=c.specid if
referencing config and spec. do not include the word 'where'
at the beginning or the semicolon at the end. all other common
mysql syntax rules apply. Ex: 's.specid>1 and s.specid<=20 and
c.beamnum!=8 and c.specid=s.specid'. don't include hit table.
freqtype is the frequency type for the pcolormesh, either
'binnum' or 'topo'.
output is xarr, a 1D array representing the x-axis, yarr, a 1D
array representing the y-axis, and data, a 2D array of the
data"""
import MySQLFunction, struct, threeplot, pylab, numpy, command
#Create command to send to database
if ('c.specid=s.specid' or 's.specid=c.specid') not in where:
where = where + ' and c.specid=s.specid'
cmd = command.generate('s.coarsespec,c.obstime,c.IF1_rfFreq','spec s, config c', where=where)
#Fetch data from mysql
fromdb = MySQLFunction.mysqlcommand(cmd)
#Create arrays from data
length = len(fromdb)
coarsespec = [fromdb[x][0] for x in xrange(length)]
time = numpy.array([fromdb[x][1] for x in xrange(length)])
#Freq type?
if freqtype=='topo':
rfFreq = fromdb[0][2]
RFLO = rfFreq - 50000000
yarr = numpy.linspace(RFLO, RFLO+200000000, 4096)
yarr = yarr / 1000000
else:
yarr = numpy.arange(1,4097,1)
#Create seconds array
time = time-min(time)
xarr = time*86400
#Unpack data from blob
data = [numpy.array(struct.unpack('i'*4096,coarsespec[x])) for x in xrange(length)]
data = numpy.transpose(data)
return (xarr,yarr,data)
def update(output): """Takes output from interpolate and updates database in increments of 50000.""" #Determine limits for each loop limits=numpy.arange(0,len(output),50000) limits=numpy.append(limits,len(output)) #Perform loop for x in range(len(limits)-1): #Select output data in current range output_temp = output(limits[x]:limits[x+1]) #Create mysql query conditions_string = '\n'.join(["WHEN %s THEN %5.14s" %(x,y) for x,y in output_temp]) where_string = ', '.join([z[0] for z in output_temp]) query = "UPDATE config \nSET obstime = CASE specid \n%s \nEND \nWHERE specid in (%s)" %(conditions_string,where_string) #Send query to database MySQLFunction.mysqlcommand(query)
def fetchdata(freq_type='topo',increment_log=7,where='h.specid<100',savedata=''):
"""Fetches data for creating a histogram of hits versus
frequency, where frequency (or binnum) has been grouped by
increments of 10^(increment_log).
freq_type is either 'topo', 'bary', or 'binnum'. default is
topo
increment_log is the log of the spacing between bins. must
be an integer
where is a string to include additional information to
narrow the results. typically it will be used to specify a
range of specids. each column name MUST be prefixed with
the first letter of the table name and a period, like
c.obstime. include c.specid=h.specid if referncing both hit and
config tables. do not include the word 'where' at the beginning
or the semicolon at the end. all other common mysql syntax
rules apply. Ex: 'h.specid>1 and h.specid<=20
and c.beamnum!=8 and h.specid=c.specid'. by default it will
grab only the first 99 specids.
savedata allows the user to have the data saved by
inputting the file name. data is saved as an ASCII file. if
left empty, no file will be saved
returns two arrays, bins and count"""
import MySQLFunction, command, numpy, histcommand
#Generate mysql command using histcommand
cmd = histcommand.hist(freq_type,increment_log,where)
#Pass command to mysql and return results
data = MySQLFunction.mysqlcommand(cmd)
#Split data into two arrays
length = len(data)
bins = numpy.asarray([data[x][0] for x in range(length)])
count = numpy.asarray([data[x][1] for x in range(length)])
#Save data?
if savedata!='':
numpy.savetxt('%s' %savedata,(bins,count))
return (bins, count)
def grabdata(start,end): """ Grabs mjd value, specid, AGC_SysTime and AGC_Time from config table in given range. Output is the correct format for the interpolate function. Inputs: start - first specid in range to update end - last specid in range to update Output: data - array of obstime,specid,AGC_Systime,and AGC_Time, in tuples """ #Grab values from database cmd = 'select obstime,specid,AGC_SysTime,AGC_Time from config where specid>=%s and specid<=%s;' %(start,end) data = numpy.array(MySQLFunction.mysqlcommand(cmd)) return data
def extract(rawfile='',spec=[]):
"""Returns a list of tuples of start/stop
specids for continuous observing intervals where
drift rate is zero.
Input is EITHER the rawfile in which the intervals
are to be computed, or a list of the FIRST and LAST
spectra between which all intervals are computed.
Use one or the other but not both."""
import MySQLFunction, numpy, sys
#Create query depending on input type
if rawfile!='' and spec==[]:
cmd = 'select straight_join distinct c.specid from config c,hit h where h.reserved=0 and c.rawfile="%s" and c.specid=h.specid;' %rawfile
elif spec!=[] and rawfile=='' and len(spec)==2:
cmd = 'select straight_join distinct specid from hit where reserved=0 and specid>=%s and specid<=%s;' %(spec[0],spec[1])
elif len(spec)!=2:
sys.exit('Spec is a list of TWO specids only')
elif spec!=[] and rawfile!='':
sys.exit('Please only use ONE input type')
#Send query to db, fetch results
data = MySQLFunction.mysqlcommand(cmd)
data = [int(x[0]) for x in data]
#Compute differences between each spectra in results
diff = [data[x+1]-data[x] for x in range(len(data)-1)]
#Determine indices where diff>10
indices=[x for x in range(len(diff)) if diff[x]>20]
#Create tuples of start/stop specids for pointings
tuples = [(data[indices[x]+1],data[indices[x+1]]) for x in range(len(indices)-1) if (data[indices[x+1]]-data[indices[x]+1])>25]
return tuples
def update(spec_init,spec_final):
for beam in range(14):
#Create command to send to mysql
cmd = 'select ra,decl,specid from config where specid>=%i and specid<=%i and beamnum=%i;' %(spec_init,spec_final,beam)
#Execute command and return results
alldata = numpy.array(MySQLFunction.mysqlcommand(cmd))
#Run script on each set of consecutive spectra
for v in range(2):
if v==0:
ra,dec,specid=numpy.transpose(alldata[::2])
elif v==1:
ra,dec,specid=numpy.transpose(alldata[1::2])
#Calculate difference in ra,dec
delta_ra = 15*(ra[1:]-ra[:-1])
delta_dec = dec[1:]-dec[:-1]
#Correct for absolute declination
dec_rad = 0.5*(numpy.radians(dec[1:]+dec[:-1]))
delta_dec = numpy.degrees(numpy.cos(dec_rad)*numpy.radians(delta_dec))
#Compute vector quadratically
vec=numpy.sqrt(numpy.power(delta_ra,2)+numpy.power(delta_dec,2))
#Calculate drift rate
diff = specid[1:]-specid[:-1]
time = .67108864*diff
driftrate = vec/time
#Separate by observing type
specid=specid.astype(int)
targeted=specid[numpy.where(driftrate<=.0001)[0]]
slowdrift=specid[numpy.where((driftrate>.0001) & (driftrate<.0039))[0]]
skydrift=specid[numpy.where((driftrate>=.0039) & (driftrate<=.0043))[0]]
fastdrift=specid[numpy.where(driftrate>.0043)[0]]
#Last spectrum gets previous spectrum's designation
if driftrate[-1]<=.0001:
targeted=numpy.append(targeted,specid[-1])
elif driftrate[-1]>.0001 and driftrate[-1]<.0039:
slowdrift=numpy.append(slowdrift,specid[-1])
elif driftrate[-1]>=.0039 and driftrate[-1]<=.0043:
skydrift=numpy.append(skydrift,specid[-1])
else:
fastdrift=numpy.append(fastdrift,specid[-1])
#Compose strings, create query, and pass to db
if len(targeted)>0:
targeted_string = ', '.join([str(z) for z in targeted])
query0 = 'update hit set reserved=0 where specid in (%s);'%targeted_string
MySQLFunction.mysqlcommand(query0)
if len(slowdrift)>0:
slowdrift_string = ', '.join([str(z) for z in slowdrift])
query1 = 'update hit set reserved=1 where specid in (%s);'%slowdrift_string
MySQLFunction.mysqlcommand(query1)
if len(skydrift)>0:
skydrift_string = ', '.join([str(z) for z in skydrift])
query2 = 'update hit set reserved=2 where specid in (%s);'%skydrift_string
MySQLFunction.mysqlcommand(query2)
if len(fastdrift)>0:
fastdrift_string = ', '.join([str(z) for z in fastdrift])
query3 = 'update hit set reserved=3 where specid in (%s);'%fastdrift_string
MySQLFunction.mysqlcommand(query3)
print 'beam %s updated %s/2'%(beam,v+1)
return
def makeplot(hitfreq,eventpower,rfi_found,coarsespec_freq,coarsepowers,specid=1,dolog='False', dotext=True, saveplot=''):
"""Plots both coarse power (for coarse spectrum) as a blue line
and event power (for hits) as red X's on the same figure.
hitfreq is a list of frequencies for hits in a spectrum, eventpower
is a list of the eventpowers, coarsespec_freq is a list of
frequencies for the coarse spectra bins, and coarsepowers
is a list of the powers in each bin.
freq_type is either 'topo' or 'bary' (default is topo)
specid is the value of the specid desired
if dolog='True', the power on the y-axis will be logarithmic
where is an optional string to further modify which hits are plotted.
prefix all columns with an 'h' and a period, such as h.eventpower. do
not include the phrase 'where' or the semicolon at the end. all other
syntax rules apply.
saveplot='' allows you to save the plot by inputting its name as a
string. if left blank, no plot is saved
output is a figure."""
import pylab, numpy, MySQLFunction, jd2gd, math
# pylab.rc('text', usetex=True)
# pylab.rc('font', family='serif')
#Get additional info for text header
cmd = 'select beamnum, ra, decl, obstime, IF1_rfFreq, thrscale from config where specid=%d' %specid
data = MySQLFunction.mysqlcommand(cmd)
#Combine systime with obstime to get complete time
obstime = data[0][3]
#Create Gregorian date from obstime
gd = jd2gd.caldate(obstime)
dates = ['January','February','March','April','May','June','July',
'August','September','October','November','December']
gd = [str(gd[x]) for x in range(len(gd))]
#Insert zeros to make formatting nice
if float(gd[2])<10:
gd[2] = '0' + gd[2]
if float(gd[3])<10:
gd[3] = '0' + gd[3]
if float(gd[4])<10:
gd[4] = '0' + gd[4]
if float(gd[5])<10:
gd[5] = '0' + gd[5]
#Compile one date string
date = gd[0] + ' ' + dates[int(gd[1])-1] + ' ' + gd[2] + ' ' + gd[3] + ':' + gd[4] + ':' + gd[5][:2]
#Calculate center frequency
rfFreq=int(data[0][4])
rfFreq = rfFreq/1000000
rfFreq = rfFreq-50
cfreq = rfFreq + 100
#Determine beam and polarization
beam = int(data[0][0])
beamnum = str(math.floor(float(beam)/2))[0]
frac = float(beam)/2
if math.modf(frac)[0]==0.0:
newbeam = beamnum + 'a'
else: newbeam = beamnum + 'b'
#thrscale
thrscale=float(data[0][5])/4.0
#PLOTTING
#Initialize figure
fig=pylab.figure(figsize=(12,7))
ax1 = fig.add_axes([0.1, 0.12, 0.85, 0.75])
#Divide into RFI flagged or not
ginds=numpy.where(rfi_found==0)[0]
rinds=numpy.where(rfi_found>0)[0]
coarsepowers/=thrscale
#Log or not?
if dolog=='True':
pylab.semilogy(coarsespec_freq/1000000,coarsepowers,'k')
pylab.semilogy(hitfreq[rinds]/1000000,eventpower[rinds],'rx')
pylab.semilogy(hitfreq[ginds]/1000000,eventpower[ginds],'bx')
elif dolog=='False':
pylab.plot(coarsespec_freq/1000000,coarsepowers,'k')
pylab.plot(hitfreq[rinds]/1000000,eventpower[rinds],'rx')
pylab.plot(hitfreq[ginds]/1000000,eventpower[ginds],'bx')
#Add text to figure
if dotext:
pylab.figtext(0.1,.97,'Beam: %s' % newbeam)
pylab.figtext(0.3,.97,'RA: %s' %data[0][1])
pylab.figtext(0.5,.97,'Dec: %s' %data[0][2])
pylab.figtext(0.95,.97,'Date: %s' %date, ha='right')
pylab.figtext(0.1,.92,'Hit Count: %s' %len(eventpower))
pylab.figtext(0.95,.92,'Center Freq: %s MHz' %cfreq, ha='right')
#Set x-scale
lowbound = cfreq - 100
uppbound = cfreq + 100
xticks = numpy.linspace(lowbound,uppbound,21)
pylab.xticks(xticks)
#Rotate x-labels
for i in ax1.get_xticklabels(): i.set_rotation(45)
#Set labels and title
pylab.xlabel('Frequency (MHz)', size=14)
pylab.ylabel('Power (arb units)', size=14)
# pylab.title(' Coarse Spectrum and Hits for Specid=%d' %specid)
#Set axis limits
if len(eventpower) != 0:
v = [lowbound,uppbound,min(coarsepowers),max(eventpower)+1000]
else: v=[lowbound,uppbound,min(coarsepowers),max(coarsepowers)]
pylab.axis(v)
#Add grid
pylab.grid(True)
#Save plot?
if saveplot != '':
pylab.savefig('%s' %saveplot)
return fig
def fetchdata(freq_type='topo', specid=1, where='',writedata='False'):
""" Fetches the coarse spectrum and event power data for
the input parameters.
freq_type is either 'topo' or 'bary'. The default is 'topo'
specid is the specid to be plotted. If no value is
assigned, data from the first specid will be plotted.
where is an optional string to further modify which hits
are plotted.only include columns from table "hit". do not
include the word 'where' or the semicolon at the end. all
other syntax rules apply.
writedata='True' allows the user to have the data saved.
data are stored as ASCII files, one for each the coarse
spectrum and the hits, each containing two arrays of the
same length. writedata is set to 'False' by default. Files,
if saved, are named CoarseSpec_specid.txt and
Hits_specid.txt, where specid is the corresponding
numerical value.
returns four arrays: hit_frequency, hit_eventpower,
coarsespec_frequency, coarsespec_power"""
import MySQLFunction, command, numpy, struct
#Create full column name from freq_type
if freq_type=='topo':
freq_type = 'topocentric_freq'
elif freq_type=='bary':
freq_type = 'barycentric_freq'
#Create strings to put in command function
col_name1 = '%s, eventpower, rfi_found' % freq_type
table_name1 = 'hit'
where1 = 'specid=%d' % specid
#Change where1 if where string isn't empty
if where != '' :
where1 = where1 + ' and %s' %where
col_name2 = 's.coarsespec, c.IF1_rfFreq'
table_name2 = 'spec s, config c'
where2 = 's.specid=%d and s.specid=c.specid' % specid
#Create command to send to MySQL
command1 = command.generate(col_name1,table_name1,where=where1)
command2 = command.generate(col_name2, table_name2,where=where2)
#Execute commands in MySQL
data = MySQLFunction.mysqlcommand(command1)
allelse = MySQLFunction.mysqlcommand(command2)
#Separate data into component arrays
length = len(data)
hitfreq = numpy.asarray([data[x][0] for x in range(length)])
eventpower = numpy.asarray([data[x][1] for x in range(length)])
rfi_found = numpy.asarray([data[x][2] for x in range(length)])
blob = allelse[0][0]
rfFreq = allelse[0][1]
#Unpack blob
coarsepowers = numpy.asarray(struct.unpack('i'*4096,blob))
#Compute coarse spectrum frequencies, centered on each bin
RFLO = rfFreq - 50000000
coarsespec_freq = numpy.linspace(RFLO, RFLO+200000000, 4096)
#Save data?
if writedata == 'True':
numpy.savetxt('CoarseSpec_%d.txt' %specid,(coarsespec_freq,coarsepowers))
numpy.savetxt('Hits_%d.txt' %specid,(freq,eventpower))
return (hitfreq, eventpower, rfi_found, coarsespec_freq, coarsepowers)
def fetchdata(where='',cumulative='False',dolog='False',savedata=''):
""" Fetches data for a histogram of hits per mean power bin.
where is a string to include additional information to
narrow the results. typically it will be used to specify a
range of specids. each column name MUST be prefixed with
the first letter of the table name and a period, like
c.obstime. don't forget to include 'h.specid=c.specid if
referencing config and hit. do not include the word 'where'
at the beginning or the semicolon at the end. all other common
mysql syntax rules apply. Ex: 'h.specid>1 and h.specid<=20 and
c.beamnum!=8 and c.specid=h.specid'.
setting cumulative='True' makes a cumulative histogram as mean
power increases
savedata allows the user to have the data saved by
inputting the file name. data is saved as an ASCII file. if
left empty, no file will be saved
returns two arrays, bins and count. bins are LEFT edges"""
import numpy, MySQLFunction, command
#Create command to pass to mysql
if 'c.' in where:
cmd = command.generate('h.meanpower','config c, hit h', where=where)
else:
cmd = command.generate('h.meanpower','hit h',where=where)
#Send command to mysql, return results
data = MySQLFunction.mysqlcommand(cmd)
#Data not empty?
if len(data)>0:
#Turn data into array
data = numpy.array([x[0] for x in data])
#Create linearly spaced bins
bins = list(numpy.linspace(50,500,91))
bins.append(max(data))
#Histogram data
count,bins = numpy.histogram(data,bins)
#Normalize data if not log
if dolog!='True':
norml = float(numpy.sum(count))
count = [x/norml for x in count]
#Cumulative?
if cumulative=='True':
count = list(numpy.cumsum(count))
#Save data?
if savedata!='':
numpy.savetxt('%s' %savedata,(bins,count))
else: bins,count=[],[]
return (bins,count)
def fetchdata(maximum=24,histtype='fracmax',where='',savedata=''):
"""Fetches data for event count per coarse frequency bin.
maximum is the maximum number of allowed hits per coarse bin
histtype specifies the type of histogram you intend to
plot:
'fracmax' is the fraction of spectra that reach the
maximum for each coarse bin, and the default option,
'%full' is the percent full each coarse bin is after adding
events for all desired spectra, and
'maxcount' is the number of times each bin reaches the maximum
where is a string to include additional information to
narrow the results. typically it will be used to specify a
range of specids. each column name MUST be prefixed with
the first letter of the table name and a period, like
c.obstime. don't forget to include 'h.specid=c.specid if
referencing table config. do not include the word 'where'
at the beginning or the semicolon at the end. all other common
mysql syntax rules apply. Ex: 'h.specid>1 and h.specid<=20 and
c.beamnum!=8 and c.specid=h.specid'.
savedata allows the user to have the data saved by
inputting the file name. data is saved as an ASCII file. if
left empty, no file will be saved
returns two arrays, bin and count"""
import MySQLFunction, command, math, numpy, collections, itertools
#Create command to pass to mysql
if 'c.' in where:
cmd = command.generate('h.binnum,h.specid','config c, hit h', where=where)
else:
cmd = command.generate('h.binnum,h.specid','hit h',where=where)
#Send command and return results
data = MySQLFunction.mysqlcommand(cmd)
#Round bin numbers down to nearest coarsebin
data = [(math.floor(data[x][0]/32768),data[x][1]) for x in xrange(len(data))]
#Initialize bin array
bins = numpy.arange(0,4097,1)
#Depending on histtype, compile data:
if histtype!='%full':
#Group data by specid
res = collections.defaultdict(list)
for v,k in data: res[k].append(v)
data = [{'specid':k, 'coarsebin':v} for k,v in res.items()]
#Histogram each spectrum
length = len(data)
hist_temp = [(numpy.histogram(data[x]['coarsebin'],bins)) for x in xrange(length)]
#Make a list of lists of full bins
fullbins = [[y for y in numpy.where(hist_temp[x][0]==maximum)[0]] for x in xrange(length)]
#Unpack fullbins into a single list
fullbins = list(itertools.chain(*fullbins))
#Histogram the full bins
count = numpy.histogram(fullbins,bins)[0]
if histtype=='fracmax':
#Divide by number of spectra to get fraction
count = numpy.array(count/float(length))
elif histtype=='%full':
length = len(data)
#Pull out coarse bins, specids from data
coarsebins = [data[x][0] for x in xrange(length)]
specids = [data[x][1] for x in xrange(length)]
#Histogram data
count = numpy.histogram(coarsebins,bins)[0]
#Determine total number of spectra
uniq_IDs = list(set(specids))
numspec = len(uniq_IDs)
#Divide by total number possible to get percent
count = numpy.array(count/float(numspec*24))
#Cut off right-most bin
bins = bins[:-1]
#Save data?
if savedata!='':
numpy.savetxt('%s' %savedata,(bins,count))
return (bins,count)
def makeplot(ra,dec,specid,where='',figtype='cart',ellipses='no',saveplot=''):
""" Makes a plot of RA and DEC.
where is a string specifying which data from mysql to grab.
ONLY include columns from table 'config'. do not include
the word 'where' at the beginning or the semicolon at the
end. all other common mysql syntax rules apply.
figtype is the figure type. default is 'cart' for cartesian
coordinates. 'sky' plots the ra and dec as a Mollweide
projection
if ellipses='yes', the spectra are plotted as individual ellipses
of fixed size corresponding to ALFA beam width (only for Cartesian
coordinate axis)
saveplot allows you to save the figure by specifying the
file name as a string. if left empty, the figure is not
saved.
returns a figure"""
import jd2gd, command
#Determine values to include in figure
var1 = min(specid)
var2 = max(specid)
var3 = len(specid)
#Get time info from database
cmd = command.generate('obstime','config',where=where)
data = MySQLFunction.mysqlcommand(cmd)
#Create list
day = numpy.asarray([data[x][0] for x in xrange(len(data))])
#Determine start and end dates
start = min(day)
end = max(day)
#Create Gregorian date from obstime
start = jd2gd.caldate(start)
end = jd2gd.caldate(end)
dates = ['January','February','March','April','May','June','July',
'August','September','October','November','December']
start = [str(start[x]) for x in range(len(start))]
end = [str(end[x]) for x in range(len(end))]
#Insert zeros to make formatting nice
if float(start[2])<10:
start[2] = '0' + start[2]
if float(start[3])<10:
start[3] = '0' + start[3]
if float(start[4])<10:
start[4] = '0' + start[4]
if float(start[5])<10:
start[5] = '0' + start[5]
if float(end[2])<10:
end[2] = '0' + end[2]
if float(end[3])<10:
end[3] = '0' + end[3]
if float(end[4])<10:
end[4] = '0' + end[4]
if float(end[5])<10:
end[5] = '0' + end[5]
#Compile date strings
date1 = start[0]+' '+dates[int(start[1])-1]+' '+start[2]+' '+start[3]+':'+start[4]+':'+start[5][:4]
date2 = end[0]+' '+dates[int(end[1])-1]+' '+end[2]+' '+end[3]+':'+end[4]+':'+end[5][:4]
#Which fig type?
patches=[]
if figtype=='cart':
#Initialize figure
fig=pylab.figure(figsize=(12,7))
ax1 = fig.add_axes([0.1, 0.12, 0.85, 0.75])
#Plot data
if ellipses=='yes':
for x,y in zip(ra,dec):
circle=matplotlib.patches.Ellipse((x,y),width=.0038888,height=.058333,facecolor='b',edgecolor='b')
patches.append(circle)
p=matplotlib.collections.PatchCollection(patches)
ax1.add_collection(p)
else: ax1.scatter(ra,dec,s=.1,label='spectra')
else:
#Convert RA to degrees
length = len(ra)
ra = [ra[x]*15 for x in range(length)]
#Convert arrays to radians
ra = [math.radians(ra[x]) for x in range(length)]
dec = [math.radians(dec[x]) for x in range(length)]
#Adjust ra scale
ra = [ra[x]-numpy.pi for x in range(length)]
#Create figure
fig = pylab.figure(figsize=(12,7))
ax1 = fig.add_axes([.1,.12,.70,.85],projection='mollweide')
#Plot data
ax1.scatter(ra,dec,label='spectra',marker='+',c='b')
#Set xtick labels
ax1.set_xticklabels(['2','4','6','8','10','12','14','16','18','20','22'])
#Create galactic plane lines
#Create l,b arrays
b = [math.radians(5)]*360 + [math.radians(-5)]*360
l = list(numpy.linspace(0,2*numpy.pi,360))*2
#Pass l,b pairs to gal2eq
bounds = []
for x in range(720):
bounds.append(astroconvert.gal2eq(l[x],b[x]))
#Separate into component arrays
ra_bounds = [bounds[x][0] for x in range(720)]
dec_bounds = [bounds[x][1] for x in range(720)]
#Convert to deg if necessary, and plot
if figtype=='sky':
ra_bounds = [ra_bounds[x]-numpy.pi for x in range(720)]
ax1.scatter(ra_bounds,dec_bounds,edgecolor='none',c='r',label='galactic plane')
else:
ra_bounds = [math.degrees(ra_bounds[x])/15 for x in range(720)]
dec_bounds = [math.degrees(dec_bounds[x]) for x in range(720)]
ax1.plot(ra_bounds,dec_bounds,'r',label='galactic plane')
#Set axis limits
ax1.set_xlim(0, 24)
ax1.set_ylim(0, 40)
#Set axis labels and title
pylab.xlabel('RA (hours)')
pylab.ylabel('Dec (degrees)')
pylab.title('Right Ascension and Declination')
#Add legend
ax1.legend(bbox_to_anchor=(0,0,1,1),loc=0)
#Add text to figure
pylab.figtext(0.1,.97,'First Specid: %d' %var1)
pylab.figtext(0.1,.945,'Last Specid: %d' %var2)
pylab.figtext(0.1,.92,'Count: %d spectra' %var3)
pylab.figtext(0.95,.97,' Start: %s' %date1, ha='right')
pylab.figtext(0.95,.945,'End: %s' %date2, ha='right')
#Include grid
pylab.grid(True)
#Save figure?
if saveplot!='':
pylab.savefig('%s'%saveplot)
return fig
def check(spec1,spec2,width,length):
"""Checks a range of specids for frequency
clustering.
spec1 and spec2 are the first and last
spectra to be checked for clustering.
width is the frequency interval width in
which to look for clusters.
length is the number of consecutive spectra
that consistitutes a cluster.
All spectra must share a common center
frequency.
Output is a list of frequency bands in
which clustering was observed."""
import MySQLFunction, numpy, sys
#Obtain frequency information for loops
cmd = 'select distinct IF1_rfFreq from config where specid>=%s and specid<=%s;'%(spec1,spec2)
centerfreq = MySQLFunction.mysqlcommand(cmd)
centerfreq = [freq[0] for freq in centerfreq]
#Only one center frequency?
if len(centerfreq)>1:
sys.exit('More than one center frequency in this range')
else:
center = centerfreq[0]
#Create frequency band ranges
uppbound = center + 100000000
lowbound = center - 100000000
freqs = numpy.arange(lowbound,uppbound+1,width)
#Loop through freq bands
results = []
for x in range(len(freqs)-1):
#Get spectra where a hit is found in the specified freq range
cmd = 'select distinct specid from hit where specid>=%s and specid<=%s and topocentric_freq>%s and topocentric_freq<%s and rfi_found=0 and reserved=0;'%(spec1,spec2,freqs[x],freqs[x+1])
spectra = MySQLFunction.mysqlcommand(cmd)
spectra = [b[0] for b in spectra]
#Does a cluster occur?
if len(spectra)>(width-1):
for y in range(len(spectra)-(width-1)):
#Create list of differences
diff = [spectra[z+1]-spectra[z] for z in numpy.arange(y,y+width,1)]
for val in diff:
if val!=1:
diff[val]=0
#All consecutive?
if all(diff):
results.append(freqs[x]+(width/2))
break
print x
return results
def makeplot(xarr,yarr,data,where='',freqtype='binnum',vlim=(-1,-1), tslim=(-1,-1),saveplot=''):
"""Method to produce the three panel
dynamic spectrum plot. The main panel
is a pcolormesh figure of data with
the mesh defined by xarr and yarr.
The bottom panel shows a time series of
the 2D data, and the right panel shows
the average bandpass
freqtype is the frequency type for the pcolormesh, either
'binnum' or 'topo'.
Input:
where is a string to include additional information to
narrow the results. typically it will be used to specify a
range of specids. each column name MUST be prefixed with
the first letter of the table name and a period, like
c.obstime. don't forget to include 's.specid=c.specid if
referencing config and spec. do not include the word 'where'
at the beginning or the semicolon at the end. all other common
mysql syntax rules apply. Ex: 's.specid>1 and s.specid<=20 and
c.beamnum!=8 and c.specid=s.specid'. don't include hit table.
saveplot='' allows you to save the plot by inputting its name
as a string. if left blank, no plot is saved
Output:
Figure instance
"""
import numpy, pylab, jd2gd, MySQLFunction, command
#Calculate the time series and average bandpass
# for the subpanel plots
tseries=numpy.mean(data, axis=0)
bandpass=numpy.mean(data, axis=1)
#If no plot limits specified,
if vlim==(-1,-1):
vlim=(numpy.min(data), numpy.max(data))
if tslim==(-1,-1):
tslim=(numpy.min(tseries), numpy.max(tseries))
#Create figure instance, add axes and turn off labels
fig=pylab.figure(figsize=(12,7))
ax1 = fig.add_axes([0.1, 0.3, 0.6, 0.6])
ax2 = fig.add_axes([0.1, 0.1, 0.6, 0.2], sharex=ax1)
ax3 = fig.add_axes([0.7, 0.3, 0.2, 0.6], sharey=ax1)
for i in ax3.get_yticklabels(): i.set_visible(False)
for i in ax3.get_xticklabels(): i.set_rotation(270)
for i in ax1.get_xticklabels(): i.set_visible(False)
#Generate 2D mesh
T,F=numpy.meshgrid(xarr,yarr)
#Add plots
ax1.pcolormesh(T,F,data, vmin=vlim[0], vmax=vlim[1])
ax2.plot(xarr, tseries, 'r.')
ax3.step(bandpass, yarr, 'g-')
#Set axes labels
ax2.set_xlabel('Time (Seconds)')
if freqtype=='binnum':
ax1.set_ylabel('Frequency channel')
elif freqtype=='topo':
ax1.set_ylabel('Frequency (MHz)')
ax1.set_title('Dynamic Spectra - Coarse Bins')
ax2.set_ylabel('Mean Intensity')
ax1.set_xlim((min(xarr), max(xarr)))
ax1.set_ylim((min(yarr), max(yarr)))
ax2.set_ylim((tslim[0], tslim[1]))
#Gather additional info
if where=='':
cmd = command.generate('specid,obstime,AGC_Time','config')
elif 'c.' not in where:
where = where + ' and s.specid=c.specid'
cmd = command.generate('s.specid,c.obstime,c.AGC_Time','config c, spec s',where=where)
data = MySQLFunction.mysqlcommand(cmd)
#Separate into arrays
length = len(data)
specid = [data[x][0] for x in range(length)]
day = numpy.asarray([data[x][1] for x in range(length)])
fracday = numpy.asarray([float(data[x][2])/86400000 for x in range(length)])
time = day + fracday
#Get specid count
uniq_IDs = set(specid)
speccount = len(uniq_IDs)
#Determine start and end dates
start = min(time)
end = max(time)
#Create Gregorian date from obstime
start = jd2gd.caldate(start)
end = jd2gd.caldate(end)
dates = ['January','February','March','April','May','June','July',
'August','September','October','November','December']
start = [str(start[x]) for x in range(len(start))]
end = [str(end[x]) for x in range(len(end))]
#Insert zeros to make formatting nice
if float(start[2])<10:
start[2] = '0' + start[2]
if float(start[3])<10:
start[3] = '0' + start[3]
if float(start[4])<10:
start[4] = '0' + start[4]
if float(start[5])<10:
start[5] = '0' + start[5]
if float(end[2])<10:
end[2] = '0' + end[2]
if float(end[3])<10:
end[3] = '0' + end[3]
if float(end[4])<10:
end[4] = '0' + end[4]
if float(end[5])<10:
end[5] = '0' + end[5]
#Compile date strings
date1 = start[0]+' '+dates[int(start[1])-1]+' '+start[2]+' '+start[3]+':'+start[4]+':'+start[5][:4]
date2 = end[0]+' '+dates[int(end[1])-1]+' '+end[2]+' '+end[3]+':'+end[4]+':'+end[5][:4]
#Add text to figure
pylab.figtext(0.73,.175,'SpecID Count: %s' %speccount)
pylab.figtext(0.73,.15,'Start: %s' %date1)
pylab.figtext(0.73,.125,'End: %s' %date2)
#Save plot?
if saveplot != '':
pylab.savefig('%s' %saveplot)
return fig
def makeplot(bins,count,freq_type='topo',increment_log=7,where='h.specid<100',dolog='False',saveplot=''):
"""Creates a histogram of hits at different
frequencies, separated into bins by increments of
10^(increment_log).
freq_type is either 'topo', 'bary', or 'binnum'. default is
topo
increment_log is the log of the spacing between bins. must
be an integer
where is a string to include additional information to
narrow the results. typically it will be used to specify a
range of specids. each column name MUST be prefixed with
the first letter of the table name and a period, like
c.obstime. do not include the word 'where' at the beginning
or the semicolon at the end. all other common mysql syntax
rules apply. Ex: 'h.specid>1 and h.specid<=20 and
c.beamnum!=8'. by default it will grab only the first 99
specids.
if dolog='True', the hit count is plotted on a log scale
saveplot allows the user to have the plot saved by inputting
the file name. if left empty, no file will be saved.
returns a figure"""
import pylab, numpy, math, MySQLFunction, command, jd2gd
#Are there data to plot?
if len(bins)!=0:
#Initialize figure
fig=pylab.figure(figsize=(12,7))
ax1 = fig.add_axes([0.1, 0.14, 0.85, 0.75])
#Configure x-ticks
if freq_type!='binnum':
lowbound = min(bins)/1000000
uppbound = max(bins)/1000000
xticks = numpy.arange(lowbound,uppbound+10**(increment_log-6),10)
pylab.xticks(xticks)
else:
lowbound = min(bins)
uppbound = max(bins)
xticks = numpy.arange(lowbound,uppbound+10**(increment_log),5000000)
pylab.xticks(xticks)
#Rotate xticks
for i in ax1.get_xticklabels(): i.set_rotation(45)
#Create xlabels
if freq_type=='topo':
xlabel = 'Topocentric Frequency (MHz)'
elif freq_type=='bary':
xlabel = 'Barycentric Frequency (MHz)'
else: xlabel = 'Bin Number'
#Determine bar width
if freq_type != 'binnum':
width = (10**increment_log)/1000000
else: width = 10**increment_log
#Make plots
if dolog=='True' and freq_type!='binnum':
pylab.bar(bins/1000000,count,log='True',width=width,align='center')
ylabel = 'Count (Logarithmic)'
elif dolog=='True':
pylab.bar(bins,count,log='True',width=width,align='center')
ylabel = 'Count (Logarithmic)'
elif freq_type!='binnum':
pylab.bar(bins/1000000,count,width=width,align='center')
ylabel = 'Count'
else:
pylab.bar(bins,count,width=width,align='center')
ylabel = 'Count'
#Determine y-axis lower limit
if dolog=='True':
bottom = int(math.floor(numpy.log10(min(count)+1)))
var = 10**bottom
else:
var = min(count)*0.5
#Set axes limits
if freq_type=='binnum':
v = [0,uppbound+width/2,var,max(count)*1.1]
pylab.axis(v)
else:
v=[lowbound-width/2,uppbound+width/2,var,max(count)*1.1]
pylab.axis(v)
#Add grid
pylab.grid(True)
#Add labels
pylab.xlabel('%s'%xlabel)
pylab.ylabel('%s'%ylabel)
pylab.title('Event Count Per Bin')
#Get extra info for plot
if where=='':
cmd = command.generate('specid,obstime,AGC_Time','config')
elif 'c.' not in where:
where = where + ' and h.specid=c.specid'
cmd = command.generate('c.specid,c.obstime','config c, hit h',where=where)
#Send command to mysql, return results
data = MySQLFunction.mysqlcommand(cmd)
#Separate into arrays
length = len(data)
specid = [data[x][0] for x in range(length)]
time = numpy.asarray([data[x][1] for x in range(length)])
#Get hit count and specid count
uniq_IDs = set(specid)
speccount = len(uniq_IDs)
hitcount = sum(count)
#Determine start and end dates
start = min(time)
end = max(time)
#Create Gregorian date from obstime
start = jd2gd.caldate(start)
end = jd2gd.caldate(end)
dates = ['January','February','March','April','May','June','July',
'August','September','October','November','December']
start = [str(start[x]) for x in range(len(start))]
end = [str(end[x]) for x in range(len(end))]
#Insert zeros to make formatting nice
if float(start[2])<10:
start[2] = '0' + start[2]
if float(start[3])<10:
start[3] = '0' + start[3]
if float(start[4])<10:
start[4] = '0' + start[4]
if float(start[5])<10:
start[5] = '0' + start[5]
if float(end[2])<10:
end[2] = '0' + end[2]
if float(end[3])<10:
end[3] = '0' + end[3]
if float(end[4])<10:
end[4] = '0' + end[4]
if float(end[5])<10:
end[5] = '0' + end[5]
#Compile date strings
date1 = start[0]+' '+dates[int(start[1])-1]+' '+start[2]+' '+start[3]+':'+start[4]+':'+start[5][:2]
date2 = end[0]+' '+dates[int(end[1])-1]+' '+end[2]+' '+end[3]+':'+end[4]+':'+end[5][:2]
#Create dictionary of bin resolutions
if freq_type!='binnum':
binres = {0: '1 Hz', 1: '10 Hz', 2: '100 Hz', 3: '1 KHz', 4: '10 KHz', 5: '100 KHz',
6: '1 MHz', 7: '10 MHz', 8: '100MHz', 9: '1 GHz'}
else:
binres = {0: '1 Bin', 1: '10 Bins', 2: '100 Bins', 3: '1,000 Bins', 4: '10,000 Bins', 5: '100,000 Bins',
6: '1,000,000 Bins', 7: '10,000,000 Bins', 8: '100,000,000 Bins'}
#Add text to figure
pylab.figtext(0.1,.97,'Hit Count: %s' %hitcount)
pylab.figtext(0.1,.92,'Bin Resolution: %s' %binres[increment_log])
pylab.figtext(0.1,.945,'SpecID Count: %s' %speccount)
pylab.figtext(0.95,.97,' Start: %s' %date1, ha='right')
pylab.figtext(0.95,.945,'End: %s' %date2, ha='right')
#Save figure?
if saveplot!='':
pylab.savefig('%s'%saveplot)
return fig
def makeplot(bins,count,where='',dolog='False',cumulative='False',saveplot=''):
"""Creates histogram of hits per mean power bin.
where is a string to include additional information to
narrow the results. typically it will be used to specify a
range of specids. each column name MUST be prefixed with
the first letter of the table name and a period, like
c.obstime. don't forget to include 'h.specid=c.specid if
referencing config and hit. do not include the word 'where'
at the beginning or the semicolon at the end. all other common
mysql syntax rules apply. Ex: 'h.specid>1 and h.specid<=20 and
c.beamnum!=8 and c.specid=h.specid'.
setting cumulative='True' makes a cumulative histogram as mean
power increases
setting dolog='True' makes the y-axis (count) logarithmic
saveplot allows the user to have the figure saved by
inputting the file name. if left empty, no file will be saved
returns a figure"""
import pylab, MySQLFunction, command, jd2gd, math, numpy
#Data to plot?
if len(bins) != 0:
#Initialize figure
fig=pylab.figure(figsize=(12,7))
ax1 = fig.add_axes([0.1, 0.14, 0.85, 0.75])
#Determine bar width
width = bins[1]-bins[0]
#Logarithmic?
if dolog != 'True':
pylab.bar(bins[:-1],count,width=width)
ylabel = 'Count'
var = min(count)*0.5
else:
pylab.bar(bins[:-1],count,width=width,log='True')
ylabel = 'Count (Logarithmic)'
bottom = int(math.floor(numpy.log10(min(count)+1)))
var = 10**(bottom)
#Configure ticks
pylab.xticks([0,50,100,150,200,250,300,350,400,450,500,500+width],['0','50','100','150','200','250','300','350','400','450','500','%d'%max(bins)])
#Rotate xticks
for i in ax1.get_xticklabels(): i.set_rotation(45)
#Cumulative?
if cumulative != 'True':
cumltv = ''
else:
cumltv = '(Cumulative)'
#Set axes limits
v = [min(bins),500+width,var,max(count)*1.1]
pylab.axis(v)
#Add grid
pylab.grid(True,which='both')
#Add labels
pylab.xlabel('Mean Power')
pylab.ylabel('%s'%ylabel)
pylab.title('Hit Count Per Mean Power Bin %s'%cumltv)
#Get extra info for plot
if where=='':
cmd = command.generate('specid,obstime,AGC_Time','config')
elif 'c.' not in where:
where = where + ' and h.specid=c.specid'
cmd = command.generate('c.specid,c.obstime','config c, hit h',where=where)
data = MySQLFunction.mysqlcommand(cmd)
#Separate into arrays
length = len(data)
specid = [data[x][0] for x in range(length)]
time = numpy.asarray([data[x][1] for x in range(length)])
#Get specid count
uniq_IDs = set(specid)
speccount = len(uniq_IDs)
# determine start and end dates
start = min(time)
end = max(time)
#Create Gregorian date from obstime
start = jd2gd.caldate(start)
end = jd2gd.caldate(end)
dates = ['January','February','March','April','May','June','July',
'August','September','October','November','December']
start = [str(start[x]) for x in range(len(start))]
end = [str(end[x]) for x in range(len(end))]
#Insert zeros to make formatting nice
if float(start[2])<10:
start[2] = '0' + start[2]
if float(start[3])<10:
start[3] = '0' + start[3]
if float(start[4])<10:
start[4] = '0' + start[4]
if float(start[5])<10:
start[5] = '0' + start[5]
if float(end[2])<10:
end[2] = '0' + end[2]
if float(end[3])<10:
end[3] = '0' + end[3]
if float(end[4])<10:
end[4] = '0' + end[4]
if float(end[5])<10:
end[5] = '0' + end[5]
#Compile date strings
date1 = start[0]+' '+dates[int(start[1])-1]+' '+start[2]+' '+start[3]+':'+start[4]+':'+start[5][:2]
date2 = end[0]+' '+dates[int(end[1])-1]+' '+end[2]+' '+end[3]+':'+end[4]+':'+end[5][:2]
#Add text to figure
pylab.figtext(0.1,.945,'SpecID Count: %s' %speccount)
pylab.figtext(0.95,.97,'Start: %s' %date1, ha='right')
pylab.figtext(0.95,.945,'End: %s' %date2, ha='right')
#Save figure?
if saveplot!='':
pylab.savefig('%s'%saveplot)
return fig
def makeplot(eventpower,freq,time,errbar=[0], where='',freqtype='topo',vlim=(-1,-1),frac=0.9,saveplot=''):
"""Produces a 'confetti plot' of spectra dynamically.
freqtype is the frequency type, either 'binnum', 'bary', or
'topo'.
Input:
where is a string to include additional information to
narrow the results. typically it will be used to specify a
range of specids. each column name MUST be prefixed with
the first letter of the table name and a period, like
c.obstime. don't forget to include 's.specid=c.specid if
referencing config and spec. do not include the word 'where'
at the beginning or the semicolon at the end. all other common
mysql syntax rules apply. Ex: 's.specid>1 and s.specid<=20 and
c.beamnum!=8 and c.specid=s.specid'. don't include hit table.
vlim is a tuple of (vmin,vmax). Values are used in conjuction
with norm to normalize luminance data.
frac is the fraction of hits, sorted by eventpower, before which
the size is 0.1 and after which the size is 1
saveplot='' allows you to save the plot by inputting its name
as a string. if left blank, no plot is saved
Output:
Figure instance
"""
import pylab, numpy, MySQLFunction, command, jd2gd, math
# initialize figure
fig=pylab.figure(figsize=(12,7))
ax1 = fig.add_axes([0.1, 0.14, 0.85, 0.75])
#If no plot limits specified,
if vlim==(-1,-1):
vlim=(numpy.min(eventpower), numpy.max(eventpower))
# create array of point sizes
sorted = numpy.sort(eventpower)
index = math.floor(frac*len(eventpower))
cutoff = eventpower[index]
size = [.1 if eventpower[x]<cutoff else 1 for x in xrange(len(eventpower))]
# plot data
if len(errbar)>1:
size=20
pylab.errorbar(time,freq,xerr=errbar,fmt=None,ecolor='k', capsize=0.0)
pylab.scatter(time,freq,s=size,c=eventpower,edgecolors='none',vmin=vlim[0],vmax=vlim[1])
# add grid
pylab.grid(True,which='both')
# add labels
ax1.set_xlabel('Time (seconds)')
ax1.set_title('Dynamic Spectra - Hits')
if freqtype=='binnum':
ax1.set_ylabel('Frequency channel')
elif freqtype=='topo':
ax1.set_ylabel('Topocentric Frequency (MHz)')
elif freqtype=='bary':
ax1.set_ylabel('Barycentric Frequency (MHz)')
# gather additional info
if where=='':
cmd = command.generate('specid,obstime,AGC_Time,IF1_rfFreq','config')
elif 'c.' not in where:
where = where + ' and h.specid=c.specid'
cmd = command.generate('h.specid,c.obstime,c.AGC_Time,c.IF1_rfFreq','hit h, config c',where=where)
else:
where = where + ' and h.specid=c.specid'
cmd = command.generate('h.specid,c.obstime,c.AGC_Time,c.IF1_rfFreq','hit h, config c',where=where)
data = MySQLFunction.mysqlcommand(cmd)
# separate into arrays
length = len(data)
specid = [data[x][0] for x in xrange(length)]
day = numpy.asarray([data[x][1] for x in xrange(length)])
# get specid and hit count
uniq_IDs = set(specid)
speccount = len(uniq_IDs)
hitcount = len(eventpower)
# determine start and end dates
start = min(day)
end = max(day)
# calculate the min and max RF from the center freq
# scale y axis accordinly
rfctr=float(data[0][3])
rflo=rfctr-50e6
rfhi=rflo+200e6
rflo-=5e6 #Add offsets to get hits away from plot edges
rfhi+=5e6
# guess whether data is given in Hz or MHz
if numpy.log10(freq[0])<4:
rflo/=1e6
rfhi/=1e6
# set axes limits
v = [0,max(time),rflo,rfhi]
pylab.axis(v)
if min(freq)<rflo or max(freq)>rfhi: print "WARNING: There are hits outside freq limits"
# create Gregorian date from obstime
start = jd2gd.caldate(start)
end = jd2gd.caldate(end)
dates = ['January','February','March','April','May','June','July',
'August','September','October','November','December']
start = [str(start[x]) for x in range(len(start))]
end = [str(end[x]) for x in range(len(end))]
# insert zeros to make formatting nice
if float(start[2])<10:
start[2] = '0' + start[2]
if float(start[3])<10:
start[3] = '0' + start[3]
if float(start[4])<10:
start[4] = '0' + start[4]
if float(start[5])<10:
start[5] = '0' + start[5]
if float(end[2])<10:
end[2] = '0' + end[2]
if float(end[3])<10:
end[3] = '0' + end[3]
if float(end[4])<10:
end[4] = '0' + end[4]
if float(end[5])<10:
end[5] = '0' + end[5]
# compile date strings
date1 = start[0]+' '+dates[int(start[1])-1]+' '+start[2]+' '+start[3]+':'+start[4]+':'+start[5][:2]
date2 = end[0]+' '+dates[int(end[1])-1]+' '+end[2]+' '+end[3]+':'+end[4]+':'+end[5][:2]
# add text to figure
pylab.figtext(0.4,.96,'Spectra Count: %s' %speccount)
pylab.figtext(0.4,.935,'Hit Count: %s' %hitcount)
pylab.figtext(0.61,.96,'Vmin,Vmax: %s %s' % (vlim[0], vlim[1]))
pylab.figtext(0.61,.935,'ALFA RFctr: %4.1f' % (rfctr/1e6))
pylab.figtext(0.8,.96,'Max Power: %s' %max(eventpower))
pylab.figtext(0.8,.935,'Min Power: %s' %min(eventpower))
pylab.figtext(0.1,.96,'Start: %s' %date1)
pylab.figtext(0.1,.935,'End: %s' %date2)
# save plot?
if saveplot != '':
pylab.savefig('%s' %saveplot)
return fig
def makeplot(bins,count,maximum=24,histtype='fracmax',where='',saveplot=''):
"""Plots data for event count per coarse frequency bin.
maximum is the maximum number of allowed hits per coarse bin
histtype specifies the type of histogram you intend to
plot:
'fracmax' is the fraction of spectra that reach the
max (24) for each coarse bin, and the default option,
'%full' is the percent full each coarse bin is after adding
events for all desired spectra, and
'maxcount' is the number of times each bin reaches the max
(24)
where is a string to include additional information to
narrow the results. typically it will be used to specify a
range of specids. each column name MUST be prefixed with
the first letter of the table name and a period, like
c.obstime. do not include the word 'where' at the beginning
or the semicolon at the end. all other common mysql syntax
rules apply. Ex: 'h.specid>1 and h.specid<=20 and
c.beamnum!=8'.
saveplot allows the user to have the figure saved by
inputting the file name. if left empty, no file will be saved
returns a figure"""
import pylab, MySQLFunction, jd2gd, numpy, command
#Data to plot?
if len(bins) != 0:
#Initialize figure
fig=pylab.figure(figsize=(12,7))
ax1 = fig.add_axes([0.1, 0.14, 0.85, 0.75])
#Set axes limits
v = [0,4096,0,max(count)]
ax1.axis(v)
#Configure xticks
xticks = numpy.arange(0,4097,256)
ax1.set_xticks(xticks)
#Rotate xticks
for i in ax1.get_xticklabels(): i.set_rotation(45)
#Add grid
pylab.grid(True)
#Plot data
pylab.bar(bins,count,width=1,align='center')
#Get extra info for plot
if 'c.' not in where:
where = where + ' and h.specid=c.specid'
cmd = command.generate('c.specid,c.obstime','config c, hit h',where=where)
data = MySQLFunction.mysqlcommand(cmd)
#Separate into arrays
length = len(data)
specid = numpy.asarray([data[x][0] for x in range(length)])
time = numpy.asarray([data[x][1] for x in range(length)])
#Get specid count
uniq_IDs = set(specid)
speccount = len(uniq_IDs)
num = sum(count)
#Get hit count and labels
if histtype=='fracmax':
title = 'Fraction of Spectra That Reach Maximum Allowed Hit Count Per Coarse Bin'
hitcount = int(num*speccount)
countlabel = 'Max Count'
if histtype=='%full':
title = 'Percent of Maximum Allowed Hits Per Coarse Bin'
hitcount = int(num*speccount*24)
countlabel = 'Hit Count'
if histtype=='maxcount':
title = 'Number of Times Each Coarse Bin Reached The Maximum Allowed Hit Count'
hitcount = sum(count)
countlabel = 'Max Count'
pylab.xlabel('Coarse Bin Number')
pylab.ylabel('Count')
pylab.title('%s' %title)
#Determine start and end dates
start = min(time)
end = max(time)
#Create Gregorian date from obstime
start = jd2gd.caldate(start)
end = jd2gd.caldate(end)
dates = ['January','February','March','April','May','June','July',
'August','September','October','November','December']
start = [str(start[x]) for x in range(len(start))]
end = [str(end[x]) for x in range(len(end))]
#Insert zeros to make formatting nice
if float(start[2])<10:
start[2] = '0' + start[2]
if float(start[3])<10:
start[3] = '0' + start[3]
if float(start[4])<10:
start[4] = '0' + start[4]
if float(start[5])<10:
start[5] = '0' + start[5]
if float(end[2])<10:
end[2] = '0' + end[2]
if float(end[3])<10:
end[3] = '0' + end[3]
if float(end[4])<10:
end[4] = '0' + end[4]
if float(end[5])<10:
end[5] = '0' + end[5]
#Compile date strings
date1 = start[0]+' '+dates[int(start[1])-1]+' '+start[2]+' '+start[3]+':'+start[4]+':'+start[5][:2]
date2 = end[0]+' '+dates[int(end[1])-1]+' '+end[2]+' '+end[3]+':'+end[4]+':'+end[5][:2]
#Add text to figure
pylab.figtext(0.1,.97,'%s: %d' % (countlabel,hitcount))
pylab.figtext(0.1,.945,'SpecID Count: %s' %speccount)
pylab.figtext(0.95,.97,'Start: %s' %date1, ha='right')
pylab.figtext(0.95,.945,'End: %s' %date2, ha='right')
pylab.figtext(0.1,.92,'Limit: %d' %maximum)
#Save figure?
if saveplot!='':
pylab.savefig('%s'%saveplot)
return fig
