def chk_sched(lines):
# Post-creation check on the schedule to address the fact that
# 3C273 is too close to the Sun from 9/15 to 10/13 each year (day-of-year 259-287).
doy = int(Time(lines[0][:10]).yday[5:8])
next = -1 # Impossible line number
if doy >= 259 and doy <= 287:
# This is a date range when the 27-m antenna should not point at source 1229+020
# Simply replace 1229+020 with 1331+305 (3C286). That is a much weaker source, but it
# may be okay for a PHASECAL
if doy < 272:
dt = (
3000. - (doy - 259) * 240.
) / 86400. # Starts at 50 min and decreases by 4 min each day
else:
dt = 0.
for i in range(len(lines)):
if lines[i].find('1229+020') != -1:
# This line has to be adjusted by dt later and change source to '1331+305'
lines[i] = lines[i].replace('1229+020', '1331+305')
lines[i] = (Time(lines[i][:19]) + dt).iso[:19] + lines[i][19:]
next = i + 1 # Set to next line number, which also has to increment by dt
elif next == i:
lines[i] = (Time(lines[i][:19]) + dt).iso[:19] + lines[i][19:]
next = -1
return lines
def get_projects_nosql(t):
''' Read all projects from FDB file for the current date and return a summary
as a dictionary with keys Timestamp, Project, and EOS (another timestamp)
'''
import dump_tsys as dt
# timerange is 12 UT to 12 UT on next day, relative to the day in Time() object t
trange = Time([int(t.mjd) + 12. / 24, int(t.mjd) + 36. / 24], format='mjd')
tstart = t.iso[2:10].replace('-', '') + '120000'
t2 = Time(int(t.mjd) + 1, format='mjd')
tend = t2.iso[2:10].replace('-', '') + '120000'
fdb = dt.rd_fdb(t)
fdb2 = dt.rd_fdb(t2)
if fdb == {}:
# No FDB file found, so return empty project dictionary
print 'No Project data [FDB file] found for the given date.'
return {}
if fdb == {}:
pass
else:
# Concatenate the two dicts into one
fdb = dict([(k, np.concatenate((fdb.get(k, []), fdb2.get(k, []))))
for k in set(fdb) | set(fdb2)])
# Get "good" indexes for times between 12 UT on date and 12 UT on next date
gidx, = np.where(
np.logical_and(fdb['SCANID'] > tstart, fdb['SCANID'] < tend))
scanid, idx = np.unique(fdb['SCANID'][gidx], return_index=True)
sidx = gidx[idx] # Indexes into fdb for the start of each scan
# Get the project IDs for scans during the period
projdict = {
'Timestamp': fdb['ST_TS'][sidx].astype(float),
'Project': fdb['PROJECTID'][sidx],
'EOS': fdb['EN_TS'][sidx].astype(float)
}
return projdict
def sp_bsize(x, y):
''' Make nice plots of the beamsize.
'''
import matplotlib.pyplot as plt
nfrq, npnt, nant = x['rao'].shape
nrow = 2
ncol = (nant + 1) / 2
f, ax = plt.subplots(nrow, ncol, sharex='col', sharey='row')
f.set_size_inches(2 * nant, 7, forward=True)
t1 = Time(x['ut_mjd'][0], format='mjd')
f.suptitle('X-feed Beam Widths for SOLPNT scan at ' + t1.iso[:19] + ' UT',
fontsize=18)
fout, a, aout = disk_conv()
fgood = np.where(x['fghz'] > 2.48)[0]
for ant in range(nant):
ax[ant % nrow,
ant / nrow].set_title('Ant ' + str(ant + 1) + ' [blue=RA, red=Dec]')
if ant % ncol == 0:
ax[ant / ncol, 0].set_ylabel('Beam FWHM [deg]')
if ant >= ncol:
ax[1, ant - ncol].set_xlabel('Frequency [GHz]')
ax[ant % nrow, ant / nrow].set_ylim(0.5, 5)
ax[ant % nrow, ant / nrow].set_xlim(1, 20)
ax[ant % nrow, ant / nrow].set_yscale('log')
ax[ant % nrow, ant / nrow].set_xscale('log')
ax[ant % nrow, ant / nrow].plot(
x['fghz'][fgood],
x['raparms'][2, fgood, ant] * 2 * np.sqrt(np.log(2)) / 10000, 'b.')
ax[ant % nrow, ant / nrow].plot(fout, aout, color='black')
ax[ant % nrow, ant / nrow].plot(
x['fghz'][fgood],
x['decparms'][2, fgood, ant] * 2 * np.sqrt(np.log(2)) / 10000,
'r.')
plt.draw()
f, ax = plt.subplots(nrow, ncol, sharex='col', sharey='row')
f.set_size_inches(2 * nant, 7, forward=True)
t1 = Time(x['ut_mjd'][0], format='mjd')
f.suptitle('Y-feed Beam Widths for SOLPNT scan at ' + t1.iso[:19] + ' UT',
fontsize=18)
for ant in range(nant):
ax[ant % nrow,
ant / nrow].set_title('Ant ' + str(ant + 1) + ' [blue=RA, red=Dec]')
if ant % ncol == 0:
ax[ant / ncol, 0].set_ylabel('Beam FWHM [deg]')
if ant >= ncol:
ax[1, ant - ncol].set_xlabel('Frequency [GHz]')
ax[ant % nrow, ant / nrow].set_ylim(0.5, 5)
ax[ant % nrow, ant / nrow].set_xlim(1, 20)
ax[ant % nrow, ant / nrow].set_yscale('log')
ax[ant % nrow, ant / nrow].set_xscale('log')
ax[ant % nrow, ant / nrow].plot(
y['fghz'][fgood],
y['raparms'][2, fgood, ant] * 2 * np.sqrt(np.log(2)) / 10000, 'b.')
ax[ant % nrow, ant / nrow].plot(fout, aout, color='black')
ax[ant % nrow, ant / nrow].plot(
y['fghz'][fgood],
y['decparms'][2, fgood, ant] * 2 * np.sqrt(np.log(2)) / 10000,
'r.')
plt.draw()
def sql2refcal(t, lohi=False):
'''Supply a timestamp in Time format, return the closest refcal data'''
import cal_header as ch
import stateframe as stf
if lohi:
caltype = 12
else:
caltype = 8
xml, buf = ch.read_cal(caltype, t=t)
refcal = stf.extract(
buf, xml['Refcal_Real']) + stf.extract(buf, xml['Refcal_Imag']) * 1j
flag = stf.extract(buf, xml['Refcal_Flag'])
fghz = stf.extract(buf, xml['Fghz'])
sigma = stf.extract(buf, xml['Refcal_Sigma'])
timestamp = Time(stf.extract(buf, xml['Timestamp']), format='lv')
tbg = Time(stf.extract(buf, xml['T_beg']), format='lv')
ted = Time(stf.extract(buf, xml['T_end']), format='lv')
pha = np.angle(refcal)
amp = np.absolute(refcal)
return {
'pha': pha,
'amp': amp,
'flag': flag,
'fghz': fghz,
'sigma': sigma,
'timestamp': timestamp,
't_bg': tbg,
't_ed': ted
}
def apply_color_filter_to_image(item,
dir,
save=True,
display=False,
hole_size=1000,
object_size=600):
t = Time()
print('Processing slide: {}'.format(item))
image_path = os.path.join(dir, item)
np_orig = open_image_np(image_path)
item = item.split('.')[0]
filtered_np_img, filtered_mask, filter_np_img_noholes, filtered_mask_noholes = color_filter(
np_orig,
item,
save_dir=FILTER_ING_DIR,
save=False,
display=display,
hole_size=hole_size,
object_size=object_size)
save_suffix = '.png'
item = item + save_suffix
if save:
if not os.path.isdir(FILTER_DIR):
os.makedirs(FILTER_DIR)
if not os.path.isdir(FILTER_MASK_DIR):
os.makedirs(FILTER_MASK_DIR)
if not os.path.isdir(FILTER_NH_DIR):
os.makedirs(FILTER_NH_DIR)
if not os.path.isdir(FILTER_MASK_NH_DIR):
os.makedirs(FILTER_MASK_NH_DIR)
t1 = Time()
filter_path = os.path.join(FILTER_DIR, item)
pil_img = np_to_pil(filtered_np_img)
pil_img.save(filter_path)
print("%-20s | Time: %-14s Name: %s" %
("Save Filtered", str(t1.elapsed()), filter_path))
t1 = Time()
mask_path = os.path.join(FILTER_MASK_DIR, item)
pil_mask = np_to_pil(filtered_mask)
pil_mask.save(mask_path)
print("%-20s | Time: %-14s Name: %s" %
("Save Mask", str(t1.elapsed()), mask_path))
t1 = Time()
filter_nh_path = os.path.join(FILTER_NH_DIR, item)
pil_img_nh = np_to_pil(filter_np_img_noholes)
pil_img_nh.save(filter_nh_path)
print("%-20s | Time: %-14s Name: %s" %
("Save Filtered NoHoles", str(t1.elapsed()), filter_nh_path))
t1 = Time()
mask_nh_path = os.path.join(FILTER_MASK_NH_DIR, item)
pil_mask_nh = np_to_pil(filtered_mask_noholes)
pil_mask_nh.save(mask_nh_path)
print("%-20s | Time: %-14s Name: %s" %
("Save Mask NoHoles", str(t1.elapsed()), mask_nh_path))
def sql2rstn(t=None):
"""This function extracts the RSTN data from SQL with SQL_timestamp
0300 on the date supplied. If the values could be extracted then the
data is returned in a list as follows:
0 - timestamp: Astropy Time which is the date on which the data was
collected. This should match dt
1 - freq: A float32 numpy array containing the 9 frequencies in GHz
2 - data: The flux data which is a 9x7 int16 numpy array.
SQL_timestamp is also returned."""
if t is None: t = Time.now()
sqlt = Time(np.floor(t.mjd) + 0.125, format='mjd')
xml, buf = ch.read_cal(12, sqlt)
if buf is None: return None, None
sqlt_read = Time(extract(buf, xml['SQL_timestamp']), format='lv')
#if np.floor(sqlt.mjd) != np.floor(sqlt_read.mjd): return None, None
data = []
data.append(Time(extract(buf, xml['Timestamp']), format='lv'))
data.append(extract(buf, xml['FGHz']))
data.append(extract(buf, xml['Flux']))
return data, sqlt_read
def get_gain_corr(trange, tref=None, fghz=None):
''' Calls get_gain_state() for a timerange and a reference time,
and returns the gain difference table to apply to data in the
given timerange. If no reference time is provided, the gain
state is referred to the nearest earlier REFCAL.
Returns a dictionary containing:
antgain Array of size (15, 2, 34, nt) = (nant, npol, nbands, nt)
times A Time() object corresponding to the times in
antgain
'''
if tref is None:
# No reference time specified, so get nearest earlier REFCAL
xml, buf = ch.read_cal(8,t=trange[0])
tref = Time(stf.extract(buf,xml['Timestamp']),format='lv')
# Get the gain state at the reference time (actually median over 1 minute)
trefrange = Time([tref.iso,Time(tref.lv+61,format='lv').iso])
ref_gs = get_gain_state(trefrange) # refcal gain state for 60 s
# Get median of refcal gain state (which should be constant anyway)
ref_gs['h1'] = np.median(ref_gs['h1'],1)
ref_gs['h2'] = np.median(ref_gs['h2'],1)
ref_gs['v1'] = np.median(ref_gs['v1'],1)
ref_gs['v2'] = np.median(ref_gs['v2'],1)
# Get the gain state of the requested timerange
src_gs = get_gain_state(trange) # solar gain state for timerange of file
nt = len(src_gs['times'])
antgain = np.zeros((15,2,34,nt),np.float32) # Antenna-based gains vs. band
for i in range(15):
for j in range(34):
antgain[i,0,j] = src_gs['h1'][i] + src_gs['h2'][i] - ref_gs['h1'][i] - ref_gs['h2'][i] + src_gs['dcmattn'][i,0,j] - ref_gs['dcmattn'][i,0,j]
antgain[i,1,j] = src_gs['v1'][i] + src_gs['v2'][i] - ref_gs['v1'][i] - ref_gs['v2'][i] + src_gs['dcmattn'][i,1,j] - ref_gs['dcmattn'][i,1,j]
return {'antgain': antgain, 'times': src_gs['times']}
def unrot_refcal(refcal_in):
''' Apply feed-rotation correction to data read with rd_refcal(), returning updated data in
the same format for further processing.
'''
import dbutil as db
import copy
import chan_util_bc as cu
import cal_header as ch
from stateframe import extract
refcal = copy.deepcopy(refcal_in)
xml, buf = ch.read_cal(11, Time(refcal['times'][0][0], format='jd'))
dph = extract(buf, xml['XYphase'])
xi_rot = extract(buf, xml['Xi_Rot'])
freq = extract(buf, xml['FGHz'])
freq = freq[np.where(freq != 0)]
band = []
for f in freq:
band.append(cu.freq2bdname(f))
bds, sidx = np.unique(band, return_index=True)
nbd = len(bds)
eidx = np.append(sidx[1:], len(band))
dxy = np.zeros((14, 34), dtype=np.float)
xi = np.zeros(34, dtype=np.float)
fghz = np.zeros(34)
# average dph and xi_rot frequencies within each band, to convert to 34-band representation
for b, bd in enumerate(bds):
fghz[bd - 1] = np.nanmean(freq[sidx[b]:eidx[b]])
xi[bd - 1] = np.nanmean(xi_rot[sidx[b]:eidx[b]])
for a in range(14):
dxy[a, bd - 1] = np.angle(np.sum(np.exp(1j * dph[a, sidx[b]:eidx[b]])))
nscans = len(refcal['scanlist'])
for i in range(nscans):
# Read parallactic angles for this scan
trange = Time([refcal['tstlist'][i].iso, refcal['tedlist'][i].iso])
times, chi = db.get_chi(trange)
tchi = times.jd
t = refcal['times'][i]
if len(t) > 0:
vis = copy.deepcopy(refcal['vis'][i])
idx = nearest_val_idx(t, tchi)
pa = chi[idx] # Parallactic angle for the times of this refcal.
pa[:, [8, 9, 10, 12]] = 0.0
nt = len(idx) # Number of times in this refcal
# Apply X-Y delay phase correction
for a in range(13):
a1 = lobe(dxy[a] - dxy[13])
a2 = -dxy[13] - xi
a3 = dxy[a] - xi + np.pi
for j in range(nt):
vis[a, 1, :, j] *= np.exp(1j * a1)
vis[a, 2, :, j] *= np.exp(1j * a2)
vis[a, 3, :, j] *= np.exp(1j * a3)
for j in range(nt):
for a in range(13):
refcal['vis'][i][a, 0, :, j] = vis[a, 0, :, j] * np.cos(pa[j, a]) + vis[a, 3, :, j] * np.sin(pa[j, a])
refcal['vis'][i][a, 2, :, j] = vis[a, 2, :, j] * np.cos(pa[j, a]) + vis[a, 1, :, j] * np.sin(pa[j, a])
refcal['vis'][i][a, 3, :, j] = vis[a, 3, :, j] * np.cos(pa[j, a]) - vis[a, 0, :, j] * np.sin(pa[j, a])
refcal['vis'][i][a, 1, :, j] = vis[a, 1, :, j] * np.cos(pa[j, a]) - vis[a, 2, :, j] * np.sin(pa[j, a])
return refcal
def sp_bsize(x, y):
''' Make nice plots of the beamsize, currently only for the first eight
antennas. This routine will have to be modified to work with more.
'''
import matplotlib.pyplot as plt
nfrq, npnt, nant = x['rao'].shape
f, ax = plt.subplots(2, nant / 2, sharex='col', sharey='row')
f.set_size_inches(2 * nant, 7, forward=True)
t1 = Time(x['ut_mjd'][0], format='mjd')
f.suptitle('X-feed Beam Widths for SOLPNT scan at ' + t1.iso[:19] + ' UT',
fontsize=18)
fout, a, aout = disk_conv()
fgood = np.where(x['fghz'] > 2.48)[0]
for ant in range(nant):
ax[ant % 2,
ant / 2].set_title('Ant ' + str(ant + 1) + ' [blue=RA, red=Dec]')
if ant % 4 == 0:
ax[ant / 4, 0].set_ylabel('Beam FWHM [deg]')
if ant >= nant / 2:
ax[1, ant - nant / 2].set_xlabel('Frequency [GHz]')
ax[ant % 2, ant / 2].set_ylim(0.5, 5)
ax[ant % 2, ant / 2].set_xlim(1, 20)
ax[ant % 2, ant / 2].set_yscale('log')
ax[ant % 2, ant / 2].set_xscale('log')
ax[ant % 2, ant / 2].plot(
x['fghz'][fgood],
x['raparms'][2, fgood, ant] * 2 * np.sqrt(np.log(2)) / 10000, 'b.')
ax[ant % 2, ant / 2].plot(fout, aout, color='black')
ax[ant % 2, ant / 2].plot(
x['fghz'][fgood],
x['decparms'][2, fgood, ant] * 2 * np.sqrt(np.log(2)) / 10000,
'r.')
plt.draw()
f, ax = plt.subplots(2, nant / 2, sharex='col', sharey='row')
f.set_size_inches(2 * nant, 7, forward=True)
t1 = Time(x['ut_mjd'][0], format='mjd')
f.suptitle('Y-feed Beam Widths for SOLPNT scan at ' + t1.iso[:19] + ' UT',
fontsize=18)
for ant in range(nant):
ax[ant % 2,
ant / 2].set_title('Ant ' + str(ant + 1) + ' [blue=RA, red=Dec]')
if ant % 4 == 0:
ax[ant / 4, 0].set_ylabel('Beam FWHM [deg]')
if ant >= nant / 2:
ax[1, ant - nant / 2].set_xlabel('Frequency [GHz]')
ax[ant % 2, ant / 2].set_ylim(0.5, 5)
ax[ant % 2, ant / 2].set_xlim(1, 20)
ax[ant % 2, ant / 2].set_yscale('log')
ax[ant % 2, ant / 2].set_xscale('log')
ax[ant % 2, ant / 2].plot(
y['fghz'][fgood],
y['raparms'][2, fgood, ant] * 2 * np.sqrt(np.log(2)) / 10000, 'b.')
ax[ant % 2, ant / 2].plot(fout, aout, color='black')
ax[ant % 2, ant / 2].plot(
y['fghz'][fgood],
y['decparms'][2, fgood, ant] * 2 * np.sqrt(np.log(2)) / 10000,
'r.')
plt.draw()
def rstntext2sql(startdt, enddt, logfile=None):
"""This routine extracts data from the old archive text file and
writes it to SQL. It will output a list of dates that were not
archived. The SQL time will be the date of the data at 0300.
The program checks to see if there is current data already for
the date range. It will NOT overwrite a record if it is already
present.
startdt and enddt are the dates that will be written to SQL from
startdt up to but not including enddt."""
data = rstnfluxfromtextarchive(startdt, enddt)
if data is None:
print "No RSTN data found in range ", startdt.iso, " to ", enddt.iso
return
print "Processing data from: ", startdt.iso, " to ", enddt.iso
offset = int(np.floor(startdt.mjd))
days = int(np.floor(enddt.mjd)) - offset
processed = np.zeros((days), dtype=bool)
recordswritten = 0
existingrecords = 0
for d in data:
print "Processing Date: ", d[0].iso
i = int(np.floor(d[0].mjd)) - offset
processed[i] = True
sqltime = Time(np.floor(d[0].mjd) + 0.125, format='mjd')
xml, buf = ch.read_cal(12, sqltime)
if buf is not None:
sqltime_read = Time(extract(buf, xml['SQL_timestamp']),
format='lv')
if np.floor(sqltime_read.mjd) != np.floor(sqltime.mjd): buf = None
if buf is None:
if ch.rstnflux2sql(d, sqltime):
recordswritten += 1
print "Record Written"
else:
print "Record Write Failed!"
else:
print "Record Exists."
existingrecords += 1
if logfile is None: logfile = "/tmp/missingrstn.txt"
f = open(logfile, "w")
for i in range(days):
if not processed[i]:
f.write(Time(float(i + offset) + 0.125, format='mjd').iso + "\n")
f.close()
print "Number of days searched: ", days
print "Number of existing records: ", existingrecords
print "Records Written: ", recordswritten
print "Missing Records: ", days - (recordswritten +
existingrecords)
def rstnfluxfromcurrentnoaa():
"""This extracts the first RSTN data block from:
ftp://ftp.swpc.noaa.gov/pub/lists/radio/rad.txt
A list is returned with each element as follows:
0 - timestamp: Astropy Time which is the date on which the data was
collected. This should match dt
1 - freq: A float32 numpy array containing the 9 frequencies in GHz
2 - data: The flux data which is a 9x7 int16 numpy array.
If the ftp failed or the specified date is not found None is returned."""
data = []
success = True
noaa_url = 'ftp://ftp.swpc.noaa.gov/pub/lists/radio/rad.txt'
try:
f = urllib2.urlopen(noaa_url)
lines = f.readlines()
except:
print "Could not read from ", noaa_url
return None
for i, line in enumerate(lines): # Provides counter "i"
line = line.strip()
if line[:3] == 'MHZ':
break
if i + 1 == len(lines):
print "No data found in ", noaa_url
return None
lines = lines[i + 1:]
lines = np.array(lines) # Converts to numpy array, for where()
clean_lines = lines[np.where(lines != '\n')] # Eliminates all empty lines
clean_lines = [l.replace('\n', '')
for l in clean_lines] # Removes \n in remaining lines
datestr = "%04d-%02d-%02d" % (int(
clean_lines[0][0:4]), strptime(clean_lines[0][5:8],
'%b').tm_mon, int(clean_lines[0][9:]))
timestamp = Time(datestr, out_subfmt='date')
t = Time.now()
if np.floor(timestamp.mjd) != np.floor(t.mjd - 1.0):
print "No data found for previous day (", Time(t.mjd - 1.0,
'mjd',
out_subfmt='date'), ")"
return None
data = np.zeros((9, 7), dtype=np.int16)
for j in range(1, 10):
d = clean_lines[j].split()
data[j - 1] = list(map(int, d[1:8]))
return [timestamp, freq, data]
def get_goes_data(t=None, sat_num=None):
''' Reads GOES data from https://umbra.nascom.nasa.gov/ repository, for date
and satellite number provided. If sat_num is None, data for all available
satellites are downloaded, with some sanity check used to decide the best.
If the Time() object t is None, data for the day before the current date
are read (since there is a delay of 1 day in availability of the data).
Returns:
goes_t GOES time array in plot_date format
goes_data GOES 1-8 A lightcurve
'''
from sunpy.util.config import get_and_create_download_dir
import shutil
from astropy.io import fits
import urllib2
if t is None:
t = Time(Time.now().mjd - 1, format='mjd')
yr = t.iso[:4]
datstr = t.iso[:10].replace('-', '')
if sat_num is None:
f = urllib2.urlopen('https://umbra.nascom.nasa.gov/goes/fits/' + yr)
lines = f.readlines()
sat_num = []
for line in lines:
idx = line.find(datstr)
if idx != -1:
sat_num.append(line[idx - 2:idx])
if type(sat_num) is int:
sat_num = [str(sat_num)]
filenames = []
for sat in sat_num:
filename = 'go' + sat + datstr + '.fits'
url = 'https://umbra.nascom.nasa.gov/goes/fits/' + yr + '/' + filename
f = urllib2.urlopen(url)
with open(get_and_create_download_dir() + '/' + filename, 'wb') as g:
shutil.copyfileobj(f, g)
filenames.append(get_and_create_download_dir() + '/' + filename)
pmerit = 0
for file in filenames:
gfits = fits.open(file)
data = gfits[2].data['FLUX'][0][:, 0]
good, = np.where(data > 1.e-8)
tsecs = gfits[2].data['TIME'][0]
merit = len(good)
date_elements = gfits[0].header['DATE-OBS'].split('/')
if merit > pmerit:
print 'File:', file, 'is best'
pmerit = merit
goes_data = data
goes_t = Time(date_elements[2] + '-' + date_elements[1] + '-' +
date_elements[0]).plot_date + tsecs / 86400.
try:
return goes_t, goes_data
except:
print 'No good GOES data for', datstr
return None, None
def pcal_anal(trange, path=None):
import os
import os.path
import socket
import glob
if path is None:
path = ''
out = findfile(trange)
if out is None:
return
host = socket.gethostname()
filelist = out['scanlist']
statuslist = out['status']
starttimelist = out['tstlist']
nscans = len(filelist)
print 'Found', nscans, 'scans to process.'
for i in range(nscans):
good, = np.where(np.array(statuslist[i]) == 'done')
flist = np.array(filelist[i])[good].tolist() # List of "done" files
first_file = filelist[i][0]
last_file = filelist[i][-1]
mjd = fname2mjd(last_file)
tdif = Time.now().mjd - mjd
if len(good) == len(filelist[i]) and tdif > ten_minutes:
# All files in this scan are marked "done", so process the scan only if the plots do not already exist
tmark = fname2mjd(first_file)
tmarkp = tmark + one_minute
tmarkn = tmark - one_minute
tmark = Time(tmark, format='mjd').iso.replace('-', '').replace(
':', '').replace(' ', '')[:12]
tmarkp = Time(tmarkp, format='mjd').iso.replace('-', '').replace(
':', '').replace(' ', '')[:12]
tmarkn = Time(tmarkn, format='mjd').iso.replace('-', '').replace(
':', '').replace(' ', '')[:12]
f1 = glob.glob(path + 'pcT*' + tmark + '*.png')
f2 = glob.glob(path + 'pcT*' + tmarkp + '*.png')
f3 = glob.glob(path + 'pcT*' + tmarkn + '*.png')
if f1 == [] and f2 == [] and f3 == []:
#print 'No files:',tmarkn,tmark,tmarkp,'found.'
print 'Processing completed scan', i + 1
graph(filelist[i], path=path)
else:
print 'Scan processing already complete. Skipping scan', i + 1
elif len(good) == len(filelist[i]) and tdif < ten_minutes:
# All files in this scan are marked "done", but it has been less than 10 min, so process the scan
print 'Processing completed scan', i + 1
graph(flist, path=path)
elif len(good) == len(filelist[i]) - 1:
# This scan is still active, so process all files up to this point.
print 'Processing active scan', i + 1
graph(flist, path=path)
def get_xy_corr(out, doplot=True):
''' Analyze a pair of parallel and cross polarization calibration scans and
return the X vs. Y delay phase corrections on all antennas 1-14.
Required keyword:
out a 2-element array of dicts representing two scans, the first
being the parallel-feed scan, and the second being the
crossed-feed scan.
Optional keyword:
doplot True => plot the final result, False => no plot
'''
if doplot: import matplotlib.pylab as plt
tstr = Time(out[0]['time'][0],format='jd').iso[:19].replace('-','').replace(' ','').replace(':','')
ph0 = np.angle(np.sum(out[0]['x'][ri.bl2ord[:13,13]],3))
ph1 = np.angle(np.sum(out[1]['x'][ri.bl2ord[:13,13]],3))
ph0[:,2:] = ph1[:,2:] # Insert crossed-feed phases from ph1 into ph0
fghz = out[0]['fghz']
nf = len(fghz)
dph = np.zeros((14,nf),np.float)
# Determine xi_rot
xi2 = ph0[:,2] - ph0[:,0] + ph0[:,3] - ph0[:,1] # This is 2 * xi, measured separately on each of 13 antennas
xi_rot = np.unwrap(np.angle(np.sum(np.exp(1j*xi2),0)))/2. # Very clever average does not suffer from wrapping issues
# Form differential delay phase from channels, and average them
# dph14 = XY - XX and YY - YX + pi
#dph14 = np.concatenate((lobe(ph0[:,2] - ph0[:,0] + np.pi/2),lobe(ph0[:,1] - ph0[:,3] - np.pi/2))) # 26 values for Ant 14
#dph[13] = np.angle(np.sum(np.exp(1j*dph14),0)) # Very clever average does not suffer from wrapping issues
# dphi = XX - YX and XY - YY + pi
#dphi = np.array((lobe(ph0[:,0] - ph0[:,3] - np.pi/2),lobe(ph0[:,2] - ph0[:,1] + np.pi/2))) # 2 values for Ant 14
#dph[:13] = np.angle(np.sum(np.exp(1j*dphi),0))
# dph14 = XY - XX - xi_rot and YY - YX + xi_rot
dph14 = np.concatenate((lobe(ph0[:,2] - ph0[:,0] - xi_rot),lobe(ph0[:,1] - ph0[:,3] + xi_rot))) # 26 values for Ant 14
dph[13] = np.angle(np.sum(np.exp(1j*dph14),0)) # Very clever average does not suffer from wrapping issues
# dphi = XX - YX + xi_rot and XY - YY - xi_rot
dphi = np.array((lobe(ph0[:,0] - ph0[:,3] + xi_rot),lobe(ph0[:,2] - ph0[:,1] - xi_rot))) # 2 values for Ant 14
dph[:13] = np.angle(np.sum(np.exp(1j*dphi),0))
if doplot:
f, ax = plt.subplots(4, 4, num='XY_Phase')
ax.shape = (16,)
for i in range(13):
ax[i].plot(fghz,dphi[0,i],'.')
ax[i].plot(fghz,dphi[1,i],'.')
ax[i].plot(fghz,dph[i],'k.')
for i in range(26):
ax[13].plot(fghz,dph14[i],'.')
ax[13].plot(fghz,dph[13],'k.')
for i in range(14): ax[i].set_ylim(-4,4)
f.suptitle('Multicolor: Measurements, Black: Final Results')
np.savez('/common/tmp/Feed_rotation/'+tstr+'_delay_phase.npz',fghz=fghz,dph=dph,xi_rot=xi_rot)
xy_phase = {'timestamp':Time(out[0]['time'][0],format='jd').lv,'fghz':fghz,'xyphase':dph,'xi_rot':xi_rot, 'dphi':dphi, 'dph14':dph14}
return xy_phase
def show_spikes(az, el, filename=None):
naz = len(az)
nel = len(el)
n = naz
if nel > n:
n = nel
if filename == None:
print ' Azimuth | Elevation'
print 'Ant Universal Time Current (mA) | Ant Universal Time Current (mA)'
for i in range(n):
if i < naz:
line = ' %2d %19s %12.3f |' % (
az[i][0], Time(az[i][1], format='lv').iso[:19], az[i][2])
else:
line = ' |'
if i < nel:
line = line + ' %2d %19s %12.3f' % (
el[i][0], Time(el[i][1], format='lv').iso[:19], el[i][2])
else:
line = line + ' '
print line
else:
file1 = open(filename, "w")
file1.write(
' Azimuth | Elevation\n'
)
file1.write(
'Ant Universal Time Current (mA) | Ant Universal Time Current (mA)\n'
)
for i in range(n):
if i < naz:
line = ' %2d %19s %12.3f |' % (
az[i][0], Time(az[i][1], format='lv').iso[:19], az[i][2])
else:
line = ' |'
if i < nel:
line = line + ' %2d %19s %12.3f\n' % (
el[i][0], Time(el[i][1], format='lv').iso[:19], el[i][2])
else:
line = line + ' \n'
file1.write(line)
file1.close()
def __init__(self, trange):
''' Create the object for the specified timerange specified by the
2-element Time() trange. The timerange is used to create a list
of Miriad database files to read, and the data are read.
'''
# Read data
out = dump_tsys.rd_miriad_tsys(trange)
nt, nf, nant = out['xtsys'].shape
self.xdata = out['xtsys']
self.ydata = out['ytsys']
self.fghz = out['fghz']
self.time = Time(out['ut_mjd'], format='mjd')
self.tidx = [0, len(self.time)]
self.fidx = [116, 448]
# Read calibration
fghz, self.calfac, self.offsun = offline.read_calfac(trange[0])
self.bidx = [0, 100]
self.drange = [None, None]
self.antlist = range(nant)
self.cbar = True
self.showants = range(nant)
self.domedian = True
self.docal = True
self.dolog = False
self.dosub = True
self.ax = None
self.version = __version__
def fname2mjd(filename):
fstem = filename.split('/')[-1]
fstr = fstem[3:7] + '-' + fstem[7:9] + '-' + fstem[9:11] + ' ' + fstem[11:13] + ':' + fstem[
13:15] + ':' + fstem[
15:17]
t = Time(fstr)
return t.mjd
def fname2mjd(filename):
from util import Time
fstem = filename.split(os.sep)[-1]
fstr = fstem[2:6] + '-' + fstem[6:8] + '-' + fstem[8:10] + ' ' + fstem[
10:12] + ':' + fstem[12:14] + ':' + fstem[14:16]
print fstr
return Time(fstr).mjd
def sp_write_calfac(x, y, calfac, offsun):
''' Given input dictionaries x, y, and the calibration factors calfac and offsun,
generated by sp_get_calfac(), write standard binary files with the name
TPCALyyyymmdd_a_bbb_c.dat, where a = number of feeds (2),
bbb = number of frequencies,
c = number of antennas.
These values are needed in order to know how to read the file...
The file contains a list of bbb frequencies, and the arrays calfac and offsun,
both of dimensions [a,bbb,c].
'''
fghz = x['fghz']
nf = len(fghz)
t1 = Time(x['ut_mjd'][0], format='mjd')
datstr = t1.iso[:10].replace('-', '')
dims = calfac.shape
siz = np.prod(dims)
buf = struct.pack(str(nf) + 'f', *fghz)
buf += struct.pack(str(siz) + 'f', *calfac.reshape(siz))
buf += struct.pack(str(siz) + 'f', *offsun.reshape(siz))
filename = '/data1/TPCAL/TPCAL' + datstr + '_' + str(dims[0]) + '_' + str(
dims[1]) + '_' + str(dims[2]) + '.dat'
f = open(filename, 'wb')
f.write(buf)
f.close()
return
def filter_blue_pen(rgb, output_type="bool"):
"""
Create a mask to filter out blue pen marks from a slide.
Args:
rgb: RGB image as a NumPy array.
output_type: Type of array to return (bool, float, or uint8).
Returns:
NumPy array representing the mask.
"""
t = Time()
result = filter_blue(rgb, red_upper_thresh=60, green_upper_thresh=120, blue_lower_thresh=190) & \
filter_blue(rgb, red_upper_thresh=120, green_upper_thresh=170, blue_lower_thresh=200) & \
filter_blue(rgb, red_upper_thresh=175, green_upper_thresh=210, blue_lower_thresh=230) & \
filter_blue(rgb, red_upper_thresh=145, green_upper_thresh=180, blue_lower_thresh=210) & \
filter_blue(rgb, red_upper_thresh=37, green_upper_thresh=95, blue_lower_thresh=160) & \
filter_blue(rgb, red_upper_thresh=30, green_upper_thresh=65, blue_lower_thresh=130) & \
filter_blue(rgb, red_upper_thresh=130, green_upper_thresh=155, blue_lower_thresh=180) & \
filter_blue(rgb, red_upper_thresh=40, green_upper_thresh=35, blue_lower_thresh=85) & \
filter_blue(rgb, red_upper_thresh=30, green_upper_thresh=20, blue_lower_thresh=65) & \
filter_blue(rgb, red_upper_thresh=90, green_upper_thresh=90, blue_lower_thresh=140) & \
filter_blue(rgb, red_upper_thresh=60, green_upper_thresh=60, blue_lower_thresh=120) & \
filter_blue(rgb, red_upper_thresh=110, green_upper_thresh=110, blue_lower_thresh=175)
if output_type == "bool":
pass
elif output_type == "float":
result = result.astype(float)
else:
result = result.astype("uint8") * 255
util.np_info(result, "Filter Blue Pen", t.elapsed())
return result
def filter_grays(rgb, tolerance=15, output_type="bool"):
"""
Create a mask to filter out pixels where the red, green, and blue channel values are similar.
Args:
np_img: RGB image as a NumPy array.
tolerance: Tolerance value to determine how similar the values must be in order to be filtered out
output_type: Type of array to return (bool, float, or uint8).
Returns:
NumPy array representing a mask where pixels with similar red, green, and blue values have been masked out.
"""
t = Time()
(h, w, c) = rgb.shape
rgb = rgb.astype(np.int)
rg_diff = abs(rgb[:, :, 0] - rgb[:, :, 1]) <= tolerance
rb_diff = abs(rgb[:, :, 0] - rgb[:, :, 2]) <= tolerance
gb_diff = abs(rgb[:, :, 1] - rgb[:, :, 2]) <= tolerance
result = ~(rg_diff & rb_diff & gb_diff)
if output_type == "bool":
pass
elif output_type == "float":
result = result.astype(float)
else:
result = result.astype("uint8") * 255
util.np_info(result, "Filter Grays", t.elapsed())
return result
def filter_red_pen(rgb, output_type="bool"):
"""
Create a mask to filter out red pen marks from a slide.
Args:
rgb: RGB image as a NumPy array.
output_type: Type of array to return (bool, float, or uint8).
Returns:
NumPy array representing the mask.
"""
t = Time()
result = filter_red(rgb, red_lower_thresh=150, green_upper_thresh=80, blue_upper_thresh=90) & \
filter_red(rgb, red_lower_thresh=110, green_upper_thresh=20, blue_upper_thresh=30) & \
filter_red(rgb, red_lower_thresh=185, green_upper_thresh=65, blue_upper_thresh=105) & \
filter_red(rgb, red_lower_thresh=195, green_upper_thresh=85, blue_upper_thresh=125) & \
filter_red(rgb, red_lower_thresh=220, green_upper_thresh=115, blue_upper_thresh=145) & \
filter_red(rgb, red_lower_thresh=125, green_upper_thresh=40, blue_upper_thresh=70) & \
filter_red(rgb, red_lower_thresh=200, green_upper_thresh=120, blue_upper_thresh=150) & \
filter_red(rgb, red_lower_thresh=100, green_upper_thresh=50, blue_upper_thresh=65) & \
filter_red(rgb, red_lower_thresh=85, green_upper_thresh=25, blue_upper_thresh=45)
if output_type == "bool":
pass
elif output_type == "float":
result = result.astype(float)
else:
result = result.astype("uint8") * 255
util.np_info(result, "Filter Red Pen", t.elapsed())
return result
def filter_blue(rgb, red_upper_thresh, green_upper_thresh, blue_lower_thresh, output_type="bool",
display_np_info=False):
"""
Create a mask to filter out blueish colors, where the mask is based on a pixel being below a
red channel threshold value, below a green channel threshold value, and above a blue channel threshold value.
Args:
rgb: RGB image as a NumPy array.
red_upper_thresh: Red channel upper threshold value.
green_upper_thresh: Green channel upper threshold value.
blue_lower_thresh: Blue channel lower threshold value.
output_type: Type of array to return (bool, float, or uint8).
display_np_info: If True, display NumPy array info and filter time.
Returns:
NumPy array representing the mask.
"""
if display_np_info:
t = Time()
r = rgb[:, :, 0] < red_upper_thresh
g = rgb[:, :, 1] < green_upper_thresh
b = rgb[:, :, 2] > blue_lower_thresh
result = ~(r & g & b)
if output_type == "bool":
pass
elif output_type == "float":
result = result.astype(float)
else:
result = result.astype("uint8") * 255
if display_np_info:
util.np_info(result, "Filter Blue", t.elapsed())
return result
def filter_binary_dilation(np_img, disk_size=5, iterations=1, output_type="uint8"):
"""
Dilate a binary object (bool, float, or uint8).
Args:
np_img: Binary image as a NumPy array.
disk_size: Radius of the disk structuring element used for dilation.
iterations: How many times to repeat the dilation.
output_type: Type of array to return (bool, float, or uint8).
Returns:
NumPy array (bool, float, or uint8) where edges have been dilated.
"""
t = Time()
if np_img.dtype == "uint8":
np_img = np_img / 255
result = sc_morph.binary_dilation(np_img, sk_morphology.disk(disk_size), iterations=iterations)
if output_type == "bool":
pass
elif output_type == "float":
result = result.astype(float)
else:
result = result.astype("uint8") * 255
util.np_info(result, "Binary Dilation", t.elapsed())
return result
def filter_binary_closing(np_img, disk_size=3, iterations=1, output_type="uint8"):
"""
Close a binary object (bool, float, or uint8). Closing is a dilation followed by an erosion.
Closing can be used to remove small holes.
Args:
np_img: Binary image as a NumPy array.
disk_size: Radius of the disk structuring element used for closing.
iterations: How many times to repeat.
output_type: Type of array to return (bool, float, or uint8).
Returns:
NumPy array (bool, float, or uint8) following binary closing.
"""
t = Time()
if np_img.dtype == "uint8":
np_img = np_img / 255
result = sc_morph.binary_closing(np_img, sk_morphology.disk(disk_size), iterations=iterations)
if output_type == "bool":
pass
elif output_type == "float":
result = result.astype(float)
else:
result = result.astype("uint8") * 255
util.np_info(result, "Binary Closing", t.elapsed())
return result
def filter_remove_small_holes(np_img, min_size=3000, output_type="uint8"):
"""
Filter image to remove small holes less than a particular size.
Args:
np_img: Image as a NumPy array of type bool.
min_size: Remove small holes below this size.
output_type: Type of array to return (bool, float, or uint8).
Returns:
NumPy array (bool, float, or uint8).
"""
t = Time()
rem_sm = sk_morphology.remove_small_holes(np_img, min_size=min_size)
if output_type == "bool":
pass
elif output_type == "float":
rem_sm = rem_sm.astype(float)
else:
rem_sm = rem_sm.astype("uint8") * 255
util.np_info(rem_sm, "Remove Small Holes", t.elapsed())
return rem_sm
def writesfdata_anta(fld,
trange,
outfile,
delim=" ",
ignore=None,
interval=None):
data, msg = loadsfdata_anta(fld, trange, interval)
print msg
if msg != "Success":
return
dt = Time(data['Timestamp'].astype(float), format='lv').iso
lines = []
for i, t in enumerate(dt):
l = (t[0:19])
mask = False
for f in fld:
if ignore != None:
if data[f][i] == ignore:
mask = True
break
else:
l = l + delim + str(data[f][i])
else:
l = l + delim + str(data[f][i])
if not mask:
lines.append(l)
if len(l) > 0:
with open(outfile, 'w') as filehandle:
filehandle.writelines("%s\n" % l for l in lines)
def get_reboot(trange, previous=False):
''' Get the times of any ROACH (correlator) reboots in the given timerange
Returns Time() object of reboots, or None. If previous is True, it
returns the time of the previous reboot.
'''
import numpy as np
t0, t1 = trange.lv.astype(np.int)
tmjd = trange[0].mjd
cursor = get_cursor()
ver = find_table_version(cursor, t0, scan_header=True)
query = 'select Timestamp,TimeAtAcc0 from hV' + ver + '_vD1 where Timestamp between ' + str(
t0) + ' and ' + str(t1) + ' order by Timestamp'
data, msg = do_query(cursor, query)
t0, idx = np.unique(data['TimeAtAcc0'], return_index=True)
t_reboot = data['TimeAtAcc0'][idx].astype(float)
if previous:
pass
else:
if t_reboot.size == 1:
if t_reboot < tmjd:
return None
elif t_reboot[0] < tmjd:
t_reboot = t_reboot[1:]
return Time(t_reboot, format='mjd')
def a14_wscram(trange):
''' Get the Antenna 14 windscram state, and the average wind speed, for a
given time range.
Returns:
times as a Time() object, or error message if failure
wscram array of windscram state, 0 = not in wind scram, 1 = in windscram
avgwind array of average wind speeds, in MPH, or error message if failure
'''
tstart, tend = [str(i) for i in trange.lv]
cursor = get_cursor()
ver = find_table_version(cursor, trange[0].lv)
query = 'select Timestamp,Ante_Fron_Wind_State from fV' + ver + '_vD15 where (I15 = 13) and Timestamp between ' + tstart + ' and ' + tend
data, msg = do_query(cursor, query)
if msg == 'Success':
try:
times = Time(data['Timestamp'].astype('int'), format='lv')
wscram = data['Ante_Fron_Wind_State']
except:
return 'Error: Unknown Error', None, None
else:
return 'Error: ' + msg, None, None
query = 'select Timestamp,Sche_Data_Weat_AvgWind from fV' + ver + '_vD1 where Timestamp between ' + tstart + ' and ' + tend
data, msg = do_query(cursor, query)
if msg == 'Success':
avgwind = data['Sche_Data_Weat_AvgWind']
else:
return times, wscram, 'Error: ' + msg
cursor.close()
return times, wscram, avgwind
def get_sql_info(trange):
''' Get all antenna information from the SQL database for a given
timerange, including TrackFlag and Parallactic Angle
Also determines if the RFSwitch state (i.e. which 27-m receiver
is being used).
'''
cursor = db.get_cursor()
sqldict = db.get_dbrecs(cursor, dimension=15, timestamp=trange)
azeldict = stateframe.azel_from_sqldict(sqldict)
time = Time(sqldict['Timestamp'][:, 0].astype(int), format='lv')
azeldict.update({'Time': time})
sqldict = db.get_dbrecs(cursor, dimension=1, timestamp=trange)
azeldict.update({'RFSwitch': sqldict['FEMA_Powe_RFSwitchStatus']})
azeldict.update({'LF_Rcvr': sqldict['FEMA_Rece_LoFreqEnabled']})
if np.median(azeldict['RFSwitch']) == 0.0 and np.median(
azeldict['LF_Rcvr']) == 1.0:
azeldict.update({'Receiver': 'Low'})
elif np.median(azeldict['RFSwitch']) == 1.0 and np.median(
azeldict['LF_Rcvr']) == 0.0:
azeldict.update({'Receiver': 'High'})
else:
azeldict.update({'Receiver': 'Unknown'})
cursor.close()
return azeldict
