def test_transit_delta_t():
t_start = 21351.34
t1 = ephemeris.transit_times(123.0, t_start)[0]
t2 = ephemeris.transit_times(125.0, t_start)[0]
delta = (t2 - t1) % 86400
assert delta == approx(8.0 * 60.0 * ephemeris.STELLAR_S, abs=0.1)
def test_transit_sources():
# Check at an early time (this is close to the UNIX epoch)
t_start = 12315.123
t1 = ephemeris.transit_times(350.8664, t_start)[0] # This is CasA's RA
t2 = ephemeris.transit_times(ephemeris.CasA, t_start)[0]
# Due to precession of the polar axis this only matches within ~30s
assert t1 == approx(t2, abs=30)
# Check at an early time (this is close to the UNIX epoch)
t_start = datetime(2001, 1, 1)
t1 = ephemeris.transit_times(350.8664, t_start)[0] # This is CasA's RA
t2 = ephemeris.transit_times(ephemeris.CasA, t_start)[0]
# This is the J2000 epoch and so the precession should be ~0.
assert t1 == approx(t2, abs=0.1)
def print_timestamp(fn):
src_dict = {'CasA': eph.CasA, 'TauA': eph.TauA, 'CygA': eph.CygA, 'VirA': eph.VirA,
'1929': 292.0, '0329': 54.0}
try:
f = h5py.File(fn, 'r')
except IOError:
print "File couldn't be opened"
return
times = f['index_map']['time'].value['ctime'][:]
print "-------------------------------------"
print "start time %s in PT\n" % (eph.unix_to_datetime(times[0]))
print "RA range %f : %f" % (np.round(eph.transit_RA(times[0]), 1),
np.round(eph.transit_RA(times[-1]),1))
print "-------------------------------------"
srcz = []
for src in src_dict:
if eph.transit_times(src_dict[src], times[0])[0] < times[-1]:
print "%s is in this file" % src
srcz.append(src)
return srcz
def test_transit_against_transit_ra():
t_start = 65422.2
ra = 234.54234
t = ephemeris.transit_times(ra, t_start)[0]
ra_back_calculated = ephemeris.lsa(t)
assert ra == approx(ra_back_calculated, abs=1e-2)
def fs_from_file(filename, frq, src,
del_t=900, transposed=True, subtract_avg=False):
f = h5py.File(filename, 'r')
times = f['index_map']['time'].value['ctime'] + 10.6
src_trans = eph.transit_times(src, times[0])
# try to account for differential arrival time from cylinder rotation.
del_phi = (src._dec - np.radians(eph.CHIMELATITUDE)) * np.sin(np.radians(1.988))
del_phi *= (24 * 3600.0) / (2 * np.pi)
# Adjust the transit time accordingly
src_trans += del_phi
# Select +- del_t of transit, accounting for the mispointing
t_range = np.where((times < src_trans + del_t) & (times > src_trans - del_t))[0]
times = times[t_range[0]:t_range[-1]]#[offp::2] test
print "Time range:", times[0], times[-1]
print "\n...... This data is from %s starting at RA: %f ...... \n" \
% (eph.unix_to_datetime(times[0]), eph.transit_RA(times[0]))
if transposed is True:
v = f['vis'][frq[0]:frq[-1]+1, :]
v = v[..., t_range[0]:t_range[-1]]
vis = v['r'] + 1j * v['i']
del v
# Read in time and freq slice if data has not yet been transposed
if transposed is False:
v = f['vis'][t_range[0]:t_range[-1], frq[0]:frq[-1]+1, :]
vis = v['r'][:] + 1j * v['i'][:]
del v
vis = np.transpose(vis, (1, 2, 0))
inp = gen_inp()[0]
# Remove offset from galaxy
if subtract_avg is True:
vis -= 0.5 * (vis[..., 0] + vis[..., -1])[..., np.newaxis]
freq_MHZ = 800.0 - np.array(frq) / 1024.0 * 400.
print len(inp)
baddies = np.where(np.isnan(tools.get_feed_positions(inp)[:, 0]))[0]
# Fringestop to location of "src"
data_fs = tools.fringestop_pathfinder(vis, eph.transit_RA(times), freq_MHZ, inp, src)
# data_fs = fringestop_pathfinder(vis, eph.transit_RA(times), freq_MHZ, inp, src)
return data_fs
def transit_flag(name, time, nsig=1.0, freq=400.0, pol='X'):
flag = np.zeros(time.size, dtype=np.bool)
if name.lower() == 'sun':
window_sec = nsig * utils.get_window(freq, pol=pol, dec=np.radians(23.45), deg=False)
ttrans = ephemeris.solar_transit(time[0] - 24.0 * 3600.0, time[-1] + 24.0 * 3600.0)
else:
src = FluxCatalog[name].skyfield
window_sec = nsig * utils.get_window(freq, pol=pol, dec=np.radians(FluxCatalog[name].dec), deg=False)
ttrans = ephemeris.transit_times(src, time[0] - 24.0 * 3600.0, time[-1] + 24.0 * 3600.0)
for tt in ttrans:
# Flag +/- window_sec around peak location
begin = tt - window_sec
end = tt + window_sec
flag |= ((time >= begin) & (time <= end))
return flag
def print_timestamp(fn):
src_dict = {
'CasA': eph.CasA,
'TauA': eph.TauA,
'CygA': eph.CygA,
'VirA': eph.VirA,
'1929': 292.0,
'0329': 54.0
}
try:
f = h5py.File(fn, 'r')
except IOError:
print "File couldn't be opened"
return
times = f['index_map']['time'].value['ctime'][:]
print "-------------------------------------"
print "start time %s in PT\n" % (eph.unix_to_datetime(times[0]))
print "RA range %f : %f" % (np.round(eph.transit_RA(
times[0]), 1), np.round(eph.transit_RA(times[-1]), 1))
print "-------------------------------------"
srcz = []
for src in src_dict:
if eph.transit_times(src_dict[src], times[0])[0] < times[-1]:
print "%s is in this file" % src
srcz.append(src)
return srcz
def get_sunfree_srcs(self, srcs=None):
"""This method uses the attributes 'night_acq_list' and
'acq_list' to determine the srcs that transit
in the available data. If these attributes do not
exist, then the method 'set_acq_list' is called.
If srcs is not specified, then it defaults to the
brightest four radio point sources in the sky:
CygA, CasA, TauA, and VirA.
"""
if self.acq_list is None:
self.set_acq_list()
if srcs is None:
srcs = [
ephemeris.CygA, ephemeris.CasA, ephemeris.TauA, ephemeris.VirA
]
Ns = len(srcs)
clr = [False] * Ns
ntt = [False] * Ns # night transit
for ii, src in enumerate(srcs):
night_transit = np.array([])
for acq in self.night_acq_list:
night_transit = np.append(
night_transit, ephemeris.transit_times(src, *acq[1]))
if night_transit.size:
ntt[ii] = True
if src.name in ["CygA", "CasA"]:
transit = np.array([])
for acq in self.acq_list:
transit = np.append(transit,
ephemeris.transit_times(src, *acq[1]))
if transit.size:
clr[ii] = True
else:
clr[ii] = ntt[ii]
return clr, ntt, srcs
def test_ha_src(self):
times = self.times
ha_src = pv.ha_src(times, self.RA_src)
diff_true = np.round(np.mod(eph.CHIMELONGITUDE - (-78.0730), 360))
trans_time = eph.transit_times(self.RA_src + diff_true, times[0])
ha_src_trans = np.mod(np.round(pv.ha_src(trans_time, self.RA_src)), 360)
ha_src_pretrans = pv.ha_src(trans_time - 1000, self.RA_src)
assert ha_src_trans == 0.0
assert (ha_src < 360).all()
assert ha_src_pretrans < 0.0
def noise_src_phase(fn_ns_sol, fn_sky_sol, src=eph.CasA, trb=10):
""" Get noise source phase at the time sky solution was calculated
(e.g. time of CasA transit) and use that as baseline.
Parameters
----------
fn_ns_sol : np.str
file name with noise source solution
fn_sky_sol : np.str
file name with sky solution
src : ephemeris.Object
object that gave sky solution in fn_sky_sol
tbr : np.int
time rebin factor
Returns
-------
phase solution : array_like
(nfeed, nt) real array of phases
"""
f = h5py.File(fn_ns_sol, 'r')
fsky = h5py.File(fn_sky_sol, 'r')
gx = fsky['gainsx'][:]
gy = fsky['gainsy'][:]
gains_ns = f['gains'][:]
feeds = f['channels'][:]
freq = f['freq_bins'][:]
toff = f['timestamp_off']
nt = len(toff)
nfeed = len(feeds)
nfreq = len(freq)
toff = toff[::10]
gains_sky = cp.construct_gain_mat(gx, gy, 64)
gains_sky = gains_sky[freq, feeds]
gains_ns = gains_ns[..., :nt // trb * trb].reshape(nfreq, nfeed, -1,
trb).mean(-1)
src_trans = eph.transit_times(src, toff[0])
trans_pix = np.argmin(abs(src_trans - toff))
phase = np.angle(gains_ns) - np.angle(gains_ns[..., trans_pix, np.newaxis])
return np.angle(gains_sky)[:, np.newaxis] + phase
def find_transit_file(dir_nm,
unix_time=None,
src=None,
trans=True,
verbose=True):
flist = glob.glob(dir_nm)
tdel = []
print "Looking at %d files" % len(flist)
for ii in range(len(flist)):
if trans is True:
try:
r = andata.Reader(flist[ii])
if ii == 0 and src != None:
time_trans = eph.transit_times(src, r.time[0])
elif unix_time != None:
time_trans = unix_time
else:
continue
tdel.append(np.min(abs(time_trans - r.time)))
if verbose is True:
print flist[ii]
print "%f hours away \n" % np.min(
abs(time_trans - r.time)) / 3600.0
except (KeyError, IOError):
print "That one didn't work"
elif trans is False:
try:
f = h5py.File(flist[ii], 'r')
times = f['index_map']['time'].value['ctime'][0]
f.close()
time_trans = unix_time
tdel.append(np.min(abs(time_trans - times)))
except (ValueError, IOError):
pass
tdel = np.array(tdel)
return flist[np.argmin(tdel)], tdel.min()
def noise_src_phase(fn_ns_sol, fn_sky_sol, src=eph.CasA, trb=10):
""" Get noise source phase at the time sky solution was calculated
(e.g. time of CasA transit) and use that as baseline.
Parameters
----------
fn_ns_sol : np.str
file name with noise source solution
fn_sky_sol : np.str
file name with sky solution
src : ephemeris.Object
object that gave sky solution in fn_sky_sol
tbr : np.int
time rebin factor
Returns
-------
phase solution : array_like
(nfeed, nt) real array of phases
"""
f = h5py.File(fn_ns_sol, 'r')
fsky = h5py.File(fn_sky_sol, 'r')
gx = fsky['gainsx'][:]
gy = fsky['gainsy'][:]
gains_ns = f['gains'][:]
feeds = f['channels'][:]
freq = f['freq_bins'][:]
toff = f['timestamp_off']
nt = len(toff)
nfeed = len(feeds)
nfreq = len(freq)
toff = toff[::10]
gains_sky = cp.construct_gain_mat(gx, gy, 64)
gains_sky = gains_sky[freq, feeds]
gains_ns = gains_ns[..., :nt // trb * trb].reshape(nfreq, nfeed, -1, trb).mean(-1)
src_trans = eph.transit_times(src, toff[0])
trans_pix = np.argmin(abs(src_trans - toff))
phase = np.angle(gains_ns) - np.angle(gains_ns[..., trans_pix, np.newaxis])
return np.angle(gains_sky)[:, np.newaxis] + phase
def find_transit_file(dir_nm, unix_time=None, src=None, trans=True, verbose=True):
flist = glob.glob(dir_nm)
tdel = []
print "Looking at %d files" % len(flist)
for ii in range(len(flist)):
if trans is True:
try:
r = andata.Reader(flist[ii])
if ii==0 and src != None:
time_trans = eph.transit_times(src, r.time[0])
elif unix_time != None:
time_trans = unix_time
else:
continue
tdel.append(np.min(abs(time_trans - r.time)))
if verbose is True:
print flist[ii]
print "%f hours away \n" % np.min(abs(time_trans - r.time)) / 3600.0
except (KeyError, IOError):
print "That one didn't work"
elif trans is False:
try:
f = h5py.File(flist[ii], 'r')
times = f['index_map']['time'].value['ctime'][0]
f.close()
time_trans = unix_time
tdel.append(np.min(abs(time_trans - times)))
except (ValueError, IOError):
pass
tdel = np.array(tdel)
return flist[np.argmin(tdel)], tdel.min()
def read_data(reader_obj, src, prod_sel, freq_sel=None, del_t=50):
""" Use andata tools to select freq, products,
and times.
"""
R = reader_obj
# Figure out when calibration source transits
src_trans = eph.transit_times(src, R.time[0])
# Select +-100 seconds of transit
time_range = np.where((R.time < src_trans + del_t) & (R.time > src_trans - del_t))[0]
R.time_sel = [time_range[0], time_range[-1]]
R.prod_sel = prod_sel
R.freq_sel = freq_sel
and_obj = R.read()
return and_obj
def read_data(reader_obj, src, prod_sel, freq_sel=None, del_t=50):
""" Use andata tools to select freq, products,
and times.
"""
R = reader_obj
# Figure out when calibration source transits
src_trans = eph.transit_times(src, R.time[0])
# Select +-100 seconds of transit
time_range = np.where((R.time < src_trans + del_t)
& (R.time > src_trans - del_t))[0]
R.time_sel = [time_range[0], time_range[-1]]
R.prod_sel = prod_sel
R.freq_sel = freq_sel
and_obj = R.read()
return and_obj
def find_transit(time0='Now', src=eph.CasA):
import time
if time0 == 'Now':
time0 = time.time()
# Get reference datetime from unixtime
dt_now = eph.unix_to_datetime(time0)
dt_now = dt_now.isoformat()
# Only use relevant characters in datetime string
dt_str = dt_now.replace("-", "")[:7]
dirnm = '/mnt/gong/archive/' + dt_str
filelist = glob.glob(dirnm + '*/*h5')
# Step through each file and find that day's transit
for ff in filelist:
try:
andataReader = andata.Reader(ff)
acqtimes = andataReader.time
trans_time = eph.transit_times(src, time0)
#print ff
# print eph.transit_RA(trans_time), eph.transit_RA(acqtimes[0])
del andataReader
if np.abs(acqtimes - trans_time[0]).min() < 1000.0:
# print "On ", eph.unix_to_datetime(trans_time[0])
# print "foundit in %s \n" % ff
return ff
break
except (KeyError, ValueError, IOError):
pass
return None
def find_transit(time0='Now', src=eph.CasA):
import time
if time0 == 'Now':
time0 = time.time()
# Get reference datetime from unixtime
dt_now = eph.unix_to_datetime(time0)
dt_now = dt_now.isoformat()
# Only use relevant characters in datetime string
dt_str = dt_now.replace("-", "")[:7]
dirnm = '/mnt/gong/archive/' + dt_str
filelist = glob.glob(dirnm + '*/*h5')
# Step through each file and find that day's transit
for ff in filelist:
try:
andataReader = andata.Reader(ff)
acqtimes = andataReader.time
trans_time = eph.transit_times(src, time0)
#print ff
# print eph.transit_RA(trans_time), eph.transit_RA(acqtimes[0])
del andataReader
if np.abs(acqtimes - trans_time[0]).min() < 1000.0:
# print "On ", eph.unix_to_datetime(trans_time[0])
# print "foundit in %s \n" % ff
return ff
break
except (KeyError, ValueError, IOError):
pass
return None
def fringestop_and_sum(fn, feeds, freq, src, transposed=True,
return_unfs=True, meridian=False, del_t=1500, frick=None):
""" Take an input file fn and a set of feeds and return
a formed beam on src.
"""
if transposed is True:
r = andata.Reader(fn)
r.freq_sel = freq
X = r.read()
times = r.time
else:
f = h5py.File(fn, 'r')
times = f['index_map']['time'].value['ctime']
print "Read in data"
# Get transit time for source
src_trans = eph.transit_times(src, times[0])
del_phi = 1.30 * (src._dec - np.radians(eph.CHIMELATITUDE)) * np.sin(np.radians(1.988))
del_phi *= (24 * 3600.0) / (2 * np.pi)
# Adjust the transit time accordingly
src_trans += del_phi
# Select +- del_t of transit, accounting for the mispointing
t_range = np.where((times < src_trans + del_t) & (times > src_trans - del_t))[0]
times = times[t_range[0]:t_range[-1]]#[offp::2] test
print "Time range:", times[0], times[-1]
# Generate correctly ordered corrinputs
corrinput_real = gen_inp()[0]
inp = np.array(corrinput_real)
# Ensure vis array is in correct order (freq, prod, time)
if transposed is True:
data = X.vis[:, :, t_range[0]:t_range[-1]]
freq = X.freq
else:
v = f['vis'][t_range[0]:t_range[-1], freq, :]
vis = v['r'] + 1j * v['i']
data = vis.transpose()[np.newaxis]
del vis
freq = 800 - 400.0 / 1024 * freq
freq = np.array([freq])
# autos = auto_corrs(256)
# offp = (abs(data[:, autos, 0::2]).mean() > (abs(data[:, autos, 1::2]).mean())).astype(int)
# data = data[..., offp::2] test
data_unfs = sum_corrs(data.copy(), feeds)
ra_ = eph.transit_RA(times)
ra_2 = nolan_ra(times)
#ra_ = ra_2.copy()
if meridian is True:
ra = np.ones_like(ra_) * np.degrees(src._ra)
else:
ra = ra_
print len(inp)
dfs = tools.fringestop_pathfinder(data.copy(), ra, freq, inp, src)
#dfs = fringestop_pathfinder(data.copy(), ra_2, freq, inp, src, frick=frick)
# dfs = fringestop_pathfinder(data.copy(), ra_1, freq, inp, src, frick=frick)
# fp = np.loadtxt('/home/connor/feed_layout_decrease.txt')
# PH = fill_nolan(times, src._ra * 180.0 / np.pi, src._dec * 180.0 / np.pi, fp)
dfs_sum = sum_corrs(dfs, feeds)
if return_unfs is True:
return dfs_sum, ra_, dfs, data
else:
return dfs_sum, ra_
def process(self, tstream):
"""Apply the timing correction to the input timestream.
Parameters
----------
tstream : andata.CorrData, containers.TimeStream, or containers.SiderealStream
Apply the timing correction to the visibilities stored in this container.
Returns
-------
tstream_corr : same as `tstream`
Timestream with corrected visibilities.
"""
# Determine times
if "time" in tstream.index_map:
timestamp = tstream.time
else:
csd = (tstream.attrs["lsd"]
if "lsd" in tstream.attrs else tstream.attrs["csd"])
timestamp = ephemeris.csd_to_unix(csd + tstream.ra / 360.0)
# Extract local frequencies
tstream.redistribute("freq")
nfreq = tstream.vis.local_shape[0]
sfreq = tstream.vis.local_offset[0]
efreq = sfreq + nfreq
freq = tstream.freq[sfreq:efreq]
# If requested, extract the input flags
input_flags = tstream.input_flags[:] if self.use_input_flags else None
# Check needs_timing_correction flags time ranges to see if input timestream
# falls within the interval
needs_timing_correction = False
for flag in self.flags:
if (timestamp[0] >= flag["start_time"]
and timestamp[0] <= flag["finish_time"]):
if timestamp[-1] >= flag["finish_time"]:
raise PipelineRuntimeError(
f"Data covering {timestamp[0]} to {timestamp[-1]} partially overlaps "
f"needs_timing_correction DataFlag covering {ephemeris.unix_to_datetime(flag['start_time']).strftime('%Y%m%dT%H%M%SZ')} "
f"to {ephemeris.unix_to_datetime(flag['finish_time']).strftime('%Y%m%dT%H%M%SZ')}."
)
else:
self.log.info(
f"Data covering {timestamp[0]} to {timestamp[-1]} flagged by "
f"needs_timing_correction DataFlag covering {ephemeris.unix_to_datetime(flag['start_time']).strftime('%Y%m%dT%H%M%SZ')} "
f"to {ephemeris.unix_to_datetime(flag['finish_time']).strftime('%Y%m%dT%H%M%SZ')}. Timing correction will be applied."
)
needs_timing_correction = True
break
if not needs_timing_correction:
self.log.info(
f"Data in span {ephemeris.unix_to_datetime(timestamp[0]).strftime('%Y%m%dT%H%M%SZ')} to {ephemeris.unix_to_datetime(timestamp[-1]).strftime('%Y%m%dT%H%M%SZ')} does not need timing correction"
)
return tstream
# Find the right timing correction
for tcorr in self.tcorr:
if timestamp[0] >= tcorr.time[0] and timestamp[-1] <= tcorr.time[
-1]:
break
else:
msg = (
"Could not find timing correction file covering "
"range of timestream data (%s to %s)." % tuple(
ephemeris.unix_to_datetime([timestamp[0], timestamp[-1]])))
if self.pass_if_missing:
self.log.warning(msg + " Doing nothing.")
return tstream
raise RuntimeError(msg)
self.log.info("Using correction file %s" % tcorr.attrs["tag"])
# If requested, reference the timing correct with respect to source transit time
if self.refer_to_transit:
# First check for transit_time attribute in file
ttrans = tstream.attrs.get("transit_time", None)
if ttrans is None:
source = tstream.attrs["source_name"]
ttrans = ephemeris.transit_times(
ephemeris.source_dictionary[source],
tstream.time[0],
tstream.time[-1],
)
if ttrans.size != 1:
raise RuntimeError(
"Found %d transits of %s in timestream. "
"Require single transit." % (ttrans.size, source))
else:
ttrans = ttrans[0]
self.log.info(
"Referencing timing correction to %s (RA=%0.1f deg)." % (
ephemeris.unix_to_datetime(ttrans).strftime(
"%Y%m%dT%H%M%SZ"),
ephemeris.lsa(ttrans),
))
tcorr.set_global_reference_time(ttrans,
window=self.transit_window,
interpolate=True,
interp="linear")
# Apply the timing correction
tcorr.apply_timing_correction(tstream,
time=timestamp,
freq=freq,
input_flags=input_flags,
copy=False)
return tstream
def solve_ps_transit(filename, corrs, feeds, inp,
src, nfreq=1024, transposed=False, nfeed=128):
""" Function that fringestops time slice
where point source is in the beam, takes
all correlations for a given polarization, and then
eigendecomposes the correlation matrix freq by freq
after removing the fpga phases. It will also
plot intermediate steps to verify the phase solution.
Parameters
----------
filename : np.str
Full-path filename
corrs : list
List of correlations to use in solver
feeds : list
List of feeds to use
inp :
Correlator inputs (output of ch_util.tools.get_correlator_inputs)
src : ephem.FixedBody
Source to calibrate off of. e.g. ch_util.ephemeris.TauA
Returns
-------
Gains : np.array
Complex gain array (nfreq, nfeed)
"""
nsplit = 32 # Number of freq chunks to divide nfreq into
del_t = 800
f = h5py.File(filename, 'r')
# Add half an integration time to each. Hack.
times = f['index_map']['time'].value['ctime'] + 10.50
src_trans = eph.transit_times(src, times[0])
# try to account for differential arrival time from
# cylinder rotation.
del_phi = (src._dec - np.radians(eph.CHIMELATITUDE)) \
* np.sin(np.radians(1.988))
del_phi *= (24 * 3600.0) / (2 * np.pi)
# Adjust the transit time accordingly
src_trans += del_phi
# Select +- del_t of transit, accounting for the mispointing
t_range = np.where((times < src_trans +
del_t) & (times > src_trans - del_t))[0]
print "\n...... This data is from %s starting at RA: %f ...... \n" \
% (eph.unix_to_datetime(times[0]), eph.transit_RA(times[0]))
assert (len(t_range) > 0), "Source is not in this acq"
# Create gains array to fill in solution
Gains = np.zeros([nfreq, nfeed], np.complex128)
print "Starting the solver"
times = times[t_range[0]:t_range[-1]]
k=0
# Start at a strong freq channel that can be plotted
# and from which we can find the noise source on-sample
for i in range(12, nsplit) + range(0, 12):
k+=1
# Divides the arrays up into nfreq / nsplit freq chunks and solves those
frq = range(i * nfreq // nsplit, (i+1) * nfreq // nsplit)
print " %d:%d \n" % (frq[0], frq[-1])
# Read in time and freq slice if data has already been transposed
if transposed is True:
v = f['vis'][frq[0]:frq[-1]+1, corrs, :]
v = v[..., t_range[0]:t_range[-1]]
vis = v['r'] + 1j * v['i']
if k==1:
autos = auto_corrs(nfeed)
offp = (abs(vis[:, autos, 0::2]).mean() > \
(abs(vis[:, autos, 1::2]).mean())).astype(int)
times = times[offp::2]
vis = vis[..., offp::2]
gg = f['gain_coeff'][frq[0]:frq[-1]+1,
feeds, t_range[0]:t_range[-1]][..., offp::2]
gain_coeff = gg['r'] + 1j * gg['i']
del gg
# Read in time and freq slice if data has not yet been transposed
if transposed is False:
print "TRANSPOSED V OF CODE DOESN'T WORK YET!"
v = f['vis'][t_range[0]:t_range[-1]:2, frq[0]:frq[-1]+1, corrs]
vis = v['r'][:] + 1j * v['i'][:]
del v
gg = f['gain_coeff'][0, frq[0]:frq[-1]+1, feeds]
gain_coeff = gg['r'][:] + 1j * gg['i'][:]
vis = vis[..., offp::2]
vis = np.transpose(vis, (1, 2, 0))
# Remove fpga gains from data
vis = remove_fpga_gains(vis, gain_coeff, nfeed=nfeed, triu=False)
# Remove offset from galaxy
vis -= 0.5 * (vis[..., 0] + vis[..., -1])[..., np.newaxis]
# Get physical freq for fringestopper
freq_MHZ = 800.0 - np.array(frq) / 1024.0 * 400.
baddies = np.where(np.isnan(tools.get_feed_positions(inp)[:, 0]))[0]
a, b, c = select_corrs(baddies, nfeed=128)
vis[:, a + b] = 0.0
# Fringestop to location of "src"
data_fs = tools.fringestop_pathfinder(vis, eph.transit_RA(times), freq_MHZ, inp, src)
del vis
dr, sol_arr = solve_gain(data_fs)
# Find index of point source transit
drlist = np.argmax(dr, axis=-1)
# If multiple freq channels are zerod, the trans_pix
# will end up being 0. This is bad, so ensure that
# you are only looking for non-zero transit pixels.
drlist = [x for x in drlist if x != 0]
trans_pix = np.argmax(np.bincount(drlist))
assert trans_pix != 0.0
Gains[frq] = sol_arr[..., trans_pix-3:trans_pix+4].mean(-1)
zz = h5py.File('data' + str(i) + '.hdf5','w')
zz.create_dataset('data', data=dr)
zz.close()
print "%f, %d Nans out of %d" % (np.isnan(sol_arr).sum(), np.isnan(Gains[frq]).sum(), np.isnan(Gains[frq]).sum())
print trans_pix, sol_arr[..., trans_pix-3:trans_pix+4].mean(-1).sum(), sol_arr.mean(-1).sum()
# Plot up post-fs phases to see if everything has been fixed
if frq[0] == 12 * nsplit:
print "======================"
print " Plotting up freq: %d" % frq[0]
print "======================"
img_nm = './phs_plots/dfs' + np.str(frq[17]) + np.str(np.int(time.time())) +'.png'
img_nmcorr = './phs_plots/dfs' + np.str(frq[17]) + np.str(np.int(time.time())) +'corr.png'
plt_gains(data_fs, 0, img_name=img_nm, bad_chans=baddies)
dfs_corr = correct_dfs(data_fs, np.angle(Gains[frq])[..., np.newaxis], nfeed=128)
plt_gains(dfs_corr, 0, img_name=img_nmcorr, bad_chans=baddies)
del dfs_corr
del data_fs, a
return Gains
def __init__(
self,
vis1,
vis2,
tm1,
tm2,
src1,
src2,
freqs,
prods,
inputs,
pstns0,
bsipts=None,
):
"""Basic initialization method"""
self.adj_freqs = False # frequencies adjacent in data? (used for some tests)
self.VERBOSE = False
self.PATH = True # True if Pathfinder, False if CHIME
self.vis1 = vis1
self.vis2 = vis2
self.tm1 = tm1
self.tm2 = tm2
self.freqs = freqs
self.prods = prods
self.inputs = inputs
self.inprds = np.array([[self.inputs[prod[0]], self.inputs[prod[1]]]
for prod in self.prods])
self.pstns0 = pstns0
self.bsipts = bsipts
self.Nfr = len(freqs)
self.Npr = len(prods)
self.Nip = len(inputs)
self.c_bslns0, self.bslns0 = self.set_bslns0()
self.source1 = src1
self.source2 = src2
self.dec1 = self.source1._dec
if self.source2 is not None:
self.dec2 = self.source2._dec
self.tt1 = (ephemeris.transit_times(self.source1, self.tm1[0],
self.tm1[-1])[0] +
TTCORR[self.source1.name])
if self.source2 is not None:
self.tt2 = (ephemeris.transit_times(self.source2, self.tm2[0],
self.tm2[-1])[0] +
TTCORR[self.source2.name])
# Results place holders:
self.pass_xd1, self.pass_xd2 = None, None
self.pass_cont1, self.pass_cont2 = None, None
self.good_prods = None
self.good_freqs = None
self.good_prods_cons = None
self.good_freqs_cons = None
# TODO: delete all incidences of self.dists_computed. Subs by
# initializing here to self.xdists1 = None, etc..
self.dists_computed = False
# TODO: this might not be used...
# Possible to scramble expected baselines. For debug purposes only:
self.bslins0_scrambled = False
def fringestop_and_sum(fn,
feeds,
freq,
src,
transposed=True,
return_unfs=True,
meridian=False,
del_t=1500,
frick=None):
""" Take an input file fn and a set of feeds and return
a formed beam on src.
"""
if transposed is True:
r = andata.Reader(fn)
r.freq_sel = freq
X = r.read()
times = r.time
else:
f = h5py.File(fn, 'r')
times = f['index_map']['time'].value['ctime']
print "Read in data"
# Get transit time for source
src_trans = eph.transit_times(src, times[0])
del_phi = 1.30 * (src._dec - np.radians(eph.CHIMELATITUDE)) * np.sin(
np.radians(1.988))
del_phi *= (24 * 3600.0) / (2 * np.pi)
# Adjust the transit time accordingly
src_trans += del_phi
# Select +- del_t of transit, accounting for the mispointing
t_range = np.where((times < src_trans + del_t)
& (times > src_trans - del_t))[0]
times = times[t_range[0]:t_range[-1]] #[offp::2] test
print "Time range:", times[0], times[-1]
# Generate correctly ordered corrinputs
corrinput_real = gen_inp()[0]
inp = np.array(corrinput_real)
# Ensure vis array is in correct order (freq, prod, time)
if transposed is True:
data = X.vis[:, :, t_range[0]:t_range[-1]]
freq = X.freq
else:
v = f['vis'][t_range[0]:t_range[-1], freq, :]
vis = v['r'] + 1j * v['i']
data = vis.transpose()[np.newaxis]
del vis
freq = 800 - 400.0 / 1024 * freq
freq = np.array([freq])
# autos = auto_corrs(256)
# offp = (abs(data[:, autos, 0::2]).mean() > (abs(data[:, autos, 1::2]).mean())).astype(int)
# data = data[..., offp::2] test
data_unfs = sum_corrs(data.copy(), feeds)
ra_ = eph.transit_RA(times)
ra_2 = nolan_ra(times)
#ra_ = ra_2.copy()
if meridian is True:
ra = np.ones_like(ra_) * np.degrees(src._ra)
else:
ra = ra_
print len(inp)
dfs = tools.fringestop_pathfinder(data.copy(), ra, freq, inp, src)
#dfs = fringestop_pathfinder(data.copy(), ra_2, freq, inp, src, frick=frick)
# dfs = fringestop_pathfinder(data.copy(), ra_1, freq, inp, src, frick=frick)
# fp = np.loadtxt('/home/connor/feed_layout_decrease.txt')
# PH = fill_nolan(times, src._ra * 180.0 / np.pi, src._dec * 180.0 / np.pi, fp)
dfs_sum = sum_corrs(dfs, feeds)
if return_unfs is True:
return dfs_sum, ra_, dfs, data
else:
return dfs_sum, ra_
def main(config_file=None, logging_params=DEFAULT_LOGGING):
# Load config
config = DEFAULTS.deepcopy()
if config_file is not None:
config.merge(NameSpace(load_yaml_config(config_file)))
# Setup logging
log.setup_logging(logging_params)
logger = log.get_logger(__name__)
## Load data for flagging
# Load fpga restarts
time_fpga_restart = []
if config.fpga_restart_file is not None:
with open(config.fpga_restart_file, 'r') as handler:
for line in handler:
time_fpga_restart.append(
ephemeris.datetime_to_unix(
ephemeris.timestr_to_datetime(line.split('_')[0])))
time_fpga_restart = np.array(time_fpga_restart)
# Load housekeeping flag
if config.housekeeping_file is not None:
ftemp = TempData.from_acq_h5(config.housekeeping_file,
datasets=["time_flag"])
else:
ftemp = None
# Load jump data
if config.jump_file is not None:
with h5py.File(config.jump_file, 'r') as handler:
jump_time = handler["time"][:]
jump_size = handler["jump_size"][:]
else:
jump_time = None
jump_size = None
# Load rain data
if config.rain_file is not None:
with h5py.File(config.rain_file, 'r') as handler:
rain_ranges = handler["time_range_conservative"][:]
else:
rain_ranges = []
# Load data flags
data_flags = {}
if config.data_flags:
finder.connect_database()
flag_types = finder.DataFlagType.select()
possible_data_flags = []
for ft in flag_types:
possible_data_flags.append(ft.name)
if ft.name in config.data_flags:
new_data_flags = finder.DataFlag.select().where(
finder.DataFlag.type == ft)
data_flags[ft.name] = list(new_data_flags)
# Set desired range of time
start_time = (ephemeris.datetime_to_unix(
datetime.datetime(
*config.start_date)) if config.start_date is not None else None)
end_time = (ephemeris.datetime_to_unix(datetime.datetime(
*config.end_date)) if config.end_date is not None else None)
## Find gain files
files = {}
for src in config.sources:
files[src] = sorted(
glob.glob(
os.path.join(config.directory, src.lower(),
"%s_%s_lsd_*.h5" % (
config.prefix,
src.lower(),
))))
csd = {}
for src in config.sources:
csd[src] = np.array(
[int(os.path.splitext(ff)[0][-4:]) for ff in files[src]])
for src in config.sources:
logger.info("%s: %d files" % (src, len(csd[src])))
## Remove files that occur during flag
csd_flag = {}
for src in config.sources:
body = ephemeris.source_dictionary[src]
csd_flag[src] = np.ones(csd[src].size, dtype=np.bool)
for ii, cc in enumerate(csd[src][:]):
ttrans = ephemeris.transit_times(body,
ephemeris.csd_to_unix(cc))[0]
if (start_time is not None) and (ttrans < start_time):
csd_flag[src][ii] = False
continue
if (end_time is not None) and (ttrans > end_time):
csd_flag[src][ii] = False
continue
# If requested, remove daytime transits
if not config.include_daytime.get(
src, config.include_daytime.default) and daytime_flag(
ttrans)[0]:
logger.info("%s CSD %d: daytime transit" % (src, cc))
csd_flag[src][ii] = False
continue
# Remove transits during HKP drop out
if ftemp is not None:
itemp = np.flatnonzero(
(ftemp.time[:] >= (ttrans - config.transit_window))
& (ftemp.time[:] <= (ttrans + config.transit_window)))
tempflg = ftemp['time_flag'][itemp]
if (tempflg.size == 0) or ((np.sum(tempflg, dtype=np.float32) /
float(tempflg.size)) < 0.50):
logger.info("%s CSD %d: no housekeeping" % (src, cc))
csd_flag[src][ii] = False
continue
# Remove transits near jumps
if jump_time is not None:
njump = np.sum((jump_size > config.min_jump_size)
& (jump_time > (ttrans - config.jump_window))
& (jump_time < ttrans))
if njump > config.max_njump:
logger.info("%s CSD %d: %d jumps before" %
(src, cc, njump))
csd_flag[src][ii] = False
continue
# Remove transits near rain
for rng in rain_ranges:
if (((ttrans - config.transit_window) <= rng[1])
and ((ttrans + config.transit_window) >= rng[0])):
logger.info("%s CSD %d: during rain" % (src, cc))
csd_flag[src][ii] = False
break
# Remove transits during data flag
for name, flag_list in data_flags.items():
if csd_flag[src][ii]:
for flg in flag_list:
if (((ttrans - config.transit_window) <=
flg.finish_time)
and ((ttrans + config.transit_window) >=
flg.start_time)):
logger.info("%s CSD %d: %s flag" %
(src, cc, name))
csd_flag[src][ii] = False
break
# Print number of files left after flagging
for src in config.sources:
logger.info("%s: %d files (after flagging)" %
(src, np.sum(csd_flag[src])))
## Construct pair wise differences
npair = len(config.diff_pair)
shift = [nd * 24.0 * 3600.0 for nd in config.nday_shift]
calmap = []
calpair = []
for (tsrc, csrc), sh in zip(config.diff_pair, shift):
body_test = ephemeris.source_dictionary[tsrc]
body_cal = ephemeris.source_dictionary[csrc]
for ii, cc in enumerate(csd[tsrc]):
if csd_flag[tsrc][ii]:
test_transit = ephemeris.transit_times(
body_test, ephemeris.csd_to_unix(cc))[0]
cal_transit = ephemeris.transit_times(body_cal,
test_transit + sh)[0]
cal_csd = int(np.fix(ephemeris.unix_to_csd(cal_transit)))
ttrans = np.sort([test_transit, cal_transit])
if cal_csd in csd[csrc]:
jj = list(csd[csrc]).index(cal_csd)
if csd_flag[csrc][jj] and not np.any(
(time_fpga_restart >= ttrans[0])
& (time_fpga_restart <= ttrans[1])):
calmap.append([ii, jj])
calpair.append([tsrc, csrc])
calmap = np.array(calmap)
calpair = np.array(calpair)
ntransit = calmap.shape[0]
logger.info("%d total transit pairs" % ntransit)
for ii in range(ntransit):
t1 = ephemeris.transit_times(
ephemeris.source_dictionary[calpair[ii, 0]],
ephemeris.csd_to_unix(csd[calpair[ii, 0]][calmap[ii, 0]]))[0]
t2 = ephemeris.transit_times(
ephemeris.source_dictionary[calpair[ii, 1]],
ephemeris.csd_to_unix(csd[calpair[ii, 1]][calmap[ii, 1]]))[0]
logger.info("%s (%d) - %s (%d): %0.1f hr" %
(calpair[ii, 0], csd_flag[calpair[ii, 0]][calmap[ii, 0]],
calpair[ii, 1], csd_flag[calpair[ii, 1]][calmap[ii, 1]],
(t1 - t2) / 3600.0))
# Determine unique diff pairs
diff_name = np.array(['%s/%s' % tuple(cp) for cp in calpair])
uniq_diff, lbl_diff, cnt_diff = np.unique(diff_name,
return_inverse=True,
return_counts=True)
ndiff = uniq_diff.size
for ud, udcnt in zip(uniq_diff, cnt_diff):
logger.info("%s: %d transit pairs" % (ud, udcnt))
## Load gains
inputmap = tools.get_correlator_inputs(datetime.datetime.utcnow(),
correlator='chime')
ninput = len(inputmap)
nfreq = 1024
# Set up gain arrays
gain = np.zeros((2, nfreq, ninput, ntransit), dtype=np.complex64)
weight = np.zeros((2, nfreq, ninput, ntransit), dtype=np.float32)
input_sort = np.zeros((2, ninput, ntransit), dtype=np.int)
kcsd = np.zeros((2, ntransit), dtype=np.float32)
timestamp = np.zeros((2, ntransit), dtype=np.float64)
is_daytime = np.zeros((2, ntransit), dtype=np.bool)
for tt in range(ntransit):
for kk, (src, ind) in enumerate(zip(calpair[tt], calmap[tt])):
body = ephemeris.source_dictionary[src]
filename = files[src][ind]
logger.info("%s: %s" % (src, filename))
temp = containers.StaticGainData.from_file(filename)
freq = temp.freq[:]
inputs = temp.input[:]
isort = reorder_inputs(inputmap, inputs)
inputs = inputs[isort]
gain[kk, :, :, tt] = temp.gain[:, isort]
weight[kk, :, :, tt] = temp.weight[:, isort]
input_sort[kk, :, tt] = isort
kcsd[kk, tt] = temp.attrs['lsd']
timestamp[kk, tt] = ephemeris.transit_times(
body, ephemeris.csd_to_unix(kcsd[kk, tt]))[0]
is_daytime[kk, tt] = daytime_flag(timestamp[kk, tt])[0]
if np.any(isort != np.arange(isort.size)):
logger.info("Input ordering has changed: %s" %
ephemeris.unix_to_datetime(
timestamp[kk, tt]).strftime("%Y-%m-%d"))
logger.info("")
inputs = np.array([(inp.id, inp.input_sn) for inp in inputmap],
dtype=[('chan_id', 'u2'), ('correlator_input', 'S32')])
## Load input flags
inpflg = np.ones((2, ninput, ntransit), dtype=np.bool)
min_flag_time = np.min(timestamp) - 7.0 * 24.0 * 60.0 * 60.0
max_flag_time = np.max(timestamp) + 7.0 * 24.0 * 60.0 * 60.0
flaginput_files = sorted(
glob.glob(
os.path.join(config.flaginput_dir, "*" + config.flaginput_suffix,
"*.h5")))
if flaginput_files:
logger.info("Found %d flaginput files." % len(flaginput_files))
tmp = andata.FlagInputData.from_acq_h5(flaginput_files, datasets=())
start, stop = [
int(yy) for yy in np.percentile(
np.flatnonzero((tmp.time[:] >= min_flag_time)
& (tmp.time[:] <= max_flag_time)), [0, 100])
]
cont = andata.FlagInputData.from_acq_h5(flaginput_files,
start=start,
stop=stop,
datasets=['flag'])
for kk in range(2):
inpflg[kk, :, :] = cont.resample('flag',
timestamp[kk],
transpose=True)
logger.info("Flaginput time offsets in minutes (pair %d):" % kk)
logger.info(
str(
np.fix((cont.time[cont.search_update_time(timestamp[kk])] -
timestamp[kk]) / 60.0).astype(np.int)))
# Sort flags so they are in same order
for tt in range(ntransit):
for kk in range(2):
inpflg[kk, :, tt] = inpflg[kk, input_sort[kk, :, tt], tt]
# Do not apply input flag to phase reference
for ii in config.index_phase_ref:
inpflg[:, ii, :] = True
## Flag out gains with high uncertainty and frequencies with large fraction of data flagged
frac_err = tools.invert_no_zero(np.sqrt(weight) * np.abs(gain))
flag = np.all((weight > 0.0) & (np.abs(gain) > 0.0) &
(frac_err < config.max_uncertainty),
axis=0)
freq_flag = ((np.sum(flag, axis=(1, 2), dtype=np.float32) /
float(np.prod(flag.shape[1:]))) > config.freq_threshold)
if config.apply_rfi_mask:
freq_flag &= np.logical_not(rfi.frequency_mask(freq))
flag = flag & freq_flag[:, np.newaxis, np.newaxis]
good_freq = np.flatnonzero(freq_flag)
logger.info("Number good frequencies %d" % good_freq.size)
## Generate flags with more conservative cuts on frequency
c_flag = flag & np.all(frac_err < config.conservative.max_uncertainty,
axis=0)
c_freq_flag = ((np.sum(c_flag, axis=(1, 2), dtype=np.float32) /
float(np.prod(c_flag.shape[1:]))) >
config.conservative.freq_threshold)
if config.conservative.apply_rfi_mask:
c_freq_flag &= np.logical_not(rfi.frequency_mask(freq))
c_flag = c_flag & c_freq_flag[:, np.newaxis, np.newaxis]
c_good_freq = np.flatnonzero(c_freq_flag)
logger.info("Number good frequencies (conservative thresholds) %d" %
c_good_freq.size)
## Apply input flags
flag &= np.all(inpflg[:, np.newaxis, :, :], axis=0)
## Update flags based on beam flag
if config.beam_flag_file is not None:
dbeam = andata.BaseData.from_acq_h5(config.beam_flag_file)
db_csd = np.floor(ephemeris.unix_to_csd(dbeam.index_map['time'][:]))
for ii, name in enumerate(config.beam_flag_datasets):
logger.info("Applying %s beam flag." % name)
if not ii:
db_flag = dbeam.flags[name][:]
else:
db_flag &= dbeam.flags[name][:]
cnt = 0
for ii, dbc in enumerate(db_csd):
this_csd = np.flatnonzero(np.any(kcsd == dbc, axis=0))
if this_csd.size > 0:
logger.info("Beam flag for %d matches %s." %
(dbc, str(kcsd[:, this_csd])))
flag[:, :, this_csd] &= db_flag[np.newaxis, :, ii, np.newaxis]
cnt += 1
logger.info("Applied %0.1f percent of the beam flags" %
(100.0 * cnt / float(db_csd.size), ))
## Flag inputs with large amount of missing data
input_frac_flagged = (
np.sum(flag[good_freq, :, :], axis=(0, 2), dtype=np.float32) /
float(good_freq.size * ntransit))
input_flag = input_frac_flagged > config.input_threshold
for ii in config.index_phase_ref:
logger.info("Phase reference %d has %0.3f fraction of data flagged." %
(ii, input_frac_flagged[ii]))
input_flag[ii] = True
good_input = np.flatnonzero(input_flag)
flag = flag & input_flag[np.newaxis, :, np.newaxis]
logger.info("Number good inputs %d" % good_input.size)
## Calibrate
gaincal = gain[0] * tools.invert_no_zero(gain[1])
frac_err_cal = np.sqrt(frac_err[0]**2 + frac_err[1]**2)
count = np.sum(flag, axis=-1, dtype=np.int)
stat_flag = count > config.min_num_transit
## Calculate phase
amp = np.abs(gaincal)
phi = np.angle(gaincal)
## Calculate polarisation groups
pol_dict = {'E': 'X', 'S': 'Y'}
cyl_dict = {2: 'A', 3: 'B', 4: 'C', 5: 'D'}
if config.group_by_cyl:
group_id = [
(inp.pol,
inp.cyl) if tools.is_chime(inp) and (ii in good_input) else None
for ii, inp in enumerate(inputmap)
]
else:
group_id = [
inp.pol if tools.is_chime(inp) and (ii in good_input) else None
for ii, inp in enumerate(inputmap)
]
ugroup_id = sorted([uidd for uidd in set(group_id) if uidd is not None])
ngroup = len(ugroup_id)
group_list_noref = [
np.array([
gg for gg, gid in enumerate(group_id)
if (gid == ugid) and gg not in config.index_phase_ref
]) for ugid in ugroup_id
]
group_list = [
np.array([gg for gg, gid in enumerate(group_id) if gid == ugid])
for ugid in ugroup_id
]
if config.group_by_cyl:
group_str = [
"%s-%s" % (pol_dict[pol], cyl_dict[cyl]) for pol, cyl in ugroup_id
]
else:
group_str = [pol_dict[pol] for pol in ugroup_id]
index_phase_ref = []
for gstr, igroup in zip(group_str, group_list):
candidate = [ii for ii in config.index_phase_ref if ii in igroup]
if len(candidate) != 1:
index_phase_ref.append(None)
else:
index_phase_ref.append(candidate[0])
logger.info(
"Phase reference: %s" %
', '.join(['%s = %s' % tpl
for tpl in zip(group_str, index_phase_ref)]))
## Apply thermal correction to amplitude
if config.amp_thermal.enabled:
logger.info("Applying thermal correction.")
# Load the temperatures
tdata = TempData.from_acq_h5(config.amp_thermal.filename)
index = tdata.search_sensors(config.amp_thermal.sensor)[0]
temp = tdata.datasets[config.amp_thermal.field][index]
temp_func = scipy.interpolate.interp1d(tdata.time, temp,
**config.amp_thermal.interp)
itemp = temp_func(timestamp)
dtemp = itemp[0] - itemp[1]
flag_func = scipy.interpolate.interp1d(
tdata.time, tdata.datasets['flag'][index].astype(np.float32),
**config.amp_thermal.interp)
dtemp_flag = np.all(flag_func(timestamp) == 1.0, axis=0)
flag &= dtemp_flag[np.newaxis, np.newaxis, :]
for gstr, igroup in zip(group_str, group_list):
pstr = gstr[0]
thermal_coeff = np.polyval(config.amp_thermal.coeff[pstr], freq)
gthermal = 1.0 + thermal_coeff[:, np.newaxis, np.newaxis] * dtemp[
np.newaxis, np.newaxis, :]
amp[:, igroup, :] *= tools.invert_no_zero(gthermal)
## Compute common mode
if config.subtract_common_mode_before:
logger.info("Calculating common mode amplitude and phase.")
cmn_amp, flag_cmn_amp = compute_common_mode(amp,
flag,
group_list_noref,
median=False)
cmn_phi, flag_cmn_phi = compute_common_mode(phi,
flag,
group_list_noref,
median=False)
# Subtract common mode (from phase only)
logger.info("Subtracting common mode phase.")
group_flag = np.zeros((ngroup, ninput), dtype=np.bool)
for gg, igroup in enumerate(group_list):
group_flag[gg, igroup] = True
phi[:,
igroup, :] = phi[:, igroup, :] - cmn_phi[:, gg, np.newaxis, :]
for iref in index_phase_ref:
if (iref is not None) and (iref in igroup):
flag[:, iref, :] = flag_cmn_phi[:, gg, :]
## If requested, determine and subtract a delay template
if config.fit_delay_before:
logger.info("Fitting delay template.")
omega = timing.FREQ_TO_OMEGA * freq
tau, tau_flag, _ = construct_delay_template(
omega,
phi,
c_flag & flag,
min_num_freq_for_delay_fit=config.min_num_freq_for_delay_fit)
# Compute residuals
logger.info("Subtracting delay template.")
phi = phi - tau[np.newaxis, :, :] * omega[:, np.newaxis, np.newaxis]
## Normalize by median over time
logger.info("Calculating median amplitude and phase.")
med_amp = np.zeros((nfreq, ninput, ndiff), dtype=amp.dtype)
med_phi = np.zeros((nfreq, ninput, ndiff), dtype=phi.dtype)
count_by_diff = np.zeros((nfreq, ninput, ndiff), dtype=np.int)
stat_flag_by_diff = np.zeros((nfreq, ninput, ndiff), dtype=np.bool)
def weighted_mean(yy, ww, axis=-1):
return np.sum(ww * yy, axis=axis) * tools.invert_no_zero(
np.sum(ww, axis=axis))
for dd in range(ndiff):
this_diff = np.flatnonzero(lbl_diff == dd)
this_flag = flag[:, :, this_diff]
this_amp = amp[:, :, this_diff]
this_amp_err = this_amp * frac_err_cal[:, :,
this_diff] * this_flag.astype(
np.float32)
this_phi = phi[:, :, this_diff]
this_phi_err = frac_err_cal[:, :, this_diff] * this_flag.astype(
np.float32)
count_by_diff[:, :, dd] = np.sum(this_flag, axis=-1, dtype=np.int)
stat_flag_by_diff[:, :,
dd] = count_by_diff[:, :,
dd] > config.min_num_transit
if config.weighted_mean == 2:
logger.info("Calculating inverse variance weighted mean.")
med_amp[:, :,
dd] = weighted_mean(this_amp,
tools.invert_no_zero(this_amp_err**2),
axis=-1)
med_phi[:, :,
dd] = weighted_mean(this_phi,
tools.invert_no_zero(this_phi_err**2),
axis=-1)
elif config.weighted_mean == 1:
logger.info("Calculating uniform weighted mean.")
med_amp[:, :, dd] = weighted_mean(this_amp,
this_flag.astype(np.float32),
axis=-1)
med_phi[:, :, dd] = weighted_mean(this_phi,
this_flag.astype(np.float32),
axis=-1)
else:
logger.info("Calculating median value.")
for ff in range(nfreq):
for ii in range(ninput):
if np.any(this_flag[ff, ii, :]):
med_amp[ff, ii, dd] = wq.median(
this_amp[ff, ii, :],
this_flag[ff, ii, :].astype(np.float32))
med_phi[ff, ii, dd] = wq.median(
this_phi[ff, ii, :],
this_flag[ff, ii, :].astype(np.float32))
damp = np.zeros_like(amp)
dphi = np.zeros_like(phi)
for dd in range(ndiff):
this_diff = np.flatnonzero(lbl_diff == dd)
damp[:, :, this_diff] = amp[:, :, this_diff] * tools.invert_no_zero(
med_amp[:, :, dd, np.newaxis]) - 1.0
dphi[:, :,
this_diff] = phi[:, :, this_diff] - med_phi[:, :, dd, np.newaxis]
# Compute common mode
if not config.subtract_common_mode_before:
logger.info("Calculating common mode amplitude and phase.")
cmn_amp, flag_cmn_amp = compute_common_mode(damp,
flag,
group_list_noref,
median=True)
cmn_phi, flag_cmn_phi = compute_common_mode(dphi,
flag,
group_list_noref,
median=True)
# Subtract common mode (from phase only)
logger.info("Subtracting common mode phase.")
group_flag = np.zeros((ngroup, ninput), dtype=np.bool)
for gg, igroup in enumerate(group_list):
group_flag[gg, igroup] = True
dphi[:, igroup, :] = dphi[:, igroup, :] - cmn_phi[:, gg,
np.newaxis, :]
for iref in index_phase_ref:
if (iref is not None) and (iref in igroup):
flag[:, iref, :] = flag_cmn_phi[:, gg, :]
## Compute RMS
logger.info("Calculating RMS of amplitude and phase.")
mad_amp = np.zeros((nfreq, ninput), dtype=amp.dtype)
std_amp = np.zeros((nfreq, ninput), dtype=amp.dtype)
mad_phi = np.zeros((nfreq, ninput), dtype=phi.dtype)
std_phi = np.zeros((nfreq, ninput), dtype=phi.dtype)
mad_amp_by_diff = np.zeros((nfreq, ninput, ndiff), dtype=amp.dtype)
std_amp_by_diff = np.zeros((nfreq, ninput, ndiff), dtype=amp.dtype)
mad_phi_by_diff = np.zeros((nfreq, ninput, ndiff), dtype=phi.dtype)
std_phi_by_diff = np.zeros((nfreq, ninput, ndiff), dtype=phi.dtype)
for ff in range(nfreq):
for ii in range(ninput):
this_flag = flag[ff, ii, :]
if np.any(this_flag):
std_amp[ff, ii] = np.std(damp[ff, ii, this_flag])
std_phi[ff, ii] = np.std(dphi[ff, ii, this_flag])
mad_amp[ff, ii] = 1.48625 * wq.median(
np.abs(damp[ff, ii, :]), this_flag.astype(np.float32))
mad_phi[ff, ii] = 1.48625 * wq.median(
np.abs(dphi[ff, ii, :]), this_flag.astype(np.float32))
for dd in range(ndiff):
this_diff = this_flag & (lbl_diff == dd)
if np.any(this_diff):
std_amp_by_diff[ff, ii, dd] = np.std(damp[ff, ii,
this_diff])
std_phi_by_diff[ff, ii, dd] = np.std(dphi[ff, ii,
this_diff])
mad_amp_by_diff[ff, ii, dd] = 1.48625 * wq.median(
np.abs(damp[ff, ii, :]),
this_diff.astype(np.float32))
mad_phi_by_diff[ff, ii, dd] = 1.48625 * wq.median(
np.abs(dphi[ff, ii, :]),
this_diff.astype(np.float32))
## Construct delay template
if not config.fit_delay_before:
logger.info("Fitting delay template.")
omega = timing.FREQ_TO_OMEGA * freq
tau, tau_flag, _ = construct_delay_template(
omega,
dphi,
c_flag & flag,
min_num_freq_for_delay_fit=config.min_num_freq_for_delay_fit)
# Compute residuals
logger.info("Subtracting delay template from phase.")
resid = (dphi - tau[np.newaxis, :, :] *
omega[:, np.newaxis, np.newaxis]) * flag.astype(np.float32)
else:
resid = dphi
tau_count = np.sum(tau_flag, axis=-1, dtype=np.int)
tau_stat_flag = tau_count > config.min_num_transit
tau_count_by_diff = np.zeros((ninput, ndiff), dtype=np.int)
tau_stat_flag_by_diff = np.zeros((ninput, ndiff), dtype=np.bool)
for dd in range(ndiff):
this_diff = np.flatnonzero(lbl_diff == dd)
tau_count_by_diff[:, dd] = np.sum(tau_flag[:, this_diff],
axis=-1,
dtype=np.int)
tau_stat_flag_by_diff[:,
dd] = tau_count_by_diff[:,
dd] > config.min_num_transit
## Calculate statistics of residuals
std_resid = np.zeros((nfreq, ninput), dtype=phi.dtype)
mad_resid = np.zeros((nfreq, ninput), dtype=phi.dtype)
std_resid_by_diff = np.zeros((nfreq, ninput, ndiff), dtype=phi.dtype)
mad_resid_by_diff = np.zeros((nfreq, ninput, ndiff), dtype=phi.dtype)
for ff in range(nfreq):
for ii in range(ninput):
this_flag = flag[ff, ii, :]
if np.any(this_flag):
std_resid[ff, ii] = np.std(resid[ff, ii, this_flag])
mad_resid[ff, ii] = 1.48625 * wq.median(
np.abs(resid[ff, ii, :]), this_flag.astype(np.float32))
for dd in range(ndiff):
this_diff = this_flag & (lbl_diff == dd)
if np.any(this_diff):
std_resid_by_diff[ff, ii,
dd] = np.std(resid[ff, ii,
this_diff])
mad_resid_by_diff[ff, ii, dd] = 1.48625 * wq.median(
np.abs(resid[ff, ii, :]),
this_diff.astype(np.float32))
## Calculate statistics of delay template
mad_tau = np.zeros((ninput, ), dtype=phi.dtype)
std_tau = np.zeros((ninput, ), dtype=phi.dtype)
mad_tau_by_diff = np.zeros((ninput, ndiff), dtype=phi.dtype)
std_tau_by_diff = np.zeros((ninput, ndiff), dtype=phi.dtype)
for ii in range(ninput):
this_flag = tau_flag[ii]
if np.any(this_flag):
std_tau[ii] = np.std(tau[ii, this_flag])
mad_tau[ii] = 1.48625 * wq.median(np.abs(tau[ii]),
this_flag.astype(np.float32))
for dd in range(ndiff):
this_diff = this_flag & (lbl_diff == dd)
if np.any(this_diff):
std_tau_by_diff[ii, dd] = np.std(tau[ii, this_diff])
mad_tau_by_diff[ii, dd] = 1.48625 * wq.median(
np.abs(tau[ii]), this_diff.astype(np.float32))
## Define output
res = {
"timestamp": {
"data": timestamp,
"axis": ["div", "time"]
},
"is_daytime": {
"data": is_daytime,
"axis": ["div", "time"]
},
"csd": {
"data": kcsd,
"axis": ["div", "time"]
},
"pair_map": {
"data": lbl_diff,
"axis": ["time"]
},
"pair_count": {
"data": cnt_diff,
"axis": ["pair"]
},
"gain": {
"data": gaincal,
"axis": ["freq", "input", "time"]
},
"frac_err": {
"data": frac_err_cal,
"axis": ["freq", "input", "time"]
},
"flags/gain": {
"data": flag,
"axis": ["freq", "input", "time"],
"flag": True
},
"flags/gain_conservative": {
"data": c_flag,
"axis": ["freq", "input", "time"],
"flag": True
},
"flags/count": {
"data": count,
"axis": ["freq", "input"],
"flag": True
},
"flags/stat": {
"data": stat_flag,
"axis": ["freq", "input"],
"flag": True
},
"flags/count_by_pair": {
"data": count_by_diff,
"axis": ["freq", "input", "pair"],
"flag": True
},
"flags/stat_by_pair": {
"data": stat_flag_by_diff,
"axis": ["freq", "input", "pair"],
"flag": True
},
"med_amp": {
"data": med_amp,
"axis": ["freq", "input", "pair"]
},
"med_phi": {
"data": med_phi,
"axis": ["freq", "input", "pair"]
},
"flags/group_flag": {
"data": group_flag,
"axis": ["group", "input"],
"flag": True
},
"cmn_amp": {
"data": cmn_amp,
"axis": ["freq", "group", "time"]
},
"cmn_phi": {
"data": cmn_phi,
"axis": ["freq", "group", "time"]
},
"amp": {
"data": damp,
"axis": ["freq", "input", "time"]
},
"phi": {
"data": dphi,
"axis": ["freq", "input", "time"]
},
"std_amp": {
"data": std_amp,
"axis": ["freq", "input"]
},
"std_amp_by_pair": {
"data": std_amp_by_diff,
"axis": ["freq", "input", "pair"]
},
"mad_amp": {
"data": mad_amp,
"axis": ["freq", "input"]
},
"mad_amp_by_pair": {
"data": mad_amp_by_diff,
"axis": ["freq", "input", "pair"]
},
"std_phi": {
"data": std_phi,
"axis": ["freq", "input"]
},
"std_phi_by_pair": {
"data": std_phi_by_diff,
"axis": ["freq", "input", "pair"]
},
"mad_phi": {
"data": mad_phi,
"axis": ["freq", "input"]
},
"mad_phi_by_pair": {
"data": mad_phi_by_diff,
"axis": ["freq", "input", "pair"]
},
"tau": {
"data": tau,
"axis": ["input", "time"]
},
"flags/tau": {
"data": tau_flag,
"axis": ["input", "time"],
"flag": True
},
"flags/tau_count": {
"data": tau_count,
"axis": ["input"],
"flag": True
},
"flags/tau_stat": {
"data": tau_stat_flag,
"axis": ["input"],
"flag": True
},
"flags/tau_count_by_pair": {
"data": tau_count_by_diff,
"axis": ["input", "pair"],
"flag": True
},
"flags/tau_stat_by_pair": {
"data": tau_stat_flag_by_diff,
"axis": ["input", "pair"],
"flag": True
},
"std_tau": {
"data": std_tau,
"axis": ["input"]
},
"std_tau_by_pair": {
"data": std_tau_by_diff,
"axis": ["input", "pair"]
},
"mad_tau": {
"data": mad_tau,
"axis": ["input"]
},
"mad_tau_by_pair": {
"data": mad_tau_by_diff,
"axis": ["input", "pair"]
},
"resid_phi": {
"data": resid,
"axis": ["freq", "input", "time"]
},
"std_resid_phi": {
"data": std_resid,
"axis": ["freq", "input"]
},
"std_resid_phi_by_pair": {
"data": std_resid_by_diff,
"axis": ["freq", "input", "pair"]
},
"mad_resid_phi": {
"data": mad_resid,
"axis": ["freq", "input"]
},
"mad_resid_phi_by_pair": {
"data": mad_resid_by_diff,
"axis": ["freq", "input", "pair"]
},
}
## Create the output container
logger.info("Creating StabilityData container.")
data = StabilityData()
data.create_index_map(
"div", np.array(["numerator", "denominator"], dtype=np.string_))
data.create_index_map("pair", np.array(uniq_diff, dtype=np.string_))
data.create_index_map("group", np.array(group_str, dtype=np.string_))
data.create_index_map("freq", freq)
data.create_index_map("input", inputs)
data.create_index_map("time", timestamp[0, :])
logger.info("Writing datsets to container.")
for name, dct in res.iteritems():
is_flag = dct.get('flag', False)
if is_flag:
dset = data.create_flag(name.split('/')[-1], data=dct['data'])
else:
dset = data.create_dataset(name, data=dct['data'])
dset.attrs['axis'] = np.array(dct['axis'], dtype=np.string_)
data.attrs['phase_ref'] = np.array(
[iref for iref in index_phase_ref if iref is not None])
# Determine the output filename and save results
start_time, end_time = ephemeris.unix_to_datetime(
np.percentile(timestamp, [0, 100]))
tfmt = "%Y%m%d"
night_str = 'night_' if not np.any(is_daytime) else ''
output_file = os.path.join(
config.output_dir, "%s_%s_%sraw_stability_data.h5" %
(start_time.strftime(tfmt), end_time.strftime(tfmt), night_str))
logger.info("Saving results to %s." % output_file)
data.save(output_file)
def offline_point_source_calibration(file_list,
source,
inputmap=None,
start=None,
stop=None,
physical_freq=None,
tcorr=None,
logging_params=DEFAULT_LOGGING,
**kwargs):
# Load config
config = DEFAULTS.deepcopy()
config.merge(NameSpace(kwargs))
# Setup logging
log.setup_logging(logging_params)
mlog = log.get_logger(__name__)
mlog.info("ephemeris file: %s" % ephemeris.__file__)
# Set the model to use
fitter_function = utils.fit_point_source_transit
model_function = utils.model_point_source_transit
farg = inspect.getargspec(fitter_function)
defaults = {
key: val
for key, val in zip(farg.args[-len(farg.defaults):], farg.defaults)
}
poly_deg_amp = kwargs.get('poly_deg_amp', defaults['poly_deg_amp'])
poly_deg_phi = kwargs.get('poly_deg_phi', defaults['poly_deg_phi'])
poly_type = kwargs.get('poly_type', defaults['poly_type'])
param_name = ([
'%s_poly_amp_coeff%d' % (poly_type, cc)
for cc in range(poly_deg_amp + 1)
] + [
'%s_poly_phi_coeff%d' % (poly_type, cc)
for cc in range(poly_deg_phi + 1)
])
model_kwargs = [('poly_deg_amp', poly_deg_amp),
('poly_deg_phi', poly_deg_phi), ('poly_type', poly_type)]
model_name = '.'.join(
[getattr(model_function, key) for key in ['__module__', '__name__']])
tval = {}
# Set where to evaluate gain
ha_eval_str = ['raw_transit']
if config.multi_sample:
ha_eval_str += ['transit', 'peak']
ha_eval = [0.0, None]
fitslc = slice(1, 3)
ind_eval = ha_eval_str.index(config.evaluate_gain_at)
# Determine dimensions
direction = ['amp', 'phi']
nparam = len(param_name)
ngain = len(ha_eval_str)
ndir = len(direction)
# Determine frequencies
data = andata.CorrData.from_acq_h5(file_list,
datasets=(),
start=start,
stop=stop)
freq = data.freq
if physical_freq is not None:
index_freq = np.array(
[np.argmin(np.abs(ff - freq)) for ff in physical_freq])
freq_sel = utils.convert_to_slice(index_freq)
freq = freq[index_freq]
else:
index_freq = np.arange(freq.size)
freq_sel = None
nfreq = freq.size
# Compute flux of source
inv_rt_flux_density = tools.invert_no_zero(
np.sqrt(FluxCatalog[source].predict_flux(freq)))
# Read in the eigenvaluess for all frequencies
data = andata.CorrData.from_acq_h5(file_list,
datasets=['erms', 'eval'],
freq_sel=freq_sel,
start=start,
stop=stop)
# Determine source coordinates
this_csd = np.floor(ephemeris.unix_to_csd(np.median(data.time)))
timestamp0 = ephemeris.transit_times(FluxCatalog[source].skyfield,
ephemeris.csd_to_unix(this_csd))[0]
src_ra, src_dec = ephemeris.object_coords(FluxCatalog[source].skyfield,
date=timestamp0,
deg=True)
ra = ephemeris.lsa(data.time)
ha = ra - src_ra
ha = ha - (ha > 180.0) * 360.0 + (ha < -180.0) * 360.0
ha = np.radians(ha)
itrans = np.argmin(np.abs(ha))
window = 0.75 * np.max(np.abs(ha))
off_source = np.abs(ha) > window
mlog.info("CSD %d" % this_csd)
mlog.info("Hour angle at transit (%d of %d): %0.2f deg " %
(itrans, len(ha), np.degrees(ha[itrans])))
mlog.info("Hour angle off source: %0.2f deg" %
np.median(np.abs(np.degrees(ha[off_source]))))
src_dec = np.radians(src_dec)
lat = np.radians(ephemeris.CHIMELATITUDE)
# Determine division of frequencies
ninput = data.ninput
ntime = data.ntime
nblock_freq = int(np.ceil(nfreq / float(config.nfreq_per_block)))
# Determine bad inputs
eps = 10.0 * np.finfo(data['erms'].dtype).eps
good_freq = np.flatnonzero(np.all(data['erms'][:] > eps, axis=-1))
ind_sub_freq = good_freq[slice(0, good_freq.size,
max(int(good_freq.size / 10), 1))]
tmp_data = andata.CorrData.from_acq_h5(file_list,
datasets=['evec'],
freq_sel=ind_sub_freq,
start=start,
stop=stop)
eps = 10.0 * np.finfo(tmp_data['evec'].dtype).eps
bad_input = np.flatnonzero(
np.all(np.abs(tmp_data['evec'][:, 0]) < eps, axis=(0, 2)))
input_axis = tmp_data.input.copy()
del tmp_data
# Query layout database for correlator inputs
if inputmap is None:
inputmap = tools.get_correlator_inputs(
datetime.datetime.utcfromtimestamp(data.time[itrans]),
correlator='chime')
inputmap = tools.reorder_correlator_inputs(input_axis, inputmap)
tools.change_chime_location(rotation=config.telescope_rotation)
# Determine x and y pol index
xfeeds = np.array([
idf for idf, inp in enumerate(inputmap)
if (idf not in bad_input) and tools.is_array_x(inp)
])
yfeeds = np.array([
idf for idf, inp in enumerate(inputmap)
if (idf not in bad_input) and tools.is_array_y(inp)
])
nfeed = xfeeds.size + yfeeds.size
pol = [yfeeds, xfeeds]
polstr = ['Y', 'X']
npol = len(pol)
neigen = min(max(npol, config.neigen), data['eval'].shape[1])
phase_ref = config.phase_reference_index
phase_ref_by_pol = [
pol[pp].tolist().index(phase_ref[pp]) for pp in range(npol)
]
# Calculate dynamic range
eval0_off_source = np.median(data['eval'][:, 0, off_source], axis=-1)
dyn = data['eval'][:, 1, :] * tools.invert_no_zero(
eval0_off_source[:, np.newaxis])
# Determine frequencies to mask
not_rfi = np.ones((nfreq, 1), dtype=np.bool)
if config.mask_rfi is not None:
for frng in config.mask_rfi:
not_rfi[:, 0] &= ((freq < frng[0]) | (freq > frng[1]))
mlog.info("%0.1f percent of frequencies available after masking RFI." %
(100.0 * np.sum(not_rfi, dtype=np.float32) / float(nfreq), ))
#dyn_flg = utils.contiguous_flag(dyn > config.dyn_rng_threshold, centre=itrans)
if source in config.dyn_rng_threshold:
dyn_rng_threshold = config.dyn_rng_threshold[source]
else:
dyn_rng_threshold = config.dyn_rng_threshold.default
mlog.info("Dynamic range threshold set to %0.1f." % dyn_rng_threshold)
dyn_flg = dyn > dyn_rng_threshold
# Calculate fit flag
fit_flag = np.zeros((nfreq, npol, ntime), dtype=np.bool)
for pp in range(npol):
mlog.info("Dynamic Range Nsample, Pol %d: %s" % (pp, ','.join([
"%d" % xx for xx in np.percentile(np.sum(dyn_flg, axis=-1),
[25, 50, 75, 100])
])))
if config.nsigma1 is None:
fit_flag[:, pp, :] = dyn_flg & not_rfi
else:
fit_window = config.nsigma1 * np.radians(
utils.get_window(freq, pol=polstr[pp], dec=src_dec, deg=True))
win_flg = np.abs(ha)[np.newaxis, :] <= fit_window[:, np.newaxis]
fit_flag[:, pp, :] = (dyn_flg & win_flg & not_rfi)
# Calculate base error
base_err = data['erms'][:, np.newaxis, :]
# Check for sign flips
ref_resp = andata.CorrData.from_acq_h5(file_list,
datasets=['evec'],
input_sel=config.eigen_reference,
freq_sel=freq_sel,
start=start,
stop=stop)['evec'][:, 0:neigen,
0, :]
sign0 = 1.0 - 2.0 * (ref_resp.real < 0.0)
# Check that we have the correct reference feed
if np.any(np.abs(ref_resp.imag) > 0.0):
ValueError("Reference feed %d is incorrect." % config.eigen_reference)
del ref_resp
# Save index_map
results = {}
results['model'] = model_name
results['param'] = param_name
results['freq'] = data.index_map['freq'][:]
results['input'] = input_axis
results['eval'] = ha_eval_str
results['dir'] = direction
for key, val in model_kwargs:
results[key] = val
# Initialize numpy arrays to hold results
if config.return_response:
results['response'] = np.zeros((nfreq, ninput, ntime),
dtype=np.complex64)
results['response_err'] = np.zeros((nfreq, ninput, ntime),
dtype=np.float32)
results['fit_flag'] = fit_flag
results['ha_axis'] = ha
results['ra'] = ra
else:
results['gain_eval'] = np.zeros((nfreq, ninput, ngain),
dtype=np.complex64)
results['weight_eval'] = np.zeros((nfreq, ninput, ngain),
dtype=np.float32)
results['frac_gain_err'] = np.zeros((nfreq, ninput, ngain, ndir),
dtype=np.float32)
results['parameter'] = np.zeros((nfreq, ninput, nparam),
dtype=np.float32)
results['parameter_err'] = np.zeros((nfreq, ninput, nparam),
dtype=np.float32)
results['index_eval'] = np.full((nfreq, ninput), -1, dtype=np.int8)
results['gain'] = np.zeros((nfreq, ninput), dtype=np.complex64)
results['weight'] = np.zeros((nfreq, ninput), dtype=np.float32)
results['ndof'] = np.zeros((nfreq, ninput, ndir), dtype=np.float32)
results['chisq'] = np.zeros((nfreq, ninput, ndir), dtype=np.float32)
results['timing'] = np.zeros((nfreq, ninput), dtype=np.complex64)
# Initialize metric like variables
results['runtime'] = np.zeros((nblock_freq, 2), dtype=np.float64)
# Compute distances
dist = tools.get_feed_positions(inputmap)
for pp, feeds in enumerate(pol):
dist[feeds, :] -= dist[phase_ref[pp], np.newaxis, :]
# Loop over frequency blocks
for gg in range(nblock_freq):
mlog.info("Frequency block %d of %d." % (gg, nblock_freq))
fstart = gg * config.nfreq_per_block
fstop = min((gg + 1) * config.nfreq_per_block, nfreq)
findex = np.arange(fstart, fstop)
ngroup = findex.size
freq_sel = utils.convert_to_slice(index_freq[findex])
timeit_start_gg = time.time()
#
if config.return_response:
gstart = start
gstop = stop
tslc = slice(0, ntime)
else:
good_times = np.flatnonzero(np.any(fit_flag[findex], axis=(0, 1)))
if good_times.size == 0:
continue
gstart = int(np.min(good_times))
gstop = int(np.max(good_times)) + 1
tslc = slice(gstart, gstop)
gstart += start
gstop += start
hag = ha[tslc]
itrans = np.argmin(np.abs(hag))
# Load eigenvectors.
nudata = andata.CorrData.from_acq_h5(
file_list,
datasets=['evec', 'vis', 'flags/vis_weight'],
apply_gain=False,
freq_sel=freq_sel,
start=gstart,
stop=gstop)
# Save time to load data
results['runtime'][gg, 0] = time.time() - timeit_start_gg
timeit_start_gg = time.time()
mlog.info("Time to load (per frequency): %0.3f sec" %
(results['runtime'][gg, 0] / ngroup, ))
# Loop over polarizations
for pp, feeds in enumerate(pol):
# Get timing correction
if tcorr is not None:
tgain = tcorr.get_gain(nudata.freq, nudata.input[feeds],
nudata.time)
tgain *= tgain[:, phase_ref_by_pol[pp], np.newaxis, :].conj()
tgain_transit = tgain[:, :, itrans].copy()
tgain *= tgain_transit[:, :, np.newaxis].conj()
# Create the polarization masking vector
P = np.zeros((1, ninput, 1), dtype=np.float64)
P[:, feeds, :] = 1.0
# Loop over frequencies
for gff, ff in enumerate(findex):
flg = fit_flag[ff, pp, tslc]
if (2 * int(np.sum(flg))) < (nparam +
1) and not config.return_response:
continue
# Normalize by eigenvalue and correct for pi phase flips in process.
resp = (nudata['evec'][gff, 0:neigen, :, :] *
np.sqrt(data['eval'][ff, 0:neigen, np.newaxis, tslc]) *
sign0[ff, :, np.newaxis, tslc])
# Rotate to single-pol response
# Move time to first axis for the matrix multiplication
invL = tools.invert_no_zero(
np.rollaxis(data['eval'][ff, 0:neigen, np.newaxis, tslc],
-1, 0))
UT = np.rollaxis(resp, -1, 0)
U = np.swapaxes(UT, -1, -2)
mu, vp = np.linalg.eigh(np.matmul(UT.conj(), P * U))
rsign0 = (1.0 - 2.0 * (vp[:, 0, np.newaxis, :].real < 0.0))
resp = mu[:, np.newaxis, :] * np.matmul(U, rsign0 * vp * invL)
# Extract feeds of this pol
# Transpose so that time is back to last axis
resp = resp[:, feeds, -1].T
# Compute error on response
dataflg = ((nudata.weight[gff, feeds, :] > 0.0)
& np.isfinite(nudata.weight[gff, feeds, :])).astype(
np.float32)
resp_err = dataflg * base_err[ff, :, tslc] * np.sqrt(
nudata.vis[gff, feeds, :].real) * tools.invert_no_zero(
np.sqrt(mu[np.newaxis, :, -1]))
# Reference to specific input
resp *= np.exp(
-1.0J *
np.angle(resp[phase_ref_by_pol[pp], np.newaxis, :]))
# Apply timing correction
if tcorr is not None:
resp *= tgain[gff]
results['timing'][ff, feeds] = tgain_transit[gff]
# Fringestop
lmbda = scipy.constants.c * 1e-6 / nudata.freq[gff]
resp *= tools.fringestop_phase(
hag[np.newaxis, :], lat, src_dec,
dist[feeds, 0, np.newaxis] / lmbda,
dist[feeds, 1, np.newaxis] / lmbda)
# Normalize by source flux
resp *= inv_rt_flux_density[ff]
resp_err *= inv_rt_flux_density[ff]
# If requested, reference phase to the median value
if config.med_phase_ref:
phi0 = np.angle(resp[:, itrans, np.newaxis])
resp *= np.exp(-1.0J * phi0)
resp *= np.exp(
-1.0J *
np.median(np.angle(resp), axis=0, keepdims=True))
resp *= np.exp(1.0J * phi0)
# Check if return_response flag was set by user
if not config.return_response:
if config.multi_sample:
moving_window = config.nsigma2 and config.nsigma2 * np.radians(
utils.get_window(nudata.freq[gff],
pol=polstr[pp],
dec=src_dec,
deg=True))
# Loop over inputs
for pii, ii in enumerate(feeds):
is_good = flg & (np.abs(resp[pii, :]) >
0.0) & (resp_err[pii, :] > 0.0)
# Set the intial gains based on raw response at transit
if is_good[itrans]:
results['gain_eval'][ff, ii,
0] = tools.invert_no_zero(
resp[pii, itrans])
results['frac_gain_err'][ff, ii, 0, :] = (
resp_err[pii, itrans] * tools.invert_no_zero(
np.abs(resp[pii, itrans])))
results['weight_eval'][ff, ii, 0] = 0.5 * (
np.abs(resp[pii, itrans])**2 *
tools.invert_no_zero(resp_err[pii, itrans]))**2
results['index_eval'][ff, ii] = 0
results['gain'][ff,
ii] = results['gain_eval'][ff, ii,
0]
results['weight'][ff,
ii] = results['weight_eval'][ff,
ii,
0]
# Exit if not performing multi time sample fit
if not config.multi_sample:
continue
if (2 * int(np.sum(is_good))) < (nparam + 1):
continue
try:
param, param_err, gain, gain_err, ndof, chisq, tval = fitter_function(
hag[is_good],
resp[pii, is_good],
resp_err[pii, is_good],
ha_eval,
window=moving_window,
tval=tval,
**config.fit)
except Exception as rex:
if config.verbose:
mlog.info(
"Frequency %0.2f, Feed %d failed with error: %s"
% (nudata.freq[gff], ii, rex))
continue
# Check for nan
wfit = (np.abs(gain) *
tools.invert_no_zero(np.abs(gain_err)))**2
if np.any(~np.isfinite(np.abs(gain))) or np.any(
~np.isfinite(wfit)):
continue
# Save to results using the convention that you should *multiply* the visibilites by the gains
results['gain_eval'][
ff, ii, fitslc] = tools.invert_no_zero(gain)
results['frac_gain_err'][ff, ii, fitslc,
0] = gain_err.real
results['frac_gain_err'][ff, ii, fitslc,
1] = gain_err.imag
results['weight_eval'][ff, ii, fitslc] = wfit
results['parameter'][ff, ii, :] = param
results['parameter_err'][ff, ii, :] = param_err
results['ndof'][ff, ii, :] = ndof
results['chisq'][ff, ii, :] = chisq
# Check if the fit was succesful and update the gain evaluation index appropriately
if np.all((chisq / ndof.astype(np.float32)
) <= config.chisq_per_dof_threshold):
results['index_eval'][ff, ii] = ind_eval
results['gain'][ff, ii] = results['gain_eval'][
ff, ii, ind_eval]
results['weight'][ff, ii] = results['weight_eval'][
ff, ii, ind_eval]
else:
# Return response only (do not fit model)
results['response'][ff, feeds, :] = resp
results['response_err'][ff, feeds, :] = resp_err
# Save time to fit data
results['runtime'][gg, 1] = time.time() - timeit_start_gg
mlog.info("Time to fit (per frequency): %0.3f sec" %
(results['runtime'][gg, 1] / ngroup, ))
# Clean up
del nudata
gc.collect()
# Print total run time
mlog.info("TOTAL TIME TO LOAD: %0.3f min" %
(np.sum(results['runtime'][:, 0]) / 60.0, ))
mlog.info("TOTAL TIME TO FIT: %0.3f min" %
(np.sum(results['runtime'][:, 1]) / 60.0, ))
# Set the best estimate of the gain
if not config.return_response:
flag = results['index_eval'] >= 0
gain = results['gain']
# Compute amplitude
amp = np.abs(gain)
# Hard cutoffs on the amplitude
med_amp = np.median(amp[flag])
min_amp = med_amp * config.min_amp_scale_factor
max_amp = med_amp * config.max_amp_scale_factor
flag &= ((amp >= min_amp) & (amp <= max_amp))
# Flag outliers in amplitude for each frequency
for pp, feeds in enumerate(pol):
med_amp_by_pol = np.zeros(nfreq, dtype=np.float32)
sig_amp_by_pol = np.zeros(nfreq, dtype=np.float32)
for ff in range(nfreq):
this_flag = flag[ff, feeds]
if np.any(this_flag):
med, slow, shigh = utils.estimate_directional_scale(
amp[ff, feeds[this_flag]])
lower = med - config.nsigma_outlier * slow
upper = med + config.nsigma_outlier * shigh
flag[ff, feeds] &= ((amp[ff, feeds] >= lower) &
(amp[ff, feeds] <= upper))
med_amp_by_pol[ff] = med
sig_amp_by_pol[ff] = 0.5 * (shigh - slow) / np.sqrt(
np.sum(this_flag, dtype=np.float32))
if config.nsigma_med_outlier:
med_flag = med_amp_by_pol > 0.0
not_outlier = flag_outliers(med_amp_by_pol,
med_flag,
window=config.window_med_outlier,
nsigma=config.nsigma_med_outlier)
flag[:, feeds] &= not_outlier[:, np.newaxis]
mlog.info("Pol %s: %d frequencies are outliers." %
(polstr[pp],
np.sum(~not_outlier & med_flag, dtype=np.int)))
# Determine bad frequencies
flag_freq = (np.sum(flag, axis=1, dtype=np.float32) /
float(ninput)) > config.threshold_good_freq
good_freq = np.flatnonzero(flag_freq)
# Determine bad inputs
fraction_good = np.sum(flag[good_freq, :], axis=0,
dtype=np.float32) / float(good_freq.size)
flag_input = fraction_good > config.threshold_good_input
# Finalize flag
flag &= (flag_freq[:, np.newaxis] & flag_input[np.newaxis, :])
# Interpolate gains
interp_gain, interp_weight = interpolate_gain(
freq,
gain,
results['weight'],
flag=flag,
length_scale=config.interpolation_length_scale,
mlog=mlog)
# Save gains to object
results['flag'] = flag
results['gain'] = interp_gain
results['weight'] = interp_weight
# Return results
return results
def solve_ps_transit(filename,
corrs,
feeds,
inp,
src,
nfreq=1024,
transposed=False,
nfeed=128):
""" Function that fringestops time slice
where point source is in the beam, takes
all correlations for a given polarization, and then
eigendecomposes the correlation matrix freq by freq
after removing the fpga phases. It will also
plot intermediate steps to verify the phase solution.
Parameters
----------
filename : np.str
Full-path filename
corrs : list
List of correlations to use in solver
feeds : list
List of feeds to use
inp :
Correlator inputs (output of ch_util.tools.get_correlator_inputs)
src : ephem.FixedBody
Source to calibrate off of. e.g. ch_util.ephemeris.TauA
Returns
-------
Gains : np.array
Complex gain array (nfreq, nfeed)
"""
nsplit = 32 # Number of freq chunks to divide nfreq into
del_t = 800
f = h5py.File(filename, 'r')
# Add half an integration time to each. Hack.
times = f['index_map']['time'].value['ctime'] + 10.50
src_trans = eph.transit_times(src, times[0])
# try to account for differential arrival time from
# cylinder rotation.
del_phi = (src._dec - np.radians(eph.CHIMELATITUDE)) \
* np.sin(np.radians(1.988))
del_phi *= (24 * 3600.0) / (2 * np.pi)
# Adjust the transit time accordingly
src_trans += del_phi
# Select +- del_t of transit, accounting for the mispointing
t_range = np.where((times < src_trans + del_t)
& (times > src_trans - del_t))[0]
print "\n...... This data is from %s starting at RA: %f ...... \n" \
% (eph.unix_to_datetime(times[0]), eph.transit_RA(times[0]))
assert (len(t_range) > 0), "Source is not in this acq"
# Create gains array to fill in solution
Gains = np.zeros([nfreq, nfeed], np.complex128)
print "Starting the solver"
times = times[t_range[0]:t_range[-1]]
k = 0
# Start at a strong freq channel that can be plotted
# and from which we can find the noise source on-sample
for i in range(12, nsplit) + range(0, 12):
k += 1
# Divides the arrays up into nfreq / nsplit freq chunks and solves those
frq = range(i * nfreq // nsplit, (i + 1) * nfreq // nsplit)
print " %d:%d \n" % (frq[0], frq[-1])
# Read in time and freq slice if data has already been transposed
if transposed is True:
v = f['vis'][frq[0]:frq[-1] + 1, corrs, :]
v = v[..., t_range[0]:t_range[-1]]
vis = v['r'] + 1j * v['i']
if k == 1:
autos = auto_corrs(nfeed)
offp = (abs(vis[:, autos, 0::2]).mean() > \
(abs(vis[:, autos, 1::2]).mean())).astype(int)
times = times[offp::2]
vis = vis[..., offp::2]
gg = f['gain_coeff'][frq[0]:frq[-1] + 1, feeds,
t_range[0]:t_range[-1]][..., offp::2]
gain_coeff = gg['r'] + 1j * gg['i']
del gg
# Read in time and freq slice if data has not yet been transposed
if transposed is False:
print "TRANSPOSED V OF CODE DOESN'T WORK YET!"
v = f['vis'][t_range[0]:t_range[-1]:2, frq[0]:frq[-1] + 1, corrs]
vis = v['r'][:] + 1j * v['i'][:]
del v
gg = f['gain_coeff'][0, frq[0]:frq[-1] + 1, feeds]
gain_coeff = gg['r'][:] + 1j * gg['i'][:]
vis = vis[..., offp::2]
vis = np.transpose(vis, (1, 2, 0))
# Remove fpga gains from data
vis = remove_fpga_gains(vis, gain_coeff, nfeed=nfeed, triu=False)
# Remove offset from galaxy
vis -= 0.5 * (vis[..., 0] + vis[..., -1])[..., np.newaxis]
# Get physical freq for fringestopper
freq_MHZ = 800.0 - np.array(frq) / 1024.0 * 400.
baddies = np.where(np.isnan(tools.get_feed_positions(inp)[:, 0]))[0]
a, b, c = select_corrs(baddies, nfeed=128)
vis[:, a + b] = 0.0
# Fringestop to location of "src"
data_fs = tools.fringestop_pathfinder(vis, eph.transit_RA(times),
freq_MHZ, inp, src)
del vis
dr, sol_arr = solve_gain(data_fs)
# Find index of point source transit
drlist = np.argmax(dr, axis=-1)
# If multiple freq channels are zerod, the trans_pix
# will end up being 0. This is bad, so ensure that
# you are only looking for non-zero transit pixels.
drlist = [x for x in drlist if x != 0]
trans_pix = np.argmax(np.bincount(drlist))
assert trans_pix != 0.0
Gains[frq] = sol_arr[..., trans_pix - 3:trans_pix + 4].mean(-1)
zz = h5py.File('data' + str(i) + '.hdf5', 'w')
zz.create_dataset('data', data=dr)
zz.close()
print "%f, %d Nans out of %d" % (np.isnan(sol_arr).sum(),
np.isnan(Gains[frq]).sum(),
np.isnan(Gains[frq]).sum())
print trans_pix, sol_arr[..., trans_pix - 3:trans_pix +
4].mean(-1).sum(), sol_arr.mean(-1).sum()
# Plot up post-fs phases to see if everything has been fixed
if frq[0] == 12 * nsplit:
print "======================"
print " Plotting up freq: %d" % frq[0]
print "======================"
img_nm = './phs_plots/dfs' + np.str(frq[17]) + np.str(
np.int(time.time())) + '.png'
img_nmcorr = './phs_plots/dfs' + np.str(frq[17]) + np.str(
np.int(time.time())) + 'corr.png'
plt_gains(data_fs, 0, img_name=img_nm, bad_chans=baddies)
dfs_corr = correct_dfs(data_fs,
np.angle(Gains[frq])[..., np.newaxis],
nfeed=128)
plt_gains(dfs_corr, 0, img_name=img_nmcorr, bad_chans=baddies)
del dfs_corr
del data_fs, a
return Gains
def main(config_file=None, logging_params=DEFAULT_LOGGING):
# Setup logging
log.setup_logging(logging_params)
mlog = log.get_logger(__name__)
# Set config
config = DEFAULTS.deepcopy()
if config_file is not None:
config.merge(NameSpace(load_yaml_config(config_file)))
# Create transit tracker
source_list = FluxCatalog.sort(
) if not config.source_list else config.source_list
cal_list = [
name for name, obj in FluxCatalog.iteritems()
if (obj.dec >= config.min_dec) and (
obj.predict_flux(config.freq_nominal) >= config.min_flux) and (
name in source_list)
]
if not cal_list:
raise RuntimeError("No calibrators found.")
# Sort list by flux at nominal frequency
cal_list.sort(
key=lambda name: FluxCatalog[name].predict_flux(config.freq_nominal))
# Add to transit tracker
transit_tracker = containers.TransitTrackerOffline(
nsigma=config.nsigma_source, extend_night=config.extend_night)
for name in cal_list:
transit_tracker[name] = FluxCatalog[name].skyfield
mlog.info("Initializing offline point source processing.")
search_time = config.start_time or 0
# Find all calibration files
all_files = sorted(
glob.glob(
os.path.join(config.acq_dir,
'*' + config.correlator + config.acq_suffix, '*.h5')))
if not all_files:
return
# Remove files whose last modified time is before the time of the most recent update
all_files = [
ff for ff in all_files if (os.path.getmtime(ff) > search_time)
]
if not all_files:
return
# Remove files that are currently locked
all_files = [
ff for ff in all_files
if not os.path.isfile(os.path.splitext(ff)[0] + '.lock')
]
if not all_files:
return
# Add files to transit tracker
for ff in all_files:
transit_tracker.add_file(ff)
# Extract point source transits ready for analysis
all_transits = transit_tracker.get_transits()
# Create dictionary to hold results
h5_psrc_fit = {}
inputmap = None
# Loop over transits
for transit in all_transits:
src, csd, is_day, files, start, stop = transit
# Discard any point sources with unusual csd value
if (csd < config.min_csd) or (csd > config.max_csd):
continue
# Discard any point sources transiting during the day
if is_day > config.process_daytime:
continue
mlog.info(
'Processing %s transit on CSD %d (%d files, %d time samples)' %
(src, csd, len(files), stop - start + 1))
# Load inputmap
if inputmap is None:
if config.inputmap is None:
inputmap = tools.get_correlator_inputs(
ephemeris.unix_to_datetime(ephemeris.csd_to_unix(csd)),
correlator=config.correlator)
else:
with open(config.inputmap, 'r') as handler:
inputmap = pickle.load(handler)
# Grab the timing correction for this transit
tcorr = None
if config.apply_timing:
if config.timing_glob is not None:
mlog.info(
"Loading timing correction from extended timing solutions."
)
timing_files = sorted(glob.glob(config.timing_glob))
if timing_files:
try:
tcorr = search_extended_timing_solutions(
timing_files, ephemeris.csd_to_unix(csd))
except Exception as e:
mlog.error(
'search_extended_timing_solutions failed with error: %s'
% e)
else:
mlog.info(str(tcorr))
if tcorr is None:
mlog.info(
"Loading timing correction from chimetiming acquisitions.")
try:
tcorr = timing.load_timing_correction(
files,
start=start,
stop=stop,
window=config.timing_window,
instrument=config.correlator)
except Exception as e:
mlog.error(
'timing.load_timing_correction failed with error: %s' %
e)
mlog.warning(
'No timing correction applied to %s transit on CSD %d.'
% (src, csd))
else:
mlog.info(str(tcorr))
# Call the main routine to process data
try:
outdct = offline_cal.offline_point_source_calibration(
files,
src,
start=start,
stop=stop,
inputmap=inputmap,
tcorr=tcorr,
logging_params=logging_params,
**config.analysis.as_dict())
except Exception as e:
msg = 'offline_cal.offline_point_source_calibration failed with error: %s' % e
mlog.error(msg)
continue
#raise RuntimeError(msg)
# Find existing gain files for this particular point source
if src not in h5_psrc_fit:
output_files = find_files(config, psrc=src)
if output_files is not None:
output_files = output_files[-1]
mlog.info('Writing %s transit on CSD %d to existing file %s.' %
(src, csd, output_files))
h5_psrc_fit[src] = containers.PointSourceWriter(
src,
output_file=output_files,
output_dir=config.output_dir,
output_suffix=point_source_name_to_file_suffix(src),
instrument=config.correlator,
max_file_size=config.max_file_size,
max_num=config.max_num_time,
memory_size=0)
# Associate this gain calibration to the transit time
this_time = ephemeris.transit_times(FluxCatalog[src].skyfield,
ephemeris.csd_to_unix(csd))[0]
outdct['csd'] = csd
outdct['is_daytime'] = is_day
outdct['acquisition'] = os.path.basename(os.path.dirname(files[0]))
# Write to output file
mlog.info('Writing to disk results from %s transit on CSD %d.' %
(src, csd))
h5_psrc_fit[src].write(this_time, **outdct)
# Dump an individual file for this point source transit
mlog.info('Dumping to disk single file for %s transit on CSD %d.' %
(src, csd))
dump_dir = os.path.join(config.output_dir, 'point_source_gains')
containers.mkdir(dump_dir)
dump_file = os.path.join(dump_dir, '%s_csd_%d.h5' % (src.lower(), csd))
h5_psrc_fit[src].dump(dump_file,
datasets=[
'csd', 'acquisition', 'is_daytime', 'gain',
'weight', 'timing', 'model'
])
mlog.info('Finished analysis of %s transit on CSD %d.' % (src, csd))
def fs_from_file(filename,
frq,
src,
del_t=900,
transposed=True,
subtract_avg=False):
f = h5py.File(filename, 'r')
times = f['index_map']['time'].value['ctime'] + 10.6
src_trans = eph.transit_times(src, times[0])
# try to account for differential arrival time from cylinder rotation.
del_phi = (src._dec - np.radians(eph.CHIMELATITUDE)) * np.sin(
np.radians(1.988))
del_phi *= (24 * 3600.0) / (2 * np.pi)
# Adjust the transit time accordingly
src_trans += del_phi
# Select +- del_t of transit, accounting for the mispointing
t_range = np.where((times < src_trans + del_t)
& (times > src_trans - del_t))[0]
times = times[t_range[0]:t_range[-1]] #[offp::2] test
print "Time range:", times[0], times[-1]
print "\n...... This data is from %s starting at RA: %f ...... \n" \
% (eph.unix_to_datetime(times[0]), eph.transit_RA(times[0]))
if transposed is True:
v = f['vis'][frq[0]:frq[-1] + 1, :]
v = v[..., t_range[0]:t_range[-1]]
vis = v['r'] + 1j * v['i']
del v
# Read in time and freq slice if data has not yet been transposed
if transposed is False:
v = f['vis'][t_range[0]:t_range[-1], frq[0]:frq[-1] + 1, :]
vis = v['r'][:] + 1j * v['i'][:]
del v
vis = np.transpose(vis, (1, 2, 0))
inp = gen_inp()[0]
# Remove offset from galaxy
if subtract_avg is True:
vis -= 0.5 * (vis[..., 0] + vis[..., -1])[..., np.newaxis]
freq_MHZ = 800.0 - np.array(frq) / 1024.0 * 400.
print len(inp)
baddies = np.where(np.isnan(tools.get_feed_positions(inp)[:, 0]))[0]
# Fringestop to location of "src"
data_fs = tools.fringestop_pathfinder(vis, eph.transit_RA(times), freq_MHZ,
inp, src)
# data_fs = fringestop_pathfinder(vis, eph.transit_RA(times), freq_MHZ, inp, src)
return data_fs
