def _translate_single_opinion(self, opinion, timestamp):
flag = None
if opinion.decision == "bad":
# Translate from opinions LSD to flags start- and finish-time.
start_time = csd_to_unix(opinion.lsd)
finish_time = csd_to_unix(opinion.lsd + 1)
flag = DataFlag.create_flag(
"vote",
start_time,
finish_time,
opinion.freq,
opinion.instrument,
opinion.inputs,
)
client, _ = DataFlagClient.get_or_create(client_name=__name__,
client_version=__version__)
vote = DataFlagVote.create(
time=timestamp,
mode=self.mode,
client=client,
flag=flag,
revision=self.revision,
lsd=opinion.lsd,
)
DataFlagVoteOpinion.create(vote=vote, opinion=opinion)
return flag
def is_daytime(self, key, csd):
src = self[key] if isinstance(key, basestring) else key
is_daytime = 0
src_ra, src_dec = ephemeris.object_coords(src.body, date=ephemeris.csd_to_unix(csd), deg=True)
transit_start = ephemeris.csd_to_unix(csd + (src_ra - src.window) / 360.0)
transit_end = ephemeris.csd_to_unix(csd + (src_ra + src.window) / 360.0)
solar_rise = ephemeris.solar_rising(transit_start - 24.0*3600.0, end_time=transit_end)
for rr in solar_rise:
ss = ephemeris.solar_setting(rr)[0]
rrex = rr + self._extend_night
ssex = ss - self._extend_night
if ((transit_start <= ssex) and (rrex <= transit_end)):
is_daytime += 1
tt = ephemeris.solar_transit(rr)[0]
if (transit_start <= tt) and (tt <= transit_end):
is_daytime += 1
break
return is_daytime
def _csds_available_data(self, csds, filenames_online,
filenames_that_exist):
"""
Return the subset of csds in `csds` for whom all files are online.
`filenames_online` and `filenames_that_exist` are a list of tuples
(start_time, finish_time)
All 3 lists should be sorted.
"""
csds_available = []
for csd in csds:
start_time = ephemeris.csd_to_unix(csd)
end_time = ephemeris.csd_to_unix(csd + 1)
# online - list of filenames that are online between start_time and end_time
# index_online, the final index in which data was located
online, index_online = self._files_in_timespan(
start_time, end_time, filenames_online)
exists, index_exists = self._files_in_timespan(
start_time, end_time, filenames_that_exist)
if (len(online) == len(exists)) and (len(online) != 0):
csds_available.append(csd)
# The final file in the span may contain more than one sidereal day
index_online = max(index_online - 1, 0)
index_exists = max(index_exists - 1, 0)
filenames_online = filenames_online[index_online:]
filenames_that_exist = filenames_that_exist[index_exists:]
return csds_available
def _flags_mask(self, index_map_ra):
if self._cache_flags:
flag_time_spans = get_flags_cached(self.flags,
self._cache_reset_time)
else:
flag_time_spans = get_flags(
self.flags,
csd_to_unix(self.lsd.lsd),
csd_to_unix(self.lsd.lsd + 1),
)
csd_arr = self.lsd.lsd + index_map_ra / 360.0
flag_mask = np.zeros_like(csd_arr, dtype=np.bool)
for type_, ca, cb in flag_time_spans:
flag_mask[(csd_arr > unix_to_csd(ca))
& (csd_arr < unix_to_csd(cb))] = True
return flag_mask[:, np.newaxis]
def process(self, sstream):
"""Calculate the mean(median) over the sidereal day.
Parameters
----------
sstream : andata.CorrData or containers.SiderealStream
Timestream or sidereal stream.
Returns
-------
mustream : same as sstream
Sidereal stream containing only the mean(median) value.
"""
from .flagging import daytime_flag, transit_flag
# Make sure we are distributed over frequency
sstream.redistribute("freq")
# Extract lsd
lsd = sstream.attrs[
"lsd"] if "lsd" in sstream.attrs else sstream.attrs["csd"]
lsd_list = lsd if hasattr(lsd, "__iter__") else [lsd]
# Calculate the right ascension, method differs depending on input container
if "ra" in sstream.index_map:
ra = sstream.ra
timestamp = {
dd: ephemeris.csd_to_unix(dd + ra / 360.0)
for dd in lsd_list
}
flag_quiet = np.ones(ra.size, dtype=np.bool)
elif "time" in sstream.index_map:
ra = ephemeris.lsa(sstream.time)
timestamp = {lsd: sstream.time}
flag_quiet = np.fix(ephemeris.unix_to_csd(sstream.time)) == lsd
else:
raise RuntimeError("Format of `sstream` argument is unknown.")
# If requested, determine "quiet" region of sky.
# In the case of a SiderealStack, there will be multiple LSDs and the
# mask will be the logical AND of the mask from each individual LSDs.
if self.mask_day:
for dd, time_dd in timestamp.items():
# Mask daytime data
flag_quiet &= ~daytime_flag(time_dd)
if self.mask_sources:
for dd, time_dd in timestamp.items():
# Mask data near bright source transits
for body in self.body:
flag_quiet &= ~transit_flag(
body, time_dd, nsigma=self.nsigma)
if self.mask_ra:
# Only use data within user specified ranges of RA
mask_ra = np.zeros(ra.size, dtype=np.bool)
for ra_range in self.mask_ra:
self.log.info("Using data between RA = [%0.2f, %0.2f] deg" %
tuple(ra_range))
mask_ra |= (ra >= ra_range[0]) & (ra <= ra_range[1])
flag_quiet &= mask_ra
# Create output container
newra = np.mean(ra[flag_quiet], keepdims=True)
mustream = containers.SiderealStream(
ra=newra,
axes_from=sstream,
attrs_from=sstream,
distributed=True,
comm=sstream.comm,
)
mustream.redistribute("freq")
mustream.attrs["statistic"] = self._name_of_statistic
# Dereference visibilities
all_vis = sstream.vis[:].view(np.ndarray)
mu_vis = mustream.vis[:].view(np.ndarray)
# Combine the visibility weights with the quiet flag
all_weight = sstream.weight[:].view(np.ndarray) * flag_quiet.astype(
np.float32)
if not self.inverse_variance:
all_weight = (all_weight > 0.0).astype(np.float32)
# Only include freqs/baselines where enough data is actually present
frac_present = all_weight.sum(axis=-1) / flag_quiet.sum(axis=-1)
all_weight *= (frac_present > self.missing_threshold)[..., np.newaxis]
num_freq_missing_local = int(
(frac_present < self.missing_threshold).all(axis=1).sum())
num_freq_missing = self.comm.allreduce(num_freq_missing_local,
op=MPI.SUM)
self.log.info(
"Cannot estimate a sidereal mean for "
f"{100.0 * num_freq_missing / len(mustream.freq):.2f}% of all frequencies."
)
# Save the total number of nonzero samples as the weight dataset of the output
# container
mustream.weight[:] = np.sum(all_weight, axis=-1, keepdims=True)
# If requested, compute median (requires loop over frequencies and baselines)
if self.median:
mu_vis[..., 0].real = weighted_median(all_vis.real.copy(),
all_weight)
mu_vis[..., 0].imag = weighted_median(all_vis.imag.copy(),
all_weight)
# Where all the weights are zero explicitly set the median to zero
missing = ~(all_weight.any(axis=-1))
mu_vis[missing, 0] = 0.0
else:
# Otherwise calculate the mean
mu_vis[:] = np.sum(all_weight * all_vis, axis=-1, keepdims=True)
mu_vis[:] *= tools.invert_no_zero(mustream.weight[:])
# Return sidereal stream containing the mean value
return mustream
def view(self):
if self.lsd is None:
return panel.pane.Markdown("No data selected.")
try:
if self.intercylinder_only:
name = "ringmap_intercyl"
else:
name = "ringmap"
container = self.data.load_file(self.revision, self.lsd, name)
except DataError as err:
return panel.pane.Markdown(
f"Error: {str(err)}. Please report this problem."
)
# Index map for ra (x-axis)
index_map_ra = container.index_map["ra"]
axis_name_ra = "RA [degrees]"
# Index map for sin(ZA)/sin(theta) (y-axis)
index_map_el = container.index_map["el"]
axis_name_el = "sin(\u03B8)"
# Apply data selections
sel_beam = np.where(container.index_map["beam"] == self.beam)[0]
sel_freq = np.where(
[f[0] for f in container.index_map["freq"]] == self.frequency
)[0]
if self.polarization == self.mean_pol_text:
sel_pol = np.where(
(container.index_map["pol"] == "XX")
| (container.index_map["pol"] == "YY")
)[0]
rmap = np.squeeze(container.map[sel_beam, sel_pol, sel_freq])
rmap = np.nanmean(rmap, axis=0)
else:
sel_pol = np.where(container.index_map["pol"] == self.polarization)[0]
rmap = np.squeeze(container.map[sel_beam, sel_pol, sel_freq])
if self.flag_mask:
rmap = np.where(self._flags_mask(container.index_map["ra"]), np.nan, rmap)
if self.weight_mask:
try:
rms = np.squeeze(container.rms[sel_pol, sel_freq])
except IndexError:
logger.error(
f"rms dataset of ringmap file for rev {self.revision} lsd "
f"{self.lsd} is missing [{sel_pol}, {sel_freq}] (polarization, "
f"frequency). rms has shape {container.rms.shape}"
)
self.weight_mask = False
else:
rmap = np.where(self._weights_mask(rms), np.nan, rmap)
# Set flagged data to nan
rmap = np.where(rmap == 0, np.nan, rmap)
if self.crosstalk_removal:
# The mean of an all-nan slice (masked?) is nan. We don't need a warning about that.
with warnings.catch_warnings():
warnings.filterwarnings("ignore", r"All-NaN slice encountered")
rmap -= np.nanmedian(rmap, axis=0)
if self.template_subtraction:
try:
rm_stack = self.data.load_file_from_path(
self._stack_path, ccontainers.RingMap
)
except DataError as err:
return panel.pane.Markdown(
f"Error: {str(err)}. Please report this problem."
)
# The stack file has all polarizations, so we can't reuse sel_pol
if self.polarization == self.mean_pol_text:
stack_sel_pol = np.where(
(rm_stack.index_map["pol"] == "XX")
| (rm_stack.index_map["pol"] == "YY")
)[0]
else:
stack_sel_pol = np.where(
rm_stack.index_map["pol"] == self.polarization
)[0]
try:
rm_stack = np.squeeze(rm_stack.map[sel_beam, stack_sel_pol, sel_freq])
except IndexError as err:
logger.error(
f"map dataset of ringmap stack file "
f"is missing [{sel_beam}, {stack_sel_pol}, {sel_freq}] (beam, polarization, "
f"frequency). map has shape {rm_stack.map.shape}:\n{err}"
)
self.template_subtraction = False
else:
if self.polarization == self.mean_pol_text:
rm_stack = np.nanmean(rm_stack, axis=0)
# FIXME: this is a hack. remove when rinmap stack file fixed.
rmap -= rm_stack.reshape(rm_stack.shape[0], -1, 2).mean(axis=-1)
if self.transpose:
rmap = rmap.T
index_x = index_map_ra
index_y = index_map_el
axis_names = [axis_name_ra, axis_name_el]
xlim, ylim = self.ylim, self.xlim
else:
index_x = index_map_el
index_y = index_map_ra
axis_names = [axis_name_el, axis_name_ra]
xlim, ylim = self.xlim, self.ylim
img = hv.Image(
(index_x, index_y, rmap),
datatype=["image", "grid"],
kdims=axis_names,
).opts(
clim=self.colormap_range,
logz=self.logarithmic_colorscale,
cmap=process_cmap("inferno", provider="matplotlib"),
colorbar=True,
xlim=xlim,
ylim=ylim,
)
if self.serverside_rendering is not None:
# set colormap
cmap_inferno = copy.copy(matplotlib_cm.get_cmap("inferno"))
cmap_inferno.set_under("black")
cmap_inferno.set_bad("lightgray")
# Set z-axis normalization (other possible values are 'eq_hist', 'cbrt').
if self.logarithmic_colorscale:
normalization = "log"
else:
normalization = "linear"
# datashade/rasterize the image
img = self.serverside_rendering(
img,
cmap=cmap_inferno,
precompute=True,
x_range=xlim,
y_range=ylim,
normalization=normalization,
)
if self.mark_moon:
# Put a ring around the location of the moon if it transits on this day
eph = skyfield_wrapper.ephemeris
# Start and end times of the CSD
st = csd_to_unix(self.lsd.lsd)
et = csd_to_unix(self.lsd.lsd + 1)
moon_time, moon_dec = chime.transit_times(
eph["moon"], st, et, return_dec=True
)
if len(moon_time):
lunar_transit = unix_to_csd(moon_time[0])
lunar_dec = moon_dec[0]
lunar_ra = (lunar_transit % 1) * 360.0
lunar_za = np.sin(np.radians(lunar_dec - 49.0))
if self.transpose:
img *= hv.Ellipse(lunar_ra, lunar_za, (5.5, 0.15))
else:
img *= hv.Ellipse(lunar_za, lunar_ra, (0.04, 21))
if self.mark_day_time:
# Calculate the sun rise/set times on this sidereal day
# Start and end times of the CSD
start_time = csd_to_unix(self.lsd.lsd)
end_time = csd_to_unix(self.lsd.lsd + 1)
times, rises = chime.rise_set_times(
skyfield_wrapper.ephemeris["sun"],
start_time,
end_time,
diameter=-10,
)
sun_rise = 0
sun_set = 0
for t, r in zip(times, rises):
if r:
sun_rise = (unix_to_csd(t) % 1) * 360
else:
sun_set = (unix_to_csd(t) % 1) * 360
# Highlight the day time data
opts = {
"color": "grey",
"alpha": 0.5,
"line_width": 1,
"line_color": "black",
"line_dash": "dashed",
}
if self.transpose:
if sun_rise < sun_set:
img *= hv.VSpan(sun_rise, sun_set).opts(**opts)
else:
img *= hv.VSpan(self.ylim[0], sun_set).opts(**opts)
img *= hv.VSpan(sun_rise, self.ylim[1]).opts(**opts)
else:
if sun_rise < sun_set:
img *= hv.HSpan(sun_rise, sun_set).opts(**opts)
else:
img *= hv.HSpan(self.ylim[0], sun_set).opts(**opts)
img *= hv.HSpan(sun_rise, self.ylim[1]).opts(**opts)
img.opts(
# Fix height, but make width responsive
height=self.height,
responsive=True,
shared_axes=True,
bgcolor="lightgray",
)
return panel.Row(img, width_policy="max")
def main(config_file=None, logging_params=DEFAULT_LOGGING):
# Load config
config = DEFAULTS.deepcopy()
if config_file is not None:
config.merge(NameSpace(load_yaml_config(config_file)))
# Setup logging
log.setup_logging(logging_params)
logger = log.get_logger(__name__)
## Load data for flagging
# Load fpga restarts
time_fpga_restart = []
if config.fpga_restart_file is not None:
with open(config.fpga_restart_file, 'r') as handler:
for line in handler:
time_fpga_restart.append(
ephemeris.datetime_to_unix(
ephemeris.timestr_to_datetime(line.split('_')[0])))
time_fpga_restart = np.array(time_fpga_restart)
# Load housekeeping flag
if config.housekeeping_file is not None:
ftemp = TempData.from_acq_h5(config.housekeeping_file,
datasets=["time_flag"])
else:
ftemp = None
# Load jump data
if config.jump_file is not None:
with h5py.File(config.jump_file, 'r') as handler:
jump_time = handler["time"][:]
jump_size = handler["jump_size"][:]
else:
jump_time = None
jump_size = None
# Load rain data
if config.rain_file is not None:
with h5py.File(config.rain_file, 'r') as handler:
rain_ranges = handler["time_range_conservative"][:]
else:
rain_ranges = []
# Load data flags
data_flags = {}
if config.data_flags:
finder.connect_database()
flag_types = finder.DataFlagType.select()
possible_data_flags = []
for ft in flag_types:
possible_data_flags.append(ft.name)
if ft.name in config.data_flags:
new_data_flags = finder.DataFlag.select().where(
finder.DataFlag.type == ft)
data_flags[ft.name] = list(new_data_flags)
# Set desired range of time
start_time = (ephemeris.datetime_to_unix(
datetime.datetime(
*config.start_date)) if config.start_date is not None else None)
end_time = (ephemeris.datetime_to_unix(datetime.datetime(
*config.end_date)) if config.end_date is not None else None)
## Find gain files
files = {}
for src in config.sources:
files[src] = sorted(
glob.glob(
os.path.join(config.directory, src.lower(),
"%s_%s_lsd_*.h5" % (
config.prefix,
src.lower(),
))))
csd = {}
for src in config.sources:
csd[src] = np.array(
[int(os.path.splitext(ff)[0][-4:]) for ff in files[src]])
for src in config.sources:
logger.info("%s: %d files" % (src, len(csd[src])))
## Remove files that occur during flag
csd_flag = {}
for src in config.sources:
body = ephemeris.source_dictionary[src]
csd_flag[src] = np.ones(csd[src].size, dtype=np.bool)
for ii, cc in enumerate(csd[src][:]):
ttrans = ephemeris.transit_times(body,
ephemeris.csd_to_unix(cc))[0]
if (start_time is not None) and (ttrans < start_time):
csd_flag[src][ii] = False
continue
if (end_time is not None) and (ttrans > end_time):
csd_flag[src][ii] = False
continue
# If requested, remove daytime transits
if not config.include_daytime.get(
src, config.include_daytime.default) and daytime_flag(
ttrans)[0]:
logger.info("%s CSD %d: daytime transit" % (src, cc))
csd_flag[src][ii] = False
continue
# Remove transits during HKP drop out
if ftemp is not None:
itemp = np.flatnonzero(
(ftemp.time[:] >= (ttrans - config.transit_window))
& (ftemp.time[:] <= (ttrans + config.transit_window)))
tempflg = ftemp['time_flag'][itemp]
if (tempflg.size == 0) or ((np.sum(tempflg, dtype=np.float32) /
float(tempflg.size)) < 0.50):
logger.info("%s CSD %d: no housekeeping" % (src, cc))
csd_flag[src][ii] = False
continue
# Remove transits near jumps
if jump_time is not None:
njump = np.sum((jump_size > config.min_jump_size)
& (jump_time > (ttrans - config.jump_window))
& (jump_time < ttrans))
if njump > config.max_njump:
logger.info("%s CSD %d: %d jumps before" %
(src, cc, njump))
csd_flag[src][ii] = False
continue
# Remove transits near rain
for rng in rain_ranges:
if (((ttrans - config.transit_window) <= rng[1])
and ((ttrans + config.transit_window) >= rng[0])):
logger.info("%s CSD %d: during rain" % (src, cc))
csd_flag[src][ii] = False
break
# Remove transits during data flag
for name, flag_list in data_flags.items():
if csd_flag[src][ii]:
for flg in flag_list:
if (((ttrans - config.transit_window) <=
flg.finish_time)
and ((ttrans + config.transit_window) >=
flg.start_time)):
logger.info("%s CSD %d: %s flag" %
(src, cc, name))
csd_flag[src][ii] = False
break
# Print number of files left after flagging
for src in config.sources:
logger.info("%s: %d files (after flagging)" %
(src, np.sum(csd_flag[src])))
## Construct pair wise differences
npair = len(config.diff_pair)
shift = [nd * 24.0 * 3600.0 for nd in config.nday_shift]
calmap = []
calpair = []
for (tsrc, csrc), sh in zip(config.diff_pair, shift):
body_test = ephemeris.source_dictionary[tsrc]
body_cal = ephemeris.source_dictionary[csrc]
for ii, cc in enumerate(csd[tsrc]):
if csd_flag[tsrc][ii]:
test_transit = ephemeris.transit_times(
body_test, ephemeris.csd_to_unix(cc))[0]
cal_transit = ephemeris.transit_times(body_cal,
test_transit + sh)[0]
cal_csd = int(np.fix(ephemeris.unix_to_csd(cal_transit)))
ttrans = np.sort([test_transit, cal_transit])
if cal_csd in csd[csrc]:
jj = list(csd[csrc]).index(cal_csd)
if csd_flag[csrc][jj] and not np.any(
(time_fpga_restart >= ttrans[0])
& (time_fpga_restart <= ttrans[1])):
calmap.append([ii, jj])
calpair.append([tsrc, csrc])
calmap = np.array(calmap)
calpair = np.array(calpair)
ntransit = calmap.shape[0]
logger.info("%d total transit pairs" % ntransit)
for ii in range(ntransit):
t1 = ephemeris.transit_times(
ephemeris.source_dictionary[calpair[ii, 0]],
ephemeris.csd_to_unix(csd[calpair[ii, 0]][calmap[ii, 0]]))[0]
t2 = ephemeris.transit_times(
ephemeris.source_dictionary[calpair[ii, 1]],
ephemeris.csd_to_unix(csd[calpair[ii, 1]][calmap[ii, 1]]))[0]
logger.info("%s (%d) - %s (%d): %0.1f hr" %
(calpair[ii, 0], csd_flag[calpair[ii, 0]][calmap[ii, 0]],
calpair[ii, 1], csd_flag[calpair[ii, 1]][calmap[ii, 1]],
(t1 - t2) / 3600.0))
# Determine unique diff pairs
diff_name = np.array(['%s/%s' % tuple(cp) for cp in calpair])
uniq_diff, lbl_diff, cnt_diff = np.unique(diff_name,
return_inverse=True,
return_counts=True)
ndiff = uniq_diff.size
for ud, udcnt in zip(uniq_diff, cnt_diff):
logger.info("%s: %d transit pairs" % (ud, udcnt))
## Load gains
inputmap = tools.get_correlator_inputs(datetime.datetime.utcnow(),
correlator='chime')
ninput = len(inputmap)
nfreq = 1024
# Set up gain arrays
gain = np.zeros((2, nfreq, ninput, ntransit), dtype=np.complex64)
weight = np.zeros((2, nfreq, ninput, ntransit), dtype=np.float32)
input_sort = np.zeros((2, ninput, ntransit), dtype=np.int)
kcsd = np.zeros((2, ntransit), dtype=np.float32)
timestamp = np.zeros((2, ntransit), dtype=np.float64)
is_daytime = np.zeros((2, ntransit), dtype=np.bool)
for tt in range(ntransit):
for kk, (src, ind) in enumerate(zip(calpair[tt], calmap[tt])):
body = ephemeris.source_dictionary[src]
filename = files[src][ind]
logger.info("%s: %s" % (src, filename))
temp = containers.StaticGainData.from_file(filename)
freq = temp.freq[:]
inputs = temp.input[:]
isort = reorder_inputs(inputmap, inputs)
inputs = inputs[isort]
gain[kk, :, :, tt] = temp.gain[:, isort]
weight[kk, :, :, tt] = temp.weight[:, isort]
input_sort[kk, :, tt] = isort
kcsd[kk, tt] = temp.attrs['lsd']
timestamp[kk, tt] = ephemeris.transit_times(
body, ephemeris.csd_to_unix(kcsd[kk, tt]))[0]
is_daytime[kk, tt] = daytime_flag(timestamp[kk, tt])[0]
if np.any(isort != np.arange(isort.size)):
logger.info("Input ordering has changed: %s" %
ephemeris.unix_to_datetime(
timestamp[kk, tt]).strftime("%Y-%m-%d"))
logger.info("")
inputs = np.array([(inp.id, inp.input_sn) for inp in inputmap],
dtype=[('chan_id', 'u2'), ('correlator_input', 'S32')])
## Load input flags
inpflg = np.ones((2, ninput, ntransit), dtype=np.bool)
min_flag_time = np.min(timestamp) - 7.0 * 24.0 * 60.0 * 60.0
max_flag_time = np.max(timestamp) + 7.0 * 24.0 * 60.0 * 60.0
flaginput_files = sorted(
glob.glob(
os.path.join(config.flaginput_dir, "*" + config.flaginput_suffix,
"*.h5")))
if flaginput_files:
logger.info("Found %d flaginput files." % len(flaginput_files))
tmp = andata.FlagInputData.from_acq_h5(flaginput_files, datasets=())
start, stop = [
int(yy) for yy in np.percentile(
np.flatnonzero((tmp.time[:] >= min_flag_time)
& (tmp.time[:] <= max_flag_time)), [0, 100])
]
cont = andata.FlagInputData.from_acq_h5(flaginput_files,
start=start,
stop=stop,
datasets=['flag'])
for kk in range(2):
inpflg[kk, :, :] = cont.resample('flag',
timestamp[kk],
transpose=True)
logger.info("Flaginput time offsets in minutes (pair %d):" % kk)
logger.info(
str(
np.fix((cont.time[cont.search_update_time(timestamp[kk])] -
timestamp[kk]) / 60.0).astype(np.int)))
# Sort flags so they are in same order
for tt in range(ntransit):
for kk in range(2):
inpflg[kk, :, tt] = inpflg[kk, input_sort[kk, :, tt], tt]
# Do not apply input flag to phase reference
for ii in config.index_phase_ref:
inpflg[:, ii, :] = True
## Flag out gains with high uncertainty and frequencies with large fraction of data flagged
frac_err = tools.invert_no_zero(np.sqrt(weight) * np.abs(gain))
flag = np.all((weight > 0.0) & (np.abs(gain) > 0.0) &
(frac_err < config.max_uncertainty),
axis=0)
freq_flag = ((np.sum(flag, axis=(1, 2), dtype=np.float32) /
float(np.prod(flag.shape[1:]))) > config.freq_threshold)
if config.apply_rfi_mask:
freq_flag &= np.logical_not(rfi.frequency_mask(freq))
flag = flag & freq_flag[:, np.newaxis, np.newaxis]
good_freq = np.flatnonzero(freq_flag)
logger.info("Number good frequencies %d" % good_freq.size)
## Generate flags with more conservative cuts on frequency
c_flag = flag & np.all(frac_err < config.conservative.max_uncertainty,
axis=0)
c_freq_flag = ((np.sum(c_flag, axis=(1, 2), dtype=np.float32) /
float(np.prod(c_flag.shape[1:]))) >
config.conservative.freq_threshold)
if config.conservative.apply_rfi_mask:
c_freq_flag &= np.logical_not(rfi.frequency_mask(freq))
c_flag = c_flag & c_freq_flag[:, np.newaxis, np.newaxis]
c_good_freq = np.flatnonzero(c_freq_flag)
logger.info("Number good frequencies (conservative thresholds) %d" %
c_good_freq.size)
## Apply input flags
flag &= np.all(inpflg[:, np.newaxis, :, :], axis=0)
## Update flags based on beam flag
if config.beam_flag_file is not None:
dbeam = andata.BaseData.from_acq_h5(config.beam_flag_file)
db_csd = np.floor(ephemeris.unix_to_csd(dbeam.index_map['time'][:]))
for ii, name in enumerate(config.beam_flag_datasets):
logger.info("Applying %s beam flag." % name)
if not ii:
db_flag = dbeam.flags[name][:]
else:
db_flag &= dbeam.flags[name][:]
cnt = 0
for ii, dbc in enumerate(db_csd):
this_csd = np.flatnonzero(np.any(kcsd == dbc, axis=0))
if this_csd.size > 0:
logger.info("Beam flag for %d matches %s." %
(dbc, str(kcsd[:, this_csd])))
flag[:, :, this_csd] &= db_flag[np.newaxis, :, ii, np.newaxis]
cnt += 1
logger.info("Applied %0.1f percent of the beam flags" %
(100.0 * cnt / float(db_csd.size), ))
## Flag inputs with large amount of missing data
input_frac_flagged = (
np.sum(flag[good_freq, :, :], axis=(0, 2), dtype=np.float32) /
float(good_freq.size * ntransit))
input_flag = input_frac_flagged > config.input_threshold
for ii in config.index_phase_ref:
logger.info("Phase reference %d has %0.3f fraction of data flagged." %
(ii, input_frac_flagged[ii]))
input_flag[ii] = True
good_input = np.flatnonzero(input_flag)
flag = flag & input_flag[np.newaxis, :, np.newaxis]
logger.info("Number good inputs %d" % good_input.size)
## Calibrate
gaincal = gain[0] * tools.invert_no_zero(gain[1])
frac_err_cal = np.sqrt(frac_err[0]**2 + frac_err[1]**2)
count = np.sum(flag, axis=-1, dtype=np.int)
stat_flag = count > config.min_num_transit
## Calculate phase
amp = np.abs(gaincal)
phi = np.angle(gaincal)
## Calculate polarisation groups
pol_dict = {'E': 'X', 'S': 'Y'}
cyl_dict = {2: 'A', 3: 'B', 4: 'C', 5: 'D'}
if config.group_by_cyl:
group_id = [
(inp.pol,
inp.cyl) if tools.is_chime(inp) and (ii in good_input) else None
for ii, inp in enumerate(inputmap)
]
else:
group_id = [
inp.pol if tools.is_chime(inp) and (ii in good_input) else None
for ii, inp in enumerate(inputmap)
]
ugroup_id = sorted([uidd for uidd in set(group_id) if uidd is not None])
ngroup = len(ugroup_id)
group_list_noref = [
np.array([
gg for gg, gid in enumerate(group_id)
if (gid == ugid) and gg not in config.index_phase_ref
]) for ugid in ugroup_id
]
group_list = [
np.array([gg for gg, gid in enumerate(group_id) if gid == ugid])
for ugid in ugroup_id
]
if config.group_by_cyl:
group_str = [
"%s-%s" % (pol_dict[pol], cyl_dict[cyl]) for pol, cyl in ugroup_id
]
else:
group_str = [pol_dict[pol] for pol in ugroup_id]
index_phase_ref = []
for gstr, igroup in zip(group_str, group_list):
candidate = [ii for ii in config.index_phase_ref if ii in igroup]
if len(candidate) != 1:
index_phase_ref.append(None)
else:
index_phase_ref.append(candidate[0])
logger.info(
"Phase reference: %s" %
', '.join(['%s = %s' % tpl
for tpl in zip(group_str, index_phase_ref)]))
## Apply thermal correction to amplitude
if config.amp_thermal.enabled:
logger.info("Applying thermal correction.")
# Load the temperatures
tdata = TempData.from_acq_h5(config.amp_thermal.filename)
index = tdata.search_sensors(config.amp_thermal.sensor)[0]
temp = tdata.datasets[config.amp_thermal.field][index]
temp_func = scipy.interpolate.interp1d(tdata.time, temp,
**config.amp_thermal.interp)
itemp = temp_func(timestamp)
dtemp = itemp[0] - itemp[1]
flag_func = scipy.interpolate.interp1d(
tdata.time, tdata.datasets['flag'][index].astype(np.float32),
**config.amp_thermal.interp)
dtemp_flag = np.all(flag_func(timestamp) == 1.0, axis=0)
flag &= dtemp_flag[np.newaxis, np.newaxis, :]
for gstr, igroup in zip(group_str, group_list):
pstr = gstr[0]
thermal_coeff = np.polyval(config.amp_thermal.coeff[pstr], freq)
gthermal = 1.0 + thermal_coeff[:, np.newaxis, np.newaxis] * dtemp[
np.newaxis, np.newaxis, :]
amp[:, igroup, :] *= tools.invert_no_zero(gthermal)
## Compute common mode
if config.subtract_common_mode_before:
logger.info("Calculating common mode amplitude and phase.")
cmn_amp, flag_cmn_amp = compute_common_mode(amp,
flag,
group_list_noref,
median=False)
cmn_phi, flag_cmn_phi = compute_common_mode(phi,
flag,
group_list_noref,
median=False)
# Subtract common mode (from phase only)
logger.info("Subtracting common mode phase.")
group_flag = np.zeros((ngroup, ninput), dtype=np.bool)
for gg, igroup in enumerate(group_list):
group_flag[gg, igroup] = True
phi[:,
igroup, :] = phi[:, igroup, :] - cmn_phi[:, gg, np.newaxis, :]
for iref in index_phase_ref:
if (iref is not None) and (iref in igroup):
flag[:, iref, :] = flag_cmn_phi[:, gg, :]
## If requested, determine and subtract a delay template
if config.fit_delay_before:
logger.info("Fitting delay template.")
omega = timing.FREQ_TO_OMEGA * freq
tau, tau_flag, _ = construct_delay_template(
omega,
phi,
c_flag & flag,
min_num_freq_for_delay_fit=config.min_num_freq_for_delay_fit)
# Compute residuals
logger.info("Subtracting delay template.")
phi = phi - tau[np.newaxis, :, :] * omega[:, np.newaxis, np.newaxis]
## Normalize by median over time
logger.info("Calculating median amplitude and phase.")
med_amp = np.zeros((nfreq, ninput, ndiff), dtype=amp.dtype)
med_phi = np.zeros((nfreq, ninput, ndiff), dtype=phi.dtype)
count_by_diff = np.zeros((nfreq, ninput, ndiff), dtype=np.int)
stat_flag_by_diff = np.zeros((nfreq, ninput, ndiff), dtype=np.bool)
def weighted_mean(yy, ww, axis=-1):
return np.sum(ww * yy, axis=axis) * tools.invert_no_zero(
np.sum(ww, axis=axis))
for dd in range(ndiff):
this_diff = np.flatnonzero(lbl_diff == dd)
this_flag = flag[:, :, this_diff]
this_amp = amp[:, :, this_diff]
this_amp_err = this_amp * frac_err_cal[:, :,
this_diff] * this_flag.astype(
np.float32)
this_phi = phi[:, :, this_diff]
this_phi_err = frac_err_cal[:, :, this_diff] * this_flag.astype(
np.float32)
count_by_diff[:, :, dd] = np.sum(this_flag, axis=-1, dtype=np.int)
stat_flag_by_diff[:, :,
dd] = count_by_diff[:, :,
dd] > config.min_num_transit
if config.weighted_mean == 2:
logger.info("Calculating inverse variance weighted mean.")
med_amp[:, :,
dd] = weighted_mean(this_amp,
tools.invert_no_zero(this_amp_err**2),
axis=-1)
med_phi[:, :,
dd] = weighted_mean(this_phi,
tools.invert_no_zero(this_phi_err**2),
axis=-1)
elif config.weighted_mean == 1:
logger.info("Calculating uniform weighted mean.")
med_amp[:, :, dd] = weighted_mean(this_amp,
this_flag.astype(np.float32),
axis=-1)
med_phi[:, :, dd] = weighted_mean(this_phi,
this_flag.astype(np.float32),
axis=-1)
else:
logger.info("Calculating median value.")
for ff in range(nfreq):
for ii in range(ninput):
if np.any(this_flag[ff, ii, :]):
med_amp[ff, ii, dd] = wq.median(
this_amp[ff, ii, :],
this_flag[ff, ii, :].astype(np.float32))
med_phi[ff, ii, dd] = wq.median(
this_phi[ff, ii, :],
this_flag[ff, ii, :].astype(np.float32))
damp = np.zeros_like(amp)
dphi = np.zeros_like(phi)
for dd in range(ndiff):
this_diff = np.flatnonzero(lbl_diff == dd)
damp[:, :, this_diff] = amp[:, :, this_diff] * tools.invert_no_zero(
med_amp[:, :, dd, np.newaxis]) - 1.0
dphi[:, :,
this_diff] = phi[:, :, this_diff] - med_phi[:, :, dd, np.newaxis]
# Compute common mode
if not config.subtract_common_mode_before:
logger.info("Calculating common mode amplitude and phase.")
cmn_amp, flag_cmn_amp = compute_common_mode(damp,
flag,
group_list_noref,
median=True)
cmn_phi, flag_cmn_phi = compute_common_mode(dphi,
flag,
group_list_noref,
median=True)
# Subtract common mode (from phase only)
logger.info("Subtracting common mode phase.")
group_flag = np.zeros((ngroup, ninput), dtype=np.bool)
for gg, igroup in enumerate(group_list):
group_flag[gg, igroup] = True
dphi[:, igroup, :] = dphi[:, igroup, :] - cmn_phi[:, gg,
np.newaxis, :]
for iref in index_phase_ref:
if (iref is not None) and (iref in igroup):
flag[:, iref, :] = flag_cmn_phi[:, gg, :]
## Compute RMS
logger.info("Calculating RMS of amplitude and phase.")
mad_amp = np.zeros((nfreq, ninput), dtype=amp.dtype)
std_amp = np.zeros((nfreq, ninput), dtype=amp.dtype)
mad_phi = np.zeros((nfreq, ninput), dtype=phi.dtype)
std_phi = np.zeros((nfreq, ninput), dtype=phi.dtype)
mad_amp_by_diff = np.zeros((nfreq, ninput, ndiff), dtype=amp.dtype)
std_amp_by_diff = np.zeros((nfreq, ninput, ndiff), dtype=amp.dtype)
mad_phi_by_diff = np.zeros((nfreq, ninput, ndiff), dtype=phi.dtype)
std_phi_by_diff = np.zeros((nfreq, ninput, ndiff), dtype=phi.dtype)
for ff in range(nfreq):
for ii in range(ninput):
this_flag = flag[ff, ii, :]
if np.any(this_flag):
std_amp[ff, ii] = np.std(damp[ff, ii, this_flag])
std_phi[ff, ii] = np.std(dphi[ff, ii, this_flag])
mad_amp[ff, ii] = 1.48625 * wq.median(
np.abs(damp[ff, ii, :]), this_flag.astype(np.float32))
mad_phi[ff, ii] = 1.48625 * wq.median(
np.abs(dphi[ff, ii, :]), this_flag.astype(np.float32))
for dd in range(ndiff):
this_diff = this_flag & (lbl_diff == dd)
if np.any(this_diff):
std_amp_by_diff[ff, ii, dd] = np.std(damp[ff, ii,
this_diff])
std_phi_by_diff[ff, ii, dd] = np.std(dphi[ff, ii,
this_diff])
mad_amp_by_diff[ff, ii, dd] = 1.48625 * wq.median(
np.abs(damp[ff, ii, :]),
this_diff.astype(np.float32))
mad_phi_by_diff[ff, ii, dd] = 1.48625 * wq.median(
np.abs(dphi[ff, ii, :]),
this_diff.astype(np.float32))
## Construct delay template
if not config.fit_delay_before:
logger.info("Fitting delay template.")
omega = timing.FREQ_TO_OMEGA * freq
tau, tau_flag, _ = construct_delay_template(
omega,
dphi,
c_flag & flag,
min_num_freq_for_delay_fit=config.min_num_freq_for_delay_fit)
# Compute residuals
logger.info("Subtracting delay template from phase.")
resid = (dphi - tau[np.newaxis, :, :] *
omega[:, np.newaxis, np.newaxis]) * flag.astype(np.float32)
else:
resid = dphi
tau_count = np.sum(tau_flag, axis=-1, dtype=np.int)
tau_stat_flag = tau_count > config.min_num_transit
tau_count_by_diff = np.zeros((ninput, ndiff), dtype=np.int)
tau_stat_flag_by_diff = np.zeros((ninput, ndiff), dtype=np.bool)
for dd in range(ndiff):
this_diff = np.flatnonzero(lbl_diff == dd)
tau_count_by_diff[:, dd] = np.sum(tau_flag[:, this_diff],
axis=-1,
dtype=np.int)
tau_stat_flag_by_diff[:,
dd] = tau_count_by_diff[:,
dd] > config.min_num_transit
## Calculate statistics of residuals
std_resid = np.zeros((nfreq, ninput), dtype=phi.dtype)
mad_resid = np.zeros((nfreq, ninput), dtype=phi.dtype)
std_resid_by_diff = np.zeros((nfreq, ninput, ndiff), dtype=phi.dtype)
mad_resid_by_diff = np.zeros((nfreq, ninput, ndiff), dtype=phi.dtype)
for ff in range(nfreq):
for ii in range(ninput):
this_flag = flag[ff, ii, :]
if np.any(this_flag):
std_resid[ff, ii] = np.std(resid[ff, ii, this_flag])
mad_resid[ff, ii] = 1.48625 * wq.median(
np.abs(resid[ff, ii, :]), this_flag.astype(np.float32))
for dd in range(ndiff):
this_diff = this_flag & (lbl_diff == dd)
if np.any(this_diff):
std_resid_by_diff[ff, ii,
dd] = np.std(resid[ff, ii,
this_diff])
mad_resid_by_diff[ff, ii, dd] = 1.48625 * wq.median(
np.abs(resid[ff, ii, :]),
this_diff.astype(np.float32))
## Calculate statistics of delay template
mad_tau = np.zeros((ninput, ), dtype=phi.dtype)
std_tau = np.zeros((ninput, ), dtype=phi.dtype)
mad_tau_by_diff = np.zeros((ninput, ndiff), dtype=phi.dtype)
std_tau_by_diff = np.zeros((ninput, ndiff), dtype=phi.dtype)
for ii in range(ninput):
this_flag = tau_flag[ii]
if np.any(this_flag):
std_tau[ii] = np.std(tau[ii, this_flag])
mad_tau[ii] = 1.48625 * wq.median(np.abs(tau[ii]),
this_flag.astype(np.float32))
for dd in range(ndiff):
this_diff = this_flag & (lbl_diff == dd)
if np.any(this_diff):
std_tau_by_diff[ii, dd] = np.std(tau[ii, this_diff])
mad_tau_by_diff[ii, dd] = 1.48625 * wq.median(
np.abs(tau[ii]), this_diff.astype(np.float32))
## Define output
res = {
"timestamp": {
"data": timestamp,
"axis": ["div", "time"]
},
"is_daytime": {
"data": is_daytime,
"axis": ["div", "time"]
},
"csd": {
"data": kcsd,
"axis": ["div", "time"]
},
"pair_map": {
"data": lbl_diff,
"axis": ["time"]
},
"pair_count": {
"data": cnt_diff,
"axis": ["pair"]
},
"gain": {
"data": gaincal,
"axis": ["freq", "input", "time"]
},
"frac_err": {
"data": frac_err_cal,
"axis": ["freq", "input", "time"]
},
"flags/gain": {
"data": flag,
"axis": ["freq", "input", "time"],
"flag": True
},
"flags/gain_conservative": {
"data": c_flag,
"axis": ["freq", "input", "time"],
"flag": True
},
"flags/count": {
"data": count,
"axis": ["freq", "input"],
"flag": True
},
"flags/stat": {
"data": stat_flag,
"axis": ["freq", "input"],
"flag": True
},
"flags/count_by_pair": {
"data": count_by_diff,
"axis": ["freq", "input", "pair"],
"flag": True
},
"flags/stat_by_pair": {
"data": stat_flag_by_diff,
"axis": ["freq", "input", "pair"],
"flag": True
},
"med_amp": {
"data": med_amp,
"axis": ["freq", "input", "pair"]
},
"med_phi": {
"data": med_phi,
"axis": ["freq", "input", "pair"]
},
"flags/group_flag": {
"data": group_flag,
"axis": ["group", "input"],
"flag": True
},
"cmn_amp": {
"data": cmn_amp,
"axis": ["freq", "group", "time"]
},
"cmn_phi": {
"data": cmn_phi,
"axis": ["freq", "group", "time"]
},
"amp": {
"data": damp,
"axis": ["freq", "input", "time"]
},
"phi": {
"data": dphi,
"axis": ["freq", "input", "time"]
},
"std_amp": {
"data": std_amp,
"axis": ["freq", "input"]
},
"std_amp_by_pair": {
"data": std_amp_by_diff,
"axis": ["freq", "input", "pair"]
},
"mad_amp": {
"data": mad_amp,
"axis": ["freq", "input"]
},
"mad_amp_by_pair": {
"data": mad_amp_by_diff,
"axis": ["freq", "input", "pair"]
},
"std_phi": {
"data": std_phi,
"axis": ["freq", "input"]
},
"std_phi_by_pair": {
"data": std_phi_by_diff,
"axis": ["freq", "input", "pair"]
},
"mad_phi": {
"data": mad_phi,
"axis": ["freq", "input"]
},
"mad_phi_by_pair": {
"data": mad_phi_by_diff,
"axis": ["freq", "input", "pair"]
},
"tau": {
"data": tau,
"axis": ["input", "time"]
},
"flags/tau": {
"data": tau_flag,
"axis": ["input", "time"],
"flag": True
},
"flags/tau_count": {
"data": tau_count,
"axis": ["input"],
"flag": True
},
"flags/tau_stat": {
"data": tau_stat_flag,
"axis": ["input"],
"flag": True
},
"flags/tau_count_by_pair": {
"data": tau_count_by_diff,
"axis": ["input", "pair"],
"flag": True
},
"flags/tau_stat_by_pair": {
"data": tau_stat_flag_by_diff,
"axis": ["input", "pair"],
"flag": True
},
"std_tau": {
"data": std_tau,
"axis": ["input"]
},
"std_tau_by_pair": {
"data": std_tau_by_diff,
"axis": ["input", "pair"]
},
"mad_tau": {
"data": mad_tau,
"axis": ["input"]
},
"mad_tau_by_pair": {
"data": mad_tau_by_diff,
"axis": ["input", "pair"]
},
"resid_phi": {
"data": resid,
"axis": ["freq", "input", "time"]
},
"std_resid_phi": {
"data": std_resid,
"axis": ["freq", "input"]
},
"std_resid_phi_by_pair": {
"data": std_resid_by_diff,
"axis": ["freq", "input", "pair"]
},
"mad_resid_phi": {
"data": mad_resid,
"axis": ["freq", "input"]
},
"mad_resid_phi_by_pair": {
"data": mad_resid_by_diff,
"axis": ["freq", "input", "pair"]
},
}
## Create the output container
logger.info("Creating StabilityData container.")
data = StabilityData()
data.create_index_map(
"div", np.array(["numerator", "denominator"], dtype=np.string_))
data.create_index_map("pair", np.array(uniq_diff, dtype=np.string_))
data.create_index_map("group", np.array(group_str, dtype=np.string_))
data.create_index_map("freq", freq)
data.create_index_map("input", inputs)
data.create_index_map("time", timestamp[0, :])
logger.info("Writing datsets to container.")
for name, dct in res.iteritems():
is_flag = dct.get('flag', False)
if is_flag:
dset = data.create_flag(name.split('/')[-1], data=dct['data'])
else:
dset = data.create_dataset(name, data=dct['data'])
dset.attrs['axis'] = np.array(dct['axis'], dtype=np.string_)
data.attrs['phase_ref'] = np.array(
[iref for iref in index_phase_ref if iref is not None])
# Determine the output filename and save results
start_time, end_time = ephemeris.unix_to_datetime(
np.percentile(timestamp, [0, 100]))
tfmt = "%Y%m%d"
night_str = 'night_' if not np.any(is_daytime) else ''
output_file = os.path.join(
config.output_dir, "%s_%s_%sraw_stability_data.h5" %
(start_time.strftime(tfmt), end_time.strftime(tfmt), night_str))
logger.info("Saving results to %s." % output_file)
data.save(output_file)
def process(self, sstream, suntrans, inputmap):
"""Clean the sun.
Parameters
----------
sstream : containers.SiderealStream
Sidereal stream.
suntrans : containers.SolarTransit
Response to the sun.
inputmap : list of :class:`CorrInput`
A list describing the inputs as they are in the file.
Returns
-------
mstream : containers.SiderealStream
Sidereal stream with sun removed
"""
sstream.redistribute("freq")
suntrans.redistribute("freq")
# Determine time mapping
if hasattr(sstream, "time"):
stime = sstream.time[:]
else:
ra = sstream.index_map["ra"][:]
csd = (sstream.attrs["lsd"]
if "lsd" in sstream.attrs else sstream.attrs["csd"])
stime = ephemeris.csd_to_unix(csd + ra / 360.0)
# Extract gain array
gtime = suntrans.time[:]
gain = suntrans.response[:].view(np.ndarray)
ninput = gain.shape[1]
# Determine product map
prod_map = sstream.index_map["prod"][:]
nprod = prod_map.size
if nprod != (ninput * (ninput + 1) // 2):
raise Exception(
"Number of inputs does not match the number of products.")
feed_list = [(inputmap[ii], inputmap[jj]) for ii, jj in prod_map]
# Determine polarisation for each visibility
same_pol = np.zeros(nprod, dtype=np.bool)
for pp, (ii, jj) in enumerate(feed_list):
if tools.is_chime(ii) and tools.is_chime(jj):
same_pol[pp] = tools.is_chime_y(ii) == tools.is_chime_y(jj)
# Match ra
match = np.array([np.argmin(np.abs(gt - stime)) for gt in gtime])
# Loop over frequencies and products
for lfi, fi in sstream.vis[:].enumerate(0):
for pp in range(nprod):
if same_pol[pp]:
ii, jj = prod_map[pp]
# Fetch the gains
gi = gain[lfi, ii, :]
gj = gain[lfi, jj, :].conj()
# Subtract the gains
sstream.vis[fi, pp, match] -= gi * gj
# Return the clean sidereal stream
return sstream
def end_time(self):
return csd_to_unix(self._lsd + 1)
def from_lsd(cls, lsd: int):
unix = csd_to_unix(lsd)
lsd = int(lsd)
date = unix_to_datetime(unix).date()
day = cls(lsd, date)
return day
def _available_files(self, start_csd, end_csd):
"""
Return chimestack files available in cedar_online between start_csd and end_csd, if all of the files for that period are available online.
Return an empty list if files between start_csd and end_csd are only partially available online.
Total file count is verified by checking files that exist everywhere.
Parameters
----------
start_csd : int
Start date in sidereal day format
end_csd : int
End date in sidereal day format
Returns
-------
list
List contains the chimestack files available in the timespan, if all of them are available online
"""
# Connect to databases
db.connect()
# Get timestamps in unix format
# Needed for queries
start_time = ephemeris.csd_to_unix(start_csd)
end_time = ephemeris.csd_to_unix(end_csd)
# We will want to know which files are in chime_online and nearline on cedar
online_node = di.StorageNode.get(name="cedar_online", active=True)
chimestack_inst = di.ArchiveInst.get(name="chimestack")
# TODO if the time range is so small that it’s completely contained within a single file, nothing will be returned
# have to special-case it by looking for files which start before the start time and end after the end time).
archive_files = (di.ArchiveFileCopy.select(
di.CorrFileInfo.start_time,
di.CorrFileInfo.finish_time,
).join(di.ArchiveFile).join(di.ArchiveAcq).switch(di.ArchiveFile).join(
di.CorrFileInfo))
# chimestack files available online which include between start and end_time
files_that_exist = archive_files.where(
di.ArchiveAcq.inst ==
chimestack_inst, # specifically looking for chimestack files
di.CorrFileInfo.start_time <
end_time, # which contain data that includes start time and end time
di.CorrFileInfo.finish_time >= start_time,
di.ArchiveFileCopy.has_file == "Y",
)
files_online = files_that_exist.where(
di.ArchiveFileCopy.node == online_node, # that are online
)
filenames_online = sorted([t for t in files_online.tuples()])
# files_that_exist might contain the same file multiple files
# if it exists in multiple locations (nearline, online, gossec, etc)
# we only want to include it once
filenames_that_exist = sorted(
list(set(t for t in files_that_exist.tuples())))
return filenames_online, filenames_that_exist
def process(self, tstream):
"""Apply the timing correction to the input timestream.
Parameters
----------
tstream : andata.CorrData, containers.TimeStream, or containers.SiderealStream
Apply the timing correction to the visibilities stored in this container.
Returns
-------
tstream_corr : same as `tstream`
Timestream with corrected visibilities.
"""
# Determine times
if "time" in tstream.index_map:
timestamp = tstream.time
else:
csd = (tstream.attrs["lsd"]
if "lsd" in tstream.attrs else tstream.attrs["csd"])
timestamp = ephemeris.csd_to_unix(csd + tstream.ra / 360.0)
# Extract local frequencies
tstream.redistribute("freq")
nfreq = tstream.vis.local_shape[0]
sfreq = tstream.vis.local_offset[0]
efreq = sfreq + nfreq
freq = tstream.freq[sfreq:efreq]
# If requested, extract the input flags
input_flags = tstream.input_flags[:] if self.use_input_flags else None
# Check needs_timing_correction flags time ranges to see if input timestream
# falls within the interval
needs_timing_correction = False
for flag in self.flags:
if (timestamp[0] >= flag["start_time"]
and timestamp[0] <= flag["finish_time"]):
if timestamp[-1] >= flag["finish_time"]:
raise PipelineRuntimeError(
f"Data covering {timestamp[0]} to {timestamp[-1]} partially overlaps "
f"needs_timing_correction DataFlag covering {ephemeris.unix_to_datetime(flag['start_time']).strftime('%Y%m%dT%H%M%SZ')} "
f"to {ephemeris.unix_to_datetime(flag['finish_time']).strftime('%Y%m%dT%H%M%SZ')}."
)
else:
self.log.info(
f"Data covering {timestamp[0]} to {timestamp[-1]} flagged by "
f"needs_timing_correction DataFlag covering {ephemeris.unix_to_datetime(flag['start_time']).strftime('%Y%m%dT%H%M%SZ')} "
f"to {ephemeris.unix_to_datetime(flag['finish_time']).strftime('%Y%m%dT%H%M%SZ')}. Timing correction will be applied."
)
needs_timing_correction = True
break
if not needs_timing_correction:
self.log.info(
f"Data in span {ephemeris.unix_to_datetime(timestamp[0]).strftime('%Y%m%dT%H%M%SZ')} to {ephemeris.unix_to_datetime(timestamp[-1]).strftime('%Y%m%dT%H%M%SZ')} does not need timing correction"
)
return tstream
# Find the right timing correction
for tcorr in self.tcorr:
if timestamp[0] >= tcorr.time[0] and timestamp[-1] <= tcorr.time[
-1]:
break
else:
msg = (
"Could not find timing correction file covering "
"range of timestream data (%s to %s)." % tuple(
ephemeris.unix_to_datetime([timestamp[0], timestamp[-1]])))
if self.pass_if_missing:
self.log.warning(msg + " Doing nothing.")
return tstream
raise RuntimeError(msg)
self.log.info("Using correction file %s" % tcorr.attrs["tag"])
# If requested, reference the timing correct with respect to source transit time
if self.refer_to_transit:
# First check for transit_time attribute in file
ttrans = tstream.attrs.get("transit_time", None)
if ttrans is None:
source = tstream.attrs["source_name"]
ttrans = ephemeris.transit_times(
ephemeris.source_dictionary[source],
tstream.time[0],
tstream.time[-1],
)
if ttrans.size != 1:
raise RuntimeError(
"Found %d transits of %s in timestream. "
"Require single transit." % (ttrans.size, source))
else:
ttrans = ttrans[0]
self.log.info(
"Referencing timing correction to %s (RA=%0.1f deg)." % (
ephemeris.unix_to_datetime(ttrans).strftime(
"%Y%m%dT%H%M%SZ"),
ephemeris.lsa(ttrans),
))
tcorr.set_global_reference_time(ttrans,
window=self.transit_window,
interpolate=True,
interp="linear")
# Apply the timing correction
tcorr.apply_timing_correction(tstream,
time=timestamp,
freq=freq,
input_flags=input_flags,
copy=False)
return tstream
def view(self):
if self.lsd is None:
return panel.pane.Markdown("No data selected.")
try:
sens_container = self.data.load_file(self.revision, self.lsd,
"sensitivity")
except DataError as err:
return panel.pane.Markdown(
f"Error: {str(err)}. Please report this problem.")
# Index map for ra (x-axis)
sens_csd = csd(sens_container.time)
index_map_ra = (sens_csd - self.lsd.lsd) * 360
axis_name_ra = "RA [degrees]"
# Index map for frequency (y-axis)
index_map_f = np.linspace(800.0, 400.0, 1024, endpoint=False)
axis_name_f = "Frequency [MHz]"
# Apply data selections
if self.polarization == self.mean_pol_text:
sel_pol = np.where((sens_container.index_map["pol"] == "XX")
| (sens_container.index_map["pol"] == "YY"))[0]
sens = np.squeeze(sens_container.measured[:, sel_pol])
sens = np.squeeze(np.nanmean(sens, axis=1))
else:
sel_pol = np.where(
sens_container.index_map["pol"] == self.polarization)[0]
sens = np.squeeze(sens_container.measured[:, sel_pol])
if self.flag_mask:
sens = np.where(self._flags_mask(index_map_ra).T, np.nan, sens)
# Set flagged data to nan
sens = np.where(sens == 0, np.nan, sens)
if self.mask_rfi:
try:
rfi_container = self.data.load_file(self.revision, self.lsd,
"rfi")
except DataError as err:
return panel.pane.Markdown(
f"Error: {str(err)}. Please report this problem.")
rfi = np.squeeze(rfi_container.mask[:])
# calculate percentage masked to print later
rfi_percentage = round(np.count_nonzero(rfi) / rfi.size * 100)
sens *= np.where(rfi, np.nan, 1)
if self.divide_by_estimate:
estimate = np.squeeze(sens_container.radiometer[:, sel_pol])
if self.polarization == self.mean_pol_text:
estimate = np.squeeze(np.nanmean(estimate, axis=1))
estimate = np.where(estimate == 0, np.nan, estimate)
sens = sens / estimate
if self.transpose:
sens = sens.T
index_x = index_map_f
index_y = index_map_ra
axis_names = [axis_name_f, axis_name_ra]
xlim, ylim = self.ylim, self.xlim
else:
index_x = index_map_ra
index_y = index_map_f
axis_names = [axis_name_ra, axis_name_f]
xlim, ylim = self.xlim, self.ylim
image_opts = {
"clim": self.colormap_range,
"logz": self.logarithmic_colorscale,
"cmap": process_cmap("viridis", provider="matplotlib"),
"colorbar": True,
"xticks": [0, 60, 120, 180, 240, 300, 360],
}
if self.mask_rfi:
image_opts["title"] = f"RFI mask: {rfi_percentage}%"
overlay_opts = {
"xlim": xlim,
"ylim": ylim,
}
# Fill in missing data
img = hv_image_with_gaps(index_x,
index_y,
sens,
opts=image_opts,
kdims=axis_names).opts(**overlay_opts)
if self.serverside_rendering is not None:
# set colormap
cmap_inferno = copy.copy(matplotlib_cm.get_cmap("viridis"))
# Set z-axis normalization (other possible values are 'eq_hist', 'cbrt').
if self.logarithmic_colorscale:
normalization = "log"
else:
normalization = "linear"
# datashade/rasterize the image
img = self.serverside_rendering(
img,
cmap=cmap_inferno,
precompute=True,
x_range=xlim,
y_range=ylim,
normalization=normalization,
# TODO: set xticks like above
)
if self.mark_day_time:
# Calculate the sun rise/set times on this sidereal day
# Start and end times of the CSD
start_time = csd_to_unix(self.lsd.lsd)
end_time = csd_to_unix(self.lsd.lsd + 1)
times, rises = chime.rise_set_times(
skyfield_wrapper.ephemeris["sun"],
start_time,
end_time,
diameter=-10,
)
sun_rise = 0
sun_set = 0
for t, r in zip(times, rises):
if r:
sun_rise = (unix_to_csd(t) % 1) * 360
else:
sun_set = (unix_to_csd(t) % 1) * 360
# Highlight the day time data
opts = {
"color": "grey",
"alpha": 0.5,
"line_width": 1,
"line_color": "black",
"line_dash": "dashed",
}
span = hv.HSpan if self.transpose else hv.VSpan
if sun_rise < sun_set:
img *= span(sun_rise, sun_set).opts(**opts)
else:
img *= span(self.xlim[0], sun_set).opts(**opts)
img *= span(sun_rise, self.xlim[-1]).opts(**opts)
img.opts(
# Fix height, but make width responsive
height=self.height,
responsive=True,
bgcolor="lightgray",
shared_axes=True,
)
return panel.Row(img, width_policy="max")
def process(self, sstream, inputmap):
"""Clean the sun.
Parameters
----------
sstream : containers.SiderealStream
Sidereal stream.
Returns
-------
mstream : containers.SiderealStream
Sidereal stack with sun projected out.
"""
sstream.redistribute("freq")
# Get array of CSDs for each sample
ra = sstream.index_map["ra"][:]
csd = sstream.attrs[
"lsd"] if "lsd" in sstream.attrs else sstream.attrs["csd"]
csd = csd + ra / 360.0
nprod = len(sstream.index_map["prod"])
# Get position of sun at every time sample
times = ephemeris.csd_to_unix(csd)
sun_pos = np.array([
ra_dec_of(ephemeris.skyfield_wrapper.ephemeris["sun"], t)
for t in times
])
# Get hour angle and dec of sun, in radians
ha = 2 * np.pi * (ra / 360.0) - sun_pos[:, 0]
dec = sun_pos[:, 1]
el = sun_pos[:, 2]
# Construct baseline vector for each visibility
feed_pos = tools.get_feed_positions(inputmap)
vis_pos = np.array([
feed_pos[ii] - feed_pos[ij]
for ii, ij in sstream.index_map["prod"][:]
])
feed_list = [(inputmap[fi], inputmap[fj])
for fi, fj in sstream.index_map["prod"][:]]
# Determine polarisation for each visibility
pol_ind = np.full(len(feed_list), -1, dtype=np.int)
for ii, (fi, fj) in enumerate(feed_list):
if tools.is_chime(fi) and tools.is_chime(fj):
pol_ind[ii] = 2 * tools.is_chime_y(fi) + tools.is_chime_y(fj)
# Change vis_pos for non-CHIME feeds from NaN to 0.0
vis_pos[(pol_ind == -1), :] = 0.0
# Initialise new container
sscut = sstream.__class__(axes_from=sstream, attrs_from=sstream)
sscut.redistribute("freq")
wv = 3e2 / sstream.index_map["freq"]["centre"]
# Iterate over frequencies and polarisations to null out the sun
for lfi, fi in sstream.vis[:].enumerate(0):
# Get the baselines in wavelengths
u = vis_pos[:, 0] / wv[fi]
v = vis_pos[:, 1] / wv[fi]
# Loop over ra to reduce memory usage
for ri in range(len(ra)):
# Copy over the visiblities and weights
vis = sstream.vis[fi, :, ri]
weight = sstream.weight[fi, :, ri]
sscut.vis[fi, :, ri] = vis
sscut.weight[fi, :, ri] = weight
# Check if sun has set
if el[ri] > 0.0:
# Calculate the phase that the sun would have using the fringestop routine
sun_vis = tools.fringestop_phase(
ha[ri], np.radians(ephemeris.CHIMELATITUDE), dec[ri],
u, v)
# Calculate the visibility vector for the sun
sun_vis = sun_vis.conj()
# Mask out the auto-correlations
sun_vis *= np.logical_or(u != 0.0, v != 0.0)
# Iterate over polarisations to do projection independently for each.
# This is needed because of the different beams for each pol.
for pol in range(4):
# Mask out other polarisations in the visibility vector
sun_vis_pol = sun_vis * (pol_ind == pol)
# Calculate various projections
vds = (vis * sun_vis_pol.conj() * weight).sum(axis=0)
sds = (sun_vis_pol * sun_vis_pol.conj() *
weight).sum(axis=0)
isds = tools.invert_no_zero(sds)
# Subtract sun contribution from visibilities and place in new array
sscut.vis[fi, :, ri] -= sun_vis_pol * vds * isds
# Return the clean sidereal stream
return sscut
def setup(self):
"""Query the database and fetch the files
Returns
-------
files : list
List of files to load
"""
files = None
# Query the database on rank=0 only, and broadcast to everywhere else
if mpiutil.rank0:
if self.run_name:
return self.QueryRun()
layout.connect_database()
f = finder.Finder(node_spoof=self.node_spoof)
f.filter_acqs(di.AcqType.name == self.acqtype)
if self.instrument is not None:
f.filter_acqs(di.ArchiveInst.name == self.instrument)
if self.accept_all_global_flags:
f.accept_all_global_flags()
# Use start and end times if set, or try and use the start and end CSDs
if self.start_time:
st, et = self.start_time, self.end_time
elif self.start_csd:
st = ephemeris.csd_to_unix(self.start_csd)
et = (
ephemeris.csd_to_unix(self.end_csd)
if self.end_csd is not None
else None
)
# Note: include_time_interval includes the specified time interval
# Using this instead of set_time_range, which only narrows the interval
# f.include_time_interval(self.start_time, self.end_time)
f.set_time_range(st, et)
if self.start_RA and self.end_RA:
f.include_RA_interval(self.start_RA, self.end_RA)
elif self.start_RA or self.start_RA:
self.log.warning(
"One but not both of start_RA and end_RA " "are set. Ignoring both."
)
f.filter_acqs(di.ArchiveInst.name == self.instrument)
if self.exclude_daytime:
f.exclude_daytime()
if self.exclude_sun:
f.exclude_sun(
time_delta=self.exclude_sun_time_delta,
time_delta_rise_set=self.exclude_sun_time_delta_rise_set,
)
if self.include_transits:
time_delta = self.include_transits_time_delta
ntime_delta = len(time_delta)
if (ntime_delta > 1) and (ntime_delta < len(self.include_transits)):
raise ValueError(
"Must specify `time_delta` for each source in "
"`include_transits` or provide single value for all sources."
)
for ss, src in enumerate(self.include_transits):
tdelta = time_delta[ss % ntime_delta] if ntime_delta > 0 else None
bdy = (
ephemeris.source_dictionary[src]
if isinstance(src, str)
else src
)
f.include_transits(bdy, time_delta=tdelta)
if self.exclude_transits:
time_delta = self.exclude_transits_time_delta
ntime_delta = len(time_delta)
if (ntime_delta > 1) and (ntime_delta < len(self.exclude_transits)):
raise ValueError(
"Must specify `time_delta` for each source in "
"`exclude_transits` or provide single value for all sources."
)
for ss, src in enumerate(self.exclude_transits):
tdelta = time_delta[ss % ntime_delta] if ntime_delta > 0 else None
bdy = (
ephemeris.source_dictionary[src]
if isinstance(src, str)
else src
)
f.exclude_transits(bdy, time_delta=tdelta)
if self.source_26m:
f.include_26m_obs(self.source_26m)
if len(self.exclude_data_flag_types) > 0:
f.exclude_data_flag_type(self.exclude_data_flag_types)
results = f.get_results()
if not self.return_intervals:
files = [fname for result in results for fname in result[0]]
files.sort()
else:
files = results
files.sort(key=lambda x: x[1][0])
files = mpiutil.world.bcast(files, root=0)
# Make sure all nodes have container before return
mpiutil.world.Barrier()
return files
def offline_point_source_calibration(file_list,
source,
inputmap=None,
start=None,
stop=None,
physical_freq=None,
tcorr=None,
logging_params=DEFAULT_LOGGING,
**kwargs):
# Load config
config = DEFAULTS.deepcopy()
config.merge(NameSpace(kwargs))
# Setup logging
log.setup_logging(logging_params)
mlog = log.get_logger(__name__)
mlog.info("ephemeris file: %s" % ephemeris.__file__)
# Set the model to use
fitter_function = utils.fit_point_source_transit
model_function = utils.model_point_source_transit
farg = inspect.getargspec(fitter_function)
defaults = {
key: val
for key, val in zip(farg.args[-len(farg.defaults):], farg.defaults)
}
poly_deg_amp = kwargs.get('poly_deg_amp', defaults['poly_deg_amp'])
poly_deg_phi = kwargs.get('poly_deg_phi', defaults['poly_deg_phi'])
poly_type = kwargs.get('poly_type', defaults['poly_type'])
param_name = ([
'%s_poly_amp_coeff%d' % (poly_type, cc)
for cc in range(poly_deg_amp + 1)
] + [
'%s_poly_phi_coeff%d' % (poly_type, cc)
for cc in range(poly_deg_phi + 1)
])
model_kwargs = [('poly_deg_amp', poly_deg_amp),
('poly_deg_phi', poly_deg_phi), ('poly_type', poly_type)]
model_name = '.'.join(
[getattr(model_function, key) for key in ['__module__', '__name__']])
tval = {}
# Set where to evaluate gain
ha_eval_str = ['raw_transit']
if config.multi_sample:
ha_eval_str += ['transit', 'peak']
ha_eval = [0.0, None]
fitslc = slice(1, 3)
ind_eval = ha_eval_str.index(config.evaluate_gain_at)
# Determine dimensions
direction = ['amp', 'phi']
nparam = len(param_name)
ngain = len(ha_eval_str)
ndir = len(direction)
# Determine frequencies
data = andata.CorrData.from_acq_h5(file_list,
datasets=(),
start=start,
stop=stop)
freq = data.freq
if physical_freq is not None:
index_freq = np.array(
[np.argmin(np.abs(ff - freq)) for ff in physical_freq])
freq_sel = utils.convert_to_slice(index_freq)
freq = freq[index_freq]
else:
index_freq = np.arange(freq.size)
freq_sel = None
nfreq = freq.size
# Compute flux of source
inv_rt_flux_density = tools.invert_no_zero(
np.sqrt(FluxCatalog[source].predict_flux(freq)))
# Read in the eigenvaluess for all frequencies
data = andata.CorrData.from_acq_h5(file_list,
datasets=['erms', 'eval'],
freq_sel=freq_sel,
start=start,
stop=stop)
# Determine source coordinates
this_csd = np.floor(ephemeris.unix_to_csd(np.median(data.time)))
timestamp0 = ephemeris.transit_times(FluxCatalog[source].skyfield,
ephemeris.csd_to_unix(this_csd))[0]
src_ra, src_dec = ephemeris.object_coords(FluxCatalog[source].skyfield,
date=timestamp0,
deg=True)
ra = ephemeris.lsa(data.time)
ha = ra - src_ra
ha = ha - (ha > 180.0) * 360.0 + (ha < -180.0) * 360.0
ha = np.radians(ha)
itrans = np.argmin(np.abs(ha))
window = 0.75 * np.max(np.abs(ha))
off_source = np.abs(ha) > window
mlog.info("CSD %d" % this_csd)
mlog.info("Hour angle at transit (%d of %d): %0.2f deg " %
(itrans, len(ha), np.degrees(ha[itrans])))
mlog.info("Hour angle off source: %0.2f deg" %
np.median(np.abs(np.degrees(ha[off_source]))))
src_dec = np.radians(src_dec)
lat = np.radians(ephemeris.CHIMELATITUDE)
# Determine division of frequencies
ninput = data.ninput
ntime = data.ntime
nblock_freq = int(np.ceil(nfreq / float(config.nfreq_per_block)))
# Determine bad inputs
eps = 10.0 * np.finfo(data['erms'].dtype).eps
good_freq = np.flatnonzero(np.all(data['erms'][:] > eps, axis=-1))
ind_sub_freq = good_freq[slice(0, good_freq.size,
max(int(good_freq.size / 10), 1))]
tmp_data = andata.CorrData.from_acq_h5(file_list,
datasets=['evec'],
freq_sel=ind_sub_freq,
start=start,
stop=stop)
eps = 10.0 * np.finfo(tmp_data['evec'].dtype).eps
bad_input = np.flatnonzero(
np.all(np.abs(tmp_data['evec'][:, 0]) < eps, axis=(0, 2)))
input_axis = tmp_data.input.copy()
del tmp_data
# Query layout database for correlator inputs
if inputmap is None:
inputmap = tools.get_correlator_inputs(
datetime.datetime.utcfromtimestamp(data.time[itrans]),
correlator='chime')
inputmap = tools.reorder_correlator_inputs(input_axis, inputmap)
tools.change_chime_location(rotation=config.telescope_rotation)
# Determine x and y pol index
xfeeds = np.array([
idf for idf, inp in enumerate(inputmap)
if (idf not in bad_input) and tools.is_array_x(inp)
])
yfeeds = np.array([
idf for idf, inp in enumerate(inputmap)
if (idf not in bad_input) and tools.is_array_y(inp)
])
nfeed = xfeeds.size + yfeeds.size
pol = [yfeeds, xfeeds]
polstr = ['Y', 'X']
npol = len(pol)
neigen = min(max(npol, config.neigen), data['eval'].shape[1])
phase_ref = config.phase_reference_index
phase_ref_by_pol = [
pol[pp].tolist().index(phase_ref[pp]) for pp in range(npol)
]
# Calculate dynamic range
eval0_off_source = np.median(data['eval'][:, 0, off_source], axis=-1)
dyn = data['eval'][:, 1, :] * tools.invert_no_zero(
eval0_off_source[:, np.newaxis])
# Determine frequencies to mask
not_rfi = np.ones((nfreq, 1), dtype=np.bool)
if config.mask_rfi is not None:
for frng in config.mask_rfi:
not_rfi[:, 0] &= ((freq < frng[0]) | (freq > frng[1]))
mlog.info("%0.1f percent of frequencies available after masking RFI." %
(100.0 * np.sum(not_rfi, dtype=np.float32) / float(nfreq), ))
#dyn_flg = utils.contiguous_flag(dyn > config.dyn_rng_threshold, centre=itrans)
if source in config.dyn_rng_threshold:
dyn_rng_threshold = config.dyn_rng_threshold[source]
else:
dyn_rng_threshold = config.dyn_rng_threshold.default
mlog.info("Dynamic range threshold set to %0.1f." % dyn_rng_threshold)
dyn_flg = dyn > dyn_rng_threshold
# Calculate fit flag
fit_flag = np.zeros((nfreq, npol, ntime), dtype=np.bool)
for pp in range(npol):
mlog.info("Dynamic Range Nsample, Pol %d: %s" % (pp, ','.join([
"%d" % xx for xx in np.percentile(np.sum(dyn_flg, axis=-1),
[25, 50, 75, 100])
])))
if config.nsigma1 is None:
fit_flag[:, pp, :] = dyn_flg & not_rfi
else:
fit_window = config.nsigma1 * np.radians(
utils.get_window(freq, pol=polstr[pp], dec=src_dec, deg=True))
win_flg = np.abs(ha)[np.newaxis, :] <= fit_window[:, np.newaxis]
fit_flag[:, pp, :] = (dyn_flg & win_flg & not_rfi)
# Calculate base error
base_err = data['erms'][:, np.newaxis, :]
# Check for sign flips
ref_resp = andata.CorrData.from_acq_h5(file_list,
datasets=['evec'],
input_sel=config.eigen_reference,
freq_sel=freq_sel,
start=start,
stop=stop)['evec'][:, 0:neigen,
0, :]
sign0 = 1.0 - 2.0 * (ref_resp.real < 0.0)
# Check that we have the correct reference feed
if np.any(np.abs(ref_resp.imag) > 0.0):
ValueError("Reference feed %d is incorrect." % config.eigen_reference)
del ref_resp
# Save index_map
results = {}
results['model'] = model_name
results['param'] = param_name
results['freq'] = data.index_map['freq'][:]
results['input'] = input_axis
results['eval'] = ha_eval_str
results['dir'] = direction
for key, val in model_kwargs:
results[key] = val
# Initialize numpy arrays to hold results
if config.return_response:
results['response'] = np.zeros((nfreq, ninput, ntime),
dtype=np.complex64)
results['response_err'] = np.zeros((nfreq, ninput, ntime),
dtype=np.float32)
results['fit_flag'] = fit_flag
results['ha_axis'] = ha
results['ra'] = ra
else:
results['gain_eval'] = np.zeros((nfreq, ninput, ngain),
dtype=np.complex64)
results['weight_eval'] = np.zeros((nfreq, ninput, ngain),
dtype=np.float32)
results['frac_gain_err'] = np.zeros((nfreq, ninput, ngain, ndir),
dtype=np.float32)
results['parameter'] = np.zeros((nfreq, ninput, nparam),
dtype=np.float32)
results['parameter_err'] = np.zeros((nfreq, ninput, nparam),
dtype=np.float32)
results['index_eval'] = np.full((nfreq, ninput), -1, dtype=np.int8)
results['gain'] = np.zeros((nfreq, ninput), dtype=np.complex64)
results['weight'] = np.zeros((nfreq, ninput), dtype=np.float32)
results['ndof'] = np.zeros((nfreq, ninput, ndir), dtype=np.float32)
results['chisq'] = np.zeros((nfreq, ninput, ndir), dtype=np.float32)
results['timing'] = np.zeros((nfreq, ninput), dtype=np.complex64)
# Initialize metric like variables
results['runtime'] = np.zeros((nblock_freq, 2), dtype=np.float64)
# Compute distances
dist = tools.get_feed_positions(inputmap)
for pp, feeds in enumerate(pol):
dist[feeds, :] -= dist[phase_ref[pp], np.newaxis, :]
# Loop over frequency blocks
for gg in range(nblock_freq):
mlog.info("Frequency block %d of %d." % (gg, nblock_freq))
fstart = gg * config.nfreq_per_block
fstop = min((gg + 1) * config.nfreq_per_block, nfreq)
findex = np.arange(fstart, fstop)
ngroup = findex.size
freq_sel = utils.convert_to_slice(index_freq[findex])
timeit_start_gg = time.time()
#
if config.return_response:
gstart = start
gstop = stop
tslc = slice(0, ntime)
else:
good_times = np.flatnonzero(np.any(fit_flag[findex], axis=(0, 1)))
if good_times.size == 0:
continue
gstart = int(np.min(good_times))
gstop = int(np.max(good_times)) + 1
tslc = slice(gstart, gstop)
gstart += start
gstop += start
hag = ha[tslc]
itrans = np.argmin(np.abs(hag))
# Load eigenvectors.
nudata = andata.CorrData.from_acq_h5(
file_list,
datasets=['evec', 'vis', 'flags/vis_weight'],
apply_gain=False,
freq_sel=freq_sel,
start=gstart,
stop=gstop)
# Save time to load data
results['runtime'][gg, 0] = time.time() - timeit_start_gg
timeit_start_gg = time.time()
mlog.info("Time to load (per frequency): %0.3f sec" %
(results['runtime'][gg, 0] / ngroup, ))
# Loop over polarizations
for pp, feeds in enumerate(pol):
# Get timing correction
if tcorr is not None:
tgain = tcorr.get_gain(nudata.freq, nudata.input[feeds],
nudata.time)
tgain *= tgain[:, phase_ref_by_pol[pp], np.newaxis, :].conj()
tgain_transit = tgain[:, :, itrans].copy()
tgain *= tgain_transit[:, :, np.newaxis].conj()
# Create the polarization masking vector
P = np.zeros((1, ninput, 1), dtype=np.float64)
P[:, feeds, :] = 1.0
# Loop over frequencies
for gff, ff in enumerate(findex):
flg = fit_flag[ff, pp, tslc]
if (2 * int(np.sum(flg))) < (nparam +
1) and not config.return_response:
continue
# Normalize by eigenvalue and correct for pi phase flips in process.
resp = (nudata['evec'][gff, 0:neigen, :, :] *
np.sqrt(data['eval'][ff, 0:neigen, np.newaxis, tslc]) *
sign0[ff, :, np.newaxis, tslc])
# Rotate to single-pol response
# Move time to first axis for the matrix multiplication
invL = tools.invert_no_zero(
np.rollaxis(data['eval'][ff, 0:neigen, np.newaxis, tslc],
-1, 0))
UT = np.rollaxis(resp, -1, 0)
U = np.swapaxes(UT, -1, -2)
mu, vp = np.linalg.eigh(np.matmul(UT.conj(), P * U))
rsign0 = (1.0 - 2.0 * (vp[:, 0, np.newaxis, :].real < 0.0))
resp = mu[:, np.newaxis, :] * np.matmul(U, rsign0 * vp * invL)
# Extract feeds of this pol
# Transpose so that time is back to last axis
resp = resp[:, feeds, -1].T
# Compute error on response
dataflg = ((nudata.weight[gff, feeds, :] > 0.0)
& np.isfinite(nudata.weight[gff, feeds, :])).astype(
np.float32)
resp_err = dataflg * base_err[ff, :, tslc] * np.sqrt(
nudata.vis[gff, feeds, :].real) * tools.invert_no_zero(
np.sqrt(mu[np.newaxis, :, -1]))
# Reference to specific input
resp *= np.exp(
-1.0J *
np.angle(resp[phase_ref_by_pol[pp], np.newaxis, :]))
# Apply timing correction
if tcorr is not None:
resp *= tgain[gff]
results['timing'][ff, feeds] = tgain_transit[gff]
# Fringestop
lmbda = scipy.constants.c * 1e-6 / nudata.freq[gff]
resp *= tools.fringestop_phase(
hag[np.newaxis, :], lat, src_dec,
dist[feeds, 0, np.newaxis] / lmbda,
dist[feeds, 1, np.newaxis] / lmbda)
# Normalize by source flux
resp *= inv_rt_flux_density[ff]
resp_err *= inv_rt_flux_density[ff]
# If requested, reference phase to the median value
if config.med_phase_ref:
phi0 = np.angle(resp[:, itrans, np.newaxis])
resp *= np.exp(-1.0J * phi0)
resp *= np.exp(
-1.0J *
np.median(np.angle(resp), axis=0, keepdims=True))
resp *= np.exp(1.0J * phi0)
# Check if return_response flag was set by user
if not config.return_response:
if config.multi_sample:
moving_window = config.nsigma2 and config.nsigma2 * np.radians(
utils.get_window(nudata.freq[gff],
pol=polstr[pp],
dec=src_dec,
deg=True))
# Loop over inputs
for pii, ii in enumerate(feeds):
is_good = flg & (np.abs(resp[pii, :]) >
0.0) & (resp_err[pii, :] > 0.0)
# Set the intial gains based on raw response at transit
if is_good[itrans]:
results['gain_eval'][ff, ii,
0] = tools.invert_no_zero(
resp[pii, itrans])
results['frac_gain_err'][ff, ii, 0, :] = (
resp_err[pii, itrans] * tools.invert_no_zero(
np.abs(resp[pii, itrans])))
results['weight_eval'][ff, ii, 0] = 0.5 * (
np.abs(resp[pii, itrans])**2 *
tools.invert_no_zero(resp_err[pii, itrans]))**2
results['index_eval'][ff, ii] = 0
results['gain'][ff,
ii] = results['gain_eval'][ff, ii,
0]
results['weight'][ff,
ii] = results['weight_eval'][ff,
ii,
0]
# Exit if not performing multi time sample fit
if not config.multi_sample:
continue
if (2 * int(np.sum(is_good))) < (nparam + 1):
continue
try:
param, param_err, gain, gain_err, ndof, chisq, tval = fitter_function(
hag[is_good],
resp[pii, is_good],
resp_err[pii, is_good],
ha_eval,
window=moving_window,
tval=tval,
**config.fit)
except Exception as rex:
if config.verbose:
mlog.info(
"Frequency %0.2f, Feed %d failed with error: %s"
% (nudata.freq[gff], ii, rex))
continue
# Check for nan
wfit = (np.abs(gain) *
tools.invert_no_zero(np.abs(gain_err)))**2
if np.any(~np.isfinite(np.abs(gain))) or np.any(
~np.isfinite(wfit)):
continue
# Save to results using the convention that you should *multiply* the visibilites by the gains
results['gain_eval'][
ff, ii, fitslc] = tools.invert_no_zero(gain)
results['frac_gain_err'][ff, ii, fitslc,
0] = gain_err.real
results['frac_gain_err'][ff, ii, fitslc,
1] = gain_err.imag
results['weight_eval'][ff, ii, fitslc] = wfit
results['parameter'][ff, ii, :] = param
results['parameter_err'][ff, ii, :] = param_err
results['ndof'][ff, ii, :] = ndof
results['chisq'][ff, ii, :] = chisq
# Check if the fit was succesful and update the gain evaluation index appropriately
if np.all((chisq / ndof.astype(np.float32)
) <= config.chisq_per_dof_threshold):
results['index_eval'][ff, ii] = ind_eval
results['gain'][ff, ii] = results['gain_eval'][
ff, ii, ind_eval]
results['weight'][ff, ii] = results['weight_eval'][
ff, ii, ind_eval]
else:
# Return response only (do not fit model)
results['response'][ff, feeds, :] = resp
results['response_err'][ff, feeds, :] = resp_err
# Save time to fit data
results['runtime'][gg, 1] = time.time() - timeit_start_gg
mlog.info("Time to fit (per frequency): %0.3f sec" %
(results['runtime'][gg, 1] / ngroup, ))
# Clean up
del nudata
gc.collect()
# Print total run time
mlog.info("TOTAL TIME TO LOAD: %0.3f min" %
(np.sum(results['runtime'][:, 0]) / 60.0, ))
mlog.info("TOTAL TIME TO FIT: %0.3f min" %
(np.sum(results['runtime'][:, 1]) / 60.0, ))
# Set the best estimate of the gain
if not config.return_response:
flag = results['index_eval'] >= 0
gain = results['gain']
# Compute amplitude
amp = np.abs(gain)
# Hard cutoffs on the amplitude
med_amp = np.median(amp[flag])
min_amp = med_amp * config.min_amp_scale_factor
max_amp = med_amp * config.max_amp_scale_factor
flag &= ((amp >= min_amp) & (amp <= max_amp))
# Flag outliers in amplitude for each frequency
for pp, feeds in enumerate(pol):
med_amp_by_pol = np.zeros(nfreq, dtype=np.float32)
sig_amp_by_pol = np.zeros(nfreq, dtype=np.float32)
for ff in range(nfreq):
this_flag = flag[ff, feeds]
if np.any(this_flag):
med, slow, shigh = utils.estimate_directional_scale(
amp[ff, feeds[this_flag]])
lower = med - config.nsigma_outlier * slow
upper = med + config.nsigma_outlier * shigh
flag[ff, feeds] &= ((amp[ff, feeds] >= lower) &
(amp[ff, feeds] <= upper))
med_amp_by_pol[ff] = med
sig_amp_by_pol[ff] = 0.5 * (shigh - slow) / np.sqrt(
np.sum(this_flag, dtype=np.float32))
if config.nsigma_med_outlier:
med_flag = med_amp_by_pol > 0.0
not_outlier = flag_outliers(med_amp_by_pol,
med_flag,
window=config.window_med_outlier,
nsigma=config.nsigma_med_outlier)
flag[:, feeds] &= not_outlier[:, np.newaxis]
mlog.info("Pol %s: %d frequencies are outliers." %
(polstr[pp],
np.sum(~not_outlier & med_flag, dtype=np.int)))
# Determine bad frequencies
flag_freq = (np.sum(flag, axis=1, dtype=np.float32) /
float(ninput)) > config.threshold_good_freq
good_freq = np.flatnonzero(flag_freq)
# Determine bad inputs
fraction_good = np.sum(flag[good_freq, :], axis=0,
dtype=np.float32) / float(good_freq.size)
flag_input = fraction_good > config.threshold_good_input
# Finalize flag
flag &= (flag_freq[:, np.newaxis] & flag_input[np.newaxis, :])
# Interpolate gains
interp_gain, interp_weight = interpolate_gain(
freq,
gain,
results['weight'],
flag=flag,
length_scale=config.interpolation_length_scale,
mlog=mlog)
# Save gains to object
results['flag'] = flag
results['gain'] = interp_gain
results['weight'] = interp_weight
# Return results
return results
def _create_hook(self):
"""Create the revision.
This tries to determine which days are good and bad, and partitions the
available good days into the individual stacks.
"""
days = {}
core.connect()
# Go over each revision and construct the set of LSDs we should stack, and save
# the path to each.
# NOTE: later entries is `daily_revisions` will override LSDs found in earlier
# revisions.
for rev in self.default_params["daily_revisions"]:
daily_path = (
self.root_path
if self.default_params["daily_root"] is None
else self.default_params["daily_root"]
)
daily_rev = daily.DailyProcessing(rev, root_path=daily_path)
# Get all the bad days in this revision
revision = df.DataRevision.get(name=rev)
query = (
df.DataFlagOpinion.select(df.DataFlagOpinion.lsd)
.distinct()
.where(
df.DataFlagOpinion.revision == revision,
df.DataFlagOpinion.decision == "bad",
)
)
bad_days = [x[0] for x in query.tuples()]
# Get all the good days
query = (
df.DataFlagOpinion.select(df.DataFlagOpinion.lsd)
.distinct()
.where(
df.DataFlagOpinion.revision == revision,
df.DataFlagOpinion.decision == "good",
)
)
good_days = [x[0] for x in query.tuples()]
for d in daily_rev.ls():
# Filter out known bad days here
if (int(d) in bad_days) or (int(d) not in good_days):
continue
# Insert the day and path into the dict, this will replace the entries
# from prior revisions
path = daily_rev.base_path / d
lsd = int(d)
days[lsd] = path
lsds = sorted(days)
# Map each LSD into the quarter it belongs in and find which quarters we have
# data for
dates = ctime.unix_to_datetime(ephemeris.csd_to_unix(np.array(lsds)))
yq = np.array([f"{d.year}q{(d.month - 1) // 3 + 1}" for d in dates])
quarters = np.unique(yq)
npart = self.default_params["partitions"]
lsd_partitions = {}
# For each quarter divide the LSDs it contains into a number of partitions to
# give jack knifes
for quarter in quarters:
lsds_in_quarter = sorted(np.array(lsds)[yq == quarter])
# Skip quarters with two few days in them
if len(lsds_in_quarter) < self.default_params["min_days"] * npart:
continue
for i in range(npart):
lsd_partitions[f"{quarter}p{i}"] = [
int(d) for d in lsds_in_quarter[i::npart]
]
# Save the relevant parameters into the revisions configuration
self.default_params["days"] = {
int(day): str(path) for day, path in days.items()
}
self.default_params["stacks"] = lsd_partitions
def start_time(self):
return csd_to_unix(self._lsd)
def parse_ant_logs(cls, logs, return_post_report_params=False):
"""
Unzip and parse .ANT log file output by nsched for John Galt Telescope
observations
Parameters
----------
logs : list of strings
.ZIP filenames. Each .ZIP archive should include a .ANT file and
a .POST_REPORT file. This method unzips the archive, uses
`parse_post_report` to read the .POST_REPORT file and extract
the CHIME sidereal day corresponding to the DRAO sidereal day,
and then reads the lines in the .ANT file to obtain the pointing
history of the Galt Telescope during this observation.
(The DRAO sidereal day is days since the clock in Ev Sheehan's
office at DRAO was reset. This clock is typically only reset every
few years, but it does not correspond to any defined date, so the
date must be figured out from the .POST_REPORT file, which reports
both the DRAO sidereal day and the UTC date and time.
Known reset dates: 2017-11-21, 2019-3-10)
Returns
-------
if output_params == False:
ant_data: A dictionary consisting of lists containing the LST,
hour angle, RA, and dec (all as Skyfield Angle objects),
CHIME sidereal day, and DRAO sidereal day.
if output_params == True
output_params: dictionary returned by `parse_post_report`
and
ant_data: described above
Files
-----
the .ANT and .POST_REPORT files in the input .zip archive are
extracted into /tmp/26mlog/<loginname>/
"""
from skyfield.positionlib import Angle
from caput import time as ctime
DRAO_lon = ephemeris.CHIMELONGITUDE * 24.0 / 360.0
def sidlst_to_csd(sid, lst, sid_ref, t_ref):
"""
Convert an integer DRAO sidereal day and LST to a float
CHIME sidereal day
Parameters
----------
sid : int
DRAO sidereal day
lst : float, in hours
local sidereal time
sid_ref : int
DRAO sidereal day at the reference time t_ref
t_ref : skyfield time object, Julian days
Reference time
Returns
-------
output : float
CHIME sidereal day
"""
csd_ref = int(
ephemeris.csd(ephemeris.datetime_to_unix(
t_ref.utc_datetime())))
csd = sid - sid_ref + csd_ref
return csd + lst / ephemeris.SIDEREAL_S / 24.0
ant_data_list = []
post_report_list = []
for log in logs:
doobs = True
filename = log.split("/")[-1]
basedir = "/tmp/26mlog/{}/".format(os.getlogin())
basename, extension = filename.split(".")
post_report_file = basename + ".POST_REPORT"
ant_file = basename + ".ANT"
if extension == "zip":
try:
zipfile.ZipFile(log).extract(post_report_file,
path=basedir)
except:
print(
"Failed to extract {} into {}. Moving right along...".
format(post_report_file, basedir))
doobs = False
try:
zipfile.ZipFile(log).extract(ant_file, path=basedir)
except:
print(
"Failed to extract {} into {}. Moving right along...".
format(ant_file, basedir))
doobs = False
if doobs:
try:
post_report_params = cls.parse_post_report(
basedir + post_report_file)
with open(os.path.join(basedir, ant_file), "r") as f:
lines = [line for line in f]
ant_data = {"sid": np.array([])}
lsth = []
lstm = []
lsts = []
hah = []
ham = []
has = []
decd = []
decm = []
decs = []
for l in lines:
arr = l.split()
try:
lst_hms = [float(x) for x in arr[2].split(":")]
# do last element first: if this is going to
# crash because a line in the log is incomplete,
# we don't want it to append to any of the lists
decs.append(float(arr[8].replace('"', "")))
decm.append(float(arr[7].replace("'", "")))
decd.append(float(arr[6].replace("D", "")))
has.append(float(arr[5].replace("S", "")))
ham.append(float(arr[4].replace("M", "")))
hah.append(float(arr[3].replace("H", "")))
lsts.append(float(lst_hms[2]))
lstm.append(float(lst_hms[1]))
lsth.append(float(lst_hms[0]))
ant_data["sid"] = np.append(
ant_data["sid"], int(arr[1]))
except:
print("Failed in file {} for line \n{}".format(
ant_file, l))
if len(ant_data["sid"]) != len(decs):
print("WARNING: mismatch in list lengths.")
ant_data["lst"] = Angle(hours=(lsth, lstm, lsts))
ha = Angle(hours=(hah, ham, has))
dec = Angle(degrees=(decd, decm, decs))
ant_data["ha"] = Angle(
radians=ha.radians -
ephemeris.galt_pointing_model_ha(ha, dec).radians,
preference="hours",
)
ant_data["dec_cirs"] = Angle(
radians=dec.radians -
ephemeris.galt_pointing_model_dec(ha, dec).radians,
preference="degrees",
)
ant_data["csd"] = sidlst_to_csd(
np.array(ant_data["sid"]),
ant_data["lst"].hours,
post_report_params["SID"],
post_report_params["start_time"],
)
ant_data["t"] = ephemeris.unix_to_skyfield_time(
ephemeris.csd_to_unix(ant_data["csd"]))
# Correct RA from equinox to CIRS coords (both in radians)
era = np.radians(
ctime.unix_to_era(ephemeris.ensure_unix(
ant_data["t"])))
gast = ant_data["t"].gast * 2 * np.pi / 24.0
ant_data["ra_cirs"] = Angle(
radians=ant_data["lst"].radians -
ant_data["ha"].radians + (era - gast),
preference="hours",
)
obs = ephemeris.Star_cirs(
ra=ant_data["ra_cirs"],
dec=ant_data["dec_cirs"],
epoch=ant_data["t"],
)
ant_data["ra"] = obs.ra
ant_data["dec"] = obs.dec
ant_data_list.append(ant_data)
post_report_list.append(post_report_params)
except:
print("Parsing {} failed".format(post_report_file))
if return_post_report_params:
return post_report_list, ant_data_list
return ant_data
def main(config_file=None, logging_params=DEFAULT_LOGGING):
# Setup logging
log.setup_logging(logging_params)
mlog = log.get_logger(__name__)
# Set config
config = DEFAULTS.deepcopy()
if config_file is not None:
config.merge(NameSpace(load_yaml_config(config_file)))
# Create transit tracker
source_list = FluxCatalog.sort(
) if not config.source_list else config.source_list
cal_list = [
name for name, obj in FluxCatalog.iteritems()
if (obj.dec >= config.min_dec) and (
obj.predict_flux(config.freq_nominal) >= config.min_flux) and (
name in source_list)
]
if not cal_list:
raise RuntimeError("No calibrators found.")
# Sort list by flux at nominal frequency
cal_list.sort(
key=lambda name: FluxCatalog[name].predict_flux(config.freq_nominal))
# Add to transit tracker
transit_tracker = containers.TransitTrackerOffline(
nsigma=config.nsigma_source, extend_night=config.extend_night)
for name in cal_list:
transit_tracker[name] = FluxCatalog[name].skyfield
mlog.info("Initializing offline point source processing.")
search_time = config.start_time or 0
# Find all calibration files
all_files = sorted(
glob.glob(
os.path.join(config.acq_dir,
'*' + config.correlator + config.acq_suffix, '*.h5')))
if not all_files:
return
# Remove files whose last modified time is before the time of the most recent update
all_files = [
ff for ff in all_files if (os.path.getmtime(ff) > search_time)
]
if not all_files:
return
# Remove files that are currently locked
all_files = [
ff for ff in all_files
if not os.path.isfile(os.path.splitext(ff)[0] + '.lock')
]
if not all_files:
return
# Add files to transit tracker
for ff in all_files:
transit_tracker.add_file(ff)
# Extract point source transits ready for analysis
all_transits = transit_tracker.get_transits()
# Create dictionary to hold results
h5_psrc_fit = {}
inputmap = None
# Loop over transits
for transit in all_transits:
src, csd, is_day, files, start, stop = transit
# Discard any point sources with unusual csd value
if (csd < config.min_csd) or (csd > config.max_csd):
continue
# Discard any point sources transiting during the day
if is_day > config.process_daytime:
continue
mlog.info(
'Processing %s transit on CSD %d (%d files, %d time samples)' %
(src, csd, len(files), stop - start + 1))
# Load inputmap
if inputmap is None:
if config.inputmap is None:
inputmap = tools.get_correlator_inputs(
ephemeris.unix_to_datetime(ephemeris.csd_to_unix(csd)),
correlator=config.correlator)
else:
with open(config.inputmap, 'r') as handler:
inputmap = pickle.load(handler)
# Grab the timing correction for this transit
tcorr = None
if config.apply_timing:
if config.timing_glob is not None:
mlog.info(
"Loading timing correction from extended timing solutions."
)
timing_files = sorted(glob.glob(config.timing_glob))
if timing_files:
try:
tcorr = search_extended_timing_solutions(
timing_files, ephemeris.csd_to_unix(csd))
except Exception as e:
mlog.error(
'search_extended_timing_solutions failed with error: %s'
% e)
else:
mlog.info(str(tcorr))
if tcorr is None:
mlog.info(
"Loading timing correction from chimetiming acquisitions.")
try:
tcorr = timing.load_timing_correction(
files,
start=start,
stop=stop,
window=config.timing_window,
instrument=config.correlator)
except Exception as e:
mlog.error(
'timing.load_timing_correction failed with error: %s' %
e)
mlog.warning(
'No timing correction applied to %s transit on CSD %d.'
% (src, csd))
else:
mlog.info(str(tcorr))
# Call the main routine to process data
try:
outdct = offline_cal.offline_point_source_calibration(
files,
src,
start=start,
stop=stop,
inputmap=inputmap,
tcorr=tcorr,
logging_params=logging_params,
**config.analysis.as_dict())
except Exception as e:
msg = 'offline_cal.offline_point_source_calibration failed with error: %s' % e
mlog.error(msg)
continue
#raise RuntimeError(msg)
# Find existing gain files for this particular point source
if src not in h5_psrc_fit:
output_files = find_files(config, psrc=src)
if output_files is not None:
output_files = output_files[-1]
mlog.info('Writing %s transit on CSD %d to existing file %s.' %
(src, csd, output_files))
h5_psrc_fit[src] = containers.PointSourceWriter(
src,
output_file=output_files,
output_dir=config.output_dir,
output_suffix=point_source_name_to_file_suffix(src),
instrument=config.correlator,
max_file_size=config.max_file_size,
max_num=config.max_num_time,
memory_size=0)
# Associate this gain calibration to the transit time
this_time = ephemeris.transit_times(FluxCatalog[src].skyfield,
ephemeris.csd_to_unix(csd))[0]
outdct['csd'] = csd
outdct['is_daytime'] = is_day
outdct['acquisition'] = os.path.basename(os.path.dirname(files[0]))
# Write to output file
mlog.info('Writing to disk results from %s transit on CSD %d.' %
(src, csd))
h5_psrc_fit[src].write(this_time, **outdct)
# Dump an individual file for this point source transit
mlog.info('Dumping to disk single file for %s transit on CSD %d.' %
(src, csd))
dump_dir = os.path.join(config.output_dir, 'point_source_gains')
containers.mkdir(dump_dir)
dump_file = os.path.join(dump_dir, '%s_csd_%d.h5' % (src.lower(), csd))
h5_psrc_fit[src].dump(dump_file,
datasets=[
'csd', 'acquisition', 'is_daytime', 'gain',
'weight', 'timing', 'model'
])
mlog.info('Finished analysis of %s transit on CSD %d.' % (src, csd))
def process(self, sstream, inputmap, inputmask):
"""Determine calibration from a timestream.
Parameters
----------
sstream : andata.CorrData or containers.SiderealStream
Timestream collected during the day.
inputmap : list of :class:`CorrInput`
A list describing the inputs as they are in the file.
inputmask : containers.CorrInputMask
Mask indicating which correlator inputs to use in the
eigenvalue decomposition.
Returns
-------
suntrans : containers.SunTransit
Response to the sun.
"""
from operator import itemgetter
from itertools import groupby
from .calibration import _extract_diagonal, solve_gain
# Ensure that we are distributed over frequency
sstream.redistribute("freq")
# Find the local frequencies
nfreq = sstream.vis.local_shape[0]
sfreq = sstream.vis.local_offset[0]
efreq = sfreq + nfreq
# Get the local frequency axis
freq = sstream.freq["centre"][sfreq:efreq]
wv = 3e2 / freq
# Get times
if hasattr(sstream, "time"):
time = sstream.time
ra = ephemeris.transit_RA(time)
else:
ra = sstream.index_map["ra"][:]
csd = (sstream.attrs["lsd"]
if "lsd" in sstream.attrs else sstream.attrs["csd"])
csd = csd + ra / 360.0
time = ephemeris.csd_to_unix(csd)
# Only examine data between sunrise and sunset
time_flag = np.zeros(len(time), dtype=np.bool)
rise = ephemeris.solar_rising(time[0] - 24.0 * 3600.0,
end_time=time[-1])
for rr in rise:
ss = ephemeris.solar_setting(rr)[0]
time_flag |= (time >= rr) & (time <= ss)
if not np.any(time_flag):
self.log.debug(
"No daytime data between %s and %s.",
ephemeris.unix_to_datetime(time[0]).strftime("%b %d %H:%M"),
ephemeris.unix_to_datetime(time[-1]).strftime("%b %d %H:%M"),
)
return None
# Convert boolean flag to slices
time_index = np.where(time_flag)[0]
time_slice = []
ntime = 0
for key, group in groupby(
enumerate(time_index),
lambda index_item: index_item[0] - index_item[1]):
group = list(map(itemgetter(1), group))
ngroup = len(group)
time_slice.append(
(slice(group[0], group[-1] + 1), slice(ntime, ntime + ngroup)))
ntime += ngroup
time = np.concatenate([time[slc[0]] for slc in time_slice])
ra = np.concatenate([ra[slc[0]] for slc in time_slice])
# Get ra, dec, alt of sun
sun_pos = np.array([
ra_dec_of(ephemeris.skyfield_wrapper.ephemeris["sun"], t)
for t in time
])
# Convert from ra to hour angle
sun_pos[:, 0] = np.radians(ra) - sun_pos[:, 0]
# Determine good inputs
nfeed = len(inputmap)
good_input = np.arange(
nfeed, dtype=np.int)[inputmask.datasets["input_mask"][:]]
# Use input map to figure out which are the X and Y feeds
xfeeds = np.array([
idx for idx, inp in enumerate(inputmap)
if tools.is_chime_x(inp) and (idx in good_input)
])
yfeeds = np.array([
idx for idx, inp in enumerate(inputmap)
if tools.is_chime_y(inp) and (idx in good_input)
])
self.log.debug(
"Performing sun calibration with %d/%d good feeds (%d xpol, %d ypol).",
len(good_input),
nfeed,
len(xfeeds),
len(yfeeds),
)
# Construct baseline vector for each visibility
feed_pos = tools.get_feed_positions(inputmap)
vis_pos = np.array([
feed_pos[ii] - feed_pos[ij]
for ii, ij in sstream.index_map["prod"][:]
])
vis_pos = np.where(np.isnan(vis_pos), np.zeros_like(vis_pos), vis_pos)
u = (vis_pos[np.newaxis, :, 0] / wv[:, np.newaxis])[:, :, np.newaxis]
v = (vis_pos[np.newaxis, :, 1] / wv[:, np.newaxis])[:, :, np.newaxis]
# Create container to hold results of fit
suntrans = containers.SunTransit(time=time,
pol_x=xfeeds,
pol_y=yfeeds,
axes_from=sstream)
for key in suntrans.datasets.keys():
suntrans.datasets[key][:] = 0.0
# Set coordinates
suntrans.coord[:] = sun_pos
# Loop over time slices
for slc_in, slc_out in time_slice:
# Extract visibility slice
vis_slice = sstream.vis[..., slc_in].copy()
ha = (sun_pos[slc_out, 0])[np.newaxis, np.newaxis, :]
dec = (sun_pos[slc_out, 1])[np.newaxis, np.newaxis, :]
# Extract the diagonal (to be used for weighting)
norm = (_extract_diagonal(vis_slice, axis=1).real)**0.5
norm = tools.invert_no_zero(norm)
# Fringestop
if self.fringestop:
vis_slice *= tools.fringestop_phase(
ha, np.radians(ephemeris.CHIMELATITUDE), dec, u, v)
# Solve for the point source response of each set of polarisations
ev_x, resp_x, err_resp_x = solve_gain(vis_slice,
feeds=xfeeds,
norm=norm[:, xfeeds])
ev_y, resp_y, err_resp_y = solve_gain(vis_slice,
feeds=yfeeds,
norm=norm[:, yfeeds])
# Save to container
suntrans.evalue_x[..., slc_out] = ev_x
suntrans.evalue_y[..., slc_out] = ev_y
suntrans.response[:, xfeeds, slc_out] = resp_x
suntrans.response[:, yfeeds, slc_out] = resp_y
suntrans.response_error[:, xfeeds, slc_out] = err_resp_x
suntrans.response_error[:, yfeeds, slc_out] = err_resp_y
# If requested, fit a model to the primary beam of the sun transit
if self.model_fit:
# Estimate peak RA
i_transit = np.argmin(np.abs(sun_pos[:, 0]))
body = ephemeris.skyfield_wrapper.ephemeris["sun"]
obs = ephemeris._get_chime()
obs.date = ephemeris.unix_to_ephem_time(time[i_transit])
body.compute(obs)
peak_ra = ephemeris.peak_RA(body)
dra = ra - peak_ra
dra = np.abs(dra - (dra > np.pi) * 2.0 * np.pi)[np.newaxis,
np.newaxis, :]
# Estimate FWHM
sig_x = cal_utils.guess_fwhm(freq,
pol="X",
dec=body.dec,
sigma=True)[:, np.newaxis, np.newaxis]
sig_y = cal_utils.guess_fwhm(freq,
pol="Y",
dec=body.dec,
sigma=True)[:, np.newaxis, np.newaxis]
# Only fit ra values above the specified dynamic range threshold
fit_flag = np.zeros([nfreq, nfeed, ntime], dtype=np.bool)
fit_flag[:, xfeeds, :] = dra < (self.nsig * sig_x)
fit_flag[:, yfeeds, :] = dra < (self.nsig * sig_y)
# Fit model for the complex response of each feed to the point source
param, param_cov = cal_utils.fit_point_source_transit(
ra,
suntrans.response[:],
suntrans.response_error[:],
flag=fit_flag)
# Save to container
suntrans.add_dataset("flag")
suntrans.flag[:] = fit_flag
suntrans.add_dataset("parameter")
suntrans.parameter[:] = param
suntrans.add_dataset("parameter_cov")
suntrans.parameter_cov[:] = param_cov
# Update attributes
units = "sqrt(" + sstream.vis.attrs.get("units",
"correlator-units") + ")"
suntrans.response.attrs["units"] = units
suntrans.response_error.attrs["units"] = units
suntrans.attrs["source"] = "Sun"
# Return sun transit
return suntrans