import time
import skyfield.api
from ch_util import tools, ephemeris, andata
from ch_util.fluxcat import FluxCatalog
sys.path.insert(0, "/home/ssiegel/ch_pipeline/venv/src/draco")
from draco.util import _fast_tools
###################################################
# default variables
###################################################
DEFAULTS = NameSpace(
load_yaml_config(
os.path.join(os.path.dirname(os.path.realpath(__file__)),
'defaults.yaml') + ':n2cal'))
LOG_FILE = os.environ.get(
'N2CAL_LOG_FILE',
os.path.join(os.path.dirname(os.path.realpath(__file__)), 'n2cal.log'))
DEFAULT_LOGGING = {
'formatters': {
'std': {
'format': "%(asctime)s %(levelname)s %(name)s: %(message)s",
'datefmt': "%m/%d %H:%M:%S"
},
},
'handlers': {
'stderr': {
def main(config_file=None, logging_params=DEFAULT_LOGGING):
# Setup logging
log.setup_logging(logging_params)
mlog = log.get_logger(__name__)
# Set config
config = DEFAULTS.deepcopy()
if config_file is not None:
config.merge(NameSpace(load_yaml_config(config_file)))
# Set niceness
current_niceness = os.nice(0)
os.nice(config.niceness - current_niceness)
mlog.info('Changing process niceness from %d to %d. Confirm: %d' %
(current_niceness, config.niceness, os.nice(0)))
# Find acquisition files
acq_files = sorted(glob(os.path.join(config.data_dir, config.acq, "*.h5")))
nfiles = len(acq_files)
# Determine time range of each file
findex = []
tindex = []
for ii, filename in enumerate(acq_files):
subdata = andata.CorrData.from_acq_h5(filename, datasets=())
findex += [ii] * subdata.ntime
tindex += range(subdata.ntime)
findex = np.array(findex)
tindex = np.array(tindex)
# Determine transits within these files
transits = []
data = andata.CorrData.from_acq_h5(acq_files, datasets=())
solar_rise = ephemeris.solar_rising(data.time[0] - 24.0 * 3600.0,
end_time=data.time[-1])
for rr in solar_rise:
ss = ephemeris.solar_setting(rr)[0]
solar_flag = np.flatnonzero((data.time >= rr) & (data.time <= ss))
if solar_flag.size > 0:
solar_flag = solar_flag[::config.downsample]
tval = data.time[solar_flag]
this_findex = findex[solar_flag]
this_tindex = tindex[solar_flag]
file_list, tindices = [], []
for ii in range(nfiles):
this_file = np.flatnonzero(this_findex == ii)
if this_file.size > 0:
file_list.append(acq_files[ii])
tindices.append(this_tindex[this_file])
date = ephemeris.unix_to_datetime(rr).strftime('%Y%m%dT%H%M%SZ')
transits.append((date, tval, file_list, tindices))
# Specify some parameters for algorithm
N = 2048
noffset = len(config.offsets)
if config.sep_pol:
rank = 1
cross_pol = False
pol = np.array(['S', 'E'])
pol_s = np.array(
[rr + 256 * xx for xx in range(0, 8, 2) for rr in range(256)])
pol_e = np.array(
[rr + 256 * xx for xx in range(1, 8, 2) for rr in range(256)])
prod_ss = []
prod_ee = []
else:
rank = 8
cross_pol = config.cross_pol
pol = np.array(['all'])
npol = pol.size
# Create file prefix and suffix
prefix = []
prefix.append("gain_solutions")
if config.output_prefix is not None:
prefix.append(config.output_prefix)
prefix = '_'.join(prefix)
suffix = []
suffix.append("pol_%s" % '_'.join(pol))
suffix.append("niter_%d" % config.niter)
if cross_pol:
suffix.append("zerocross")
else:
suffix.append("keepcross")
if config.normalize:
suffix.append("normed")
else:
suffix.append("notnormed")
suffix = '_'.join(suffix)
# Loop over solar transits
for date, timestamps, files, time_indices in transits:
nfiles = len(files)
mlog.info("%s (%d files) " % (date, nfiles))
output_file = os.path.join(
config.output_dir, "%s_SUN_%s_%s.pickle" % (prefix, date, suffix))
mlog.info("Saving to: %s" % output_file)
# Get info about this set of files
data = andata.CorrData.from_acq_h5(files, datasets=['flags/inputs'])
prod = data.prod
coord = sun_coord(timestamps, deg=True)
fstart = config.freq_start if config.freq_start is not None else 0
fstop = config.freq_stop if config.freq_stop is not None else data.freq.size
freq_index = range(fstart, fstop)
freq = data.freq[freq_index]
ntime = timestamps.size
nfreq = freq.size
# Determind bad inputs
if config.bad_input_file is None or not os.path.isfile(
config.bad_input_file):
bad_input = np.flatnonzero(
~np.all(data.flags['inputs'][:], axis=-1))
else:
with open(config.bad_input_file, 'r') as handler:
bad_input = pickle.load(handler)
mlog.info("%d inputs flagged as bad." % bad_input.size)
bad_prod = np.array([
ii for ii, pp in enumerate(prod)
if (pp[0] in bad_input) or (pp[1] in bad_input)
])
# Create arrays to hold the results
ores = {}
ores['date'] = date
ores['coord'] = coord
ores['time'] = timestamps
ores['freq'] = freq
ores['offsets'] = config.offsets
ores['pol'] = pol
ores['evalue'] = np.zeros((noffset, nfreq, ntime, N), dtype=np.float32)
ores['resp'] = np.zeros((noffset, nfreq, ntime, N, config.neigen),
dtype=np.complex64)
ores['resp_err'] = np.zeros((noffset, nfreq, ntime, N, config.neigen),
dtype=np.float32)
# Loop over frequencies
for ff, find in enumerate(freq_index):
mlog.info("Freq %d of %d. %0.2f MHz." % (ff + 1, nfreq, freq[ff]))
cnt = 0
# Loop over files
for ii, (filename, tind) in enumerate(zip(files, time_indices)):
ntind = len(tind)
mlog.info("Processing file %s (%d time samples)" %
(filename, ntind))
# Loop over times
for tt in tind:
t0 = time.time()
mlog.info("Time %d of %d. %d index of current file." %
(cnt + 1, ntime, tt))
# Load visibilities
with h5py.File(filename, 'r') as hf:
vis = hf['vis'][find, :, tt]
# Set bad products equal to zero
vis[bad_prod] = 0.0
# Different code if we are separating polarisations
if config.sep_pol:
if not any(prod_ss):
for pind, pp in enumerate(prod):
if (pp[0] in pol_s) and (pp[1] in pol_s):
prod_ss.append(pind)
elif (pp[0] in pol_e) and (pp[1] in pol_e):
prod_ee.append(pind)
prod_ss = np.array(prod_ss)
prod_ee = np.array(prod_ee)
mlog.info("Product sizes: %d, %d" %
(prod_ss.size, prod_ee.size))
# Loop over polarisations
for pp, (input_pol,
prod_pol) in enumerate([(pol_s, prod_ss),
(pol_e, prod_ee)]):
visp = vis[prod_pol]
mlog.info("pol %s, visibility size: %d" %
(pol[pp], visp.size))
# Loop over offsets
for oo, off in enumerate(config.offsets):
mlog.info(
"pol %s, rank %d, niter %d, offset %d, cross_pol %s, neigen %d"
% (pol[pp], rank, config.niter, off,
cross_pol, config.neigen))
ev, rr, rre = solve_gain(
visp,
cutoff=off,
cross_pol=cross_pol,
normalize=config.normalize,
rank=rank,
niter=config.niter,
neigen=config.neigen)
ores['evalue'][oo, ff, cnt, input_pol] = ev
ores['resp'][oo, ff, cnt, input_pol, :] = rr
ores['resp_err'][oo, ff, cnt,
input_pol, :] = rre
else:
# Loop over offsets
for oo, off in enumerate(config.offsets):
mlog.info(
"rank %d, niter %d, offset %d, cross_pol %s, neigen %d"
% (rank, config.niter, off, cross_pol,
config.neigen))
ev, rr, rre = solve_gain(
vis,
cutoff=off,
cross_pol=cross_pol,
normalize=config.normalize,
rank=rank,
niter=config.niter,
neigen=config.neigen)
ores['evalue'][oo, ff, cnt, :] = ev
ores['resp'][oo, ff, cnt, :, :] = rr
ores['resp_err'][oo, ff, cnt, :, :] = rre
# Increment time counter
cnt += 1
# Print time elapsed
mlog.info("Took %0.1f seconds." % (time.time() - t0, ))
# Save to pickle file
with open(output_file, 'w') as handle:
pickle.dump(ores, handle)
import log
from pychfpga import NameSpace, load_yaml_config
from calibration import utils
from ch_util import andata, tools, ephemeris, timing
from ch_util.fluxcat import FluxCatalog
###################################################
# default variables
###################################################
DEFAULTS = NameSpace(
load_yaml_config(
os.path.join(os.path.dirname(os.path.realpath(__file__)),
'defaults.yaml') + ':point_source.analysis'))
LOG_FILE = os.environ.get(
'CALIBRATION_LOG_FILE',
os.path.join(os.path.dirname(os.path.realpath(__file__)),
'offline_cal.log'))
DEFAULT_LOGGING = {
'formatters': {
'std': {
'format': "%(asctime)s %(levelname)s %(name)s: %(message)s",
'datefmt': "%m/%d %H:%M:%S"
},
},
'handlers': {
def main(config_file=None, logging_params=DEFAULT_LOGGING):
# Setup logging
log.setup_logging(logging_params)
mlog = log.get_logger(__name__)
# Set config
config = DEFAULTS.deepcopy()
if config_file is not None:
config.merge(NameSpace(load_yaml_config(config_file)))
# Set niceness
current_niceness = os.nice(0)
os.nice(config.niceness - current_niceness)
mlog.info('Changing process niceness from %d to %d. Confirm: %d' %
(current_niceness, config.niceness, os.nice(0)))
# Find acquisition files
acq_files = sorted(glob(os.path.join(config.data_dir, config.acq, "*.h5")))
nfiles = len(acq_files)
# Determine time range of each file
findex = []
tindex = []
for ii, filename in enumerate(acq_files):
subdata = andata.CorrData.from_acq_h5(filename, datasets=())
findex += [ii] * subdata.ntime
tindex += range(subdata.ntime)
findex = np.array(findex)
tindex = np.array(tindex)
# Determine transits within these files
transits = []
data = andata.CorrData.from_acq_h5(acq_files, datasets=())
solar_rise = ephemeris.solar_rising(data.time[0] - 24.0 * 3600.0,
end_time=data.time[-1])
for rr in solar_rise:
ss = ephemeris.solar_setting(rr)[0]
solar_flag = np.flatnonzero((data.time >= rr) & (data.time <= ss))
if solar_flag.size > 0:
solar_flag = solar_flag[::config.downsample]
tval = data.time[solar_flag]
this_findex = findex[solar_flag]
this_tindex = tindex[solar_flag]
file_list, tindices = [], []
for ii in range(nfiles):
this_file = np.flatnonzero(this_findex == ii)
if this_file.size > 0:
file_list.append(acq_files[ii])
tindices.append(this_tindex[this_file])
date = ephemeris.unix_to_datetime(rr).strftime('%Y%m%dT%H%M%SZ')
transits.append((date, tval, file_list, tindices))
# Create file prefix and suffix
prefix = []
prefix.append("redundant_calibration")
if config.output_prefix is not None:
prefix.append(config.output_prefix)
prefix = '_'.join(prefix)
suffix = []
if config.include_auto:
suffix.append("wauto")
else:
suffix.append("noauto")
if config.include_intracyl:
suffix.append("wintra")
else:
suffix.append("nointra")
if config.fix_degen:
suffix.append("fixed_degen")
else:
suffix.append("degen")
suffix = '_'.join(suffix)
# Loop over solar transits
for date, timestamps, files, time_indices in transits:
nfiles = len(files)
mlog.info("%s (%d files) " % (date, nfiles))
output_file = os.path.join(config.output_dir,
"%s_SUN_%s_%s.h5" % (prefix, date, suffix))
mlog.info("Saving to: %s" % output_file)
# Get info about this set of files
data = andata.CorrData.from_acq_h5(files,
datasets=['flags/inputs'],
apply_gain=False,
renormalize=False)
coord = sun_coord(timestamps, deg=True)
fstart = config.freq_start if config.freq_start is not None else 0
fstop = config.freq_stop if config.freq_stop is not None else data.freq.size
freq_index = range(fstart, fstop)
freq = data.freq[freq_index]
ntime = timestamps.size
nfreq = freq.size
# Determind bad inputs
if config.bad_input_file is None or not os.path.isfile(
config.bad_input_file):
bad_input = np.flatnonzero(
~np.all(data.flags['inputs'][:], axis=-1))
else:
with open(config.bad_input_file, 'r') as handler:
bad_input = pickle.load(handler)
mlog.info("%d inputs flagged as bad." % bad_input.size)
nant = data.ninput
# Determine polarization product maps
dbinputs = tools.get_correlator_inputs(ephemeris.unix_to_datetime(
timestamps[0]),
correlator='chime')
dbinputs = tools.reorder_correlator_inputs(data.input, dbinputs)
feedpos = tools.get_feed_positions(dbinputs)
prod = defaultdict(list)
dist = defaultdict(list)
for pp, this_prod in enumerate(data.prod):
aa, bb = this_prod
inp_aa = dbinputs[aa]
inp_bb = dbinputs[bb]
if (aa in bad_input) or (bb in bad_input):
continue
if not tools.is_chime(inp_aa) or not tools.is_chime(inp_bb):
continue
if not config.include_intracyl and (inp_aa.cyl == inp_bb.cyl):
continue
if not config.include_auto and (aa == bb):
continue
this_dist = list(feedpos[aa, :] - feedpos[bb, :])
if tools.is_array_x(inp_aa) and tools.is_array_x(inp_bb):
key = 'XX'
elif tools.is_array_y(inp_aa) and tools.is_array_y(inp_bb):
key = 'YY'
elif not config.include_crosspol:
continue
elif tools.is_array_x(inp_aa) and tools.is_array_y(inp_bb):
key = 'XY'
elif tools.is_array_y(inp_aa) and tools.is_array_x(inp_bb):
key = 'YX'
else:
raise RuntimeError("CHIME feeds not polarized.")
prod[key].append(pp)
dist[key].append(this_dist)
polstr = sorted(prod.keys())
polcnt = 0
pol_sky_id = []
bmap = {}
for key in polstr:
prod[key] = np.array(prod[key])
dist[key] = np.array(dist[key])
p_bmap, p_ubaseline = generate_mapping(dist[key])
nubase = p_ubaseline.shape[0]
bmap[key] = p_bmap + polcnt
if polcnt > 0:
ubaseline = np.concatenate((ubaseline, p_ubaseline), axis=0)
pol_sky_id += [key] * nubase
else:
ubaseline = p_ubaseline.copy()
pol_sky_id = [key] * nubase
polcnt += nubase
mlog.info("%d unique baselines" % polcnt)
nsky = ubaseline.shape[0]
# Create arrays to hold the results
ores = {}
ores['freq'] = freq
ores['input'] = data.input
ores['time'] = timestamps
ores['coord'] = coord
ores['pol'] = np.array(pol_sky_id)
ores['baseline'] = ubaseline
# Create array to hold gain results
ores['gain'] = np.zeros((nfreq, nant, ntime), dtype=np.complex)
ores['sky'] = np.zeros((nfreq, nsky, ntime), dtype=np.complex)
ores['err'] = np.zeros((nfreq, nant + nsky, ntime, 2), dtype=np.float)
# Loop over polarisations
for key in polstr:
reverse_map = bmap[key]
p_prod = prod[key]
isort = np.argsort(reverse_map)
p_prod = p_prod[isort]
p_ant1 = data.prod['input_a'][p_prod]
p_ant2 = data.prod['input_b'][p_prod]
p_vismap = reverse_map[isort]
# Find the redundant groups
tmp = np.where(np.diff(p_vismap) != 0)[0]
edges = np.zeros(2 + tmp.size, dtype='int')
edges[0] = 0
edges[1:-1] = tmp + 1
edges[-1] = p_vismap.size
kept_base = np.unique(p_vismap)
# Determine the unique antennas
kept_ants = np.unique(np.concatenate([p_ant1, p_ant2]))
antmap = np.zeros(kept_ants.max() + 1, dtype='int') - 1
p_nant = kept_ants.size
for i in range(p_nant):
antmap[kept_ants[i]] = i
p_ant1_use = antmap[p_ant1].copy()
p_ant2_use = antmap[p_ant2].copy()
# Create matrix
p_nvis = p_prod.size
nred = edges.size - 1
npar = p_nant + nred
A = np.zeros((p_nvis, npar), dtype=np.float32)
B = np.zeros((p_nvis, npar), dtype=np.float32)
for kk in range(p_nant):
flag_ant1 = p_ant1_use == kk
if np.any(flag_ant1):
A[flag_ant1, kk] = 1.0
B[flag_ant1, kk] = 1.0
flag_ant2 = p_ant2_use == kk
if np.any(flag_ant2):
A[flag_ant2, kk] = 1.0
B[flag_ant2, kk] = -1.0
for ee in range(nred):
A[edges[ee]:edges[ee + 1], p_nant + ee] = 1.0
B[edges[ee]:edges[ee + 1], p_nant + ee] = 1.0
# Add equations to break degeneracy
if config.fix_degen:
A = np.concatenate((A, np.zeros((1, npar), dtype=np.float32)))
A[-1, 0:p_nant] = 1.0
B = np.concatenate((B, np.zeros((3, npar), dtype=np.float32)))
B[-3, 0:p_nant] = 1.0
B[-2, 0:p_nant] = feedpos[kept_ants, 0]
B[-1, 0:p_nant] = feedpos[kept_ants, 1]
# Loop over frequencies
for ff, find in enumerate(freq_index):
mlog.info("Freq %d of %d. %0.2f MHz." %
(ff + 1, nfreq, freq[ff]))
cnt = 0
# Loop over files
for ii, (filename, tind) in enumerate(zip(files,
time_indices)):
ntind = len(tind)
mlog.info("Processing file %s (%d time samples)" %
(filename, ntind))
# Compute noise weight
with h5py.File(filename, 'r') as hf:
wnoise = np.median(hf['flags/vis_weight'][find, :, :],
axis=-1)
# Loop over times
for tt in tind:
t0 = time.time()
mlog.info("Time %d of %d. %d index of current file." %
(cnt + 1, ntime, tt))
# Load visibilities
with h5py.File(filename, 'r') as hf:
snap = hf['vis'][find, :, tt]
wsnap = wnoise * (
(hf['flags/vis_weight'][find, :, tt] > 0.0) &
(np.abs(snap) > 0.0)).astype(np.float32)
# Extract relevant products for this polarization
snap = snap[p_prod]
wsnap = wsnap[p_prod]
# Turn into amplitude and phase, avoiding NaN
mask = (wsnap > 0.0)
amp = np.where(mask, np.log(np.abs(snap)), 0.0)
phi = np.where(mask, np.angle(snap), 0.0)
# Deal with phase wrapping
for aa, bb in zip(edges[:-1], edges[1:]):
dphi = phi[aa:bb] - np.sort(phi[aa:bb])[int(
(bb - aa) / 2)]
phi[aa:bb] += (2.0 * np.pi * (dphi < -np.pi) -
2.0 * np.pi * (dphi > np.pi))
# Add elements to fix degeneracy
if config.fix_degen:
amp = np.concatenate((amp, np.zeros(1)))
phi = np.concatenate((phi, np.zeros(3)))
# Determine noise matrix
inv_diagC = wsnap * np.abs(snap)**2 * 2.0
if config.fix_degen:
inv_diagC = np.concatenate((inv_diagC, np.ones(1)))
# Amplitude estimate and covariance
amp_param_cov = np.linalg.inv(
np.dot(A.T, inv_diagC[:, np.newaxis] * A))
amp_param = np.dot(amp_param_cov,
np.dot(A.T, inv_diagC * amp))
# Phase estimate and covariance
if config.fix_degen:
inv_diagC = np.concatenate((inv_diagC, np.ones(2)))
phi_param_cov = np.linalg.inv(
np.dot(B.T, inv_diagC[:, np.newaxis] * B))
phi_param = np.dot(phi_param_cov,
np.dot(B.T, inv_diagC * phi))
# Save to large array
ores['gain'][ff, kept_ants,
cnt] = np.exp(amp_param[0:p_nant] +
1.0J * phi_param[0:p_nant])
ores['sky'][ff, kept_base,
cnt] = np.exp(amp_param[p_nant:] +
1.0J * phi_param[p_nant:])
ores['err'][ff, kept_ants, cnt,
0] = np.diag(amp_param_cov[0:p_nant,
0:p_nant])
ores['err'][ff, nant + kept_base, cnt,
0] = np.diag(amp_param_cov[p_nant:,
p_nant:])
ores['err'][ff, kept_ants, cnt,
1] = np.diag(phi_param_cov[0:p_nant,
0:p_nant])
ores['err'][ff, nant + kept_base, cnt,
1] = np.diag(phi_param_cov[p_nant:,
p_nant:])
# Increment time counter
cnt += 1
# Print time elapsed
mlog.info("Took %0.1f seconds." % (time.time() - t0, ))
# Save to pickle file
with h5py.File(output_file, 'w') as handler:
handler.attrs['date'] = date
for key, val in ores.iteritems():
handler.create_dataset(key, data=val)
import os
import datetime
import time
import numpy as np
import h5py
from pychfpga import NameSpace, load_yaml_config
from pychfpga import Hdf5Writer
from ch_util import ephemeris
from calibration.utils import get_window
from version import __version__
DEFAULTS = NameSpace(load_yaml_config(os.path.join(os.path.dirname(os.path.realpath(__file__)),
'defaults.yaml') + ':point_source'))
def mkdir(directory):
""" Make a directory if it does not already exist.
"""
try:
os.makedirs(directory)
except OSError:
if not os.path.isdir(directory):
raise
class PointSourceWriter(Hdf5Writer):
""" Interface to an Hdf5Writer containing fits to point source transit.
"""
def main(config_file=None, logging_params=DEFAULT_LOGGING):
# Setup logging
log.setup_logging(logging_params)
mlog = log.get_logger(__name__)
# Set config
config = DEFAULTS.deepcopy()
if config_file is not None:
config.merge(NameSpace(load_yaml_config(config_file)))
# Create transit tracker
source_list = FluxCatalog.sort(
) if not config.source_list else config.source_list
cal_list = [
name for name, obj in FluxCatalog.iteritems()
if (obj.dec >= config.min_dec) and (
obj.predict_flux(config.freq_nominal) >= config.min_flux) and (
name in source_list)
]
if not cal_list:
raise RuntimeError("No calibrators found.")
# Sort list by flux at nominal frequency
cal_list.sort(
key=lambda name: FluxCatalog[name].predict_flux(config.freq_nominal))
# Add to transit tracker
transit_tracker = containers.TransitTrackerOffline(
nsigma=config.nsigma_source, extend_night=config.extend_night)
for name in cal_list:
transit_tracker[name] = FluxCatalog[name].skyfield
mlog.info("Initializing offline point source processing.")
search_time = config.start_time or 0
# Find all calibration files
all_files = sorted(
glob.glob(
os.path.join(config.acq_dir,
'*' + config.correlator + config.acq_suffix, '*.h5')))
if not all_files:
return
# Remove files whose last modified time is before the time of the most recent update
all_files = [
ff for ff in all_files if (os.path.getmtime(ff) > search_time)
]
if not all_files:
return
# Remove files that are currently locked
all_files = [
ff for ff in all_files
if not os.path.isfile(os.path.splitext(ff)[0] + '.lock')
]
if not all_files:
return
# Add files to transit tracker
for ff in all_files:
transit_tracker.add_file(ff)
# Extract point source transits ready for analysis
all_transits = transit_tracker.get_transits()
# Create dictionary to hold results
h5_psrc_fit = {}
inputmap = None
# Loop over transits
for transit in all_transits:
src, csd, is_day, files, start, stop = transit
# Discard any point sources with unusual csd value
if (csd < config.min_csd) or (csd > config.max_csd):
continue
# Discard any point sources transiting during the day
if is_day > config.process_daytime:
continue
mlog.info(
'Processing %s transit on CSD %d (%d files, %d time samples)' %
(src, csd, len(files), stop - start + 1))
# Load inputmap
if inputmap is None:
if config.inputmap is None:
inputmap = tools.get_correlator_inputs(
ephemeris.unix_to_datetime(ephemeris.csd_to_unix(csd)),
correlator=config.correlator)
else:
with open(config.inputmap, 'r') as handler:
inputmap = pickle.load(handler)
# Grab the timing correction for this transit
tcorr = None
if config.apply_timing:
if config.timing_glob is not None:
mlog.info(
"Loading timing correction from extended timing solutions."
)
timing_files = sorted(glob.glob(config.timing_glob))
if timing_files:
try:
tcorr = search_extended_timing_solutions(
timing_files, ephemeris.csd_to_unix(csd))
except Exception as e:
mlog.error(
'search_extended_timing_solutions failed with error: %s'
% e)
else:
mlog.info(str(tcorr))
if tcorr is None:
mlog.info(
"Loading timing correction from chimetiming acquisitions.")
try:
tcorr = timing.load_timing_correction(
files,
start=start,
stop=stop,
window=config.timing_window,
instrument=config.correlator)
except Exception as e:
mlog.error(
'timing.load_timing_correction failed with error: %s' %
e)
mlog.warning(
'No timing correction applied to %s transit on CSD %d.'
% (src, csd))
else:
mlog.info(str(tcorr))
# Call the main routine to process data
try:
outdct = offline_cal.offline_point_source_calibration(
files,
src,
start=start,
stop=stop,
inputmap=inputmap,
tcorr=tcorr,
logging_params=logging_params,
**config.analysis.as_dict())
except Exception as e:
msg = 'offline_cal.offline_point_source_calibration failed with error: %s' % e
mlog.error(msg)
continue
#raise RuntimeError(msg)
# Find existing gain files for this particular point source
if src not in h5_psrc_fit:
output_files = find_files(config, psrc=src)
if output_files is not None:
output_files = output_files[-1]
mlog.info('Writing %s transit on CSD %d to existing file %s.' %
(src, csd, output_files))
h5_psrc_fit[src] = containers.PointSourceWriter(
src,
output_file=output_files,
output_dir=config.output_dir,
output_suffix=point_source_name_to_file_suffix(src),
instrument=config.correlator,
max_file_size=config.max_file_size,
max_num=config.max_num_time,
memory_size=0)
# Associate this gain calibration to the transit time
this_time = ephemeris.transit_times(FluxCatalog[src].skyfield,
ephemeris.csd_to_unix(csd))[0]
outdct['csd'] = csd
outdct['is_daytime'] = is_day
outdct['acquisition'] = os.path.basename(os.path.dirname(files[0]))
# Write to output file
mlog.info('Writing to disk results from %s transit on CSD %d.' %
(src, csd))
h5_psrc_fit[src].write(this_time, **outdct)
# Dump an individual file for this point source transit
mlog.info('Dumping to disk single file for %s transit on CSD %d.' %
(src, csd))
dump_dir = os.path.join(config.output_dir, 'point_source_gains')
containers.mkdir(dump_dir)
dump_file = os.path.join(dump_dir, '%s_csd_%d.h5' % (src.lower(), csd))
h5_psrc_fit[src].dump(dump_file,
datasets=[
'csd', 'acquisition', 'is_daytime', 'gain',
'weight', 'timing', 'model'
])
mlog.info('Finished analysis of %s transit on CSD %d.' % (src, csd))
def main(config_file=None, logging_params=DEFAULT_LOGGING):
# Setup logging
log.setup_logging(logging_params)
mlog = log.get_logger(__name__)
# Set config
config = DEFAULTS.deepcopy()
if config_file is not None:
config.merge(NameSpace(load_yaml_config(config_file)))
# Set niceness
current_niceness = os.nice(0)
os.nice(config.niceness - current_niceness)
mlog.info('Changing process niceness from %d to %d. Confirm: %d' %
(current_niceness, config.niceness, os.nice(0)))
# Create output suffix
output_suffix = config.output_suffix if config.output_suffix is not None else "jumps"
# Calculate the wavelet transform for the following scales
nwin = 2 * config.max_scale + 1
nhwin = nwin // 2
if config.log_scale:
mlog.info("Using log scale.")
scale = np.logspace(np.log10(config.min_scale), np.log10(nwin), num=config.num_points, dtype=np.int)
else:
mlog.info("Using linear scale.")
scale = np.arange(config.min_scale, nwin, dtype=np.int)
# Loop over acquisitions
for acq in config.acq:
# Find acquisition files
all_data_files = sorted(glob(os.path.join(config.data_dir, acq, "*.h5")))
nfiles = len(all_data_files)
if nfiles == 0:
continue
mlog.info("Now processing acquisition %s (%d files)" % (acq, nfiles))
# Determine list of feeds to examine
dset = ['flags/inputs'] if config.use_input_flag else ()
rdr = andata.CorrData.from_acq_h5(all_data_files, datasets=dset,
apply_gain=False, renormalize=False)
inputmap = tools.get_correlator_inputs(ephemeris.unix_to_datetime(rdr.time[0]),
correlator='chime')
# Extract good inputs
if config.use_input_flag:
ifeed = np.flatnonzero((np.sum(rdr.flags['inputs'][:], axis=-1, dtype=np.int) /
float(rdr.flags['inputs'].shape[-1])) > config.input_threshold)
else:
ifeed = np.array([ii for ii, inp in enumerate(inputmap) if tools.is_chime(inp)])
ninp = len(ifeed)
mlog.info("Processing %d feeds." % ninp)
# Create list of candidates
cfreq, cinput, ctime, cindex = [], [], [], []
jump_flag, jump_time, jump_auto = [], [], []
ncandidate = 0
# Determine number of files to process at once
if config.max_num_file is None:
chunk_size = nfiles
else:
chunk_size = min(config.max_num_file, nfiles)
# Loop over chunks of files
for chnk, data_files in enumerate(chunks(all_data_files, chunk_size)):
mlog.info("Now processing chunk %d (%d files)" % (chnk, len(data_files)))
# Deteremine selections along the various axes
rdr = andata.CorrData.from_acq_h5(data_files, datasets=())
auto_sel = np.array([ii for ii, pp in enumerate(rdr.prod) if pp[0] == pp[1]])
auto_sel = andata._convert_to_slice(auto_sel)
if config.time_start is None:
ind_start = 0
else:
time_start = ephemeris.datetime_to_unix(datetime.datetime(*config.time_start))
ind_start = int(np.argmin(np.abs(rdr.time - time_start)))
if config.time_stop is None:
ind_stop = rdr.ntime
else:
time_stop = ephemeris.datetime_to_unix(datetime.datetime(*config.time_stop))
ind_stop = int(np.argmin(np.abs(rdr.time - time_stop)))
if config.freq_physical is not None:
if hasattr(config.freq_physical, '__iter__'):
freq_physical = config.freq_physical
else:
freq_physical = [config.freq_physical]
freq_sel = [np.argmin(np.abs(ff - rdr.freq)) for ff in freq_physical]
freq_sel = andata._convert_to_slice(freq_sel)
else:
fstart = config.freq_start if config.freq_start is not None else 0
fstop = config.freq_stop if config.freq_stop is not None else rdr.freq.size
freq_sel = slice(fstart, fstop)
# Load autocorrelations
t0 = time.time()
data = andata.CorrData.from_acq_h5(data_files, datasets=['vis'], start=ind_start, stop=ind_stop,
freq_sel=freq_sel, prod_sel=auto_sel,
apply_gain=False, renormalize=False)
mlog.info("Took %0.1f seconds to load autocorrelations." % (time.time() - t0,))
# If first chunk, save the frequencies that are being used
if not chnk:
all_freq = data.freq.copy()
# If requested do not consider data during day or near bright source transits
flag_quiet = np.ones(data.ntime, dtype=np.bool)
if config.ignore_sun:
flag_quiet &= ~transit_flag('sun', data.time, freq=np.min(data.freq), pol='X', nsig=1.0)
if config.only_quiet:
flag_quiet &= ~daytime_flag(data.time)
for ss in ["CYG_A", "CAS_A", "TAU_A", "VIR_A"]:
flag_quiet &= ~transit_flag(ss, data.time, freq=np.min(data.freq), pol='X', nsig=1.0)
# Loop over frequencies
for ff, freq in enumerate(data.freq):
print_cnt = 0
mlog.info("FREQ %d (%0.2f MHz)" % (ff, freq))
auto = data.vis[ff, :, :].real
fractional_auto = auto * tools.invert_no_zero(np.median(auto, axis=-1, keepdims=True)) - 1.0
# Loop over inputs
for ii in ifeed:
print_cnt += 1
do_print = not (print_cnt % 100)
if do_print:
mlog.info("INPUT %d" % ii)
t0 = time.time()
signal = fractional_auto[ii, :]
# Perform wavelet transform
coef, freqs = pywt.cwt(signal, scale, config.wavelet_name)
if do_print:
mlog.info("Took %0.1f seconds to perform wavelet transform." % (time.time() - t0,))
t0 = time.time()
# Find local modulus maxima
flg_mod_max, mod_max = mod_max_finder(scale, coef, threshold=config.thresh, search_span=config.search_span)
if do_print:
mlog.info("Took %0.1f seconds to find modulus maxima." % (time.time() - t0,))
t0 = time.time()
# Find persisent modulus maxima across scales
candidates, cmm, pdrift, start, stop, lbl = finger_finder(scale, flg_mod_max, mod_max,
istart=max(config.min_rise - config.min_scale, 0),
do_fill=False)
if do_print:
mlog.info("Took %0.1f seconds to find fingers." % (time.time() - t0,))
t0 = time.time()
if candidates is None:
continue
# Cut bad candidates
index_good_candidates = np.flatnonzero((scale[stop] >= config.max_scale) &
flag_quiet[candidates[start, np.arange(start.size)]] &
(pdrift <= config.psigma_max))
ngood = index_good_candidates.size
if ngood == 0:
continue
mlog.info("Input %d has %d jumps" % (ii, ngood))
# Add remaining candidates to list
ncandidate += ngood
cfreq += [freq] * ngood
cinput += [ii] * ngood
for igc in index_good_candidates:
icenter = candidates[start[igc], igc]
cindex.append(icenter)
ctime.append(data.time[icenter])
aa = max(0, icenter - nhwin)
bb = min(data.ntime, icenter + nhwin + 1)
ncut = bb - aa
temp_var = np.zeros(nwin, dtype=np.bool)
temp_var[0:ncut] = True
jump_flag.append(temp_var)
temp_var = np.zeros(nwin, dtype=data.time.dtype)
temp_var[0:ncut] = data.time[aa:bb].copy()
jump_time.append(temp_var)
temp_var = np.zeros(nwin, dtype=auto.dtype)
temp_var[0:ncut] = auto[ii, aa:bb].copy()
jump_auto.append(temp_var)
# Garbage collect
del data
gc.collect()
# If we found any jumps, write them to a file.
if ncandidate > 0:
output_file = os.path.join(config.output_dir, "%s_%s.h5" % (acq, output_suffix))
mlog.info("Writing %d jumps to: %s" % (ncandidate, output_file))
# Write to output file
with h5py.File(output_file, 'w') as handler:
handler.attrs['files'] = all_data_files
handler.attrs['chan_id'] = ifeed
handler.attrs['freq'] = all_freq
index_map = handler.create_group('index_map')
index_map.create_dataset('jump', data=np.arange(ncandidate))
index_map.create_dataset('window', data=np.arange(nwin))
ax = np.array(['jump'])
dset = handler.create_dataset('freq', data=np.array(cfreq))
dset.attrs['axis'] = ax
dset = handler.create_dataset('input', data=np.array(cinput))
dset.attrs['axis'] = ax
dset = handler.create_dataset('time', data=np.array(ctime))
dset.attrs['axis'] = ax
dset = handler.create_dataset('time_index', data=np.array(cindex))
dset.attrs['axis'] = ax
ax = np.array(['jump', 'window'])
dset = handler.create_dataset('jump_flag', data=np.array(jump_flag))
dset.attrs['axis'] = ax
dset = handler.create_dataset('jump_time', data=np.array(jump_time))
dset.attrs['axis'] = ax
dset = handler.create_dataset('jump_auto', data=np.array(jump_auto))
dset.attrs['axis'] = ax
else:
mlog.info("No jumps found for %s acquisition." % acq)