def __init__(self, stix_packets):
self.num_samples = len(stix_packets['coarse_time'])
# Header
self.scet_coarse = stix_packets['coarse_time']
self.scet_fine = stix_packets['fine_time']
self.obs_utc = scet_to_datetime(
f'{self.scet_coarse[0]}:{self.scet_fine[0]}')
self.obs_beg = self.obs_utc
self.obs_end = scet_to_datetime(
f'{self.scet_coarse[-1]}:{self.scet_fine[-1]}')
self.obs_avg = self.obs_beg + (self.obs_end - self.obs_beg) / 2.0
# Create array of times as dt from date_obs
times = [
scet_to_datetime(f"{stix_packets['coarse_time'][i]}:"
f"{stix_packets['fine_time'][i]}")
for i in range(self.num_samples)
]
time = np.array(times) - times[0]
# Data
self.time = [t.total_seconds() for t in time]
self.sw_running = stix_packets.get('NIXD0021')
self.instrument_number = stix_packets.get('NIXD0022')
self.instrument_mode = stix_packets.get('NIXD0023')
self.hk_dpu_pcb_t = stix_packets.get('NIXD0025')
self.hk_dpu_fpga_t = stix_packets.get('NIXD0026')
self.hk_dpu_3v3_c = stix_packets.get('NIXD0027')
self.hk_dpu_2v5_c = stix_packets.get('NIXD0028')
self.hk_dpu_1v5_c = stix_packets.get('NIXD0029')
self.hk_dpu_spw_c = stix_packets.get('NIXD0030')
self.hk_dpu_spw0_v = stix_packets.get('NIXD0031')
self.hk_dpu_spw1_v = stix_packets.get('NIXD0032')
self.sw_version = stix_packets.get('NIXD0001')
self.cpu_load = stix_packets.get('NIXD0002')
self.archive_memory_usage = stix_packets.get('NIXD0003')
self.autonomous_asw_boot_stat = stix_packets.get('NIXD0166')
self.memory_load_ena_flag = stix_packets.get('NIXD0167')
self.idpu_identifier = stix_packets.get('NIXD0004')
self.active_spw_link = stix_packets.get('NIXD0005')
self.overruns_for_tasks = stix_packets.get('NIXD0168')
self.watchdog_state = stix_packets.get('NIXD0169')
self.received_spw_packets = stix_packets.get('NIXD0079')
self.rejected_spw_packets = stix_packets.get('NIXD0079')
self.hk_dpu_1v5_v = stix_packets.get('NIXD0035')
self.hk_ref_2v5_v = stix_packets.get('NIXD0036')
self.hk_dpu_2v9_v = stix_packets.get('NIXD0037')
self.hk_psu_temp_t = stix_packets.get('NIXD0024')
self.fdir_status = stix_packets.get('NIX00085')
self.fdir_status_mask_of_hk_temperature = stix_packets.get('NIX00161')
self.fdir_status_mask_of_hk_voltage = stix_packets.get('NIX00162')
self.hk_selftest_status_flag = stix_packets.get('NIXD0163')
self.memory_status_flag = stix_packets.get('NIXD0164')
self.fdir_status_mask_of_hk_current = stix_packets.get('NIXD0165')
self.number_executed_tc = stix_packets.get('NIX00166')
self.number_sent_tm = stix_packets.get('NIX00167')
self.number_failed_tm_gen = stix_packets.get('NIX00168')
def _get_time(self):
# Replicate packet time for each sample
base_times = Time(
list(
chain(*[[
scet_to_datetime(
f'{self["scet_coarse"][i]}:{self["scet_fine"][i]}')
] * n for i, n in enumerate(self['num_samples'])])))
# For each sample generate sample number and multiply by duration and apply unit
start_delta = np.hstack([
(np.arange(ns) * it)
for ns, it in self[['num_samples', 'integration_time']]
])
# hstack op loses unit
start_delta = start_delta.value * self['integration_time'].unit
duration = np.hstack([
np.ones(num_sample) * int_time for num_sample, int_time in self[
['num_samples', 'integration_time']]
])
duration = duration.value * self['integration_time'].unit
# TODO Write out and simplify
end_delta = start_delta + duration
# Add the delta time to base times and convert to relative from start time
times = base_times + start_delta + (end_delta - start_delta) / 2
# times -= times[0]
return times, duration
def from_packets(cls, packets, eng_packets):
tmp = QTable()
tmp['scet_coarse'] = packets['coarse_time']
tmp['scet_fine'] = packets['coarse_time']
control = Control(tmp)
data = Data()
if 'parameters' in packets:
control['ubsd_counter'] = packets.get('NIX00285')[0]
control['pald_counter'] = packets.get('NIX00286')[0]
control['num_samples'] = packets.get('NIX00286')[0]
# DATA
data['start_scet_coarse'] = packets.get('NIX00287')
data['end_scet_coarse'] = packets.get('NIX00287')
data['obs_utc'] = scet_to_datetime(
f"{data['start_scet_coarse']}:0")
data['highest_flareflag'] = packets.get('NIX00289')[0]
data['tm_byte_volume'] = packets.get('NIX00290')[0]
data['average_z_loc'] = packets.get('NIX00291')[0]
data['average_y_loc'] = packets.get('NIX00292')[0]
data['processing_mask'] = packets.get('NIX00293')[0]
return cls(control=control, data=data)
else:
return None
def from_packets(cls, packets, eng_packets):
# Header
control = Control()
scet_coarse = packets['NIX00445']
scet_fine = packets['NIX00446']
start_times = Time([
scet_to_datetime(f'{scet_coarse[i]}:{scet_fine[i]}')
for i in range(len(scet_coarse))
])
control['summing_value'] = packets['NIX00088']
control['averaging_value'] = packets['NIX00490']
control['index'] = range(len(control))
delta_time = ((control['summing_value'] * control['averaging_value']) /
1000.0)
samples = packets['NIX00089']
offsets = [
delta_time[i] * 0.5 * np.arange(ns) * u.s
for i, ns in enumerate(samples)
]
time = Time(
np.hstack(
[start_times[i] + offsets[i] for i in range(len(offsets))]))
timedel = np.hstack(offsets).value * u.s
# Data
try:
data = Data()
data['time'] = time
data['timedel'] = timedel
data['cha_diode0'] = packets['NIX00090']
data['cha_diode1'] = packets['NIX00091']
data['chb_diode0'] = packets['NIX00092']
data['chb_diode1'] = packets['NIX00093']
data['control_index'] = np.hstack(
[np.full(ns, i) for i, ns in enumerate(samples)])
except ValueError as e:
logger.warning(e)
return None
return cls(control=control, data=data)
def _get_time(cls, control, num_energies, packets, pad_after):
times = []
durations = []
start = 0
for i, (ns, it) in enumerate(control['num_samples',
'integration_time']):
off_sets = np.array(packets.get('NIX00485')[start:start + ns]) * it
base_time = Time(
scet_to_datetime(
f'{control["scet_coarse"][i]}:{control["scet_fine"][i]}'))
start_times = base_time + off_sets
end_times = base_time + off_sets + it
cur_time = start_times + (end_times - start_times) / 2
times.extend(cur_time)
durations.extend([it] * ns)
start += ns
time = Time(times)
time = Time(
np.pad(time.datetime64, (0, pad_after),
constant_values=time[-1].datetime64))
time = time.reshape(-1, num_energies)
duration = np.pad(np.hstack(durations),
(0, pad_after)).reshape(-1, num_energies) * it.unit
return duration, time
def __init__(self, stix_packets):
self.num_samples = len(stix_packets['coarse_time'])
# Header
self.scet_coarse = stix_packets['coarse_time']
self.scet_fine = stix_packets['fine_time']
self.obs_utc = scet_to_datetime(
f'{self.scet_coarse[0]}:{self.scet_fine[0]}')
self.obs_beg = self.obs_utc
self.obs_end = scet_to_datetime(
f'{self.scet_coarse[-1]}:{self.scet_fine[-1]}')
self.obs_avg = self.obs_beg + (self.obs_end - self.obs_beg) / 2.0
# Create array of times as dt from date_obs
times = [
scet_to_datetime(f"{stix_packets['coarse_time'][i]}:"
f"{stix_packets['fine_time'][i]}")
for i in range(self.num_samples)
]
time = np.array(times) - times[0]
# Data
self.time = [t.total_seconds() for t in time]
self.sw_running = stix_packets.get('NIXD0021')
self.instrument_number = stix_packets.get('NIXD0022')
self.instrument_mode = stix_packets.get('NIXD0023')
self.hk_dpu_pcb_t = stix_packets.get('NIXD0025')
self.hk_dpu_fpga_t = stix_packets.get('NIXD0026')
self.hk_dpu_3v3_c = stix_packets.get('NIXD0027')
self.hk_dpu_2v5_c = stix_packets.get('NIXD0028')
self.hk_dpu_1v5_c = stix_packets.get('NIXD0029')
self.hk_dpu_spw_c = stix_packets.get('NIXD0030')
self.hk_dpu_spw0_v = stix_packets.get('NIXD0031')
self.hk_dpu_spw1_v = stix_packets.get('NIXD0032')
self.hk_asp_ref_2v5a_v = stix_packets.get('NIXD0038')
self.hk_asp_ref_2v5b_v = stix_packets.get('NIXD0039')
self.hk_asp_tim01_t = stix_packets.get('NIXD0040')
self.hk_asp_tim02_t = stix_packets.get('NIXD0041')
self.hk_asp_tim03_t = stix_packets.get('NIXD0042')
self.hk_asp_tim04_t = stix_packets.get('NIXD0043')
self.hk_asp_tim05_t = stix_packets.get('NIXD0044')
self.hk_asp_tim06_t = stix_packets.get('NIXD0045')
self.hk_asp_tim07_t = stix_packets.get('NIXD0046')
self.hk_asp_tim08_t = stix_packets.get('NIXD0047')
self.hk_asp_vsensa_v = stix_packets.get('NIXD0048')
self.hk_asp_vsensb_v = stix_packets.get('NIXD0049')
self.hk_att_v = stix_packets.get('NIXD0050')
self.hk_att_t = stix_packets.get('NIXD0051')
self.hk_hv_01_16_v = stix_packets.get('NIXD0052')
self.hk_hv_17_32_v = stix_packets.get('NIXD0053')
self.det_q1_t = stix_packets.get('NIXD0054')
self.det_q2_t = stix_packets.get('NIXD0055')
self.det_q3_t = stix_packets.get('NIXD0056')
self.det_q4_t = stix_packets.get('NIXD0057')
self.hk_dpu_1v5_v = stix_packets.get('NIXD0035')
self.hk_ref_2v5_v = stix_packets.get('NIXD0036')
self.hk_dpu_2v9_v = stix_packets.get('NIXD0037')
self.hk_psu_temp_t = stix_packets.get('NIXD0024')
self.sw_version = stix_packets.get('NIXD0001')
self.cpu_load = stix_packets.get('NIXD0002')
self.archive_memory_usage = stix_packets.get('NIXD0003')
self.autonomous_asw_boot_stat = stix_packets.get('NIXD0166')
self.memory_load_ena_flag = stix_packets.get('NIXD0167')
self.idpu_identifier = stix_packets.get('NIXD0004')
self.active_spw_link = stix_packets.get('NIXD0005')
self.overruns_for_tasks = stix_packets.get('NIXD0168')
self.watchdog_state = stix_packets.get('NIXD0169')
self.received_spw_packets = stix_packets.get('NIXD0079')
self.rejected_spw_packets = stix_packets.get('NIXD0078')
self.endis_detector_status = stix_packets.get('NIXD0070')
self.spw1_power_status = stix_packets.get('NIXD0080')
self.spw0_power_status = stix_packets.get('NIXD0081')
self.q4_power_status = stix_packets.get('NIXD0082')
self.q3_power_status = stix_packets.get('NIXD0083')
self.q2_power_status = stix_packets.get('NIXD0084')
self.q1_power_status = stix_packets.get('NIXD0085')
self.aspect_b_power_status = stix_packets.get('NIXD0086')
self.aspect_a_power_status = stix_packets.get('NIXD0087')
self.att_m2_moving = stix_packets.get('NIXD0088')
self.att_m1_moving = stix_packets.get('NIXD0089')
self.hv17_32_enabled_status = stix_packets.get('NIXD0090')
self.hv01_16_enabled_status = stix_packets.get('NIXD0091')
self.lv_enabled_status = stix_packets.get('NIXD0092')
self.hv1_depolar_in_progress = stix_packets.get('NIXD0066')
self.hv2_depolar_in_progress = stix_packets.get('NIXD0067')
self.att_ab_flag_open = stix_packets.get('NIXD0068')
self.att_bc_flag_closed = stix_packets.get('NIXD0069')
self.med_value_trg_acc = stix_packets.get('NIX00072')
self.max_value_of_trig_acc = stix_packets.get('NIX00073')
self.hv_regulators_mask = stix_packets.get('NIXD0074')
self.tc_20_128_seq_cnt = stix_packets.get('NIXD0077')
self.attenuator_motions = stix_packets.get('NIX00076')
self.hk_asp_photoa0_v = stix_packets.get('NIX00078')
self.hk_asp_photoa1_v = stix_packets.get('NIX00079')
self.hk_asp_photob0_v = stix_packets.get('NIX00080')
self.hk_asp_photob1_v = stix_packets.get('NIX00081')
self.attenuator_currents = stix_packets.get('NIXD0075')
self.hk_att_c = stix_packets.get('NIXD0075')
self.hk_det_c = stix_packets.get('NIXD0058')
self.fdir_function_status = stix_packets.get('NIX00085')
def from_packets(cls, packets, eng_packets):
control = Control.from_packets(packets)
control['integration_time'] = (
np.array(packets['NIX00122'], np.uint32) + 1) * 0.1 * u.s
# control['obs_beg'] = control['obs_utc']
# control['.obs_end'] = control['obs_beg'] + timedelta(seconds=control['duration'].astype('float'))
# control['.obs_avg'] = control['obs_beg'] + (control['obs_end'] - control['obs_beg']) / 2
# Control
control['quiet_time'] = np.array(packets['NIX00123'], np.uint16)
control['live_time'] = np.array(packets['NIX00124'], np.uint32)
control['average_temperature'] = np.array(packets['NIX00125'],
np.uint16)
control['detector_mask'] = _get_detector_mask(packets)
control['pixel_mask'] = _get_pixel_mask(packets)
control['subspectrum_mask'] = _get_sub_spectrum_mask(packets)
control['compression_scheme_counts_skm'] = _get_compression_scheme(
packets, 'NIXD0126', 'NIXD0127', 'NIXD0128')
subspec_data = {}
j = 129
for subspec, i in enumerate(range(300, 308)):
subspec_data[subspec + 1] = {
'num_points': packets.get(f'NIXD0{j}')[0],
'num_summed_channel': packets.get(f'NIXD0{j + 1}')[0],
'lowest_channel': packets.get(f'NIXD0{j + 2}')[0]
}
j += 3
control['num_samples'] = np.array(packets.get('NIX00159'), np.uint16)
# control.remove_column('index')
# control = unique(control)
# control['index'] = np.arange(len(control))
control['subspec_num_points'] = np.array(
[v['num_points'] for v in subspec_data.values()]).reshape(1, -1)
control['subspec_num_summed_channel'] = np.array([
v['num_summed_channel'] for v in subspec_data.values()
]).reshape(1, -1)
control['subspec_lowest_channel'] = np.array([
v['lowest_channel'] for v in subspec_data.values()
]).reshape(1, -1)
subspec_index = np.argwhere(
control['subspectrum_mask'][0].flatten() == 1)
num_sub_spectra = control['subspectrum_mask'].sum(axis=1)
sub_channels = [
np.arange(control['subspec_num_points'][0, index] + 1) *
(control['subspec_num_summed_channel'][0, index] + 1) +
control['subspec_lowest_channel'][0, index]
for index in subspec_index
]
channels = list(chain(*[ch.tolist() for ch in sub_channels]))
control['num_channels'] = len(channels)
# Data
data = Data()
data['control_index'] = [0]
data['time'] = (Time(
scet_to_datetime(f"{control['scet_coarse'][0]}"
f":{control['scet_fine'][0]}")) +
control['integration_time'][0] / 2).reshape(1)
data['timedel'] = control['integration_time'][0]
# data['detector_id'] = np.array(packets.get('NIXD0155'), np.ubyte)
# data['pixel_id'] = np.array(packets.get('NIXD0156'), np.ubyte)
# data['subspec_id'] = np.array(packets.get('NIXD0157'), np.ubyte)
num_spec_points = np.array(packets.get('NIX00146'))
cs, ck, cm = control['compression_scheme_counts_skm'][0]
counts, counts_var = decompress(packets.get('NIX00158'),
s=cs,
k=ck,
m=cm,
return_variance=True)
counts_rebinned = np.apply_along_axis(
rebin_proportional, 1, counts.reshape(-1, len(channels)), channels,
np.arange(1025))
counts_var_rebinned = np.apply_along_axis(
rebin_proportional, 1, counts_var.reshape(-1, len(channels)),
channels, np.arange(1025))
dids = np.array(packets.get('NIXD0155'),
np.ubyte).reshape(-1, num_sub_spectra[0])[:, 0]
pids = np.array(packets.get('NIXD0156'),
np.ubyte).reshape(-1, num_sub_spectra[0])[:, 0]
full_counts = np.zeros((32, 12, 1024))
full_counts[dids, pids] = counts_rebinned
full_counts_var = np.zeros((32, 12, 1024))
full_counts_var[dids, pids] = counts_var_rebinned
data['counts'] = full_counts.reshape((1, *full_counts.shape))
data['counts_err'] = np.sqrt(full_counts_var).reshape(
(1, *full_counts_var.shape))
return cls(control=control, data=data)
def from_packets(cls, packets, eng_packets):
# Control
control = Control.from_packets(packets)
control['pixel_mask'] = np.unique(_get_pixel_mask(packets), axis=0)
control['detector_mask'] = np.unique(_get_detector_mask(packets),
axis=0)
control['rcr'] = np.unique(packets['NIX00401']).astype(np.int16)
control['index'] = range(len(control))
e_min = np.array(packets['NIXD0442'])
e_max = np.array(packets['NIXD0443'])
energy_unit = np.array(packets['NIXD0019']) + 1
num_times = np.array(packets['NIX00089'])
total_num_times = num_times.sum()
cs, ck, cm = control['compression_scheme_counts_skm'][0]
counts, counts_var = decompress(packets['NIX00268'],
s=cs,
k=ck,
m=cm,
return_variance=True)
counts = counts.reshape(total_num_times, -1)
counts_var = counts_var.reshape(total_num_times, -1)
full_counts = np.zeros((total_num_times, 32))
full_counts_var = np.zeros((total_num_times, 32))
cids = [
np.arange(emin, emax + 1, eunit)
for (emin, emax, eunit) in zip(e_min, e_max, energy_unit)
]
control['energy_bin_mask'] = np.full((1, 32), False, np.ubyte)
control['energy_bin_mask'][:, cids] = True
dl_energies = np.array([[ENERGY_CHANNELS[ch]['e_lower']
for ch in chs] +
[ENERGY_CHANNELS[chs[-1]]['e_upper']]
for chs in cids][0])
sci_energies = np.hstack(
[[ENERGY_CHANNELS[ch]['e_lower'] for ch in range(32)],
ENERGY_CHANNELS[31]['e_upper']])
ind = 0
for nt in num_times:
e_ch_start = 0
e_ch_end = counts.shape[1]
if dl_energies[0] == 0:
full_counts[ind:ind + nt, 0] = counts[ind:ind + nt, 0]
full_counts_var[ind:ind + nt, 0] = counts_var[ind:ind + nt, 0]
e_ch_start = 1
if dl_energies[-1] == np.inf:
full_counts[ind:ind + nt, -1] = counts[ind:ind + nt, -1]
full_counts_var[ind:ind + nt, -1] = counts[ind:ind + nt, -1]
e_ch_end -= 1
torebin = np.where((dl_energies >= 4.0) & (dl_energies <= 150.0))
full_counts[ind:ind + nt, 1:-1] = np.apply_along_axis(
rebin_proportional, 1, counts[ind:ind + nt,
e_ch_start:e_ch_end],
dl_energies[torebin], sci_energies[1:-1])
full_counts_var[ind:ind + nt, 1:-1] = np.apply_along_axis(
rebin_proportional, 1, counts_var[ind:ind + nt,
e_ch_start:e_ch_end],
dl_energies[torebin], sci_energies[1:-1])
ind += nt
if counts.sum() != full_counts.sum():
raise ValueError(
'Original and reformatted count totals do not match')
delta_time = (np.array(packets['NIX00441'], np.uint16)) * 0.1 * u.s
closing_time_offset = (np.array(packets['NIX00269'],
np.uint16)) * 0.1 * u.s
# TODO incorporate into main loop above
centers = []
deltas = []
last = 0
for i, nt in enumerate(num_times):
edge = np.hstack([
delta_time[last:last + nt],
delta_time[last + nt - 1] + closing_time_offset[i]
])
delta = np.diff(edge)
center = edge[:-1] + delta / 2
centers.append(center)
deltas.append(delta)
last = last + nt
centers = np.hstack(centers)
deltas = np.hstack(deltas)
# Data
data = Data()
data['time'] = Time(scet_to_datetime(f'{int(control["time_stamp"][0])}:0')) \
+ centers
data['timedel'] = deltas
ts, tk, tm = control['compression_scheme_triggers_skm'][0]
triggers, triggers_var = decompress(packets['NIX00267'],
s=ts,
k=tk,
m=tm,
return_variance=True)
data['triggers'] = triggers
data['triggers_err'] = np.sqrt(triggers_var)
data['counts'] = full_counts * u.ct
data['counts_err'] = np.sqrt(full_counts_var) * u.ct
data['control_index'] = 0
return cls(control=control, data=data)
def from_packets(cls, packets, eng_packets):
# Control
control = Control.from_packets(packets)
control.remove_column('num_structures')
control = unique(control)
if len(control) != 1:
raise ValueError()
control['index'] = range(len(control))
data = Data()
data['control_index'] = np.full(len(packets['NIX00441']), 0)
data['delta_time'] = (np.array(packets['NIX00441'],
np.uint16)) * 0.1 * u.s
unique_times = np.unique(data['delta_time'])
# time = np.array([])
# for dt in set(self.delta_time):
# i, = np.where(self.delta_time == dt)
# nt = sum(np.array(packets['NIX00258'])[i])
# time = np.append(time, np.repeat(dt, nt))
# self.time = time
data['rcr'] = packets['NIX00401']
data['pixel_mask1'] = _get_pixel_mask(packets, 'NIXD0407')
data['pixel_mask2'] = _get_pixel_mask(packets, 'NIXD0444')
data['pixel_mask3'] = _get_pixel_mask(packets, 'NIXD0445')
data['pixel_mask4'] = _get_pixel_mask(packets, 'NIXD0446')
data['pixel_mask5'] = _get_pixel_mask(packets, 'NIXD0447')
data['detector_masks'] = _get_detector_mask(packets)
data['integration_time'] = (np.array(packets['NIX00405'])) * 0.1
ts, tk, tm = control['compression_scheme_triggers_skm'][0]
triggers, triggers_var = decompress(
[packets[f'NIX00{i}'] for i in range(242, 258)],
s=ts,
k=tk,
m=tm,
return_variance=True)
data['triggers'] = triggers.T
data['triggers_err'] = np.sqrt(triggers_var).T
tids = np.searchsorted(data['delta_time'], unique_times)
data = data[tids]
num_energy_groups = sum(packets['NIX00258'])
# Data
vis = np.zeros((unique_times.size, 32, 32), dtype=complex)
vis_err = np.zeros((unique_times.size, 32, 32), dtype=complex)
e_low = np.array(packets['NIXD0016'])
e_high = np.array(packets['NIXD0017'])
# TODO create energy bin mask
control['energy_bin_mask'] = np.full((1, 32), False, np.ubyte)
all_energies = set(np.hstack([e_low, e_high]))
control['energy_bin_mask'][:, list(all_energies)] = True
data['flux'] = np.array(packets['NIX00261']).reshape(
unique_times.size, -1)
num_detectors = packets['NIX00262'][0]
detector_id = np.array(packets['NIX00100']).reshape(
unique_times.size, -1, num_detectors)
# vis[:, detector_id[0], e_low.reshape(unique_times.size, -1)[0]] = (
# np.array(packets['NIX00263']) + np.array(packets['NIX00264'])
# * 1j).reshape(unique_times.size, num_detectors, -1)
ds, dk, dm = control['compression_scheme_counts_skm'][0]
real, real_var = decompress(packets['NIX00263'],
s=ds,
k=dk,
m=dm,
return_variance=True)
imaginary, imaginary_var = decompress(packets['NIX00264'],
s=ds,
k=dk,
m=dm,
return_variance=True)
mesh = np.ix_(np.arange(unique_times.size), detector_id[0][0],
e_low.reshape(unique_times.size, -1)[0])
vis[mesh] = (real + imaginary * 1j).reshape(unique_times.size,
num_detectors, -1)
# TODO this doesn't seem correct prob need combine in a better
vis_err[mesh] = (np.sqrt(real_var) +
np.sqrt(imaginary_var) * 1j).reshape(
unique_times.size, num_detectors, -1)
data['visibility'] = vis
data['visibility_err'] = vis_err
data['time'] = Time(scet_to_datetime(f'{int(control["time_stamp"][0])}:0')) \
+ data['delta_time'] + data['integration_time'] / 2
data['timedel'] = data['integration_time']
return cls(control=control, data=data)
def from_packets(cls, packets, eng_packets):
# Control
ssid = packets['SSID'][0]
control = Control.from_packets(packets)
control.remove_column('num_structures')
control = unique(control)
if len(control) != 1:
raise ValueError(
'Creating a science product form packets from multiple products'
)
control['index'] = 0
data = Data()
data['delta_time'] = (np.array(packets['NIX00441'],
np.int32)) * 0.1 * u.s
unique_times = np.unique(data['delta_time'])
data['rcr'] = np.array(packets['NIX00401'], np.ubyte)
data['num_pixel_sets'] = np.array(packets['NIX00442'], np.ubyte)
pixel_masks = _get_pixel_mask(packets, 'NIXD0407')
pixel_masks = pixel_masks.reshape(-1, data['num_pixel_sets'][0], 12)
if ssid == 21 and data['num_pixel_sets'][0] != 12:
pixel_masks = np.pad(pixel_masks,
((0, 0), (0, 12 - data['num_pixel_sets'][0]),
(0, 0)))
data['pixel_masks'] = pixel_masks
data['detector_masks'] = _get_detector_mask(packets)
data['integration_time'] = (np.array(packets.get('NIX00405'),
np.uint16)) * 0.1 * u.s
# TODO change once FSW fixed
ts, tk, tm = control['compression_scheme_counts_skm'][0]
triggers, triggers_var = decompress(
[packets.get(f'NIX00{i}') for i in range(242, 258)],
s=ts,
k=tk,
m=tm,
return_variance=True)
data['triggers'] = triggers.T
data['triggers_err'] = np.sqrt(triggers_var).T
data['num_energy_groups'] = np.array(packets['NIX00258'], np.ubyte)
tmp = dict()
tmp['e_low'] = np.array(packets['NIXD0016'], np.ubyte)
tmp['e_high'] = np.array(packets['NIXD0017'], np.ubyte)
tmp['num_data_elements'] = np.array(packets['NIX00259'])
unique_energies_low = np.unique(tmp['e_low'])
unique_energies_high = np.unique(tmp['e_high'])
# counts = np.array(eng_packets['NIX00260'], np.uint32)
cs, ck, cm = control['compression_scheme_counts_skm'][0]
counts, counts_var = decompress(packets.get('NIX00260'),
s=cs,
k=ck,
m=cm,
return_variance=True)
counts = counts.reshape(unique_times.size, unique_energies_low.size,
data['detector_masks'][0].sum(),
data['num_pixel_sets'][0].sum())
counts_var = counts_var.reshape(unique_times.size,
unique_energies_low.size,
data['detector_masks'][0].sum(),
data['num_pixel_sets'][0].sum())
# t x e x d x p -> t x d x p x e
counts = counts.transpose((0, 2, 3, 1))
counts_var = np.sqrt(counts_var.transpose((0, 2, 3, 1)))
if ssid == 21:
out_counts = np.zeros((unique_times.size, 32, 12, 32))
out_var = np.zeros((unique_times.size, 32, 12, 32))
elif ssid == 22:
out_counts = np.zeros((unique_times.size, 32, 4, 32))
out_var = np.zeros((unique_times.size, 32, 4, 32))
# energy_index = 0
# count_index = 0
# for i, time in enumerate(unique_times):
# inds = np.where(data['delta_time'] == time)
# cur_num_energies = data['num_energy_groups'][inds].astype(int).sum()
# low = np.unique(tmp['e_low'][energy_index:energy_index+cur_num_energies])
# high = np.unique(tmp['e_high'][energy_index:energy_index + cur_num_energies])
# cur_num_energies = low.size
# num_counts = tmp['num_data_elements'][energy_index:energy_index+cur_num_energies].sum()
# cur_counts = counts[count_index:count_index+num_counts]
# count_index += num_counts
# pids = data[inds[0][0]]['pixel_masks']
# dids = np.where(data[inds[0][0]]['detector_masks'] == True)
# cids = np.full(32, False)
# cids[low] = True
#
# if ssid == 21:
# cur_counts = cur_counts.reshape(cur_num_energies, dids[0].size, pids.sum())
# elif ssid == 22:
# cur_counts = cur_counts.reshape(cur_num_energies, dids[0].size, 4)
#
dl_energies = np.array([
[ENERGY_CHANNELS[lch]['e_lower'], ENERGY_CHANNELS[hch]['e_upper']]
for lch, hch in zip(unique_energies_low, unique_energies_high)
]).reshape(-1)
dl_energies = np.unique(dl_energies)
sci_energies = np.hstack(
[[ENERGY_CHANNELS[ch]['e_lower'] for ch in range(32)],
ENERGY_CHANNELS[31]['e_upper']])
# If there is any onboard summing of energy channels rebin back to standard sci channels
if (unique_energies_high - unique_energies_low).sum() > 0:
rebinned_counts = np.zeros((*counts.shape[:-1], 32))
rebinned_counts_var = np.zeros((*counts_var.shape[:-1], 32))
e_ch_start = 0
e_ch_end = counts.shape[-1]
if dl_energies[0] == 0.0:
rebinned_counts[..., 0] = counts[..., 0]
rebinned_counts_var[..., 0] = counts_var[..., 0]
e_ch_start += 1
elif dl_energies[-1] == np.inf:
rebinned_counts[..., -1] = counts[..., -1]
rebinned_counts_var[..., -1] = counts_var[..., -1]
e_ch_end -= 1
torebin = np.where((dl_energies >= 4.0) & (dl_energies <= 150.0))
rebinned_counts[..., 1:-1] = np.apply_along_axis(
rebin_proportional, -1,
counts[...,
e_ch_start:e_ch_end].reshape(-1, e_ch_end - e_ch_start),
dl_energies[torebin], sci_energies[1:-1]).reshape(
(*counts.shape[:-1], 30))
rebinned_counts_var[..., 1:-1] = np.apply_along_axis(
rebin_proportional, -1,
counts_var[..., e_ch_start:e_ch_end].reshape(
-1, e_ch_end - e_ch_start), dl_energies[torebin],
sci_energies[1:-1]).reshape((*counts_var.shape[:-1], 30))
energy_indices = np.full(32, True)
energy_indices[[0, -1]] = False
ix = np.ix_(np.full(unique_times.size, True),
data['detector_masks'][0].astype(bool),
np.ones(data['num_pixel_sets'][0], dtype=bool),
np.full(32, True))
out_counts[ix] = rebinned_counts
out_var[ix] = rebinned_counts_var
else:
energy_indices = np.full(32, False)
energy_indices[unique_energies_low.min(
):unique_energies_high.max() + 1] = True
ix = np.ix_(np.full(unique_times.size,
True), data['detector_masks'][0].astype(bool),
np.ones(data['num_pixel_sets'][0], dtype=bool),
energy_indices)
out_counts[ix] = counts
out_var[ix] = counts_var
# if (high - low).sum() > 0:
# raise NotImplementedError()
# #full_counts = rebin_proportional(dl_energies, cur_counts, sci_energies)
#
# dids2 = data[inds[0][0]]['detector_masks']
# cids2 = np.full(32, False)
# cids2[low] = True
# tids2 = time == unique_times
#
# if ssid == 21:
# out_counts[np.ix_(tids2, cids2, dids2, pids)] = cur_counts
# elif ssid == 22:
# out_counts[np.ix_(tids2, cids2, dids2)] = cur_counts
if counts.sum() != out_counts.sum():
import ipdb
ipdb.set_trace()
raise ValueError(
'Original and reformatted count totals do not match')
control['energy_bin_mask'] = np.full((1, 32), False, np.ubyte)
all_energies = set(np.hstack([tmp['e_low'], tmp['e_high']]))
control['energy_bin_mask'][:, list(all_energies)] = True
# time x energy x detector x pixel
# counts = np.array(
# eng_packets['NIX00260'], np.uint16).reshape(unique_times.size, num_energies,
# num_detectors, num_pixels)
# time x channel x detector x pixel need to transpose to time x detector x pixel x channel
sub_index = np.searchsorted(data['delta_time'], unique_times)
data = data[sub_index]
data['time'] = Time(scet_to_datetime(f'{int(control["time_stamp"][0])}:0')) \
+ data['delta_time'] + data['integration_time'] / 2
data['timedel'] = data['integration_time']
data['counts'] = out_counts * u.ct
data['counts_err'] = out_var * u.ct
data['control_index'] = control['index'][0]
data.remove_columns(['delta_time', 'integration_time'])
data = data['time', 'timedel', 'rcr', 'pixel_masks', 'detector_masks',
'num_pixel_sets', 'num_energy_groups', 'triggers',
'triggers_err', 'counts', 'counts_err']
data['control_index'] = 0
return cls(control=control, data=data)
def from_packets(cls, packets, eng_packets):
control = Control.from_packets(packets)
control.remove_column('num_structures')
control = unique(control)
if len(control) != 1:
raise ValueError(
'Creating a science product form packets from multiple products'
)
control['index'] = 0
data = Data()
data['start_time'] = (np.array(packets.get('NIX00404'),
np.uint16)) * 0.1 * u.s
data['rcr'] = np.array(packets.get('NIX00401')[0], np.ubyte)
data['integration_time'] = (np.array(
packets.get('NIX00405')[0], np.int16)) * 0.1 * u.s
data['pixel_masks'] = _get_pixel_mask(packets, 'NIXD0407')
data['detector_masks'] = _get_detector_mask(packets)
data['triggers'] = np.array(
[packets.get(f'NIX00{i}') for i in range(408, 424)], np.int64).T
data['num_samples'] = np.array(packets.get('NIX00406'), np.int16)
num_detectors = 32
num_energies = 32
num_pixels = 12
# Data
tmp = dict()
tmp['pixel_id'] = np.array(packets.get('NIXD0158'), np.ubyte)
tmp['detector_id'] = np.array(packets.get('NIXD0153'), np.ubyte)
tmp['channel'] = np.array(packets.get('NIXD0154'), np.ubyte)
tmp['continuation_bits'] = packets.get('NIXD0159', np.ubyte)
control['energy_bin_mask'] = np.full((1, 32), False, np.ubyte)
all_energies = set(tmp['channel'])
control['energy_bin_mask'][:, list(all_energies)] = True
# Find contiguous time indices
unique_times = np.unique(data['start_time'])
time_indices = np.searchsorted(unique_times, data['start_time'])
# Create full count array 0s are not send down, if cb = 0 1 count, for cb 1 just extract
# and for cb 2 extract and sum
raw_counts = packets.get('NIX00065')
counts_1d = []
raw_count_index = 0
for cb in tmp['continuation_bits']:
if cb == 0:
counts_1d.append(1)
elif cb == 1:
cur_count = raw_counts[raw_count_index]
counts_1d.append(cur_count)
raw_count_index += cb
elif cb == 2:
cur_count = raw_counts[raw_count_index:(raw_count_index + cb)]
combined_count = int.from_bytes(
(cur_count[0] + 1).to_bytes(2, 'big') +
cur_count[1].to_bytes(1, 'big'), 'big')
counts_1d.append(combined_count)
raw_count_index += cb
else:
raise ValueError(
f'Continuation bits value of {cb} not allowed (0, 1, 2)')
counts_1d = np.array(counts_1d, np.uint16)
# raw_counts = counts_1d
end_inds = np.cumsum(data['num_samples'])
start_inds = np.hstack([0, end_inds[:-1]])
dd = [(tmp['pixel_id'][s:e], tmp['detector_id'][s:e],
tmp['channel'][s:e], counts_1d[s:e])
for s, e in zip(start_inds.astype(int), end_inds)]
counts = np.zeros(
(len(unique_times), num_detectors, num_pixels, num_energies),
np.uint32)
for i, (pid, did, cid, cc) in enumerate(dd):
counts[time_indices[i], did, pid, cid] = cc
# Create final count array with 4 dimensions: unique times, 32 det, 32 energies, 12 pixels
# for i in range(self.num_samples):
# tid = np.argwhere(self.raw_counts == unique_times)
# start_index = 0
# for i, time_index in enumerate(time_indices):
# end_index = np.uint32(start_index + np.sum(data['num_samples'][time_index]))
#
# for did, cid, pid in zip(tmp['detector_id'], tmp['channel'], tmp['pixel_id']):
# index_1d = ((tmp['detector_id'] == did) & (tmp['channel'] == cid)
# & (tmp['pixel_id'] == pid))
# cur_count = counts_1d[start_index:end_index][index_1d[start_index:end_index]]
# # If we have a count assign it other wise do nothing as 0
# if cur_count:
# counts[i, did, cid, pid] = cur_count[0]
#
# start_index = end_index
sub_index = np.searchsorted(data['start_time'], unique_times)
data = data[sub_index]
data['time'] = Time(scet_to_datetime(f'{int(control["time_stamp"][0])}:0'))\
+ data['start_time'] + data['integration_time']/2
data['timedel'] = data['integration_time']
data['counts'] = counts * u.ct
data['control_index'] = control['index'][0]
data.remove_columns(['start_time', 'integration_time', 'num_samples'])
return cls(control=control, data=data)
