def test_deltatime_sub():
t1 = SCETime(100, 2)
dt1 = SCETimeDelta(100, 1)
dt2 = SCETimeDelta(200, 2)
with pytest.raises(TypeError, match=r'Unsupported operation for types SCETimeDelta and int'):
_ = dt1 - 1
with pytest.raises(TypeError, match=r'Quantity could not be converted to SCETimeDelta'):
_ = dt1 - (1*u.m)
# test sub deltatimes and quantities
dt1_dt2 = dt1 - dt2
dt1_float = dt1 - (200 + 2 / MAX_FINE) * u.s
assert dt1_dt2 == dt1_float
assert dt1_dt2.coarse == -100
assert dt1_dt2.fine == -1
dt2_dt1 = dt2 - dt1
float_dt1 = (200 + 2/MAX_FINE) * u.s - dt1
assert dt2_dt1 == float_dt1
assert dt2_dt1.coarse == 100
assert dt2_dt1.fine == 1
# test sub times
with pytest.raises(TypeError, match=f'Unsupported operation for types.*'):
dt1 - t1
t2 = t1 - dt1
assert t2.coarse == 0
assert t2.fine == 1
with pytest.raises(ValueError, match=r'Coarse time must be in range.*'):
t1 - dt2
def test_timedelta_eq():
dt1 = SCETimeDelta(123, 456)
dt2 = SCETimeDelta((123 + 456/MAX_FINE)*u.s)
dt3 = SCETimeDelta(-1, -1)
assert dt1 == dt2
assert dt1 != dt3
assert dt1 == (123 + 456/MAX_FINE)*u.s
def _get_time(self):
# Replicate the start time of each for the number of samples in that packet
base_coarse, base_fine = zip(
*[([ct] * ns, [ft] * ns) for ns, ct, ft in self[
['num_samples', 'scet_coarse', 'scet_fine']]])
base_coarse = np.hstack(base_coarse)
base_fine = np.hstack(base_fine)
bases = SCETime(base_coarse, base_fine)
# Create start time for each time bin by multiplying the duration by the sample number
deltas = SCETimeDelta(
np.hstack([(np.arange(ns) * it)
for ns, it in self[['num_samples', 'integration_time']]
]))
# Create integration time for each sample in each packet by replicating the duration for
# number of sample in each packet
durations = SCETimeDelta(
np.hstack([
np.ones(num_sample) * int_time for num_sample, int_time in
self[['num_samples', 'integration_time']]
]))
# Add the delta time to base times and convert to bin centers
times = bases + deltas + (durations / 2)
# Create a time range object covering the total observation time
tr = SCETimeRange(start=bases[0],
end=bases[-1] + deltas[-1] + durations[-1])
return times, durations, tr
def test_timedelta_add():
t1 = SCETime(1, 1)
dt1 = SCETimeDelta(100, 1)
dt2 = SCETimeDelta(200, 2)
# test time plus timedelta
t1_dt1 = dt1 + t1
dt1_t1 = t1 + dt1
assert t1_dt1 == dt1_t1
assert t1_dt1.coarse == 101
assert t1_dt1.fine == 2
with pytest.raises(ValueError, match=f'.*are not convertible'):
_ = dt1 + (1*u.m)
# test timedelta plus timedelta/quantity
dt1_dt2 = dt1 + dt2
dt1_float = dt1 + (200+2/MAX_FINE)*u.s
dt2_dt1 = dt2 + dt1
float_dt2 = (100 + 1/MAX_FINE) * u.s + dt2
assert dt1_dt2 == dt2_dt1
assert dt1_float == dt1_dt2
assert float_dt2 == dt1_dt2
assert dt1_dt2.coarse == 300
assert dt1_dt2.fine == 3
def test_timedelta_init():
dt1 = SCETimeDelta(0, 0)
dt2 = SCETimeDelta.from_float(0*u.s)
dt3 = SCETimeDelta(dt1)
assert dt1 == dt2
assert dt2 == dt3
assert dt1 == dt3
with pytest.raises(ValueError):
_ = SCETimeDelta(2 ** 32 + 1, 0)
with pytest.raises(ValueError):
SCETime(0, 2**16+1)
with pytest.raises(ValueError):
SCETime(0.0, 0)
def _get_time(cls, control, num_energies, packets, pad_after):
times = []
durations = []
start = 0
for i, (ns, it) in enumerate(control['num_samples',
'integration_time']):
off_sets = packets.get_value('NIX00485')[start:start + ns] * it
base_time = SCETime(control["scet_coarse"][i],
control["scet_fine"][i])
start_times = base_time + off_sets
end_times = base_time + off_sets + it
cur_time = start_times + (end_times - start_times) / 2
times.extend(cur_time)
durations.extend([it] * ns)
start += ns
time = np.array([(t.coarse, t.fine) for t in times])
time = np.pad(time, ((0, pad_after), (0, 0)), mode='edge')
time = SCETime(time[:, 0], time[:, 1]).reshape(-1, num_energies)
duration = SCETimeDelta(
np.pad(np.hstack(durations),
(0, pad_after)).reshape(-1, num_energies))
scet_timerange = SCETimeRange(start=time[0, 0] - duration[0, 0] / 2,
end=time[-1, -1] + duration[-1, 0] / 2)
return duration, time, scet_timerange
def from_levelb(cls, levelb, parent=''):
packets, idb_versions, control = HKProduct.from_levelb(levelb,
parent=parent)
# Create array of times as dt from date_obs
times = SCETime(control['scet_coarse'], control['scet_fine'])
# Data
data = Data()
data['time'] = times
data['timedel'] = SCETimeDelta(0, 0)
for nix, param in packets.data[0].__dict__.items():
if nix.startswith("NIXG") or nix == 'NIX00020':
continue
name = param.idb_info.get_product_attribute_name()
data.add_basic(name=name, nix=nix, attr='value', packets=packets)
data['control_index'] = range(len(control))
return cls(service_type=packets.service_type,
service_subtype=packets.service_subtype,
ssid=packets.ssid,
control=control,
data=data,
idb_versions=idb_versions)
def test_time_minmax():
assert SCETime.min_time() == SCETime(coarse=0, fine=0)
assert SCETime.max_time() == SCETime(coarse=MAX_COARSE, fine=MAX_FINE)
# TODO enable after https://github.com/i4Ds/STIXCore/issues/102
# assert SCETime.min_time() - SCETime(coarse=0, fine=1) == SCETime.min_time()
with pytest.raises(ValueError, match=r'Coarse time must be in range.*'):
m = SCETime.max_time()
dt = SCETimeDelta(0, 1)
nm = m + dt
print(nm)
def from_levelb(cls, levelb, parent):
packets, idb_versions = GenericProduct.getLeveL0Packets(levelb)
control = Control()
control['scet_coarse'] = packets.get('scet_coarse')
control['scet_fine'] = packets.get('scet_fine')
control['integration_time'] = 0
control['index'] = np.arange(len(control)).astype(
get_min_uint(len(control)))
control['raw_file'] = levelb.control['raw_file']
control['packet'] = levelb.control['packet']
control['parent'] = parent
# Create array of times as dt from date_obs
times = SCETime(control['scet_coarse'], control['scet_fine'])
# Data
data = Data()
data['time'] = times
data['timedel'] = SCETimeDelta(0, 0)
reshape_nixs = {'NIX00103', 'NIX00104'}
reshape = False
if reshape_nixs.intersection(packets.data[0].__dict__.keys()):
reshape = True
for nix, param in packets.data[0].__dict__.items():
name = param.idb_info.get_product_attribute_name()
data.add_basic(name=name,
nix=nix,
attr='value',
packets=packets,
reshape=reshape)
data['control_index'] = np.arange(len(control)).astype(
get_min_uint(len(control)))
return cls(service_type=packets.service_type,
service_subtype=packets.service_subtype,
ssid=packets.ssid,
control=control,
data=data,
idb_versions=idb_versions,
packets=packets)
def from_levelb(cls, levelb, parent=''):
packets, idb_versions = GenericProduct.getLeveL0Packets(levelb)
control = Control()
control['scet_coarse'] = packets.get('scet_coarse')
control['scet_fine'] = packets.get('scet_fine')
control['index'] = np.arange(len(control)).astype(get_min_uint(len(control)))
# When the packets are parsed empty packets are dropped but in LB we don't parse so this
# is not known need to compare control and levelb.control and only use matching rows
if len(levelb.control) > len(control):
matching_index = np.argwhere(
np.in1d(levelb.control['scet_coarse'], np.array(packets.get('scet_coarse'))))
control['raw_file'] = levelb.control['raw_file'][matching_index].reshape(-1)
control['packet'] = levelb.control['packet'][matching_index].reshape(-1)
else:
control['raw_file'] = levelb.control['raw_file'].reshape(-1)
control['packet'] = levelb.control['packet'].reshape(-1)
control['parent'] = parent
tmp = Data()
tmp.add_basic(name='ubsd_counter', nix='NIX00285', packets=packets, dtype=np.uint32)
tmp.add_basic(name='pald_counter', nix='NIX00286', packets=packets, dtype=np.uint32)
tmp.add_basic(name='num_flares', nix='NIX00294', packets=packets, dtype=np.uint16)
colnames = ['start_scet_coarse', 'end_scet_coarse', 'highest_flareflag', 'tm_byte_volume',
'average_z_loc', 'average_y_loc', 'processing_mask']
flares = Data()
if tmp['num_flares'].sum() > 0:
flares.add_basic(name='start_scet_coarse', nix='NIX00287', packets=packets)
flares.add_basic(name='end_scet_coarse', nix='NIX00288', packets=packets)
flares.add_basic(name='highest_flareflag', nix='NIX00289', packets=packets,
dtype=np.byte)
flares.add_basic(name='tm_byte_volume', nix='NIX00290', packets=packets, dtype=np.byte)
flares.add_basic(name='average_z_loc', nix='NIX00291', packets=packets, dtype=np.byte)
flares.add_basic(name='average_y_loc', nix='NIX00292', packets=packets, dtype=np.byte)
flares.add_basic(name='processing_mask', nix='NIX00293', packets=packets, dtype=np.byte)
tmp_data = defaultdict(list)
start = 0
for i, (ubsd, pald, n_flares) in enumerate(tmp):
end = start + n_flares
if n_flares == 0:
tmp_data['control_index'].append(i)
tmp_data['coarse'].append(control['scet_coarse'][i])
tmp_data['fine'].append(control['scet_fine'][i])
tmp_data['ubsd'].append(ubsd)
tmp_data['pald'].append(pald)
for name in colnames:
tmp_data[name].append(0)
else:
tmp_data['control_index'].append([i] * n_flares)
tmp_data['coarse'].append([control['scet_coarse'][i]] * n_flares)
tmp_data['fine'].append([control['scet_fine'][i]] * n_flares)
ubsd['ubsd']([ubsd] * n_flares)
pald['pald'].append([pald] * n_flares)
for name in colnames:
tmp_data[name].extend(flares[name][start:end])
start = end
data = Data(tmp_data)
data['time'] = SCETime(tmp_data['coarse'], tmp_data['fine'])
data['timedel'] = SCETimeDelta(np.full(len(data), 0), np.full(len(data), 0))
data.remove_columns(['coarse', 'fine'])
return cls(service_type=packets.service_type,
service_subtype=packets.service_subtype,
ssid=packets.ssid,
control=control,
data=data,
idb_versions=idb_versions,
packets=packets)
def from_levelb(cls, levelb, parent=''):
packets, idb_versions, control = QLProduct.from_levelb(levelb, parent=parent)
# Control
control.add_basic(name='integration_time', nix='NIX00122', packets=packets,
dtype=np.uint32, attr='value')
control.add_basic(name='quiet_time', nix='NIX00123', packets=packets,
dtype=np.uint16, attr='value')
control.add_basic(name='live_time', nix='NIX00124', packets=packets,
dtype=np.uint32, attr='value')
control.add_basic(name='average_temperature', nix='NIX00125', packets=packets,
dtype=np.uint16, attr='value')
control.add_data('detector_mask', _get_detector_mask(packets))
control.add_data('pixel_mask', _get_pixel_mask(packets))
control.add_data('subspectrum_mask', _get_sub_spectrum_mask(packets))
control.add_data('compression_scheme_counts_skm',
_get_compression_scheme(packets, 'NIX00158'))
subspec_data = {}
j = 129
for subspec, i in enumerate(range(300, 308)):
subspec_data[subspec+1] = {'num_points': packets.get_value(f'NIXD0{j}'),
'num_summed_channel': packets.get_value(f'NIXD0{j + 1}'),
'lowest_channel': packets.get_value(f'NIXD0{j + 2}')}
j += 3
control.add_basic(name='num_samples', nix='NIX00159', packets=packets, dtype=np.uint16)
control['subspec_num_points'] = (
np.vstack([v['num_points'] for v in subspec_data.values()]).T + 1).astype(np.uint16)
control['subspec_num_summed_channel'] = (np.vstack(
[v['num_summed_channel'] for v in subspec_data.values()]).T + 1).astype(np.uint16)
control['subspec_lowest_channel'] = (
np.vstack([v['lowest_channel'] for v in subspec_data.values()]).T).astype(np.uint16)
channels = []
for i, subspectrum_mask in enumerate(control['subspectrum_mask']):
subspec_index = np.argwhere(subspectrum_mask == 1)
sub_channels = [np.arange(control['subspec_num_points'][i, index])
* (control['subspec_num_summed_channel'][i, index])
+ control['subspec_lowest_channel'][i, index] for index in
subspec_index]
channels.append(list(chain(*[ch.tolist() for ch in sub_channels])))
control['num_channels'] = [len(c) for c in channels]
duration = SCETimeDelta(packets.get_value('NIX00122').astype(np.uint32))
time = SCETime(control['scet_coarse'], control['scet_fine']) + duration / 2
dids = packets.get_value('NIXD0155')
pids = packets.get_value('NIXD0156')
ssids = packets.get_value('NIXD0157')
num_spec_points = packets.get_value('NIX00146')
unique_times, unique_time_indices = np.unique(time.as_float(), return_index=True)
unique_times_lookup = {k: v for k, v in zip(unique_times, np.arange(unique_times.size))}
# should really do the other way make a smaller lookup rather than repeating many many times
tids = np.hstack([[unique_times_lookup[t.as_float()]] * n
for t, n in zip(time, control['num_samples'])])
c_in = list(chain.from_iterable([repeat(c, n)
for c, n in zip(channels, control['num_samples'])]))
counts = packets.get_value('NIX00158')
counts_var = packets.get_value('NIX00158', attr='error')
c_out = np.arange(1025)
start = 0
count_map = defaultdict(list)
counts_var_map = defaultdict(list)
for tid, did, pid, ssid, nps, cin in zip(tids, dids, pids, ssids, num_spec_points, c_in):
end = start + nps
logger.debug('%d, %d, %d, %d, %d, %d', tid, did, pid, ssid, nps, end)
count_map[tid, did, pid].append(counts[start:end])
counts_var_map[tid, did, pid].append(counts_var[start:end])
start = end
full_counts = np.zeros((unique_times.size, 32, 12, 1024))
full_counts_var = np.zeros((unique_times.size, 32, 12, 1024))
for tid, did, pid in count_map.keys():
cur_counts = count_map[tid, did, pid]
cur_counts_var = counts_var_map[tid, did, pid]
counts_rebinned = rebin_proportional(np.hstack(cur_counts), cin, c_out)
counts_var_rebinned = rebin_proportional(np.hstack(cur_counts_var), cin, c_out)
full_counts[tid, did, pid] = counts_rebinned
full_counts_var[tid, did, pid] = counts_var_rebinned
control = control[unique_time_indices]
control['index'] = np.arange(len(control))
# Data
data = Data()
data['time'] = time[unique_time_indices]
data['timedel'] = duration[unique_time_indices]
data.add_meta(name='timedel', nix='NIX00122', packets=packets)
data['counts'] = (full_counts*u.ct).astype(get_min_uint(full_counts))
data.add_meta(name='counts', nix='NIX00158', packets=packets)
data['counts_err'] = (np.sqrt(full_counts_var)*u.ct).astype(np.float32)
data['control_index'] = np.arange(len(control)).astype(np.uint16)
return cls(service_type=packets.service_type,
service_subtype=packets.service_subtype,
ssid=packets.ssid,
control=control,
data=data,
idb_versions=idb_versions,
packets=packets)
def test_timedelta_from_float():
dt = SCETimeDelta.from_float(-1 * u.s)
assert dt == SCETimeDelta(coarse=-1, fine=0)
def test_timedelta_as_float():
dt = SCETimeDelta(coarse=-1, fine=0)
assert dt.as_float() == -1.0 * u.s
def __call__(self, *args, **kwargs):
if len(args) == 1 and len(kwargs) == 0:
if isinstance(args[0], (str, Path)):
file_path = Path(args[0])
header = fits.getheader(file_path)
service_type = int(
header.get('stype')) if 'stype' in header else 0
service_subtype = int(
header.get('sstype')) if 'sstype' in header else 0
parent = header.get('parent').split(
';') if 'parent' in header else ''
raw = header.get('raw_file').split(
';') if 'raw_file' in header else ''
try:
ssid = int(header.get('ssid'))
except (ValueError, TypeError):
ssid = None
level = header.get('Level')
header.get('TIMESYS')
hdul = fits.open(file_path)
control = read_qtable(file_path, hdu='CONTROL', hdul=hdul)
# Weird issue where time_stamp wasn't a proper table column?
if 'time_stamp' in control.colnames:
ts = control['time_stamp'].value
control.remove_column('time_stamp')
control['time_stamp'] = ts * u.s
data = read_qtable(file_path, hdu='DATA', hdul=hdul)
if level == 'LL01':
# TODO remova that hack in favor for a proper header information after
# https://github.com/i4Ds/STIXCore/issues/224 is solved
if "lightcurve" in args[0]:
service_type = 21
service_subtype = 6
ssid = 30
if "flareflag" in args[0]:
service_type = 21
service_subtype = 6
ssid = 34
if level not in ['LB', 'LL01']:
data['timedel'] = SCETimeDelta(data['timedel'])
offset = SCETime.from_float(header['OBT_BEG'] * u.s)
try:
control['time_stamp'] = SCETime.from_float(
control['time_stamp'])
except KeyError:
pass
data['time'] = offset + data['time']
energies = None
if level == 'L1':
try:
energies = read_qtable(file_path, hdu='ENERGIES')
except KeyError:
logger.info(
f"no ENERGIES data found in FITS: {file_path}")
idb_versions = defaultdict(SCETimeRange)
if level in ('L0', 'L1'):
try:
idbt = read_qtable(file_path, hdu='IDB_VERSIONS')
for row in idbt.iterrows():
idb_versions[row[0]] = SCETimeRange(
start=SCETime.from_float(row[1]),
end=SCETime.from_float(row[2]))
except KeyError:
logger.warn(f"no IDB data found in FITS: {file_path}")
Product = self._check_registered_widget(
level=level,
service_type=service_type,
service_subtype=service_subtype,
ssid=ssid,
control=control,
data=data,
energies=energies)
return Product(level=level,
service_type=service_type,
service_subtype=service_subtype,
ssid=ssid,
control=control,
data=data,
energies=energies,
idb_versions=idb_versions,
raw=raw,
parent=parent)
def __call__(self, *args, **kwargs):
if len(args) == 1 and len(kwargs) == 0:
if isinstance(args[0], (str, Path)):
file_path = Path(args[0])
header = fits.getheader(file_path)
service_type = int(header.get('stype'))
service_subtype = int(header.get('sstype'))
parent = header.get('parent').split(';')
raw = header.get('raw_file').split(';')
try:
ssid = int(header.get('ssid'))
except ValueError:
ssid = None
level = header.get('Level')
header.get('TIMESYS')
hdul = fits.open(file_path)
control = read_qtable(file_path, hdu='CONTROL', hdul=hdul)
# Weird issue where time_stamp wasn't a proper table column?
if 'time_stamp' in control.colnames:
ts = control['time_stamp'].value
control.remove_column('time_stamp')
control['time_stamp'] = ts * u.s
data = read_qtable(file_path, hdu='DATA', hdul=hdul)
if level != 'LB':
data['timedel'] = SCETimeDelta(data['timedel'])
offset = SCETime.from_float(header['OBT_BEG'] * u.s)
try:
control['time_stamp'] = SCETime.from_float(
control['time_stamp'])
except KeyError:
pass
data['time'] = offset + data['time']
energies = None
if level == 'L1':
try:
energies = read_qtable(file_path, hdu='ENERGIES')
except KeyError:
logger.info(
f"no ENERGIES data found in FITS: {file_path}")
idb_versions = defaultdict(SCETimeRange)
if level in ('L0', 'L1'):
try:
idbt = read_qtable(file_path, hdu='IDB_VERSIONS')
for row in idbt.iterrows():
idb_versions[row[0]] = SCETimeRange(
start=SCETime.from_float(row[1]),
end=SCETime.from_float(row[2]))
except KeyError:
logger.warn(f"no IDB data found in FITS: {file_path}")
Product = self._check_registered_widget(
level=level,
service_type=service_type,
service_subtype=service_subtype,
ssid=ssid,
control=control,
data=data,
energies=energies)
return Product(level=level,
service_type=service_type,
service_subtype=service_subtype,
ssid=ssid,
control=control,
data=data,
energies=energies,
idb_versions=idb_versions,
raw=raw,
parent=parent)