def test_level1_processor_generate_filename(config, product):
config.get.side_effect = [td.soop.DIR, '.']
processor = FitsL1Processor('some/path')
config.get.side_effect = [td.soop.DIR, '.']
product.control.colnames = []
beg = SCETime(coarse=0, fine=0)
end = SCETime(coarse=1, fine=2**15)
avg = beg + (end - beg) / 2
product.obt_beg = beg
product.obt_avg = avg
product.obt_end = end
product.obs_beg = beg.to_datetime()
product.obs_avg = avg.to_datetime()
product.obs_end = end.to_datetime()
product.type = 'ql'
product.scet_timerange = SCETimeRange(start=SCETime(0, 0),
end=SCETime(coarse=0,
fine=2**16 - 1))
product.utc_timerange = product.scet_timerange.to_timerange()
product.level = 'L1'
product.name = 'a_name'
filename = processor.generate_filename(product, version=1)
assert filename == 'solo_L1_stix-ql-a-name_20000101_V01.fits'
product.type = 'sci'
filename = processor.generate_filename(product, version=1)
assert filename == 'solo_L1_stix-sci-a-name_20000101T000000-20000101T000001_V01.fits'
def _get_time(self):
# Replicate the start time of each for the number of samples in that packet
base_coarse, base_fine = zip(
*[([ct] * ns, [ft] * ns) for ns, ct, ft in self[
['num_samples', 'scet_coarse', 'scet_fine']]])
base_coarse = np.hstack(base_coarse)
base_fine = np.hstack(base_fine)
bases = SCETime(base_coarse, base_fine)
# Create start time for each time bin by multiplying the duration by the sample number
deltas = SCETimeDelta(
np.hstack([(np.arange(ns) * it)
for ns, it in self[['num_samples', 'integration_time']]
]))
# Create integration time for each sample in each packet by replicating the duration for
# number of sample in each packet
durations = SCETimeDelta(
np.hstack([
np.ones(num_sample) * int_time for num_sample, int_time in
self[['num_samples', 'integration_time']]
]))
# Add the delta time to base times and convert to bin centers
times = bases + deltas + (durations / 2)
# Create a time range object covering the total observation time
tr = SCETimeRange(start=bases[0],
end=bases[-1] + deltas[-1] + durations[-1])
return times, durations, tr
def _get_time(cls, control, num_energies, packets, pad_after):
times = []
durations = []
start = 0
for i, (ns, it) in enumerate(control['num_samples',
'integration_time']):
off_sets = packets.get_value('NIX00485')[start:start + ns] * it
base_time = SCETime(control["scet_coarse"][i],
control["scet_fine"][i])
start_times = base_time + off_sets
end_times = base_time + off_sets + it
cur_time = start_times + (end_times - start_times) / 2
times.extend(cur_time)
durations.extend([it] * ns)
start += ns
time = np.array([(t.coarse, t.fine) for t in times])
time = np.pad(time, ((0, pad_after), (0, 0)), mode='edge')
time = SCETime(time[:, 0], time[:, 1]).reshape(-1, num_energies)
duration = SCETimeDelta(
np.pad(np.hstack(durations),
(0, pad_after)).reshape(-1, num_energies))
scet_timerange = SCETimeRange(start=time[0, 0] - duration[0, 0] / 2,
end=time[-1, -1] + duration[-1, 0] / 2)
return duration, time, scet_timerange
def test_level0_processor_generate_filename():
with patch('stixcore.products.level0.quicklookL0.QLProduct') as product:
processor = FitsL0Processor('some/path')
product.control.colnames = []
product.type = 'ql'
product.scet_timerange = SCETimeRange(start=SCETime(0, 0),
end=SCETime(1234, 1234))
product.level = 'LB'
product.name = 'a_name'
filename = processor.generate_filename(product, version=1)
assert filename == 'solo_LB_stix-ql-a-name_0000000000_V01.fits'
with patch('stixcore.products.level0.scienceL0.ScienceProduct') as product:
product.type = 'sci'
product.control.colnames = []
product.obs_avg.coarse = 0
product.level = 'L0'
product.name = 'a_name'
product.scet_timerange = SCETimeRange(start=SCETime(12345, 6789),
end=SCETime(98765, 4321))
filename = processor.generate_filename(product, version=1)
assert filename == 'solo_L0_stix-sci-a-name_0000012345-0000098765_V01.fits'
dummy_control_data = {
'request_id': [123456],
'tc_packet_seq_control': [98765]
}
product.control.__getitem__.side_effect = dummy_control_data.__getitem__
product.control.colnames = ['request_id']
filename = processor.generate_filename(product, version=1)
assert filename == 'solo_L0_stix-sci-a-name' \
'_0000012345-0000098765_V01_123456.fits'
product.control.colnames = ['request_id', 'tc_packet_seq_control']
filename = processor.generate_filename(product, version=1)
assert filename == 'solo_L0_stix-sci-a-name' \
'_0000012345-0000098765_V01_123456-98765.fits'
def test_level1_processor_generate_primary_header(config, product):
config.get.side_effect = [td.soop.DIR, '.']
processor = FitsL1Processor('some/path')
beg = SCETime(coarse=683769519, fine=0)
end = SCETime(coarse=beg.coarse + 24 * 60 * 60)
beg + (end - beg) / 2
product.scet_timerange = SCETimeRange(start=beg, end=end)
product.utc_timerange = product.scet_timerange.to_timerange()
product.raw = ['packet1.xml', 'packet2.xml']
product.parent = ['l01.fits', 'l02.fts']
product.level = 'L1'
product.type = "ql"
product.service_type = 1
product.service_subtype = 2
product.ssid = 3
test_data = {
'FILENAME': 'a_filename.fits',
'OBT_BEG': beg.as_float().value,
'OBT_END': end.as_float().value,
'DATE_OBS': product.utc_timerange.start.fits,
'DATE_BEG': product.utc_timerange.start.fits,
'DATE_AVG': product.utc_timerange.center.fits,
'DATE_END': product.utc_timerange.end.fits,
'STYPE': product.service_type,
'SSTYPE': product.service_subtype,
'SSID': product.ssid,
'TIMESYS': 'UTC',
'LEVEL': 'L1',
'OBS_ID': 'SSTX_040A_000_000_5Md2_112;SSTX_040A_000_000_vFLg_11Y',
'OBS_TYPE': '5Md2;vFLg',
'OBS_MODE': 'STIX_ANALYSIS;STIX_BASIC',
'SOOPNAME': '',
'SOOPTYPE': '',
'TARGET': '',
'RSUN_ARC': 1589.33,
'HGLT_OBS': -0.32,
'HGLN_OBS': -66.52,
'RAW_FILE': 'packet1.xml;packet2.xml',
'PARENT': 'l01.fits;l02.fts'
}
header = processor.generate_primary_header('a_filename.fits', product)
for name, value, *comment in header:
if name in test_data.keys():
if isinstance(value, float):
assert np.allclose(test_data[name], value)
else:
assert value == test_data[name]
def test_timerange():
tr = SCETimeRange(start=SCETime(coarse=100, fine=0), end=SCETime(coarse=200, fine=0))
tp_in = SCETime(coarse=150, fine=0)
tr_in = SCETimeRange(start=SCETime(coarse=150, fine=0), end=SCETime(coarse=160, fine=0))
tr_out = SCETimeRange(start=SCETime(coarse=150, fine=0), end=SCETime(coarse=250, fine=0))
tp_out = SCETime(coarse=250, fine=0)
assert tp_in in tr
assert tp_out not in tr
assert tr_in in tr
assert tr_out not in tr
tr.expand(tp_out)
tr.expand(tr_out)
assert tp_out in tr
assert tr_out in tr
def test_level0_processor_generate_primary_header(datetime, product):
processor = FitsL0Processor('some/path')
datetime.now().isoformat.return_value = '1234-05-07T01:02:03.346'
product.obs_beg = SCETime(coarse=0, fine=0)
product.obs_avg = SCETime(coarse=0, fine=2**15)
product.obs_end = SCETime(coarse=1, fine=2**15)
product.scet_timerange = SCETimeRange(start=product.obs_beg,
end=product.obs_end)
product.raw = ['packet1.xml', 'packet2.xml']
product.parent = ['lb1.fits', 'lb2.fts']
product.service_type = 1
product.service_subtype = 2
product.ssid = 3
product.level = 'L0'
test_data = {
'FILENAME': 'a_filename.fits',
'DATE': '1234-05-07T01:02:03.346',
'OBT_BEG': 0.0,
'OBT_END': 1.5000076295109483,
'DATE_OBS': '0000000000:00000',
'DATE_BEG': '0000000000:00000',
'DATE_AVG': '0000000000:49152',
'DATE_END': '0000000001:32768',
'STYPE': 1,
'SSTYPE': 2,
'SSID': 3,
'TIMESYS': "OBT",
'LEVEL': 'L0',
'RAW_FILE': 'packet1.xml;packet2.xml',
'PARENT': 'lb1.fits;lb2.fts'
}
header = processor.generate_primary_header('a_filename.fits', product)
for name, value, *comment in header:
if name in test_data.keys():
assert value == test_data[name]
def scet_timerange(self):
return SCETimeRange(
start=self.data['time'][0] - self.data['timedel'][0] / 2,
end=self.data['time'][-1] + self.data['timedel'][-1] / 2)
def __call__(self, *args, **kwargs):
if len(args) == 1 and len(kwargs) == 0:
if isinstance(args[0], (str, Path)):
file_path = Path(args[0])
header = fits.getheader(file_path)
service_type = int(
header.get('stype')) if 'stype' in header else 0
service_subtype = int(
header.get('sstype')) if 'sstype' in header else 0
parent = header.get('parent').split(
';') if 'parent' in header else ''
raw = header.get('raw_file').split(
';') if 'raw_file' in header else ''
try:
ssid = int(header.get('ssid'))
except (ValueError, TypeError):
ssid = None
level = header.get('Level')
header.get('TIMESYS')
hdul = fits.open(file_path)
control = read_qtable(file_path, hdu='CONTROL', hdul=hdul)
# Weird issue where time_stamp wasn't a proper table column?
if 'time_stamp' in control.colnames:
ts = control['time_stamp'].value
control.remove_column('time_stamp')
control['time_stamp'] = ts * u.s
data = read_qtable(file_path, hdu='DATA', hdul=hdul)
if level == 'LL01':
# TODO remova that hack in favor for a proper header information after
# https://github.com/i4Ds/STIXCore/issues/224 is solved
if "lightcurve" in args[0]:
service_type = 21
service_subtype = 6
ssid = 30
if "flareflag" in args[0]:
service_type = 21
service_subtype = 6
ssid = 34
if level not in ['LB', 'LL01']:
data['timedel'] = SCETimeDelta(data['timedel'])
offset = SCETime.from_float(header['OBT_BEG'] * u.s)
try:
control['time_stamp'] = SCETime.from_float(
control['time_stamp'])
except KeyError:
pass
data['time'] = offset + data['time']
energies = None
if level == 'L1':
try:
energies = read_qtable(file_path, hdu='ENERGIES')
except KeyError:
logger.info(
f"no ENERGIES data found in FITS: {file_path}")
idb_versions = defaultdict(SCETimeRange)
if level in ('L0', 'L1'):
try:
idbt = read_qtable(file_path, hdu='IDB_VERSIONS')
for row in idbt.iterrows():
idb_versions[row[0]] = SCETimeRange(
start=SCETime.from_float(row[1]),
end=SCETime.from_float(row[2]))
except KeyError:
logger.warn(f"no IDB data found in FITS: {file_path}")
Product = self._check_registered_widget(
level=level,
service_type=service_type,
service_subtype=service_subtype,
ssid=ssid,
control=control,
data=data,
energies=energies)
return Product(level=level,
service_type=service_type,
service_subtype=service_subtype,
ssid=ssid,
control=control,
data=data,
energies=energies,
idb_versions=idb_versions,
raw=raw,
parent=parent)
def __call__(self, *args, **kwargs):
if len(args) == 1 and len(kwargs) == 0:
if isinstance(args[0], (str, Path)):
file_path = Path(args[0])
header = fits.getheader(file_path)
service_type = int(header.get('stype'))
service_subtype = int(header.get('sstype'))
parent = header.get('parent').split(';')
raw = header.get('raw_file').split(';')
try:
ssid = int(header.get('ssid'))
except ValueError:
ssid = None
level = header.get('Level')
header.get('TIMESYS')
hdul = fits.open(file_path)
control = read_qtable(file_path, hdu='CONTROL', hdul=hdul)
# Weird issue where time_stamp wasn't a proper table column?
if 'time_stamp' in control.colnames:
ts = control['time_stamp'].value
control.remove_column('time_stamp')
control['time_stamp'] = ts * u.s
data = read_qtable(file_path, hdu='DATA', hdul=hdul)
if level != 'LB':
data['timedel'] = SCETimeDelta(data['timedel'])
offset = SCETime.from_float(header['OBT_BEG'] * u.s)
try:
control['time_stamp'] = SCETime.from_float(
control['time_stamp'])
except KeyError:
pass
data['time'] = offset + data['time']
energies = None
if level == 'L1':
try:
energies = read_qtable(file_path, hdu='ENERGIES')
except KeyError:
logger.info(
f"no ENERGIES data found in FITS: {file_path}")
idb_versions = defaultdict(SCETimeRange)
if level in ('L0', 'L1'):
try:
idbt = read_qtable(file_path, hdu='IDB_VERSIONS')
for row in idbt.iterrows():
idb_versions[row[0]] = SCETimeRange(
start=SCETime.from_float(row[1]),
end=SCETime.from_float(row[2]))
except KeyError:
logger.warn(f"no IDB data found in FITS: {file_path}")
Product = self._check_registered_widget(
level=level,
service_type=service_type,
service_subtype=service_subtype,
ssid=ssid,
control=control,
data=data,
energies=energies)
return Product(level=level,
service_type=service_type,
service_subtype=service_subtype,
ssid=ssid,
control=control,
data=data,
energies=energies,
idb_versions=idb_versions,
raw=raw,
parent=parent)
