def test_level1_processor_generate_filename(config, product):
config.get.side_effect = [td.soop.DIR, '.']
processor = FitsL1Processor('some/path')
config.get.side_effect = [td.soop.DIR, '.']
product.control.colnames = []
beg = SCETime(coarse=0, fine=0)
end = SCETime(coarse=1, fine=2**15)
avg = beg + (end - beg) / 2
product.obt_beg = beg
product.obt_avg = avg
product.obt_end = end
product.obs_beg = beg.to_datetime()
product.obs_avg = avg.to_datetime()
product.obs_end = end.to_datetime()
product.type = 'ql'
product.scet_timerange = SCETimeRange(start=SCETime(0, 0),
end=SCETime(coarse=0,
fine=2**16 - 1))
product.utc_timerange = product.scet_timerange.to_timerange()
product.level = 'L1'
product.name = 'a_name'
filename = processor.generate_filename(product, version=1)
assert filename == 'solo_L1_stix-ql-a-name_20000101_V01.fits'
product.type = 'sci'
filename = processor.generate_filename(product, version=1)
assert filename == 'solo_L1_stix-sci-a-name_20000101T000000-20000101T000001_V01.fits'
def process_fits_files(self, files=None):
all_files = list()
if files is None:
files = self.level0_files
product_types = defaultdict(list)
for file in files:
# group by service,subservice,ssid example: 'L0/21/6/30'
product_types[str(file.parent)].append(file)
jobs = []
with ProcessPoolExecutor() as executor:
for pt, files in product_types.items():
jobs.append(
executor.submit(
process_type,
files,
processor=FitsL1Processor(self.output_dir),
# keep track of the used Spice kernel
spice_kernel_path=Spice.instance.meta_kernel_path,
config=CONFIG))
for job in jobs:
try:
new_files = job.result()
all_files.extend(new_files)
except Exception:
logger.error('error', exc_info=True)
return list(set(all_files))
def test_level1_processor_generate_primary_header(config, product):
config.get.side_effect = [td.soop.DIR, '.']
processor = FitsL1Processor('some/path')
beg = SCETime(coarse=683769519, fine=0)
end = SCETime(coarse=beg.coarse + 24 * 60 * 60)
beg + (end - beg) / 2
product.scet_timerange = SCETimeRange(start=beg, end=end)
product.utc_timerange = product.scet_timerange.to_timerange()
product.raw = ['packet1.xml', 'packet2.xml']
product.parent = ['l01.fits', 'l02.fts']
product.level = 'L1'
product.type = "ql"
product.service_type = 1
product.service_subtype = 2
product.ssid = 3
test_data = {
'FILENAME': 'a_filename.fits',
'OBT_BEG': beg.as_float().value,
'OBT_END': end.as_float().value,
'DATE_OBS': product.utc_timerange.start.fits,
'DATE_BEG': product.utc_timerange.start.fits,
'DATE_AVG': product.utc_timerange.center.fits,
'DATE_END': product.utc_timerange.end.fits,
'STYPE': product.service_type,
'SSTYPE': product.service_subtype,
'SSID': product.ssid,
'TIMESYS': 'UTC',
'LEVEL': 'L1',
'OBS_ID': 'SSTX_040A_000_000_5Md2_112;SSTX_040A_000_000_vFLg_11Y',
'OBS_TYPE': '5Md2;vFLg',
'OBS_MODE': 'STIX_ANALYSIS;STIX_BASIC',
'SOOPNAME': '',
'SOOPTYPE': '',
'TARGET': '',
'RSUN_ARC': 1589.33,
'HGLT_OBS': -0.32,
'HGLN_OBS': -66.52,
'RAW_FILE': 'packet1.xml;packet2.xml',
'PARENT': 'l01.fits;l02.fts'
}
header = processor.generate_primary_header('a_filename.fits', product)
for name, value, *comment in header:
if name in test_data.keys():
if isinstance(value, float):
assert np.allclose(test_data[name], value)
else:
assert value == test_data[name]
def test_calibration_hk(levelb, idbm, tmp_path):
with test_data.tmtc.TM_3_25_2.open('r') as file:
hex = file.readlines()
levelb.data.__getitem__.return_value = [re.sub(r"\s+", "", h) for h in hex]
levelb.control = {'raw_file': 'raw.xml', 'packet': 0}
hkl0 = MaxiReportL0.from_levelb(levelb)
hkl0.control['parent'] = ['parent.fits']
hkl0.control['raw_file'] = ['raw.xml']
hkl1 = MaxiReportL1.from_level0(hkl0)
fits_procl1 = FitsL1Processor(tmp_path)
fits_procl1.write_fits(hkl1)[0]
assert True
def test_calibration_hk_many(idbm, tmp_path):
idbm.download_version("2.26.35", force=True)
tstart = perf_counter()
prod_lb_p1 = list(LevelB.from_tm(SOCPacketFile(
test_data.io.HK_MAXI_P1)))[0]
prod_lb_p1.control['raw_file'] = ['raw.xml']
hk_p1 = MaxiReportL0.from_levelb(prod_lb_p1)
hk_p1.control['raw_file'] = ['raw.xml']
hk_p1.control['parent'] = ['parent.fits']
fits_procl0 = FitsL0Processor(tmp_path)
filename = fits_procl0.write_fits(hk_p1)[0]
hk_p1_io = Product(filename)
prod_lb_p2 = LevelB.from_tm(SOCPacketFile(test_data.io.HK_MAXI_P2))
hk_p2 = MaxiReportL0.from_levelb(list(prod_lb_p2)[0])
# fake a idb change on the same day
hk_p2.idb_versions["2.26.35"] = hk_p2.idb_versions["2.26.34"]
del hk_p2.idb_versions["2.26.34"]
hkl0 = hk_p1_io + hk_p2
hkl0.control['raw_file'] = ['raw.xml']
hkl0.control['parent'] = ['parent.fits']
hkl1 = MaxiReportL1.from_level0(hkl0, idbm=idbm)
hkl1.control['raw_file'] = ['raw.xml']
hkl1.control['parent'] = ['parent.fits']
fits_procl1 = FitsL1Processor(tmp_path)
filename = fits_procl1.write_fits(hkl1)[0]
tend = perf_counter()
logger.info('Time taken %f', tend - tstart)
def test_level1_processor_init(config):
config.get.side_effect = [td.soop.DIR, '.']
pro = FitsL1Processor('some/path')
assert pro.archive_path == 'some/path'
def __init__(self, source_dir, output_dir):
self.source_dir = Path(source_dir)
self.output_dir = Path(output_dir)
self.level0_files = sorted(list(self.source_dir.rglob('*.fits')))
self.processor = FitsL1Processor(self.output_dir)