def from_fits(cls, fitspath):
header = fits.getheader(fitspath)
control = QTable.read(fitspath, hdu='CONTROL')
data = QTable.read(fitspath, hdu='DATA')
obs_beg = Time(header['DATE_OBS'])
data['time'] = (data['time'] + obs_beg)
return cls(control=control, data=data)
def read_qtable(file, hdu, hdul=None):
"""
Read a fits file into a QTable and maintain dtypes of columns with units
Hack to work around QTable not respecting the dtype in fits file see
https://github.com/astropy/astropy/issues/12494
Parameters
----------
file : `str` or `pathlib.Path`
Fits file
hdu : `str`
The name of the extension
hdul : optional `astropy.io.fits.HDUList`
The HDU list for the fits file
Returns
-------
`astropy.table.QTable`
The corrected QTable with correct data types
"""
qtable = QTable.read(file, hdu)
if hdul is None:
hdul = fits.open(file)
for col in hdul[hdu].data.columns:
if col.unit:
logger.debug(f'Unit present dtype correction needed for {col}')
dtype = col.dtype
if col.bzero:
logger.debug(
f'Unit present dtype and bzero correction needed for {col}'
)
bits = np.log2(col.bzero)
if bits.is_integer():
dtype = BITS_TO_UINT[int(bits + 1)]
if hasattr(dtype, 'subdtype'):
dtype = dtype.base
qtable[col.name] = qtable[col.name].astype(dtype)
return qtable
def write_fits(self, product):
self.meta = []
if callable(getattr(product, 'to_days', None)):
#print("# to_days")
products = product.to_days()
else:
#print("# to_request")
products = product.to_requests()
for prod in products:
#create an fits for each request
filename = self.generate_filename(product=prod)
start_date = prod.obs_beg.to_datetime()
if prod.type == 'ql':
start_date = prod.obs_avg.to_datetime()
#path = self.archive_path.joinpath(*[prod.level, format(start_date.year, '04d'),
# format(start_date.month, '02d'),
# format(start_date.day, '02d'),
# prod.type.upper()])
path = self.archive_path
#path.mkdir(parents=True, exist_ok=True)
fitspath = path / filename
logger.info(f'Fits filename:{fitspath.as_posix()}')
#if fitspath.exists():
# logger.info(f'Fits file %s exists appending data {fitspath.name}')
# existing = prod.from_fits(fitspath)
# # logger.debug('Existing %s \n New %s', existing, prod)
# prod = prod + existing
# # logger.debug('Combined %s', prod)
control = prod.control
data = prod.data
elow, ehigh = prod.get_energies()
energies = QTable()
energies['channel'] = range(len(elow))
energies['e_low'] = elow * u.keV
energies['e_high'] = ehigh * u.keV
# Convert time to be relative to start date
data['time'] = (data['time'] - prod.obs_beg).to(u.s)
primary_header = self.generate_primary_header(filename, prod)
primary_hdu = fits.PrimaryHDU()
primary_hdu.header.update(primary_header)
primary_hdu.header.update({'HISTORY': 'Processed by STIX'})
control_hdu = fits.BinTableHDU(control)
control_hdu.name = 'CONTROL'
data_hdu = fits.BinTableHDU(data)
data_hdu.name = 'DATA'
energy_hdu = fits.BinTableHDU(energies)
energy_hdu.name = 'ENERGIES'
hdul = fits.HDUList(
[primary_hdu, control_hdu, data_hdu, energy_hdu])
#logger.debug(f'Writing fits file to {path / filename}')
full_path = path / filename
if full_path.is_file():
print("Removing existing fits:", str(full_path))
full_path.unlink()
hdul.writeto(full_path, overwrite=True, checksum=True)
_meta = {
'data_start_unix': prod.obs_beg.to_datetime().timestamp(),
'data_end_unix': prod.obs_end.to_datetime().timestamp(),
'_id': self.fits_db_id,
'filename': filename
}
if 'request_id' in control.colnames:
_meta['request_id'] = int(control['request_id'][0])
self.fits_db_id += 1
self.meta.append(_meta)
def from_fits(cls, fitspath):
control = QTable.read(fitspath, hdu='CONTROL')
data = QTable.read(fitspath, hdu='DATA')
return cls(control=control, data=data)
def raw_to_engineering_product(product, idbm):
"""Apply parameter raw to engineering conversion for the entire product.
Parameters
----------
product : `BaseProduct`
The TM product as level 0
Returns
-------
`int`
How many columns where calibrated.
"""
col_n = 0
idb_ranges = QTable(rows=[
(version, range.start.as_float(), range.end.as_float())
for version, range in product.idb_versions.items()
],
names=["version", "obt_start", "obt_end"])
idb_ranges.sort("obt_start")
idb_ranges['obt_start'][0] = SCETime.min_time().as_float()
for i in range(0, len(idb_ranges) - 1):
idb_ranges['obt_end'][i] = idb_ranges['obt_start'][i + 1]
idb_ranges['obt_end'][-1] = SCETime.max_time().as_float()
for table, timecol in [(product.data, 'time'),
(product.control, 'scet_coarse')]:
if timecol == 'scet_coarse':
if 'scet_coarse' in table.colnames:
timevector = SCETime(coarse=table['scet_coarse'],
fine=table['scet_fine']).as_float()
else:
# product per request (xray: no 'scet_coarse' in control)
# do not have engineering values in control
continue
else: # time
timevector = table['time'].as_float()
for col in table.colnames:
if not (hasattr(table[col], "meta")
# and table[col].meta.get("PCF_CURTX", None) is not None
and not isinstance(table[col].meta.get("PCF_CURTX", None),
(type(None), list)) and
table[col].meta["NIXS"] is not None):
continue
col_n += 1
c = 0
# clone the current column into a new column as the content might be replaced chunk wise
table[CCN] = table[col]
for idbversion, starttime, endtime in idb_ranges.iterrows():
idb = idbm.get_idb(idbversion)
idb_time_period = np.where((starttime <= timevector)
& (timevector < endtime))[0]
if len(idb_time_period) < 1:
continue
c += len(idb_time_period)
calib_param = idb.get_params_for_calibration(
product.service_type, product.service_subtype,
(product.ssid if hasattr(product, "ssid") else None),
table[col].meta["NIXS"], table[col].meta["PCF_CURTX"])[0]
raw = Parameter(table[col].meta["NIXS"],
table[idb_time_period][col], None)
eng = apply_raw_to_engineering(raw, (calib_param, idb))
# cast the type of the column if needed
if table[CCN].dtype != eng.engineering.dtype:
table[CCN] = table[CCN].astype(eng.engineering.dtype)
# set the unit if needed
if hasattr(eng.engineering,
"unit") and table[CCN].unit != eng.engineering.unit:
meta = table[col].meta
table[CCN].unit = eng.engineering.unit
# restore the meta info
setattr(table[CCN], "meta", meta)
# override the data into the new column
table[CCN][idb_time_period] = eng.engineering
# replace the old column with the converted
table[col] = table[CCN]
table[col].meta = table[CCN].meta
# delete the generic column for conversion
del table[CCN]
# delete the calibration key from meta as it is now processed
del table[col].meta["PCF_CURTX"]
if c != len(table):
logger.warning(
"Not all time bins got converted to engineering" +
"values due to bad idb periods." +
f"\n Converted bins: {c}\ntotal bins {len(table)}")
return col_n