def generate_transmission_tables():
from datetime import datetime
cur_date = datetime.now().strftime('%Y%m%d')
datetime.now().strftime('%Y%m%d')
trans = Transmission()
energies = np.linspace(2, 150, 1001) * u.keV
norm_sci_energies = trans.get_transmission()
norm_sci_energies.write(f'stix_transmission_sci_energies_{cur_date}.csv')
norm_high_res = trans.get_transmission(energies=energies)
norm_high_res.write(f'stix_transmission_highres_{cur_date}.csv')
comps = trans.get_transmission_by_component()
comps_sci_energies = Table(
[c.transmission(trans.energies) for c in comps.values()],
names=[k for k in comps.keys()])
comps_sci_energies['energy'] = trans.energies
comps_sci_energies.write(
f'stix_transmission_by_component_sci_energies_{cur_date}.csv')
comps_highres = Table([c.transmission(energies) for c in comps.values()],
names=[k for k in comps.keys()])
comps_highres['energy'] = energies
comps_highres.write(
f'stix_transmission_by_component_highres_{cur_date}.csv')
def from_tm(cls, tmfile):
"""Process the given SOCFile and creates LevelB FITS files.
Parameters
----------
tmfile : `SOCPacketFile`
The input data file.
"""
packet_data = defaultdict(list)
for packet_no, binary in tmfile.get_packet_binaries():
try:
packet = TMPacket(binary)
except Exception:
logger.error('Error parsing %s, %d', tmfile.name, packet_no, exc_info=True)
return
packet.source = (tmfile.file.name, packet_no)
packet_data[packet.key].append(packet)
for prod_key, packets in packet_data.items():
headers = []
hex_data = []
for packet in packets:
sh = vars(packet.source_packet_header)
bs = sh.pop('bitstream')
hex_data.append(bs.hex)
dh = vars(packet.data_header)
dh.pop('datetime')
headers.append({**sh, **dh, 'raw_file': packet.source[0],
'packet': packet.source[1]})
if len(headers) == 0 or len(hex_data) == 0:
return None
control = Table(headers)
control['index'] = np.arange(len(control), dtype=np.int64)
data = Table()
data['control_index'] = np.array(control['index'], dtype=np.int64)
data['data'] = hex_data
control = unique(control, keys=['scet_coarse', 'scet_fine', 'sequence_count'])
# Only keep data that is in the control table via index
data = data[np.nonzero(control['index'][:, None] == data['control_index'])[1]]
# now reindex both data and control
control['index'] = range(len(control))
data['control_index'] = control['index']
service_type, service_subtype, ssid = prod_key
if ssid is not None:
control['ssid'] = ssid
product = LevelB(service_type=service_type, service_subtype=service_subtype,
ssid=ssid, control=control, data=data)
yield product
def array():
# composite index
col0 = np.array([x % 2 for x in range(1, 11)])
col1 = np.array([x for x in range(1, 11)])
t = Table([col0, col1])
t = t[t.argsort()]
return SortedArray(t, t['col1'].copy())
def createMultipleGaussian(self,
stddevXRange=[2., 3],
stddevYRange=None,
fluxInPhotons=[1000., 10000],
nStars=100,
withSeed=False):
if withSeed is True:
np.random.seed(seed=12345)
xMean = np.random.uniform(1, self._shape[1] - 1, nStars)
yMean = np.random.uniform(1, self._shape[0] - 1, nStars)
sx = np.random.uniform(stddevXRange[0], stddevXRange[1], nStars)
if stddevYRange is None:
sy = sx
else:
sy = np.random.uniform(stddevYRange[0], stddevYRange[1], nStars)
theta = np.arctan2(yMean - 0.5 * self._shape[0],
xMean - 0.5 * self._shape[1]) - np.pi / 2
flux = np.random.uniform(fluxInPhotons[0], fluxInPhotons[1], nStars)
self._table = Table()
self._table['x_mean'] = xMean
self._table['y_mean'] = yMean
self._table['x_stddev'] = sx
self._table['y_stddev'] = sy
self._table['theta'] = theta
self._table['flux'] = flux
ima = self.createImage()
return ima
def createMultipleGaussianIntegerCentroids(self,
stddevXRange=[2., 3],
stddevYRange=None,
fluxInPhotons=[1000., 10000],
nStars=100):
xMean = np.random.randint(1, self._shape[1] - 1, nStars)
yMean = np.random.randint(1, self._shape[0] - 1, nStars)
sx = np.random.uniform(stddevXRange[0], stddevXRange[1], nStars)
if stddevYRange is None:
sy = sx
else:
sy = np.random.uniform(stddevYRange[0], stddevYRange[1], nStars)
theta = np.arctan2(yMean - 0.5 * self._shape[0],
xMean - 0.5 * self._shape[1]) - np.pi / 2
flux = np.random.uniform(fluxInPhotons[0], fluxInPhotons[1], nStars)
self._table = Table()
self._table['x_mean'] = xMean
self._table['y_mean'] = yMean
self._table['x_stddev'] = sx
self._table['y_stddev'] = sy
self._table['theta'] = theta
self._table['flux'] = flux
ima = self.createImage()
return ima
def test_write_jsviewer_options(tmpdir):
t = Table()
t['a'] = [1, 2, 3, 4, 5]
t['b'] = ['<b>a</b>', 'b', 'c', 'd', 'e']
t['a'].unit = 'm'
tmpfile = tmpdir.join('test.html').strpath
t.write(tmpfile,
format='jsviewer',
table_id='test',
max_lines=3,
jskwargs={'display_length': 5},
table_class='display hover',
htmldict=dict(raw_html_cols='b'))
ref = REFERENCE % dict(
lines=format_lines(t['a'][:3], t['b'][:3]),
table_class='display hover',
table_id='test',
length='5',
display_length='5, 10, 25, 50, 100, 500, 1000',
datatables_css_url=
'https://cdn.datatables.net/1.10.12/css/jquery.dataTables.css',
datatables_js_url=
'https://cdn.datatables.net/1.10.12/js/jquery.dataTables.min.js',
jquery_url='https://code.jquery.com/jquery-3.1.1.min.js')
with open(tmpfile) as f:
assert f.read().strip() == ref.strip()
def join(self, datatable):
""" add data to the current catalogue.
This is based on astropy.Table join:
"
The join() method allows one to merge these two tables into a single table
based on matching values in the “key columns”.
"
We use the join_type='outer'
(http://docs.astropy.org/en/stable/table/operations.html)
"""
# ---------------------
# - Input Test
if type(datatable) is not Table:
try:
datatable = Table(datatable)
except:
raise TypeError(
"the given datatable is not an astropy. Table and cannot be converted into."
)
from astropy.table import join
self._properties["data"] = join(self.data,
datatable,
join_type='outer')
self._update_fovmask_()
def test_write_jsviewer_local(tmpdir):
t = Table()
t['a'] = [1, 2, 3, 4, 5]
t['b'] = ['a', 'b', 'c', 'd', 'e']
t['a'].unit = 'm'
tmpfile = tmpdir.join('test.html').strpath
t.write(tmpfile,
format='jsviewer',
table_id='test',
jskwargs={'use_local_files': True})
ref = REFERENCE % dict(
lines=format_lines(t['a'], t['b']),
table_class='display compact',
table_id='test',
length='50',
display_length='10, 25, 50, 100, 500, 1000',
datatables_css_url='file://' +
join(EXTERN_DIR, 'css', 'jquery.dataTables.css'),
datatables_js_url='file://' +
join(EXTERN_DIR, 'js', 'jquery.dataTables.min.js'),
jquery_url='file://' + join(EXTERN_DIR, 'js', 'jquery-3.1.1.min.js'))
with open(tmpfile) as f:
assert f.read().strip() == ref.strip()
def get_transmission(self, energies=None, attenuator=False):
base_comps = [self.components[name] for name in ['front_window', 'rear_window', 'dem',
'mli', 'calibration_foil',
'dead_layer']]
if energies is None:
energies = self.energies
if attenuator:
base_comps.append(self.components['attenuator'])
base = Compound(base_comps)
base_trans = base.transmission(energies[:-1] + 0.5 * np.diff(energies))
fine = Compound(base_comps + [self.components['grid_covers']])
fine_trans = fine.transmission(energies[:-1] + 0.5 * np.diff(energies))
fine_grids = np.array([11, 13, 18, 12, 19, 17]) - 1
transmission = Table()
# transmission['sci_channel'] = range(1, 31)
for i in range(33):
name = f'det-{i}'
if np.isin(i, fine_grids):
transmission[name] = fine_trans
else:
transmission[name] = base_trans
return transmission
def write_absid_file(self, outfil=None):
from astropy.table import Column
from astropy.table.table import Table
wrest = self.lines.keys()
wrest.sort()
if outfil is None:
outfil = self.absid_file
# Columns
cols = [Column(np.array(wrest), name='WREST')]
clm_nms = self.lines[wrest[0]].analy.keys()
for clm_nm in clm_nms:
clist = [self.lines[iwrest].analy[clm_nm] for iwrest in wrest]
cols.append( Column(np.array(clist), name=clm_nm) )
cols.append( Column(np.ones(len(cols[0]))*self.zabs, name='ZABS') )
table = Table(cols)
prihdr = fits.Header()
prihdr['COMMENT'] = "Above are the data sources"
prihdu = fits.PrimaryHDU(header=prihdr)
table_hdu = fits.BinTableHDU.from_columns(np.array(table.filled()))
thdulist = fits.HDUList([prihdu, table_hdu])
thdulist.writeto(outfil,clobber=True)
print('Wrote AbsID file: {:s}'.format(outfil))
def _extractStars(self):
self._findStars()
self._starsTab = Table()
self._starsTab['x'] = self._selectedStars['xcentroid']
self._starsTab['y'] = self._selectedStars['ycentroid']
self._starsCut = extract_stars(NDData(data=self._image),
self._starsTab,
self._size)
def test_get_observation_type_invalid_obs_id_valueerror(self, mock_query):
with pytest.raises(ValueError):
arr = {'a': np.array([], dtype=np.int32), 'b': [], 'obs_type': []}
data_table = Table(arr)
ehst = ESAHubbleClass(self.get_dummy_tap_handler())
mock_query.return_value = data_table
dummy_obs_id = '1234'
ehst.get_observation_type(dummy_obs_id)
def test_table_index_time_warning(engine):
# Make sure that no ERFA warnings are emitted when indexing a table by
# a Time column with a non-default time scale
tab = Table()
tab['a'] = Time([1, 2, 3], format='jyear', scale='tai')
tab['b'] = [4, 3, 2]
with warnings.catch_warnings(record=True) as wlist:
tab.add_index(('a', 'b'), engine=engine)
assert len(wlist) == 0
def createMoffatImage(self, posX, posY, gamma, alpha, peak):
table = Table()
table['x_0'] = [posX]
table['y_0'] = [posY]
table['gamma'] = [gamma]
table['alpha'] = [alpha]
table['amplitude'] = [peak]
self._table = table
return self.createImage()
def pickleCatalogue(catalogue, filename, **kwargs):
"""Write a catalogue to an ascii file."""
verbose = kwargs.pop('verbose', False)
if kwargs:
raise TypeError('Unexpected **kwargs: %r' % kwargs)
outfile = open(filename, 'wb')
pickle.dump(Table(catalogue, masked=False), outfile, pickle.HIGHEST_PROTOCOL)
outfile.close()
print("Catalogue pickled." if verbose else "")
return
def createIntegratedGaussianPRFImage(self, sigma, flux, x0, y0):
table = Table()
table['sigma'] = [sigma]
table['flux'] = [flux]
table['x_0'] = [x0]
table['y_0'] = [y0]
self._table = table
ima = make_model_sources_image(self._shape, IntegratedGaussianPRF(),
table)
return ima
def alignCoordsOnMeanNGS(self):
disp = self._getDisplacementsFromMeanNGS()
dx = disp[:, 0]
dy = disp[:, 1]
self.starsTabsNew = []
for i in range(len(self._starsTabs)):
tab = Table()
tab['x_fit'] = self._starsTabs[i]['x_fit'] - dx[i]
tab['y_fit'] = self._starsTabs[i]['y_fit'] - dy[i]
tab['flux_fit'] = self._starsTabs[i]['flux_fit']
self.starsTabsNew.append(tab)
def process_tmtc_file(tmfile, basedir):
fits_processor = FitsL0Processor(basedir)
tree = Et.parse(tmfile)
root = tree.getroot()
packet_data = defaultdict(list)
for i, node in enumerate(root.iter('Packet')):
packet_binary = unhexlify(node.text)
# Not sure why guess and extra moc header
header, packet_hex = process_tm_packet(packet_binary[76:])
# if header.get('ssid', -1) == 21:
key = f"{header['service_type']}-{header['service_subtype']}"
packet_data[key].append((header, packet_hex))
for product, packet in packet_data.items():
#header and hex for the same type
# packet_data = sorted(packet_data,
# key=lambda x: (x[0]['scet_coarse'] + x[0]['scet_fine'] / 2 ** 16, x[0]['seq_count']))
#difference services
header, hex_data = zip(*packet)
control = Table(header)
control['index'] = np.arange(len(control))
data = Table()
data['control_index'] = control['index']
# data['data'] = binary_data
data['data'] = hex_data
if 'ssid' in control.colnames:
ssids = np.unique(control['ssid'])
for ssid in ssids:
index = np.nonzero(control['ssid'] == ssid)
if len(index[0]) > 0:
cur_control = control[index]
cur_data = data[index]
cur_control['index'] = np.arange(len(cur_control))
cur_data['control_index'] = np.arange(len(cur_control))
prod = SciLevel0(control=cur_control, data=cur_data)
fits_processor.write_fits(prod)
else:
prod = level0(control=control, data=data)
fits_processor.write_fits(prod)
def test__select_related_members(self, mock_query):
arr = {
'a': np.array([1, 4], dtype=np.int32),
'b': [2.0, 5.0],
'members': ['caom:HST/test', 'y']
}
data_table = Table(arr)
ehst = ESAHubbleClass(self.get_dummy_tap_handler())
mock_query.return_value = data_table
dummy_obs_id = "1234"
oids = ehst._select_related_members(dummy_obs_id)
assert oids == ['test']
def test__select_related_composite(self, mock_query):
arr = {
'a': np.array([1, 4], dtype=np.int32),
'b': [2.0, 5.0],
'observation_id': ['x', 'y']
}
data_table = Table(arr)
ehst = ESAHubbleClass(self.get_dummy_tap_handler())
mock_query.return_value = data_table
dummy_obs_id = "1234"
oids = ehst._select_related_composite(dummy_obs_id)
assert oids == ['x', 'y']
def test_get_observation_type(self, mock_query):
arr = {
'a': np.array([1, 4], dtype=np.int32),
'b': [2.0, 5.0],
'obs_type': ['HST Test', 'y']
}
data_table = Table(arr)
ehst = ESAHubbleClass(self.get_dummy_tap_handler())
mock_query.return_value = data_table
dummy_obs_id = "1234"
oids = ehst.get_observation_type(dummy_obs_id)
assert oids == 'HST Test'
def test_get_hst_link(self, mock_observation_type, mock_query):
mock_observation_type.return_value = "HST"
arr = {
'a': np.array([1], dtype=np.int32),
'b': [2.0],
'observation_id': ['1234']
}
data_table = Table(arr)
ehst = ESAHubbleClass(self.get_dummy_tap_handler())
mock_query.return_value = data_table
dummy_obs_id = "1234"
oids = ehst.get_hap_hst_link(dummy_obs_id)
assert oids == ['1234']
def test_write_jsviewer_overwrite(tmpdir):
t = Table()
t['a'] = [1, 2, 3, 4, 5]
t['b'] = ['a', 'b', 'c', 'd', 'e']
t['a'].unit = 'm'
tmpfile = tmpdir.join('test.html').strpath
# normal write
t.write(tmpfile, format='jsviewer')
# errors on overwrite
with pytest.raises(OSError, match="exists"):
t.write(tmpfile, format='jsviewer')
# unless specified
t.write(tmpfile, format='jsviewer', overwrite=True)
def test_invalid_updates(self, main_col, table_types, engine):
# using .loc and .loc_indices with a value not present should raise an exception
self._setup(main_col, table_types)
t = Table([[1, 2, 3, 4], [2, 3, 4, 5], [3, 4, 5, 6]], names=('a', 'b', 'c'), meta={'name': 'first table'})
t.add_index('a')
with pytest.raises(ValueError):
t.loc[3] = [[1,2,3]]
with pytest.raises(ValueError):
t.loc[[1, 4, 2]] = [[1, 2, 3], [4, 5, 6]]
with pytest.raises(ValueError):
t.loc[[1, 4, 2]] = [[1, 2, 3], [4, 5, 6], [2, 3]]
with pytest.raises(ValueError):
t.loc[[1, 4, 2]] = [[1, 2, 3], [4, 5], [2, 3]]
def get_transmission(self, energies=None, attenuator=False):
"""
Get the transmission for each detector at the center of the given energy bins.
If energies are not supplied will evaluate at standard science energy channels
Parameters
----------
energies : `astropy.units.Quantity`, optional
The energies to evaluate the transmission
attenuator : `bool`, optional
True for attenuator in X-ray path, False for attenuator not in X-ray path
Returns
-------
`astropy.table.Table`
Table containing the transmission values for each energy and detector
"""
base_comps = [
self.components[name] for name in [
'front_window', 'rear_window', 'dem', 'mli',
'calibration_foil', 'dead_layer'
]
]
if energies is None:
energies = self.energies
if attenuator:
base_comps.append(self.components['attenuator'])
base = Compound(base_comps)
base_trans = base.transmission(energies)
fine = Compound(base_comps + [self.components['grid_covers']])
fine_trans = fine.transmission(energies)
# TODO need to move to configuration db
fine_grids = np.array([11, 13, 18, 12, 19, 17]) - 1
transmission = Table()
# transmission['sci_channel'] = range(1, 31)
transmission['energies'] = energies
for i in range(32):
name = f'det-{i}'
if np.isin(i, fine_grids):
transmission[name] = fine_trans
else:
transmission[name] = base_trans
return transmission
def test_updating_row_byindex(self, main_col, table_types, engine):
self._setup(main_col, table_types)
t = Table([['a', 'b', 'c', 'd'], [2, 3, 4, 5], [3, 4, 5, 6]], names=('a', 'b', 'c'), meta={'name': 'first table'})
t.add_index('a', engine=engine)
t.add_index('b', engine=engine)
t.loc['c'] = ['g', 40, 50] # single label, with primary key 'a'
t2 = t[2]
assert list(t2) == ['g', 40, 50]
# list search
t.loc[['a', 'd', 'b']] = [['a', 20, 30], ['d', 50, 60], ['b', 30, 40]]
t2 = [['a', 20, 30], ['d', 50, 60], ['b', 30, 40]]
for i, p in zip(t2, [1, 4, 2]): # same order as input list
assert list(t[p-1]) == i
def createGaussianImage(self, posX, posY, fluxInPhotons, stdX, stdY=None):
table = Table()
table['x_mean'] = [posX]
table['y_mean'] = [posY]
table['x_stddev'] = [stdX]
if stdY is None:
table['y_stddev'] = [stdX]
else:
table['y_stddev'] = [stdY]
table['flux'] = [fluxInPhotons]
self._table = table
ima = np.zeros(self._shape)
ima += make_gaussian_sources_image(self._shape, self._table)
if self._usePoissonNoise:
ima = self._addShotNoise(ima)
return ima
def createMultipleIntegratedGaussianPRFImage(self,
stddevRange=[2., 3],
fluxInPhotons=[1000., 10000],
nStars=100):
xMean = np.random.uniform(1, self._shape[1] - 1, nStars)
yMean = np.random.uniform(1, self._shape[0] - 1, nStars)
sx = np.random.uniform(stddevRange[0], stddevRange[1], nStars)
flux = np.random.uniform(fluxInPhotons[0], fluxInPhotons[1], nStars)
self._table = Table()
self._table['x_0'] = xMean
self._table['y_0'] = yMean
self._table['sigma'] = sx
self._table['flux'] = flux
ima = make_model_sources_image(self._shape, IntegratedGaussianPRF(),
self._table)
return ima
def create(self, data, header, force_it=True, **build):
""" builds the catalogue
Parameters
----------
data: [astropy.TableColumns, dictionnary or numpy.ndarray (withkey)]
the data associated to the catalogue. It must have
ra, dec, and magnitude entries. The keys associated
to these are set in _build_properties (key_ra,
key_dec, key_mag ...). See also set_mag_keys
header: [pyfits.Header / None]
Header containing the data for the catalogue (if any)
force_it: [bool] -optional-
if data already exists, set force_it to true to overwrite it.
**build goes to the build dictorty (key_mag, data_slice etc.)
Returns
-------
Void
"""
if self.has_data() and force_it is False:
raise AttributeError("'data' is already defined."+\
" Set force_it to True if you really known what you are doing")
self._properties["data"] = Table(data)
self._properties["header"] = header if header is not None \
else pf.Header()
self.set_starsid(build.pop("key_class", None),
build.pop("value_star", None))
self._build_properties = kwargs_update(self._build_properties, **build)
# -------------------------------
# - Try to get the fundamentals
if self._build_properties['key_ra'] is None:
self._automatic_key_match_("ra")
if self._build_properties['key_dec'] is None:
self._automatic_key_match_("dec")
self._update_contours_()
def splice_fits(flg=0):
'''
Splices together the various PCA fits for SDSS or BOSS
flg: int (0)
0=BOSS, 1=SDSS
'''
import glob
from astropy.table.table import Table
if flg == 0:
outroot = 'Output/BOSS_DR10Lya_PCA_values_nocut'
outfil = 'BOSS_DR10Lya_PCA_values_nocut.fits'
elif flg == 1:
outroot = 'Output/SDSS_DR7Lya_PCA_values_nocut'
outfil = 'SDSS_DR7Lya_PCA_values_nocut.fits'
# Get all the files
files = glob.glob(outroot + '*')
for ifil in files:
print('Reading {:s}'.format(ifil))
hdu = fits.open(ifil)
tab = Table(hdu[1].data)
#
if not 'full_tab' in locals():
full_tab = tab
else:
#xdb.set_trace()
for row in tab:
full_tab.add_row(row)
# Write
prihdr = fits.Header()
if flg == 0:
prihdr['PROJECT'] = 'BOSS: z>2 quasars'
elif flg == 1:
prihdr['PROJECT'] = 'SDSS: Meant for z<2 quasars'
prihdr['COMMENT'] = 'PCA fits to the quasars'
prihdu = fits.PrimaryHDU(header=prihdr)
table_hdu = fits.BinTableHDU.from_columns(np.array(full_tab.filled()))
thdulist = fits.HDUList([prihdu, table_hdu])
print('Writing {:s} table, with {:d} rows'.format(outfil, len(full_tab)))
thdulist.writeto(outfil, clobber=True)
