def test_table_to_ts(self):
# Generate the data and the corresponding dates
base = parse_time(datetime.datetime.today())
times = base - TimeDelta(np.arange(24 * 60)*u.minute)
intensity = u.Quantity(np.sin(np.arange(0, 12 * np.pi, ((12 * np.pi) / (24*60)))), u.W/u.m**2)
# Create the units and meta objects
units = OrderedDict([('intensity', u.W/u.m**2)])
meta = MetaDict({'key':'value'})
tbl_meta = MetaDict({'t_key':'t_value'})
# Create a suitable mixin qtable
table = Table([times, intensity], names=['time', 'intensity'], meta=tbl_meta)
table.add_index('time')
# Create TS from table and check
ts_table = sunpy.timeseries.TimeSeries(table, meta, units)
assert isinstance(ts_table, sunpy.timeseries.timeseriesbase.GenericTimeSeries)
ts_table2 = sunpy.timeseries.TimeSeries(table, units, meta)
assert (ts_table2 == ts_table)
# Create TS using a tuple of values
ts_table3 = sunpy.timeseries.TimeSeries((table, meta, units))
assert isinstance(ts_table3, sunpy.timeseries.timeseriesbase.GenericTimeSeries)
# ToDo: Try an incompatible table
dual_index_table = Table([times, intensity], names=['time', 'intensity'], meta=tbl_meta)
dual_index_table.add_index(('time', 'intensity'))
with pytest.raises(ValueError):
sunpy.timeseries.TimeSeries((dual_index_table, meta, units))
def test_table_to_ts(self):
# Generate the data and the corresponding dates
base = parse_time(datetime.datetime.today())
times = base - TimeDelta(np.arange(24 * 60)*u.minute)
intensity = u.Quantity(np.sin(np.arange(0, 12 * np.pi, ((12 * np.pi) / (24*60)))), u.W/u.m**2)
# Create the units and meta objects
units = OrderedDict([('intensity', u.W/u.m**2)])
meta = MetaDict({'key':'value'})
tbl_meta = MetaDict({'t_key':'t_value'})
# Create a suitable mixin qtable
table = Table([times, intensity], names=['time', 'intensity'], meta=tbl_meta)
table.add_index('time')
# Create TS from table and check
ts_table = sunpy.timeseries.TimeSeries(table, meta, units)
assert isinstance(ts_table, sunpy.timeseries.timeseriesbase.GenericTimeSeries)
ts_table2 = sunpy.timeseries.TimeSeries(table, units, meta)
assert (ts_table2 == ts_table)
# Create TS using a tuple of values
ts_table3 = sunpy.timeseries.TimeSeries((table, meta, units))
assert isinstance(ts_table3, sunpy.timeseries.timeseriesbase.GenericTimeSeries)
# ToDo: Try an incompatible table
dual_index_table = Table([times, intensity], names=['time', 'intensity'], meta=tbl_meta)
dual_index_table.add_index(('time', 'intensity'))
with pytest.raises(ValueError):
sunpy.timeseries.TimeSeries((dual_index_table, meta, units))
def _get_fid_acq_stages():
fid_acqs = Table.read("""
warns score P2=0.0 P2=2.0 P2=2.5 P2=3.0 P2=4.0 P2=5.0 P2=6.0 P2=8.0 P2=99.0
----- ----- ------ ------ ------ ------ ------ ------ ------ ------ -------
- 0 0 1.9 2.4 2.8 3.6 4.5 5.0 6.0 6.0
Y 1 0 1.9 2.4 2.8 3.6 4.5 5.0 6.0 6.0
YY 2 0 1.9 2.4 2.75 3.4 4.2 4.2 4.5 4.5
YYY 3 0 1.9 2.4 2.7 3.2 3.5 3.5 3.5 3.5
R 4 0 1.7 2.2 2.5 3.1 3.4 3.4 3.4 3.4
RY 5 0 1.7 2.2 2.4 3.0 3.3 3.3 3.3 3.3
RYY 6 0 1.7 2.2 2.3 2.5 2.5 2.5 2.5 2.5
RR 8 0 1.5 2.0 2.0 2.0 2.0 2.0 2.0 2.0
RRY 9 0 1.5 2.0 2.0 2.0 2.0 2.0 2.0 2.0
RRR 12 -1 -1.0 -1.0 -1.0 -1.0 -1.0 -1.0 -1.0 -1.0
""",
format='ascii.fixed_width_two_line')
P2s = [
float(name[3:]) for name in fid_acqs.colnames if name.startswith('P2=')
]
funcs = []
for fid_acq in fid_acqs:
vals = [
fid_acq[name] for name in fid_acqs.colnames
if name.startswith('P2=')
]
funcs.append(interp1d(P2s, vals))
out = Table([fid_acqs['score'], funcs], names=['spoiler_score', 'min_P2'])
out.add_index('spoiler_score')
return out
def get_site_table_single_query(from_date=None, ndays=5):
if ndays > 100:
ndays = 100
sched = get_telsched(from_date=from_date, ndays=ndays, telnr=None)
t = Table(names=[
'Date', 'progID', 'Observers', 'PiFirstName', 'PiLastName', 'PiId'
] + site_list,
dtype=['a10', 'a10', 'a100', 'a50', 'a50', 'i4'] +
[int] * len(site_list))
for prog in sched:
if prog['Date'] not in list(t['Date']):
row = {
'Date': prog['Date'],
'Observers': prog['Observers'],
'progID': prog['ProjCode']
}
for site in site_list:
row[site] = 0
t.add_row(row)
if prog['Location'] == 'CIT. Hirsch,CIT,UCB,CIT':
tonights_sites = 'CIT,CIT,UCB,CIT'.split(',')
else:
tonights_sites = prog['Location'].split(',')
tonights_observers = prog['Observers'].split(',')
while len(tonights_sites) != len(tonights_observers):
if len(tonights_sites) > len(tonights_observers):
print(
f'{prog["Date"]}: N sites > N observers: removing last site'
)
tonights_sites.pop()
elif len(tonights_sites) < len(tonights_observers):
print(
f'{prog["Date"]}: N sites < N observers: adding site Other'
)
tonights_sites.append('Other')
t.add_index('Date')
rowid = t.loc_indices[prog['Date']]
for entry in tonights_sites:
if entry == 'CIT. Hirsch':
print(tonights_sites)
print('Correcting comma')
entry = 'CIT, Hirsch'
if entry in site_list:
t[rowid][entry] += 1
elif entry == 'Swin':
t[rowid]['Swinburne'] += 1
elif entry == 'Northwestern':
t[rowid]['NU'] += 1
elif entry == 'USCS':
t[rowid]['UCSC'] += 1
elif entry == 'NASA':
t[rowid]['Other'] += 1
elif entry == '':
pass
else:
print(f'Unmatched entry: "{entry}"')
return t
def test_table_index_time_warning(engine):
# Make sure that no ERFA warnings are emitted when indexing a table by
# a Time column with a non-default time scale
tab = Table()
tab['a'] = Time([1, 2, 3], format='jyear', scale='tai')
tab['b'] = [4, 3, 2]
with warnings.catch_warnings(record=True) as wlist:
tab.add_index(('a', 'b'), engine=engine)
assert len(wlist) == 0
def get_datasets(observatory):
"""
Get a list of datasets for a given observatory.
Parameters
----------
observatory : `str`
Observatory name.
Returns
-------
`astropy.table.Table`
Examples
--------
>>> from sunpy.net.cdaweb import get_datasets
>>>
>>> datasets = get_datasets('STEREOB') #doctest: +REMOTE_DATA
>>> datasets['Id'] #doctest: +REMOTE_DATA
<Column name='Id' dtype='str17' length=4>
STB_LB_IMPACT
STB_L1_IMPACT_HKP
STB_L1_HET
STB_L1_SWEA_SPEC
>>> datasets.loc['STB_L1_SWEA_SPEC']['Label'] #doctest: +REMOTE_DATA
'STEREO Behind IMPACT/SWEA Spectra - J. Luhmann (UCB/SSL)'
>>> datasets.loc['STB_L1_SWEA_SPEC'][['Start', 'End']] #doctest: +REMOTE_DATA
<Row index=3>
Start End
str24 str24
------------------------ ------------------------
2012-12-01T00:00:03.000Z 2013-12-31T23:59:41.000Z
"""
# Get a list of files for a given dataset between start and end times
url = '/'.join([
_CDAS_BASEURL,
'dataviews', _DATAVIEW,
'datasets'
])
url = f'{url}?observatory={observatory}'
response = requests.get(url, headers=_CDAS_HEADERS)
datasets = response.json()['DatasetDescription']
ids = [dataset['Id'] for dataset in datasets]
instruments = [', '.join(dataset['Instrument']) for dataset in datasets]
labels = [dataset['Label'] for dataset in datasets]
stimes = [dataset['TimeInterval']['Start'] for dataset in datasets]
etimes = [dataset['TimeInterval']['End'] for dataset in datasets]
t = Table([ids, instruments, labels, stimes, etimes],
names=['Id', 'Instruments', 'Label', 'Start', 'End'])
t.add_index('Id')
return t
def test_invalid_updates(self, main_col, table_types, engine):
# using .loc and .loc_indices with a value not present should raise an exception
self._setup(main_col, table_types)
t = Table([[1, 2, 3, 4], [2, 3, 4, 5], [3, 4, 5, 6]],
names=('a', 'b', 'c'), meta={'name': 'first table'})
t.add_index('a')
with pytest.raises(ValueError):
t.loc[3] = [[1, 2, 3]]
with pytest.raises(ValueError):
t.loc[[1, 4, 2]] = [[1, 2, 3], [4, 5, 6]]
with pytest.raises(ValueError):
t.loc[[1, 4, 2]] = [[1, 2, 3], [4, 5, 6], [2, 3]]
with pytest.raises(ValueError):
t.loc[[1, 4, 2]] = [[1, 2, 3], [4, 5], [2, 3]]
def get_observatory_groups():
"""
Get a list of observatory IDs for each observatory in CDAWeb.
An observatory group is typically a single mission, which can contain
multiple observatories, e.g. for the STEREO observatory group there are two
observatories, STEREO-A and STEREO-B.
Returns
-------
`astropy.table.Table`
Examples
--------
>>> from sunpy.net.cdaweb import get_observatory_groups
>>>
>>> groups = get_observatory_groups() #doctest: +REMOTE_DATA
>>> groups['Group'] #doctest: +REMOTE_DATA
<Column name='Group' dtype='str55' length=75>
ACE
AMPTE
...
Voyager
Wind
>>> groups.loc['STEREO'] #doctest: +REMOTE_DATA
<Row index=62>
Group Observatories
str55 str518
------ -----------------------------------------------------------------------------
STEREO 'Ahead', 'Behind', 'STA', 'STB', 'STEREO', 'STEREOA', 'STEREOB', 'sta', 'stb'
"""
# Get a list of files for a given dataset between start and end times
url = '/'.join(
[_CDAS_BASEURL, 'dataviews', _DATAVIEW, 'observatoryGroups'])
response = requests.get(url, headers=_CDAS_HEADERS)
obs_groups = response.json()
names = [obs['Name'] for obs in obs_groups['ObservatoryGroupDescription']]
obs_ids = [
obs['ObservatoryId']
for obs in obs_groups['ObservatoryGroupDescription']
]
# Join all IDs into a single string
obs_ids = ["'" + "', '".join(id) + "'" for id in obs_ids]
t = Table([names, obs_ids], names=['Group', 'Observatories'])
t.add_index('Group')
return t
def test_updating_row_byindex(self, main_col, table_types, engine):
self._setup(main_col, table_types)
t = Table([['a', 'b', 'c', 'd'], [2, 3, 4, 5], [3, 4, 5, 6]],
names=('a', 'b', 'c'), meta={'name': 'first table'})
t.add_index('a', engine=engine)
t.add_index('b', engine=engine)
t.loc['c'] = ['g', 40, 50] # single label, with primary key 'a'
t2 = t[2]
assert list(t2) == ['g', 40, 50]
# list search
t.loc[['a', 'd', 'b']] = [['a', 20, 30], ['d', 50, 60], ['b', 30, 40]]
t2 = [['a', 20, 30], ['d', 50, 60], ['b', 30, 40]]
for i, p in zip(t2, [1, 4, 2]): # same order as input list
assert list(t[p - 1]) == i
def sortBf(data, model):
if model=="b2s1_ode":
col_names = ['branch','stability','p','L2','b1','b2','s']
# stability = data[1]>=0
data_cols = [data[0],data[1],data[4],data[5],data[6],data[7],data[8]]
t = Table(data_cols, names=col_names
,meta={'name': model+' ODE AUTO bifurcation diagram'}
,dtype=('i4', 'i4','f8','f8','f8','f8','f8'))
t.add_index('branch')
elif model=="b2s2":
col_names = ['branch','stability','p','L2','b1','b2','s1','s2']
# stability = data[1]>=0
data_cols = [data[0],data[1],data[4],data[5],data[6],data[7],data[8],data[9]]
t = Table(data_cols, names=col_names
,meta={'name': model+' ODE AUTO bifurcation diagram'}
,dtype=('i4', 'i4','f8','f8','f8','f8','f8','f8'))
t.add_index('branch')
return t
def table_ts():
# Generate the data and the corresponding dates
base = parse_time(datetime.datetime.today())
times = base - TimeDelta(np.arange(24 * 60)*u.minute)
intensity = u.Quantity(
np.sin(np.arange(0, 12 * np.pi, ((12 * np.pi) / (24 * 60)))), u.W / u.m ** 2)
# Create the units and meta objects
units = OrderedDict([('intensity', u.W / u.m**2)])
meta = MetaDict({'key': 'value'})
tbl_meta = MetaDict({'t_key': 't_value'})
# Create a suitable mixin qtable
table = Table(
[times, intensity], names=['time', 'intensity'], meta=tbl_meta)
table.add_index('time')
# Create TS from dataframe and check
return sunpy.timeseries.TimeSeries(table, meta, units)
def table_ts():
# Generate the data and the corresponding dates
base = parse_time(datetime.datetime.today())
times = base - TimeDelta(np.arange(24 * 60)*u.minute)
intensity = u.Quantity(
np.sin(np.arange(0, 12 * np.pi, ((12 * np.pi) / (24 * 60)))), u.W / u.m ** 2)
# Create the units and meta objects
units = OrderedDict([('intensity', u.W / u.m**2)])
meta = MetaDict({'key': 'value'})
tbl_meta = MetaDict({'t_key': 't_value'})
# Create a suitable mixin qtable
table = Table(
[times, intensity], names=['time', 'intensity'], meta=tbl_meta)
table.add_index('time')
# Create TS from dataframe and check
return sunpy.timeseries.TimeSeries(table, meta, units)
def to_table(self, format=".6e"):
"""Convert covariance matrix to table
Parameters
----------
format : str
Column format string
Returns
-------
table : `~astropy.table.Table`
Covariance table
"""
table = Table()
table["name"] = self.parameters.names
for idx, par in enumerate(self.parameters):
vals = self.data[idx]
table[par.name] = vals
table[par.name].format = format
table.add_index("name")
return table
def test_get_index():
a = [1, 4, 5, 2, 7, 4, 45]
b = [2.0, 5.0, 8.2, 3.7, 4.3, 6.5, 3.3]
t = Table([a, b], names=('a', 'b'), meta={'name': 'first table'})
t.add_index(['a'])
# Getting the values of index using names
x1 = get_index(t, names=['a'])
assert isinstance(x1, SlicedIndex)
assert len(x1.columns) == 1
assert len(x1.columns[0]) == 7
assert x1.columns[0].info.name == 'a'
# Getting the vales of index using table_copy
x2 = get_index(t, table_copy=t[['a']])
assert isinstance(x2, SlicedIndex)
assert len(x2.columns) == 1
assert len(x2.columns[0]) == 7
assert x2.columns[0].info.name == 'a'
with pytest.raises(ValueError):
get_index(t, names=['a'], table_copy=t[['a']])
with pytest.raises(ValueError):
get_index(t, names=None, table_copy=None)
lcat = 'WENSS'
if 'WENSS' in labels:
nu = [pardict[hcat][0], pardict[lcat][0]]
s = [pardict[hcat][1], pardict[lcat][1]]
e_s = [pardict[hcat][2], pardict[lcat][2]]
if ~np.isnan(s).any():
sp, e_sp = dquants.two_pt_alpha(nu, s, e_s)
al_calcs[ii]['al_l_WENSS_NVSS', 'e_al_l_WENSS_NVSS'] = sp, e_sp
lcol = lcat + '_Limit'
if lcol in sp_row.colnames:
if sp_row[lcol] == 'U':
al_calcs[ii]['l_al_l_WENSS_NVSS'] = True
al_calcs = join(al_calcs, sp_class, keys='Source_name', join_type='left')
al_calcs.add_index('Source_name')
sed_fit_params = Table(
names=('Source_name', 'SED_Class', 'al_10_1.4', 'e_al_10_1.4',
'l_al_10_1.4', 'al_1.4_.15', 'e_al_1.4_.15', 'l_al_1.4_.15', 'nu_p',
'e_nu_p', 'l_nu_p', 's_p', 'e_s_p', 'q', 'e_q', 'al_highc',
'e_al_highc', 'l_al_highc', 'al_lowc', 'e_al_lowc', 'l_al_lowc',
'quality', 'ccode', 's0_highc', 'e_s0_highc', 's0_lowc',
'e_s0_lowc', 'e_nu_p_eh', 'e_nu_p_el', 'e_s_p_eh', 'e_s_p_el'),
dtype=('S8', 'S3', 'f8', 'f8', 'bool', 'f8', 'f8', 'bool', 'f8', 'f8',
'bool', 'f8', 'f8', 'f8', 'f8', 'f8', 'f8', 'bool', 'f8', 'f8',
'bool', 'S2', 'S1', 'f8', 'f8', 'f8', 'f8', 'f8', 'f8', 'f8', 'f8'))
fit_results_ext_tab.add_index('source')
for row in final_unq_jvla:
def main(name,version,HSTband,scalepc):
'''match nebulae and association catalogue
'''
print(f'parameters: {name} {version} {HSTband} {scalepc}')
# =====================================================================
# Read in the data
# =====================================================================
#p = {x:sample_table.loc[name][x] for x in sample_table.columns}
# DAP linemaps (Halpha and OIII)
filename = data_ext / 'MUSE' / 'DR2.1' / 'copt' / 'MUSEDAP'
filename = [x for x in filename.iterdir() if x.stem.startswith(name)][0]
with fits.open(filename) as hdul:
Halpha = NDData(data=hdul['HA6562_FLUX'].data,
uncertainty=StdDevUncertainty(hdul['HA6562_FLUX_ERR'].data),
mask=np.isnan(hdul['HA6562_FLUX'].data),
meta=hdul['HA6562_FLUX'].header,
wcs=WCS(hdul['HA6562_FLUX'].header))
OIII = NDData(data=hdul['OIII5006_FLUX'].data,
uncertainty=StdDevUncertainty(hdul['OIII5006_FLUX_ERR'].data),
mask=np.isnan(hdul['OIII5006_FLUX'].data),
meta=hdul['OIII5006_FLUX'].header,
wcs=WCS(hdul['OIII5006_FLUX'].header))
# the original catalogue from Francesco
with fits.open(nebulae_file) as hdul:
nebulae = Table(hdul[1].data)
nebulae['SkyCoord'] = SkyCoord(nebulae['cen_ra']*u.deg,nebulae['cen_dec']*u.deg,frame='icrs')
'''
with fits.open(basedir/'data'/'interim'/f'Nebulae_Catalogue_v2p1_dig.fits') as hdul:
dig = Table(hdul[1].data)
with fits.open(basedir/'data'/'interim'/f'Nebulae_Catalogue_v2p1_fuv.fits') as hdul:
fuv = Table(hdul[1].data)
with fits.open(basedir/'data'/'interim'/f'Nebulae_Catalogue_v2p1_eq.fits') as hdul:
eq_width = Table(hdul[1].data)
nebulae = join(nebulae,fuv,keys=['gal_name','region_ID'])
nebulae = join(nebulae,eq_width,keys=['gal_name','region_ID'])
nebulae = join(nebulae,dig,keys=['gal_name','region_ID'])'
'''
nebulae.rename_columns(['cen_x','cen_y'],['x','y'])
with np.errstate(divide='ignore',invalid='ignore'):
nebulae['[SIII]/[SII]'] = np.nan
SII = nebulae['SII6716_FLUX_CORR']+nebulae['SII6730_FLUX_CORR']
SIII = nebulae['SIII6312_FLUX_CORR']+nebulae['SIII9068_FLUX_CORR']
nebulae[SII>0]['[SIII]/[SII]'] = SIII[SII>0]/SII[SII>0]
#nebulae['HA/FUV'] = nebulae['HA6562_FLUX_CORR']/nebulae['FUV_FLUX_CORR']
#nebulae['HA/FUV_err'] = nebulae['HA/FUV']*np.sqrt((nebulae['HA6562_FLUX_CORR_ERR']/nebulae['HA6562_FLUX_CORR'])**2+(nebulae['FUV_FLUX_CORR_ERR']/nebulae['FUV_FLUX_CORR'])**2)
nebulae = nebulae[nebulae['gal_name']==name]
nebulae.add_index('region_ID')
filename = data_ext / 'Products' / 'Nebulae_catalogs'/'Nebulae_catalogue_v2' /'spatial_masks'/f'{name}_nebulae_mask_V2.fits'
with fits.open(filename) as hdul:
nebulae_mask = NDData(hdul[0].data.astype(float),mask=Halpha.mask,meta=hdul[0].header,wcs=WCS(hdul[0].header))
nebulae_mask.data[nebulae_mask.data==-1] = np.nan
#print(f'{name}: {len(nebulae)} HII-regions in final catalogue')
# the association catalogue and mask
target = name.lower()
associations, associations_mask = read_associations(folder=association_folder,
target=target,scalepc=scalepc,
HSTband=HSTband,version=version,data='all')
if not associations:
return 0
# enviornmental masks
with fits.open(env_masks_folder / f'{name}_simple.fits') as hdul:
mask = reproject_interp(hdul[0],Halpha.meta,order='nearest-neighbor',return_footprint=False)
env_masks_neb = NDData(data=mask,
meta=hdul[0].header,
wcs=Halpha.wcs)
#print(f'{name}: {len(associations)} associations in catalogue')
# =====================================================================
# reproject and match catalogues
# =====================================================================
nebulae_hst, _ = reproject_interp(nebulae_mask,
output_projection=associations_mask.wcs,
shape_out=associations_mask.data.shape,
order='nearest-neighbor')
# we scale the associations such that the the id is in the decimal
scale = 10**np.ceil(np.log10(max(associations_mask.data[~np.isnan(associations_mask.data)])))
s_arr = associations_mask.data/scale+nebulae_hst
#print(f'masks reprojected')
# ids of associations, nebulae and combination (sum) of both
a_id = np.unique(associations_mask.data[~np.isnan(associations_mask.data)]).astype(int)
n_id = np.unique(nebulae_mask.data[~np.isnan(nebulae_mask.data)]).astype(int)
s_id = np.unique(s_arr[~np.isnan(s_arr)])
# this splits the sum into two parts (nebulae and associations)
a_modf,n_modf = np.modf(s_id)
n_modf = n_modf.astype(int)
a_modf = np.round(a_modf*scale).astype(int)
unique_a, count_a = np.unique(a_modf,return_counts=True)
unique_n, count_n = np.unique(n_modf,return_counts=True)
nebulae_dict = {int(n) : a_modf[n_modf==n].tolist() for n in n_id}
associations_dict = {int(a) : n_modf[a_modf==a].tolist() for a in a_id}
# so far we ensured that the nebulae in unique_n have only one association,
# but it is possible that this association goes beyond the nebulae and into
# a second nebulae. Those objects are excluded here
isolated_nebulae = set()
isolated_assoc = set()
for n,v in nebulae_dict.items():
if len(v)==1:
if len(associations_dict[v[0]])==1:
isolated_nebulae.add(n)
isolated_assoc.add(v[0])
#print(f'n_associations = {len(associations_dict)}')
#print(f'n_nebulae = {len(nebulae_dict)}')
#print(f'1to1 match = {len(isolated_nebulae)}')
# we save those two dicts so we do not have to redo this everytime
with open(basedir/version/HSTband/f'{scalepc}pc'/f'{name}_{HSTband}_{scalepc}pc_nebulae.yml','w+') as f:
yaml.dump(nebulae_dict,f)
with open(basedir/version/HSTband/f'{scalepc}pc'/f'{name}_{HSTband}_{scalepc}pc_associations.yml','w+') as f:
yaml.dump(associations_dict,f)
# find all assoc that have at least one pixel outside of the nebulae masks
mask = associations_mask.data.copy()
mask[~np.isnan(nebulae_hst)] = np.nan
outside = np.unique(mask[~np.isnan(mask)].astype(int))
# find all assoc that have at least one pixel inside of the nebulea masks
mask = associations_mask.data.copy()
mask[np.isnan(nebulae_hst)] = np.nan
inside = np.unique(mask[~np.isnan(mask)].astype(int))
contained = np.setdiff1d(inside,outside)
partial = np.intersect1d(inside,outside)
isolated = np.setdiff1d(outside,inside)
#print(f'contained: {len(contained)}\npartial: {len(partial)}\nisolated: {len(isolated)}')
assoc_tmp = associations[['assoc_ID']].copy()
assoc_tmp.add_index('assoc_ID')
x_asc,y_asc = associations['SkyCoord'].to_pixel(env_masks_neb.wcs)
outside = (x_asc > env_masks_neb.data.shape[1]) | (y_asc > env_masks_neb.data.shape[0])
x_asc[outside] = 0
y_asc[outside] = 0
assoc_tmp['env_asc'] = [environment_dict[env_masks_neb.data[y,x]] for
x,y in zip(x_asc.astype(int),y_asc.astype(int))]
assoc_tmp[outside]['env_asc'] = ''
assoc_tmp['overlap'] = np.empty(len(associations),dtype='U9')
assoc_tmp['overlap'][np.isin(assoc_tmp['assoc_ID'],contained)] = 'contained'
assoc_tmp['overlap'][np.isin(assoc_tmp['assoc_ID'],partial)] = 'partial'
assoc_tmp['overlap'][np.isin(assoc_tmp['assoc_ID'],isolated)] = 'isolated'
assoc_tmp['1to1'] = False
assoc_tmp['1to1'][np.isin(assoc_tmp['assoc_ID'],list(isolated_assoc))] = True
assoc_tmp['Nnebulae'] = [len(associations_dict[k]) for k in assoc_tmp['assoc_ID']]
assoc_tmp['region_ID'] = np.nan
assoc_tmp['region_ID'][assoc_tmp['1to1']] = [associations_dict[k][0] for k in assoc_tmp[assoc_tmp['1to1']]['assoc_ID']]
overlap = join(
Table(np.unique(associations_mask.data[~np.isnan(associations_mask.data)],return_counts=True),names=['assoc_ID','size']),
Table(np.unique(associations_mask.data[~np.isnan(nebulae_hst) & ~np.isnan(associations_mask.data)],return_counts=True),names=['assoc_ID','overlap_size']),
keys=['assoc_ID'],join_type='outer')
overlap = overlap.filled(0)
overlap['overlap_asc'] = overlap['overlap_size']/overlap['size']
overlap['overlap_asc'].info.format = '%.2f'
assoc_tmp = join(assoc_tmp,overlap[['assoc_ID','overlap_asc']],keys='assoc_ID')
#print('write to file')
hdu = fits.BinTableHDU(assoc_tmp,name='joined catalogue')
hdu.writeto(basedir/version/HSTband/f'{scalepc}pc'/f'{name}_{HSTband}_{scalepc}pc_associations.fits',overwrite=True)
nebulae_tmp = nebulae[['region_ID','x','y']].copy()
nebulae_tmp.add_index('region_ID')
nebulae_tmp['env_neb'] = [environment_dict[env_masks_neb.data[y,x]] for
x,y in zip(nebulae_tmp['x'].astype(int),nebulae_tmp['y'].astype(int))]
nebulae_tmp['neighbors'] = np.nan
for row in nebulae_tmp:
row['neighbors'] = len(find_neighbors(nebulae_mask.data,tuple(row[['x','y']]),row['region_ID'],plot=False))
del nebulae_tmp[['x','y']]
nebulae_tmp['1to1'] = False
nebulae_tmp['1to1'][np.isin(nebulae_tmp['region_ID'],list(isolated_nebulae))] = True
nebulae_tmp['Nassoc'] = [len(nebulae_dict[k]) for k in nebulae_tmp['region_ID']]
nebulae_tmp['assoc_ID'] = np.nan
nebulae_tmp['assoc_ID'][nebulae_tmp['1to1']] = [nebulae_dict[k][0] for k in nebulae_tmp[nebulae_tmp['1to1']]['region_ID']]
overlap = join(
Table(np.unique(nebulae_hst[~np.isnan(nebulae_hst)],return_counts=True),names=['region_ID','size']),
Table(np.unique(nebulae_hst[~np.isnan(nebulae_hst) & ~np.isnan(associations_mask.data)],return_counts=True),names=['region_ID','overlap_size']),
keys=['region_ID'],join_type='outer')
overlap = overlap.filled(0)
overlap['overlap_neb'] = overlap['overlap_size']/overlap['size']
overlap['overlap_neb'].info.format = '%.2f'
nebulae_tmp = join(nebulae_tmp,overlap[['region_ID','overlap_neb']],keys='region_ID')
hdu = fits.BinTableHDU(nebulae_tmp,name='joined catalogue')
hdu.writeto(basedir/version/HSTband/f'{scalepc}pc'/f'{name}_{HSTband}_{scalepc}pc_nebulae.fits',overwrite=True)
#del nebulae_tmp['1to1']
#print(f'{np.sum(nebulae_tmp["neighbors"]==0)} nebulae have no neighbors')
catalogue = join(assoc_tmp,nebulae_tmp,keys=['assoc_ID','region_ID'])
catalogue = join(catalogue,nebulae,keys='region_ID')
catalogue = join(catalogue,associations,keys='assoc_ID')
# pay attention to the order of assoc, neb
catalogue.rename_columns(['X','Y','x','y','RA','DEC','cen_ra','cen_dec',
'reg_area','region_area',
'EBV_1','EBV_2','EBV_err','EBV_ERR',
'SkyCoord_1','SkyCoord_2'],
['x_asc','y_asc','x_neb','y_neb','ra_asc','dec_asc','ra_neb','dec_neb',
'area_asc','area_neb',
'EBV_balmer','EBV_stars','EBV_balmer_err','EBV_stars_err',
'SkyCoord_asc','SkyCoord_neb'])
# separation to other associations and nebulae
idx,sep_asc,_= match_coordinates_sky(catalogue['SkyCoord_asc'],associations['SkyCoord'],nthneighbor=2)
idx,sep_neb,_= match_coordinates_sky(catalogue['SkyCoord_neb'],nebulae['SkyCoord'],nthneighbor=2)
catalogue['sep_asc'] = sep_asc.to(u.arcsec)
catalogue['sep_neb'] = sep_neb.to(u.arcsec)
# select the columns of the joined catalogue
columns = ['assoc_ID','region_ID','x_asc','y_asc','x_neb','y_neb',
'ra_asc','dec_asc','ra_neb','dec_neb','SkyCoord_asc','SkyCoord_neb',
'env_asc','env_neb','area_asc','area_neb',
'sep_asc','sep_neb','neighbors','Nassoc','overlap','overlap_asc','overlap_neb',
'age','age_err','mass','mass_err','EBV_stars','EBV_stars_err','EBV_balmer','EBV_balmer_err',
'met_scal','met_scal_err','logq_D91','logq_D91_err',] + \
[x for x in nebulae.columns if x.endswith('_FLUX_CORR')] + \
[x for x in nebulae.columns if x.endswith('_FLUX_CORR_ERR')] + \
['NUV_FLUX','NUV_FLUX_ERR','U_FLUX','U_FLUX_ERR','B_FLUX','B_FLUX_ERR',
'V_FLUX','V_FLUX_ERR','I_FLUX','I_FLUX_ERR']
catalogue = catalogue[columns]
catalogue.rename_columns([col for col in catalogue.columns if col.endswith('FLUX_CORR')],
[col.replace('FLUX_CORR','flux') for col in catalogue.columns if col.endswith('FLUX_CORR')])
catalogue.rename_columns([col for col in catalogue.columns if col.endswith('FLUX_CORR_ERR')],
[col.replace('FLUX_CORR_ERR','flux_err') for col in catalogue.columns if col.endswith('FLUX_CORR_ERR')])
catalogue['assoc_ID'] = catalogue['assoc_ID'].astype('int')
catalogue['region_ID'] = catalogue['region_ID'].astype('int')
catalogue.info.description = 'Joined catalogue between associations and nebulae'
mean_sep = np.mean(catalogue['SkyCoord_asc'].separation(catalogue['SkyCoord_neb']))
#print(f'{len(catalogue)} objects in catalogue')
#print(f'the mean separation between cluster and association center is {mean_sep.to(u.arcsecond):.2f}')
export = catalogue.copy() #[catalogue['contained']]
#export.add_column(export['SkyCoord_asc'].to_string(style='hmsdms',precision=2),index=6,name='RaDec_asc')
#export.add_column(export['SkyCoord_neb'].to_string(style='hmsdms',precision=2),index=8,name='RaDec_neb')
RA_asc ,DEC_asc = zip(*[x.split(' ') for x in export['SkyCoord_asc'].to_string(style='hmsdms',precision=2)])
RA_neb ,DEC_neb = zip(*[x.split(' ') for x in export['SkyCoord_neb'].to_string(style='hmsdms',precision=2)])
export.add_column(RA_asc,index=6,name='Ra_asc')
export.add_column(DEC_asc,index=8,name='Dec_asc')
export.add_column(RA_neb,index=10,name='Ra_neb')
export.add_column(DEC_neb,index=12,name='Dec_neb')
for col in export.columns:
if col not in ['Ra_asc','Dec_asc','Ra_neb','Dec_neb','region_ID','cluster_ID','overlap','env_asc','env_neb']:
export[col].info.format = '%.2f'
del export[['ra_asc','dec_asc','ra_neb','dec_neb','SkyCoord_neb','SkyCoord_asc']]
hdu = fits.BinTableHDU(export,name='joined catalogue')
hdu.writeto(basedir/version/HSTband/f'{scalepc}pc'/f'{name}_{HSTband}_{scalepc}pc_associations_and_nebulae_joined.fits',overwrite=True)
class ClusterFinder:
'''
Identify clusters in mock data.
'''
def __init__(self, galaxy, sort_key, d_proj, central_idx=None, \
periodic=True, boxsize=0.0, max_r=3.0, background=None):
'''
Parameters
------------
galaxy
Input galaxy catalog. The first three columns should be
coordinates ['x', 'y', 'z'] and the catalog will be projected
along z-axis.
sort_key : str
Galaxies will be sorted according to sort_key of the catalog
for the initial percolation.
d_proj : float
Projection length. Cluster finder will see any galaxies within
d_c +- d_proj as members. It should use the same units as
coordinates.
central_idx
Index of central galaxies to be considered as potential
centers. If set to None, will consider all input galaxies.
periodic : bool
Whether the catalog has periodic boundary condition.
boxsize : float
Boxsize of the mock data. Used when dealing with periodic
boundary conditions.
max_r : float
Only galaxies within max_r of central galaxy will be considered
in indentification.
background : float
Background galaxy density. If None, will assume V=boxsize**3 to
calculate.
'''
self.galaxy = Table(galaxy).copy()
self.ng = len(self.galaxy)
self.dproj = d_proj
self.periodic = periodic
self.boxsize = boxsize
self.max_r = max_r
self.sort_key = sort_key
self.galaxy.add_columns([np.ones(self.ng, dtype=float), np.arange(self.ng)], \
names=['pfree', 'ID'])
if central_idx is None:
self.init_central = self.galaxy.copy()
else:
self.init_central = self.galaxy[central_idx].copy()
self.init_central.remove_columns(['pfree'])
self.init_central.add_column(np.zeros(len(self.init_central)),
name='lambda')
self.init_central.sort(sort_key, reverse=True)
self.galaxy.add_index('ID')
if background is None:
self.background = self.ng / self.boxsize**3 * 2 * self.dproj
else:
self.background = background
def initial_run(self, init_aperture=0.5):
assigned = set()
N_init_c = len(self.init_central)
for i in range(N_init_c):
print('Initial run... Finished:{}; Total:{}'.format(i, N_init_c),
end='\r')
if self.init_central[i]['ID'] in assigned:
continue
temp_cluster = self.init_central[i]
temp_galaxy = self.galaxy[self.galaxy['pfree'] != 0]
temp_member = cylinder_filter(temp_galaxy, temp_cluster['pos'], \
init_aperture, self.dproj, periodic=self.periodic, boxsize=self.boxsize)
self.init_central['lambda'][i] = len(temp_member)
assigned = assigned | set(temp_member['ID'])
self.init_central = self.init_central[self.init_central['lambda'] >= 3]
self.init_central.sort('lambda', reverse=True)
self.init_central.add_index('ID')
def percolation(self, compare_key=None):
central_to_process = self.init_central.copy()
# Identify the first cluster
_central = central_to_process[0]
f_cat, f_central = self.identify_single(_central, compare_key)
f_central['cID'], f_cat['cID'] = 0, 0
self.final_central = [f_central]
self.final_member = [f_cat]
tmem_gal = self.galaxy.loc[f_cat['ID']]
tmem_gal['pfree'] = tmem_gal['pfree'] * (1 - f_cat['pmem'])
central_to_process.remove_row(0)
# Remove the potential centers having low pfree.
low_pfree_id = tmem_gal[tmem_gal['pfree'] < 0.5]['ID']
low_pfree_cluster = np.intersect1d(low_pfree_id,
central_to_process['ID'])
if len(low_pfree_cluster) > 0:
low_pfree_cidx = central_to_process.loc_indices[low_pfree_cluster]
central_to_process.remove_rows(low_pfree_cidx)
# Loop over the list and do percolation.
while len(central_to_process) > 0:
N_central = len(self.final_central)
print('Percolation... Finished:{}; Remaining:{}'.format(
N_central, len(central_to_process)),
end='\r')
_central = central_to_process[0]
f_cat, f_central = self.identify_single(_central, compare_key)
f_central['cID'], f_cat['cID'] = N_central, N_central
self.final_central.append(f_central)
self.final_member.append(f_cat)
tmem_gal = self.galaxy.loc[f_cat['ID']]
tmem_gal['pfree'] = tmem_gal['pfree'] * (1 - f_cat['pmem'])
central_to_process.remove_row(0)
low_pfree_id = tmem_gal[tmem_gal['pfree'] < 0.5]['ID']
low_pfree_cluster = np.intersect1d(low_pfree_id,
central_to_process['ID'])
if len(low_pfree_cluster) > 0:
low_pfree_cidx = central_to_process.loc_indices[
low_pfree_cluster]
central_to_process.remove_rows(low_pfree_cidx)
def identify_single(self, init_central, compare_key=None):
if compare_key is None:
compare_key = self.sort_key
# Only consider galaxies close enough to the central.
temp_central = Table(init_central)
temp_cat = cylinder_filter(self.galaxy, temp_central['pos'],
self.max_r, self.dproj)
temp_cat.add_column(0.0, name='pmem')
solver = lambda_solver(temp_cat['R'], temp_cat['pfree'],
self.background)
# Galaxy must be outside of Rc if its pmem=0 and vice versa.
temp_cat = temp_cat[solver['pmem'] > 0]
# Iteration until the central galaxy is the most dominant one within Rc.
while temp_cat[compare_key].max() != temp_central[compare_key]:
temp_central = Table(temp_cat[temp_cat[compare_key].argmax()])
temp_central.remove_columns(['pfree', 'R'])
temp_central.add_column(0.0, name='lambda')
temp_cat = cylinder_filter(self.galaxy, temp_central['pos'],
self.max_r, self.dproj)
temp_cat.add_column(0.0, name='pmem')
solver = lambda_solver(temp_cat['R'], temp_cat['pfree'],
self.background)
temp_cat = temp_cat[solver['pmem'] > 0]
temp_cat['pmem'] = solver['pmem'][solver['pmem'] > 0]
temp_central['lambda'] = solver['lambda']
temp_central.add_column(-1, name='cID')
temp_cat.add_column(-1, name='cID')
return temp_cat, temp_central
class halo_props:
'''
Systematically analyse the halo X-ray properties based
on other modules.
Attributes
-----------
datatype : str
A copy of the input type of simulation data.
catalogue_original : pynbody.halo.HaloCatalogue
The input halo catalogue.
length : length of the input catalogue.
host_id_of_top_level
How catalogue record "hostHalo" for those halos
without a host halo. Default is 0.
errorlist : list
Record when the host halo ID of a certain subhalo is not
recorded in the catalogue (weird but will happen in
ahf sometimes).
rho_crit
Critical density of the current snapshot in Msol kpc**-3.
ovdens
Virial overdensity factor :math:`\Delta_{vir}` of the current snapshot.
dict : astropy.table.Table
A copy of the halo.properties dictionary but in a table form
to make future reference more convenient.
haloid
List of halo_id given by property dictionary.
IDlist
Table of halo_id and corresponding #ID given in the property
dictionary.
hostid
List of the halo_id of the host halo of each halo (originally
recorded in the property dictionary in the form of #ID).
new_catalogue : dict
The new catalogue which includes all the subhalo particles
in its host halo. The keys of the dictionary are the indexes of
halos in `catalogue_original`.
prop
Table of quantities corresponding to input field.
host_list
List of host halos.
tophost
halo_ids of the top-level host halo for each halo.
children : list of sets
Each set corresponds to the one-level down children of each halo.
galaxy_list
List of all galaxies (as long as n_star > 0).
lumi_galaxy_list
List of all luminous galaxies (self_m_star > galaxy_low_limit).
galaxies : list of sets
Each set corresponds to the embeded galaxies of each halo. All
the subhalos will not be considered and will have an empty set.
And for host halos it will include all the galaxies within it,
including the galaxies actually embedded in the subhalo (i.e.,
the children of subhalo).
lumi_galaxies
Each set corresponds to the embeded luminous galaxies of each
halo. Same as `galaxies`, only care about host halo and include
all the luminous galaxies within.
n_lgal
Number of total luminous galaxies embedded in each halo. Again,
only care about host halos and the galaxies within subhalos
(i.e., subhalos themselves) will also be taken into account.
group_list
halo_id of the halo identified as group in the catalogue.
'''
def __init__(self,
halocatalogue,
datatype,
field=default_field,
host_id_of_top_level=0,
verbose=True):
'''
Initialization routine.
Input
-----
ahfcatalogue : pynbody.halo.HaloCatalogue
Only has been tested for pynbody.halo.AHFCatalogue
field
Quantities to calculate. When changing specific_mass_field,
luminosity_field and temp_field, source codes must be modified.
datatype : str
What kind of simulation data you are dealing with.
Accepted datatype for now: 'gizmo_ahf' and 'tipsy_ahf'.
host_id_of_top_level
How catalogue record "hostHalo" for those halos
without a host halo. Default is 0.
'''
self.datatype = datatype
self.catalogue_original = halocatalogue
self.length = len(self.catalogue_original)
init_zeros = np.zeros(self.length)
self.host_id_of_top_level = host_id_of_top_level
self.errorlist = [{}, {}, {}]
self.verbose = verbose
self.rho_crit = pnb.analysis.cosmology.rho_crit(
f=self.catalogue_original[1], unit='Msol kpc**-3')
self.ovdens = cosmology.Delta_vir(self.catalogue_original[1])
self.dict = []
k = 0
for j in range(self.length):
i = j + 1
prop = self.catalogue_original[i].properties
# currently pynbody can not load the *substructure files
# generated by MPI AHF correctly, so here we remove
# substructure info even if it can be successfully loaded
# with the none-MPI AHF files.
prop.pop('children', None)
prop.pop('parentid', None)
hid = prop['halo_id']
if i != hid:
raise Exception('Attention! halo_id doesn\'t equal i !!!')
self.dict.append(prop)
if ((i // 100) != (k // 100)) and self.verbose:
print('Loading properties... {:7} / {}'.format(i, self.length),
end='\r')
k = i
self.dict = Table(self.dict)
self.haloid = self.dict['halo_id']
IDs = self.dict['#ID']
self.ID_list = Table([IDs, self.haloid], names=['#ID', 'halo_id'])
self.ID_list.add_row([host_id_of_top_level, host_id_of_top_level])
self.ID_list.add_index('#ID')
host_in_IDlist = np.isin(self.dict['hostHalo'], self.ID_list['#ID'])
# Some hostHalo id will not be listed in #ID list, this is probably due to AHF algorithm
in_idx, = np.where(host_in_IDlist)
_not_in_ = np.invert(host_in_IDlist)
not_in_idx, = np.where(_not_in_)
self.hostid = np.zeros(self.length, dtype=np.int)
self.hostid[in_idx] = self.ID_list.loc[self.dict['hostHalo']
[in_idx]]['halo_id']
self.hostid = np.ma.array(self.hostid, dtype=np.int, mask=_not_in_)
# loc method enables using #ID as index
if len(not_in_idx) > 0:
for error in not_in_idx:
self.errorlist[0][
self.haloid[error]] = self.dict['hostHalo'][error]
# prop initialization
self.prop = {}
for field_type in default_units:
init_prop_table = [
init_zeros for _ in range(len(field[field_type]))
]
self.prop[field_type] = Table(init_prop_table,
names=field[field_type])
# astropy.table.Table is only used for generating a structured array more conveniently
self.prop[field_type] = pnb.array.SimArray(
self.prop[field_type], units=default_units[field_type])
self.field = default_field
self.field_units = default_units
self._have_children = False
self._have_galaxy = False
self._have_group = False
self._have_radii = False
self._have_temp = False
self._have_new_catalogue = False
self._have_center = False
def init_relationship(self,
galaxy_low_limit,
include_sub=False,
galaxy_mode='only stellar',
N_galaxy=3):
'''
Get basic information regarding groups, hosts, children, etc.
Parameters
------------
galaxy_low_limit : pynbody.array.SimArray
Required by get_galaxy(). Limit above which galaxies will
be identified as luminous galaxies.
include_sub
Whether or not to include all the subhalo particles when
generating the new catalogue. See get_new_catalogue() for
details.
N_galaxy : int
Required by get_group_list(). Number of luminous galaxies
above which host halos are considered as groups.
'''
self.get_children()
self.get_new_catalogue(include_=include_sub)
self.get_galaxy(g_low_limit=galaxy_low_limit, mode=galaxy_mode)
self.get_group_list(N_galaxy)
self.get_center()
def calcu_radii_masses(self,
halo_id_list=[],
rdict=None,
precision=1e-2,
rmax=None):
'''
Calculate radii (Rvir, R200, etc) and corresponding masses.
Parameters
-----------
halo_id_list
List of halo_ids to calculate radii and masses.
If set to empty list, then will use self.group_list.
rdict : dict
names and values for overdensity factors. Default is:
{'vir': self.ovdens, '200': 200, '500': 500, '2500': 2500}
precision : float
Precision for calculate radius. See get_index() in
calculate_R.py documentation for detail.
rmax
Maximum value for the shrinking sphere method. See
get_index() in calculate_R.py documentation for detail.
'''
halo_id_list = np.array(halo_id_list, dtype=np.int).reshape(-1)
if len(halo_id_list) == 0:
if not self._have_group:
raise Exception(
'Must get_group_list (or init_relationship) first!')
halo_id_list = self.group_list
if not self._have_center:
raise Exception('Must get_center first!')
if rdict == None:
rdict = {'vir': self.ovdens, '200': 200, '500': 500, '2500': 2500}
t1 = 0
t2 = 0
list_length = np.array(list(halo_id_list)).max()
k = 0
for j in halo_id_list:
i = j - 1
prop = self.dict[i]
t1 = time.time()
MassRadii = cR.get_radius(self.new_catalogue[j], \
overdensities=list(rdict.values()), rho_crit=self.rho_crit, \
prop=prop, precision=precision, cen=self.center[i], rmax=rmax)
for key in rdict:
self.prop['R'][key][i] = MassRadii[1][rdict[key]]
self.prop['M'][key][i] = MassRadii[0][rdict[key]]
t2 = time.time()
if ((i // 100) != (k // 100)) and self.verbose:
print('Calculating radii and masses... {:7} / {}, time: \
{:.5f}s'.format(j, list_length, t2 - t1),
end='\r')
k = i
self._have_radii = True
def calcu_specific_masses(self, halo_id_list=[], \
calcu_field=radii_to_cal_sepcific_mass, \
temp_cut='5e5 K', nh_cut='0.13 cm**-3'):
'''
Calculate some specific masses, such as baryon, IGrM, etc.
Parameters
-----------
halo_id_list
List of halo_ids to calculate masses.
If set to empty list, then will use self.group_list.
calcu_field
Radii to calculate specific masses within.
temp_cut
Temperature limit above which gas particles are
considered as hot.
nh_cut
nh limit above which gas particles are considered
as star forming.
'''
halo_id_list = np.array(halo_id_list, dtype=np.int).reshape(-1)
if len(halo_id_list) == 0:
if not self._have_group:
raise Exception(
'Must get_group_list (or init_relationship) first!')
halo_id_list = self.group_list
if not self._have_radii:
raise Exception('Must get_radii_masses first!')
list_length = np.array(list(halo_id_list)).max()
k = 0
for j in halo_id_list:
i = j - 1
prop = self.dict[i]
center = self.center[i]
halo = self.new_catalogue[j]
tx = pnb.transformation.inverse_translate(halo, center)
with tx:
boxsize = halo.properties['boxsize'].in_units('kpc')
original_pos = halo['pos'].copy()
halo['pos'] = correct_pos(halo['pos'], boxsize)
for r in calcu_field:
# Apply filters
subsim = halo[pnb.filt.Sphere(
self.prop['R'][i:i + 1][r].in_units('kpc'))]
cold_diffuse_gas = subsim.gas[pnb.filt.LowPass('temp', temp_cut) & \
pnb.filt.LowPass('nh', nh_cut)]
ISM = subsim.gas[pnb.filt.HighPass('nh', nh_cut)]
hot_diffuse_gas_ = subsim.gas[pnb.filt.HighPass('temp', temp_cut) & \
pnb.filt.LowPass('nh', nh_cut)]
# Calculate masses
self.prop['M']['star' + r][i] = subsim.star['mass'].sum()
self.prop['M']['gas' + r][i] = subsim.gas['mass'].sum()
self.prop['M']['bar' + r][i] = self.prop['M']['star' + r][i] \
+ self.prop['M']['gas' + r][i]
self.prop['M']['ism' + r][i] = ISM['mass'].sum()
self.prop['M']['cold' + r][i] = cold_diffuse_gas['mass'].sum() \
+ self.prop['M']['ism' + r][i]
self.prop['M']['igrm' +
r][i] = hot_diffuse_gas_['mass'].sum()
halo['pos'] = original_pos
if ((i // 100) != (k // 100)) and self.verbose:
print('Calculating specific masses... {:7} / {}'.format(
j, list_length),
end='\r')
k = i
def calcu_temp_lumi(self, cal_file, halo_id_list=[], \
core_corr_factor=0.15, calcu_field='500', \
temp_cut='5e5 K', nh_cut='0.13 cm**-3', \
additional_filt=None):
'''
Calculate all the temperatures and luminosities listed in
temp_field and luminosity_field.
Parameters
-----------
cal_file
Calibration file used for calculating Tspec.
halo_id_list
List of halo_ids to calculate temperatures
and luminosities. If set to empty list, then will use
self.group_list.
core_corr_factor
Inner radius for calculating core-corrected
temperatures. Gas particles within
(core_corr_factor*R, R) will be used for calculation.
calcu_field
Radius to calculate temperatures and luminosities
within. Must be in radius_field. Default: R_500.
temp_cut
Temperature limit above which gas particles are
considered as hot.
nh_cut
nh limit above which gas particles are considered
as star forming.
additional_filt
Any additional filter used to constrain the hot diffuse
gas we are investigating.
'''
halo_id_list = np.array(halo_id_list, dtype=np.int).reshape(-1)
if len(halo_id_list) == 0:
if not self._have_group:
raise Exception(
'Must get_group_list (or init_relationship) first!')
halo_id_list = self.group_list
if not self._have_radii:
raise Exception('Must get_radii_masses first!')
list_length = np.array(list(halo_id_list)).max()
k = 0
for j in halo_id_list:
i = j - 1
center = self.center[i]
halo = self.new_catalogue[j]
R = self.prop['R'][i:i + 1][calcu_field].in_units('kpc')
tx = pnb.transformation.inverse_translate(halo, center)
with tx:
boxsize = halo.properties['boxsize'].in_units('kpc')
original_pos = halo['pos'].copy()
halo['pos'] = correct_pos(halo['pos'], boxsize)
subsim = halo[pnb.filt.Sphere(R)]
if additional_filt is None:
hot_diffuse_filt = pnb.filt.HighPass('temp', temp_cut) & \
pnb.filt.LowPass('nh', nh_cut)
else:
hot_diffuse_filt = pnb.filt.HighPass('temp', temp_cut) & \
pnb.filt.LowPass('nh', nh_cut) & additional_filt
hot_diffuse_gas_ = subsim.gas[hot_diffuse_filt]
# cal_tweight can return the sum of weight_type at the same time.
self.prop['T']['x'][i], self.prop['L']['x'][i] = \
cal_tweight(hot_diffuse_gas_, weight_type='Lx')
self.prop['T']['x_cont'][i], self.prop['L']['x_cont'][i] = \
cal_tweight(hot_diffuse_gas_, weight_type='Lx_cont')
self.prop['T']['mass'][i], _ = cal_tweight(hot_diffuse_gas_,
weight_type='mass')
self.prop['T']['spec'][i] = pnb.array.SimArray(cal_tspec(hot_diffuse_gas_, \
cal_f=cal_file, datatype=self.datatype), units='keV')
self.prop['T']['xb'][i], self.prop['L']['xb'][i] = \
cal_tweight(hot_diffuse_gas_, weight_type='Lxb')
self.prop['L']['xb_cont'][i] = hot_diffuse_gas_[
'Lxb_cont'].sum()
# Core-corrected temperatures:
# Filter:
corr_hot_ = hot_diffuse_gas_[~pnb.filt.
Sphere(core_corr_factor * R)]
self.prop['T']['spec_corr'][i] = pnb.array.SimArray(cal_tspec(corr_hot_, \
cal_f=cal_file, datatype=self.datatype), units='keV')
self.prop['T']['x_corr'][i], self.prop['L']['x_corr'][
i] = cal_tweight(corr_hot_, weight_type='Lx')
self.prop['T']['xb_corr'][i], self.prop['L']['xb_corr'][
i] = cal_tweight(corr_hot_, weight_type='Lxb')
self.prop['T']['x_corr_cont'][i], _ = \
cal_tweight(corr_hot_, weight_type='Lx_cont')
self.prop['T']['mass_corr'][i], _ = cal_tweight(
corr_hot_, weight_type='mass')
halo['pos'] = original_pos
if ((i // 100) != (k // 100)) and self.verbose:
print('Calculating temperatures and luminosities... {:7} / {}'\
.format(j, list_length), end='\r')
k = i
self._have_temp = True
def calcu_entropy(self, cal_file, n_par=9, halo_id_list=[], \
calcu_field=entropy_field, thickness=0.05, volume_type='full', \
temp_cut='5e5 K', nh_cut='0.13 cm**-3', additional_filt=None):
'''
Calculate all entropy within a thin spherical shell
centered at halo.
Parameters
-----------
cal_file
Calibration file used for calculating Tspec.
n_par : int
Number of particles the shell must contain,
below which entropy will not be calculated.
halo_id_list
List of halo_ids to calculate entropies.
If set to empty list, then will use self.group_list.
calcu_field
Radii of the thin shell to calculate entropies.
thickness : float
Thickness Devided by radius of the spherical shell, i.e.,
the shell will be R~(1+thickness)*R.
volume : str
Volume used for calculating average electron number
density. 'gas' means only using the sum over the volumes
of all hot diffuse gas particles. 'full' means to use
4*pi*R^2*dR.
temp_cut
Temperature limit above which gas particles are
considered as hot.
nh_cut
nh limit above which gas particles are considered
as star forming.
additional_filt
Any additional filter used to constrain the hot diffuse
gas we are investigating.
'''
# thickness = pnb.array.SimArray(thickness, 'kpc')
halo_id_list = np.array(halo_id_list, dtype=np.int).reshape(-1)
if len(halo_id_list) == 0:
if not self._have_group:
raise Exception(
'Must get_group_list (or init_relationship) first!')
halo_id_list = self.group_list
if not self._have_radii:
raise Exception('Must get_radii_masses first!')
list_length = np.array(list(halo_id_list)).max()
k = 0
for j in halo_id_list:
i = j - 1
center = self.center[i]
halo = self.new_catalogue[j]
tx = pnb.transformation.inverse_translate(halo, center)
with tx:
boxsize = halo.properties['boxsize'].in_units('kpc')
original_pos = halo['pos'].copy()
halo['pos'] = correct_pos(halo['pos'], boxsize)
for r in calcu_field:
R = self.prop['R'][i:i + 1][r].in_units('kpc')
subgas = halo.gas[pnb.filt.Annulus(R, (thickness + 1) * R)]
if additional_filt is None:
hot_diffuse_filt = pnb.filt.HighPass('temp', temp_cut) & \
pnb.filt.LowPass('nh', nh_cut)
else:
hot_diffuse_filt = pnb.filt.HighPass('temp', temp_cut) & \
pnb.filt.LowPass('nh', nh_cut) & additional_filt
hot_diffuse_gas_ = subgas[hot_diffuse_filt]
if len(hot_diffuse_gas_) < n_par:
self.prop['S'][r][i] = np.nan
self.prop['T']['spec' + r][i] = np.nan
else:
tempTspec = pnb.array.SimArray(cal_tspec(hot_diffuse_gas_, \
cal_f=cal_file, datatype=self.datatype), units='keV')
if volume_type == 'gas':
temp_volume = hot_diffuse_gas_['volume'].sum()
elif volume_type == 'full':
temp_volume = 4 / 3 * np.pi * ((
(thickness + 1) * R)**3 - R**3)
elif volume_type == 'full_approx':
temp_volume = 4 * np.pi * R**2 * thickness * R
else:
raise Exception("volume_type is not accepted!")
avg_ne = ((hot_diffuse_gas_['ne'] * hot_diffuse_gas_['volume']).sum() \
/ temp_volume).in_units('cm**-3')
avg_nh = ((hot_diffuse_gas_['nh'] * hot_diffuse_gas_['volume']).sum() \
/ temp_volume).in_units('cm**-3')
self.prop['T']['spec' + r][i] = tempTspec
self.prop['ne'][r][i] = avg_ne
self.prop['nh'][r][i] = avg_nh
self.prop['S'][r][i] = tempTspec / (avg_ne)**(2, 3)
halo['pos'] = original_pos
if ((i // 100) != (k // 100)) and self.verbose:
print(' Calculating entropies... {:7} / {}'\
.format(j, list_length), end='\r')
k = i
def calcu_metallicity(self, halo_id_list=[], elements=['H', 'O', 'Si', 'Fe'], \
radii=['500'], temp_cut='5e5 K', nh_cut='0.13 cm**-3', \
additional_filt=None, weight_types=['mass', 'Lx']):
if self.datatype[:5] == 'gizmo':
self.metal_idx = {'He': 1, 'C': 2, 'N': 3, 'O': 4, \
'Ne': 5, 'Mg': 6, 'Si': 7, 'S': 8, 'Ca': 9, 'Fe': 10}
else:
raise Exception('Currently only support GIZMO.')
init_zeros = np.zeros(self.length)
field_names = []
for ele in elements:
for rad in radii:
for weight_type in weight_types:
field_names.append('Z_' + ele + rad + weight_type)
init_prop_table = Table([init_zeros for _ in range(len(field_names))])
self.prop['metals'] = Table(init_prop_table, names=field_names)
self.prop['metals'] = pnb.array.SimArray(self.prop['metals'],
units='cm**-3')
self.field['metals'] = field_names
self.field_units['metals'] = 'cm**-3'
halo_id_list = np.array(halo_id_list, dtype=np.int).reshape(-1)
if len(halo_id_list) == 0:
if not self._have_group:
raise Exception(
'Must get_group_list (or init_relationship) first!')
halo_id_list = self.group_list
if not self._have_radii:
raise Exception('Must get_radii_masses first!')
list_length = np.array(list(halo_id_list)).max()
k = 0
for j in halo_id_list:
i = j - 1
center = self.center[i]
halo = self.new_catalogue[j]
tx = pnb.transformation.inverse_translate(halo, center)
with tx:
boxsize = halo.properties['boxsize'].in_units('kpc')
original_pos = halo['pos'].copy()
halo['pos'] = correct_pos(halo['pos'], boxsize)
for r in radii:
R = self.prop['R'][i:i + 1][r].in_units('kpc')
subgas = halo.gas[pnb.filt.Sphere(R)]
if additional_filt is None:
hot_diffuse_filt = pnb.filt.HighPass('temp', temp_cut) & \
pnb.filt.LowPass('nh', nh_cut)
else:
hot_diffuse_filt = pnb.filt.HighPass('temp', temp_cut) & \
pnb.filt.LowPass('nh', nh_cut) & additional_filt
igrm = subgas[hot_diffuse_filt]
for weight_type in weight_types:
weight_sum = igrm[weight_type].sum()
for ele in elements:
if ele != 'H':
# gas_nx = g_p.n_X(igrm['rho'], igrm['metals'][:, self.metal_idx[ele]], ele)
totZx = (
igrm['metals'][:, self.metal_idx[ele]] *
igrm[weight_type]).sum()
else:
# totNx = (igrm['nh'] * igrm[weight_type]).sum()
totZx = (igrm['X_H'] * igrm[weight_type]).sum()
self.prop['metals']['Z_' + ele + r + weight_type][i] = \
(totZx/weight_sum)
halo['pos'] = original_pos
if ((i // 100) != (k // 100)) and self.verbose:
print(' Calculating metallicities... {:7} / {}'\
.format(j, list_length), end='\r')
k = i
def savedata(self, filename, field=None, halo_id_list=[], units=None):
'''
Save the data in hdf5 format. Will save halo_id_list
(key: 'halo_id') and the quantities listed in field.
Parameters
-----------
filename
Filename of the hdf5 file.
field
Type of information to save.
halo_id_list
List of halo_ids to save.If set to empty list,
then will use self.group_list.
units
Convert the data into specified inits and save.
'''
if field is None:
field = self.field
if units is None:
field_units = self.field_units
halo_id_list = np.array(halo_id_list, dtype=np.int).reshape(-1)
if len(halo_id_list) == 0:
halo_id_list = self.group_list
with h5py.File(filename, "w") as f:
dataset = f.create_dataset("halo_id", data=halo_id_list)
dataset.attrs[
'Description'] = 'halo_ids of halos saved in this file.'
dataset2 = f.create_dataset("N_lgal",
data=self.n_lgal[halo_id_list - 1])
dataset2.attrs['Description'] = 'Number of luminous galaxies'
for attr in field:
grp = f.create_group(attr)
infos = field[attr]
for info in infos:
data_to_save = self.prop[attr][info][halo_id_list - 1]
data_to_save.convert_units(field_units[attr])
dset = grp.create_dataset(info, data=data_to_save)
dset.attrs['units'] = str(data_to_save.units)
def get_children(self):
'''
Generate list of children (subhalos) for each halo.
Subhalo itself can also have children. And this list
will not contain "grandchildren" (i.e., the children
of children).
'''
self.host_list = []
self.tophost = np.zeros(self.length).astype(np.int)
self.children = [set() for _ in range(self.length)]
k = 0
for i in range(self.length):
j = self.haloid[i] #j = i + 1
if ((j // 100) != (k // 100)) and self.verbose:
print('Generating children list... Halo: {:7} / {}'.format(
j, self.length),
end='\r')
k = j
prop = self.dict[i]
hostID = prop['hostHalo']
if j in self.errorlist[0]:
self.errorlist[1][j] = hostID
continue
try:
if hostID == self.host_id_of_top_level:
self.host_list.append(j)
self.tophost[i] = j
else:
if hostID < 0:
print(
'Make sure you\'ve used the correct host ID of the top-level halos!'
)
host_haloid = self.ID_list.loc[hostID]['halo_id']
self.children[host_haloid - 1].add(j)
temphost = j
while temphost != self.host_id_of_top_level:
temphost2 = temphost
temphost = self.hostid[temphost - 1]
self.tophost[i] = temphost2
except IndexError:
self.errorlist[1][j] = hostID
self._have_children = True
def get_new_catalogue(self, include_):
'''
Generate a new catalogue based on catalogue_original,
the new catalogue will include all the subhalo particles
in its host halo.
Parameters
-------------
include_ : bool
If True, then will include all the subhalo particles.
Otherwise will just be a copy of catalogue_original.
'''
if not self._have_children:
raise Exception('Must get_children first!')
if include_:
self.new_catalogue = {}
k = 0
for i in range(self.length):
j = self.haloid[i]
if ((i // 100) != (k // 100)) and self.verbose:
print('Generating new catalogue... Halo: {:7} / {}'.format(
j, self.length),
end='\r')
k = i
if len(self.children[i]) == 0:
self.new_catalogue[j] = self.catalogue_original[j]
else:
union_list = [j] + list(self.children[i])
self.new_catalogue[j] = get_union(self.catalogue_original,
union_list)
else:
self.new_catalogue = self.catalogue_original
self._have_new_catalogue = True
def get_galaxy(self, g_low_limit, mode='only stellar'):
'''
Generate list of galaxies for each host halo. The subsubhalo
will also be included in the hosthalo galaxy list. And it won't
generate list for the subhalos even if there are galaxies within.
Parameters
-------------
g_low_limit : pynbody.array.SimArray
Limit above which galaxies will be identified as luminous
galaxies.
'''
if not self._have_children:
raise Exception('Must get_children first!')
if not self._have_new_catalogue:
raise Exception('Must get_new_catalogue first!')
self.galaxy_list = [] # List of all galaxies (as long as n_star > 0).
self.lumi_galaxy_list = [
] # List of all luminous galaxies (self_m_star > galaxy_low_limit).
self.galaxies = [set() for _ in range(self.length)]
self.lumi_galaxies = [set() for _ in range(self.length)]
self.n_lgal = np.zeros(
self.length
) # Number of total luminous galaxies embedded in each host halo.
# The galaxies within subhalos (i.e., subhalos themselves) will also be taken into account.
k = 0
for i in range(self.length):
j = self.haloid[i]
if ((i // 100) != (k // 100)) and self.verbose:
print('Calculating total stellar masses... Halo: {:7} / {}'.
format(j, self.length),
end='\r')
k = i
self.prop['M']['total_star'][i] = self.new_catalogue[j].star[
'mass'].sum()
sf_gas = self.new_catalogue[j].gas[pnb.filt.LowPass(
'temp', '3e4 K')]
# sf_gas = self.new_catalogue[j].gas[pnb.filt.HighPass('nh', '0.13 cm**-3')]
self.prop['M']['total_sfgas'][i] = sf_gas['mass'].sum()
# sf_gas, i.e., star forming gas, is used in the definition of resolved galaxies in Liang's Figure2.
# But seems that Liang didn't plot Figure 2 using the concept of resolved galaxies.
low_limit = g_low_limit.in_units(self.prop['M']['total_star'].units)
k = 0
for i in range(self.length):
j = self.haloid[i]
if ((i // 100) != (k // 100)) and self.verbose:
print(' Identifying galaxies... Halo: {:7} / {}'.
format(j, self.length),
end='\r')
k = i
children_list = np.array(list(self.children[i]))
if len(children_list) == 0:
self_Mstar = self.prop['M']['total_star'][i]
# if mode == 'include cold gas':
self_Msfgas = self.prop['M']['total_sfgas'][i]
else:
children_union = get_union(self.new_catalogue,
list(children_list))
children_union_within_ = self.new_catalogue[j].intersect(
children_union)
self_Mstar = self.prop['M']['total_star'][
i] - children_union_within_.star['mass'].sum()
# if mode == 'include cold gas':
sf_gas_union = children_union_within_.gas[pnb.filt.LowPass(
'temp', '3e4 K')]
# sf_gas_union = children_union.gas[pnb.filt.HighPass('nh', '0.13 cm**-3')]
self_Msfgas = self.prop['M']['total_sfgas'][i] - sf_gas_union[
'mass'].sum()
self.prop['M']['self_star'][i] = self_Mstar
self.prop['M']['self_sfgas'][i] = self_Msfgas
try:
if mode == 'only stellar':
condition = (self_Mstar > 0)
elif mode == 'include cold gas':
condition = (self_Mstar + self_Msfgas > low_limit)
if condition:
self.galaxy_list.append(j)
temp_tophost = self.tophost[i]
self.galaxies[temp_tophost - 1].add(j)
if self_Mstar > low_limit:
self.lumi_galaxy_list.append(j)
self.n_lgal[temp_tophost - 1] += 1
self.lumi_galaxies[temp_tophost - 1].add(j)
except KeyError:
self.errorlist[2][j] = self.dict['hostHalo'][i]
self._have_galaxy = True
def get_group_list(self, N_galaxy):
'''
halo_id of the halo identified as group in the catalogue.
Parameters
-----------
N_galaxy : int
Number of luminous galaxies above which host halos
are considered as groups.
'''
if not self._have_galaxy:
raise Exception('Must get_galaxy first!')
self.group_list, = np.where(self.n_lgal >= N_galaxy)
self.group_list += 1
self._have_group = True
def calcu_tx_lx(self, halo_id_list=[], \
core_corr_factor=0.15, calcu_field='500', \
temp_cut='5e5 K', nh_cut='0.13 cm**-3', additional_filt=None):
'''
Calculate X-ray luminosities and emission weighted
temperatures listed in temp_field and luminosity_field.
Parameters
-----------
halo_id_list
List of halo_ids to calculate temperatures on.
If set to empty list, then will use self.group_list.
core_corr_factor
Inner radius for calculating core-corrected
temperatures. Gas particles within
(core_corr_factor*R, R) will be used for calculation.
calcu_field
Radius to calculate temperatures and luminosities
within. Must be in radius_field. Default: R_500.
temp_cut
Temperature limit above which gas particles are
considered as hot.
nh_cut
nh limit above which gas particles are considered
as star forming.
additional_filt
Any additional filter used to constrain the hot diffuse
gas we are investigating.
'''
halo_id_list = np.array(halo_id_list, dtype=np.int).reshape(-1)
if len(halo_id_list) == 0:
if not self._have_group:
raise Exception(
'Must get_group_list (or init_relationship) first!')
halo_id_list = self.group_list
if not self._have_radii:
raise Exception('Must get_radii_masses first!')
if not self._have_new_catalogue:
raise Exception('Must get_new_catalogue first!')
list_length = np.array(list(halo_id_list)).max()
for j in halo_id_list:
i = j - 1
if self.verbose:
print('Calculating temperatures and luminosities... {:7} / {}'\
.format(j, list_length), end='\r')
center = self.center[i]
halo = self.new_catalogue[j]
R = self.prop['R'][i:i + 1][calcu_field].in_units('kpc')
tx = pnb.transformation.inverse_translate(halo, center)
with tx:
boxsize = halo.properties['boxsize'].in_units('kpc')
original_pos = halo['pos'].copy()
halo['pos'] = correct_pos(halo['pos'], boxsize)
subsim = halo[pnb.filt.Sphere(R)]
if additional_filt is None:
hot_diffuse_filt = pnb.filt.HighPass('temp', temp_cut) & \
pnb.filt.LowPass('nh', nh_cut)
else:
hot_diffuse_filt = pnb.filt.HighPass('temp', temp_cut) & \
pnb.filt.LowPass('nh', nh_cut) & additional_filt
hot_diffuse_gas_ = subsim.gas[hot_diffuse_filt]
# cal_tweight can return the sum of weight_type at the same time.
self.prop['T']['x'][i], self.prop['L']['x'][i] = \
cal_tweight(hot_diffuse_gas_, weight_type='Lx')
self.prop['T']['x_cont'][i], self.prop['L']['x_cont'][i] = \
cal_tweight(hot_diffuse_gas_, weight_type='Lx_cont')
# Core-corrected temperatures:
# Filter:
corr_hot_ = hot_diffuse_gas_[~pnb.filt.
Sphere(core_corr_factor * R)]
self.prop['T']['x_corr'][i], _ = cal_tweight(corr_hot_,
weight_type='Lx')
self.prop['T']['x_corr_cont'][i], _ = \
cal_tweight(corr_hot_, weight_type='Lx_cont')
halo['pos'] = original_pos
def calcu_tspec(self, cal_file, halo_id_list=[], \
core_corr_factor=0.15, calcu_field='500', temp_cut='5e5 K', \
nh_cut='0.13 cm**-3', additional_filt=None):
'''
Calculate spectroscopic temperatures based on Douglas's
pytspec module.
Parameters
-----------
cal_file
Calibration file used for calculating Tspec.
halo_id_list
List of halo_ids to calculate temperatures and
luminosities. If set to empty list, then will use
self.group_list.
core_corr_factor
Inner radius for calculating core-corrected temperatures.
Gas particles within (core_corr_factor*R, R) will be used
for calculation.
calcu_field
Radius to calculate temperatures and luminosities within.
Must be in radius_field. Default: R_500.
temp_cut
Temperature limit above which gas particles are
considered as hot.
nh_cut
nh limit above which gas particles are considered
as star forming.
additional_filt
Any additional filter used to constrain the hot diffuse
gas we are investigating.
'''
halo_id_list = np.array(halo_id_list, dtype=np.int).reshape(-1)
if len(halo_id_list) == 0:
if not self._have_group:
raise Exception(
'Must get_group_list (or init_relationship) first!')
halo_id_list = self.group_list
if not self._have_radii:
raise Exception('Must get_radii_masses first!')
if not self._have_new_catalogue:
raise Exception('Must get_new_catalogue first!')
list_length = np.array(list(halo_id_list)).max()
for j in halo_id_list:
i = j - 1
if self.verbose:
print('Calculating spectroscopic temperatures... {:7} / {}'\
.format(j, list_length), end='\r')
center = self.center[i]
halo = self.new_catalogue[j]
R = self.prop['R'][i:i + 1][calcu_field].in_units('kpc')
tx = pnb.transformation.inverse_translate(halo, center)
with tx:
boxsize = halo.properties['boxsize'].in_units('kpc')
original_pos = halo['pos'].copy()
halo['pos'] = correct_pos(halo['pos'], boxsize)
subsim = halo[pnb.filt.Sphere(R)]
if additional_filt is None:
hot_diffuse_filt = pnb.filt.HighPass('temp', temp_cut) & \
pnb.filt.LowPass('nh', nh_cut)
else:
hot_diffuse_filt = pnb.filt.HighPass('temp', temp_cut) & \
pnb.filt.LowPass('nh', nh_cut) & additional_filt
hot_diffuse_gas_ = subsim.gas[hot_diffuse_filt]
self.prop['T']['spec'][i] = pnb.array.SimArray(cal_tspec(hot_diffuse_gas_, \
cal_f=cal_file, datatype=self.datatype), units='keV')
# Core-corrected temperatures:
# Filter:
corr_hot_ = hot_diffuse_gas_[~pnb.filt.
Sphere(core_corr_factor * R)]
self.prop['T']['spec_corr'][i] = pnb.array.SimArray(cal_tspec(corr_hot_, \
cal_f=cal_file, datatype=self.datatype), units='keV')
halo['pos'] = original_pos
def get_center(self):
'''
Calculate the center of the halos if an ahfcatalogue is
provided, then will automatically load the results in ahf.
Otherwise it will try to calculate the center coordinates
via gravitional potential or center of mass.
Notes
------
Due to a bug in pynbody, calculating center of mass will
lead to an incorrect result for the halos crossing the
periodical boundary of the simulation box. Make sure pynbody
has fixed it before you use.
'''
if self.datatype[-4:] == '_ahf':
axes = ['Xc', 'Yc', 'Zc']
tempcen = {}
for axis in axes:
tempcen[axis] = np.asarray(self.dict[axis],
dtype=float).reshape(-1, 1)
self.center = np.concatenate(
(tempcen['Xc'], tempcen['Yc'], tempcen['Zc']), axis=1)
self.center = pnb.array.SimArray(
self.center,
units='kpc') * self.dict['a'][0] / self.dict['h'][0]
if self.datatype == 'tipsy_ahf':
self.center -= self.dict['boxsize'][0].in_units('kpc') / 2
else:
self.center = pnb.array.SimArray(np.zeros((self.length, 3)),
units='kpc')
if 'phi' in self.new_catalogue[1].loadable_keys():
center_mode = 'pot'
else:
center_mode = 'com'
for i in range(self.length):
j = self.haloid[i]
print('Calculating center... {:7} / {}'.format(j, self.length),
end='\r')
self.center[i] = pnb.analysis.halo.center(self.new_catalogue[j], \
mode=center_mode, retcen=True, vel=False)
self._have_center = True
def make_todo(input_folder=None,
cameras=None,
ccds=None,
overwrite=False,
find_secondary_targets=True,
output_file=None):
"""
Create the TODO list which is used by the pipeline to keep track of the
targets that needs to be processed.
Will create the file `todo.sqlite` in the directory.
Parameters:
input_folder (string, optional): Input folder to create TODO list for.
If ``None``, the input directory in the environment variable ``TESSPHOT_INPUT`` is used.
cameras (iterable of integers, optional): TESS camera number (1-4). If ``None``, all cameras will be included.
ccds (iterable of integers, optional): TESS CCD number (1-4). If ``None``, all cameras will be included.
overwrite (boolean): Overwrite existing TODO file. Default=``False``.
find_secondary_targets (boolean): Should secondary targets from TPFs be included? Default=True.
output_file (string, optional): The file path where the output file should be saved.
If not specified, the file will be saved into the input directory.
Should only be used for testing, since the file would (proberly) otherwise end up with
a wrong file name for running with the rest of the pipeline.
Raises:
NotADirectoryError: If the specified ``input_folder`` is not an existing directory.
.. codeauthor:: Rasmus Handberg <*****@*****.**>
"""
logger = logging.getLogger(__name__)
# Check the input folder, and load the default if not provided:
if input_folder is None:
input_folder = os.environ.get(
'TESSPHOT_INPUT',
os.path.join(os.path.dirname(__file__), 'tests', 'input'))
# Check that the given input directory is indeed a directory:
if not os.path.isdir(input_folder):
raise NotADirectoryError(
"The given path does not exist or is not a directory")
# Make sure cameras and ccds are iterable:
cameras = (1, 2, 3, 4) if cameras is None else (cameras, )
ccds = (1, 2, 3, 4) if ccds is None else (ccds, )
# The TODO file that we want to create. Delete it if it already exits:
if output_file is None:
todo_file = os.path.join(input_folder, 'todo.sqlite')
else:
output_file = os.path.abspath(output_file)
if not output_file.endswith('.sqlite'):
output_file = output_file + '.sqlite'
todo_file = output_file
if os.path.exists(todo_file):
if overwrite:
os.remove(todo_file)
else:
logger.info("TODO file already exists")
return
# Number of threads available for parallel processing:
threads_max = int(
os.environ.get('SLURM_CPUS_PER_TASK', multiprocessing.cpu_count()))
# Load file with targets to be excluded from processing for some reason:
exclude_file = os.path.join(os.path.dirname(__file__), 'data',
'todolist-exclude.dat')
exclude = np.genfromtxt(exclude_file,
usecols=(0, 1, 2),
dtype=None,
encoding='utf-8')
exclude = set([tuple(e) for e in exclude])
# Create the TODO list as a table which we will fill with targets:
cat = Table(names=('starid', 'sector', 'camera', 'ccd', 'datasource',
'tmag', 'cbv_area', 'edge_dist'),
dtype=('int64', 'int32', 'int32', 'int32', 'S256', 'float32',
'int32', 'float32'))
# Load list of all Target Pixel files in the directory:
tpf_files = find_tpf_files(input_folder)
logger.info("Number of TPF files: %d", len(tpf_files))
if len(tpf_files) > 0:
# Open a pool of workers:
logger.info("Starting pool of workers for TPFs...")
threads = min(
threads_max, len(tpf_files)
) # No reason to use more than the number of jobs in total
logger.info("Using %d processes.", threads)
if threads > 1:
pool = multiprocessing.Pool(threads)
m = pool.imap_unordered
else:
m = map
# Run the TPF files in parallel:
tic = default_timer()
_tpf_todo_wrapper = functools.partial(
_tpf_todo,
input_folder=input_folder,
cameras=cameras,
ccds=ccds,
find_secondary_targets=find_secondary_targets,
exclude=exclude)
for cat2 in m(_tpf_todo_wrapper, tpf_files):
cat = vstack([cat, cat2], join_type='exact')
if threads > 1:
pool.close()
pool.join()
# Amount of time it took to process TPF files:
toc = default_timer()
logger.info("Elaspsed time: %f seconds (%f per file)", toc - tic,
(toc - tic) / len(tpf_files))
# Remove secondary TPF targets if they are also the primary target:
indx_remove = np.zeros(len(cat), dtype='bool')
cat.add_index('starid')
for k, row in enumerate(cat):
if row['datasource'].startswith('tpf:'):
indx = cat.loc['starid', row['starid']]['datasource'] == 'tpf'
if np.any(indx):
indx_remove[k] = True
cat.remove_indices('starid')
logger.info("Removing %d secondary TPF files as they are also primary",
np.sum(indx_remove))
cat = cat[~indx_remove]
# Find list of all HDF5 files:
hdf_files = find_hdf5_files(input_folder, camera=cameras, ccd=ccds)
logger.info("Number of HDF5 files: %d", len(hdf_files))
if len(hdf_files) > 0:
# TODO: Could we change this so we dont have to parse the filename?
inputs = []
for fname in hdf_files:
m = re.match(r'sector(\d+)_camera(\d)_ccd(\d)\.hdf5',
os.path.basename(fname))
inputs.append((input_folder, int(m.group(1)), int(m.group(2)),
int(m.group(3))))
# Open a pool of workers:
logger.info("Starting pool of workers for FFIs...")
threads = min(threads_max, len(
inputs)) # No reason to use more than the number of jobs in total
logger.info("Using %d processes.", threads)
if threads > 1:
pool = multiprocessing.Pool(threads)
m = pool.imap_unordered
else:
m = map
tic = default_timer()
ccds_done = 0
for cat2 in m(_ffi_todo_wrapper, inputs):
cat = vstack([cat, cat2], join_type='exact')
ccds_done += 1
logger.info("CCDs done: %d/%d", ccds_done, len(inputs))
# Amount of time it took to process TPF files:
toc = default_timer()
logger.info("Elaspsed time: %f seconds (%f per file)", toc - tic,
(toc - tic) / len(inputs))
if threads > 1:
pool.close()
pool.join()
# Check if any targets were found:
if len(cat) == 0:
logger.error("No targets found")
return
# Remove duplicates!
logger.info("Removing duplicate entries...")
_, idx = np.unique(cat[('starid', 'sector', 'camera', 'ccd',
'datasource')],
return_index=True,
axis=0)
cat = cat[np.sort(idx)]
# If the target is present in more than one TPF file, pick the one
# where the target is the furthest from the edge of the image
# and discard the target in all the other TPFs:
if find_secondary_targets:
# Add an index column to the table for later use:
cat.add_column(Column(name='priority', data=np.arange(len(cat))))
# Create index that will only find secondary targets:
indx = [row['datasource'].strip().startswith('tpf:') for row in cat]
# Group the table on the starids and find groups with more than 1 target:
# Equivalent to the SQL code "GROUP BY starid HAVING COUNT(*) > 1"
remove_indx = []
for g in cat[indx].group_by('starid').groups:
if len(g) > 1:
# Find the target farthest from the edge and mark the rest
# for removal:
logger.debug(g)
im = np.argmax(g['edge_dist'])
ir = np.ones(len(g), dtype='bool')
ir[im] = False
remove_indx += list(g[ir]['priority'])
# Remove the list of duplicate secondary targets:
logger.info("Removing %d secondary targets as duplicates.",
len(remove_indx))
logger.debug(remove_indx)
cat.remove_rows(remove_indx)
# Exclude targets from exclude list:
# Add an index and use that to search for starid, and then further check sector and datasource:
cat.add_index('starid')
remove_indx = []
for ex in exclude:
try:
indx = np.atleast_1d(cat.loc_indices['starid', ex[0]])
except KeyError:
indx = []
for i in indx:
if cat[i]['sector'] == ex[1] and cat[i]['datasource'] == ex[2]:
remove_indx.append(i)
if remove_indx:
del cat[remove_indx]
cat.remove_indices('starid')
# Load file with specific method settings and create lookup-table of them:
methods_file = os.path.join(os.path.dirname(__file__), 'data',
'todolist-methods.dat')
methods_file = np.genfromtxt(methods_file,
usecols=(0, 1, 2, 3),
dtype=None,
encoding='utf-8')
methods = {}
for m in methods_file:
methods[(m[0], m[1], m[2])] = m[3].strip().lower()
# Sort the final list:
cat.sort('tmag')
# Write the TODO list to the SQLite database file:
logger.info("Writing TODO file...")
with contextlib.closing(sqlite3.connect(todo_file)) as conn:
cursor = conn.cursor()
# Change settings of SQLite file:
cursor.execute("PRAGMA page_size=4096;")
cursor.execute("PRAGMA foreign_keys=ON;")
cursor.execute("PRAGMA locking_mode=EXCLUSIVE;")
cursor.execute("PRAGMA journal_mode=TRUNCATE;")
# Create TODO-list table:
cursor.execute("""CREATE TABLE todolist (
priority INTEGER PRIMARY KEY ASC NOT NULL,
starid BIGINT NOT NULL,
sector INTEGER NOT NULL,
datasource TEXT NOT NULL DEFAULT 'ffi',
camera INTEGER NOT NULL,
ccd INTEGER NOT NULL,
method TEXT DEFAULT NULL,
tmag REAL,
status INTEGER DEFAULT NULL,
cbv_area INTEGER NOT NULL
);""")
for pri, row in enumerate(cat):
# Find if there is a specific method defined for this target:
method = methods.get((int(row['starid']), int(
row['sector']), row['datasource'].strip()), None)
# Add target to TODO-list:
cursor.execute(
"INSERT INTO todolist (priority,starid,sector,camera,ccd,datasource,tmag,cbv_area,method) VALUES (?,?,?,?,?,?,?,?,?);",
(pri + 1, int(row['starid']), int(row['sector']),
int(row['camera']), int(
row['ccd']), row['datasource'].strip(), float(
row['tmag']), int(row['cbv_area']), method))
conn.commit()
cursor.execute(
"CREATE INDEX starid_datasource_idx ON todolist (starid, datasource);"
) # FIXME: Should be "UNIQUE", but something is weird in ETE-6?!
cursor.execute("CREATE INDEX status_idx ON todolist (status);")
cursor.execute("CREATE INDEX starid_idx ON todolist (starid);")
conn.commit()
# Run a VACUUM of the table which will force a recreation of the
# underlying "pages" of the file.
# Please note that we are changing the "isolation_level" of the connection here,
# but since we closing the conmnection just after, we are not changing it back
conn.isolation_level = None
cursor.execute("VACUUM;")
# Close connection:
cursor.close()
logger.info("TODO done.")
class IRISSpectrograph(object):
"""An object to hold data from multiple IRIS raster scans."""
def __init__(self, filenames, spectral_windows="All", common_axis=0):
"""Initializes an IRISSpectrograph object from IRIS level 2 files."""
# default common axis is 0.
if type(filenames) is str:
filenames = [filenames]
raster_index_to_file = []
for f, filename in enumerate(filenames):
hdulist = fits.open(filename)
if f == 0:
# collecting the window observations.
windows_in_obs = np.array([
hdulist[0].header["TDESC{0}".format(i)]
for i in range(1, hdulist[0].header["NWIN"] + 1)
])
# if spectral window is All then get every window else take the appropriate windows
if spectral_windows == "All":
spectral_windows_req = windows_in_obs
window_fits_indices = range(1, len(hdulist) - 2)
else:
if type(spectral_windows) is str:
spectral_windows_req = [spectral_windows]
spectral_windows_req = np.asarray(spectral_windows_req,
dtype="U")
window_is_in_obs = np.asarray([
window in windows_in_obs
for window in spectral_windows_req
])
if not all(window_is_in_obs):
missing_windows = window_is_in_obs == False
raise ValueError(
"Spectral windows {0} not in file {1}".format(
spectral_windows[missing_windows],
filenames[0]))
window_fits_indices = np.nonzero(
np.in1d(windows_in_obs, spectral_windows))[0] + 1
# Create table of spectral window info in OBS.
self.spectral_windows = Table(
[[
hdulist[0].header["TDESC{0}".format(i)]
for i in window_fits_indices
],
[
hdulist[0].header["TDET{0}".format(i)]
for i in window_fits_indices
],
Quantity([
hdulist[0].header["TWAVE{0}".format(i)]
for i in window_fits_indices
],
unit="angstrom"),
Quantity([
hdulist[0].header["TWMIN{0}".format(i)]
for i in window_fits_indices
],
unit="angstrom"),
Quantity([
hdulist[0].header["TWMAX{0}".format(i)]
for i in window_fits_indices
],
unit="angstrom")],
names=("name", "detector type", "brightest wavelength",
"min wavelength", "max wavelength"))
# Set spectral window name as table index.
self.spectral_windows.add_index("name")
# creating a empty list for every spectral window and each spectral window
# is a key for the dictionary.
data_dict = dict([
(window_name, list())
for window_name in self.spectral_windows["name"]
])
auxiliary_header = hdulist[-2].header
# the unchanged header of the hdulist indexed 0.
self.meta = hdulist[0].header
for i, window_name in enumerate(self.spectral_windows["name"]):
wcs_ = WCS(hdulist[window_fits_indices[i]].header)
data_nan_masked = copy.deepcopy(
hdulist[window_fits_indices[i]].data)
data_nan_masked[hdulist[window_fits_indices[i]].data ==
-200.] = np.nan
data_mask = hdulist[window_fits_indices[i]].data == -200.
# appending Cube instance to the corresponding window key in dictionary's list.
data_dict[window_name].append(
Cube(data_nan_masked, wcs_, meta=self.meta,
mask=data_mask))
scan_label = "scan{0}".format(f)
# Append to list representing the scan labels of each
# spectrum.
len_raster_axis = hdulist[1].header["NAXIS3"]
raster_index_to_file = raster_index_to_file + [scan_label
] * len_raster_axis
# Concatenate auxiliary data arrays from each file.
try:
auxiliary_data = np.concatenate(
(auxiliary_data, np.array(hdulist[-2].data)), axis=0)
except UnboundLocalError as e:
if e.args[
0] == "local variable 'auxiliary_data' referenced before assignment":
auxiliary_data = np.array(hdulist[-2].data)
else:
raise e
hdulist.close()
self.auxiliary_data = Table()
# Enter certain properties into auxiliary data table as
# quantities with units.
auxiliary_colnames = [key for key in auxiliary_header.keys()][7:]
quantity_colnames = [("TIME", "s"), ("PZTX", "arcsec"),
("PZTY", "arcsec"), ("EXPTIMEF", "s"),
("EXPTIMEN", "s"), ("XCENIX", "arcsec"),
("YCENIX", "arcsec"), ("OBS_VRIX", "m/s")]
for col in quantity_colnames:
self.auxiliary_data[col[0]] = _enter_column_into_table_as_quantity(
col[0], auxiliary_header, auxiliary_colnames, auxiliary_data,
col[1])
# Enter remaining properties into table without units/
for i, colname in enumerate(auxiliary_colnames):
self.auxiliary_data[
colname] = auxiliary_data[:, auxiliary_header[colname]]
# Reorder columns so they reflect order in data file.
self.auxiliary_data = self.auxiliary_data[[
key for key in auxiliary_header.keys()
][7:]]
# Rename some columns to be more user friendly.
rename_colnames = [("EXPTIMEF", "FUV EXPOSURE TIME"),
("EXPTIMEN", "NUV EXPOSURE TIME")]
for col in rename_colnames:
self.auxiliary_data.rename_column(col[0], col[1])
# Add column designating what scan/file number each spectra
# comes from. This can be used to determine the corresponding
# wcs object and level 1 info.
self.auxiliary_data["scan"] = raster_index_to_file
# Attach dictionary containing level 1 and wcs info for each file used.
# Calculate measurement time of each spectrum.
times = np.array([
parse_time(self.meta["STARTOBS"]) + timedelta(seconds=s)
for s in self.auxiliary_data["TIME"]
])
# making a CubeSequence of every dictionary key window.
self.data = dict([(window_name,
CubeSequence(data_dict[window_name],
meta=self.meta,
common_axis=common_axis,
time=times))
for window_name in self.spectral_windows['name']])
def __repr__(self):
spectral_window = self.spectral_windows["name"][0]
spectral_windows_info = "".join([
"\n {0}\n (raster axis, slit axis, spectral axis) {1}".
format(name, self.data[name].shape[1])
for name in self.spectral_windows["name"]
])
return "<iris.IRISSpectrograph instance\nOBS ID: {0}\n".format(self.meta["OBSID"]) + \
"OBS Description: {0}\n".format(self.meta["OBS_DESC"]) + \
"OBS period: {0} -- {1}\n".format(self.meta["STARTOBS"], self.meta["ENDOBS"]) + \
"Instance period: {0} -- {1}\n".format(self.data[spectral_window].time[0],
self.data[spectral_window].time[-1]) + \
"Number unique raster positions: {0}\n".format(self.meta["NRASTERP"]) + \
"Spectral windows{0}>".format(spectral_windows_info)
class FilterGenerator(object):
"""
Astronomical filter object generator.
"""
def __init__(self, path=data_path):
"""
Parameters
----------
path : str (optional)
Path to `fft.fits` file
"""
self.path = path
self.table = Table(fits.getdata(path))
self.table.add_index('col0')
def available_filters(self):
"""
Return the available filters in the archive
"""
return self.table['col0'].data
def reconstruct(self, identifier, model=False):
"""
Reconstruct an approximate filter transmittance curve for
a given filter.
Parameters
----------
identifier : str
Name of the filter. To see available filters, run
`~tynt.Filter.available_filters()`
model : bool
Construct a composite astropy model which approximates the
transmittance curve.
Returns
-------
filt : `~tynt.Filter`
Astronomical filter object.
"""
filt = list(self.table.loc[identifier])[1:]
n_lambda, lambda_0, delta_lambda, tr_max = filt[:4]
fft = filt[4:]
astropy_model = None
wavelength = np.arange(lambda_0,
(n_lambda + 1) * delta_lambda + lambda_0,
delta_lambda)
N = len(wavelength)
ifft = np.fft.ifft(fft, n=len(wavelength))
transmittance = ((ifft.real - ifft.real.min()) * tr_max /
ifft.real.ptp())
if model:
m = (np.sum([
models.Sine1D(
amplitude=fft[i].real / N, frequency=i / N, phase=1 / 4)
for i in range(len(fft))
]) - np.sum([
models.Sine1D(amplitude=fft[i].imag / N, frequency=i / N)
for i in range(len(fft))
]))
@models.custom_model
def fft_model(x):
"""
Approximate Fourier reconstruction of an astronomical filter
Parameters
----------
x : `~np.ndarray`
Wavelength in Angstroms.
Returns
-------
transmittance : `~np.ndarray`
Transmittance curve
"""
mo = m(
(x - wavelength.min()) / (wavelength[1] - wavelength[0]))
return (mo - mo.min()) * tr_max / mo.ptp()
astropy_model = fft_model()
return Filter(wavelength * u.Angstrom,
transmittance,
model=astropy_model)
def download_true_transmittance(self, identifier):
"""
Query the SVO service for a given filter,
return the true transmittance curve.
Parameters
----------
identifier : str
Name of the filter. To see available filters, run
`~tynt.Filter.available_filters()`
Returns
-------
filt : `~tynt.Filter`
Astronomical filter object.
"""
path = download_file('http://svo2.cab.inta-csic.es/'
'theory/fps3/fps.php?ID={0}'.format(identifier))
true_transmittance = Table.read(path, format='votable')
return Filter(true_transmittance['Wavelength'].data.data * u.Angstrom,
true_transmittance['Transmission'].data.data)
def load_tyc_teff():
"""Load the Tycho-2 effective temperatures."""
print('Loading TYC2 effective temperatures')
with tarfile.open(os.path.join('vizier', 'tyc2teff.tar.gz'), 'r:gz') as tf:
# unfortunately the behaviour of the CDS reader results in high memory
# usage during loading due to parsing non-used fields, so use a custom
# implementation
tyc_range = None
teff_range = None
with tf.extractfile('./ReadMe') as readme:
re_file = re.compile(r'tycall.dat\ +[0-9]+\ +(?P<length>[0-9]+)')
re_table = re.compile(
r'Byte-by-byte Description of file: (?P<name>\S+)$')
re_field = re.compile(
r'''\ *(?P<start>[0-9]+)\ *-\ *(?P<end>[0-9]+) # range
\ +\S+ # format
\ +\S+ # units
\ +(?P<label>\S+) # label''', re.X)
record_count = None
current_table = None
for bline in readme.readlines():
line = bline.decode('ascii')
match = re_file.match(line)
if match:
record_count = int(match.group('length'))
continue
match = re_table.match(line)
if match:
current_table = match.group('name')
continue
if current_table != 'tycall.dat':
continue
match = re_field.match(line)
if not match:
continue
if match.group('label') == 'Tycho':
tyc_range = int(match.group('start')) - 1, int(
match.group('end'))
elif match.group('label') == 'Teff':
teff_range = int(match.group('start')) - 1, int(
match.group('end'))
if tyc_range is not None and teff_range is not None:
break
if record_count is None:
raise RuntimeError('Could not get record count')
if tyc_range is None or teff_range is None:
raise RuntimeError('Could not find Tycho, Teff fields')
with tf.extractfile('./tycall.dat.gz') as gzf, gzip.open(
gzf, 'rt', encoding='ascii') as f:
tycs = np.empty(record_count, dtype=np.int64)
teffs = np.empty(record_count, dtype=np.float64)
record = 0
for line in f:
try:
tycsplit = line[tyc_range[0]:tyc_range[1]].split('-')
tyc = int(tycsplit[0]) + int(tycsplit[1]) * 10000 + int(
tycsplit[2]) * 1000000000
teff = float(line[teff_range[0]:teff_range[1]])
if teff != 99999:
tycs[record] = tyc
teffs[record] = teff
record += 1
except ValueError:
pass
data = Table([tycs[0:record], teffs[0:record]],
names=('TYC', 'teff_val'))
data['teff_val'].unit = u.K
data.add_index('TYC')
return unique(data, keys=['TYC'])
def mangle(sn, S, spec_mjd, filters, staticfilter=False, anchor_distance=100, verbose=False):
"""
:param anchor_distance:
:param spec_mjd:
:param filters:
:param staticfilter:
:param verbose:
:param sn:
:param S:
:return:
"""
if hasattr(sn, "lcfit"):
rows = OrderedDict()
filter_dict = OrderedDict()
for i, f in enumerate(filters):
filter_dict[f] = functions.load_filter(os.path.join(defaults._default_filter_dir_path, f + ".dat"))
filter_dict[f].calculate_edges()
# filter_dict[f].calculate_edges_zero()
fit_flux = sn.lcfit.spline[f](spec_mjd)
sn.phot.data_filters[f].resample_response(new_wavelength=S.wavelength)
S_filter_flux = calc_spectrum_filter_flux(filter_object=sn.phot.data_filters[f], spectrum_object=S, verbose=verbose)
S_filter_flux_no_area = calc_spectrum_filter_flux(filter_object=sn.phot.data_filters[f],
spectrum_object=S,
correct_for_area=False, verbose=verbose)
mS_filter_flux = np.nan
rows[f] = (fit_flux, S_filter_flux, S_filter_flux_no_area)
if i == 0:
data_table = Table(
names=("filter", "fitflux", "spec_filterflux", "mangledspec_filterflux", "filter_object", "mask"),
dtype=('S12', 'f4', 'f4', 'f4', object, bool))
data_table.add_row((f, fit_flux, S_filter_flux, mS_filter_flux, filter_dict[f], True))
for i, f in enumerate(data_table["filter_object"]):
## Test extent
bool_uncontained = np.logical_or(f._lower_edge < S.min_wavelength, f._upper_edge > S.max_wavelength)
if verbose: print(bool_uncontained)
if bool_uncontained:
data_table = data_table[np.where(data_table["filter"] != utils.b(f.filter_name))]
knot_colours = [j._plot_colour for j in data_table["filter_object"] if hasattr(j, "_plot_colour")]
data_table.add_column(Column(knot_colours, name="knot_colours"))
data_table["lambda_eff"] = [i.lambda_effective.value for i in data_table["filter_object"]]
if not staticfilter:
w = 0
else:
w = np.where(data_table["filter"] == staticfilter)
scale_factor = 1. / data_table[w]["fitflux"]
if verbose: print("Scale Factor", scale_factor)
norm_factor = data_table[w]["fitflux"] / data_table[w]["spec_filterflux"]
if verbose: print("norm factor", norm_factor)
data_table["fitflux"] = data_table["fitflux"] * scale_factor
# "spec flux"
data_table["spec_filterflux"] = data_table["spec_filterflux"] * scale_factor
if verbose: print("scaled ", )
S.flux = S.flux * scale_factor
S.flux = S.flux * norm_factor
S.scale_factor = scale_factor
S.norm_factor = norm_factor
data_table["spec_filterflux"] = data_table["spec_filterflux"] * norm_factor
# ## Lower
anchor_min_wavelength = np.nanmin([i._lower_edge for i in data_table["filter_object"]]) - anchor_distance
# ## Upper
anchor_max_wavelength = np.nanmax([i._upper_edge for i in data_table["filter_object"]]) + anchor_distance
if verbose: print(data_table)
if len(data_table) < 2:
S.flux = S.flux / S.scale_factor
fit_dict = OrderedDict()
fit_dict["SpectrumObject"] = S
fit_dict["final_spl"] = lambda x: np.ones(len(x))
fit_dict["data_table"] = data_table
return fit_dict
mc_l, mc_u = functions.calc_linear_terms(data_table[data_table["mask"]], key="fitflux", verbose=verbose)
anchor_l = mc_l[0] * anchor_min_wavelength + mc_l[1]
anchor_u = mc_u[0] * anchor_max_wavelength + mc_u[1]
spl_wav = S.data['wavelength'][
np.logical_and(S.data['wavelength'] >= anchor_min_wavelength, S.data['wavelength'] <= anchor_max_wavelength)]
mc_spec_l, mc_spec_u = functions.calc_linear_terms(data_table[data_table["mask"]], key="spec_filterflux", verbose=verbose)
anchor_spec_l = mc_spec_l[0] * anchor_min_wavelength + mc_spec_l[1]
anchor_spec_u = mc_spec_u[0] * anchor_max_wavelength + mc_spec_u[1]
data_table.add_row(("lower_anchor", anchor_spec_l, anchor_spec_l, anchor_spec_u, np.nan, False,
colours.hex["batman"], anchor_min_wavelength))
data_table.add_row(("upper_anchor", anchor_spec_u, anchor_spec_u, anchor_spec_u, np.nan, False,
colours.hex["batman"], anchor_max_wavelength))
data_table.add_index("lambda_eff")
data_table.sort()
for i, f in enumerate(data_table["filter_object"]):
if isinstance(f, classes.FilterClass):
mangledspec_filterflux = calc_spectrum_filter_flux(filter_object=f, spectrum_object=S, verbose=verbose)
# print(data_table["spec_filterflux"][i], mangledspec_filterflux)
data_table["mangledspec_filterflux"][i] = mangledspec_filterflux
else:
pass
wanted_flux = data_table[data_table["mask"]]["fitflux"].data
wanted_filters = data_table[data_table["mask"]]["filter_object"].data
fit_dict = manglespec3(S, spec_mjd, wanted_filters, wanted_flux, data_table)
return fit_dict
def compile_distances(name):
'''Compare the measured distance to literature values from NED
'''
distances = ascii.read(basedir / 'data' / 'literature distances' /
f'{name}.csv',
delimiter=',',
comment='#')
references = ascii.read(basedir / 'data' / 'literature distances' /
f'paper_list.csv',
encoding='utf8',
delimiter=',')
ref_dict = {
'Refcode': list(references['Refcode']),
'Authors': list(references['Authors']),
'Title': list(references['Title'])
}
new = 0
# search for missing references
for bibcode in distances['Refcode']:
if bibcode not in ref_dict['Refcode']:
# this shouldn't happen too often. So we open it in this loop
from astroquery.nasa_ads import ADS
ADS.TOKEN = open(basedir / 'notebooks' / 'ADS_DEV_KEY', 'r').read()
try:
result = ADS.query_simple(bibcode)
ref_dict['Refcode'].append(bibcode)
ref_dict['Authors'].append(';'.join(result['author'][0]))
ref_dict['Title'].append(result['title'][0][0])
new += 1
except:
print(f'can not find {bibcode} for {name}')
if new > 0:
print(f'{new} new items added')
references = Table(ref_dict)
references.sort('Refcode', reverse=True)
with open(basedir / 'data' / 'literature distances' /
f'paper_list.csv',
'w',
encoding='utf8',
newline='\n') as f:
ascii.write(references,
f,
format='csv',
overwrite=True,
delimiter=',')
references.add_index('Refcode')
references['year'] = [int(row['Refcode'][:4]) for row in references]
references['firstAuthor'] = [
row['Authors'].split(',')[0] for row in references
]
references['name'] = [
f'{row["firstAuthor"]}+{str(row["year"])[2:]}' for row in references
]
distances['year'] = [int(row['Refcode'][:4]) for row in distances]
distances['name'] = [
references.loc[ref]['name'] for ref in distances['Refcode']
]
base_url = 'https://ui.adsabs.harvard.edu/abs/'
distances['link'] = [
f'\href{{{base_url + row["Refcode"]}}}{{{row["name"]}}}'
for row in distances
]
# ugly workaround
# some papers publish more than one distance. We use only the one with the smallest uncertainty
#distances = distances[np.abs(distances['(m-M)']-distance)<1]
'''
distances['i'] = np.arange(0,len(distances))
remove = []
for i,row in enumerate(distances):
name = row['name']
sub = distances[np.where(distances['name']==name)]
if len(sub) > 1:
if row['err(m-M)'] > np.min(sub['err(m-M)']):
remove.append(i)
# only show the 5 most recent results for each method
for method in np.unique(distances['Method']):
sub = distances[distances['Method']==method].copy()
if len(sub)>5:
sub.sort('year')
remove += list(sub[:-5]['i'])
remove = list(set(remove))
remove.sort(reverse=True)
for i in remove:
distances.remove_row(i)
'''
methods = []
year = []
DM = []
errDM = []
links = []
marker = []
ref = []
names = []
for g in distances.group_by(['Refcode', 'Method']).groups:
methods.append(Methods.get(g['Method'][0], g['Method'][0]))
year.append(g['year'][0])
links.append(g['link'][0])
DM.append(g['(m-M)'].mean())
errDM.append(np.sqrt(np.sum(g['err(m-M)']**2)))
ref.append(g['Refcode'][0])
names.append(g['name'][0])
if len(g) == 1:
marker.append('o')
else:
marker.append('D')
distances = Table([methods, year, DM, errDM, links, marker, ref, names],
names=[
'Method', 'year', '(m-M)', 'err(m-M)', 'link',
'marker', 'Refcode', 'name'
])
# only show the 5 most recent results for each method
distances['i'] = np.arange(0, len(distances))
max_entries = 5
remove = []
for method in np.unique(distances['Method']):
sub = distances[distances['Method'] == method].copy()
if len(sub) > max_entries:
sub.sort('year')
remove += list(sub[:-max_entries]['i'])
remove = list(set(remove))
remove.sort(reverse=True)
for i in remove:
distances.remove_row(i)
# distances requires [Method,year,(m-M),err(m-M),link] as columns
distances['sort_order'] = [
importance.index(row['Method']) for row in distances
]
distances.sort(['sort_order', 'year'])
distances['y'] = np.arange(1, len(distances) + 1)
return distances
lst = []
for name in tqdm(np.unique(nebulae['gal_name']),
position=0,
leave=False,
colour='green'):
p = {x: sample_table.loc[name][x] for x in sample_table.columns}
filename = data_ext / 'MUSE_DR2.1' / 'Nebulae catalogue' / 'spectra' / f'{name}_VorSpectra.fits'
with fits.open(filename) as hdul:
spectra = Table(hdul[1].data)
spectral_axis = np.exp(Table(hdul[2].data)['LOGLAM']) * u.Angstrom
spectra['region_ID'] = np.arange(len(spectra))
spectra.add_index('region_ID')
H0 = 67 * u.km / u.s / u.Mpc
z = (H0 * Distance(distmod=p['(m-M)']) / c.c).decompose()
lam_HA0 = 6562.8 * u.Angstrom
lam_HA = (1 + z) * lam_HA0
sub = nebulae[nebulae['gal_name'] == name][[
'gal_name', 'region_ID', 'eq_width', 'HA6562_FLUX'
]]
for row in tqdm(sub, position=1, leave=False, colour='red', desc=name):
try:
region_ID = row['region_ID']
flux = spectra.loc[region_ID]['SPEC'] * u.erg / u.s / u.cm**2 / u.A
fit = fit_emission_line(spectral_axis, flux, lam_HA)
def get_pec_data(image_dir, ref_image='guide_000.new', img_prefix='',
observer=None, phase_length=480,
skip_solved=True, verbose=False, parallel=False, **kwargs):
assert observer is not None, "Observer required"
# Gather all the images
base_dir = os.getenv('PANDIR', '/var/panoptes')
target_name, obs_date_start = image_dir.rstrip('/').split('/', 1)
target_dir = '{}/images/fields/{}'.format(base_dir, image_dir)
guide_images = glob.glob('{}/guide_*.new'.format(target_dir))
if len(guide_images) == 0:
print("No solved guide images found")
guide_images = glob.glob('{}/guide_*.cr2'.format(target_dir))
guide_images.sort()
# WCS Information
# Solve the guide image if given a CR2
ref_image = guide_images[-1]
ref_solve_info = None
if ref_image.endswith('cr2'):
if verbose:
print("Solving guide image")
ref_solve_info = get_solve_field(ref_image, verbose=verbose)
if verbose:
print("Solved guide image info: {}".format(ref_solve_info))
ref_image = ref_image.replace('cr2', 'new')
# If no guide image, attempt a solve on similar fits
# Note: not sure this is needed any more
if not os.path.exists(ref_image):
if os.path.exists(ref_image.replace('new', 'fits')):
ref_solve_info = get_solve_field(ref_image.replace('new', 'fits'))
if verbose and ref_solve_info:
print(ref_solve_info)
assert os.path.exists(ref_image), warnings.warn("Ref image does not exist: {}".format(ref_image))
ref_header = fits.getheader(ref_image)
ref_wcs = get_wcsinfo(ref_image)
# Reference time
t0 = Time(ref_header.get('DATE-OBS', date_parser.parse(obs_date_start))).datetime
if verbose:
print("Reference image: {}".format(ref_image))
print("Reference time: {}".format(t0))
# Image sequence
image_files = glob.glob('{}/{}*.cr2'.format(target_dir, img_prefix))
image_files.sort()
if verbose:
print("Found {} images in sequence".format(len(image_files)))
img_info = []
# Solves an individual image in the sequence
def solver(img):
if verbose:
print('*' * 80)
header_info = {}
img_wcs_path = img.replace('cr2', 'wcs')
if not os.path.exists(img_wcs_path):
if verbose:
print("No WCS, solving CR2: {}".format(img))
# Give the guide image RA/Dec as a guess since it should be close
header_info = get_solve_field(
img,
ra=ref_wcs['ra_center'].value,
dec=ref_wcs['dec_center'].value,
radius=10,
verbose=verbose,
**kwargs
)
# Gather all the header information for the image
if len(header_info) == 0:
header_info.update(get_wcsinfo(img_wcs_path))
header_info.update(fits.getheader(img.replace('cr2', 'new')))
header_info.update(read_exif(img))
# Lowercase all header names
hi = dict((k.lower(), v) for k, v in header_info.items())
del(hi['history'])
del(hi['comment'])
if verbose:
pprint(hi)
# Add header info to image info
img_info.append(hi)
# Solve all of our images
# Note: Could do this in parralel
for img in image_files:
if verbose:
print("Solving for {}".format(img))
solver(img)
# Get the center RA/Dec for all images
ras = [w['ra_center'].value for w in img_info]
decs = [w['dec_center'].value for w in img_info]
# Get the center RA/Dec in arcseconds. (??? - used for HA below)
ras_as = [w['ra_center'].to(u.arcsec).value for w in img_info]
decs_as = [w['dec_center'].to(u.arcsec).value for w in img_info]
# List of times for sequqnce
time_range = [Time(w.get('date-obs', t0)) for w in img_info]
# Get the Hourangle from the observer
ha = []
ha = np.array([observer.target_hour_angle(t, SkyCoord(ras[idx], decs[idx], unit='degree')).to(u.degree).value
for idx, t in enumerate(time_range)])
ha[ha > 270] = ha[ha > 270] - 360
# Get time deltas between each timestamp
dt = np.diff([t.datetime.timestamp() for t in time_range])
# Add the offset for initial time
dt = np.insert(dt, 0, (time_range[0].datetime.timestamp() - t0.timestamp()))
# Total offset for each image
t_offset = np.cumsum(dt)
# Arcsecond difference between each image for RA
ra_diff = np.diff(ras_as)
ra_diff = np.insert(ra_diff, 0, 0)
# Arcsecond difference between each image for Dec
dec_diff = np.diff(decs_as)
dec_diff = np.insert(dec_diff, 0, 0)
# Delta arcsecond
dra_as = pd.Series(ra_diff)
ddec_as = pd.Series(dec_diff)
# Delta arcsecond rate
dra_as_rate = dra_as / dt
ddec_as_rate = ddec_as / dt
# Fill in empty values
dra_as_rate.fillna(value=0, inplace=True)
ddec_as_rate.fillna(value=0, inplace=True)
if verbose:
print(len(ra_diff))
print(len(dec_diff))
print(len(dt))
print(len(t_offset))
print(len(ras))
print(len(decs))
table = Table({
'dec': decs,
'dec_as': ddec_as,
'dec_as_rate': ddec_as_rate,
'dt': dt,
'ha': ha,
'ra': ras,
'ra_as': dra_as,
'ra_as_rate': dra_as_rate,
'offset': t_offset,
'time_range': [t.mjd for t in time_range],
}, meta={
'name': target_name,
'obs_date_start': obs_date_start,
})
table.add_index('time_range')
table['ra'].format = '%+3.3f'
table['ha'].format = '%+3.3f'
table['dec'].format = '%+3.3f'
table['dec_as_rate'].format = '%+1.5f'
table['ra_as_rate'].format = '%+1.5f'
table['time_range'].format = '%+5.5f'
table['ra_as'].format = '%+2.3f'
table['dec_as'].format = '%+3.3f'
return table
def main(args):
t = time.perf_counter()
if os.path.isdir(args.output_path):
os.chdir(args.output_path)
else:
os.makedirs(args.output_path, exist_ok=True)
if args.arm:
do_red = args.arm.lower() == 'red'
do_blue = args.arm.lower() == 'blue'
else:
do_red = True
do_blue = True
red_root = os.path.join(args.root, 'red')
blue_root = os.path.join(args.root, 'blue')
qa_dict = {}
if args.parameter_file:
blue_user_config_lines = reduction.parse_pypeit_parameter_file(args.parameter_file, 'p200_dbsp_blue')
red_user_config_lines = reduction.parse_pypeit_parameter_file(args.parameter_file, 'p200_dbsp_red')
else:
blue_user_config_lines = []
red_user_config_lines = []
if args.debug:
pypeit.display.display.connect_to_ginga(raise_err=True, allow_new=True)
if do_red:
red_files = fix_headers.main(red_root, args.no_interactive, args.no_interactive)
context = reduction.setup(red_files, args.output_path, 'p200_dbsp_red')
# optionally use interactive correction
if not args.no_interactive:
interactive_correction(context[0])
pypeit_file_red = reduction.write_setup(context, 'all', 'p200_dbsp_red', red_user_config_lines)[0]
if do_blue:
blue_files = fix_headers.main(blue_root, args.no_interactive, args.no_interactive)
context = reduction.setup(blue_files, args.output_path, 'p200_dbsp_blue')
if not args.no_interactive:
interactive_correction(context[0])
pypeit_file_blue = reduction.write_setup(context, 'all', 'p200_dbsp_blue', blue_user_config_lines)[0]
plt.switch_backend("agg")
# TODO: parallelize this
# Would need to look like
# Splitting up the .pypeit files into bits and pieces
# Oooh what if I just do the calibration first
# and then parallelize the reduction
output_spec1ds_blue = set()
output_spec1ds_red = set()
if do_red:
output_spec1ds_red, output_spec2ds_red = reduction.redux(pypeit_file_red, args.output_path)
qa_dict = qa.save_2dspecs(qa_dict, output_spec2ds_red, args.output_path, 'p200_dbsp_red')
if do_blue:
output_spec1ds_blue, output_spec2ds_blue = reduction.redux(pypeit_file_blue, args.output_path)
qa_dict = qa.save_2dspecs(qa_dict, output_spec2ds_blue, args.output_path, 'p200_dbsp_blue')
if do_red or do_blue:
qa.write_extraction_QA(qa_dict, args.output_path)
if do_red:
verification_counter = 0
red_pypeit_files = reduction.verify_spec1ds(output_spec1ds_red, verification_counter, args.output_path)
while red_pypeit_files:
verification_counter += 1
out_1d, out_2d = reduction.re_redux(red_pypeit_files, args.output_path)
red_pypeit_files = reduction.verify_spec1ds(out_1d, verification_counter, args.output_path)
qa_dict = qa.save_2dspecs(qa_dict, out_2d, args.output_path, 'p200_dbsp_red')
output_spec1ds_red |= out_1d
output_spec2ds_red |= out_2d
if do_blue:
verification_counter = 0
blue_pypeit_files = reduction.verify_spec1ds(output_spec1ds_blue, verification_counter, args.output_path)
while blue_pypeit_files:
verification_counter += 1
out_1d, out_2d = reduction.re_redux(blue_pypeit_files, args.output_path)
blue_pypeit_files = reduction.verify_spec1ds(out_1d, verification_counter, args.output_path)
qa_dict = qa.save_2dspecs(qa_dict, out_2d, args.output_path, 'p200_dbsp_blue')
output_spec1ds_blue |= out_1d
output_spec2ds_blue |= out_2d
# TODO: use a do/while loop to iterate on the manual extraction GUI until user is satisfied
if args.manual_extraction:
# wait for user acknowledgement
input("Ready for manual extraction? If using GNU screen/tmux behind ssh, make sure to check that $DISPLAY is correct.")
plt.switch_backend("Qt5Agg")
if do_red:
red_manual_pypeit_files = reduction.manual_extraction(output_spec2ds_red, pypeit_file_red, args.output_path)
if do_blue:
blue_manual_pypeit_files = reduction.manual_extraction(output_spec2ds_blue, pypeit_file_blue, args.output_path)
if do_red and red_manual_pypeit_files:
out_1d, out_2d = reduction.re_redux(red_manual_pypeit_files, args.output_path)
qa.save_2dspecs(qa_dict, out_2d, args.output_path, 'p200_dbsp_red')
output_spec1ds_red |= out_1d
output_spec2ds_red |= out_2d
if do_blue and blue_manual_pypeit_files:
out_1d, out_2d = reduction.re_redux(blue_manual_pypeit_files, args.output_path)
qa.save_2dspecs(qa_dict, out_2d, args.output_path, 'p200_dbsp_blue')
output_spec1ds_blue |= out_1d
output_spec2ds_blue |= out_2d
# spec1d_blueNNNN-OBJ_DBSPb_YYYYMMMDDTHHMMSS.SPAT.fits
fname_len = 72
# sens_blueNNNN-OBJ_DBSPb_YYYYMMMDDTHHMMSS.SPAT.fits
sensfunc_len = 70
# Find standards and make sensitivity functions
spec1d_table = Table(names=('filename', 'arm', 'object', 'frametype',
'airmass', 'mjd', 'sensfunc', 'exptime'),
dtype=(f'U{fname_len}', 'U4', 'U20', 'U8',
float, float, f'U{sensfunc_len}', float))
# Ingest spec_1d tables
spec1ds = output_spec1ds_red | output_spec1ds_blue
for spec1d in spec1ds:
path = os.path.join(args.output_path, 'Science', spec1d)
with fits.open(path) as hdul:
head0 = hdul[0].header
head1 = hdul[1].header
arm = 'red' if 'red' in head0['PYP_SPEC'] else 'blue'
spec1d_table.add_row((spec1d, arm, head0['TARGET'],
head1['OBJTYPE'], head0['AIRMASS'],
head0['MJD'], '', head0['EXPTIME']))
spec1d_table.add_index('filename')
spec1d_table.sort(['arm', 'mjd'])
if do_red:
for row in spec1d_table[(spec1d_table['arm'] == 'red') & (spec1d_table['frametype'] == 'standard')]:
sensfunc = fluxing.make_sensfunc(row['filename'], args.output_path, 'p200_dbsp_red', red_user_config_lines)
if sensfunc == "":
spec1d_table['frametype'][spec1d_table['filename'] == row['filename']] = 'science'
else:
spec1d_table['sensfunc'][spec1d_table['filename'] == row['filename']] = sensfunc
if do_blue:
for row in spec1d_table[(spec1d_table['arm'] == 'blue') & (spec1d_table['frametype'] == 'standard')]:
sensfunc = fluxing.make_sensfunc(row['filename'], args.output_path, 'p200_dbsp_blue', blue_user_config_lines)
if sensfunc == "":
spec1d_table['frametype'][spec1d_table['filename'] == row['filename']] = 'science'
else:
spec1d_table['sensfunc'][spec1d_table['filename'] == row['filename']] = sensfunc
if do_red:
arm = spec1d_table['arm'] == 'red'
stds = (spec1d_table['frametype'] == 'standard') & arm
if np.any(stds):
for row in spec1d_table[arm]:
if row['frametype'] == 'science':
best_sens = spec1d_table[stds]['sensfunc'][np.abs(spec1d_table[stds]['airmass'] - row['airmass']).argmin()]
elif row['frametype'] == 'standard':
if (stds).sum() == 1:
best_sens = spec1d_table[stds]['sensfunc'][np.abs(spec1d_table[stds]['airmass'] - row['airmass']).argmin()]
else:
best_sens = spec1d_table[stds]['sensfunc'][np.abs(spec1d_table[stds]['airmass'] - row['airmass']).argsort()[1]]
spec1d_table.loc[row['filename']]['sensfunc'] = best_sens
else:
for filename in spec1d_table[arm]['filename']:
spec1d_table.loc[filename]['sensfunc'] = ''
if do_blue:
arm = spec1d_table['arm'] == 'blue'
stds = (spec1d_table['frametype'] == 'standard') & arm
if np.any(stds):
for row in spec1d_table[arm]:
if row['frametype'] == 'science':
best_sens = spec1d_table[stds]['sensfunc'][np.abs(spec1d_table[stds]['airmass'] - row['airmass']).argmin()]
elif row['frametype'] == 'standard':
if (stds).sum() == 1:
best_sens = spec1d_table[stds]['sensfunc'][np.abs(spec1d_table[stds]['airmass'] - row['airmass']).argmin()]
else:
best_sens = spec1d_table[stds]['sensfunc'][np.abs(spec1d_table[stds]['airmass'] - row['airmass']).argsort()[1]]
spec1d_table.loc[row['filename']]['sensfunc'] = best_sens
else:
for filename in spec1d_table[arm]['filename']:
spec1d_table.loc[filename]['sensfunc'] = ''
# build fluxfile
if do_red:
spec1d_to_sensfunc = {row['filename']: row['sensfunc'] for row in spec1d_table if row['arm'] == 'red'}
red_fluxfile = fluxing.build_fluxfile(spec1d_to_sensfunc, args.output_path, 'p200_dbsp_red', red_user_config_lines)
if do_blue:
spec1d_to_sensfunc = {row['filename']: row['sensfunc'] for row in spec1d_table if row['arm'] == 'blue'}
blue_fluxfile = fluxing.build_fluxfile(spec1d_to_sensfunc, args.output_path, 'p200_dbsp_blue', blue_user_config_lines)
# flux data
if do_red:
fluxing.flux(red_fluxfile, args.output_path)
if do_blue:
fluxing.flux(blue_fluxfile, args.output_path)
# coadd - intelligent coadding of multiple files
# first make a column "coaddID" that is the same for frames to be coadded
# TODO: when there are multiple exposures of an object, splice/output all of them
coaddIDs = []
if args.null_coadd:
coaddIDs = range(len(spec1d_table))
else:
previous_row : Row = None
S_PER_DAY = 24 * 60 * 60
thresh = 15
for i, row in enumerate(spec1d_table):
if i == 0:
coaddIDs.append(0)
else:
# if this is the same object as the last one
# and they were taken consecutively
if ((row['arm'] == previous_row['arm']) and
(row['object'] == previous_row['object']) and
((row['mjd']*S_PER_DAY - previous_row['mjd']*S_PER_DAY
- previous_row['exptime']) < previous_row['exptime'])):
coaddIDs.append(coaddIDs[-1])
else:
coaddIDs.append(coaddIDs[-1] + 1)
previous_row = row
spec1d_table.add_column(coaddIDs, name="coadd_id")
# figure out where on detector likely target is
spec1d_table.add_column(Column(name="spats", dtype=object, length=len(spec1d_table)))
spec1d_table.add_column(Column(name="fracpos", dtype=object, length=len(spec1d_table)))
all_spats = []
all_fracpos = []
# for each spec1d file
for filename in spec1d_table['filename']:
path = os.path.join(args.output_path, 'Science', filename)
with fits.open(path) as hdul:
spats = []
fracpos = []
for i in range(1, len(hdul) - 1):
# grab all of its extensions' spatial positions
spats.append(int(hdul[i].name.split('-')[0].lstrip('SPAT')))
fracpos.append(hdul[i].header['SPAT_FRACPOS'])
spats.sort()
fracpos.sort()
all_spats.append(spats)
all_fracpos.append(fracpos)
spec1d_table.loc[filename]['spats'] = spats
spec1d_table.loc[filename]['fracpos'] = fracpos
# add to table???
# this needs to be dtype object to allow for variable length lists
spec1d_table.add_column(Column(name="coadds", dtype=object, length=len(spec1d_table)))
spec1d_table.add_column([False]*len(all_spats), name="processed")
# coadd
# iterate over coadd_ids
coadd_to_spec1d = {}
for coadd_id in set(coaddIDs):
subtable = spec1d_table[spec1d_table['coadd_id'] == coadd_id]
fname_spats = {row['filename']: row['spats'].copy() for row in subtable}
grouped_spats_list = coadding.group_coadds(fname_spats)
if all(subtable['arm'] == 'red'):
coadds = coadding.coadd(grouped_spats_list, args.output_path, 'p200_dbsp_red', red_user_config_lines)
if all(subtable['arm'] == 'blue'):
coadds = coadding.coadd(grouped_spats_list, args.output_path, 'p200_dbsp_blue', blue_user_config_lines)
assert all(subtable['arm'] == 'red') or all(subtable['arm'] == 'blue'),\
"Something went wrong with coadding..."
for row in subtable:
spec1d_table.loc[row['filename']]['coadds'] = coadds
for i, coadd in enumerate(coadds):
coadd_to_spec1d[coadd] = list(zip(grouped_spats_list[i]['fnames'], grouped_spats_list[i]['spats']))
if not args.skip_telluric:
# telluric correct
if do_red:
tellcorr_inputs = []
tell_coadd_fnames = set()
for row in spec1d_table[spec1d_table['arm'] == 'red']:
if isinstance(row['coadds'], list):
for obj in row['coadds']:
if not obj in tell_coadd_fnames:
tmp = (obj, args.output_path, 'p200_dbsp_red', red_user_config_lines)
tellcorr_inputs.append(tmp)
tell_coadd_fnames.add(obj)
if args.jobs == 1:
# do it in series
for tellcorr_input in tqdm.tqdm(tellcorr_inputs):
telluric.telluric_correct(*tellcorr_input)
else:
pool = multiprocessing.Pool(args.jobs)
list(tqdm.tqdm(pool.imap(telluric.picklable_telluric_correct, tellcorr_inputs), total=len(tellcorr_inputs)))
pool.close()
pool.join()
# Maybe do something here to verify that telluric correction succeeded
# and if so, change the coadd names
for coadd in tell_coadd_fnames:
tell = coadd.replace(".fits", "_tellcorr.fits")
tellpath = os.path.join(args.output_path, 'Science', tell)
coaddpath = os.path.join(args.output_path, 'Science', coadd)
# check if tell exists and is newer than coadd
if os.path.isfile(tellpath) and (os.path.getmtime(tellpath) > os.path.getmtime(coaddpath)):
# modify coadd
for row in spec1d_table:
if coadd in row['coadds']:
ix = row['coadds'].index(coadd)
spec1d_table.loc[row['filename']]['coadds'][ix] = tell
coadd_to_spec1d[tell] = coadd_to_spec1d[coadd]
del coadd_to_spec1d[coadd]
# current splicing - make sure spatial fraction is similar on blue/red
# TODO: handle multiple observations of same target throughout night with null coadding
# splice data
splicing_dict = {}
blue_mask = spec1d_table['arm'] == 'blue'
red_mask = spec1d_table['arm'] == 'red'
os.makedirs(os.path.join(args.output_path, 'spliced'), exist_ok=True)
def get_std_trace(std_path: str) -> float:
max_sn = -1
max_fracpos = -1
with fits.open(std_path) as hdul:
# loop through trace hdus
for hdu in hdul:
if not 'SPAT' in hdu.name:
continue
# look at s/n
if 'OPT_COUNTS' in hdu.data.dtype.names:
this_sn = np.nanmedian(hdu.data['OPT_COUNTS']/hdu.data['OPT_COUNTS_SIG'])
elif 'BOX_COUNTS' in hdu.data.dtype.names:
this_sn = np.nanmedian(hdu.data['BOX_COUNTS']/hdu.data['BOX_COUNTS_SIG'])
else:
this_sn = -1
if this_sn > max_sn:
max_sn = this_sn
max_fracpos = hdu.header['SPAT_FRACPOS']
if max_fracpos == -1:
raise Exception(f"Error! No HDUs in {os.path.basename(std_path)} have median S/N > 0.")
return max_fracpos
## Need to find red + blue fracpos for standards
# hopefully standards only have one star each?
# or should i actually try to do matching there
fracpos_diff_list = []
stds = spec1d_table['frametype'] == 'standard'
if do_red or do_blue:
FRACPOS_SUM = 1.0
FRACPOS_TOL = 0.05
if do_red and do_blue:
# real matching + splicing
std_fracpos_sums = []
if (stds & blue_mask).any() and (stds & red_mask).any():
for row in spec1d_table[stds]:
# find closest mjd frame of other arm
if not row['processed']:
other_arm = spec1d_table['arm'] != row['arm']
corresponding_row = spec1d_table[other_arm & stds][np.abs(spec1d_table[other_arm & stds]['mjd'] - row['mjd']).argmin()]
this_path = os.path.join(args.output_path, 'Science', row['filename'])
corresponding_path = os.path.join(args.output_path, 'Science', corresponding_row['filename'])
std_fracpos_sums.append(get_std_trace(this_path) + get_std_trace(corresponding_path))
spec1d_table.loc[row['filename']]['processed'] = True
spec1d_table.loc[corresponding_row['filename']]['processed'] = True
FRACPOS_SUM = np.mean(std_fracpos_sums)
FRACPOS_TOL = FRACPOS_SUM * .025
# setup splicing dict
splicing_dict = {}
# for each target
for row in spec1d_table:
target = row['object']
arm = row['arm']
# for each of its fracpos
for i, fracpos in enumerate(row['fracpos']):
coadd = row['coadds'][i]
targ_dict = splicing_dict.get(target)
# normalize fracpos to red
if do_red and do_blue and arm == 'blue':
fracpos = FRACPOS_SUM - fracpos
# if it's not in the dict
if targ_dict is None:
# put it in the dict
splicing_dict[target] = {fracpos: {
arm: {
'spec1ds': coadd_to_spec1d[coadd],
'coadd': coadd
}
}}
# else
else:
close_enough = False
# for each existing fracpos
for fracpos_existing in list(targ_dict):
# if its close enough
fracpos_diff_list.append(abs(fracpos_existing - fracpos))
if abs(fracpos_existing - fracpos) < FRACPOS_TOL:
# put it in the dict
splicing_dict[target][fracpos_existing][arm] = {
'spec1ds': coadd_to_spec1d[coadd],
'coadd': coadd
}
close_enough = True
break
if not close_enough:
# If this fracpos isn't close enough to any others
splicing_dict[target][fracpos] = {arm: {
'spec1ds': coadd_to_spec1d[coadd],
'coadd': coadd
}}
# And now, actually splice!
splicing.splice(splicing_dict, args.splicing_interpolate_gaps, red_root, args.output_path)
with open("fracpos_data.pickle", "wb") as f:
pickle.dump((fracpos_diff_list, FRACPOS_SUM), f)
print('Elapsed time: {0} seconds'.format(time.perf_counter() - t))
class Filter(object):
"""
Astronomical filter object.
"""
def __init__(self, path=data_path):
"""
Parameters
----------
path : str (optional)
Path to `fft.fits` file
"""
self.path = path
self.table = Table(fits.getdata(path))
self.table.add_index('col0')
def available_filters(self):
"""
Return the available filters in the archive
"""
return self.table['col0'].data
def reconstruct(self, identifier):
"""
Reconstruct an approximate filter transmittance curve for
a given filter.
Parameters
----------
identifier : str
Name of the filter. To see available filters, run
`~tynt.Filter.available_filters()`
Returns
-------
wavelength : `~numpy.ndarray`
Wavelength array in Angstroms
transmittance : `~numpy.ndarray`
Approximate transmittance as a function of wavelength
"""
filt = list(self.table.loc[identifier])[1:]
n_lambda, lambda_0, delta_lambda, tr_max = filt[:4]
fft = filt[4:]
wavelength = np.arange(lambda_0,
(n_lambda + 1) * delta_lambda + lambda_0,
delta_lambda)
ifft = np.fft.ifft(fft, n=len(wavelength))
transmittance = ((ifft.real - ifft.real.min()) * tr_max /
ifft.real.ptp())
return wavelength, transmittance
def download_true_transmittance(self, identifier):
"""
Query the SVO service for a given filter, return the true transmittance
curve.
Parameters
----------
identifier : str
Name of the filter. To see available filters, run
`~tynt.Filter.available_filters()`
Returns
-------
wavelength : `~numpy.ndarray`
True wavelength array in Angstroms
transmittance : `~numpy.ndarray`
True transmittance as a function of wavelength
"""
path = download_file('http://svo2.cab.inta-csic.es/'
'theory/fps3/fps.php?ID={0}'.format(identifier))
true_transmittance = Table.read(path, format='votable')
return (true_transmittance['Wavelength'].data.data,
true_transmittance['Transmission'].data.data)
labels, n = ndimage.label(Gdiff*Gmask)
flaremask = (labels==3)
# Creating arrays containing time-serie data
Int_Inc = []
tiempos = []
for i in range(20):
diff = Mfiles[i+1].data-Mfiles[i].data
Mdiff = Map(np.nan_to_num(np.abs(diff)),Mfiles[i].meta)
Mdiffrot = Mdiff.rotate(angle=Mdiff.meta['crota2'] * u.deg)
Int_Inc.append((Mdiffrot.data*flaremask).sum())
tiempos.append(datetime.datetime.strptime(Mdiffrot.meta["date-obs"],'%Y-%m-%dT%H:%M:%S.%f'))
tbl_meta = {'t_key':'t_value'}
table = Table([tiempos, Int_Inc/np.max(Int_Inc)], names=['time', 'Inclination'], meta=Mfiles[i].meta)
table.add_index('time')
ts_table = ts.TimeSeries(table)
# PLOT
fig, ax = plt.subplots(figsize=(10,4))
ts_table.plot(marker='o',linewidth=3)
ax.axvline(tflare, color="gray", linestyle="--")
ax.text(tflare, np.min(Int_Inc/np.max(Int_Inc)), 'flare peak', fontsize=14,color='gray',rotation=90, rotation_mode='anchor')
ax.tick_params(axis='both',labelsize=14)
ax.xaxis.set_major_formatter(mdates.DateFormatter('%H:%M'))
ax.set_xlabel('Time [hour:min]',fontsize=16)
ax.set_ylabel('Normalized B-Inclination diff ',fontsize=14)
ax.set_title(Mfiles[imask].meta['date-obs'],fontsize=24)
plt.legend('',frameon=False) # fix bug with legend
fig.savefig('20131108T0422.pdf',dpi=150,bbox_inches='tight')
plt.show()
def make_todo(input_folder=None, cameras=None, ccds=None, overwrite=False):
"""
Create the TODO list which is used by the pipeline to keep track of the
targets that needs to be processed.
Will create the file `todo.sqlite` in the directory.
Parameters:
input_folder (string, optional): Input folder to create TODO list for.
If ``None``, the input directory in the environment variable ``TESSPHOT_INPUT`` is used.
cameras (iterable of integers, optional): TESS camera number (1-4). If ``None``, all cameras will be included.
ccds (iterable of integers, optional): TESS CCD number (1-4). If ``None``, all cameras will be included.
overwrite (boolean): Overwrite existing TODO file. Default=``False``.
Raises:
IOError: If the specified ``input_folder`` is not an existing directory.
.. codeauthor:: Rasmus Handberg <*****@*****.**>
"""
logger = logging.getLogger(__name__)
# Check the input folder, and load the default if not provided:
if input_folder is None:
input_folder = os.environ.get(
'TESSPHOT_INPUT',
os.path.join(os.path.dirname(__file__), 'tests', 'input'))
# Check that the given input directory is indeed a directory:
if not os.path.isdir(input_folder):
raise IOError("The given path does not exist or is not a directory")
# Make sure cameras and ccds are iterable:
cameras = (1, 2, 3, 4) if cameras is None else (cameras, )
ccds = (1, 2, 3, 4) if ccds is None else (ccds, )
Nccds = len(cameras) * len(ccds)
# The TODO file that we want to create. Delete it if it already exits:
todo_file = os.path.join(input_folder, 'todo.sqlite')
if os.path.exists(todo_file):
if overwrite:
os.remove(todo_file)
else:
logger.info("TODO file already exists")
return
# Create the TODO list as a table which we will fill with targets:
cat = Table(names=('starid', 'camera', 'ccd', 'datasource', 'tmag',
'cbv_area'),
dtype=('int64', 'int32', 'int32', 'S256', 'float32', 'int32'))
# Load list of all Target Pixel files in the directory:
tpf_files = find_tpf_files(input_folder)
logger.info("Number of TPF files: %d", len(tpf_files))
if len(tpf_files) > 0:
# Open a pool of workers:
logger.info("Starting pool of workers for TPFs...")
threads = int(
os.environ.get('SLURM_CPUS_PER_TASK', multiprocessing.cpu_count()))
threads = min(
threads, len(tpf_files)
) # No reason to use more than the number of jobs in total
logger.info("Using %d processes.", threads)
pool = multiprocessing.Pool(threads)
# Run the TPF files in parallel:
tic = default_timer()
_tpf_todo_wrapper = functools.partial(_tpf_todo,
input_folder=input_folder,
cameras=cameras,
ccds=ccds,
find_secondary_targets=False)
for cat2 in pool.imap_unordered(_tpf_todo_wrapper, tpf_files):
cat = vstack([cat, cat2], join_type='exact')
pool.close()
pool.join()
# Amount of time it took to process TPF files:
toc = default_timer()
logger.info("Elaspsed time: %f seconds (%f per file)", toc - tic,
(toc - tic) / len(tpf_files))
# Remove secondary TPF targets if they are also the primary target:
indx_remove = np.zeros(len(cat), dtype='bool')
cat.add_index('starid')
for k, row in enumerate(cat):
if row['datasource'].startswith('tpf:'):
indx = cat.loc['starid', row['starid']]['datasource'] == 'tpf'
if np.any(indx):
indx_remove[k] = True
cat.remove_indices('starid')
logger.info("Removing %d secondary TPF files as they are also primary",
np.sum(indx_remove))
cat = cat[~indx_remove]
# Find all targets in Full Frame Images:
inputs = itertools.product([input_folder], cameras, ccds)
# Open a pool of workers:
logger.info("Starting pool of workers for FFIs...")
threads = int(
os.environ.get('SLURM_CPUS_PER_TASK', multiprocessing.cpu_count()))
threads = min(
threads,
Nccds) # No reason to use more than the number of jobs in total
logger.info("Using %d processes.", threads)
pool = multiprocessing.Pool(threads)
ccds_done = 0
for cat2 in pool.imap_unordered(_ffi_todo_wrapper, inputs):
cat = vstack([cat, cat2], join_type='exact')
ccds_done += 1
logger.info("CCDs done: %d/%d", ccds_done, Nccds)
pool.close()
pool.join()
# Remove duplicates!
logger.info("Removing duplicate entries...")
_, idx = np.unique(cat[('starid', 'camera', 'ccd', 'datasource')],
return_index=True,
axis=0)
cat = cat[np.sort(idx)]
# Sort the final list:
cat.sort('tmag')
# TODO: Can we make decisions already now on methods?
# tmag < 2.5 : Halo photometry
# tmag < 6.5 : Aperture (saturated)
# tmag > 6.5 : Aperture (with PSF fallback)
# Write the TODO list to the SQLite database file:
logger.info("Writing TODO file...")
conn = sqlite3.connect(todo_file)
cursor = conn.cursor()
cursor.execute("""CREATE TABLE todolist (
priority BIGINT NOT NULL,
starid BIGINT NOT NULL,
datasource TEXT NOT NULL DEFAULT 'ffi',
camera INT NOT NULL,
ccd INT NOT NULL,
method TEXT DEFAULT NULL,
tmag REAL,
status INT DEFAULT NULL,
cbv_area INT NOT NULL
);""")
for pri, row in enumerate(cat):
cursor.execute(
"INSERT INTO todolist (priority,starid,camera,ccd,datasource,tmag,cbv_area) VALUES (?,?,?,?,?,?,?);",
(pri + 1, int(row['starid']), int(row['camera']), int(
row['ccd']), row['datasource'].strip(), float(
row['tmag']), int(row['cbv_area'])))
conn.commit()
cursor.execute("CREATE UNIQUE INDEX priority_idx ON todolist (priority);")
cursor.execute(
"CREATE INDEX starid_datasource_idx ON todolist (starid, datasource);"
) # FIXME: Should be "UNIQUE", but something is weird in ETE-6?!
cursor.execute("CREATE INDEX status_idx ON todolist (status);")
cursor.execute("CREATE INDEX starid_idx ON todolist (starid);")
conn.commit()
# Change settings of SQLite file:
cursor.execute("PRAGMA page_size=4096;")
# Run a VACUUM of the table which will force a recreation of the
# underlying "pages" of the file.
# Please note that we are changing the "isolation_level" of the connection here,
# but since we closing the conmnection just after, we are not changing it back
conn.isolation_level = None
cursor.execute("VACUUM;")
# Close connection:
cursor.close()
conn.close()
logger.info("TODO done.")
"reference_doi": [],
"acknowledgment": [],
"notes": [],
"col_units": []
}
for this_metadata_file in _metadata_files:
mydoc = minidom.parse(this_metadata_file)
metas = mydoc.getElementsByTagName('meta')
for elem in metas:
_column_data[elem.attributes['name'].value].append(
elem.firstChild.data)
dataset_index = Table(_column_data)
dataset_index.add_index('datafilename')
def load_dataset(filename):
"""Load a dataset.
Returns
-------
result: QTable
"""
datafile_path = os.path.join(_data_directory, filename)
result = QTable(ascii.read(datafile_path, format='csv', fast_reader=False))
# add the meta data
this_row = dataset_index.loc['dem_quiet_sun.csv']
meta_dict = dict(zip(this_row.colnames, this_row))
