def test_model_output():
data = load.model_output(sn_file)
data = load.model_output(nmbr_file)
data = load.model_output(yields_file)
data = load.model_output(masses_file_bin)
data = load.model_output(sed_file)
data = load.model_output(ion_file)
data = load.model_output(hr_file, hr_type='TL')
data = load.model_output(hr_file, hr_type='Tg')
data = load.model_output(hr_file, hr_type='TTG')
del data
class TestSpectraCompiler(object):
# Initialise model_output DataFrame return a smaller single dataframe
# This reduces I/O readings
data = model_output(f"{data_path}/spectra-bin-imf135_300.z002.dat")
# Patch the model_output function
@patch("hoki.data_compilers.np.loadtxt")
@patch("hoki.data_compilers.isfile")
def test_compiler(self, mock_isfile, mock_model_output):
# Set the model_output to the DataFrame
mock_model_output.return_value = self.data.to_numpy()
mock_isfile.return_value = True
spec = SpectraCompiler(f"{data_path}", f"{data_path}", "imf135_300")
# Check if pkl file is created
assert os.path.isfile(f"{data_path}/all_spectra-bin-imf135_300.npy")
# Check output dataframe
npt.assert_allclose(spec.output[3],
self.data.loc[:,
slice("6.0", "11.0")].T.to_numpy(),
err_msg="Complied spectra is wrong.")
# Remove created pickle
os.remove(f"{data_path}/all_spectra-bin-imf135_300.npy")
class TestLoadAllEmissivities(object):
# Initialise model_output DataFrame
# This reduces I/O readings
data = load.model_output(f"{data_path}/ionizing-bin-imf135_300.z002.dat")
# Patch the model_output function
@patch("hoki.data_compilers.np.loadtxt")
@patch("hoki.data_compilers.isfile")
def test_compile_emissivities(self, mock_isfile, mock_model_output):
# Set the model_output to the DataFrame
mock_model_output.return_value = self.data.to_numpy()
mock_isfile.return_value = True
res = load.emissivities_all_z(f"{data_path}", "imf135_300")
# Check if compiled file is created
assert os.path.isfile(f"{data_path}/all_ionizing-bin-imf135_300.npy"),\
"No compiled file is created."
# Check output numpy array
npt.assert_allclose(res[3],
self.data.drop(columns='log_age').to_numpy(),
err_msg="Loading of files has failed.")
def test_load_pickled_file(self):
res = load.emissivities_all_z(f"{data_path}", "imf135_300")
# Check output numpy array
npt.assert_allclose(res[3],
self.data.drop(columns='log_age').to_numpy(),
err_msg="Loading of compiled file has failed.")
os.remove(f"{data_path}/all_ionizing-bin-imf135_300.npy")
def packnload(file):
"""Load the data from a BPASS zipped file.
Parameters
----------
file : string
A string pointing to a zipped version of BPASS data.
Returns
-------
pandas DataFrame
A pandas DataFrame containing the model data from the BPASS file,
which is a *hoki* output.
"""
gunzip(file + ".gz", file)
out = load.model_output(file)
os.remove(file)
return out
class TestLoadAllRates(object):
# Setup files to load
data = load.model_output(f"{data_path}/supernova-bin-imf135_300.zem5.dat")
# Check if function loads rates
@patch("hoki.load.model_output")
def test_load_rates(self, mock_model_output):
mock_model_output.return_value = self.data
x = load.rates_all_z(f"{data_path}", "imf135_300"),\
"The rates cannot be initialised."
# Load rates
with patch("hoki.load.model_output") as mock_model_output:
mock_model_output.return_value = data
x = load.rates_all_z(f"{data_path}", "imf135_300")
# Test wrong inputs
def test_file_not_present(self):
with pytest.raises(AssertionError):
_ = load.rates_all_z(f"{data_path}", "imf135_300"),\
"The file is not present, but the load function runs."
def test_wrong_imf(self):
with pytest.raises(HokiKeyError):
_ = load.rates_all_z(f"{data_path}", "i"),\
"An unsupported IMF is taken as an input."
# Test output
def test_output_shape(self):
assert type(self.x) == pd.DataFrame
assert (self.x.columns.get_level_values(0).unique() ==
np.array(BPASS_EVENT_TYPES)).all(),\
"wrong headers read from the file."
assert (self.x.columns.get_level_values(1).unique() ==
np.array(BPASS_NUM_METALLICITIES)).all(),\
"wrong metallicity header"
def test_output(self):
assert np.isclose(self.x.loc[:, ("Ia", 0.00001)],
self.data["Ia"]).all(),\
"Models are not loaded correctly."
class TestEmissivityCompiler(object):
# Initialise model_output DataFrame return a smaller single dataframe
# This reduces I/O readings
data = model_output(f"{data_path}/ionizing-bin-imf135_300.z002.dat")
# Patch the model_output function
@patch("hoki.data_compilers.np.loadtxt")
@patch("hoki.data_compilers.isfile")
def test_compiler(self, mock_isfile, mock_model_output):
# Set the model_output to the DataFrame
mock_model_output.return_value = self.data.to_numpy()
mock_isfile.return_value = True
res = EmissivityCompiler(f"{data_path}", f"{data_path}", "imf135_300")
assert os.path.isfile(f"{data_path}/all_ionizing-bin-imf135_300.npy")
npt.assert_allclose(res.output[3],
self.data.drop(columns='log_age').to_numpy(),
err_msg="Compiled emissivities is wrong.")
os.remove(f"{data_path}/all_ionizing-bin-imf135_300.npy")
class TestLoadAllSpectra(object):
# Initialise model_output DataFrame
# This reduces I/O readings
data = load.model_output(f"{data_path}/spectra-bin-imf135_300.z002.dat")
# Patch the model_output function
@patch("hoki.data_compilers.np.loadtxt")
@patch("hoki.data_compilers.isfile")
def test_compile_spectra(self, mock_isfile, mock_model_output):
# Set the model_output to the DataFrame
mock_model_output.return_value = self.data.to_numpy()
mock_isfile.return_value = True
spec = load.spectra_all_z(f"{data_path}", "imf135_300")
# Check if compiled file is created
assert os.path.isfile(f"{data_path}/all_spectra-bin-imf135_300.npy"),\
"No compiled file is created."
# Check output numpy array
npt.assert_allclose(spec[3],
self.data.loc[:,
slice("6.0", "11.0")].T.to_numpy(),
err_msg="Loading of files has failed.")
def test_load_pickled_file(self):
spec = load.spectra_all_z(f"{data_path}", "imf135_300")
# Check output numpy array
npt.assert_allclose(spec[3],
self.data.loc[:,
slice("6.0", "11.0")].T.to_numpy(),
err_msg="Loading of compiled file has failed.")
os.remove(f"{data_path}/all_spectra-bin-imf135_300.npy")
def __init__(self, obs_df, model):
"""
Initialisation of the AgeWizard object
Parameters
----------
obs_df: pandas.DataFrame
Observational data. MUST contain a logT and logL column (for HRD comparison) or a col and mag column
(for CMD comparison)
model: str or hoki.hrdiagrams.HRDiagrams() hoki.cmd.CMD()
Location of the modeled HRD or CMD. This can be an already instanciated HRDiagram or CMD() object, or a
path to an HR Diagram file or a pickled CMD.
"""
# Making sure the osbervational properties are given in a format we can use.
if not isinstance(obs_df, pd.DataFrame):
raise HokiFormatError(
"Observations should be stored in a Data Frame")
if 'name' not in obs_df.columns:
warnings.warn(
"We expect the name of sources to be given in the 'name' column. "
"If I can't find names I'll make my own ;)", HokiFormatWarning)
# Checking what format they giving for the model:
if isinstance(model, hoki.hrdiagrams.HRDiagram):
self.model = model
elif isinstance(model, hoki.cmd.CMD):
self.model = model
elif isinstance(model, str) and 'hrs' in model:
self.model = load.model_output(model, hr_type='TL')
elif isinstance(model, str):
try:
self.model = load.unpickle(path=model)
except AssertionError:
print('-----------------')
print(
'HOKI DEBUGGER:\nThe model param should be a path to \na BPASS HRDiagram output file or pickled CMD,'
'or\na hoki.hrdiagrams.HRDiagram or a hoki.cmd.CMD')
print('-----------------')
raise HokiFatalError('model is ' + str(type(model)))
else:
print('-----------------')
print(
'HOKI DEBUGGER:\nThe model param should be a path to \na BPASS HRDiagram output file or pickled CMD,'
'or\na hoki.hrdiagrams.HRDiagram or a hoki.cmd.CMD')
print('-----------------')
raise HokiFatalError('model is ' + str(type(model)))
self.obs_df = obs_df
self.coordinates = find_coordinates(self.obs_df, self.model)
# This line is obsolete but might need revival if we ever want to add the not normalised distributions again
# self._distributions = calculate_distributions_normalised(self.obs_df, self.model)
self.pdfs = calculate_individual_pdfs(self.obs_df,
self.model).fillna(0)
self.sources = self.pdfs.columns.to_list()
self.sample_pdf = None
self._most_likely_age = None
import hoki.age_utils as au
import hoki.load as load
import pkg_resources
import numpy as np
import pandas as pd
import pytest
from hoki.utils.exceptions import HokiFatalError, HokiUserWarning, HokiFormatError
# Loading Data
data_path = pkg_resources.resource_filename('hoki', 'data')
hr_file = data_path + '/hrs-sin-imf_chab100.zem4.dat'
cmd_file = data_path + '/cmd_bv_z002_bin_imf135_300'
myhrd = load.model_output(hr_file, hr_type='TL')
mycmd = load.unpickle(cmd_file)
# Creating Test Inputs
fake_hrd_input = pd.DataFrame.from_dict({
'name': ['star1', 'star2', 'star3'],
'logT': np.array([4.58, 4.48, 4.14]),
'logL': np.array([4.83, 5.07, 5.40])
})
bad_hrd_input = pd.DataFrame.from_dict({
'logT': np.array(['bla']),
'logL': np.array([4.83])
})
no_name_input = pd.DataFrame.from_dict({
'logT': np.array([4.58, 4.48, 4.14]),
'logL': np.array([4.83, 5.07, 5.40])
from hoki import load import pandas as pd import matplotlib.pyplot as plt import os import time start_time = time.time() # specify which BPASS file to use # BPASS_file = 'spectra-bin-imf135_300.z020.dat' BPASS_file = 'spectra-bin-imf135_100.z002.dat' count = 0 # Load the BPASS data BPASS_data = load.model_output(BPASS_file) # Convert the BPASS data to microns. The flux is in luminosity per angstrom, so it is converted by multiplying # by angstroms/micron (10^4). The wavelength is in angstrom so it needs to be converted by multiplying by # microns/angstrom (10^-4), but this is easiest done by converting the whole data set at once then correcting # the wavelength conversion by another factor of 10^-4. # BPASS_data *= 10**4 # BPASS_data.WL *= 10**-4 time_list = BPASS_data.columns[BPASS_data.columns != 'WL'] time_list_exp = np.power(10, time_list.astype(float)) delta_t = np.zeros_like(time_list_exp) for i in range(len(delta_t)):
class TestCSPEventRate():
data = model_output(f"{data_path}/supernova-bin-imf135_300.zem5.dat")
# Check initalisation
@patch("hoki.load.model_output")
def test_init(self, mock_model_output):
mock_model_output.return_value = self.data
_ = er.CSPEventRate(f"{data_path}", "imf135_300")
def test_input_functions_at_time(self):
assert np.isclose(
self.CSP.at_time([sfh_fnc], [Z_fnc], ["Ia"], 0,
sample_rate=-1)[0]["Ia"],
0.002034966495416449), "Correct input is not taken."
assert np.isclose(
self.CSP.at_time(sfh_fnc, Z_fnc, ["Ia"], 0,
sample_rate=-1)[0]["Ia"],
0.002034966495416449), "Correct input is not taken."
assert np.isclose(
self.CSP.at_time(sfh, Z_fnc, ["Ia"], 0, sample_rate=-1)[0]["Ia"],
0.002034966495416449), "Correct input is not taken."
assert np.isclose(
self.CSP.at_time([sfh], Z_fnc, ["Ia"], 0)[0]["Ia"],
0.0018987009588956765), "Correct input is not taken."
assert np.isclose(
self.CSP.at_time([sfh, sfh], [Z_fnc, Z_fnc], ["Ia"], 0)[0]["Ia"],
0.0018987009588956765), "Correct input is not taken."
def test_event_rate_wrong_input(self):
with pytest.raises(HokiFormatError):
_ = self.CSP.over_time([sfh_fnc], [Z_fnc, Z_fnc], ["Ia"], 100)
with pytest.raises(ValueError):
_ = self.CSP.over_time([sfh_fnc], [Z_fnc], ["B"], 100)
with pytest.raises(HokiFormatError):
_ = self.CSP.over_time([sfh_fnc, sfh_fnc], [Z_fnc], ["B"], 100)
@patch("hoki.load.model_output")
def test_input_over_time(self, mock_model_output):
# Load model_output with a single supernova rate file
mock_model_output.return_value = self.data
CSP = er.CSPEventRate(f"{data_path}", "imf135_300")
test_out, time_edges = CSP.over_time([sfh_fnc], [Z_fnc], ["Ia"],
100,
return_time_edges=True)
# Load model_output with a single supernova rate file
with patch("hoki.load.model_output") as mock_model_output:
mock_model_output.return_value = data
CSP = er.CSPEventRate(f"{data_path}", "imf135_300")
test_out, time_edges = CSP.over_time([sfh_fnc], [Z_fnc], ["Ia"],
100,
return_time_edges=True)
def test_bins(self):
assert np.isclose(self.time_edges, np.linspace(0, HOKI_NOW, 101)).all(),\
"time edges are not properly set."
def test_event_rate_calculation(self):
expected = np.loadtxt(f"{data_path}/csp_test_data/type_Ia_rates.txt")
assert np.isclose(self.test_out["Ia"], expected).all(),\
"The event rate calculation is wrong."
def test_event_rate_calculation_multi_type(self):
out = self.CSP.over_time([sfh_fnc], [Z_fnc], ["Ia", "II"], 100)
assert len(out) == 1, "The output of calculate_over_time is wrong."
assert len(out[0]) == 2, "The output of calculate_over_time is wrong."
def test_event_rate_calculation_multi(self):
out = self.CSP.over_time([sfh_fnc, sfh_fnc], [Z_fnc, Z_fnc], ["Ia"],
100)
assert len(out) == 2, "The output ofcalculate_over_time is wrong."
assert len(out[0]) == 1, "The output of calculate_over_time is wrong."
def test_event_rate_at_time(self):
x = self.CSP.at_time([sfh_fnc], [Z_fnc], ["Ia"], 0)
assert np.isclose(x[0]["Ia"], 0.0018987009588956765),\
"The output of CSP.at_time is wrong."
def test_vector_input(self):
assert np.isclose(
self.CSP.at_time([vec_sfh], [vec_Z], ["Ia"], 0,
sample_rate=-1)[0]["Ia"],
0.002034966495416449), "Correct input is not taken."
def test_full_grid_over_time(self):
# Build mock 2D grid
nr_time_points = len(time_points)
SFH = np.zeros((1, 13, nr_time_points), dtype=np.float64)
bpass_Z_index = np.array([
np.argmin(np.abs(Z_fnc(i) - BPASS_NUM_METALLICITIES))
for i in time_points
])
SFH[0, bpass_Z_index, range(nr_time_points)] += np.array(
[sfh_fnc(i) for i in time_points])
out = self.CSP.grid_over_time(SFH, time_points, ["Ia", "IIP"], 100)
assert out.shape == (1, 13, 2, 100), "Output shape is wrong"
assert np.allclose(out[0][:, 0].sum(axis=0),
self.test_out["Ia"]), "Not the same as over_time"
def test_full_grid_at_time(self):
# Build mock 2D grid
nr_time_points = len(time_points)
SFH = np.zeros((1, 13, nr_time_points), dtype=np.float64)
bpass_Z_index = np.array([
np.argmin(np.abs(Z_fnc(i) - BPASS_NUM_METALLICITIES))
for i in time_points
])
SFH[0, bpass_Z_index, range(nr_time_points)] += np.array(
[sfh_fnc(i) for i in time_points])
out = self.CSP.grid_at_time(SFH, time_points, ["Ia", "IIP"], 0)
assert out.shape == (1, 13, 2), "Output shape is wrong"
assert np.isclose(out[0][:, 0].sum(axis=0),
0.0018987009588956765), "Not the same as over_time"
from hoki import load
import numpy as np
from glob import glob
from h5py_utils import write_data_h5py
# model_dir = 'BPASSv2.2.1_sin-imf_chab300'
# fname = 'bpass_sin.h5'
model_dir = 'BPASSv2.2.1_bin-imf_chab100'
fname = 'bpass.h5'
models = glob(model_dir + '/*')
output_temp = load.model_output(models[0])
ages = np.array([float(a) for a in output_temp.columns[1:]])
age_mask = (10**ages / 1e9) < 18 # Gyr
ages = ages[age_mask]
wl = output_temp['WL'].values
metallicities = np.array([None] * len(models))
spec = np.zeros((len(metallicities), len(ages), len(wl)))
for i, mod in enumerate(models):
try:
metallicities[i] = float('0.' + mod[-7:-4])
except: # ...catch em# format
metallicities[i] = 10**-float(mod[-5])
# sort by increasing metallicity
Z_idx = np.argsort(metallicities)
def generate_ssp_table_bpass(
ssp_lookup_file,
Zsol=Solar['total'],
return_table=False,
model_dir='/home/rad/caesar/BPASSv2.2.1_bin-imf_chab100'):
'''
Generates an SPS lookup table from BPASS.
'''
from hoki import load
from glob import glob
mylog.info('Generating BPASS SSP lookup table %s' % (ssp_lookup_file))
mylog.info('Using BPASS files in: %s' % (model_dir))
specfiles = glob(model_dir + '/spectra*') # these must be gunzipped
smfiles = glob(model_dir + '/starmass*') # these must be gunzipped
output_temp = load.model_output(specfiles[0])
#output_temp = output_temp[(output_temp.WL>LAMBDA_LO)&(output_temp.WL<LAMBDA_HI)] # restrict wavelength range for faster calculations
#print(specfiles[0],output_temp)
ages = np.array([float(a) for a in output_temp.columns[1:]])
age_mask = (10**ages / 1e9) < 18 # Gyr
ages = ages[age_mask]
wavelengths = output_temp['WL'].values
metallicities = np.array([None] * len(specfiles))
for i, mod in enumerate(specfiles): # parse metallicities from filenames
try:
metallicities[i] = float('0.' + mod[-7:-4])
except: # ...handle em5=1e-5 and em4=1e-4 cases
metallicities[i] = 10**-float(mod[-5])
# sort by increasing metallicity
Z_idx = np.argsort(metallicities)
metallicities = metallicities[Z_idx].astype(float)
ssp_spectra = np.zeros((len(ages) * len(metallicities), len(wavelengths)))
for iZ, mod in enumerate(np.array(specfiles)[Z_idx]):
output = load.model_output(mod)
#output = output[(output.WL>LAMBDA_LO)&(output.WL<LAMBDA_HI)] # restrict wavelength range for faster calculations
for iage, a in enumerate(ages):
j = iZ * len(ages) + iage
ssp_spectra[j] = output[str(a)].values
ssp_spectra[
j] *= wavelengths**2 / CLIGHT_AA # convert from per AA to per Hz
mass_remaining = []
for i, mod in enumerate(np.array(smfiles)[Z_idx]):
output = load.model_output(mod)
mass_remaining.append(output['stellar_mass'].values)
mass_remaining = np.asarray(mass_remaining).flatten() / 1.e6 # to Mo
# convert units
ssp_ages = ages # log yr
ssp_logZ = np.log10(metallicities)
ssp_spectra /= 1e6 # to Msol
#print(np.shape(mass_remaining),mass_remaining)
with h5py.File(ssp_lookup_file, 'w') as hf:
hf.create_dataset('fsps_options', data=model_dir)
hf.create_dataset('ages', data=ssp_ages)
hf.create_dataset('logZ', data=ssp_logZ)
hf.create_dataset('mass_remaining', data=mass_remaining)
hf.create_dataset('wavelengths', data=wavelengths)
hf.create_dataset('spectra', data=ssp_spectra)
memlog('Generated BPASS lookup table with %d ages and %d metallicities' %
(len(ssp_ages), len(ssp_logZ)))
if return_table:
return ssp_ages, ssp_logZ, mass_remaining, wavelengths, ssp_spectra
def __init__(self, obs_df, model, nsamples=100):
"""
Initialisation of the AgeWizard object
Parameters
----------
obs_df: pandas.DataFrame
Observational data. MUST contain a logT and logL column (for HRD comparison) or a col and mag column
(for CMD comparison)
model: str or hoki.hrdiagrams.HRDiagrams() hoki.cmd.CMD()
Location of the modeled HRD or CMD. This can be an already instanciated HRDiagram or CMD() object, or a
path to an HR Diagram file or a pickled CMD.
nsamples: int, optional
Number of times each data point should be sampled from its error distribution. Default is 100.
This only matters if you are taking errors into account.
"""
print(f"{Dialogue.info()} AgeWizard Starting")
print(f"{Dialogue.running()} Initial Checks")
# Making sure the osbervational properties are given in a format we can use.
if not isinstance(obs_df, pd.DataFrame):
raise HokiFormatError(
"Observations should be stored in a Data Frame")
if 'name' not in obs_df.columns:
warnings.warn(
"We expect the name of sources to be given in the 'name' column. "
"If I can't find names I'll make my own ;)", HokiFormatWarning)
# Checking what format they giving for the model:
if isinstance(model, hoki.hrdiagrams.HRDiagram):
self.model = model
elif isinstance(model, hoki.cmd.CMD):
self.model = model
elif isinstance(model, str) and 'hrs' in model:
self.model = load.model_output(model, hr_type='TL')
elif isinstance(model, str):
try:
self.model = load.unpickle(path=model)
except AssertionError:
print(f'{Dialogue.ORANGE}-----------------{Dialogue.ENDC}')
print(
f'{Dialogue.debugger()}\nThe model param should be a path to \na BPASS HRDiagram output file or pickled CMD,'
'or\na hoki.hrdiagrams.HRDiagram or a hoki.cmd.CMD')
print(f'{Dialogue.ORANGE}-----------------{Dialogue.ENDC}')
raise HokiFatalError('model is ' + str(type(model)))
else:
print(f'{Dialogue.ORANGE}-----------------{Dialogue.ENDC}')
print(
f'{Dialogue.debugger()}\nThe model param should be a path to \na BPASS HRDiagram output file or pickled CMD,'
'or\na hoki.hrdiagrams.HRDiagram or a hoki.cmd.CMD')
print(f'{Dialogue.ORANGE}-----------------{Dialogue.ENDC}')
raise HokiFatalError('model is ' + str(type(model)))
print(f"{Dialogue.complete()} Initial Checks")
self.obs_df = obs_df.copy()
# not needed?
# self.coordinates = find_coordinates(self.obs_df, self.model)
# This line is obsolete but might need revival if we ever want to add the not normalised distributions again
# self._distributions = calculate_distributions_normalised(self.obs_df, self.model)
self.pdfs = au.calculate_individual_pdfs(self.obs_df,
self.model,
nsamples=nsamples).fillna(0)
self.sources = self.pdfs.columns.to_list()
self.sample_pdf = None
self._most_likely_age = None
# plt.title(r'scale for V band')
# plt.savefig('V band tau scaling.png', dpi = 200)
## ---------------------------------------------------------------------
# Plot L_Edd
# ------------------------------------------------------------------
a_min = 0.001
a_max = 1
BPASS_file = 'spectra-bin-imf100_300.z001.dat'
SM_file = BPASS_file.replace('spectra','starmass')
BPASS_data = load.model_output(BPASS_file)
BPASS_data.WL *= 10**-4
time_list = BPASS_data.columns[BPASS_data.columns != 'WL']
time_list_exp = np.power(10,time_list.astype(float))
wl_min = 0.001
wl_max = 10
BPASS_data = BPASS_data[ (BPASS_data.WL >= wl_min) & (BPASS_data.WL <= wl_max) ]
wl_list = BPASS_data.WL.to_numpy()
kappa_av_RP = np.zeros_like(time_list)
kappa_av_F = np.zeros_like(time_list)
kappa_RP = np.zeros_like(wl_list)
