def __init__(self, spectrum_library_to_analyse, workspace, pipeline):
"""
Open the spectrum library containing the spectra that we are to analyse.
:param spectrum_library_to_analyse:
The name of the spectrum library we are to analyse
:type spectrum_library_to_analyse:
str
:param workspace:
Directory where we expect to find spectrum libraries.
:type workspace:
str
:param pipeline:
The Pipeline we are to run spectra through.
:type pipeline:
Pipeline
"""
# Initialise pipeline manager
super(PipelineManagerReadFromSpectrumLibrary,
self).__init__(pipeline=pipeline)
# Open the spectrum library we are reading from
self.spectrum_library_to_analyse = spectrum_library_to_analyse
spectra = SpectrumLibrarySqlite.open_and_search(
library_spec=spectrum_library_to_analyse,
workspace=workspace,
extra_constraints={})
input_library, input_library_items = [
spectra[i] for i in ("library", "items")
]
input_library_ids = [i["specId"] for i in input_library_items]
self.input_library = input_library
self.input_library_items = input_library_items
self.input_library_ids = input_library_ids
self.spectrum_counter = 0
default="/tmp/gaussian_convolution_{}.log".format(pid),
dest="log_to",
help="Specify a log file where we log our progress.")
args = parser.parse_args()
logger.info("Calculating SNR ratios of spectra in <{}>".format(
args.input_library))
# Set path to workspace where we create libraries of spectra
workspace = args.workspace if args.workspace else os_path.join(
our_path, "../../../workspace")
os.system("mkdir -p {}".format(workspace))
# Open input SpectrumLibrary, and search for flux normalised spectra meeting our filtering constraints
spectra = SpectrumLibrarySqlite.open_and_search(
library_spec=args.input_library,
workspace=workspace,
extra_constraints={"continuum_normalised": 0})
# Get a list of the spectrum IDs which we were returned
input_library, input_spectra_ids, input_spectra_constraints = [
spectra[i] for i in ("library", "items", "constraints")
]
# Parse any definitions of SNR we were supplied on the command line
if (args.snr_definitions is None) or (len(args.snr_definitions) < 1):
snr_definitions = None
else:
snr_definitions = []
for snr_definition in args.snr_definitions:
words = snr_definition.split(",")
snr_definitions.append([words[0], float(words[1]), float(words[2])])
# Set path to workspace where we expect to find libraries of spectra
our_path = os_path.split(os_path.abspath(__file__))[0]
workspace = os_path.join(our_path, "../../../../workspace")
# Create directory to store output files in
os.system("mkdir -p {}".format(args.output_stub))
# Fetch title for this Cannon run
cannon_output = json.loads(
gzip.open(args.cannon + ".summary.json.gz", "rt").read())
description = cannon_output['description']
# Open spectrum library we originally trained the Cannon on
training_spectra_info = SpectrumLibrarySqlite.open_and_search(
library_spec=cannon_output["train_library"],
workspace=workspace,
extra_constraints={"continuum_normalised": 1})
training_library, training_library_items = [
training_spectra_info[i] for i in ("library", "items")
]
# Load training set
training_library_ids = [i["specId"] for i in training_library_items]
training_spectra = training_library.open(ids=training_library_ids)
# Recreate a Cannon instance, using the saved state
censoring_masks = cannon_output["censoring_mask"]
if censoring_masks is not None:
for key, value in censoring_masks.items():
censoring_masks[key] = np.asarray(value)
from fourgp_speclib import SpectrumLibrarySqlite
# Start logging our progress
logging.basicConfig(level=logging.INFO, format='[%(asctime)s] %(levelname)s:%(filename)s:%(message)s',
datefmt='%d/%m/%Y %H:%M:%S')
logger = logging.getLogger(__name__)
logger.info("Synthesizing spectra of pepsi")
# Instantiate base synthesizer
synthesizer = Synthesizer(library_name="turbo_pepsi_replica_3label",
logger=logger,
docstring=__doc__)
spectra = SpectrumLibrarySqlite.open_and_search(
library_spec='pepsi_4fs_hrs/',
workspace='/home/travegre/Projects/4GP/4most-4gp-scripts/workspace/',
extra_constraints={"continuum_normalised": True}
)
pepsi_library, pepsi_library_items = [spectra[i] for i in ("library", "items")]
# Load test set
pepsi_library_ids = [i["specId"] for i in pepsi_library_items]
spectra = [pepsi_library.open(ids=i).extract_item(0) for i in pepsi_library_ids]
print(spectra)
star_list = []
for spectrum in spectra:
try:
#star_list.append({'name': spectrum.metadata['Starname'], 'Teff': spectrum.metadata['Teff'], 'logg': spectrum.metadata['logg'], '[Fe/H]': spectrum.metadata['[Fe/H]'], 'microturbulence': spectrum.metadata['vmic_GES'], 'extra_metadata': {'set_id': 1}})
star_list.append({'name': spectrum.metadata['Starname'], 'Teff': spectrum.metadata['Teff'], 'logg': spectrum.metadata['logg'], '[Fe/H]': spectrum.metadata['[Fe/H]'], 'extra_metadata': {'set_id': 1}})
def main():
"""
Main entry point for running the Payne.
"""
global logger
logging.basicConfig(
level=logging.INFO,
format='[%(asctime)s] %(levelname)s:%(filename)s:%(message)s',
datefmt='%d/%m/%Y %H:%M:%S')
logger = logging.getLogger(__name__)
# Read input parameters
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument(
'--test',
required=True,
dest='test_library',
help=
"Library of spectra to test the trained Payne on. Stars may be filtered by parameters by "
"placing a comma-separated list of constraints in [] brackets after the name of the "
"library. Use the syntax [Teff=3000] to demand equality, or [0<[Fe/H]<0.2] to specify a "
"range.")
parser.add_argument(
'--train',
required=True,
dest='train_library',
help=
"Library of labelled spectra to train the Payne on. Stars may be filtered by parameters "
"by placing a comma-separated list of constraints in [] brackets after the name of the "
"library. Use the syntax [Teff=3000] to demand equality, or [0<[Fe/H]<0.2] to specify a "
"range.")
parser.add_argument(
'--workspace',
dest='workspace',
default="",
help="Directory where we expect to find spectrum libraries.")
parser.add_argument(
'--train-batch-number',
required=False,
dest='train_batch_number',
type=int,
default=0,
help=
"If training pixels in multiple batches on different machines, then this is the number of "
"the batch of pixels we are to train. It should be in the range 0 .. batch_count-1 "
"inclusive. If it is -1, then we skip training to move straight to testing."
)
parser.add_argument(
'--test-batch-number',
required=False,
dest='test_batch_number',
type=int,
default=0,
help=
"If testing spectra in multiple batches on different machines, then this is the number of "
"the batch of spectra we are to test. It should be in the range 0 .. test_batch_count-1 "
"inclusive.")
parser.add_argument(
'--num-training-workers',
required=False,
dest='train_batch_count',
type=int,
default=1,
help=
"If training pixels in multiple batches on different machines, then this is the number "
"of nodes/workers/batches.")
parser.add_argument(
'--num-testing-workers',
required=False,
dest='test_batch_count',
type=int,
default=1,
help=
"If testing spectra in multiple batches on different machines, then this is the number "
"of nodes/workers/batches.")
parser.add_argument(
'--reload-payne',
required=False,
dest='reload_payne',
default=None,
help=
"Skip training step, and reload a Payne that we've previously trained."
)
parser.add_argument('--description',
dest='description',
help="A description of this fitting run.")
parser.add_argument(
'--labels',
dest='labels',
default="Teff,logg,[Fe/H]",
help="List of the labels the Payne is to learn to estimate.")
parser.add_argument(
'--label-expressions',
dest='label_expressions',
default="",
help="List of the algebraic labels the Payne is to learn to estimate "
"(e.g. photometry_B - photometry_V).")
parser.add_argument(
'--labels-individual',
dest='labels_individual',
default="",
help="List of the labels the Payne is to fit in separate fitting runs."
)
parser.add_argument('--censor-scheme',
default="1",
dest='censor_scheme',
help="Censoring scheme version to use (1, 2 or 3).")
parser.add_argument(
'--censor',
default="",
dest='censor_line_list',
help=
"Optional list of line positions for the Payne to fit, ignoring continuum between."
)
parser.add_argument('--output-file',
default="./test_cannon.out",
dest='output_file',
help="Data file to write output to.")
parser.add_argument(
'--assume-scaled-solar',
action='store_true',
dest="assume_scaled_solar",
help=
"Assume scaled solar abundances for any elements which don't have abundances individually "
"specified. Useful for working with incomplete data sets.")
parser.add_argument(
'--no-assume-scaled-solar',
action='store_false',
dest="assume_scaled_solar",
help=
"Do not assume scaled solar abundances; throw an error if training set is has missing "
"labels.")
parser.set_defaults(assume_scaled_solar=False)
parser.add_argument('--multithread',
action='store_true',
dest="multithread",
help="Use multiple thread to speed Payne up.")
parser.add_argument(
'--nothread',
action='store_false',
dest="multithread",
help="Do not use multiple threads - use only one CPU core.")
parser.set_defaults(multithread=True)
parser.add_argument(
'--interpolate',
action='store_true',
dest="interpolate",
help=
"Interpolate the test spectra on the training spectra's wavelength raster. DANGEROUS!"
)
parser.add_argument(
'--nointerpolate',
action='store_false',
dest="interpolate",
help="Do not interpolate the test spectra onto a different raster.")
parser.set_defaults(interpolate=False)
parser.add_argument(
'--train-wavelength-window',
dest="train_wavelength_window",
help="Use only the selected wavelength region for the training")
parser.set_defaults(train_wavelength_window=False)
parser.add_argument(
'--neuron-count',
dest="neuron_count",
help="Number of neurons in each of the Payne NN layers")
parser.set_defaults(neuron_count=10)
args = parser.parse_args()
logger.info("Testing Payne with arguments <{}> <{}> <{}> <{}>".format(
args.test_library, args.train_library, args.censor_line_list,
args.output_file))
# List of labels over which we are going to test the performance of the Payne
test_label_fields = args.labels.split(",")
# List of labels we're going to fit individually
if args.labels_individual:
test_labels_individual = [
i.split("+") for i in args.labels_individual.split(",")
]
else:
test_labels_individual = [[]]
# Set path to workspace where we expect to find libraries of spectra
our_path = os_path.split(os_path.abspath(__file__))[0]
workspace = args.workspace if args.workspace else os_path.join(
our_path, "../../../workspace")
# Open training set
spectra = SpectrumLibrarySqlite.open_and_search(
library_spec=args.train_library,
workspace=workspace,
extra_constraints={"continuum_normalised": True})
training_library, training_library_items = [
spectra[i] for i in ("library", "items")
]
# Open test set
spectra = SpectrumLibrarySqlite.open_and_search(
library_spec=args.test_library,
workspace=workspace,
extra_constraints={"continuum_normalised": True})
test_library, test_library_items = [
spectra[i] for i in ("library", "items")
]
# Load training set
training_library_ids_all = [i["specId"] for i in training_library_items]
training_spectra_all = training_library.open(ids=training_library_ids_all)
raster = training_spectra_all.wavelengths
# Load test set
test_library_ids = [i["specId"] for i in test_library_items]
# Fit each set of labels we're fitting individually, one by one
for labels_individual_batch_count, test_labels_individual_batch in enumerate(
test_labels_individual):
# Create filename for the output from this Payne run
output_filename = args.output_file
# If we're fitting elements individually, individually number the runs to fit each element
if len(test_labels_individual) > 1:
output_filename += "-{:03d}".format(labels_individual_batch_count)
# If requested, fill in any missing labels on the training set by assuming scaled-solar abundances
if args.assume_scaled_solar:
training_spectra = autocomplete_scaled_solar_abundances(
input_spectra=training_spectra_all,
label_list=test_label_fields + test_labels_individual_batch)
else:
training_spectra = filter_training_spectra(
input_spectra=training_spectra_all,
label_list=test_label_fields + test_labels_individual_batch,
input_library=training_library,
input_spectrum_ids=training_library_ids_all)
# Evaluate labels which are calculated via metadata expressions
test_labels_expressions = []
if args.label_expressions.strip():
test_labels_expressions = args.label_expressions.split(",")
evaluate_computed_labels(label_expressions=test_labels_expressions,
spectra=training_spectra)
# Make combined list of all labels the Payne is going to fit
test_labels = test_label_fields + test_labels_individual_batch + test_labels_expressions
logger.info("Beginning fit of labels <{}>.".format(
",".join(test_labels)))
# If required, generate the censoring masks
censoring_masks = create_censoring_masks(
censoring_scheme=int(args.censor_scheme),
raster=raster,
censoring_line_list=args.censor_line_list,
label_fields=test_label_fields + test_labels_individual_batch,
label_expressions=test_labels_expressions,
logger=logger)
# Construct and train a model
time_training_start = time.time()
if args.train_wavelength_window:
train_window_mask = (training_spectra.wavelengths > float(args.train_wavelength_window.split('-')[0])) \
& (training_spectra.wavelengths < float(args.train_wavelength_window.split('-')[1]))
training_spectra.wavelengths = training_spectra.wavelengths[
train_window_mask]
training_spectra.values = training_spectra.values[:,
train_window_mask]
training_spectra.value_errors = training_spectra.value_errors[:,
train_window_mask]
if args.reload_payne == 'true':
model = PayneInstanceTing(
training_set=training_spectra,
label_names=test_labels,
neuron_count=int(args.neuron_count),
batch_number=args.train_batch_number,
batch_count=args.train_batch_count,
censors=censoring_masks,
threads=None if args.multithread else 1,
training_data_archive=output_filename,
load_from_archive=True,
)
else:
model = PayneInstanceTing(training_set=training_spectra,
label_names=test_labels,
neuron_count=int(args.neuron_count),
batch_number=args.train_batch_number,
batch_count=args.train_batch_count,
censors=censoring_masks,
threads=None if args.multithread else 1,
training_data_archive=output_filename)
time_training_end = time.time()
# Plot some characteristic spectra from the Payne generative model
if False:
def sigmoid_def(z):
return 1.0 / (1.0 + np.exp(-z))
payne_status = model._payne_status
w_array_0 = payne_status["w_array_0"]
w_array_1 = payne_status["w_array_1"]
w_array_2 = payne_status["w_array_2"]
b_array_0 = payne_status["b_array_0"]
b_array_1 = payne_status["b_array_1"]
b_array_2 = payne_status["b_array_2"]
x_min = payne_status["x_min"]
x_max = payne_status["x_max"]
# logging.info(w_array_0.shape) # dim1 - N pixels, dim2 - N neurons, dim3 - N labels
labels = np.array([[5000, 4, 0], [5500, 4, 0], [5000, 4, -1.],
[5000, 5, 0], [4000, 4, 0]])
labels = (labels - x_min) / (x_max - x_min) - 0.5
import matplotlib
#matplotlib.use('Agg')
import matplotlib.pyplot as plt
fig = plt.figure(figsize=(12, 8), dpi=200)
for i in training_spectra.values:
plt.plot(training_spectra.wavelengths, i, lw=0.5, alpha=0.2)
colors = ['yellow', 'orange', 'red', 'green', 'blue']
for j, i in enumerate(labels):
predict_flux = w_array_2 * sigmoid_def(
np.sum(w_array_1 *
(sigmoid_def(np.dot(w_array_0, i) + b_array_0)),
axis=1) + b_array_1) + b_array_2
plt.plot(training_spectra.wavelengths,
predict_flux,
lw=2,
c=colors[j])
plt.show()
fig.savefig("{:s}.characteristic_gen_model_plot.png".format(
output_filename),
format='png')
# Test the model
if not os.path.exists(
os.path.join(
output_filename,
"batch_{:04d}_of_{:04d}.full.json.gz".format(
args.test_batch_number, args.test_batch_count))):
N = len(test_library_ids)
time_taken = np.zeros(N)
results = []
spec_start = (N // args.test_batch_count +
1) * args.test_batch_number
spec_end = min(N, (N // args.test_batch_count + 1) *
(args.test_batch_number + 1))
threads = 1 #cpu_count()
#srng = split_seq(range(spec_start, spec_end), threads)
if not args.train_wavelength_window:
train_window_mask = False
params = [
spec_start, spec_end, args, training_spectra, censoring_masks,
model, test_labels, train_window_mask
]
manager = multiprocessing.Manager()
for batch in range(spec_start, spec_end)[::threads]:
batch = [[
i,
test_library.open(ids=test_library_ids[i]).extract_item(0)
] + params for i in range(batch, batch + threads)
if i < len(test_library_ids)]
dicti = manager.list()
ps = []
for i in batch:
spectrum = i[1]
ref_labels = []
for j in test_labels:
try:
print(j, ' ', spectrum.metadata[j])
ref_labels.append(spectrum.metadata[j])
except:
continue
#if len(ref_labels) != 3:
# continue
#for j in test_labels[3:]:
# ref_labels.append(0)
p = multiprocessing.Process(target=parallel_fit,
args=(i, dicti,
np.array(ref_labels)))
ps.append(p)
p.start()
for p in ps:
p.join()
#with Pool(threads) as pool:
# batch_results = pool.map(parallel_fit, [[i, test_library.open(ids=test_library_ids[i]).extract_item(0)]+params for i in range(batch, batch+threads) if i<len(test_library_ids)])
for result in dicti:
results.append(result)
# Report time taken
logger.info(
"Fitting of {:d} spectra completed. Took {:.2f} +/- {:.2f} sec / spectrum."
.format((spec_end - spec_start), np.mean(time_taken),
np.std(time_taken)))
# Create output data structure
censoring_output = None
if censoring_masks is not None:
censoring_output = dict([
(label, tuple([int(i) for i in mask]))
for label, mask in censoring_masks.items()
])
output_data = {
"hostname": os.uname()[1],
"generator": __file__,
"4gp_version": fourgp_version,
"cannon_version": None,
"payne_version": model.payne_version,
"start_time": time_training_start,
"end_time": time.time(),
"training_time": time_training_end - time_training_start,
"description": args.description,
"train_library": args.train_library,
"test_library": args.test_library,
"tolerance": None,
"assume_scaled_solar": args.assume_scaled_solar,
"line_list": args.censor_line_list,
"labels": test_labels,
"wavelength_raster": tuple(raster),
"censoring_mask": censoring_output
}
# Write brief summary of run to JSON file, without masses of data
#with gzip.open("{:s}.summary.json.gz".format(output_filename), "wt") as f:
# f.write(json.dumps(output_data, indent=2))
with gzip.open(
os.path.join(
output_filename,
"batch_{:04d}_of_{:04d}.summary.json.gz".format(
args.test_batch_number, args.test_batch_count)),
"wt") as f:
f.write(json.dumps(output_data, indent=2))
# Write full results to JSON file
output_data["spectra"] = results
#with gzip.open("{:s}.full.json.gz".format(output_filename), "wt") as f:
# f.write(json.dumps(output_data, indent=2))
with gzip.open(
os.path.join(
output_filename,
"batch_{:04d}_of_{:04d}.full.json.gz".format(
args.test_batch_number, args.test_batch_count)),
"wt") as f:
f.write(json.dumps(output_data, indent=2))
logging.info(
"Saving results, batch {:04d} of {:04d} completed".format(
args.test_batch_number, args.test_batch_count))
else:
# Load teh test results from batches and join them
logging.info("Loading Payne results from disk")
payne_batches_summary = {}
payne_batches_full = {}
for i in range(args.test_batch_count):
filename_summary = os.path.join(
output_filename,
"batch_{:04d}_of_{:04d}.summary.json.gz".format(
i, args.test_batch_count))
filename_full = os.path.join(
output_filename,
"batch_{:04d}_of_{:04d}.full.json.gz".format(
i, args.test_batch_count))
with gzip.open(filename_summary, "r") as f:
payne_batches_summary.update(
json.loads(f.read().decode('utf-8')))
with gzip.open(filename_full, "r") as f:
if i == 0:
payne_batches_full.update(
json.loads(f.read().decode('utf-8')))
else:
payne_batches_full['spectra'].extend(
json.loads(f.read().decode('utf-8'))['spectra'])
assert os.path.exists(filename_summary), "Could not proceed with joinning results, because " \
"test data for batch {:d} of spectra is not present " \
"on this server.".format(i)
logging.info("Payne results loaded successfully")
with gzip.open("{:s}.full.json.gz".format(output_filename),
"wt") as f:
f.write(json.dumps(payne_batches_full, indent=2))
with gzip.open("{:s}.summary.json.gz".format(output_filename),
"wt") as f:
f.write(json.dumps(payne_batches_summary, indent=2))
logging.info("Payne batches merged successfully")
def main():
"""
Main entry point for running the Cannon.
"""
logging.basicConfig(level=logging.INFO, format='[%(asctime)s] %(levelname)s:%(filename)s:%(message)s',
datefmt='%d/%m/%Y %H:%M:%S')
logger = logging.getLogger(__name__)
# Read input parameters
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('--test', required=True, dest='test_library',
help="Library of spectra to test the trained Cannon on. Stars may be filtered by parameters by "
"placing a comma-separated list of constraints in [] brackets after the name of the "
"library. Use the syntax [Teff=3000] to demand equality, or [0<[Fe/H]<0.2] to specify a "
"range.")
parser.add_argument('--train', required=False, dest='train_library', default=None,
help="Library of labelled spectra to train the Cannon on. Stars may be filtered by parameters "
"by placing a comma-separated list of constraints in [] brackets after the name of the "
"library. Use the syntax [Teff=3000] to demand equality, or [0<[Fe/H]<0.2] to specify a "
"range.")
parser.add_argument('--workspace', dest='workspace', default="",
help="Directory where we expect to find spectrum libraries.")
parser.add_argument('--cannon-version', default="casey_old", dest='cannon_version',
choices=("casey_old", "casey_new", "anna_ho"),
help="Select which implementation of the Cannon to use: Andy Casey's or Anna Ho's.")
parser.add_argument('--polynomial-order', default=2, dest='polynomial_order', type=int,
help="The maximum order of polynomials to use as basis functions in the Cannon.")
parser.add_argument('--continuum-normalisation', default="none", dest='continuum_normalisation',
help="Select continuum normalisation method: none, running_mean or polynomial.")
parser.add_argument('--reload-cannon', required=False, dest='reload_cannon', default=None,
help="Skip training step, and reload a Cannon that we've previously trained. Specify the full "
"path to the .cannon file containing the trained Cannon, but without the .cannon suffix.")
parser.add_argument('--description', dest='description',
help="A description of this fitting run.")
parser.add_argument('--labels', dest='labels',
default="Teff,logg,[Fe/H]",
help="List of the labels the Cannon is to learn to estimate.")
parser.add_argument('--label-expressions', dest='label_expressions',
default="",
help="List of the algebraic labels the Cannon is to learn to estimate "
"(e.g. photometry_B - photometry_V).")
parser.add_argument('--labels-individual', dest='labels_individual',
default="",
help="List of the labels the Cannon is to fit in separate fitting runs.")
parser.add_argument('--censor-scheme', default="1", dest='censor_scheme',
help="Censoring scheme version to use (1, 2 or 3).")
parser.add_argument('--censor', default="", dest='censor_line_list',
help="Optional list of line positions for the Cannon to fit, ignoring continuum between.")
parser.add_argument('--tolerance', default=None, dest='tolerance', type=float,
help="The parameter xtol which is passed to the scipy optimisation routines as xtol to "
"determine whether they have converged.")
parser.add_argument('--output-file', default="./test_cannon.out", dest='output_file',
help="Data file to write output to.")
parser.add_argument('--assume-scaled-solar',
action='store_true',
dest="assume_scaled_solar",
help="Assume scaled solar abundances for any elements which don't have abundances individually "
"specified. Useful for working with incomplete data sets.")
parser.add_argument('--no-assume-scaled-solar',
action='store_false',
dest="assume_scaled_solar",
help="Do not assume scaled solar abundances; throw an error if training set is has missing "
"labels.")
parser.set_defaults(assume_scaled_solar=False)
parser.add_argument('--multithread',
action='store_true',
dest="multithread",
help="Use multiple thread to speed Cannon up.")
parser.add_argument('--nothread',
action='store_false',
dest="multithread",
help="Do not use multiple threads - use only one CPU core.")
parser.set_defaults(multithread=True)
parser.add_argument('--interpolate',
action='store_true',
dest="interpolate",
help="Interpolate the test spectra on the training spectra's wavelength raster. DANGEROUS!")
parser.add_argument('--nointerpolate',
action='store_false',
dest="interpolate",
help="Do not interpolate the test spectra onto a different raster.")
parser.set_defaults(interpolate=False)
args = parser.parse_args()
logging.info("Testing Cannon with arguments <{}> <{}> <{}> <{}>".format(args.test_library,
args.train_library,
args.censor_line_list,
args.output_file))
# Pick which Cannon version to use
cannon_class, continuum_normalised_testing, continuum_normalised_training = \
select_cannon(cannon_version=args.cannon_version,
continuum_normalisation=args.continuum_normalisation)
# List of labels over which we are going to test the performance of the Cannon
test_label_fields = args.labels.split(",")
# List of labels we're going to fit individually
if args.labels_individual:
test_labels_individual = [i.split("+") for i in args.labels_individual.split(",")]
else:
test_labels_individual = [[]]
# Set path to workspace where we expect to find libraries of spectra
our_path = os_path.split(os_path.abspath(__file__))[0]
workspace = args.workspace if args.workspace else os_path.join(our_path, "../../../workspace")
# Find out whether we're reloading a previously saved Cannon
reloading_cannon = args.reload_cannon is not None
# Open training set
training_library = training_library_ids_all = None
if not reloading_cannon:
spectra = SpectrumLibrarySqlite.open_and_search(
library_spec=args.train_library,
workspace=workspace,
extra_constraints={"continuum_normalised": continuum_normalised_training}
)
training_library, training_library_items = [spectra[i] for i in ("library", "items")]
# Make list of IDs of all spectra in the training set
training_library_ids_all = [i["specId"] for i in training_library_items]
# Open test set
spectra = SpectrumLibrarySqlite.open_and_search(
library_spec=args.test_library,
workspace=workspace,
extra_constraints={"continuum_normalised": continuum_normalised_testing}
)
test_library, test_library_items = [spectra[i] for i in ("library", "items")]
# Make list of IDs of all spectra in the test set
test_library_ids = [i["specId"] for i in test_library_items]
# Fit each set of labels we're fitting individually, one by one
for labels_individual_batch_count, test_labels_individual_batch in enumerate(test_labels_individual):
# Create filename for the output from this Cannon run
output_filename = args.output_file
# If we're fitting elements individually, individually number the runs to fit each element
if len(test_labels_individual) > 1:
output_filename += "-{:03d}".format(labels_individual_batch_count)
# Sequence of tasks if we're reloading a pre-saved Cannon from disk
if reloading_cannon:
# Load the JSON data that summarises the Cannon training that we're about to reload
json_summary_filename = "{}.summary.json.gz".format(args.reload_cannon)
cannon_pickle_filename = "{}.cannon".format(args.reload_cannon)
with gzip.open(json_summary_filename, "rt") as f:
summary_json = json.loads(f.read())
raster = np.array(summary_json['wavelength_raster'])
test_labels = summary_json['labels']
training_library_ids = summary_json['training_spectra_ids']
training_library_string = summary_json['train_library']
assume_scaled_solar = summary_json['assume_scaled_solar']
tolerance = summary_json['tolerance']
line_list = summary_json['line_list']
censoring_masks = None
# If we're doing our own continuum normalisation, we need to treat each wavelength arm separately
wavelength_arm_breaks = SpectrumProperties(raster).wavelength_arms()['break_points']
time_training_start = time.time()
model = cannon_class(training_set=None,
wavelength_arms=wavelength_arm_breaks,
load_from_file=cannon_pickle_filename,
label_names=test_labels,
tolerance=args.tolerance,
polynomial_order=args.polynomial_order,
censors=None,
threads=None if args.multithread else 1
)
time_training_end = time.time()
# Sequence of tasks if we're training a Cannon from scratch
else:
training_library_string = args.train_library
assume_scaled_solar = args.assume_scaled_solar
tolerance = args.tolerance
line_list = args.censor_line_list
# If requested, fill in any missing labels on the training set by assuming scaled-solar abundances
if args.assume_scaled_solar:
training_library_ids, training_spectra = autocomplete_scaled_solar_abundances(
training_library=training_library,
training_library_ids_all=training_library_ids_all,
label_list=test_label_fields + test_labels_individual_batch
)
# Otherwise we reject any training spectra which have incomplete labels
else:
training_library_ids, training_spectra = filter_training_spectra(
training_library=training_library,
training_library_ids_all=training_library_ids_all,
label_list=test_label_fields + test_labels_individual_batch
)
# Look up the raster on which the training spectra are sampled
raster = training_spectra.wavelengths
# Evaluate labels which are calculated via metadata expressions
test_labels_expressions = []
if args.label_expressions.strip():
test_labels_expressions = args.label_expressions.split(",")
evaluate_computed_labels(label_expressions=test_labels_expressions, spectra=training_spectra)
# Make combined list of all labels the Cannon is going to fit
test_labels = test_label_fields + test_labels_individual_batch + test_labels_expressions
logging.info("Beginning fit of labels <{}>.".format(",".join(test_labels)))
# If required, generate the censoring masks
censoring_masks = create_censoring_masks(
censoring_scheme=int(args.censor_scheme),
raster=raster,
censoring_line_list=args.censor_line_list,
label_fields=test_label_fields + test_labels_individual_batch,
label_expressions=test_labels_expressions
)
# If we're doing our own continuum normalisation, we need to treat each wavelength arm separately
wavelength_arm_breaks = SpectrumProperties(raster).wavelength_arms()['break_points']
# Construct and train a model
time_training_start = time.time()
model = cannon_class(training_set=training_spectra,
wavelength_arms=wavelength_arm_breaks,
label_names=test_labels,
tolerance=args.tolerance,
polynomial_order=args.polynomial_order,
censors=censoring_masks,
threads=None if args.multithread else 1
)
time_training_end = time.time()
# Save the model
model.save_model(filename="{:s}.cannon".format(output_filename),
overwrite=True)
# Test the model
N = len(test_library_ids)
time_taken = np.zeros(N)
results = []
for index in range(N):
test_spectrum_array = test_library.open(ids=test_library_ids[index])
spectrum = test_spectrum_array.extract_item(0)
logging.info("Testing {}/{}: {}".format(index + 1, N, spectrum.metadata['Starname']))
# Calculate the time taken to process this spectrum
time_start = time.time()
# If requested, interpolate the test set onto the same raster as the training set. DANGEROUS!
if args.interpolate:
spectrum = resample_spectrum(spectrum=spectrum, training_spectra=training_spectra)
# Pass spectrum to the Cannon
labels, cov, meta = model.fit_spectrum(spectrum=spectrum)
# Check whether Cannon failed
if labels is None:
continue
# Measure the time taken
time_end = time.time()
time_taken[index] = time_end - time_start
# Identify which star it is and what the SNR is
star_name = spectrum.metadata["Starname"] if "Starname" in spectrum.metadata else ""
uid = spectrum.metadata["uid"] if "uid" in spectrum.metadata else ""
# From the label covariance matrix extract the standard deviation in each label value
# (diagonal terms in the matrix are variances)
if args.cannon_version == "anna_ho":
err_labels = cov[0]
else:
err_labels = np.sqrt(np.diag(cov[0]))
# Turn list of label values into a dictionary
cannon_output = dict(list(zip(test_labels, labels[0])))
# Add the standard deviations of each label into the dictionary
cannon_output.update(dict(list(zip(["E_{}".format(label_name) for label_name in test_labels], err_labels))))
# Add the star name and the SNR ratio of the test spectrum
result = {"Starname": star_name,
"uid": uid,
"time": time_taken[index],
"spectrum_metadata": spectrum.metadata,
"cannon_output": cannon_output
}
results.append(result)
# Report time taken
logging.info("Fitting of {:d} spectra completed. Took {:.2f} +/- {:.2f} sec / spectrum.".
format(N,
np.mean(time_taken),
np.std(time_taken)))
# Create output data structure
censoring_output = None
if reloading_cannon:
censoring_output = summary_json['censoring_mask']
else:
if censoring_masks is not None:
censoring_output = dict([(label, tuple([int(i) for i in mask]))
for label, mask in censoring_masks.items()])
output_data = {
"hostname": os.uname()[1],
"generator": __file__,
"4gp_version": fourgp_version,
"cannon_version": model.cannon_version,
"start_time": time_training_start,
"end_time": time.time(),
"training_time": time_training_end - time_training_start,
"description": args.description,
"train_library": training_library_string,
"test_library": args.test_library,
"training_spectra_ids": training_library_ids,
"tolerance": tolerance,
"assume_scaled_solar": assume_scaled_solar,
"line_list": line_list,
"labels": test_labels,
"wavelength_raster": tuple(raster),
"censoring_mask": censoring_output
}
# Write brief summary of run to JSON file, without masses of data
with gzip.open("{:s}.summary.json.gz".format(output_filename), "wt") as f:
f.write(json.dumps(output_data, indent=2))
# Write full results to JSON file
output_data["spectra"] = results
with gzip.open("{:s}.full.json.gz".format(output_filename), "wt") as f:
f.write(json.dumps(output_data, indent=2))
output_select = random.uniform(a=0, b=weights_sum)
for index, weight in enumerate(weights):
output_select -= weight
if output_select <= 0:
selected_index = index
break
return selected_index
# Open input spectrum library(s), and fetch a list of all the flux-normalised spectra within each
input_libraries = []
if args.input_library is not None:
input_libraries = [
SpectrumLibrarySqlite.open_and_search(
library_spec=item,
workspace=workspace,
extra_constraints={"continuum_normalised": 0})
for item in args.input_library
]
# Report to user how many spectra we have just found
logger.info("Opening {:d} input libraries. These contain {:s} spectra.".format(
len(input_libraries), str([len(x['items']) for x in input_libraries])))
# Open contaminating spectrum library(s), if any, and fetch a list of all the flux-normalised spectra within each
contamination_libraries = []
if args.contamination_library is not None:
contamination_libraries = [
SpectrumLibrarySqlite.open_and_search(
library_spec=item,
workspace=workspace,
def resample_templates(args, logger):
"""
Resample a spectrum library of templates onto a fixed logarithmic stride, representing each of the 4MOST arms in
turn. We use 4FS to down-sample the templates to the resolution of 4MOST observations, and automatically detect
the list of arms contained within each 4FS mock observation. We then resample the 4FS output onto a new raster
with fixed logarithmic stride.
:param args:
Object containing arguments supplied by the used, for example the name of the spectrum libraries we use for
input and output. The required fields are defined by the user interface above.
:param logger:
A python logging object.
:return:
None.
"""
# Set path to workspace where we expect to find libraries of spectra
workspace = args.workspace if args.workspace else os_path.join(
args.our_path, "../../../workspace")
# Open input template spectra
spectra = SpectrumLibrarySqlite.open_and_search(
library_spec=args.templates_in,
workspace=workspace,
extra_constraints={"continuum_normalised": 0})
templates_library, templates_library_items, templates_spectra_constraints = \
[spectra[i] for i in ("library", "items", "constraints")]
# Create new SpectrumLibrary to hold the resampled output templates
library_path = os_path.join(workspace, args.templates_out)
output_library = SpectrumLibrarySqlite(path=library_path, create=True)
# Instantiate 4FS wrapper
etc_wrapper = FourFS(path_to_4fs=os_path.join(args.binary_path,
"OpSys/ETC"),
snr_list=[250.],
magnitude=13,
snr_per_pixel=True)
for input_spectrum_id in templates_library_items:
logger.info("Working on <{}>".format(input_spectrum_id['filename']))
# Open Spectrum data from disk
input_spectrum_array = templates_library.open(
ids=input_spectrum_id['specId'])
# Load template spectrum (flux normalised)
template_flux_normalised = input_spectrum_array.extract_item(0)
# Look up the unique ID of the star we've just loaded
# Newer spectrum libraries have a uid field which is guaranteed unique; for older spectrum libraries use
# Starname instead.
# Work out which field we're using (uid or Starname)
spectrum_matching_field = 'uid' if 'uid' in template_flux_normalised.metadata else 'Starname'
# Look up the unique ID of this object
object_name = template_flux_normalised.metadata[
spectrum_matching_field]
# Search for the continuum-normalised version of this same object (which will share the same uid / name)
search_criteria = {
spectrum_matching_field: object_name,
'continuum_normalised': 1
}
continuum_normalised_spectrum_id = templates_library.search(
**search_criteria)
# Check that continuum-normalised spectrum exists and is unique
assert len(continuum_normalised_spectrum_id
) == 1, "Could not find continuum-normalised spectrum."
# Load the continuum-normalised version
template_continuum_normalised_arr = templates_library.open(
ids=continuum_normalised_spectrum_id[0]['specId'])
# Turn the SpectrumArray we got back into a single Spectrum
template_continuum_normalised = template_continuum_normalised_arr.extract_item(
0)
# Now create a mock observation of this template using 4FS
logger.info("Passing template through 4FS")
mock_observed_template = etc_wrapper.process_spectra(
spectra_list=((template_flux_normalised,
template_continuum_normalised), ))
# Loop over LRS and HRS
for mode in mock_observed_template:
# Loop over the spectra we simulated (there was only one!)
for index in mock_observed_template[mode]:
# Loop over the various SNRs we simulated (there was only one!)
for snr in mock_observed_template[mode][index]:
# Create a unique ID for this arm's data
unique_id = hashlib.md5(os.urandom(32)).hexdigest()[:16]
# Import the flux- and continuum-normalised spectra separately, but give them the same ID
for spectrum_type in mock_observed_template[mode][index][
snr]:
# Extract continuum-normalised mock observation
logger.info("Resampling {} spectrum".format(mode))
mock_observed = mock_observed_template[mode][index][
snr][spectrum_type]
# Replace errors which are nans with a large value
mock_observed.value_errors[np.isnan(
mock_observed.value_errors)] = 1000.
# Check for NaN values in spectrum itself
if not np.all(np.isfinite(mock_observed.values)):
print(
"Warning: NaN values in template <{}>".format(
template_flux_normalised.
metadata['Starname']))
mock_observed.value_errors[np.isnan(
mock_observed.values)] = 1000.
mock_observed.values[np.isnan(
mock_observed.values)] = 1.
# Resample template onto a logarithmic raster of fixed step
resampler = SpectrumResampler(mock_observed)
# Construct the raster for each wavelength arm
wavelength_arms = SpectrumProperties(
mock_observed.wavelengths).wavelength_arms()
# Resample 4FS output for each arm onto a fixed logarithmic stride
for arm_count, arm in enumerate(
wavelength_arms["wavelength_arms"]):
arm_raster, mean_pixel_width = arm
name = "{}_{}".format(mode, arm_count)
arm_info = {
"lambda_min": arm_raster[0],
"lambda_max": arm_raster[-1],
"lambda_step": mean_pixel_width
}
arm_raster = logarithmic_raster(
lambda_min=arm_info['lambda_min'],
lambda_max=arm_info['lambda_max'],
lambda_step=arm_info['lambda_step'])
# Resample 4FS output onto a fixed logarithmic step
mock_observed_arm = resampler.onto_raster(
arm_raster)
# Save it into output spectrum library
output_library.insert(
spectra=mock_observed_arm,
filenames=input_spectrum_id['filename'],
metadata_list={
"uid": unique_id,
"template_id": object_name,
"mode": mode,
"arm_name":
"{}_{}".format(mode, arm_count),
"lambda_min": arm_raster[0],
"lambda_max": arm_raster[-1],
"lambda_step": mean_pixel_width
})
help="Separator to use between fields in the CSV output.")
parser.add_argument(
'--workspace',
dest='workspace',
default="",
help="Directory where we expect to find spectrum libraries.")
args = parser.parse_args()
# Set path to workspace where we expect to find libraries of spectra
our_path = os_path.split(os_path.abspath(__file__))[0]
workspace = args.workspace if args.workspace else os_path.join(
our_path, "../../../workspace")
# Open spectrum library we're going to export from, and search for flux-normalised spectra meeting our filtering
# constraints
input_library_info = SpectrumLibrarySqlite.open_and_search(
library_spec=args.library, workspace=workspace, extra_constraints={})
# Get a list of the spectrum IDs which we were returned
input_library, library_items = [
input_library_info[i] for i in ("library", "items")
]
library_ids = [i["specId"] for i in library_items]
# Fetch list of all metadata fields, and sort it alphabetically
fields = [i.strip() for i in input_library.list_metadata_fields()]
fields.sort()
# At the top of the CSV file, write column headings with the field names
line = args.separator.join(fields)
print(line)
# Start logging our progress
logging.basicConfig(level=logging.INFO, format='[%(asctime)s] %(levelname)s:%(filename)s:%(message)s',
datefmt='%d/%m/%Y %H:%M:%S')
logger = logging.getLogger(__name__)
logger.info("Creating synthetic versions of stars from <{}>".format(input_library))
# Instantiate base synthesizer
synthesizer = Synthesizer(library_name="pepsi_synthetic",
logger=logger,
docstring=__doc__)
star_list = []
# Open input spectrum library
spectra = SpectrumLibrarySqlite.open_and_search(library_spec=input_library,
workspace=synthesizer.workspace,
extra_constraints={"continuum_normalised": 1}
)
# Get a list of the spectrum IDs which we were returned
input_library, input_spectra_ids, input_spectra_constraints = [spectra[i] for i in ("library", "items", "constraints")]
# Loop over input spectra
for input_spectrum_id in input_spectra_ids:
logger.info("Working on <{}>".format(input_spectrum_id['filename']))
# Open Spectrum data from disk
input_spectrum_array = input_library.open(ids=input_spectrum_id['specId'])
# Turn SpectrumArray object into a Spectrum object
input_spectrum = input_spectrum_array.extract_item(0)
metadata = input_spectrum.metadata
def main():
"""
Main entry point for running the Payne.
"""
global logger
logging.basicConfig(
level=logging.INFO,
format='[%(asctime)s] %(levelname)s:%(filename)s:%(message)s',
datefmt='%d/%m/%Y %H:%M:%S')
logger = logging.getLogger(__name__)
# Read input parameters
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument(
'--test',
required=True,
dest='test_library',
help=
"Library of spectra to test the trained Payne on. Stars may be filtered by parameters by "
"placing a comma-separated list of constraints in [] brackets after the name of the "
"library. Use the syntax [Teff=3000] to demand equality, or [0<[Fe/H]<0.2] to specify a "
"range.")
parser.add_argument(
'--train',
required=True,
dest='train_library',
help=
"Library of labelled spectra to train the Payne on. Stars may be filtered by parameters "
"by placing a comma-separated list of constraints in [] brackets after the name of the "
"library. Use the syntax [Teff=3000] to demand equality, or [0<[Fe/H]<0.2] to specify a "
"range.")
parser.add_argument(
'--workspace',
dest='workspace',
default="",
help="Directory where we expect to find spectrum libraries.")
parser.add_argument(
'--train-batch-number',
required=False,
dest='batch_number',
type=int,
default=0,
help=
"If training pixels in multiple batches on different machines, then this is the number of "
"the batch of pixels we are to train. It should be in the range 0 .. batch_count-1 "
"inclusive. If it is -1, then we skip training to move straight to testing."
)
parser.add_argument(
'--train-batch-count',
required=False,
dest='batch_count',
type=int,
default=1,
help=
"If training pixels in multiple batches on different machines, then this is the number "
"of batches.")
parser.add_argument('--description',
dest='description',
help="A description of this fitting run.")
parser.add_argument(
'--labels',
dest='labels',
default="Teff,logg,[Fe/H]",
help="List of the labels the Payne is to learn to estimate.")
parser.add_argument(
'--label-expressions',
dest='label_expressions',
default="",
help="List of the algebraic labels the Payne is to learn to estimate "
"(e.g. photometry_B - photometry_V).")
parser.add_argument(
'--labels-individual',
dest='labels_individual',
default="",
help="List of the labels the Payne is to fit in separate fitting runs."
)
parser.add_argument('--censor-scheme',
default="1",
dest='censor_scheme',
help="Censoring scheme version to use (1, 2 or 3).")
parser.add_argument(
'--censor',
default="",
dest='censor_line_list',
help=
"Optional list of line positions for the Payne to fit, ignoring continuum between."
)
parser.add_argument('--output-file',
default="./test_cannon.out",
dest='output_file',
help="Data file to write output to.")
parser.add_argument(
'--assume-scaled-solar',
action='store_true',
dest="assume_scaled_solar",
help=
"Assume scaled solar abundances for any elements which don't have abundances individually "
"specified. Useful for working with incomplete data sets.")
parser.add_argument(
'--no-assume-scaled-solar',
action='store_false',
dest="assume_scaled_solar",
help=
"Do not assume scaled solar abundances; throw an error if training set is has missing "
"labels.")
parser.set_defaults(assume_scaled_solar=False)
parser.add_argument('--multithread',
action='store_true',
dest="multithread",
help="Use multiple thread to speed Payne up.")
parser.add_argument(
'--nothread',
action='store_false',
dest="multithread",
help="Do not use multiple threads - use only one CPU core.")
parser.set_defaults(multithread=True)
parser.add_argument(
'--interpolate',
action='store_true',
dest="interpolate",
help=
"Interpolate the test spectra on the training spectra's wavelength raster. DANGEROUS!"
)
parser.add_argument(
'--nointerpolate',
action='store_false',
dest="interpolate",
help="Do not interpolate the test spectra onto a different raster.")
parser.set_defaults(interpolate=False)
args = parser.parse_args()
logger.info("Testing Payne with arguments <{}> <{}> <{}> <{}>".format(
args.test_library, args.train_library, args.censor_line_list,
args.output_file))
# List of labels over which we are going to test the performance of the Payne
test_label_fields = args.labels.split(",")
# List of labels we're going to fit individually
if args.labels_individual:
test_labels_individual = [
i.split("+") for i in args.labels_individual.split(",")
]
else:
test_labels_individual = [[]]
# Set path to workspace where we expect to find libraries of spectra
our_path = os_path.split(os_path.abspath(__file__))[0]
workspace = args.workspace if args.workspace else os_path.join(
our_path, "../../../workspace")
# Open training set
spectra = SpectrumLibrarySqlite.open_and_search(
library_spec=args.train_library,
workspace=workspace,
extra_constraints={"continuum_normalised": True})
training_library, training_library_items = [
spectra[i] for i in ("library", "items")
]
# Open test set
spectra = SpectrumLibrarySqlite.open_and_search(
library_spec=args.test_library,
workspace=workspace,
extra_constraints={"continuum_normalised": True})
test_library, test_library_items = [
spectra[i] for i in ("library", "items")
]
# Load training set
training_library_ids_all = [i["specId"] for i in training_library_items]
training_spectra_all = training_library.open(ids=training_library_ids_all)
raster = training_spectra_all.wavelengths
# Load test set
test_library_ids = [i["specId"] for i in test_library_items]
# Fit each set of labels we're fitting individually, one by one
for labels_individual_batch_count, test_labels_individual_batch in enumerate(
test_labels_individual):
# Create filename for the output from this Payne run
output_filename = args.output_file
# If we're fitting elements individually, individually number the runs to fit each element
if len(test_labels_individual) > 1:
output_filename += "-{:03d}".format(labels_individual_batch_count)
# If requested, fill in any missing labels on the training set by assuming scaled-solar abundances
if args.assume_scaled_solar:
training_spectra = autocomplete_scaled_solar_abundances(
input_spectra=training_spectra_all,
label_list=test_label_fields + test_labels_individual_batch)
else:
training_spectra = filter_training_spectra(
input_spectra=training_spectra_all,
label_list=test_label_fields + test_labels_individual_batch,
input_library=training_library,
input_spectrum_ids=training_library_ids_all)
# Evaluate labels which are calculated via metadata expressions
test_labels_expressions = []
if args.label_expressions.strip():
test_labels_expressions = args.label_expressions.split(",")
evaluate_computed_labels(label_expressions=test_labels_expressions,
spectra=training_spectra)
# Make combined list of all labels the Payne is going to fit
test_labels = test_label_fields + test_labels_individual_batch + test_labels_expressions
logger.info("Beginning fit of labels <{}>.".format(
",".join(test_labels)))
# If required, generate the censoring masks
censoring_masks = create_censoring_masks(
censoring_scheme=int(args.censor_scheme),
raster=raster,
censoring_line_list=args.censor_line_list,
label_fields=test_label_fields + test_labels_individual_batch,
label_expressions=test_labels_expressions)
# Construct and train a model
time_training_start = time.time()
model = PayneInstanceTing(training_set=training_spectra,
label_names=test_labels,
batch_number=args.batch_number,
batch_count=args.batch_count,
censors=censoring_masks,
threads=None if args.multithread else 1,
training_data_archive=output_filename)
time_training_end = time.time()
# Test the model
N = len(test_library_ids)
time_taken = np.zeros(N)
results = []
for index in range(N):
test_spectrum_array = test_library.open(
ids=test_library_ids[index])
spectrum = test_spectrum_array.extract_item(0)
logger.info("Testing {}/{}: {}".format(
index + 1, N, spectrum.metadata['Starname']))
# Calculate the time taken to process this spectrum
time_start = time.time()
# If requested, interpolate the test set onto the same raster as the training set. DANGEROUS!
if args.interpolate:
spectrum = resample_spectrum(spectrum=spectrum,
training_spectra=training_spectra)
# Pass spectrum to the Payne
fit_data = model.fit_spectrum(spectrum=spectrum)
# Check whether Payne failed
# if labels is None:
# continue
# Measure the time taken
time_end = time.time()
time_taken[index] = time_end - time_start
# Identify which star it is and what the SNR is
star_name = spectrum.metadata[
"Starname"] if "Starname" in spectrum.metadata else ""
uid = spectrum.metadata["uid"] if "uid" in spectrum.metadata else ""
# Fudge the errors for now until I work this out
err_labels = [0 for item in test_labels]
# Turn list of label values into a dictionary
payne_output = dict(list(zip(test_labels, fit_data['results'][0])))
# Add the standard deviations of each label into the dictionary
payne_output.update(
dict(
list(
zip([
"E_{}".format(label_name)
for label_name in test_labels
], err_labels))))
# Add the star name and the SNR ratio of the test spectrum
result = {
"Starname": star_name,
"uid": uid,
"time": time_taken[index],
"spectrum_metadata": spectrum.metadata,
"cannon_output": payne_output
}
results.append(result)
# Report time taken
logger.info(
"Fitting of {:d} spectra completed. Took {:.2f} +/- {:.2f} sec / spectrum."
.format(N, np.mean(time_taken), np.std(time_taken)))
# Create output data structure
censoring_output = None
if censoring_masks is not None:
censoring_output = dict([
(label, tuple([int(i) for i in mask]))
for label, mask in censoring_masks.items()
])
output_data = {
"hostname": os.uname()[1],
"generator": __file__,
"4gp_version": fourgp_version,
"cannon_version": None,
"payne_version": model.payne_version,
"start_time": time_training_start,
"end_time": time.time(),
"training_time": time_training_end - time_training_start,
"description": args.description,
"train_library": args.train_library,
"test_library": args.test_library,
"tolerance": None,
"assume_scaled_solar": args.assume_scaled_solar,
"line_list": args.censor_line_list,
"labels": test_labels,
"wavelength_raster": tuple(raster),
"censoring_mask": censoring_output
}
# Write brief summary of run to JSON file, without masses of data
with gzip.open("{:s}.summary.json.gz".format(output_filename),
"wt") as f:
f.write(json.dumps(output_data, indent=2))
# Write full results to JSON file
output_data["spectra"] = results
with gzip.open("{:s}.full.json.gz".format(output_filename), "wt") as f:
f.write(json.dumps(output_data, indent=2))
synthesizer.set_star_list(star_list)
# Create new SpectrumLibrary
synthesizer.create_spectrum_library()
# Iterate over the spectra we're supposed to be synthesizing
synthesizer.do_synthesis()
# Close TurboSpectrum synthesizer instance
synthesizer.clean_up()
# Load spectrum
spectra = SpectrumLibrarySqlite.open_and_search(
library_spec=synthesizer.args.library,
workspace=synthesizer.workspace,
extra_constraints={
"Starname": "Sun",
"continuum_normalised": 0
})
input_library, input_spectra_ids, input_spectra_constraints = [
spectra[i] for i in ("library", "items", "constraints")
]
input_spectrum_array = input_library.open(ids=input_spectra_ids[0]['specId'])
input_spectrum = input_spectrum_array.extract_item(0)
# Process spectra through reddening model
reddener = SpectrumReddener(input_spectrum=input_spectrum)
# Instantiate 4FS wrapper
etc_wrapper = FourFS(path_to_4fs=os_path.join(synthesizer.args.binary_path,
def tabulate_labels(library_list, label_list, output_file, workspace=None):
"""
Take a SpectrumLibrary and tabulate a list of the stellar parameters of the stars within it.
:param workspace:
Path to the workspace where we expect to find SpectrumLibraries stored
:param library_list:
A list of the SpectrumLibraries we are to tabulate the contents of
:param label_list:
A list of the labels whose values we are to tabulate
:param output_file:
The filename of the ASCII output file we are to produce
:return:
None
"""
# Set path to workspace where we expect to find libraries of spectra
if workspace is None:
our_path = os_path.split(os_path.abspath(__file__))[0]
workspace = os_path.join(our_path, "../../../../workspace")
# Open output data file
with open(output_file, "w") as output:
# Loop over each spectrum library in turn
for library in library_list:
# Extract name of spectrum library we are to open. Filter off any constraints which follow the name in []
test = re.match("([^\[]*)\[(.*)\]$", library)
if test is None:
library_name = library
else:
library_name = test.group(1)
# Open spectrum library and extract list of metadata fields which are defined on this library
library_path = os_path.join(workspace, library_name)
library_object = SpectrumLibrarySqlite(path=library_path,
create=False)
metadata_fields = library_object.list_metadata_fields()
# Now search library for spectra matching any input constraints, with additional constraint on only
# returning continuum normalised spectra, if that field is defined for this library
constraints = {}
if "continuum_normalised" in metadata_fields:
constraints["continuum_normalised"] = 1
library_spectra = SpectrumLibrarySqlite.open_and_search(
library_spec=library,
workspace=workspace,
extra_constraints=constraints)
# Write column headers at the top of the output
columns = label_list if label_list is not None else library_object.list_metadata_fields(
)
output.write("# ")
for label in columns:
output.write("{} ".format(label))
output.write("\n")
# Loop over objects in each spectrum library
for item in library_spectra["items"]:
metadata = library_object.get_metadata(ids=item['specId'])[0]
for label in columns:
output.write("{} ".format(metadata.get(label, "-")))
output.write("\n")