def setup_for_order(r, datafile, training_epochs, validation_epochs):
# load up training and validation data for order r & set up results
training_data = wobble.Data(datafile, orders=[r],
epochs=training_epochs, min_snr=0.)
training_results = wobble.Results(training_data)
validation_data = wobble.Data(datafile, orders=training_data.orders,
epochs=validation_epochs, min_snr=0.)
validation_results = wobble.Results(validation_data)
return training_data, training_results, validation_data, validation_results
if tellurics_template_fixed:
f.create_dataset('L1_template', data=np.zeros(R))
f.create_dataset('L2_template', data=np.zeros(R))
else:
f.create_dataset('L1_template', data=np.zeros(R) + 1.e4)
f.create_dataset('L2_template', data=np.zeros(R) + 1.e6)
if K_t > 0:
f.create_dataset('L1_basis_vectors', data=np.zeros(R) + 1.e3)
f.create_dataset('L2_basis_vectors', data=np.zeros(R) + 1.e8)
f.create_dataset('L2_basis_weights', data=np.ones(
R)) # never tuned, just need to pass to wobble
# set up training & validation data sets:
if True:
data = wobble.Data(starname + '_e2ds.hdf5',
filepath='../data/',
orders=orders,
min_snr=3) # to get N_epochs
validation_epochs = np.random.choice(
data.N, data.N // 8,
replace=False) # 12.5% of epochs will be validation set
training_epochs = np.delete(np.arange(data.N), validation_epochs)
else: # HACK for HD 189733
e = np.asarray([
0, 1, 6, 7, 9, 17, 18, 19, 21, 23, 24, 26, 30, 33, 34, 35, 36, 37,
38, 40, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 53, 55, 56, 61, 66,
69, 70, 72, 73, 75
]) # night of August 28, 2007
validation_epochs = np.random.choice(e, len(e) // 8, replace=False)
training_epochs = np.delete(e, validation_epochs)
plot_dir = '../regularization/{0}_Kstar{1}_Kt{2}/'.format(starname, K_star, K_t)
if not os.path.exists(plot_dir):
os.makedirs(plot_dir)
# create regularization parameter files if they don't exist:
R = len(orders)
star_filename = '../wobble/regularization/{0}_star_K{1}.hdf5'.format(starname, K_star)
if not os.path.isfile(star_filename):
generate_regularization_file(star_filename, R, star=True, tellurics=False)
tellurics_filename = '../wobble/regularization/{0}_t_K{1}.hdf5'.format(starname, K_t)
if not os.path.isfile(tellurics_filename):
generate_regularization_file(tellurics_filename, R, star=False, tellurics=True)
# set up training & validation data sets:
data = wobble.Data(datafile, filepath='data/', orders=orders, min_snr=3) # to get N_epochs
validation_epochs = np.random.choice(data.N, data.N//8, replace=False) # 12.5% of epochs will be validation set
training_epochs = np.delete(np.arange(data.N), validation_epochs)
# improve each order's regularization:
for o in enumerate(orders):
if verbose:
print('---- STARTING ORDER {0} ----'.format(o))
print("starting values:")
print("star:")
with h5py.File(star_filename, 'r') as f:
for key in list(f.keys()):
print("{0}: {1:.0e}".format(key, f[key][o]))
print("tellurics:")
with h5py.File(tellurics_filename, 'r') as f:
for key in list(f.keys()):
# create directory for plots if it doesn't exist:
if plot or plot_minimal:
plot_dir = '../regularization/{0}_Kstar{1}_Kt{2}/'.format(starname, K_star, K_t)
if not os.path.exists(plot_dir):
os.makedirs(plot_dir)
# create regularization parameter files if they don't exist:
star_filename = '../wobble/regularization/{0}_star_K{1}.hdf5'.format(starname, K_star)
if not os.path.isfile(star_filename):
generate_regularization_file(star_filename, R, type='star')
tellurics_filename = '../wobble/regularization/{0}_t_K{1}.hdf5'.format(starname, K_t)
if not os.path.isfile(tellurics_filename):
generate_regularization_file(tellurics_filename, R, type='telluric')
# load up the data we'll use for training:
data = wobble.Data(datafile, orders=orders) # to get N_epochs
# choose validation epochs:
validation_epochs = np.random.choice(data.N, data.N//6, replace=False)
# improve each order's regularization:
for r,o in enumerate(orders):
if verbose:
print('---- STARTING ORDER {0} ----'.format(o))
print("starting values:")
print("star:")
with h5py.File(star_filename, 'r') as f:
for key in list(f.keys()):
print("{0}: {1:.0e}".format(key, f[key][o]))
print("tellurics:")
with h5py.File(tellurics_filename, 'r') as f:
epochs = [0, 50] # to plot
movies = False
star_reg_file = '../wobble/regularization/{0}_star_K{1}.hdf5'.format(
starname, K_star)
tellurics_reg_file = '../wobble/regularization/{0}_t_K{1}.hdf5'.format(
starname, K_t)
plot_dir = '../results/plots_{0}_Kstar{1}_Kt{2}/'.format(
starname, K_star, K_t)
print("running wobble on star {0} with K_star = {1}, K_t = {2}".format(
starname, K_star, K_t))
start_time = time()
orders = np.arange(70, 72)
data = wobble.Data(starname + '_e2ds.hdf5',
filepath='../data/',
orders=orders)
if True: # reload data and remove all post-upgrade spectra
upgrade = 2457174.5 # June 2015
e = data.epochs[data.dates < upgrade]
data = wobble.Data(starname + '_e2ds.hdf5',
filepath='../data/',
orders=orders,
epochs=e)
orders = np.copy(data.orders)
results = wobble.Results(data=data)
print("data loaded")
print("time elapsed: {0:.2f} min".format((time() - start_time) / 60.0))
elapsed_time = time() - start_time
#NOTE: These reg file names will use the uncombined chunkwise regularisation files. the chunksize and range must therefore be matching
star_reg_file = '../wobble/regularization/{0}_star_K{1}_orders[{2},{3}).hdf5'.format(
starname, K_star, start_order, end_order)
tellurics_reg_file = '../wobble/regularization/{0}_t_K{1}_orders[{2},{3}).hdf5'.format(
starname, K_t, start_order, end_order)
plot_dir = '../results/plots_{0}_Kstar{1}_Kt{2}/'.format(
starname, K_star, K_t)
results_file = '../results/results_{0}_Kstar{1}_Kt{2}_orders[{3},{4}).hdf5'.format(
starname, K_star, K_t, start_order, end_order)
if False:
# quick test on single order
data = wobble.Data(starname + '_e2ds.hdf5',
filepath='../data/',
orders=[65, 66, 67],
min_flux=10**-5)
results = wobble.Results(data=data)
for r in range(data.R):
model = wobble.Model(data, results, r)
model.add_star('star',
variable_bases=K_star,
regularization_par_file=star_reg_file,
learning_rate_template=0.01,
learning_rate_rvs=1.)
model.add_telluric('tellurics',
rvs_fixed=True,
variable_bases=K_t,
regularization_par_file=tellurics_reg_file,
learning_rate_template=0.01)
wobble.optimize_order(model,
start_order = chunks[i][0]
end_order = chunks[i][-1]
print(
"running wobble on star {0} with K_star = {1}, K_t = {2}, orders[{3},{4})".
format(p.starname, p.K_star, p.K_t, start_order, end_order))
#orders = np.arange(start_order, end_order)
#Take intersection of chunk orders and order list to drop orders previously cut
#orders = list(set([x for x in range(start_order, end_order)]) & set(orders_list))
orders = chunks[i]
print("orders: ", orders)
data = wobble.Data(
data_file,
orders=orders,
epochs=epochs_list,
min_flux=10**-5,
#min_snr=0,
min_snr=p.min_snr,
chunkQ=True,
parameters=p)
'''
################# )
#TODO Apply continnuum norm failed arder cutting: delete data.drop_orders fromm chunk and order list
#HACK this ignores the underlying issue: WHY ARE there different results in the first place?
#DIDNN'T WORK Note self.drop_orders: [15]
#data.drop_orders [15]
#p.drop_orders [[15]] IIs of different format
try:
for order in data.drop_orders:
p.drop_orders.append(order) #append to drop orders list
#results_name = "results_Wolf294_Kstar0_Kt3__loop4_reg_snr"
#results_name = "results_GJ436_Kstar0_Kt3__nir_split_flat_reg"
#results_name = "results_GJ1148_Kstar0_Kt3__nir_split_flat_reg"
#results_name = "results_GJ3512_Kstar0_Kt3__nir_split_flat_reg"
starname = "GJ1148"
vis = True
show_mask = False
#orders = np.arange(11,53)
#lowest_optimized_order = 11
#orders = [49,50]
if vis == True:
orders = np.arange(11,53)
lowest_optimized_order = 11
data = wobble.Data(starname+'_vis'+'_e2ds.hdf5', filepath= data_directory, orders=orders, min_flux=10**-5, min_snr=0)
else:
orders = np.arange(0,56)
lowest_optimized_order = 0
data = wobble.Data(starname+'_nir_split'+'_e2ds.hdf5', filepath= data_directory, orders=orders, min_flux=10**-5, min_snr=0)
######
#plot_dir = plot_dir + results_name + "/"#+ "_bad_ep/"
plot_dir = plot_dir + results_name + "_bad_ord(49)/"
#plot_dir = plot_dir + "GJ1148_o49_snr" + "/"
os.makedirs(plot_dir, exist_ok = True)
results = wobble.Results(filename = results_directory + results_name +".hdf5")
telluric_mask = np.genfromtxt("/data/cmatthe/python/wobble_aux/telluric_masks/" +
"telluric_mask_carm_short.dat"
plot_dir = results_directory + '/plots_{0}_Kstar{1}_Kt{2}_'.format(
starname, K_star, K_t, niter) + parameter_dict["output_suffix"] + '/'
results_file = results_directory + 'results_{0}_Kstar{1}_Kt{2}_{5}_orders[{3},{4}).hdf5'.format(
starname, K_star, K_t, start_order, end_order,
parameters.dictionary["output_suffix"])
print(
"running wobble on star {0} with K_star = {1}, K_t = {2}, orders[{3},{4})"
.format(starname, K_star, K_t, start_order, end_order))
start_time = time()
orders = np.arange(start_order, end_order)
#NOTE change 25/10/2019 snr+data_suffix
data = wobble.Data(data_file,
orders=orders,
epochs=epochs_list,
min_flux=10**-5,
min_snr=0)
# min_snr=0 is part of a HACK that centralises epoch cutting to the top level script, will this interfere with order cutting?
results = wobble.Results(data=data)
''' doesn't work becaus the enoumeration of orders will restart at 0
#Load wobble results file should it already exist, so that only the current optimzed chunk gets replaced
#note this may fail if the data is not the same as the data used to create the original results file
try:
results = wobble.Results(filename=results_file)
except FileNotFoundError:
results = wobble.Results(data=data)
'''
print("data loaded")
tellurics_reg_file = '../wobble/regularization/default_t.hdf5'
plot_dir = '../results/plots_{0}_Kstar{1}_Kt{2}_def_n{3}/'.format(
starname, K_star, K_t, niter)
results_file = '../results/results_{0}_Kstar{1}_Kt{2}_n{5}_orders[{3},{4})_def.hdf5'.format(
starname, K_star, K_t, start_order, end_order, niter)
print(
"running wobble on star {0} with K_star = {1}, K_t = {2}, orders[{3},{4})"
.format(starname, K_star, K_t, start_order, end_order))
start_time = time()
orders = np.arange(start_order, end_order)
data = wobble.Data(
'../data/' + starname + '_vis' + '_e2ds.hdf5',
orders=orders,
epochs=epochs_list,
min_flux=10**-5,
min_snr=0 #deprecated, no longer used by this function as of 14_06_2019
)
# min_snr=0 is part of a HACK that centralises epoch cutting to the top level script, will this interfere with order cutting?
##NOTE Meant to put in functionality of old data.__init__
##min_flux = 1.
#min_flux=10**-5
#max_norm_flux = 2.
#padding = 2
#min_snr = 5.
#log_flux = True
#data.mask_low_pixels(min_flux=min_flux, padding=padding, min_snr=min_snr)
niter = 100 # for optimization
plots = True
epochs = [0, 50] # to plot
movies = False
star_reg_file = '../wobble/regularization/default_star.hdf5'.format(
starname, K_star)
tellurics_reg_file = '../wobble/regularization/default_t.hdf5'.format(
starname, K_t)
plot_dir = '../results/plots_{0}_Kstar{1}_Kt{2}/'.format(
starname, K_star, K_t)
if False:
# quick test on single order
data = wobble.Data(starname + '_e2ds.hdf5',
filepath='../data/',
orders=[65, 66, 67],
min_flux=10**-5)
results = wobble.Results(data=data)
for r in range(data.R):
model = wobble.Model(data, results, r)
model.add_star('star',
variable_bases=K_star,
regularization_par_file=star_reg_file,
learning_rate_template=0.01,
learning_rate_rvs=1.)
model.add_telluric('tellurics',
rvs_fixed=True,
variable_bases=K_t,
regularization_par_file=tellurics_reg_file,
learning_rate_template=0.01)
wobble.optimize_order(model,
K_t = 3
niter = 150 # for optimization
plots = True
epochs = [0, 50] # to plot
movies = False
star_reg_file = '../wobble/regularization/{0}_star_K{1}.hdf5'.format(
starname, K_star)
tellurics_reg_file = '../wobble/regularization/{0}_t_K{1}.hdf5'.format(
starname, K_t)
plot_dir = '../results/plots_{0}_Kstar{1}_Kt{2}/'.format(
starname, K_star, K_t)
if False:
# quick test on single order
data = wobble.Data(filename='../data/' + starname + '_e2ds.hdf5',
orders=[30, 56])
results = wobble.Results(data=data)
for r in range(data.R):
model = wobble.Model(data, results, r)
model.add_star('star',
variable_bases=K_star,
regularization_par_file=star_reg_file,
learning_rate_template=0.01,
learning_rate_rvs=1.)
model.add_telluric('tellurics',
rvs_fixed=True,
variable_bases=K_t,
regularization_par_file=tellurics_reg_file,
learning_rate_template=0.01)
wobble.optimize_order(model,
niter=niter,
star_reg_file = '../wobble/regularization/default_star.hdf5'
tellurics_reg_file = '../wobble/regularization/default_t.hdf5'
plot_dir = '../results/plots_{0}_Kstar{1}_Kt{2}_def_n{3}/'.format(
starname, K_star, K_t, niter)
results_file = '../results/results_{0}_Kstar{1}_Kt{2}_n{5}_orders[{3},{4})_def.hdf5'.format(
starname, K_star, K_t, start_order, end_order, niter)
print(
"running wobble on star {0} with K_star = {1}, K_t = {2}, orders[{3},{4})"
.format(starname, K_star, K_t, start_order, end_order))
start_time = time()
orders = np.arange(start_order, end_order)
data = wobble.Data(starname + '_nir_split' + '_e2ds.hdf5',
filepath='../data/',
orders=orders,
epochs=epochs_list,
min_flux=10**-5,
min_snr=0)
# min_snr=0 is part of a HACK that centralises epoch cutting to the top level script, will this interfere with order cutting?
results = wobble.Results(data=data)
''' doesn't work becaus the enoumeration of orders will restart at 0
#Load wobble results file should it already exist, so that only the current optimzed chunk gets replaced
#note this may fail if the data is not the same as the data used to create the original results file
try:
results = wobble.Results(filename=results_file)
except FileNotFoundError:
results = wobble.Results(data=data)
'''
print("data loaded")
def regularization(parameters
,run_name
,objects
,orbital_parameters_mult
,bary_starname
,servaldir
,starting_star_reg
,starting_t_reg
,top_directory
,data_directory
,max_loop_iterations
,parameter_changes
):
starname = objects[0][0]
p = parameters
reg_types = ["star", "t"]#t for tellurics NOTE(DO NOT CHANGE THIS NOMENCLATURE IT IS HARD CODED LATER ON)HACK
serval_T0_offset = find_Serval_T0_offset(orbital_parameters_mult, objects, servaldir)
base_star_reg = starting_star_reg
base_t_reg = starting_t_reg
#Generate global_epochs_list and attach to parameters object NOTE TODO must be attached to paameters object that is passed to run_wobble later
data_file= data_directory + starname + p.data_suffix + '_e2ds.hdf5'
data = wobble.Data(data_file, orders = np.arange(p.start, p.end), min_flux=10**-5, min_snr = p.min_snr,
parameters = p #passed to continuum_fit as **kwargs
)
global_epochs_list = p.global_epochs_list = data.epochs.tolist()
global_orders_list = p.global_orders_list = data.orders.tolist()
orders = global_orders_list # Drop SNR limited orders #HACK bandaid fix to remove troublesome orders in IR.
#setup output directory for this run
run_directory = top_directory + "{0}_{1}/".format(starname, run_name)
os.makedirs(run_directory, exist_ok = True)
start_time = time()
for i in range(max_loop_iterations):
#initialize parameters for this loop from input parameters object
parameters_loop = p
#create loop_directory
loop_directory = run_directory + "loop_{0}/".format(i)
os.makedirs(loop_directory, exist_ok = True)
for j,typ in enumerate(reg_types):
if typ == "star":
parameter_key_list = ["L2_template", "L1_template"]
base_reg_file = base_star_reg
if typ == "t": # t = tellurics
parameter_key_list = ["L2_template", "L1_template", "L2_basis_vectors", "L1_basis_vectors"]
base_reg_file = base_t_reg
#HACK Save a copy of the current (from previous loop) base reg file to a temporary file where it can be updated to the next_base_reg_file
temp_reg_file = loop_directory + "temp_{0}_reg".format(typ) + ".hdf5"
update_reg_file(base_reg_file, temp_reg_file, orders = [0,1,2], parameter_key = "L2_template", exponent = 0)# just makes a copy (inputs except for filenames are dummy placeholders)
for o, order in enumerate(orders):
#update loop parameters to only run 1 order
parameters_loop.start = order
parameters_loop.end = order + 1
for k,key in enumerate(parameter_key_list):
file_list = ['' for change in parameter_changes]
for c, change in enumerate(parameter_changes):
#create results_directory
results_directory = loop_directory + "results_{0}_ord{1}_{2}_{3}/".format(typ, order, key, change)# NOTE this name format is hard coded below
os.makedirs(results_directory, exist_ok = True)
parameters_loop.results_dir = results_directory
#create regularization directory, updated reg files and pass them to the parameter object
regularization_directory = results_directory + "regularization/"
os.makedirs(regularization_directory, exist_ok = True)
new_reg_file = regularization_directory + "reg_{0}_ord{1}_{2}_{3}".format(typ,order,key, change) + ".hdf5"
update_reg_file(base_reg_file, new_reg_file, orders = [order], parameter_key = key, exponent = change)
if typ == "star":
parameters_loop.reg_file_star = new_reg_file
if typ == "t":
parameters_loop.reg_file_t = new_reg_file
#check if file already exists, and if so skip(for time savings on reruns)
#NOTE HACK Assumes same hard coded file format as in chunk.py
results_file_base = results_directory + 'results_{0}_Kstar{1}_Kt{2}_'.format(starname, p.K_star, p.K_t, p.niter) + p.output_suffix
results_file = results_file_base + '.hdf5'
file_basename = os.path.basename(results_file)
if os.path.isfile(results_file):
latest_results_file = results_file
'''check if unreliable
else:
#if already present, copy existing results with identical regularization (for 0 change)
if change == 0 and not (j == 0 and k == 0):
file_to_copy = loop_directory + "results_{0}_ord{1}_{2}_{3}/".format(reg_types[0], order, parameter_key_list[0], change) + file_basename
copy_at_dest = results_directory + file_basename
shutil.copy(file_to_copy, copy_at_dest)
#change latest_results_file to copy_at_dest so it gets properly put into file_list loaded in compare_results
latest_results_file = copy_at_dest
'''
else:
#run wobble_optimization
run_wobble(parameters_loop)
latest_results_file = results_directory + file_basename
file_list[c] = latest_results_file # TODO THis feature really needs an overhaul
print("finished loop {0}, typ {1}, key {2}, change {3}".format(i, typ, key, change))
print("time elapsed gradient_search: {0:.2f} h".format((time() - start_time)/3600.0))
#Compare results, find "best", update reg
# Comparison will initially result based on standart deviation from literature planet fit
best_change = compare_results(file_list, parameter_changes, bary_starname, orbital_parameters_mult, objects, servaldir, serval_T0_offset)
#update next_base_reg_file (for next loop)
next_base_reg_file = loop_directory + "next_base_{0}_reg.hdf5".format(typ)
update_reg_file(temp_reg_file, next_base_reg_file, orders = [order], parameter_key = key, exponent = best_change)
# HACK write current next_base_reg_file to a temporary reg file, for later update the next_base_reg_file
temp_reg_file = loop_directory + "temp_{0}_reg".format(typ) + ".hdf5"
update_reg_file(next_base_reg_file, temp_reg_file, orders = [0,1,2], parameter_key = "L2_template", exponent = 0)# just makes a copy
#set new base_reg_file
base_star_reg = loop_directory + "next_base_star_reg.hdf5"
base_t_reg = loop_directory + "next_base_t_reg.hdf5"
wobble_dir = '../../compare_wobble_serval/wobbledir/'
wobble_dir_file_name = 'results_{0}_Kstar{1}_Kt{2}_'.format(
starname, K_star, K_t,
niter) + parameters.dictionary["output_suffix"] + '.hdf5'
compare_file = wobble_dir + wobble_dir_file_name
####################
start_time = time()
#make list of chunks
chunks = cr.chunk_list(start, end, chunk_size)
# HACK global epoch cutting: use epoch (and order) cutting only once on all data, then pass epochs to chunks
data = wobble.Data(data_file, orders=np.arange(start, end), min_flux=10**-5)
epochs = np.asarray(data.epochs)
epochs_list = data.epochs.tolist()
epochs_list_str = str(list(epochs))
#create file containing actually used regularization parameters forfuture reference
reg_log_file = "/data/cmatthe/wobble_data/results/reg_parameters_record/reg_par_" + parameters.dictionary[
"output_suffix"] + ".txt"
parameters.dictionary.update({
"reg_log_file": reg_log_file,
})
f = open(reg_log_file, "w+")
f.write(
"order, L1_template, L2_template, L1_basis_vector, L2_basis_vector, L2_template\n"
)
f.close()
niter = 150 # for optimization
plots = True
epochs = [0, 50] # to plot
movies = False
star_reg_file = '../wobble/regularization/{0}_star_K{1}.hdf5'.format(
starname, K_star)
tellurics_reg_file = '../wobble/regularization/{0}_t_K{1}.hdf5'.format(
starname, K_t)
plot_dir = '../results/plots_{0}_Kstar{1}_Kt{2}/'.format(
starname, K_star, K_t)
if False:
# quick test on single order
data = wobble.Data(starname + '_e2ds.hdf5',
filepath='../data/',
orders=[65, 66, 67])
results = wobble.Results(data=data)
for r in range(data.R):
model = wobble.Model(data, results, r)
model.add_star('star',
variable_bases=K_star,
regularization_par_file=star_reg_file,
learning_rate_template=0.01,
learning_rate_rvs=1.)
model.add_telluric('tellurics',
rvs_fixed=True,
variable_bases=K_t,
regularization_par_file=tellurics_reg_file,
learning_rate_template=0.01)
wobble.optimize_order(model,
tellurics_reg_file = parameter_dict["reg_file_t_chunk"]
#else:
#star_reg_file = '../wobble/regularization/{0}_star_K{1}_orders[{2},{3}).hdf5'.format(starname, K_star, start_order, end_order)
#tellurics_reg_file = '../wobble/regularization/{0}_t_K{1}_orders[{2},{3}).hdf5'.format(starname, K_t, start_order, end_order)
plot_dir = results_directory + '/plots_{0}_Kstar{1}_Kt{2}_'.format(starname, K_star, K_t, niter)+ parameter_dict["output_suffix"] +'/'
results_file = results_directory + 'results_{0}_Kstar{1}_Kt{2}_{5}_orders[{3},{4}).hdf5'.format(starname, K_star, K_t, start_order, end_order, parameters.dictionary["output_suffix"])
print("running wobble on star {0} with K_star = {1}, K_t = {2}, orders[{3},{4})".format(starname, K_star, K_t, start_order, end_order))
start_time = time()
orders = np.arange(start_order, end_order)
data = wobble.Data(starname+'_vis'+'_e2ds.hdf5', filepath= data_directory, orders=orders, epochs=epochs_list, min_flux=10**-5, min_snr=0)
# min_snr=0 is part of a HACK that centralises epoch cutting to the top level script, will this interfere with order cutting?
data = mask_tellurics(data)
results = wobble.Results(data=data)
''' doesn't work becaus the enoumeration of orders will restart at 0
#Load wobble results file should it already exist, so that only the current optimzed chunk gets replaced
#note this may fail if the data is not the same as the data used to create the original results file
try:
results = wobble.Results(filename=results_file)
except FileNotFoundError:
results = wobble.Results(data=data)
'''
print("data loaded")
niter = 150 # for optimization
plots = True
epochs = [0, 50] # to plot
movies = False
star_reg_file = '../wobble/regularization/{0}_star_K{1}.hdf5'.format(
starname, K_star)
tellurics_reg_file = '../wobble/regularization/{0}_t_K{1}.hdf5'.format(
starname, K_t)
plot_dir = '../results/plots_{0}_Kstar{1}_Kt{2}/'.format(
starname, K_star, K_t)
if False:
# quick test on two orders
data = wobble.Data(starname + '_e2ds.hdf5',
filepath='../data/',
orders=[30, 56])
results = wobble.Results(data=data)
for r in range(data.R):
model = wobble.Model(data, results, r)
model.add_star('star',
variable_bases=K_star,
regularization_par_file=star_reg_file,
learning_rate_template=0.01,
learning_rate_rvs=1.)
model.add_telluric('tellurics',
rvs_fixed=True,
variable_bases=K_t,
regularization_par_file=tellurics_reg_file,
learning_rate_template=0.01)
wobble.optimize_order(model,
if i == start:
start_order = i
end_order = i + chunk_size
chunks = [[start_order, end_order]]
else:
if i + chunk_size < end:
start_order = i
end_order = i + chunk_size
else:
start_order = i
end_order = end
chunks = np.append(chunks, [[start_order, end_order]], axis=0)
# HACK global epoch cutting: use epoch (and order) cutting only onnce on all data, then pass epochs to chunks
data = wobble.Data(starname + '_vis' + '_e2ds.hdf5',
filepath='../data/',
orders=np.arange(start, end),
min_flux=10**-5)
epochs = np.asarray(data.epochs)
epochs_list_str = str(list(epochs))
for i in range(len(chunks)):
start_order = chunks[i, 0]
end_order = chunks[i, 1]
os.system(
"python3 script_chunkwise_def.py {0} {1} {2} {3} {4} \"{5}\"".format(
start_order, end_order, starname, K_star, K_t, epochs_list_str
)) # TODO make this name context dependent/ work for any star params
#TODO pass this asa parameter file, its getting unwieldy
print("time elapsed total: {0:.2f} min".format(
(time() - start_time) / 60.0))
#tellurics_filename_final = reg_t_file_base + '_orders[{0},{1})_'.format(start, end)+ output_suffix + '.hdf5'
star_filename_final = star_reg_file
tellurics_filename_final = tellurics_reg_file
####################
start_time = time()
#make list of chunks
chunks = cr.chunk_list(start, end, chunk_size)
# HACK global epoch cutting: use epoch (and order) cutting only once on all data, then pass epochs to chunks
orders = np.arange(start, end)
data = wobble.Data(data_file, orders = orders, min_flux=10**-5)
epochs = np.asarray(data.epochs)
epochs_list = data.epochs.tolist()
epochs_list_str = str(list(epochs))
for i in range(len(chunks)):
start_chunk = int(chunks[i, 0])
end_chunk = int(chunks[i, 1])
parameters.dictionary.update({
"epochs_list" : epochs_list,
"start_chunk" : start_chunk,
"end_chunk" : end_chunk
})
#Write into YAML to pass to chunk script
starname = 'barnards'
K_star = 0
K_t = 0
niter = 150 # for optimization
plots = True
epochs = [0, 50] # to plot
movies = False
star_reg_file = '../wobble/regularization/{0}_star_K{1}.hdf5'.format(starname, K_star)
tellurics_reg_file = '../wobble/regularization/{0}_t_K{1}.hdf5'.format(starname, K_t)
plot_dir = '../results/plots_{0}_Kstar{1}_Kt{2}/'.format(starname, K_star, K_t)
print("running wobble on star {0} with K_star = {1}, K_t = {2}".format(starname, K_star, K_t))
start_time = time()
orders = np.arange(72)
data = wobble.Data(filename='../data/'+starname+'_e2ds.hdf5', orders=orders)
if True: # reload data and remove all post-upgrade spectra
upgrade = 2457174.5 # June 2015
e = data.epochs[data.dates < upgrade]
data = wobble.Data(filename='../data/'+starname+'_e2ds.hdf5', orders=orders, epochs=e)
data.drop_bad_orders()
data.drop_bad_epochs()
orders = np.copy(data.orders)
results = wobble.Results(data=data)
print("data loaded")
print("time elapsed: {0:.2f} min".format((time() - start_time)/60.0))
elapsed_time = time() - start_time
if plots:
def run_wobble(parameters):
p = parameters
results_name = 'results_{0}_Kstar{1}_Kt{2}_'.format(
p.starname, p.K_star, p.K_t, p.niter) + p.output_suffix
results_file_base = p.results_dir + results_name
results_file = results_file_base + '.hdf5'
data_file = p.data_file = p.data_dir + p.starname + p.data_suffix + '_e2ds.hdf5'
temp_dir = p.temp_dir = p.results_dir + '/temp_' + results_name + '/'
plot_dir = p.plot_dir = p.results_dir + '/plots_' + results_name + '/'
#make (output) directory
#TODO these data and results dirs should be handlesd somewhere else
os.makedirs(p.results_dir, exist_ok=True)
os.makedirs(p.data_dir, exist_ok=True)
os.makedirs(temp_dir, exist_ok=True)
os.makedirs(plot_dir, exist_ok=True)
start_time = p.start_time = time()
#generate epoch list
# if parameters has been passed a global epochs list use this going forward. This is primarily used by regularization.py
try: #skipping the except will make major issues unless dropped epochs and orders are already handled
epochs_list = p.epochs_list = p.global_epochs_list
p.drop_orders = data.drop_orders # this check here only works if data object is not yet initialized with empty drop order list.
except (AttributeError, UnboundLocalError) as e:
print(e)
print("Loading data. May take a few minutes")
try:
data = wobble.Data(data_file,
orders=np.arange(p.start, p.end),
min_flux=10**-5,
min_snr=p.min_snr,
parameters=p)
epochs_list = p.epochs_list = data.epochs.tolist()
except wobble.data.AllDataDropped:
p.min_snr = 5
print("restarting with min_snr = {}".format(p.min_snr))
data = wobble.Data(data_file,
orders=np.arange(p.start, p.end),
min_flux=10**-5,
min_snr=p.min_snr,
parameters=p)
epochs_list = p.epochs_list = data.epochs.tolist()
#orders_list = p.orders_list = data.orders.tolist() #too agressive in visible # TODO implement alternate nir and vis handling
try:
p.drop_orders = data.drop_orders
print("data.drop_orders", data.drop_orders)
except AttributeError:
print("data.drop_orders is not defined")
base_orders_list = np.arange(p.start, p.end).tolist()
orders_list = p.orders_list = [
x for x in base_orders_list if x not in p.drop_orders
]
print("orders_list", orders_list)
chunks = p.chunks = chunk_list(p.start, p.end, p.chunk_size, p.orders_list)
print("Chunks: ", chunks)
#Loop over chunks
for i in range(len(chunks)):
#pass parameters object to chunk script
p.i = i
with open(
os.path.dirname(os.path.abspath(__file__)) +
"/carmenes_aux_files/chunk_parameters.pkl", "wb") as f:
dill.dump(p, f)
#start chunk script
os.system("python3 {0}/chunk.py".format(
os.path.dirname(os.path.abspath(__file__))))
'''HACK to be removed
#import parameters back after possible changes in chunk
with open(os.path.dirname(os.path.abspath(__file__)) + "/" + "carmenes_aux_files/chunk_parameters.pkl", "rb") as f:
p = dill.load(f)
'''
print("all chunks optimized: writing combined file")
results_file_stitch(p.chunks, results_file, temp_dir)
#Combine orders
results = wobble.Results(filename=results_file)
results.combine_orders('star')
print("final RVs calculated.")
print("time elapsed: {0:.2f} minutes".format((time() - start_time) / 60.0))
results.write(results_file)
append_dates_utc(
results_file, data_file
) # cannot be done before results.write, as .write will remove dates_utc
append_parameters(p, results_file)
'''
# test
loaded_parameters = read_parameters_from_results(results_file)
attrs = dir(loaded_parameters)
print(attrs)
for attr in attrs:
#print(attr, getattr(loaded_parameters, attr))
q = getattr(p, attr) == getattr(loaded_parameters, attr)
if q == True:
print(q)
else:
print(attr, q)
'''
print("results saved as: {0}".format(results_file))
print("time elapsed: {0:.2f} minutes".format((time() - start_time) / 60.0))
#delete temp_dir which at this point only contains duplicates
shutil.rmtree(temp_dir)
print("deleted: {0}".format(temp_dir))