/
telluric_correction_from_template.py
399 lines (353 loc) · 20.3 KB
/
telluric_correction_from_template.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
import os, imp
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from scipy.interpolate import interp1d, splrep, splev
from astropy.table import Table
from match_ids import *
from lmfit import Parameters, minimize
from lmfit.models import GaussianModel, LinearModel
imp.load_source('s_collection', '../Carbon-Spectra/spectra_collection_functions.py')
from s_collection import CollectionParameters
imp.load_source('templates', '../H-alpha_map/template_spectra_function.py')
from templates import *
imp.load_source('s_helper', '../Carbon-Spectra/helper_functions.py')
from s_helper import get_spectra_dr52
# Functions
def spectra_resample(spectra, wvl_orig, wvl_target):
idx_finite = np.isfinite(spectra)
min_wvl_s = np.nanmin(wvl_orig[idx_finite])
max_wvl_s = np.nanmax(wvl_orig[idx_finite])
bspline = splrep(wvl_orig[idx_finite], spectra[idx_finite])
idx_target = np.logical_and(wvl_target >= min_wvl_s,
wvl_target <= max_wvl_s)
new_flux = splev(wvl_target[idx_target], bspline)
nex_flux_out = np.ndarray(len(wvl_target))
nex_flux_out.fill(np.nan)
nex_flux_out[idx_target] = new_flux
return nex_flux_out
def new_txt_file(filename):
temp = open(filename, 'w')
temp.close()
def append_line(filename, line_string, new_line=False):
temp = open(filename, 'a')
if new_line:
temp.write(line_string+'\n')
else:
temp.write(line_string)
temp.close()
def minimize_scale(param, template_residuum, teluric):
param_values = param.valuesdict()
return np.power(template_residuum + teluric*param_values['scale'], 2)
def minimize_scale_gauss(param, y_ref, y_fit):
param_values = param.valuesdict()
return np.power(y_ref - y_fit*param_values['scale'], 2)
def wvl_values_range_lines(lines, wvl, width=1.):
idx_pos = np.full_like(wvl, False)
for line in lines:
idx_pos = np.logical_or(idx_pos,
np.abs(wvl - line) <= width/2.)
return idx_pos
def extrema_value(vals):
min = np.nanmin(vals)
max = np.nanmax(vals)
if np.abs(min) > np.abs(max):
return min
else:
return max
print 'Reading data sets'
galah_template_dir = '/home/klemen/GALAH_data/Spectra_template_grid/galah_dr52_ccd3_6475_6745_wvlstep_0.03_lin_RF_renorm/Teff_300_logg_0.50_feh_0.20_snr_40_medianshift_std_2.5_redflag/'
galah_data_dir = '/home/klemen/GALAH_data/'
galah_spectra_dir = '/media/storage/HERMES_REDUCED/dr5.2/'
galah_param = Table.read(galah_data_dir+'sobject_iraf_52_reduced.csv', format='ascii.csv')
# determine unique numbers of observation field
observation_fields = np.int64(galah_param['sobject_id']/1000.)
all_observation_fields = np.unique(observation_fields)
selected_observation_fields = all_observation_fields
get_fields = len(selected_observation_fields)
C_LIGHT = 299792458 # m/s
wvl_min = 6500
wvl_max = 6550
shift_for_barycentric = True
print 'Reading resampled GALAH spectra'
spectra_file_csv_obs = 'galah_dr52_ccd3_6475_6745_wvlstep_0.03_lin_RF_renorm.csv'
# parse resampling settings from filename
csv_param = CollectionParameters(spectra_file_csv_obs)
ccd = csv_param.get_ccd()
wvl_start, wvl_end = csv_param.get_wvl_range()
wvl_values = csv_param.get_wvl_values()
# determine the data range to be read and read it
idx_read = np.where(np.logical_and(wvl_values > wvl_min,
wvl_values < wvl_max))
spectral_data = pd.read_csv(galah_data_dir + spectra_file_csv_obs,
sep=',', header=None, na_values='nan', usecols=idx_read[0]).values
spectral_data_size = np.shape(spectral_data)
print spectral_data_size
wvl_read = wvl_values[idx_read]
wvl_read_finer = np.arange(wvl_min, wvl_max, csv_param.get_wvl_step()/3)
# atmospheric features line list
teluric_line_list = pd.read_csv('telluric_linelist.csv')
emission_line_list = pd.read_csv('emission_linelist_2_used.csv')
# subset of atmospheric lines
line_cor_width = 1.5 # Angstrom
teluric_line_list = teluric_line_list[np.logical_and(teluric_line_list['Ang'] > wvl_min,
teluric_line_list['Ang'] < wvl_max)]
emission_line_list = emission_line_list[emission_line_list['Flux'] >= 1.]
emission_line_list = emission_line_list[np.logical_and(emission_line_list['Ang'] > wvl_min,
emission_line_list['Ang'] < wvl_max)]
out_dir = 'Correction_with_template_2'
if not os.path.exists(out_dir):
os.mkdir(out_dir)
os.chdir(out_dir)
grid_list = Table.read(galah_template_dir + 'grid_list.csv', format='ascii.csv')
selected_observation_fields = list([140117001101])
out_fields = '_fields_list.csv'
out_residuals = '_template_residuals_median.csv'
out_residuals_abs = '_template_residuals_median_abs.csv'
if not os.path.isfile(out_residuals):
new_txt_file(out_fields)
new_txt_file(out_residuals)
new_txt_file(out_residuals_abs)
for field_id in selected_observation_fields: # filter by date
print 'Working on field '+str(field_id)
spectra_row = np.where(field_id == observation_fields)
# initialize plot
n_in_field = len(spectra_row[0])
if n_in_field < 100:
continue
# create a stack of field residuals
field_residuals = np.ndarray((n_in_field, len(wvl_read_finer)))
# init plot
fig, axes = plt.subplots(2, 1)
for i_row in range(len(spectra_row[0])):
row = spectra_row[0][i_row]
object_param = galah_param[row]
object_spectra = spectral_data[row]
# get template spectra
template_file = get_best_match(object_param['teff_guess'], object_param['logg_guess'], object_param['feh_guess'], grid_list, midpoint=False)+'.csv'
template_spectra = np.loadtxt(galah_template_dir + template_file, delimiter=',')[idx_read]
# subtract spectra
spectra_residuum = object_spectra - template_spectra
# shift to observed frame
velocity_shift = object_param['rv_guess_shift'] - object_param['v_bary']
wvl_shifted = wvl_read * (1 + velocity_shift * 1000. / C_LIGHT)
# resample shifted RV/barycentric shifted to common positions - OVERSAMPLING DATA
spectra_residuum_resampled = spectra_resample(spectra_residuum, wvl_shifted, wvl_read_finer) # spectrum, in, out
field_residuals[i_row, :] = spectra_residuum_resampled
# plot graphs
axes[0].plot(wvl_shifted, spectra_residuum, color='blue', alpha=0.02, linewidth=0.8)
axes[1].plot(wvl_read, spectra_residuum, color='blue', alpha=0.02, linewidth=0.8)
# emission investigation
# idx_emission_1 = wvl_values_range_lines(list([emission_line_list['Ang'].values[0]]), wvl_read_finer, width=1.5)
# idx_emission_2 = wvl_values_range_lines(list([emission_line_list['Ang'].values[0]]), wvl_read_finer, width=1.5)
# print extrema_value(spectra_residuum_resampled[idx_emission_1]), extrema_value(spectra_residuum_resampled[idx_emission_2])
# residuals statistics
# determine a first telluric approximation for all objects in a field
residuum_median = np.nanmedian(field_residuals, axis=0)
residuum_median_abs = np.nanmedian(np.abs(field_residuals), axis=0)
axes[0].plot(wvl_read_finer, residuum_median, color='black', alpha=0.8, linewidth=0.8)
axes[0].set(xlim=(wvl_min, wvl_max), ylim=(-0.2, 0.2), ylabel='Observed')
axes[1].set(xlim=(wvl_min, wvl_max), ylim=(-0.2, 0.2), ylabel='Restframe', xlabel='Wavelength')
axes[0].grid(True)
axes[1].grid(True)
plt.tight_layout()
plt.savefig(str(field_id)+'_residuals.png', dpi=250)
plt.close()
# save results to files
append_line(out_fields, str(field_id), new_line=True)
append_line(out_residuals, ','.join([str(f) for f in residuum_median]), new_line=True)
append_line(out_residuals_abs, ','.join([str(f) for f in residuum_median_abs]), new_line=True)
# --------------------------------------------------
# READ DATA from previous processing step
fields_ids = pd.read_csv(out_fields, header=None, sep=',', na_values='nan').values.ravel()
fields_residuals = pd.read_csv(out_residuals, header=None, sep=',', na_values='nan').values
fields_residuals_abs = pd.read_csv(out_residuals_abs, header=None, sep=',', na_values='nan').values
# --------------------------------------------------
# identification of telluric wavelengths or structures
mean_field_residuals = np.nanmean(fields_residuals, axis=0)
idx_telluric_wvl = wvl_values_range_lines(teluric_line_list['Ang'], wvl_read_finer, width=1.5)
for i_f in range(fields_residuals.shape[0]):
plt.plot(wvl_read_finer, fields_residuals[i_f], color='blue', alpha=0.02, linewidth=0.8)
plt.scatter(wvl_read_finer[idx_telluric_wvl], mean_field_residuals[idx_telluric_wvl], s=3, c='black', lw=0)
for line in teluric_line_list['Ang'].values:
plt.axvline(x=line, color='red', linewidth=0.5)
for line in emission_line_list['Ang'].values:
plt.axvline(x=line, color='green', linewidth=0.5)
plt.xlim((wvl_min, wvl_max))
plt.ylim((-0.2, 0.2))
plt.savefig('_template_residuals.png', dpi=350)
plt.close()
# identification of emission wavelengths or structures
mean_field_residuals_abs = np.nanmean(fields_residuals_abs, axis=0)
idx_emission_wvl = wvl_values_range_lines(emission_line_list['Ang'], wvl_read_finer, width=1.5)
for i_f in range(fields_residuals_abs.shape[0]):
plt.plot(wvl_read_finer, fields_residuals_abs[i_f], color='blue', alpha=0.02, linewidth=0.8)
plt.scatter(wvl_read_finer[idx_emission_wvl], mean_field_residuals_abs[idx_emission_wvl], s=3, c='black', lw=0)
for line in teluric_line_list['Ang'].values:
plt.axvline(x=line, color='red', linewidth=0.5)
for line in emission_line_list['Ang'].values:
plt.axvline(x=line, color='green', linewidth=0.5)
plt.xlim((wvl_min, wvl_max))
plt.ylim((-0.2, 0.2))
plt.savefig('_template_residuals_abs.png', dpi=350)
plt.close()
# refine this approximation using the information about the wavelengths of the telluric lines
# raise SystemExit
# subset of observations - for test corrections
# i_obj_selected = np.where(observation_fields == 140309002101)[0]
# try to fit the approximation to every individual spectra in the field
for i_obj in range(0, len(galah_param), 30):
object_param = galah_param[i_obj]
object_id = object_param['sobject_id']
object_in_field = np.int64(object_id/1000.)
print 'Correcting object: '+str(object_id)+' field: '+str(object_in_field)
if object_in_field in fields_ids:
# get previously determined residual
field_residual_observed = fields_residuals[np.where(fields_ids == object_in_field)[0], :][0]
field_residual_abs_observed = fields_residuals_abs[np.where(fields_ids == object_in_field)[0], :][0]
object_spectra = spectral_data[i_obj]
velocity_shift = object_param['rv_guess_shift'] - object_param['v_bary']
# get template spectra
template_file = get_best_match(object_param['teff_guess'], object_param['logg_guess'],
object_param['feh_guess'], grid_list, midpoint=False) + '.csv'
template_spectra = np.loadtxt(galah_template_dir + template_file, delimiter=',')[idx_read]
# --------------------------------------------------
# STEP 1:
# wavelengths to be corrected - first correct part of the spectra influenced by the telluric effects
idx_emission_wvl = wvl_values_range_lines(emission_line_list['Ang'], wvl_read_finer, width=1.)
idx_telluric_wvl = wvl_values_range_lines(teluric_line_list['Ang'], wvl_read_finer, width=1.5)
idx_correction_skip = np.logical_or(np.logical_not(idx_telluric_wvl), idx_emission_wvl)
# create new temporary array from determined residuals
field_residuum_telluric = np.array(field_residual_observed)
# unset masked wavelengths
field_residuum_telluric[idx_correction_skip] = 0
# resample and shift residuals from observed to restframe
field_residuum_telluric = spectra_resample(field_residuum_telluric, wvl_read_finer/(1 + velocity_shift * 1000. / C_LIGHT), wvl_read)
field_residuum_telluric_binary = spectra_resample(np.logical_not(idx_correction_skip),
wvl_read_finer/(1 + velocity_shift * 1000. / C_LIGHT), wvl_read) > 0.5
# set the residuum outside observed wavelengths to 0
object_spectra_corrected_s1 = object_spectra - field_residuum_telluric
template_res_before_s1 = template_spectra - object_spectra
template_res_after_s1 = template_spectra - object_spectra_corrected_s1
# the best residuum scaling factor based on reference spectra
fit_param = Parameters()
fit_param.add('scale', value=1, min=0., max=10., brute_step=0.1)
fit_res = minimize(minimize_scale, fit_param, # method='brute',
args=(template_res_before_s1, field_residuum_telluric),
**{'nan_policy': 'omit'})
fit_res.params.pretty_print()
residuum_scale = fit_res.params['scale'].value
object_spectra_corrected_s1_scaled = object_spectra - residuum_scale * field_residuum_telluric
template_res_after_s1_scaled = template_spectra - object_spectra_corrected_s1_scaled
print np.nansum(template_res_after_s1**2), np.nansum(template_res_after_s1_scaled**2)
fig, axes = plt.subplots(2, 1)
suptitle = 'Guess -> teff:{:4.0f} logg:{:1.1f} feh:{:1.1f} \n correction scale:{:.1f}'.format(object_param['teff_guess'],
object_param['logg_guess'],
object_param['feh_guess'], residuum_scale)
fig.suptitle(suptitle)
axes[0].plot(wvl_read, template_spectra, color='red', linewidth=0.8)
axes[0].plot(wvl_read, object_spectra, color='black', linewidth=0.8)
axes[0].plot(wvl_read, object_spectra_corrected_s1, color='blue', linewidth=0.8)
axes[0].plot(wvl_read, object_spectra_corrected_s1_scaled, color='green', linewidth=0.8)
axes[0].set(xlim=(wvl_min, wvl_max), ylim=(0.2, 1.1), ylabel='Spectra')
on = False
off = True
for i_t in range(len(field_residuum_telluric_binary)):
f_r_b = field_residuum_telluric_binary[i_t]
if off and f_r_b:
span_s = i_t
off = False
on = True
if on and not f_r_b:
span_e = i_t
off = True
on = False
axes[1].axvspan(wvl_read[span_s], wvl_read[span_e], facecolor='black', alpha=0.1)
axes[1].plot(wvl_read, field_residuum_telluric + 0.1, color='black', linewidth=0.8)
axes[1].plot(wvl_read, template_res_before_s1, color='black', linewidth=0.8)
axes[1].plot(wvl_read, template_res_after_s1, color='blue', linewidth=0.8)
axes[1].plot(wvl_read, template_res_after_s1_scaled, color='green', linewidth=0.8)
axes[1].set(xlim=(wvl_min, wvl_max), ylim=(-0.2, 0.2), ylabel='Template residual')
axes[1].grid(True)
plt.savefig(str(object_id)+'_t.png', dpi=350)
plt.close()
# --------------------------------------------------
# --------------------------------------------------
# STEP 2:
# wavelengths to be corrected - first correct part of the spectra influenced by the telluric effects
# wavelengths to be corrected - first correct part of the spectra influenced by the telluric effects
idx_emission_wvl = wvl_values_range_lines(emission_line_list['Ang'], wvl_read_finer, width=1.5)
idx_telluric_wvl = wvl_values_range_lines(teluric_line_list['Ang'], wvl_read_finer, width=0.1)
idx_correction_skip = np.logical_or(np.logical_not(idx_emission_wvl), idx_telluric_wvl)
# create new temporary array from determined residuals
field_residuum_emission = np.array(field_residual_abs_observed)
# unset masked wavelengths
field_residuum_emission[idx_correction_skip] = 0
# resample and shift residuals from observed to restframe
field_residuum_emission = spectra_resample(field_residuum_emission,
wvl_read_finer / (1 + velocity_shift * 1000. / C_LIGHT), wvl_read)
field_residuum_emission_binary = spectra_resample(np.logical_not(idx_correction_skip),
wvl_read_finer / (1 + velocity_shift * 1000. / C_LIGHT),
wvl_read) > 0.5
# set the residuum outside observed wavelengths to 0
object_spectra_corrected_s2 = object_spectra - field_residuum_emission
template_res_before_s2 = template_spectra - object_spectra
template_res_after_s2 = template_spectra - object_spectra_corrected_s2
# get extension 0 and 2 for this spectra
object_spectra_ext0, wvl_ext0 = get_spectra_dr52(str(object_id), bands=[3], root=galah_spectra_dir, extension=0)
object_spectra_ext2, _ = get_spectra_dr52(str(object_id), bands=[3], root=galah_spectra_dir, extension=2)
sky_correction = object_spectra_ext0[0] - object_spectra_ext2[0]
# resample and shift residuals from barycentric to restframe
wvl_ext0 = wvl_ext0 / (1 + object_param['rv_guess_shift'] * 1000. / C_LIGHT)
# normalize sky correction
sky_correction -= np.median(sky_correction)
sky_correction_scaled = sky_correction/750. + 0.5
# create a comb of gaussian functions to be fitted
wvl_scale = (1 + velocity_shift * 1000. / C_LIGHT)
gauss_comb = np.full_like(object_spectra, 0.)
for l_index, line in emission_line_list.iterrows():
gauss_comb += line['Flux'] / (line['Std'] * np.sqrt(2*np.pi)) * np.exp(-0.5 * (line['Ang']/wvl_scale - wvl_read) ** 2 / line['Std'] ** 2)
# the best residuum scaling factor based on reference spectra
fit_param = Parameters()
fit_param.add('scale', value=1, min=-10., max=10., brute_step=0.1)
fit_res_comb = minimize(minimize_scale_gauss, fit_param, # method='brute',
args=(template_res_before_s2, gauss_comb),
**{'nan_policy': 'omit'})
fit_res_comb.params.pretty_print()
gauss_comb_scaled = gauss_comb * fit_res_comb.params['scale'].value
object_spectra_corrected_s2 = object_spectra + gauss_comb_scaled
fig, axes = plt.subplots(2, 1)
# suptitle = 'Guess -> teff:{:4.0f} logg:{:1.1f} feh:{:1.1f} \n correction scale:{:.1f}'.format(
# object_param['teff_guess'],
# object_param['logg_guess'],
# object_param['feh_guess'], residuum_scale)
# fig.suptitle(suptitle)
axes[0].plot(wvl_ext0[0], sky_correction_scaled, color='pink', linewidth=0.8)
axes[0].plot(wvl_read, template_spectra, color='red', linewidth=0.8)
axes[0].plot(wvl_read, object_spectra, color='black', linewidth=0.8)
# axes[0].plot(wvl_read, object_spectra_corrected_s2, color='blue', linewidth=0.8)
axes[0].plot(wvl_read, object_spectra_corrected_s2, color='green', linewidth=0.8)
axes[0].set(xlim=(wvl_min, wvl_max), ylim=(0.2, 1.1), ylabel='Spectra')
on = False
off = True
for i_t in range(len(field_residuum_emission_binary)):
f_r_b = field_residuum_emission_binary[i_t]
if off and f_r_b:
span_s = i_t
off = False
on = True
if on and not f_r_b:
span_e = i_t
off = True
on = False
axes[1].axvspan(wvl_read[span_s], wvl_read[span_e], facecolor='black', alpha=0.1)
# axes[1].plot(wvl_read, field_residuum_emission + 0.1, color='black', linewidth=0.8)
axes[1].plot(wvl_read, template_res_before_s2, color='black', linewidth=0.8)
# axes[1].plot(wvl_read, template_res_after_s2, color='blue', linewidth=0.8)
axes[1].plot(wvl_read, gauss_comb_scaled, color='green', linewidth=0.8)
axes[1].set(xlim=(wvl_min, wvl_max), ylim=(-0.3, 0.3), ylabel='Template residual')
axes[1].grid(True)
plt.savefig(str(object_id) + '_e.png', dpi=350)
plt.close()
print ''
# refine the approximations using the position of the stars in the observation field and their altitude angle