def get_min_flux(self, debug=False):
""" Obtaining the low end of the interval for sampling the S/N. Based
on the initial estimation of the radial profile of the mean frame.
"""
# Getting the radial profile in the mean frame of the cube
sampling_sep = 1
radius_int = 1
if GARRAY.ndim == 3:
global_frame = np.mean(GARRAY, axis=0)
elif GARRAY.ndim == 4:
global_frame = np.mean(GARRAY.reshape(-1, GARRAY.shape[2],
GARRAY.shape[3]),
axis=0)
me = frame_average_radprofile(global_frame,
sep=sampling_sep,
init_rad=radius_int,
plot=False)
radprof = np.array(me.radprof)
radprof = radprof[np.array(self.distances) + 1]
radprof[radprof < 0] = 0.01
self.radprof = radprof
print(
"Estimating the lower flux interval for sampling the S/N vs flux "
"function")
flux_min = pool_map(self.n_proc, _get_min_flux, iterable(self.n_dist),
self.distances, radprof, self.fwhm, self.plsc,
iterable(self.min_snr), self.wavelengths,
self.spectrum, self.algo, self.scaling,
self.svd_mode, self.random_seed, debug)
self.min_fluxes = flux_min
timing(self.starttime)
def _sample_flux_snr(distances,
fwhm,
plsc,
n_injections,
flux_min,
flux_max,
nproc=10,
random_seed=42,
wavelengths=None,
mode='median',
ncomp=2):
"""
Sensible flux intervals depend on a combination of factors, # of frames,
range of rotation, correlation, glare intensity.
"""
starttime = time_ini()
if GARRAY.ndim == 3:
frsize = int(GARRAY.shape[1])
elif GARRAY.ndim == 4:
frsize = int(GARRAY.shape[2])
ninj = n_injections
random_state = np.random.RandomState(random_seed)
flux_dist_theta_all = list()
snrs_list = list()
fluxes_list = list()
for i, d in enumerate(distances):
yy, xx = get_annulus_segments((frsize, frsize), d, 1, 1)[0]
num_patches = yy.shape[0]
fluxes_dist = random_state.uniform(flux_min[i], flux_max[i], size=ninj)
inds_inj = random_state.randint(0, num_patches, size=ninj)
for j in range(ninj):
injx = xx[inds_inj[j]]
injy = yy[inds_inj[j]]
injx -= frame_center(GARRAY[0])[1]
injy -= frame_center(GARRAY[0])[0]
dist = np.sqrt(injx**2 + injy**2)
theta = np.mod(np.arctan2(injy, injx) / np.pi * 180, 360)
flux_dist_theta_all.append((fluxes_dist[j], dist, theta))
# Multiprocessing pool
res = pool_map(nproc, _get_adi_snrs, GARRPSF, GARRPA, fwhm, plsc,
fixed(flux_dist_theta_all), wavelengths, mode, ncomp)
for i in range(len(distances)):
flux_dist = []
snr_dist = []
for j in range(ninj):
flux_dist.append(res[j + (ninj * i)][0])
snr_dist.append(res[j + (ninj * i)][1])
fluxes_list.append(flux_dist)
snrs_list.append(snr_dist)
timing(starttime)
return fluxes_list, snrs_list
def sampling(self):
""" Using the computed interval of fluxes for sampling the flux vs SNR
relationship.
"""
if not self.min_fluxes:
self.get_min_flux()
if not self.max_fluxes:
self.get_max_flux()
print("Sampling by injecting fake companions")
res = _sample_flux_snr(self.distances, self.fwhm, self.plsc,
self.n_injections, self.min_fluxes,
self.max_fluxes, self.n_proc, self.random_seed,
self.wavelengths, self.spectrum, self.algo,
self.scaling)
self.sampled_fluxes, self.sampled_snrs = res
timing(self.starttime)
def get_max_flux(self, debug=False):
""" Obtaining the high end of the interval for sampling the S/N.
"""
if self.min_fluxes is None:
self.get_min_flux()
print(
"Estimating the upper flux interval for sampling the S/N vs flux "
"function")
flux_max = pool_map(self.n_proc, _get_max_flux, iterable(self.n_dist),
self.distances,
self.min_fluxes, self.fwhm, self.plsc,
iterable(self.max_snr), self.wavelengths,
self.spectrum, self.algo, self.scaling,
self.svd_mode, self.random_seed, debug)
self.max_fluxes = flux_max
timing(self.starttime)
def train_3dnet(X,
Y,
test_size=0.1,
validation_split=0.1,
random_state=0,
layer_type=('conv3d', 'conv3d'),
nconvlayers=2,
conv_nfilters=(40, 80),
kernel_sizes=((3, 3, 3), (2, 2, 2)),
conv_strides=((1, 1, 1), (1, 1, 1)),
conv_padding='same',
dilation_rate=((1, 1, 1), (1, 1, 1)),
pool_layers=2,
pool_func='ave',
pool_sizes=((2, 2, 2), (2, 2, 2)),
pool_strides=((2, 2, 2), (2, 2, 2)),
dense_units=128,
rec_hidden_states=64,
activation='relu',
learnrate=0.003,
batchsize=64,
epochs=20,
patience=2,
min_delta=0.01,
retrain=None,
verb=1,
summary=True,
gpu_id='0',
plot='tb',
tb_path='./logs',
full_output=False):
""" 3D Convolutional network or Convolutional LSTM network for SODINN-PW
and SODINN-SVD.
Parameters
----------
...
layer_type : {'conv3d', 'clstm'} str optional
batchsize :
Batch size per GPU (no need to increase it when ``ngpus`` > 1).
"""
clear_session()
graph = get_default_graph()
config = tf.ConfigProto()
# Don't pre-allocate memory; allocate as-needed
config.gpu_options.allow_growth = True
# Only allow a total of half the GPU memory to be allocated
# config.gpu_options.per_process_gpu_memory_fraction = 0.5
config.gpu_options.visible_device_list = gpu_id
session = tf.Session(graph=graph, config=config)
set_session(session)
ngpus = len(gpu_id.split(','))
batchsize *= ngpus
if not nconvlayers == len(conv_nfilters):
raise ValueError('`conv_nfilters` has a wrong length')
if not nconvlayers == len(kernel_sizes):
raise ValueError('`kernel_sizes` has a wrong length')
if not nconvlayers == len(conv_strides):
raise ValueError('`conv_strides` has a wrong length')
if not nconvlayers == len(dilation_rate):
raise ValueError('`dilation_rate` has a wrong length')
if pool_layers > 0:
if not pool_layers == len(pool_sizes):
raise ValueError('`pool_sizes` has a wrong length')
if pool_strides is not None:
if not pool_layers == len(pool_strides):
raise ValueError('`pool_strides` has a wrong length')
else:
pool_strides = [None] * pool_layers
if isinstance(layer_type, str):
layer_type = [layer_type for _ in range(nconvlayers)]
starttime = time_ini()
# Mixed train/test sets with Sklearn split
res = train_test_split(X,
Y,
test_size=test_size,
random_state=random_state)
X_train, X_test, y_train, y_test = res
msg = 'Zeros in train: {} | Ones in train: {}'
print(msg.format(y_train.tolist().count(0), y_train.tolist().count(1)))
msg = 'Zeros in test: {} | Ones in test: {}'
print(msg.format(y_test.tolist().count(0), y_test.tolist().count(1)))
# adding the "channels" dimension (1)
X_train = X_train.reshape(X_train.shape[0], X_train.shape[1],
X_train.shape[2], X_train.shape[3], 1)
X_test = X_test.reshape(X_test.shape[0], X_test.shape[1], X_test.shape[2],
X_test.shape[3], 1)
print("\nShapes of train and test:")
print(X_train.shape, y_train.shape, X_test.shape, y_test.shape, '\n')
# --------------------------------------------------------------------------
if retrain is not None:
M = retrain
# Training the network
M.compile(loss='binary_crossentropy',
metrics=['accuracy'],
optimizer=Adam(lr=learnrate, decay=1e-2))
early_stopping = EarlyStopping(monitor='val_loss',
patience=patience,
min_delta=min_delta,
verbose=verb)
hist = M.fit(X_train,
y_train,
batch_size=batchsize,
epochs=epochs,
initial_epoch=0,
verbose=verb,
validation_split=0.1,
callbacks=[early_stopping],
shuffle=True)
score = M.evaluate(X_test, y_test, verbose=verb)
print('\n Test score/loss:', score[0])
print(' Test accuracy:', score[1])
timing(starttime)
fintime = time_fin(starttime)
if full_output:
return M, hist.history, score, fintime
else:
return M
# --------------------------------------------------------------------------
# Creating the NN model
if ngpus > 1:
with tf.device('/cpu:0'):
M = Sequential()
else:
M = Sequential()
kernel_init = 'glorot_uniform'
bias_init = 'random_normal'
rec_act = 'hard_sigmoid'
rec_init = 'orthogonal'
temp_dim = X_train.shape[1] # npcs or pw slices
patch_size = X_train.shape[2]
input_shape = (temp_dim, patch_size, patch_size, 1)
# Stack of Conv3d, (B)CLSTM, (B)LRCN or (B)GRCN layers
for i in range(nconvlayers):
if layer_type[i] in ('conv3d', 'clstm', 'bclstm'):
if pool_func == 'ave':
pooling_func = AveragePooling3D
elif pool_func == 'max':
pooling_func = MaxPooling3D
elif layer_type[i] in ('lrcn', 'blrcn', 'grcn', 'bgrcn'):
if pool_func == 'ave':
pooling_func = AveragePooling2D
elif pool_func == 'max':
pooling_func = MaxPooling2D
else:
raise ValueError('pool_func is not recognized')
if layer_type[i] == 'conv3d':
if not len(kernel_sizes[i]) == 3:
raise ValueError(
'Kernel sizes for Conv3d are tuples of 3 values')
if not len(conv_strides[i]) == 3:
raise ValueError('Strides for Conv3d are tuples of 3 values')
if not len(dilation_rate[i]) == 3:
raise ValueError('Dilation for Conv3d is a tuple of 3 values')
if i == 0:
M.add(
Conv3D(filters=conv_nfilters[i],
kernel_size=kernel_sizes[i],
strides=conv_strides[i],
padding=conv_padding,
kernel_initializer=kernel_init,
bias_initializer=bias_init,
name='conv3d_layer1',
dilation_rate=dilation_rate[i],
data_format='channels_last',
input_shape=input_shape))
M.add(SpatialDropout3D(0.5))
else:
M.add(
Conv3D(filters=conv_nfilters[i],
kernel_size=kernel_sizes[i],
strides=conv_strides[i],
padding=conv_padding,
kernel_initializer=kernel_init,
dilation_rate=dilation_rate[i],
name='conv3d_layer' + str(i + 1)))
M.add(SpatialDropout3D(0.25))
M.add(Activation(activation, name='activ_layer' + str(i + 1)))
if pool_layers != 0:
M.add(
pooling_func(pool_size=pool_sizes[i],
strides=pool_strides[i],
padding='valid'))
pool_layers -= 1
M.add(Dropout(rate=0.25, name='dropout_layer' + str(i + 1)))
elif layer_type[i] == 'clstm':
msg = 'are tuples of 2 integers'
if not len(kernel_sizes[i]) == 2:
raise ValueError('Kernel sizes for ConvLSTM' + msg)
if not len(conv_strides[i]) == 2:
raise ValueError('Strides for ConvLSTM')
if not len(dilation_rate[i]) == 2:
raise ValueError('Dilation rates for ConvLSTM')
if i == 0:
M.add(
ConvLSTM2D(
filters=conv_nfilters[i],
kernel_size=kernel_sizes[i],
strides=conv_strides[i],
padding=conv_padding,
kernel_initializer=kernel_init,
input_shape=input_shape,
name='convlstm_layer1',
return_sequences=True,
dilation_rate=dilation_rate[i],
activation='tanh',
recurrent_activation=rec_act,
use_bias=True,
recurrent_initializer=rec_init,
bias_initializer='zeros',
unit_forget_bias=True,
# TODO: Errors when using dropout, Keras bug?
dropout=0.0,
recurrent_dropout=0.0))
M.add(SpatialDropout3D(0.5))
else:
M.add(
ConvLSTM2D(
filters=conv_nfilters[i],
kernel_size=kernel_sizes[i],
strides=conv_strides[i],
padding=conv_padding,
kernel_initializer=kernel_init,
name='convlstm_layer' + str(i + 1),
return_sequences=True,
dilation_rate=dilation_rate[i],
activation='tanh',
recurrent_activation=rec_act,
use_bias=True,
recurrent_initializer=rec_init,
bias_initializer='zeros',
unit_forget_bias=True,
# TODO: Errors when using dropout, Keras bug?
dropout=0.0,
recurrent_dropout=0.0))
M.add(SpatialDropout3D(0.25))
if pool_layers != 0:
M.add(
pooling_func(pool_size=pool_sizes[i],
strides=pool_strides[i],
padding='valid'))
pool_layers -= 1
elif layer_type[i] == 'bclstm':
msg = 'are tuples of 2 integers'
if not len(kernel_sizes[i]) == 2:
raise ValueError('Kernel sizes for ConvLSTM' + msg)
if not len(conv_strides[i]) == 2:
raise ValueError('Strides for ConvLSTM')
if not len(dilation_rate[i]) == 2:
raise ValueError('Dilation rates for ConvLSTM')
if i == 0:
M.add(
Bidirectional(ConvLSTM2D(filters=conv_nfilters[i],
kernel_size=kernel_sizes[i],
strides=conv_strides[i],
padding=conv_padding,
kernel_initializer=kernel_init,
name='convlstm_layer1',
return_sequences=True,
dilation_rate=dilation_rate[i],
activation='tanh',
recurrent_activation=rec_act,
use_bias=True,
recurrent_initializer=rec_init,
bias_initializer='zeros',
unit_forget_bias=True),
input_shape=input_shape))
M.add(SpatialDropout3D(0.5))
else:
M.add(
Bidirectional(
ConvLSTM2D(filters=conv_nfilters[i],
kernel_size=kernel_sizes[i],
strides=conv_strides[i],
padding=conv_padding,
kernel_initializer=kernel_init,
name='convlstm_layer' + str(i + 1),
return_sequences=True,
dilation_rate=dilation_rate[i],
activation='tanh',
recurrent_activation=rec_act,
use_bias=True,
recurrent_initializer=rec_init,
bias_initializer='zeros',
unit_forget_bias=True)))
M.add(SpatialDropout3D(0.25))
if pool_layers != 0:
M.add(
pooling_func(pool_size=pool_sizes[i],
strides=pool_strides[i],
padding='valid'))
pool_layers -= 1
elif layer_type[i] in ('lrcn', 'blrcn', 'grcn', 'bgrcn'):
if not len(kernel_sizes[i]) == 2:
raise ValueError(
'Kernel sizes for LRCN are tuples of 2 values')
if not len(conv_strides[i]) == 2:
raise ValueError('Strides for LRCN are tuples of 2 values')
if not len(dilation_rate[i]) == 2:
raise ValueError('Dilation for LRCN is a tuple of 2 values')
if i == 0:
# TimeDistributed wrapper applies a layer to every temporal
# slice of an input. The input should be at least 3D and the
# dimension of index one will be considered to be the temporal
# dimension.
M.add(
TimeDistributed(Conv2D(filters=conv_nfilters[i],
kernel_size=kernel_sizes[i],
strides=conv_strides[i],
padding=conv_padding,
name='lrcn_layer1',
activation=activation),
input_shape=input_shape))
# This version performs the same function as Dropout, however it
# drops entire 2D feature maps instead of individual elements.
M.add(TimeDistributed(SpatialDropout2D(0.5)))
else:
M.add(
TimeDistributed(
Conv2D(filters=conv_nfilters[i],
kernel_size=kernel_sizes[i],
strides=conv_strides[i],
padding=conv_padding,
name='lrcn_layer' + str(i + 1),
activation=activation)))
M.add(TimeDistributed(SpatialDropout2D(0.25)))
if pool_layers != 0:
M.add(
TimeDistributed(
pooling_func(pool_size=pool_sizes[i],
strides=pool_strides[i],
padding='valid')))
pool_layers -= 1
# (B)LRCN or (B)GRCN on Conv2d extracted features
if layer_type[-1] == 'lrcn':
M.add(TimeDistributed(Flatten(name='flatten')))
M.add(Dropout(0.5, name='dropout_flatten'))
M.add(
CuDNNLSTM(rec_hidden_states,
kernel_initializer=kernel_init,
return_sequences=False))
M.add(Dropout(0.5, name='dropout_lstm'))
elif layer_type[-1] == 'blrcn':
M.add(TimeDistributed(Flatten(name='flatten')))
M.add(Dropout(0.5, name='dropout_flatten'))
M.add(
Bidirectional(
LSTM(
rec_hidden_states, # TODO: bug CuDNNLSTM?
kernel_initializer=kernel_init,
return_sequences=False)))
M.add(Dropout(0.5, name='dropout_lstm'))
elif layer_type[-1] == 'grcn':
M.add(TimeDistributed(Flatten(name='flatten')))
M.add(Dropout(0.5, name='dropout_flatten'))
M.add(
CuDNNGRU(rec_hidden_states,
kernel_initializer=kernel_init,
return_sequences=False))
M.add(Dropout(0.5, name='dropout_lstm'))
elif layer_type[-1] == 'bgrcn':
M.add(TimeDistributed(Flatten(name='flatten')))
M.add(Dropout(0.5, name='dropout_flatten'))
M.add(
Bidirectional(
CuDNNGRU(rec_hidden_states,
kernel_initializer=kernel_init,
return_sequences=False)))
M.add(Dropout(0.5, name='dropout_lstm'))
# Otherwise we just flatten and go to dense layers
else:
M.add(Flatten(name='flatten'))
# Fully-connected or dense layer
M.add(Dense(units=dense_units, name='dense_layer'))
M.add(Activation(activation, name='activ_dense'))
M.add(Dropout(rate=0.5, name='dropout_dense'))
# Sigmoid unit
M.add(Dense(units=1, name='dense_1unit'))
M.add(Activation('sigmoid', name='activ_out'))
if summary:
M.summary()
# Callbacks
early_stopping = EarlyStopping(monitor='val_loss',
patience=patience,
min_delta=min_delta,
verbose=verb)
if plot is not None:
if plot == 'tb':
tensorboard = TensorBoard(log_dir=tb_path,
histogram_freq=1,
write_graph=True,
write_images=True)
callbacks = [early_stopping, tensorboard]
elif plot == 'llp':
plotlosses = livelossplot.PlotLossesKeras()
callbacks = [early_stopping, plotlosses]
else:
raise ValueError("`plot` method not recognized")
else:
callbacks = [early_stopping]
# Training the network
if ngpus > 1:
# Multi-GPUs
Mpar = multi_gpu_model(M, gpus=ngpus)
# Training the network
Mpar.compile(loss='binary_crossentropy',
metrics=['accuracy'],
optimizer=Adam(lr=learnrate, decay=1e-2))
hist = Mpar.fit(X_train,
y_train,
batch_size=batchsize,
epochs=epochs,
initial_epoch=0,
verbose=verb,
shuffle=True,
validation_split=validation_split,
callbacks=callbacks)
score = Mpar.evaluate(X_test, y_test, verbose=verb)
else:
# Single GPU
M.compile(loss='binary_crossentropy',
metrics=['accuracy'],
optimizer=Adam(lr=learnrate, decay=1e-2))
hist = M.fit(X_train,
y_train,
batch_size=batchsize,
epochs=epochs,
initial_epoch=0,
verbose=verb,
shuffle=True,
validation_split=validation_split,
callbacks=callbacks)
score = M.evaluate(X_test, y_test, verbose=verb)
print('\nTest score/loss:', score[0], '\nTest accuracy:', score[1])
timing(starttime)
fintime = time_fin(starttime)
if full_output:
return M, hist.history, score, fintime
else:
return M
def train_4dnet(X,
Y,
test_size=0.1,
validation_split=0.1,
random_state=0,
layer_type=('conv3d', 'conv3d'),
nconvlayers=2,
conv_nfilters=(40, 80),
kernel_sizes=((3, 3, 3), (2, 2, 2)),
conv_strides=((1, 1, 1), (1, 1, 1)),
conv_padding='same',
dilation_rate=((1, 1, 1), (1, 1, 1)),
pool_layers=2,
pool_func='ave',
pool_sizes=((2, 2, 2), (2, 2, 2)),
pool_strides=((2, 2, 2), (2, 2, 2)),
dense_units=128,
rec_hidden_states=64,
activation='relu',
learnrate=0.003,
batchsize=64,
epochs=20,
patience=2,
min_delta=0.01,
retrain=None,
verb=1,
summary=True,
gpu_id='0',
plot='tb',
tb_path='./logs',
full_output=False):
"""
"""
clear_session()
graph = get_default_graph()
config = tf.ConfigProto()
# Don't pre-allocate memory; allocate as-needed
config.gpu_options.allow_growth = True
# Only allow a total of half the GPU memory to be allocated
# config.gpu_options.per_process_gpu_memory_fraction = 0.5
config.gpu_options.visible_device_list = gpu_id
session = tf.Session(graph=graph, config=config)
set_session(session)
ngpus = len(gpu_id.split(','))
batchsize *= ngpus
if not nconvlayers == len(conv_nfilters):
raise ValueError('`conv_nfilters` has a wrong length')
if not nconvlayers == len(kernel_sizes):
raise ValueError('`kernel_sizes` has a wrong length')
if not nconvlayers == len(conv_strides):
raise ValueError('`conv_strides` has a wrong length')
if not nconvlayers == len(dilation_rate):
raise ValueError('`dilation_rate` has a wrong length')
if pool_layers > 0:
if not pool_layers == len(pool_sizes):
raise ValueError('`pool_sizes` has a wrong length')
if pool_strides is not None:
if not pool_layers == len(pool_strides):
raise ValueError('`pool_strides` has a wrong length')
else:
pool_strides = [None] * pool_layers
if isinstance(layer_type, str):
layer_type = [layer_type for _ in range(nconvlayers)]
starttime = time_ini()
# Mixed train/test sets with Sklearn split
res = train_test_split(X,
Y,
test_size=test_size,
random_state=random_state)
X_train, X_test, y_train, y_test = res
msg = 'Zeros in train: {} | Ones in train: {}'
print(msg.format(y_train.tolist().count(0), y_train.tolist().count(1)))
msg = 'Zeros in test: {} | Ones in test: {}'
print(msg.format(y_test.tolist().count(0), y_test.tolist().count(1)))
# adding the "channels" dimension (1)
X_train = X_train.reshape(X_train.shape[0], X_train.shape[1],
X_train.shape[2], X_train.shape[3],
X_train.shape[4], 1)
X_test = X_test.reshape(X_test.shape[0], X_test.shape[1], X_test.shape[2],
X_test.shape[3], X_train.shape[4], 1)
print("\nShapes of train and test:")
print(X_train.shape, y_train.shape, X_test.shape, y_test.shape, '\n')
# --------------------------------------------------------------------------
kernel_init = 'glorot_uniform'
bias_init = 'random_normal'
rec_act = 'hard_sigmoid'
rec_init = 'orthogonal'
temp_dim = X_train.shape[1]
k_dim = X_train.shape[2]
patch_size = X_train.shape[3]
input_shape_3d = (temp_dim, patch_size, patch_size, 1)
if pool_func == 'ave':
pooling_func = AveragePooling3D
elif pool_func == 'max':
pooling_func = MaxPooling3D
# --------------------------------------------------------------------------
# Per branch model
# --------------------------------------------------------------------------
input_layer = Input(shape=input_shape_3d,
name='input_layer',
dtype='float32')
for i in range(nconvlayers):
# Stack of Conv3d, (B)CLSTM, (B)LRCN or (B)GRCN layers
if layer_type[i] in ('conv3d', 'clstm', 'bclstm'):
if pool_func == 'ave':
pooling_func = AveragePooling3D
elif pool_func == 'max':
pooling_func = MaxPooling3D
elif layer_type[i] in ('lrcn', 'blrcn', 'grcn', 'bgrcn'):
if pool_func == 'ave':
pooling_func = AveragePooling2D
elif pool_func == 'max':
pooling_func = MaxPooling2D
else:
raise ValueError('pool_func is not recognized')
if layer_type[i] == 'conv3d':
if not len(kernel_sizes[i]) == 3:
raise ValueError(
'Kernel sizes for Conv3d are tuples of 3 values')
if not len(conv_strides[i]) == 3:
raise ValueError('Strides for Conv3d are tuples of 3 values')
if not len(dilation_rate[i]) == 3:
raise ValueError('Dilation for Conv3d is a tuple of 3 values')
if i == 0:
x = Conv3D(filters=conv_nfilters[i],
kernel_size=kernel_sizes[i],
strides=conv_strides[i],
padding=conv_padding,
kernel_initializer=kernel_init,
bias_initializer=bias_init,
name='conv3d_layer1',
dilation_rate=dilation_rate[i],
data_format='channels_last',
input_shape=input_shape_3d)(input_layer)
x = SpatialDropout3D(0.5)(x)
x = Activation(activation, name='activ_layer1')(x)
x = pooling_func(pool_size=pool_sizes[i],
strides=pool_strides[i],
padding='valid')(x)
else:
x = Conv3D(filters=conv_nfilters[i],
kernel_size=kernel_sizes[i],
strides=conv_strides[i],
padding=conv_padding,
kernel_initializer=kernel_init,
bias_initializer=bias_init,
name='conv3d_layer' + str(i + 1),
dilation_rate=dilation_rate[i])(x)
x = SpatialDropout3D(0.25)(x)
x = Activation(activation, name='activ_layer' + str(i + 1))(x)
x = pooling_func(pool_size=pool_sizes[i],
strides=pool_strides[i],
padding='valid')(x)
elif layer_type[i] == 'clstm':
msg = 'are tuples of 2 integers'
if not len(kernel_sizes[0]) == 2:
raise ValueError('Kernel sizes for ConvLSTM' + msg)
if not len(conv_strides[0]) == 2:
raise ValueError('Strides for ConvLSTM')
if not len(dilation_rate[0]) == 2:
raise ValueError('Dilation rates for ConvLSTM')
if i == 0:
x = ConvLSTM2D(filters=conv_nfilters[i],
kernel_size=kernel_sizes[i],
strides=conv_strides[i],
padding=conv_padding,
kernel_initializer=kernel_init,
input_shape=input_shape_3d,
name='convlstm_layer1',
return_sequences=True,
dilation_rate=dilation_rate[i],
activation='tanh',
recurrent_activation=rec_act,
use_bias=True,
recurrent_initializer=rec_init,
bias_initializer='zeros',
unit_forget_bias=True,
dropout=0.0,
recurrent_dropout=0.0)(input_layer)
x = SpatialDropout3D(0.5)(x)
x = pooling_func(pool_size=pool_sizes[i],
strides=pool_strides[i],
padding='valid')(x)
else:
x = ConvLSTM2D(filters=conv_nfilters[i],
kernel_size=kernel_sizes[i],
strides=conv_strides[i],
padding=conv_padding,
kernel_initializer=kernel_init,
name='convlstm_layer' + str(i + 1),
return_sequences=True,
dilation_rate=dilation_rate[i],
activation='tanh',
recurrent_activation=rec_act,
use_bias=True,
recurrent_initializer=rec_init,
bias_initializer='zeros',
unit_forget_bias=True,
dropout=0.0,
recurrent_dropout=0.0)(x)
x = SpatialDropout3D(0.25)(x)
x = pooling_func(pool_size=pool_sizes[1],
strides=pool_strides[1],
padding='valid')(x)
elif layer_type[i] in ('lrcn', 'blrcn', 'grcn', 'bgrcn'):
if not len(kernel_sizes[i]) == 2:
raise ValueError(
'Kernel sizes for LRCN are tuples of 2 values')
if not len(conv_strides[i]) == 2:
raise ValueError('Strides for LRCN are tuples of 2 values')
if not len(dilation_rate[i]) == 2:
raise ValueError('Dilation for LRCN is a tuple of 2 values')
# TimeDistributed wrapper applies a layer to every temporal
# slice of an input. The input should be at least 3D and the
# dimension of index one will be considered to be the temporal
# dimension.
if i == 0:
x = TimeDistributed(Conv2D(filters=conv_nfilters[i],
kernel_size=kernel_sizes[i],
strides=conv_strides[i],
padding=conv_padding,
name='lrcn_layer1',
activation=activation),
input_shape=input_shape_3d)(input_layer)
# This version performs the same function as Dropout, however it
# drops entire 2D feature maps instead of individual elements.
x = TimeDistributed(SpatialDropout2D(0.5))(x)
x = TimeDistributed(
pooling_func(pool_size=pool_sizes[i],
strides=pool_strides[i],
padding='valid'))(x)
else:
x = TimeDistributed(
Conv2D(filters=conv_nfilters[1],
kernel_size=kernel_sizes[1],
strides=conv_strides[1],
padding=conv_padding,
name='lrcn_layer' + str(i + 1),
activation=activation))(x)
x = TimeDistributed(SpatialDropout2D(0.25))(x)
x = TimeDistributed(
pooling_func(pool_size=pool_sizes[1],
strides=pool_strides[1],
padding='valid'))(x)
# Final layer
if layer_type[-1] in ('conv3d', 'clstm'):
flatten_layer = Flatten(name='flatten')(x)
# Fully-connected or dense layer
output = Dense(units=dense_units, name='dense_layer')(flatten_layer)
output = Activation(activation, name='activ_dense')(output)
output = Dropout(rate=0.5, name='dropout_dense')(output)
elif layer_type[-1] in ('lrcn', 'blrcn', 'grcn', 'bgrcn'):
output = TimeDistributed(Flatten(name='flatten'))(x)
output = Dropout(0.5, name='dropout_flatten')(output)
model_branch = KerasModel(inputs=input_layer, outputs=output)
# --------------------------------------------------------------------------
# --------------------------------------------------------------------------
# Multi-input model
# --------------------------------------------------------------------------
inputs = []
outputs = []
for i in range(k_dim):
input_ = Input(shape=input_shape_3d,
name='input_' + str(i + 1),
dtype='float32')
output_ = model_branch(input_)
inputs.append(input_)
outputs.append(output_)
# Concatenating the branches. Shape [samples, time steps, features*k_dim]
concatenated = concatenate(outputs)
if layer_type[1] == 'lrcn':
lstm = CuDNNLSTM(rec_hidden_states,
kernel_initializer=kernel_init,
return_sequences=False)(concatenated)
concatenated = Dropout(0.5, name='dropout_lstm')(lstm)
elif layer_type[1] == 'blrcn':
blstm = Bidirectional(
LSTM(
rec_hidden_states, # TODO: bug CuDNNLSTM?
kernel_initializer=kernel_init,
return_sequences=False))(concatenated)
concatenated = Dropout(0.5, name='dropout_lstm')(blstm)
elif layer_type[1] == 'grcn':
gru = CuDNNGRU(rec_hidden_states,
kernel_initializer=kernel_init,
return_sequences=False)(concatenated)
concatenated = Dropout(0.5, name='dropout_gru')(gru)
elif layer_type[1] == 'bgrcn':
bgru = Bidirectional(
CuDNNGRU(rec_hidden_states,
kernel_initializer=kernel_init,
return_sequences=False))(concatenated)
concatenated = Dropout(0.5, name='dropout_gru')(bgru)
# Sigmoid unit
prediction = Dense(units=1,
name='sigmoid_output_unit',
activation='sigmoid')(concatenated)
model_final = KerasModel(inputs=inputs, outputs=prediction)
# --------------------------------------------------------------------------
if summary:
model_branch.summary()
# model_final.summary()
# Callbacks
early_stopping = EarlyStopping(monitor='val_loss',
patience=patience,
min_delta=min_delta,
verbose=verb)
if plot is not None:
if plot == 'tb':
tensorboard = TensorBoard(log_dir=tb_path,
histogram_freq=1,
write_graph=True,
write_images=True)
callbacks = [early_stopping, tensorboard]
elif plot == 'llp':
plotlosses = livelossplot.PlotLossesKeras()
callbacks = [early_stopping, plotlosses]
else:
raise ValueError("`plot` method not recognized")
else:
callbacks = [early_stopping]
X_train = list(np.moveaxis(X_train, 2, 0))
X_test = list(np.moveaxis(X_test, 2, 0))
# Training the network
if ngpus > 1:
# Multi-GPUs
Mpar = multi_gpu_model(model_final, gpus=ngpus)
# Training the network
Mpar.compile(loss='binary_crossentropy',
metrics=['accuracy'],
optimizer=Adam(lr=learnrate, decay=1e-2))
hist = Mpar.fit(X_train,
y_train,
batch_size=batchsize,
shuffle=True,
epochs=epochs,
initial_epoch=0,
verbose=verb,
validation_split=validation_split,
callbacks=callbacks)
score = Mpar.evaluate(X_test, y_test, verbose=verb)
else:
# Single GPU
model_final.compile(loss='binary_crossentropy',
metrics=['accuracy'],
optimizer=Adam(lr=learnrate, decay=1e-2))
hist = model_final.fit(X_train,
y_train,
batch_size=batchsize,
epochs=epochs,
initial_epoch=0,
verbose=verb,
validation_split=validation_split,
callbacks=callbacks,
shuffle=True)
score = model_final.evaluate(X_test, y_test, verbose=verb)
print('\nTest score/loss:', score[0], '\nTest accuracy:', score[1])
timing(starttime)
fintime = time_fin(starttime)
if full_output:
return model_final, hist.history, score, fintime
else:
return model_final
def run(self, dpi=100):
""" Obtaining the flux vs S/N relationship.
dpi : int, optional
DPI of the figures.
"""
if not self.sampled_fluxes or not self.sampled_snrs:
self.sampling()
nsubplots = len(self.distances)
ncols = min(4, nsubplots)
if nsubplots > 1 and nsubplots % 2 != 0:
nsubplots -= 1
if nsubplots < 3:
figsize = (10, 2)
if nsubplots == 2:
figsizex = figsize[0] * 0.66
elif nsubplots == 1:
figsizex = figsize[0] * 0.33
nrows = 1
else:
if nsubplots <= 8:
figsize = (10, 4)
else:
figsize = (10, 6)
figsizex = figsize[0]
nrows = int(nsubplots / ncols) + 1
fig, axs = plt.subplots(nrows,
ncols,
figsize=(figsizex, figsize[1]),
dpi=dpi,
sharey='row')
fig.subplots_adjust(wspace=0.05, hspace=0.3)
if isinstance(axs, np.ndarray):
axs = axs.ravel()
fhi = list()
flo = list()
print("Interpolating the Flux vs S/N function")
# Regression for each distance
for i, d in enumerate(self.distances):
plotvlines = [self.min_snr[i], self.max_snr[i]]
if isinstance(axs, np.ndarray):
axis = axs[i]
else:
axis = axs
fluxes = np.array(self.sampled_fluxes[i])
snrs = np.array(self.sampled_snrs[i])
mask = np.where(snrs > 0.1)
snrs = snrs[mask]
fluxes = fluxes[mask]
f = interp1d(np.sort(snrs),
np.sort(fluxes),
kind='slinear',
fill_value='extrapolate')
minsnr = max(self.min_snr[i], min(snrs))
maxsnr = min(self.max_snr[i], max(snrs))
snrs_pred = np.linspace(minsnr, maxsnr, num=50)
fluxes_pred = f(snrs_pred)
flux_for_lowsnr = f(minsnr)
flux_for_higsnr = f(maxsnr)
fhi.append(flux_for_higsnr)
flo.append(flux_for_lowsnr)
# Figure of flux vs s/n
axis.xaxis.set_tick_params(labelsize=6)
axis.yaxis.set_tick_params(labelsize=6)
axis.plot(fluxes, snrs, '.', alpha=0.2, markersize=4)
axis.plot(fluxes_pred, snrs_pred, '-', alpha=1, color='orangered')
axis.grid(which='major', alpha=0.3)
axis.set_xlim(0)
for l in plotvlines:
axis.plot((0, max(fluxes)), (l, l), ':', color='darksalmon')
axis = fig.add_subplot(111, frame_on=False)
axis.set_xticks([])
axis.set_yticks([])
axis.set_xlabel('Fakecomp flux scaling', labelpad=25, size=8)
axis.set_ylabel('Signal to noise ratio', labelpad=25, size=8)
if isinstance(axs, np.ndarray):
for i in range(len(self.distances), len(axs)):
axs[i].axis('off')
plt.show()
flo = np.array(flo).flatten()
fhi = np.array(fhi).flatten()
if self.inter_extrap and len(self.distances) > 2:
x = self.distances
f1 = interpolate.interp1d(x, flo, fill_value='extrapolate')
f2 = interpolate.interp1d(x, fhi, fill_value='extrapolate')
fhi = f2(self.inter_extrap_dist)
flo = f1(self.inter_extrap_dist)
plot_x = self.inter_extrap_dist
else:
plot_x = self.distances
self.estimated_fluxes_high = fhi
self.estimated_fluxes_low = flo
# figure with fluxes as a function of the separation
if len(self.distances) > 1 and isinstance(self.min_snr, (float, int)) \
and isinstance(self.max_snr, (float, int)):
plt.figure(figsize=(10, 4), dpi=dpi)
plt.plot(self.distances,
self.radprof,
'--',
alpha=0.8,
color='gray',
lw=2,
label='average radial profile')
plt.plot(plot_x,
flo,
'.-',
alpha=0.6,
lw=2,
color='dodgerblue',
label='flux lower bound')
plt.plot(plot_x,
fhi,
'.-',
alpha=0.6,
color='dodgerblue',
lw=2,
label='flux upper bound')
plt.fill_between(plot_x,
flo,
fhi,
where=flo <= fhi,
alpha=0.2,
facecolor='dodgerblue',
interpolate=True)
plt.grid(which='major', alpha=0.4)
plt.xlabel('Distance from the center [Pixels]')
plt.ylabel('Fakecomp flux scaling [Counts]')
plt.minorticks_on()
plt.xlim(0)
plt.ylim(0)
plt.legend()
plt.show()
timing(self.starttime)
def predict_pairwise(model,
cube,
angle_list,
fwhm,
patch_size_px,
delta_rot,
radius_int=None,
high_pass='******',
kernel_size=5,
normalization='slice',
imlib='opencv',
interpolation='bilinear',
nproc=1,
verbose=True,
chunks_per_proc=2):
"""
Parameters
----------
model : Keras model
cube : 3d ndarray
angle_list : 1d ndarray
fwhm : float
patch_size : int, optional
delta_rot : float, optional
verbose: bool or int, optional
0 / False: no output
1 / True: full output (timing + progress bar)
2: progress bar only
[...]
Returns
-------
probmap : 2d ndarray
Notes
-----
- support for 4D cubes?
"""
starttime = time_ini(verbose=verbose)
if radius_int is None:
radius_int = fwhm
n_frames, sizey, sizex = cube.shape
width = int(sizey / 2 - patch_size_px / 2 - radius_int)
ind = get_annulus_segments(cube[0],
inner_radius=radius_int,
width=width,
nsegm=1)
probmap = np.zeros((sizey, sizex))
if high_pass is not None:
cube = cube_filter_highpass(cube,
high_pass,
kernel_size=kernel_size,
verbose=False)
indices = list(range(ind[0][0].shape[0]))
if nproc is None:
nproc = cpu_count() // 2 # Hyper-threading doubles the # of cores
# prepare patches in parallel
nchunks = nproc * chunks_per_proc
print("Grabbing patches with {} processes".format(nproc))
res_ = list(
Progressbar(pool_imap(nproc, _parallel_make_patches_chunk,
iterable(make_chunks(indices, nchunks)), ind,
cube, angle_list, fwhm, patch_size_px, delta_rot,
normalization, imlib, interpolation),
total=nchunks,
leave=False,
verbose=False))
xx = []
yy = []
pats = []
for r in res_:
x, y, pp = r
for i in range(len(x)):
xx.append(x[i])
yy.append(y[i])
pats.append(pp[i])
if verbose == 1:
timing(starttime)
print("Prediction on patches:")
probas = predict_from_patches(model, pats, len(xx))
for i in range(len(xx)):
probmap[xx[i], yy[i]] = probas[i]
if verbose == 1:
timing(starttime)
return probmap
def estimate_fluxes(cube,
psf,
distances,
angles,
fwhm,
plsc,
wavelengths=None,
n_injections=10,
mode='median',
ncomp=2,
min_adi_snr=2,
max_adi_snr=5,
random_seed=42,
kernel='rbf',
epsilon=0.1,
c=1e4,
gamma=1e-2,
figsize=(10, 5),
dpi=100,
n_proc=2,
**kwargs):
"""
Automatic estimation of the scaling factors for injecting the
companions (brightness or contrast of fake companions).
Epsilon-Support Vector Regression (important parameters in the model are
C and epsilon). The implementation is based on scikit-learn (libsvm).
Parameters
----------
C : float, optional (default=1.0)
Penalty parameter C of the error term.
epsilon : float, optional (default=0.1)
Epsilon in the epsilon-SVR model. It specifies the epsilon-tube
within which no penalty is associated in the training loss function
with points predicted within a distance epsilon from the actual
value.
kernel : string, optional (default=’rbf’)
Specifies the kernel type to be used in the algorithm. It must be
one of ‘linear’, ‘poly’, ‘rbf’, ‘sigmoid’, ‘precomputed’ or a
callable. If none is given, ‘rbf’ will be used. If a callable is
given it is used to precompute the kernel matrix.
gamma : float, optional (default=’auto’)
Kernel coefficient for ‘rbf’, ‘poly’ and ‘sigmoid’. If gamma is
‘auto’ then 1/n_features will be used instead.
"""
starttime = time_ini()
global GARRAY
GARRAY = cube
global GARRPSF
GARRPSF = psf
global GARRPA
GARRPA = angles
global GARRWL
GARRWL = wavelengths
if cube.ndim == 4:
if wavelengths is None:
raise ValueError('`wavelengths` parameter must be provided')
# Getting the radial profile in the mean frame of the cube
sampling_sep = 1
radius_int = 1
if cube.ndim == 3:
global_frame = np.mean(cube, axis=0)
elif cube.ndim == 4:
global_frame = np.mean(cube.reshape(-1, cube.shape[2], cube.shape[3]),
axis=0)
me = frame_average_radprofile(global_frame,
sep=sampling_sep,
init_rad=radius_int,
plot=False)
radprof = np.array(me.radprof)
radprof = radprof[np.array(distances) + 1]
flux_min = radprof * 0.1
flux_min[flux_min < 0] = 0.1
# Multiprocessing pool
flux_max = pool_map(n_proc, _get_max_flux, fixed(range(len(distances))),
distances, radprof, fwhm, plsc, max_adi_snr,
wavelengths)
flux_max = np.array(flux_max)
fluxes_list, snrs_list = _sample_flux_snr(distances, fwhm, plsc,
n_injections, flux_min, flux_max,
n_proc, random_seed, wavelengths,
mode, ncomp)
plotvlines = [min_adi_snr, max_adi_snr]
nsubplots = len(distances)
if nsubplots % 2 != 0:
nsubplots -= 1
ncols = 4
nrows = int(nsubplots / ncols) + 1
fig, axs = plt.subplots(nrows,
ncols,
figsize=figsize,
dpi=dpi,
sharey='row')
fig.subplots_adjust(wspace=0.05, hspace=0.3)
axs = axs.ravel()
fhi = list()
flo = list()
# Regression for each distance
for i, d in enumerate(distances):
fluxes = np.array(fluxes_list[i])
snrs = np.array(snrs_list[i])
mask = np.where(snrs > 0.1)
snrs = snrs[mask].reshape(-1, 1)
fluxes = fluxes[mask].reshape(-1, 1)
model = SVR(kernel=kernel, epsilon=epsilon, C=c, gamma=gamma, **kwargs)
model.fit(X=snrs, y=fluxes)
flux_for_lowsnr = model.predict(min_adi_snr)
flux_for_higsnr = model.predict(max_adi_snr)
fhi.append(flux_for_higsnr[0])
flo.append(flux_for_lowsnr[0])
snrminp = min_adi_snr / 2
snrs_pred = np.linspace(snrminp, max_adi_snr + snrminp,
num=50).reshape(-1, 1)
fluxes_pred = model.predict(snrs_pred)
# Figure of flux vs s/n
axs[i].xaxis.set_tick_params(labelsize=6)
axs[i].yaxis.set_tick_params(labelsize=6)
axs[i].plot(fluxes, snrs, '.', alpha=0.2, markersize=4)
axs[i].plot(fluxes_pred,
snrs_pred,
'-',
alpha=0.99,
label='S/N regression model',
color='orangered')
axs[i].grid(which='major', alpha=0.3)
axs[i].legend(fontsize=6)
for l in plotvlines:
axs[i].plot((0, max(fluxes)), (l, l), ':', color='darksalmon')
ax0 = fig.add_subplot(111, frame_on=False)
ax0.set_xticks([])
ax0.set_yticks([])
ax0.set_xlabel('Fakecomp flux scaling [Counts]', labelpad=25, size=8)
ax0.set_ylabel('ADI-medsub median S/N (3 equidist. angles)',
labelpad=25,
size=8)
for i in range(len(distances), len(axs)):
axs[i].axis('off')
timing(starttime)
flo = np.array(flo).flatten()
fhi = np.array(fhi).flatten()
# x = distances
# f1 = interpolate.interp1d(x, flo, fill_value='extrapolate')
# f2 = interpolate.interp1d(x, fhi, fill_value='extrapolate')
# fhi = f2(distances_init)
# flo = f1(distances_init)
plt.figure(figsize=(10, 4), dpi=dpi)
plt.plot(distances,
radprof,
'--',
alpha=0.8,
color='gray',
lw=2,
label='average radial profile')
plt.plot(distances,
flo,
'.-',
alpha=0.6,
lw=2,
color='dodgerblue',
label='flux lower interval')
plt.plot(distances,
fhi,
'.-',
alpha=0.6,
color='dodgerblue',
lw=2,
label='flux upper interval')
plt.fill_between(distances,
flo,
fhi,
where=flo <= fhi,
alpha=0.2,
facecolor='dodgerblue',
interpolate=True)
plt.grid(which='major', alpha=0.4)
plt.xlabel('Distance from the center [Pixels]')
plt.ylabel('Fakecomp flux scaling [Counts]')
plt.minorticks_on()
plt.xlim(0)
plt.ylim(0)
plt.legend()
plt.show()
return flo, fhi
def train_2dconvnet(X,
Y,
test_size=0.1,
validation_split=0.1,
random_state=0,
pseudo3d=False,
nconvlayers=2,
conv_nfilters=(40, 80),
kernel_sizes=((3, 3), (3, 3)),
conv_strides=((1, 1), (1, 1)),
pool_layers=2,
pool_sizes=((2, 2), (2, 2)),
pool_strides=((2, 2), (2, 2)),
dense_units=128,
activation='relu',
learnrate=0.003,
batchsize=64,
epochs=20,
patience=2,
min_delta=0.01,
retrain=None,
verb=1,
summary=True,
gpu_id='0',
full_output=False,
plot='tb',
tb_path='./logs'):
""" 2D Convolutional network for pairwise subtracted patches.
Parameters
----------
...
Notes
-----
Multi-GPU with Keras:
https://keras.io/utils/#multi_gpu_model
"""
clear_session()
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
config.gpu_options.visible_device_list = gpu_id
set_session(tf.Session(config=config))
ngpus = len(gpu_id.split(','))
if not nconvlayers == len(conv_nfilters):
raise ValueError('`conv_nfilters` has a wrong length')
if not nconvlayers == len(kernel_sizes):
raise ValueError('`kernel_sizes` has a wrong length')
if not nconvlayers == len(conv_strides):
raise ValueError('`conv_strides` has a wrong length')
if pool_layers > 0:
if not pool_layers == len(pool_sizes):
raise ValueError('`pool_sizes` has a wrong length')
if pool_strides is not None:
if not pool_layers == len(pool_strides):
raise ValueError('`pool_strides` has a wrong length')
else:
pool_strides = [None] * pool_layers
if pseudo3d:
if not X.ndim == 4:
raise ValueError('X must contain 4D samples')
starttime = time_ini()
patch_size = X.shape[-1]
# Mixed train/test sets with Sklearn split
resplit = train_test_split(X,
Y,
test_size=test_size,
random_state=random_state)
X_train, X_test, y_train, y_test = resplit
msg = 'Zeros in train: {} | Ones in train: {}'
print(msg.format(y_train.tolist().count(0), y_train.tolist().count(1)))
msg = 'Zeros in test: {} | Ones in test: {}'
print(msg.format(y_test.tolist().count(0), y_test.tolist().count(1)))
if pseudo3d:
if not X.ndim == 4:
raise ValueError('`X` has wrong number of dimensions')
# moving the temporal dimension to the channels dimension (last)
X_train = np.moveaxis(X_train, 1, -1)
X_test = np.moveaxis(X_test, 1, -1)
input_shape = (patch_size, patch_size, X_train.shape[-1])
else:
# adding the channels dimension
X_train = X_train.reshape(X_train.shape[0], patch_size, patch_size, 1)
X_test = X_test.reshape(X_test.shape[0], patch_size, patch_size, 1)
input_shape = (patch_size, patch_size, 1)
print("\nShapes of train and test sets:")
print(X_train.shape, y_train.shape, X_test.shape, y_test.shape, '\n')
# --------------------------------------------------------------------------
if retrain is not None:
M = retrain
# re-training the network
M.compile(loss='binary_crossentropy',
metrics=['accuracy'],
optimizer=Adam(lr=learnrate, decay=1e-2))
early_stopping = EarlyStopping(monitor='val_loss',
patience=patience,
min_delta=min_delta,
verbose=verb)
hist = M.fit(X_train,
y_train,
batch_size=batchsize,
epochs=epochs,
initial_epoch=0,
verbose=verb,
validation_split=0.1,
callbacks=[early_stopping],
shuffle=True)
score = M.evaluate(X_test, y_test, verbose=verb)
print('\nTest score/loss:', score[0], '\n', 'Test accuracy:', score[1])
timing(starttime)
fintime = time_fin(starttime)
if full_output:
return M, hist.history, score, fintime
else:
return M
# --------------------------------------------------------------------------
# Creating the NN model
if ngpus > 1:
with tf.device('/cpu:0'):
M = Sequential()
else:
M = Sequential()
# Stack of 2d convolutional layers
kernel_init = 'glorot_uniform'
bias_init = 'random_normal'
for i in range(nconvlayers):
if i == 0:
M.add(
Conv2D(filters=conv_nfilters[i],
kernel_size=kernel_sizes[i],
strides=conv_strides[i],
padding='same',
kernel_initializer=kernel_init,
bias_initializer=bias_init,
name='conv2d_layer1',
data_format='channels_last',
input_shape=input_shape))
else:
M.add(
Conv2D(filters=conv_nfilters[i],
kernel_size=kernel_sizes[i],
strides=conv_strides[i],
padding='same',
kernel_initializer=kernel_init,
bias_initializer=bias_init,
name='conv2d_layer' + str(i + 1)))
M.add(Activation(activation, name='activ_layer' + str(i + 1)))
if pool_layers != 0:
M.add(
MaxPooling2D(pool_size=pool_sizes[i],
strides=pool_strides[i],
padding='valid'))
pool_layers -= 1
M.add(Dropout(rate=0.25, name='dropout_layer' + str(i + 1)))
M.add(Flatten(name='flatten'))
# Dense or fully-connected layer
M.add(Dense(units=dense_units, name='dense_128units'))
M.add(Activation(activation, name='activ_dense'))
# M.add(BatchNormalization())
M.add(Dropout(rate=0.5, name='dropout_dense'))
M.add(Dense(units=1, name='dense_1unit'))
M.add(Activation('sigmoid', name='activ_out'))
if summary:
M.summary()
# Callbacks
early_stopping = EarlyStopping(monitor='val_loss',
patience=patience,
min_delta=min_delta,
verbose=verb)
if plot is not None:
if plot == 'tb':
tensorboard = TensorBoard(log_dir=tb_path,
histogram_freq=1,
write_graph=True,
write_images=True)
callbacks = [early_stopping, tensorboard]
elif plot == 'llp':
plotlosses = livelossplot.PlotLossesKeras()
callbacks = [early_stopping, plotlosses]
else:
raise ValueError("`plot` method not recognized")
else:
callbacks = [early_stopping]
# Multi-GPUs
if ngpus > 1:
Mpar = multi_gpu_model(M, gpus=ngpus)
# Training the network
Mpar.compile(loss='binary_crossentropy',
metrics=['accuracy'],
optimizer=Adam(lr=learnrate, decay=1e-2))
hist = Mpar.fit(X_train,
y_train,
batch_size=batchsize * ngpus,
epochs=epochs,
initial_epoch=0,
verbose=verb,
validation_split=validation_split,
callbacks=callbacks,
shuffle=True)
score = Mpar.evaluate(X_test, y_test, verbose=verb)
else:
# Training the network
M.compile(loss='binary_crossentropy',
metrics=['accuracy'],
optimizer=Adam(lr=learnrate, decay=1e-2))
hist = M.fit(X_train,
y_train,
batch_size=batchsize * ngpus,
epochs=epochs,
initial_epoch=0,
verbose=verb,
validation_split=validation_split,
callbacks=callbacks,
shuffle=True)
score = M.evaluate(X_test, y_test, verbose=verb)
print('\nTest score/loss:', score[0], '\n', 'Test accuracy:', score[1])
timing(starttime)
fintime = time_fin(starttime)
if full_output:
return M, hist.history, score, fintime
else:
return M
def correct_bad_pixels(self, verbose=True, debug=False):
sci_list = []
with open(self.outpath + "sci_list.txt", "r") as f:
tmp = f.readlines()
for line in tmp:
sci_list.append(line.split('\n')[0])
sky_list = []
with open(self.outpath + "sky_list.txt", "r") as f:
tmp = f.readlines()
for line in tmp:
sky_list.append(line.split('\n')[0])
unsat_list = []
with open(self.outpath + "unsat_list.txt", "r") as f:
tmp = f.readlines()
for line in tmp:
unsat_list.append(line.split('\n')[0])
n_sci = len(sci_list)
ndit_sci = fits_info.ndit_sci
n_sky = len(sky_list)
ndit_sky = fits_info.ndit_sky
tmp = open_fits(self.outpath + '1_crop_unsat_' + unsat_list[-1],
header=False)
nx_unsat_crop = tmp.shape[2]
master_flat_frame = open_fits(self.outpath + 'master_flat_field.fits')
# Create bpix map
bpix = np.where(np.abs(master_flat_frame - 1.09) >
0.41) # i.e. for QE < 0.68 and QE > 1.5
bpix_map = np.zeros([self.com_sz, self.com_sz])
bpix_map[bpix] = 1
if nx_unsat_crop < bpix_map.shape[1]:
bpix_map_unsat = frame_crop(bpix_map, nx_unsat_crop)
else:
bpix_map_unsat = bpix_map
#number of bad pixels
nbpix = int(np.sum(bpix_map))
ntotpix = self.com_sz**2
print("total number of bpix: ", nbpix)
print("total number of pixels: ", ntotpix)
print("=> {}% of bad pixels.".format(100 * nbpix / ntotpix))
write_fits(self.outpath + 'master_bpix_map.fits', bpix_map)
write_fits(self.outpath + 'master_bpix_map_unsat.fits', bpix_map_unsat)
plot_frames(bpix_map, bpix_map_unsat)
#update final crop size
final_sz = self.get_final_sz()
#crop frames to that size
for sc, fits_name in enumerate(sci_list):
tmp = open_fits(self.outpath + '2_nan_corr_' + fits_name,
verbose=False)
tmp_tmp = cube_crop_frames(tmp, final_sz, xy=self.agpm_pos)
write_fits(self.outpath + '2_crop_' + fits_name, tmp_tmp)
if not debug:
os.system("rm " + self.outpath + '2_nan_corr_' + fits_name)
for sk, fits_name in enumerate(sky_list):
tmp = open_fits(self.outpath + '2_nan_corr_' + fits_name,
verbose=False)
tmp_tmp = cube_crop_frames(tmp, final_sz, xy=self.agpm_pos)
write_fits(self.outpath + '2_crop_' + fits_name, tmp_tmp)
if not debug:
os.system("rm " + self.outpath + '2_nan_corr_' + fits_name)
if not debug:
tmp = open_fits(self.outpath + '2_crop_' + sci_list[0])[-1]
tmp_tmp = open_fits(self.outpath + '2_crop_' + sci_list[-1])[-1]
plot_frames(tmp, tmp_tmp)
else:
# COMPARE BEFORE AND AFTER NAN_CORR + CROP
old_tmp = open_fits(self.outpath + '2_ff_' + sci_list[0])[-1]
old_tmp_tmp = open_fits(self.outpath + '2_ff_' + sci_list[-1])[-1]
tmp = open_fits(self.outpath + '2_crop_' + sci_list[0])[-1]
tmp_tmp = open_fits(self.outpath + '2_crop_' + sci_list[-1])[-1]
plot_frames(old_tmp, tmp, old_tmp_tmp, tmp_tmp)
# Crop the bpix map in a same way
bpix_map = open_fits(self.outpath + 'master_bpix_map.fits')
bpix_map_2ndcrop = frame_crop(bpix_map, final_sz, cenxy=self.agpm_pos)
write_fits(self.outpath + 'master_bpix_map_2ndcrop.fits',
bpix_map_2ndcrop)
bpix_map = open_fits(self.outpath + 'master_bpix_map_2ndcrop.fits')
t0 = time_ini()
for sc, fits_name in enumerate(sci_list):
tmp = open_fits(self.outpath + '2_crop_' + fits_name,
verbose=False)
# first with the bp max defined from the flat field (without protecting radius)
tmp_tmp = cube_fix_badpix_isolated(tmp,
bpm_mask=bpix_map,
sigma_clip=7,
num_neig=5,
size=5,
protect_mask=True,
radius=9,
verbose=False,
debug=False)
write_fits(self.outpath + '2_bpix_corr_' + fits_name,
tmp_tmp,
verbose=False)
timing(t0)
# second, residual hot pixels
tmp_tmp, bpm = cube_fix_badpix_isolated(tmp_tmp,
bpm_mask=None,
sigma_clip=8,
num_neig=5,
size=5,
protect_mask=True,
radius=10,
verbose=False,
debug=False,
full_output=True)
write_fits(self.outpath + '2_bpix_corr2_' + fits_name, tmp_tmp)
write_fits(self.outpath + '2_bpix_corr2_map_' + fits_name, bpm)
timing(t0)
if not debug:
os.system("rm " + self.outpath + '2_crop_' + fits_name)
bpix_map = open_fits(self.outpath + 'master_bpix_map_2ndcrop.fits')
t0 = time_ini()
for sk, fits_name in enumerate(sky_list):
tmp = open_fits(self.outpath + '2_crop_' + fits_name,
verbose=False)
# first with the bp max defined from the flat field (without protecting radius)
tmp_tmp = cube_fix_badpix_isolated(tmp,
bpm_mask=bpix_map,
sigma_clip=7,
num_neig=5,
size=5,
protect_mask=True,
radius=9,
verbose=False,
debug=False)
write_fits(self.outpath + '2_bpix_corr_' + fits_name,
tmp_tmp,
verbose=False)
timing(t0)
# second, residual hot pixels
tmp_tmp, bpm = cube_fix_badpix_isolated(tmp_tmp,
bpm_mask=None,
sigma_clip=8,
num_neig=5,
size=5,
protect_mask=True,
radius=10,
verbose=False,
debug=False,
full_output=True)
write_fits(self.outpath + '2_bpix_corr2_' + fits_name, tmp_tmp)
write_fits(self.outpath + '2_bpix_corr2_map_' + fits_name, bpm)
timing(t0)
if not debug:
os.system("rm " + self.outpath + '2_crop_' + fits_name)
bpix_map_unsat = open_fits(self.outpath + 'master_bpix_map_unsat.fits')
t0 = time_ini()
for un, fits_name in enumerate(unsat_list):
tmp = open_fits(self.outpath + '2_nan_corr_unsat_' + fits_name,
verbose=False)
# first with the bp max defined from the flat field (without protecting radius)
tmp_tmp = cube_fix_badpix_isolated(tmp,
bpm_mask=bpix_map_unsat,
sigma_clip=7,
num_neig=5,
size=5,
protect_mask=True,
radius=9,
verbose=False,
debug=False)
write_fits(self.outpath + '2_bpix_corr_unsat_' + fits_name,
tmp_tmp)
timing(t0)
# second, residual hot pixels
tmp_tmp, bpm = cube_fix_badpix_isolated(tmp_tmp,
bpm_mask=None,
sigma_clip=8,
num_neig=5,
size=5,
protect_mask=True,
radius=10,
verbose=False,
debug=False,
full_output=True)
write_fits(self.outpath + '2_bpix_corr2_unsat_' + fits_name,
tmp_tmp)
write_fits(self.outpath + '2_bpix_corr2_map_unsat_' + fits_name,
bpm)
timing(t0)
if not debug:
os.system("rm " + self.outpath + '2_nan_corr_unsat_' +
fits_name)
# FIRST CREATE MASTER CUBE FOR SCI
tmp_tmp_tmp = open_fits(self.outpath + '2_bpix_corr2_' + sci_list[0],
verbose=False)
n_y = tmp_tmp_tmp.shape[1]
n_x = tmp_tmp_tmp.shape[2]
tmp_tmp_tmp = np.zeros([n_sci, n_y, n_x])
for sc, fits_name in enumerate(sci_list):
tmp_tmp_tmp[sc] = open_fits(
self.outpath + '2_bpix_corr2_' + fits_name,
verbose=False)[int(random.randrange(min(ndit_sci)))]
tmp_tmp_tmp = np.median(tmp_tmp_tmp, axis=0)
write_fits(self.outpath + 'TMP_2_master_median_SCI.fits', tmp_tmp_tmp)
# THEN CREATE MASTER CUBE FOR SKY
tmp_tmp_tmp = open_fits(self.outpath + '2_bpix_corr2_' + sky_list[0],
verbose=False)
n_y = tmp_tmp_tmp.shape[1]
n_x = tmp_tmp_tmp.shape[2]
tmp_tmp_tmp = np.zeros([n_sky, n_y, n_x])
for sk, fits_name in enumerate(sky_list):
tmp_tmp_tmp[sk] = open_fits(
self.outpath + '2_bpix_corr2_' + fits_name,
verbose=False)[int(random.randrange(min(ndit_sky)))]
tmp_tmp_tmp = np.median(tmp_tmp_tmp, axis=0)
write_fits(self.outpath + 'TMP_2_master_median_SKY.fits', tmp_tmp_tmp)
bpix_map_ori = open_fits(self.outpath + 'master_bpix_map_2ndcrop.fits')
bpix_map_sci_0 = open_fits(self.outpath + '2_bpix_corr2_map_' +
sci_list[0])
bpix_map_sci_1 = open_fits(self.outpath + '2_bpix_corr2_map_' +
sci_list[-1])
bpix_map_sky_0 = open_fits(self.outpath + '2_bpix_corr2_map_' +
sky_list[0])
bpix_map_sky_1 = open_fits(self.outpath + '2_bpix_corr2_map_' +
sky_list[-1])
bpix_map_unsat_0 = open_fits(self.outpath + '2_bpix_corr2_map_unsat_' +
unsat_list[0])
bpix_map_unsat_1 = open_fits(self.outpath + '2_bpix_corr2_map_unsat_' +
unsat_list[-1])
plot_frames(
bpix_map_ori,
bpix_map_sci_0,
bpix_map_sci_1, #tmp_tmp_tmp, #bpix_tmp,
bpix_map_sky_0,
bpix_map_sky_1, #, #tmp_tmp_tmp_tmp, #bpix_tmp_tmp, tmpSCI-tmpSKY
bpix_map_unsat_0,
bpix_map_unsat_1 #, #tmp_tmp_tmp_tmp, #bpix_tmp_tmp, tmpSCI-tmpSKY
)
tmpSCI = open_fits(self.outpath + 'TMP_2_master_median_SCI.fits')
tmpSKY = open_fits(self.outpath + 'TMP_2_master_median_SKY.fits')
if not debug:
tmp_tmp = open_fits(self.outpath + '2_bpix_corr2_' +
sci_list[1])[-1]
tmp_tmp2 = open_fits(self.outpath + '2_bpix_corr2_' +
sci_list[-1])[-1]
plot_frames( #tmp, tmp-tmpSKY, #tmp_tmp_tmp, #bpix_tmp,
tmp_tmp,
tmp_tmp -
tmpSKY, #, #tmp_tmp_tmp_tmp, #bpix_tmp_tmp, tmpSCI-tmpSKY
#tmp2, tmp2-tmpSKY, #tmp_tmp_tmp, #bpix_tmp,
tmp_tmp2,
tmp_tmp2 -
tmpSKY #, #tmp_tmp_tmp_tmp, #bpix_tmp_tmp, tmpSCI-tmpSKY
)
else:
# COMPARE BEFORE AND AFTER BPIX CORR (without sky subtr)
tmp = open_fits(self.outpath + '2_crop_' + sci_list[-1])[-1]
tmp_tmp = open_fits(self.outpath + '2_bpix_corr2_' +
sci_list[-1])[-1]
tmp2 = open_fits(self.outpath + '2_crop_' + sky_list[-1])[-1]
tmp_tmp2 = open_fits(self.outpath + '2_bpix_corr2_' +
sky_list[-1])[-1]
(
tmp,
tmp - tmpSKY, #tmp_tmp_tmp, #bpix_tmp,
tmp_tmp,
tmp_tmp -
tmpSKY, #, #tmp_tmp_tmp_tmp, #bpix_tmp_tmp, tmpSCI-tmpSKY
tmp2,
tmp2 - tmpSKY, #tmp_tmp_tmp, #bpix_tmp,
tmp_tmp2,
tmp_tmp2 -
tmpSKY #, #tmp_tmp_tmp_tmp, #bpix_tmp_tmp, tmpSCI-tmpSKY
)
