def get_min_flux(self, debug=False): """ Obtaining the low end of the interval for sampling the S/N. Based on the initial estimation of the radial profile of the mean frame. """ # Getting the radial profile in the mean frame of the cube sampling_sep = 1 radius_int = 1 if GARRAY.ndim == 3: global_frame = np.mean(GARRAY, axis=0) elif GARRAY.ndim == 4: global_frame = np.mean(GARRAY.reshape(-1, GARRAY.shape[2], GARRAY.shape[3]), axis=0) me = frame_average_radprofile(global_frame, sep=sampling_sep, init_rad=radius_int, plot=False) radprof = np.array(me.radprof) radprof = radprof[np.array(self.distances) + 1] radprof[radprof < 0] = 0.01 self.radprof = radprof print( "Estimating the lower flux interval for sampling the S/N vs flux " "function") flux_min = pool_map(self.n_proc, _get_min_flux, iterable(self.n_dist), self.distances, radprof, self.fwhm, self.plsc, iterable(self.min_snr), self.wavelengths, self.spectrum, self.algo, self.scaling, self.svd_mode, self.random_seed, debug) self.min_fluxes = flux_min timing(self.starttime)
def _sample_flux_snr(distances, fwhm, plsc, n_injections, flux_min, flux_max, nproc=10, random_seed=42, wavelengths=None, mode='median', ncomp=2): """ Sensible flux intervals depend on a combination of factors, # of frames, range of rotation, correlation, glare intensity. """ starttime = time_ini() if GARRAY.ndim == 3: frsize = int(GARRAY.shape[1]) elif GARRAY.ndim == 4: frsize = int(GARRAY.shape[2]) ninj = n_injections random_state = np.random.RandomState(random_seed) flux_dist_theta_all = list() snrs_list = list() fluxes_list = list() for i, d in enumerate(distances): yy, xx = get_annulus_segments((frsize, frsize), d, 1, 1)[0] num_patches = yy.shape[0] fluxes_dist = random_state.uniform(flux_min[i], flux_max[i], size=ninj) inds_inj = random_state.randint(0, num_patches, size=ninj) for j in range(ninj): injx = xx[inds_inj[j]] injy = yy[inds_inj[j]] injx -= frame_center(GARRAY[0])[1] injy -= frame_center(GARRAY[0])[0] dist = np.sqrt(injx**2 + injy**2) theta = np.mod(np.arctan2(injy, injx) / np.pi * 180, 360) flux_dist_theta_all.append((fluxes_dist[j], dist, theta)) # Multiprocessing pool res = pool_map(nproc, _get_adi_snrs, GARRPSF, GARRPA, fwhm, plsc, fixed(flux_dist_theta_all), wavelengths, mode, ncomp) for i in range(len(distances)): flux_dist = [] snr_dist = [] for j in range(ninj): flux_dist.append(res[j + (ninj * i)][0]) snr_dist.append(res[j + (ninj * i)][1]) fluxes_list.append(flux_dist) snrs_list.append(snr_dist) timing(starttime) return fluxes_list, snrs_list
def sampling(self): """ Using the computed interval of fluxes for sampling the flux vs SNR relationship. """ if not self.min_fluxes: self.get_min_flux() if not self.max_fluxes: self.get_max_flux() print("Sampling by injecting fake companions") res = _sample_flux_snr(self.distances, self.fwhm, self.plsc, self.n_injections, self.min_fluxes, self.max_fluxes, self.n_proc, self.random_seed, self.wavelengths, self.spectrum, self.algo, self.scaling) self.sampled_fluxes, self.sampled_snrs = res timing(self.starttime)
def get_max_flux(self, debug=False): """ Obtaining the high end of the interval for sampling the S/N. """ if self.min_fluxes is None: self.get_min_flux() print( "Estimating the upper flux interval for sampling the S/N vs flux " "function") flux_max = pool_map(self.n_proc, _get_max_flux, iterable(self.n_dist), self.distances, self.min_fluxes, self.fwhm, self.plsc, iterable(self.max_snr), self.wavelengths, self.spectrum, self.algo, self.scaling, self.svd_mode, self.random_seed, debug) self.max_fluxes = flux_max timing(self.starttime)
def train_3dnet(X, Y, test_size=0.1, validation_split=0.1, random_state=0, layer_type=('conv3d', 'conv3d'), nconvlayers=2, conv_nfilters=(40, 80), kernel_sizes=((3, 3, 3), (2, 2, 2)), conv_strides=((1, 1, 1), (1, 1, 1)), conv_padding='same', dilation_rate=((1, 1, 1), (1, 1, 1)), pool_layers=2, pool_func='ave', pool_sizes=((2, 2, 2), (2, 2, 2)), pool_strides=((2, 2, 2), (2, 2, 2)), dense_units=128, rec_hidden_states=64, activation='relu', learnrate=0.003, batchsize=64, epochs=20, patience=2, min_delta=0.01, retrain=None, verb=1, summary=True, gpu_id='0', plot='tb', tb_path='./logs', full_output=False): """ 3D Convolutional network or Convolutional LSTM network for SODINN-PW and SODINN-SVD. Parameters ---------- ... layer_type : {'conv3d', 'clstm'} str optional batchsize : Batch size per GPU (no need to increase it when ``ngpus`` > 1). """ clear_session() graph = get_default_graph() config = tf.ConfigProto() # Don't pre-allocate memory; allocate as-needed config.gpu_options.allow_growth = True # Only allow a total of half the GPU memory to be allocated # config.gpu_options.per_process_gpu_memory_fraction = 0.5 config.gpu_options.visible_device_list = gpu_id session = tf.Session(graph=graph, config=config) set_session(session) ngpus = len(gpu_id.split(',')) batchsize *= ngpus if not nconvlayers == len(conv_nfilters): raise ValueError('`conv_nfilters` has a wrong length') if not nconvlayers == len(kernel_sizes): raise ValueError('`kernel_sizes` has a wrong length') if not nconvlayers == len(conv_strides): raise ValueError('`conv_strides` has a wrong length') if not nconvlayers == len(dilation_rate): raise ValueError('`dilation_rate` has a wrong length') if pool_layers > 0: if not pool_layers == len(pool_sizes): raise ValueError('`pool_sizes` has a wrong length') if pool_strides is not None: if not pool_layers == len(pool_strides): raise ValueError('`pool_strides` has a wrong length') else: pool_strides = [None] * pool_layers if isinstance(layer_type, str): layer_type = [layer_type for _ in range(nconvlayers)] starttime = time_ini() # Mixed train/test sets with Sklearn split res = train_test_split(X, Y, test_size=test_size, random_state=random_state) X_train, X_test, y_train, y_test = res msg = 'Zeros in train: {} | Ones in train: {}' print(msg.format(y_train.tolist().count(0), y_train.tolist().count(1))) msg = 'Zeros in test: {} | Ones in test: {}' print(msg.format(y_test.tolist().count(0), y_test.tolist().count(1))) # adding the "channels" dimension (1) X_train = X_train.reshape(X_train.shape[0], X_train.shape[1], X_train.shape[2], X_train.shape[3], 1) X_test = X_test.reshape(X_test.shape[0], X_test.shape[1], X_test.shape[2], X_test.shape[3], 1) print("\nShapes of train and test:") print(X_train.shape, y_train.shape, X_test.shape, y_test.shape, '\n') # -------------------------------------------------------------------------- if retrain is not None: M = retrain # Training the network M.compile(loss='binary_crossentropy', metrics=['accuracy'], optimizer=Adam(lr=learnrate, decay=1e-2)) early_stopping = EarlyStopping(monitor='val_loss', patience=patience, min_delta=min_delta, verbose=verb) hist = M.fit(X_train, y_train, batch_size=batchsize, epochs=epochs, initial_epoch=0, verbose=verb, validation_split=0.1, callbacks=[early_stopping], shuffle=True) score = M.evaluate(X_test, y_test, verbose=verb) print('\n Test score/loss:', score[0]) print(' Test accuracy:', score[1]) timing(starttime) fintime = time_fin(starttime) if full_output: return M, hist.history, score, fintime else: return M # -------------------------------------------------------------------------- # Creating the NN model if ngpus > 1: with tf.device('/cpu:0'): M = Sequential() else: M = Sequential() kernel_init = 'glorot_uniform' bias_init = 'random_normal' rec_act = 'hard_sigmoid' rec_init = 'orthogonal' temp_dim = X_train.shape[1] # npcs or pw slices patch_size = X_train.shape[2] input_shape = (temp_dim, patch_size, patch_size, 1) # Stack of Conv3d, (B)CLSTM, (B)LRCN or (B)GRCN layers for i in range(nconvlayers): if layer_type[i] in ('conv3d', 'clstm', 'bclstm'): if pool_func == 'ave': pooling_func = AveragePooling3D elif pool_func == 'max': pooling_func = MaxPooling3D elif layer_type[i] in ('lrcn', 'blrcn', 'grcn', 'bgrcn'): if pool_func == 'ave': pooling_func = AveragePooling2D elif pool_func == 'max': pooling_func = MaxPooling2D else: raise ValueError('pool_func is not recognized') if layer_type[i] == 'conv3d': if not len(kernel_sizes[i]) == 3: raise ValueError( 'Kernel sizes for Conv3d are tuples of 3 values') if not len(conv_strides[i]) == 3: raise ValueError('Strides for Conv3d are tuples of 3 values') if not len(dilation_rate[i]) == 3: raise ValueError('Dilation for Conv3d is a tuple of 3 values') if i == 0: M.add( Conv3D(filters=conv_nfilters[i], kernel_size=kernel_sizes[i], strides=conv_strides[i], padding=conv_padding, kernel_initializer=kernel_init, bias_initializer=bias_init, name='conv3d_layer1', dilation_rate=dilation_rate[i], data_format='channels_last', input_shape=input_shape)) M.add(SpatialDropout3D(0.5)) else: M.add( Conv3D(filters=conv_nfilters[i], kernel_size=kernel_sizes[i], strides=conv_strides[i], padding=conv_padding, kernel_initializer=kernel_init, dilation_rate=dilation_rate[i], name='conv3d_layer' + str(i + 1))) M.add(SpatialDropout3D(0.25)) M.add(Activation(activation, name='activ_layer' + str(i + 1))) if pool_layers != 0: M.add( pooling_func(pool_size=pool_sizes[i], strides=pool_strides[i], padding='valid')) pool_layers -= 1 M.add(Dropout(rate=0.25, name='dropout_layer' + str(i + 1))) elif layer_type[i] == 'clstm': msg = 'are tuples of 2 integers' if not len(kernel_sizes[i]) == 2: raise ValueError('Kernel sizes for ConvLSTM' + msg) if not len(conv_strides[i]) == 2: raise ValueError('Strides for ConvLSTM') if not len(dilation_rate[i]) == 2: raise ValueError('Dilation rates for ConvLSTM') if i == 0: M.add( ConvLSTM2D( filters=conv_nfilters[i], kernel_size=kernel_sizes[i], strides=conv_strides[i], padding=conv_padding, kernel_initializer=kernel_init, input_shape=input_shape, name='convlstm_layer1', return_sequences=True, dilation_rate=dilation_rate[i], activation='tanh', recurrent_activation=rec_act, use_bias=True, recurrent_initializer=rec_init, bias_initializer='zeros', unit_forget_bias=True, # TODO: Errors when using dropout, Keras bug? dropout=0.0, recurrent_dropout=0.0)) M.add(SpatialDropout3D(0.5)) else: M.add( ConvLSTM2D( filters=conv_nfilters[i], kernel_size=kernel_sizes[i], strides=conv_strides[i], padding=conv_padding, kernel_initializer=kernel_init, name='convlstm_layer' + str(i + 1), return_sequences=True, dilation_rate=dilation_rate[i], activation='tanh', recurrent_activation=rec_act, use_bias=True, recurrent_initializer=rec_init, bias_initializer='zeros', unit_forget_bias=True, # TODO: Errors when using dropout, Keras bug? dropout=0.0, recurrent_dropout=0.0)) M.add(SpatialDropout3D(0.25)) if pool_layers != 0: M.add( pooling_func(pool_size=pool_sizes[i], strides=pool_strides[i], padding='valid')) pool_layers -= 1 elif layer_type[i] == 'bclstm': msg = 'are tuples of 2 integers' if not len(kernel_sizes[i]) == 2: raise ValueError('Kernel sizes for ConvLSTM' + msg) if not len(conv_strides[i]) == 2: raise ValueError('Strides for ConvLSTM') if not len(dilation_rate[i]) == 2: raise ValueError('Dilation rates for ConvLSTM') if i == 0: M.add( Bidirectional(ConvLSTM2D(filters=conv_nfilters[i], kernel_size=kernel_sizes[i], strides=conv_strides[i], padding=conv_padding, kernel_initializer=kernel_init, name='convlstm_layer1', return_sequences=True, dilation_rate=dilation_rate[i], activation='tanh', recurrent_activation=rec_act, use_bias=True, recurrent_initializer=rec_init, bias_initializer='zeros', unit_forget_bias=True), input_shape=input_shape)) M.add(SpatialDropout3D(0.5)) else: M.add( Bidirectional( ConvLSTM2D(filters=conv_nfilters[i], kernel_size=kernel_sizes[i], strides=conv_strides[i], padding=conv_padding, kernel_initializer=kernel_init, name='convlstm_layer' + str(i + 1), return_sequences=True, dilation_rate=dilation_rate[i], activation='tanh', recurrent_activation=rec_act, use_bias=True, recurrent_initializer=rec_init, bias_initializer='zeros', unit_forget_bias=True))) M.add(SpatialDropout3D(0.25)) if pool_layers != 0: M.add( pooling_func(pool_size=pool_sizes[i], strides=pool_strides[i], padding='valid')) pool_layers -= 1 elif layer_type[i] in ('lrcn', 'blrcn', 'grcn', 'bgrcn'): if not len(kernel_sizes[i]) == 2: raise ValueError( 'Kernel sizes for LRCN are tuples of 2 values') if not len(conv_strides[i]) == 2: raise ValueError('Strides for LRCN are tuples of 2 values') if not len(dilation_rate[i]) == 2: raise ValueError('Dilation for LRCN is a tuple of 2 values') if i == 0: # TimeDistributed wrapper applies a layer to every temporal # slice of an input. The input should be at least 3D and the # dimension of index one will be considered to be the temporal # dimension. M.add( TimeDistributed(Conv2D(filters=conv_nfilters[i], kernel_size=kernel_sizes[i], strides=conv_strides[i], padding=conv_padding, name='lrcn_layer1', activation=activation), input_shape=input_shape)) # This version performs the same function as Dropout, however it # drops entire 2D feature maps instead of individual elements. M.add(TimeDistributed(SpatialDropout2D(0.5))) else: M.add( TimeDistributed( Conv2D(filters=conv_nfilters[i], kernel_size=kernel_sizes[i], strides=conv_strides[i], padding=conv_padding, name='lrcn_layer' + str(i + 1), activation=activation))) M.add(TimeDistributed(SpatialDropout2D(0.25))) if pool_layers != 0: M.add( TimeDistributed( pooling_func(pool_size=pool_sizes[i], strides=pool_strides[i], padding='valid'))) pool_layers -= 1 # (B)LRCN or (B)GRCN on Conv2d extracted features if layer_type[-1] == 'lrcn': M.add(TimeDistributed(Flatten(name='flatten'))) M.add(Dropout(0.5, name='dropout_flatten')) M.add( CuDNNLSTM(rec_hidden_states, kernel_initializer=kernel_init, return_sequences=False)) M.add(Dropout(0.5, name='dropout_lstm')) elif layer_type[-1] == 'blrcn': M.add(TimeDistributed(Flatten(name='flatten'))) M.add(Dropout(0.5, name='dropout_flatten')) M.add( Bidirectional( LSTM( rec_hidden_states, # TODO: bug CuDNNLSTM? kernel_initializer=kernel_init, return_sequences=False))) M.add(Dropout(0.5, name='dropout_lstm')) elif layer_type[-1] == 'grcn': M.add(TimeDistributed(Flatten(name='flatten'))) M.add(Dropout(0.5, name='dropout_flatten')) M.add( CuDNNGRU(rec_hidden_states, kernel_initializer=kernel_init, return_sequences=False)) M.add(Dropout(0.5, name='dropout_lstm')) elif layer_type[-1] == 'bgrcn': M.add(TimeDistributed(Flatten(name='flatten'))) M.add(Dropout(0.5, name='dropout_flatten')) M.add( Bidirectional( CuDNNGRU(rec_hidden_states, kernel_initializer=kernel_init, return_sequences=False))) M.add(Dropout(0.5, name='dropout_lstm')) # Otherwise we just flatten and go to dense layers else: M.add(Flatten(name='flatten')) # Fully-connected or dense layer M.add(Dense(units=dense_units, name='dense_layer')) M.add(Activation(activation, name='activ_dense')) M.add(Dropout(rate=0.5, name='dropout_dense')) # Sigmoid unit M.add(Dense(units=1, name='dense_1unit')) M.add(Activation('sigmoid', name='activ_out')) if summary: M.summary() # Callbacks early_stopping = EarlyStopping(monitor='val_loss', patience=patience, min_delta=min_delta, verbose=verb) if plot is not None: if plot == 'tb': tensorboard = TensorBoard(log_dir=tb_path, histogram_freq=1, write_graph=True, write_images=True) callbacks = [early_stopping, tensorboard] elif plot == 'llp': plotlosses = livelossplot.PlotLossesKeras() callbacks = [early_stopping, plotlosses] else: raise ValueError("`plot` method not recognized") else: callbacks = [early_stopping] # Training the network if ngpus > 1: # Multi-GPUs Mpar = multi_gpu_model(M, gpus=ngpus) # Training the network Mpar.compile(loss='binary_crossentropy', metrics=['accuracy'], optimizer=Adam(lr=learnrate, decay=1e-2)) hist = Mpar.fit(X_train, y_train, batch_size=batchsize, epochs=epochs, initial_epoch=0, verbose=verb, shuffle=True, validation_split=validation_split, callbacks=callbacks) score = Mpar.evaluate(X_test, y_test, verbose=verb) else: # Single GPU M.compile(loss='binary_crossentropy', metrics=['accuracy'], optimizer=Adam(lr=learnrate, decay=1e-2)) hist = M.fit(X_train, y_train, batch_size=batchsize, epochs=epochs, initial_epoch=0, verbose=verb, shuffle=True, validation_split=validation_split, callbacks=callbacks) score = M.evaluate(X_test, y_test, verbose=verb) print('\nTest score/loss:', score[0], '\nTest accuracy:', score[1]) timing(starttime) fintime = time_fin(starttime) if full_output: return M, hist.history, score, fintime else: return M
def train_4dnet(X, Y, test_size=0.1, validation_split=0.1, random_state=0, layer_type=('conv3d', 'conv3d'), nconvlayers=2, conv_nfilters=(40, 80), kernel_sizes=((3, 3, 3), (2, 2, 2)), conv_strides=((1, 1, 1), (1, 1, 1)), conv_padding='same', dilation_rate=((1, 1, 1), (1, 1, 1)), pool_layers=2, pool_func='ave', pool_sizes=((2, 2, 2), (2, 2, 2)), pool_strides=((2, 2, 2), (2, 2, 2)), dense_units=128, rec_hidden_states=64, activation='relu', learnrate=0.003, batchsize=64, epochs=20, patience=2, min_delta=0.01, retrain=None, verb=1, summary=True, gpu_id='0', plot='tb', tb_path='./logs', full_output=False): """ """ clear_session() graph = get_default_graph() config = tf.ConfigProto() # Don't pre-allocate memory; allocate as-needed config.gpu_options.allow_growth = True # Only allow a total of half the GPU memory to be allocated # config.gpu_options.per_process_gpu_memory_fraction = 0.5 config.gpu_options.visible_device_list = gpu_id session = tf.Session(graph=graph, config=config) set_session(session) ngpus = len(gpu_id.split(',')) batchsize *= ngpus if not nconvlayers == len(conv_nfilters): raise ValueError('`conv_nfilters` has a wrong length') if not nconvlayers == len(kernel_sizes): raise ValueError('`kernel_sizes` has a wrong length') if not nconvlayers == len(conv_strides): raise ValueError('`conv_strides` has a wrong length') if not nconvlayers == len(dilation_rate): raise ValueError('`dilation_rate` has a wrong length') if pool_layers > 0: if not pool_layers == len(pool_sizes): raise ValueError('`pool_sizes` has a wrong length') if pool_strides is not None: if not pool_layers == len(pool_strides): raise ValueError('`pool_strides` has a wrong length') else: pool_strides = [None] * pool_layers if isinstance(layer_type, str): layer_type = [layer_type for _ in range(nconvlayers)] starttime = time_ini() # Mixed train/test sets with Sklearn split res = train_test_split(X, Y, test_size=test_size, random_state=random_state) X_train, X_test, y_train, y_test = res msg = 'Zeros in train: {} | Ones in train: {}' print(msg.format(y_train.tolist().count(0), y_train.tolist().count(1))) msg = 'Zeros in test: {} | Ones in test: {}' print(msg.format(y_test.tolist().count(0), y_test.tolist().count(1))) # adding the "channels" dimension (1) X_train = X_train.reshape(X_train.shape[0], X_train.shape[1], X_train.shape[2], X_train.shape[3], X_train.shape[4], 1) X_test = X_test.reshape(X_test.shape[0], X_test.shape[1], X_test.shape[2], X_test.shape[3], X_train.shape[4], 1) print("\nShapes of train and test:") print(X_train.shape, y_train.shape, X_test.shape, y_test.shape, '\n') # -------------------------------------------------------------------------- kernel_init = 'glorot_uniform' bias_init = 'random_normal' rec_act = 'hard_sigmoid' rec_init = 'orthogonal' temp_dim = X_train.shape[1] k_dim = X_train.shape[2] patch_size = X_train.shape[3] input_shape_3d = (temp_dim, patch_size, patch_size, 1) if pool_func == 'ave': pooling_func = AveragePooling3D elif pool_func == 'max': pooling_func = MaxPooling3D # -------------------------------------------------------------------------- # Per branch model # -------------------------------------------------------------------------- input_layer = Input(shape=input_shape_3d, name='input_layer', dtype='float32') for i in range(nconvlayers): # Stack of Conv3d, (B)CLSTM, (B)LRCN or (B)GRCN layers if layer_type[i] in ('conv3d', 'clstm', 'bclstm'): if pool_func == 'ave': pooling_func = AveragePooling3D elif pool_func == 'max': pooling_func = MaxPooling3D elif layer_type[i] in ('lrcn', 'blrcn', 'grcn', 'bgrcn'): if pool_func == 'ave': pooling_func = AveragePooling2D elif pool_func == 'max': pooling_func = MaxPooling2D else: raise ValueError('pool_func is not recognized') if layer_type[i] == 'conv3d': if not len(kernel_sizes[i]) == 3: raise ValueError( 'Kernel sizes for Conv3d are tuples of 3 values') if not len(conv_strides[i]) == 3: raise ValueError('Strides for Conv3d are tuples of 3 values') if not len(dilation_rate[i]) == 3: raise ValueError('Dilation for Conv3d is a tuple of 3 values') if i == 0: x = Conv3D(filters=conv_nfilters[i], kernel_size=kernel_sizes[i], strides=conv_strides[i], padding=conv_padding, kernel_initializer=kernel_init, bias_initializer=bias_init, name='conv3d_layer1', dilation_rate=dilation_rate[i], data_format='channels_last', input_shape=input_shape_3d)(input_layer) x = SpatialDropout3D(0.5)(x) x = Activation(activation, name='activ_layer1')(x) x = pooling_func(pool_size=pool_sizes[i], strides=pool_strides[i], padding='valid')(x) else: x = Conv3D(filters=conv_nfilters[i], kernel_size=kernel_sizes[i], strides=conv_strides[i], padding=conv_padding, kernel_initializer=kernel_init, bias_initializer=bias_init, name='conv3d_layer' + str(i + 1), dilation_rate=dilation_rate[i])(x) x = SpatialDropout3D(0.25)(x) x = Activation(activation, name='activ_layer' + str(i + 1))(x) x = pooling_func(pool_size=pool_sizes[i], strides=pool_strides[i], padding='valid')(x) elif layer_type[i] == 'clstm': msg = 'are tuples of 2 integers' if not len(kernel_sizes[0]) == 2: raise ValueError('Kernel sizes for ConvLSTM' + msg) if not len(conv_strides[0]) == 2: raise ValueError('Strides for ConvLSTM') if not len(dilation_rate[0]) == 2: raise ValueError('Dilation rates for ConvLSTM') if i == 0: x = ConvLSTM2D(filters=conv_nfilters[i], kernel_size=kernel_sizes[i], strides=conv_strides[i], padding=conv_padding, kernel_initializer=kernel_init, input_shape=input_shape_3d, name='convlstm_layer1', return_sequences=True, dilation_rate=dilation_rate[i], activation='tanh', recurrent_activation=rec_act, use_bias=True, recurrent_initializer=rec_init, bias_initializer='zeros', unit_forget_bias=True, dropout=0.0, recurrent_dropout=0.0)(input_layer) x = SpatialDropout3D(0.5)(x) x = pooling_func(pool_size=pool_sizes[i], strides=pool_strides[i], padding='valid')(x) else: x = ConvLSTM2D(filters=conv_nfilters[i], kernel_size=kernel_sizes[i], strides=conv_strides[i], padding=conv_padding, kernel_initializer=kernel_init, name='convlstm_layer' + str(i + 1), return_sequences=True, dilation_rate=dilation_rate[i], activation='tanh', recurrent_activation=rec_act, use_bias=True, recurrent_initializer=rec_init, bias_initializer='zeros', unit_forget_bias=True, dropout=0.0, recurrent_dropout=0.0)(x) x = SpatialDropout3D(0.25)(x) x = pooling_func(pool_size=pool_sizes[1], strides=pool_strides[1], padding='valid')(x) elif layer_type[i] in ('lrcn', 'blrcn', 'grcn', 'bgrcn'): if not len(kernel_sizes[i]) == 2: raise ValueError( 'Kernel sizes for LRCN are tuples of 2 values') if not len(conv_strides[i]) == 2: raise ValueError('Strides for LRCN are tuples of 2 values') if not len(dilation_rate[i]) == 2: raise ValueError('Dilation for LRCN is a tuple of 2 values') # TimeDistributed wrapper applies a layer to every temporal # slice of an input. The input should be at least 3D and the # dimension of index one will be considered to be the temporal # dimension. if i == 0: x = TimeDistributed(Conv2D(filters=conv_nfilters[i], kernel_size=kernel_sizes[i], strides=conv_strides[i], padding=conv_padding, name='lrcn_layer1', activation=activation), input_shape=input_shape_3d)(input_layer) # This version performs the same function as Dropout, however it # drops entire 2D feature maps instead of individual elements. x = TimeDistributed(SpatialDropout2D(0.5))(x) x = TimeDistributed( pooling_func(pool_size=pool_sizes[i], strides=pool_strides[i], padding='valid'))(x) else: x = TimeDistributed( Conv2D(filters=conv_nfilters[1], kernel_size=kernel_sizes[1], strides=conv_strides[1], padding=conv_padding, name='lrcn_layer' + str(i + 1), activation=activation))(x) x = TimeDistributed(SpatialDropout2D(0.25))(x) x = TimeDistributed( pooling_func(pool_size=pool_sizes[1], strides=pool_strides[1], padding='valid'))(x) # Final layer if layer_type[-1] in ('conv3d', 'clstm'): flatten_layer = Flatten(name='flatten')(x) # Fully-connected or dense layer output = Dense(units=dense_units, name='dense_layer')(flatten_layer) output = Activation(activation, name='activ_dense')(output) output = Dropout(rate=0.5, name='dropout_dense')(output) elif layer_type[-1] in ('lrcn', 'blrcn', 'grcn', 'bgrcn'): output = TimeDistributed(Flatten(name='flatten'))(x) output = Dropout(0.5, name='dropout_flatten')(output) model_branch = KerasModel(inputs=input_layer, outputs=output) # -------------------------------------------------------------------------- # -------------------------------------------------------------------------- # Multi-input model # -------------------------------------------------------------------------- inputs = [] outputs = [] for i in range(k_dim): input_ = Input(shape=input_shape_3d, name='input_' + str(i + 1), dtype='float32') output_ = model_branch(input_) inputs.append(input_) outputs.append(output_) # Concatenating the branches. Shape [samples, time steps, features*k_dim] concatenated = concatenate(outputs) if layer_type[1] == 'lrcn': lstm = CuDNNLSTM(rec_hidden_states, kernel_initializer=kernel_init, return_sequences=False)(concatenated) concatenated = Dropout(0.5, name='dropout_lstm')(lstm) elif layer_type[1] == 'blrcn': blstm = Bidirectional( LSTM( rec_hidden_states, # TODO: bug CuDNNLSTM? kernel_initializer=kernel_init, return_sequences=False))(concatenated) concatenated = Dropout(0.5, name='dropout_lstm')(blstm) elif layer_type[1] == 'grcn': gru = CuDNNGRU(rec_hidden_states, kernel_initializer=kernel_init, return_sequences=False)(concatenated) concatenated = Dropout(0.5, name='dropout_gru')(gru) elif layer_type[1] == 'bgrcn': bgru = Bidirectional( CuDNNGRU(rec_hidden_states, kernel_initializer=kernel_init, return_sequences=False))(concatenated) concatenated = Dropout(0.5, name='dropout_gru')(bgru) # Sigmoid unit prediction = Dense(units=1, name='sigmoid_output_unit', activation='sigmoid')(concatenated) model_final = KerasModel(inputs=inputs, outputs=prediction) # -------------------------------------------------------------------------- if summary: model_branch.summary() # model_final.summary() # Callbacks early_stopping = EarlyStopping(monitor='val_loss', patience=patience, min_delta=min_delta, verbose=verb) if plot is not None: if plot == 'tb': tensorboard = TensorBoard(log_dir=tb_path, histogram_freq=1, write_graph=True, write_images=True) callbacks = [early_stopping, tensorboard] elif plot == 'llp': plotlosses = livelossplot.PlotLossesKeras() callbacks = [early_stopping, plotlosses] else: raise ValueError("`plot` method not recognized") else: callbacks = [early_stopping] X_train = list(np.moveaxis(X_train, 2, 0)) X_test = list(np.moveaxis(X_test, 2, 0)) # Training the network if ngpus > 1: # Multi-GPUs Mpar = multi_gpu_model(model_final, gpus=ngpus) # Training the network Mpar.compile(loss='binary_crossentropy', metrics=['accuracy'], optimizer=Adam(lr=learnrate, decay=1e-2)) hist = Mpar.fit(X_train, y_train, batch_size=batchsize, shuffle=True, epochs=epochs, initial_epoch=0, verbose=verb, validation_split=validation_split, callbacks=callbacks) score = Mpar.evaluate(X_test, y_test, verbose=verb) else: # Single GPU model_final.compile(loss='binary_crossentropy', metrics=['accuracy'], optimizer=Adam(lr=learnrate, decay=1e-2)) hist = model_final.fit(X_train, y_train, batch_size=batchsize, epochs=epochs, initial_epoch=0, verbose=verb, validation_split=validation_split, callbacks=callbacks, shuffle=True) score = model_final.evaluate(X_test, y_test, verbose=verb) print('\nTest score/loss:', score[0], '\nTest accuracy:', score[1]) timing(starttime) fintime = time_fin(starttime) if full_output: return model_final, hist.history, score, fintime else: return model_final
def run(self, dpi=100): """ Obtaining the flux vs S/N relationship. dpi : int, optional DPI of the figures. """ if not self.sampled_fluxes or not self.sampled_snrs: self.sampling() nsubplots = len(self.distances) ncols = min(4, nsubplots) if nsubplots > 1 and nsubplots % 2 != 0: nsubplots -= 1 if nsubplots < 3: figsize = (10, 2) if nsubplots == 2: figsizex = figsize[0] * 0.66 elif nsubplots == 1: figsizex = figsize[0] * 0.33 nrows = 1 else: if nsubplots <= 8: figsize = (10, 4) else: figsize = (10, 6) figsizex = figsize[0] nrows = int(nsubplots / ncols) + 1 fig, axs = plt.subplots(nrows, ncols, figsize=(figsizex, figsize[1]), dpi=dpi, sharey='row') fig.subplots_adjust(wspace=0.05, hspace=0.3) if isinstance(axs, np.ndarray): axs = axs.ravel() fhi = list() flo = list() print("Interpolating the Flux vs S/N function") # Regression for each distance for i, d in enumerate(self.distances): plotvlines = [self.min_snr[i], self.max_snr[i]] if isinstance(axs, np.ndarray): axis = axs[i] else: axis = axs fluxes = np.array(self.sampled_fluxes[i]) snrs = np.array(self.sampled_snrs[i]) mask = np.where(snrs > 0.1) snrs = snrs[mask] fluxes = fluxes[mask] f = interp1d(np.sort(snrs), np.sort(fluxes), kind='slinear', fill_value='extrapolate') minsnr = max(self.min_snr[i], min(snrs)) maxsnr = min(self.max_snr[i], max(snrs)) snrs_pred = np.linspace(minsnr, maxsnr, num=50) fluxes_pred = f(snrs_pred) flux_for_lowsnr = f(minsnr) flux_for_higsnr = f(maxsnr) fhi.append(flux_for_higsnr) flo.append(flux_for_lowsnr) # Figure of flux vs s/n axis.xaxis.set_tick_params(labelsize=6) axis.yaxis.set_tick_params(labelsize=6) axis.plot(fluxes, snrs, '.', alpha=0.2, markersize=4) axis.plot(fluxes_pred, snrs_pred, '-', alpha=1, color='orangered') axis.grid(which='major', alpha=0.3) axis.set_xlim(0) for l in plotvlines: axis.plot((0, max(fluxes)), (l, l), ':', color='darksalmon') axis = fig.add_subplot(111, frame_on=False) axis.set_xticks([]) axis.set_yticks([]) axis.set_xlabel('Fakecomp flux scaling', labelpad=25, size=8) axis.set_ylabel('Signal to noise ratio', labelpad=25, size=8) if isinstance(axs, np.ndarray): for i in range(len(self.distances), len(axs)): axs[i].axis('off') plt.show() flo = np.array(flo).flatten() fhi = np.array(fhi).flatten() if self.inter_extrap and len(self.distances) > 2: x = self.distances f1 = interpolate.interp1d(x, flo, fill_value='extrapolate') f2 = interpolate.interp1d(x, fhi, fill_value='extrapolate') fhi = f2(self.inter_extrap_dist) flo = f1(self.inter_extrap_dist) plot_x = self.inter_extrap_dist else: plot_x = self.distances self.estimated_fluxes_high = fhi self.estimated_fluxes_low = flo # figure with fluxes as a function of the separation if len(self.distances) > 1 and isinstance(self.min_snr, (float, int)) \ and isinstance(self.max_snr, (float, int)): plt.figure(figsize=(10, 4), dpi=dpi) plt.plot(self.distances, self.radprof, '--', alpha=0.8, color='gray', lw=2, label='average radial profile') plt.plot(plot_x, flo, '.-', alpha=0.6, lw=2, color='dodgerblue', label='flux lower bound') plt.plot(plot_x, fhi, '.-', alpha=0.6, color='dodgerblue', lw=2, label='flux upper bound') plt.fill_between(plot_x, flo, fhi, where=flo <= fhi, alpha=0.2, facecolor='dodgerblue', interpolate=True) plt.grid(which='major', alpha=0.4) plt.xlabel('Distance from the center [Pixels]') plt.ylabel('Fakecomp flux scaling [Counts]') plt.minorticks_on() plt.xlim(0) plt.ylim(0) plt.legend() plt.show() timing(self.starttime)
def predict_pairwise(model, cube, angle_list, fwhm, patch_size_px, delta_rot, radius_int=None, high_pass='******', kernel_size=5, normalization='slice', imlib='opencv', interpolation='bilinear', nproc=1, verbose=True, chunks_per_proc=2): """ Parameters ---------- model : Keras model cube : 3d ndarray angle_list : 1d ndarray fwhm : float patch_size : int, optional delta_rot : float, optional verbose: bool or int, optional 0 / False: no output 1 / True: full output (timing + progress bar) 2: progress bar only [...] Returns ------- probmap : 2d ndarray Notes ----- - support for 4D cubes? """ starttime = time_ini(verbose=verbose) if radius_int is None: radius_int = fwhm n_frames, sizey, sizex = cube.shape width = int(sizey / 2 - patch_size_px / 2 - radius_int) ind = get_annulus_segments(cube[0], inner_radius=radius_int, width=width, nsegm=1) probmap = np.zeros((sizey, sizex)) if high_pass is not None: cube = cube_filter_highpass(cube, high_pass, kernel_size=kernel_size, verbose=False) indices = list(range(ind[0][0].shape[0])) if nproc is None: nproc = cpu_count() // 2 # Hyper-threading doubles the # of cores # prepare patches in parallel nchunks = nproc * chunks_per_proc print("Grabbing patches with {} processes".format(nproc)) res_ = list( Progressbar(pool_imap(nproc, _parallel_make_patches_chunk, iterable(make_chunks(indices, nchunks)), ind, cube, angle_list, fwhm, patch_size_px, delta_rot, normalization, imlib, interpolation), total=nchunks, leave=False, verbose=False)) xx = [] yy = [] pats = [] for r in res_: x, y, pp = r for i in range(len(x)): xx.append(x[i]) yy.append(y[i]) pats.append(pp[i]) if verbose == 1: timing(starttime) print("Prediction on patches:") probas = predict_from_patches(model, pats, len(xx)) for i in range(len(xx)): probmap[xx[i], yy[i]] = probas[i] if verbose == 1: timing(starttime) return probmap
def estimate_fluxes(cube, psf, distances, angles, fwhm, plsc, wavelengths=None, n_injections=10, mode='median', ncomp=2, min_adi_snr=2, max_adi_snr=5, random_seed=42, kernel='rbf', epsilon=0.1, c=1e4, gamma=1e-2, figsize=(10, 5), dpi=100, n_proc=2, **kwargs): """ Automatic estimation of the scaling factors for injecting the companions (brightness or contrast of fake companions). Epsilon-Support Vector Regression (important parameters in the model are C and epsilon). The implementation is based on scikit-learn (libsvm). Parameters ---------- C : float, optional (default=1.0) Penalty parameter C of the error term. epsilon : float, optional (default=0.1) Epsilon in the epsilon-SVR model. It specifies the epsilon-tube within which no penalty is associated in the training loss function with points predicted within a distance epsilon from the actual value. kernel : string, optional (default=’rbf’) Specifies the kernel type to be used in the algorithm. It must be one of ‘linear’, ‘poly’, ‘rbf’, ‘sigmoid’, ‘precomputed’ or a callable. If none is given, ‘rbf’ will be used. If a callable is given it is used to precompute the kernel matrix. gamma : float, optional (default=’auto’) Kernel coefficient for ‘rbf’, ‘poly’ and ‘sigmoid’. If gamma is ‘auto’ then 1/n_features will be used instead. """ starttime = time_ini() global GARRAY GARRAY = cube global GARRPSF GARRPSF = psf global GARRPA GARRPA = angles global GARRWL GARRWL = wavelengths if cube.ndim == 4: if wavelengths is None: raise ValueError('`wavelengths` parameter must be provided') # Getting the radial profile in the mean frame of the cube sampling_sep = 1 radius_int = 1 if cube.ndim == 3: global_frame = np.mean(cube, axis=0) elif cube.ndim == 4: global_frame = np.mean(cube.reshape(-1, cube.shape[2], cube.shape[3]), axis=0) me = frame_average_radprofile(global_frame, sep=sampling_sep, init_rad=radius_int, plot=False) radprof = np.array(me.radprof) radprof = radprof[np.array(distances) + 1] flux_min = radprof * 0.1 flux_min[flux_min < 0] = 0.1 # Multiprocessing pool flux_max = pool_map(n_proc, _get_max_flux, fixed(range(len(distances))), distances, radprof, fwhm, plsc, max_adi_snr, wavelengths) flux_max = np.array(flux_max) fluxes_list, snrs_list = _sample_flux_snr(distances, fwhm, plsc, n_injections, flux_min, flux_max, n_proc, random_seed, wavelengths, mode, ncomp) plotvlines = [min_adi_snr, max_adi_snr] nsubplots = len(distances) if nsubplots % 2 != 0: nsubplots -= 1 ncols = 4 nrows = int(nsubplots / ncols) + 1 fig, axs = plt.subplots(nrows, ncols, figsize=figsize, dpi=dpi, sharey='row') fig.subplots_adjust(wspace=0.05, hspace=0.3) axs = axs.ravel() fhi = list() flo = list() # Regression for each distance for i, d in enumerate(distances): fluxes = np.array(fluxes_list[i]) snrs = np.array(snrs_list[i]) mask = np.where(snrs > 0.1) snrs = snrs[mask].reshape(-1, 1) fluxes = fluxes[mask].reshape(-1, 1) model = SVR(kernel=kernel, epsilon=epsilon, C=c, gamma=gamma, **kwargs) model.fit(X=snrs, y=fluxes) flux_for_lowsnr = model.predict(min_adi_snr) flux_for_higsnr = model.predict(max_adi_snr) fhi.append(flux_for_higsnr[0]) flo.append(flux_for_lowsnr[0]) snrminp = min_adi_snr / 2 snrs_pred = np.linspace(snrminp, max_adi_snr + snrminp, num=50).reshape(-1, 1) fluxes_pred = model.predict(snrs_pred) # Figure of flux vs s/n axs[i].xaxis.set_tick_params(labelsize=6) axs[i].yaxis.set_tick_params(labelsize=6) axs[i].plot(fluxes, snrs, '.', alpha=0.2, markersize=4) axs[i].plot(fluxes_pred, snrs_pred, '-', alpha=0.99, label='S/N regression model', color='orangered') axs[i].grid(which='major', alpha=0.3) axs[i].legend(fontsize=6) for l in plotvlines: axs[i].plot((0, max(fluxes)), (l, l), ':', color='darksalmon') ax0 = fig.add_subplot(111, frame_on=False) ax0.set_xticks([]) ax0.set_yticks([]) ax0.set_xlabel('Fakecomp flux scaling [Counts]', labelpad=25, size=8) ax0.set_ylabel('ADI-medsub median S/N (3 equidist. angles)', labelpad=25, size=8) for i in range(len(distances), len(axs)): axs[i].axis('off') timing(starttime) flo = np.array(flo).flatten() fhi = np.array(fhi).flatten() # x = distances # f1 = interpolate.interp1d(x, flo, fill_value='extrapolate') # f2 = interpolate.interp1d(x, fhi, fill_value='extrapolate') # fhi = f2(distances_init) # flo = f1(distances_init) plt.figure(figsize=(10, 4), dpi=dpi) plt.plot(distances, radprof, '--', alpha=0.8, color='gray', lw=2, label='average radial profile') plt.plot(distances, flo, '.-', alpha=0.6, lw=2, color='dodgerblue', label='flux lower interval') plt.plot(distances, fhi, '.-', alpha=0.6, color='dodgerblue', lw=2, label='flux upper interval') plt.fill_between(distances, flo, fhi, where=flo <= fhi, alpha=0.2, facecolor='dodgerblue', interpolate=True) plt.grid(which='major', alpha=0.4) plt.xlabel('Distance from the center [Pixels]') plt.ylabel('Fakecomp flux scaling [Counts]') plt.minorticks_on() plt.xlim(0) plt.ylim(0) plt.legend() plt.show() return flo, fhi
def train_2dconvnet(X, Y, test_size=0.1, validation_split=0.1, random_state=0, pseudo3d=False, nconvlayers=2, conv_nfilters=(40, 80), kernel_sizes=((3, 3), (3, 3)), conv_strides=((1, 1), (1, 1)), pool_layers=2, pool_sizes=((2, 2), (2, 2)), pool_strides=((2, 2), (2, 2)), dense_units=128, activation='relu', learnrate=0.003, batchsize=64, epochs=20, patience=2, min_delta=0.01, retrain=None, verb=1, summary=True, gpu_id='0', full_output=False, plot='tb', tb_path='./logs'): """ 2D Convolutional network for pairwise subtracted patches. Parameters ---------- ... Notes ----- Multi-GPU with Keras: https://keras.io/utils/#multi_gpu_model """ clear_session() config = tf.ConfigProto() config.gpu_options.allow_growth = True config.gpu_options.visible_device_list = gpu_id set_session(tf.Session(config=config)) ngpus = len(gpu_id.split(',')) if not nconvlayers == len(conv_nfilters): raise ValueError('`conv_nfilters` has a wrong length') if not nconvlayers == len(kernel_sizes): raise ValueError('`kernel_sizes` has a wrong length') if not nconvlayers == len(conv_strides): raise ValueError('`conv_strides` has a wrong length') if pool_layers > 0: if not pool_layers == len(pool_sizes): raise ValueError('`pool_sizes` has a wrong length') if pool_strides is not None: if not pool_layers == len(pool_strides): raise ValueError('`pool_strides` has a wrong length') else: pool_strides = [None] * pool_layers if pseudo3d: if not X.ndim == 4: raise ValueError('X must contain 4D samples') starttime = time_ini() patch_size = X.shape[-1] # Mixed train/test sets with Sklearn split resplit = train_test_split(X, Y, test_size=test_size, random_state=random_state) X_train, X_test, y_train, y_test = resplit msg = 'Zeros in train: {} | Ones in train: {}' print(msg.format(y_train.tolist().count(0), y_train.tolist().count(1))) msg = 'Zeros in test: {} | Ones in test: {}' print(msg.format(y_test.tolist().count(0), y_test.tolist().count(1))) if pseudo3d: if not X.ndim == 4: raise ValueError('`X` has wrong number of dimensions') # moving the temporal dimension to the channels dimension (last) X_train = np.moveaxis(X_train, 1, -1) X_test = np.moveaxis(X_test, 1, -1) input_shape = (patch_size, patch_size, X_train.shape[-1]) else: # adding the channels dimension X_train = X_train.reshape(X_train.shape[0], patch_size, patch_size, 1) X_test = X_test.reshape(X_test.shape[0], patch_size, patch_size, 1) input_shape = (patch_size, patch_size, 1) print("\nShapes of train and test sets:") print(X_train.shape, y_train.shape, X_test.shape, y_test.shape, '\n') # -------------------------------------------------------------------------- if retrain is not None: M = retrain # re-training the network M.compile(loss='binary_crossentropy', metrics=['accuracy'], optimizer=Adam(lr=learnrate, decay=1e-2)) early_stopping = EarlyStopping(monitor='val_loss', patience=patience, min_delta=min_delta, verbose=verb) hist = M.fit(X_train, y_train, batch_size=batchsize, epochs=epochs, initial_epoch=0, verbose=verb, validation_split=0.1, callbacks=[early_stopping], shuffle=True) score = M.evaluate(X_test, y_test, verbose=verb) print('\nTest score/loss:', score[0], '\n', 'Test accuracy:', score[1]) timing(starttime) fintime = time_fin(starttime) if full_output: return M, hist.history, score, fintime else: return M # -------------------------------------------------------------------------- # Creating the NN model if ngpus > 1: with tf.device('/cpu:0'): M = Sequential() else: M = Sequential() # Stack of 2d convolutional layers kernel_init = 'glorot_uniform' bias_init = 'random_normal' for i in range(nconvlayers): if i == 0: M.add( Conv2D(filters=conv_nfilters[i], kernel_size=kernel_sizes[i], strides=conv_strides[i], padding='same', kernel_initializer=kernel_init, bias_initializer=bias_init, name='conv2d_layer1', data_format='channels_last', input_shape=input_shape)) else: M.add( Conv2D(filters=conv_nfilters[i], kernel_size=kernel_sizes[i], strides=conv_strides[i], padding='same', kernel_initializer=kernel_init, bias_initializer=bias_init, name='conv2d_layer' + str(i + 1))) M.add(Activation(activation, name='activ_layer' + str(i + 1))) if pool_layers != 0: M.add( MaxPooling2D(pool_size=pool_sizes[i], strides=pool_strides[i], padding='valid')) pool_layers -= 1 M.add(Dropout(rate=0.25, name='dropout_layer' + str(i + 1))) M.add(Flatten(name='flatten')) # Dense or fully-connected layer M.add(Dense(units=dense_units, name='dense_128units')) M.add(Activation(activation, name='activ_dense')) # M.add(BatchNormalization()) M.add(Dropout(rate=0.5, name='dropout_dense')) M.add(Dense(units=1, name='dense_1unit')) M.add(Activation('sigmoid', name='activ_out')) if summary: M.summary() # Callbacks early_stopping = EarlyStopping(monitor='val_loss', patience=patience, min_delta=min_delta, verbose=verb) if plot is not None: if plot == 'tb': tensorboard = TensorBoard(log_dir=tb_path, histogram_freq=1, write_graph=True, write_images=True) callbacks = [early_stopping, tensorboard] elif plot == 'llp': plotlosses = livelossplot.PlotLossesKeras() callbacks = [early_stopping, plotlosses] else: raise ValueError("`plot` method not recognized") else: callbacks = [early_stopping] # Multi-GPUs if ngpus > 1: Mpar = multi_gpu_model(M, gpus=ngpus) # Training the network Mpar.compile(loss='binary_crossentropy', metrics=['accuracy'], optimizer=Adam(lr=learnrate, decay=1e-2)) hist = Mpar.fit(X_train, y_train, batch_size=batchsize * ngpus, epochs=epochs, initial_epoch=0, verbose=verb, validation_split=validation_split, callbacks=callbacks, shuffle=True) score = Mpar.evaluate(X_test, y_test, verbose=verb) else: # Training the network M.compile(loss='binary_crossentropy', metrics=['accuracy'], optimizer=Adam(lr=learnrate, decay=1e-2)) hist = M.fit(X_train, y_train, batch_size=batchsize * ngpus, epochs=epochs, initial_epoch=0, verbose=verb, validation_split=validation_split, callbacks=callbacks, shuffle=True) score = M.evaluate(X_test, y_test, verbose=verb) print('\nTest score/loss:', score[0], '\n', 'Test accuracy:', score[1]) timing(starttime) fintime = time_fin(starttime) if full_output: return M, hist.history, score, fintime else: return M
def correct_bad_pixels(self, verbose=True, debug=False): sci_list = [] with open(self.outpath + "sci_list.txt", "r") as f: tmp = f.readlines() for line in tmp: sci_list.append(line.split('\n')[0]) sky_list = [] with open(self.outpath + "sky_list.txt", "r") as f: tmp = f.readlines() for line in tmp: sky_list.append(line.split('\n')[0]) unsat_list = [] with open(self.outpath + "unsat_list.txt", "r") as f: tmp = f.readlines() for line in tmp: unsat_list.append(line.split('\n')[0]) n_sci = len(sci_list) ndit_sci = fits_info.ndit_sci n_sky = len(sky_list) ndit_sky = fits_info.ndit_sky tmp = open_fits(self.outpath + '1_crop_unsat_' + unsat_list[-1], header=False) nx_unsat_crop = tmp.shape[2] master_flat_frame = open_fits(self.outpath + 'master_flat_field.fits') # Create bpix map bpix = np.where(np.abs(master_flat_frame - 1.09) > 0.41) # i.e. for QE < 0.68 and QE > 1.5 bpix_map = np.zeros([self.com_sz, self.com_sz]) bpix_map[bpix] = 1 if nx_unsat_crop < bpix_map.shape[1]: bpix_map_unsat = frame_crop(bpix_map, nx_unsat_crop) else: bpix_map_unsat = bpix_map #number of bad pixels nbpix = int(np.sum(bpix_map)) ntotpix = self.com_sz**2 print("total number of bpix: ", nbpix) print("total number of pixels: ", ntotpix) print("=> {}% of bad pixels.".format(100 * nbpix / ntotpix)) write_fits(self.outpath + 'master_bpix_map.fits', bpix_map) write_fits(self.outpath + 'master_bpix_map_unsat.fits', bpix_map_unsat) plot_frames(bpix_map, bpix_map_unsat) #update final crop size final_sz = self.get_final_sz() #crop frames to that size for sc, fits_name in enumerate(sci_list): tmp = open_fits(self.outpath + '2_nan_corr_' + fits_name, verbose=False) tmp_tmp = cube_crop_frames(tmp, final_sz, xy=self.agpm_pos) write_fits(self.outpath + '2_crop_' + fits_name, tmp_tmp) if not debug: os.system("rm " + self.outpath + '2_nan_corr_' + fits_name) for sk, fits_name in enumerate(sky_list): tmp = open_fits(self.outpath + '2_nan_corr_' + fits_name, verbose=False) tmp_tmp = cube_crop_frames(tmp, final_sz, xy=self.agpm_pos) write_fits(self.outpath + '2_crop_' + fits_name, tmp_tmp) if not debug: os.system("rm " + self.outpath + '2_nan_corr_' + fits_name) if not debug: tmp = open_fits(self.outpath + '2_crop_' + sci_list[0])[-1] tmp_tmp = open_fits(self.outpath + '2_crop_' + sci_list[-1])[-1] plot_frames(tmp, tmp_tmp) else: # COMPARE BEFORE AND AFTER NAN_CORR + CROP old_tmp = open_fits(self.outpath + '2_ff_' + sci_list[0])[-1] old_tmp_tmp = open_fits(self.outpath + '2_ff_' + sci_list[-1])[-1] tmp = open_fits(self.outpath + '2_crop_' + sci_list[0])[-1] tmp_tmp = open_fits(self.outpath + '2_crop_' + sci_list[-1])[-1] plot_frames(old_tmp, tmp, old_tmp_tmp, tmp_tmp) # Crop the bpix map in a same way bpix_map = open_fits(self.outpath + 'master_bpix_map.fits') bpix_map_2ndcrop = frame_crop(bpix_map, final_sz, cenxy=self.agpm_pos) write_fits(self.outpath + 'master_bpix_map_2ndcrop.fits', bpix_map_2ndcrop) bpix_map = open_fits(self.outpath + 'master_bpix_map_2ndcrop.fits') t0 = time_ini() for sc, fits_name in enumerate(sci_list): tmp = open_fits(self.outpath + '2_crop_' + fits_name, verbose=False) # first with the bp max defined from the flat field (without protecting radius) tmp_tmp = cube_fix_badpix_isolated(tmp, bpm_mask=bpix_map, sigma_clip=7, num_neig=5, size=5, protect_mask=True, radius=9, verbose=False, debug=False) write_fits(self.outpath + '2_bpix_corr_' + fits_name, tmp_tmp, verbose=False) timing(t0) # second, residual hot pixels tmp_tmp, bpm = cube_fix_badpix_isolated(tmp_tmp, bpm_mask=None, sigma_clip=8, num_neig=5, size=5, protect_mask=True, radius=10, verbose=False, debug=False, full_output=True) write_fits(self.outpath + '2_bpix_corr2_' + fits_name, tmp_tmp) write_fits(self.outpath + '2_bpix_corr2_map_' + fits_name, bpm) timing(t0) if not debug: os.system("rm " + self.outpath + '2_crop_' + fits_name) bpix_map = open_fits(self.outpath + 'master_bpix_map_2ndcrop.fits') t0 = time_ini() for sk, fits_name in enumerate(sky_list): tmp = open_fits(self.outpath + '2_crop_' + fits_name, verbose=False) # first with the bp max defined from the flat field (without protecting radius) tmp_tmp = cube_fix_badpix_isolated(tmp, bpm_mask=bpix_map, sigma_clip=7, num_neig=5, size=5, protect_mask=True, radius=9, verbose=False, debug=False) write_fits(self.outpath + '2_bpix_corr_' + fits_name, tmp_tmp, verbose=False) timing(t0) # second, residual hot pixels tmp_tmp, bpm = cube_fix_badpix_isolated(tmp_tmp, bpm_mask=None, sigma_clip=8, num_neig=5, size=5, protect_mask=True, radius=10, verbose=False, debug=False, full_output=True) write_fits(self.outpath + '2_bpix_corr2_' + fits_name, tmp_tmp) write_fits(self.outpath + '2_bpix_corr2_map_' + fits_name, bpm) timing(t0) if not debug: os.system("rm " + self.outpath + '2_crop_' + fits_name) bpix_map_unsat = open_fits(self.outpath + 'master_bpix_map_unsat.fits') t0 = time_ini() for un, fits_name in enumerate(unsat_list): tmp = open_fits(self.outpath + '2_nan_corr_unsat_' + fits_name, verbose=False) # first with the bp max defined from the flat field (without protecting radius) tmp_tmp = cube_fix_badpix_isolated(tmp, bpm_mask=bpix_map_unsat, sigma_clip=7, num_neig=5, size=5, protect_mask=True, radius=9, verbose=False, debug=False) write_fits(self.outpath + '2_bpix_corr_unsat_' + fits_name, tmp_tmp) timing(t0) # second, residual hot pixels tmp_tmp, bpm = cube_fix_badpix_isolated(tmp_tmp, bpm_mask=None, sigma_clip=8, num_neig=5, size=5, protect_mask=True, radius=10, verbose=False, debug=False, full_output=True) write_fits(self.outpath + '2_bpix_corr2_unsat_' + fits_name, tmp_tmp) write_fits(self.outpath + '2_bpix_corr2_map_unsat_' + fits_name, bpm) timing(t0) if not debug: os.system("rm " + self.outpath + '2_nan_corr_unsat_' + fits_name) # FIRST CREATE MASTER CUBE FOR SCI tmp_tmp_tmp = open_fits(self.outpath + '2_bpix_corr2_' + sci_list[0], verbose=False) n_y = tmp_tmp_tmp.shape[1] n_x = tmp_tmp_tmp.shape[2] tmp_tmp_tmp = np.zeros([n_sci, n_y, n_x]) for sc, fits_name in enumerate(sci_list): tmp_tmp_tmp[sc] = open_fits( self.outpath + '2_bpix_corr2_' + fits_name, verbose=False)[int(random.randrange(min(ndit_sci)))] tmp_tmp_tmp = np.median(tmp_tmp_tmp, axis=0) write_fits(self.outpath + 'TMP_2_master_median_SCI.fits', tmp_tmp_tmp) # THEN CREATE MASTER CUBE FOR SKY tmp_tmp_tmp = open_fits(self.outpath + '2_bpix_corr2_' + sky_list[0], verbose=False) n_y = tmp_tmp_tmp.shape[1] n_x = tmp_tmp_tmp.shape[2] tmp_tmp_tmp = np.zeros([n_sky, n_y, n_x]) for sk, fits_name in enumerate(sky_list): tmp_tmp_tmp[sk] = open_fits( self.outpath + '2_bpix_corr2_' + fits_name, verbose=False)[int(random.randrange(min(ndit_sky)))] tmp_tmp_tmp = np.median(tmp_tmp_tmp, axis=0) write_fits(self.outpath + 'TMP_2_master_median_SKY.fits', tmp_tmp_tmp) bpix_map_ori = open_fits(self.outpath + 'master_bpix_map_2ndcrop.fits') bpix_map_sci_0 = open_fits(self.outpath + '2_bpix_corr2_map_' + sci_list[0]) bpix_map_sci_1 = open_fits(self.outpath + '2_bpix_corr2_map_' + sci_list[-1]) bpix_map_sky_0 = open_fits(self.outpath + '2_bpix_corr2_map_' + sky_list[0]) bpix_map_sky_1 = open_fits(self.outpath + '2_bpix_corr2_map_' + sky_list[-1]) bpix_map_unsat_0 = open_fits(self.outpath + '2_bpix_corr2_map_unsat_' + unsat_list[0]) bpix_map_unsat_1 = open_fits(self.outpath + '2_bpix_corr2_map_unsat_' + unsat_list[-1]) plot_frames( bpix_map_ori, bpix_map_sci_0, bpix_map_sci_1, #tmp_tmp_tmp, #bpix_tmp, bpix_map_sky_0, bpix_map_sky_1, #, #tmp_tmp_tmp_tmp, #bpix_tmp_tmp, tmpSCI-tmpSKY bpix_map_unsat_0, bpix_map_unsat_1 #, #tmp_tmp_tmp_tmp, #bpix_tmp_tmp, tmpSCI-tmpSKY ) tmpSCI = open_fits(self.outpath + 'TMP_2_master_median_SCI.fits') tmpSKY = open_fits(self.outpath + 'TMP_2_master_median_SKY.fits') if not debug: tmp_tmp = open_fits(self.outpath + '2_bpix_corr2_' + sci_list[1])[-1] tmp_tmp2 = open_fits(self.outpath + '2_bpix_corr2_' + sci_list[-1])[-1] plot_frames( #tmp, tmp-tmpSKY, #tmp_tmp_tmp, #bpix_tmp, tmp_tmp, tmp_tmp - tmpSKY, #, #tmp_tmp_tmp_tmp, #bpix_tmp_tmp, tmpSCI-tmpSKY #tmp2, tmp2-tmpSKY, #tmp_tmp_tmp, #bpix_tmp, tmp_tmp2, tmp_tmp2 - tmpSKY #, #tmp_tmp_tmp_tmp, #bpix_tmp_tmp, tmpSCI-tmpSKY ) else: # COMPARE BEFORE AND AFTER BPIX CORR (without sky subtr) tmp = open_fits(self.outpath + '2_crop_' + sci_list[-1])[-1] tmp_tmp = open_fits(self.outpath + '2_bpix_corr2_' + sci_list[-1])[-1] tmp2 = open_fits(self.outpath + '2_crop_' + sky_list[-1])[-1] tmp_tmp2 = open_fits(self.outpath + '2_bpix_corr2_' + sky_list[-1])[-1] ( tmp, tmp - tmpSKY, #tmp_tmp_tmp, #bpix_tmp, tmp_tmp, tmp_tmp - tmpSKY, #, #tmp_tmp_tmp_tmp, #bpix_tmp_tmp, tmpSCI-tmpSKY tmp2, tmp2 - tmpSKY, #tmp_tmp_tmp, #bpix_tmp, tmp_tmp2, tmp_tmp2 - tmpSKY #, #tmp_tmp_tmp_tmp, #bpix_tmp_tmp, tmpSCI-tmpSKY )