def set_tf_allow_growth(self): import tensorflow as tf from keras.backend.tensorflow_backend import set_session config = tf.ConfigProto() config.gpu_options.allow_growth = True config.gpu_options.visible_device_list="0" set_session(tf.Session(config=config))
def __init__(self, root, noise): # Only allocate needed memory config = tf.ConfigProto() config.gpu_options.allow_growth=True session = tf.Session(config=config) ktf.set_session(session) self.root = root self.n_filters = 64 self.kernel_size = 3 self.batch_size = 32 self.n_conv_layers = 10 self.input_file_images_validation = "/net/duna/scratch1/aasensio/deepLearning/stars/database/database_validation.h5" f = h5py.File(self.input_file_images_validation, 'r') self.n_validation_orig, self.nx, self.ny, self.n_images = f.get("image").shape f.close() self.batchs_per_epoch_validation = int(self.n_validation_orig / self.batch_size) self.n_validation = self.batchs_per_epoch_validation * self.batch_size print("Original validation set size: {0}".format(self.n_validation_orig)) print(" - Final validation set size: {0}".format(self.n_validation)) print(" - Batch size: {0}".format(self.batch_size)) print(" - Batches per epoch: {0}".format(self.batchs_per_epoch_validation))
def tf_allow_growth(): import tensorflow as tf from keras.backend.tensorflow_backend import set_session tf_config = tf.ConfigProto() tf_config.gpu_options.allow_growth = True sess = tf.Session(config=tf_config) set_session(sess)
def check_session_cores(NUM_CORES): import tensorflow as tf import keras.backend.tensorflow_backend as KTF sess = tf.Session( config=tf.ConfigProto(inter_op_parallelism_threads=int(NUM_CORES), intra_op_parallelism_threads=int(NUM_CORES))) print(KTF) KTF.set_session(sess) print("Setting session to have {} cores".format(NUM_CORES))
def set_tf_allow_growth(): """ Allow TensorFlow to manage VRAM growth """ # pylint: disable=no-member logger.debug("Setting Tensorflow 'allow_growth' option") config = tf.ConfigProto() config.gpu_options.allow_growth = True config.gpu_options.visible_device_list = "0" set_session(tf.Session(config=config)) logger.debug("Set Tensorflow 'allow_growth' option")
def __init__(self, config): tf_config = tf.ConfigProto() tf_config.gpu_options.per_process_gpu_memory_fraction = config.gpu_memory_fraction tf_config.gpu_options.visible_device_list = "0" set_session(tf.Session(config=tf_config)) self.is_deploy = config.is_deploy super(FullyConvolutionalModel, self).__init__(config)
def __init__(self, root, noise, option, lr, lr_multiplier, batch_size, nkernels): # Only allocate needed memory config = tf.ConfigProto() config.gpu_options.allow_growth=True session = tf.Session(config=config) ktf.set_session(session) self.normalizations = np.load('normalization.npy') self.root = root self.option = option self.noise = noise self.n_filters = nkernels self.lr = lr self.lr_multiplier = lr_multiplier self.batch_size = batch_size self.input_training = "/net/viga/scratch1/deepLearning/doppler_imaging/database/training_stars.h5" f = h5py.File(self.input_training, 'r') self.n_training = len(f['modulus']) f.close() self.input_validation = "/net/viga/scratch1/deepLearning/doppler_imaging/database/validation_stars.h5" f = h5py.File(self.input_validation, 'r') self.n_validation = len(f['modulus']) f.close() self.batchs_per_epoch_training = int(self.n_training / self.batch_size) self.batchs_per_epoch_validation = int(self.n_validation / self.batch_size) print("Original training set size: {0}".format(self.n_training)) print(" - Batch size: {0}".format(self.batch_size)) print(" - Batches per epoch: {0}".format(self.batchs_per_epoch_training)) print("Original validation set size: {0}".format(self.n_validation)) print(" - Batch size: {0}".format(self.batch_size)) print(" - Batches per epoch: {0}".format(self.batchs_per_epoch_validation)) if (self.n_training % self.batch_size != 0): print("ALERT! Something wrong with the batch size") # Normalization of alpha, beta, gamma lmax = 3 k = 3.0 l = np.arange(lmax + 1) cl = [] for l in range(lmax+1): for i in range(2*l+1): cl.append(np.sqrt(1.0 / (1.0+(l/1.0)**k))) self.norm_spher_harm = np.array(cl)
def __init__(self, root, depth): # Only allocate needed memory config = tf.ConfigProto() config.gpu_options.allow_growth=True session = tf.Session(config=config) ktf.set_session(session) self.root = root self.nx = 700 self.ny = 700 self.depth = depth
def make_predictions_gpu(conf,shot_list,loader,custom_path=None): loader.set_inference_mode(True) if backend == 'tf' or backend == 'tensorflow': first_time = "tensorflow" not in sys.modules if first_time: import tensorflow as tf os.environ['KERAS_BACKEND'] = 'tensorflow' from keras.backend.tensorflow_backend import set_session config = tf.ConfigProto(device_count={"GPU":1}) set_session(tf.Session(config=config)) else: os.environ['THEANO_FLAGS'] = 'device=gpu,floatX=float32' import theano from keras.utils.generic_utils import Progbar from plasma.models.builder import ModelBuilder specific_builder = ModelBuilder(conf) y_prime = [] y_gold = [] disruptive = [] model = specific_builder.build_model(True) model.compile(optimizer=optimizer_class(),loss=conf['data']['target'].loss) specific_builder.load_model_weights(model,custom_path) model.reset_states() pbar = Progbar(len(shot_list)) shot_sublists = shot_list.sublists(conf['model']['pred_batch_size'],do_shuffle=False,equal_size=True) for (i,shot_sublist) in enumerate(shot_sublists): X,y,shot_lengths,disr = loader.load_as_X_y_pred(shot_sublist) #load data and fit on data y_p = model.predict(X, batch_size=conf['model']['pred_batch_size']) model.reset_states() y_p = loader.batch_output_to_array(y_p) y = loader.batch_output_to_array(y) #cut arrays back y_p = [arr[:shot_lengths[j]] for (j,arr) in enumerate(y_p)] y = [arr[:shot_lengths[j]] for (j,arr) in enumerate(y)] pbar.add(1.0*len(shot_sublist)) loader.verbose=False#True during the first iteration y_prime += y_p y_gold += y disruptive += disr y_prime = y_prime[:len(shot_list)] y_gold = y_gold[:len(shot_list)] disruptive = disruptive[:len(shot_list)] loader.set_inference_mode(False) return y_prime,y_gold,disruptive
def __init__(self, root, noise, option, depth, network_type, activation, lr, lr_multiplier, batch_size, nkernels, n_inputs): # Only allocate needed memory config = tf.ConfigProto() config.gpu_options.allow_growth=True session = tf.Session(config=config) ktf.set_session(session) self.root = root self.option = option self.noise = noise self.depth = depth self.n_filters = nkernels self.network_type = network_type self.activation = activation self.lr = lr self.lr_multiplier = lr_multiplier self.batch_size = batch_size self.n_inputs = n_inputs self.median = np.loadtxt('/net/duna/scratch1/aasensio/deepLearning/mfbd/database/normalizations.txt') # self.input_file_images_training = "/net/duna/scratch1/aasensio/deepLearning/mfbd/database/learning_hypercube.fits" self.input_file_images_training = "/net/duna/scratch1/aasensio/deepLearning/mfbd/database/8542_learning_hypercube.fits" f = fits.open(self.input_file_images_training, memmap=True) self.n_training_orig, self.nx, self.ny, _ = f[0].data.shape f.close() # self.input_file_images_validation = "/net/duna/scratch1/aasensio/deepLearning/mfbd/database/validating_hypercube.fits" self.input_file_images_validation = "/net/duna/scratch1/aasensio/deepLearning/mfbd/database/8542_validation_hypercube.fits" f = fits.open(self.input_file_images_validation, memmap=True) self.n_validation_orig, self.nx, self.ny, _ = f[0].data.shape f.close() self.batchs_per_epoch_training = int(self.n_training_orig / self.batch_size) self.batchs_per_epoch_validation = int(self.n_validation_orig / self.batch_size) self.n_training = self.batchs_per_epoch_training * self.batch_size self.n_validation = self.batchs_per_epoch_validation * self.batch_size print("Original training set size: {0}".format(self.n_training_orig)) print(" - Final training set size: {0}".format(self.n_training)) print(" - Batch size: {0}".format(self.batch_size)) print(" - Batches per epoch: {0}".format(self.batchs_per_epoch_training)) print("Original validation set size: {0}".format(self.n_validation_orig)) print(" - Final validation set size: {0}".format(self.n_validation)) print(" - Batch size: {0}".format(self.batch_size)) print(" - Batches per epoch: {0}".format(self.batchs_per_epoch_validation))
def make_evaluations_gpu(conf,shot_list,loader): loader.set_inference_mode(True) if backend == 'tf' or backend == 'tensorflow': first_time = "tensorflow" not in sys.modules if first_time: import tensorflow as tf os.environ['KERAS_BACKEND'] = 'tensorflow' from keras.backend.tensorflow_backend import set_session config = tf.ConfigProto(device_count={"GPU":1}) set_session(tf.Session(config=config)) else: os.environ['THEANO_FLAGS'] = 'device=gpu,floatX=float32' import theano from keras.utils.generic_utils import Progbar from plasma.models.builder import ModelBuilder specific_builder = ModelBuilder(conf) y_prime = [] y_gold = [] disruptive = [] batch_size = min(len(shot_list),conf['model']['pred_batch_size']) pbar = Progbar(len(shot_list)) print('evaluating {} shots using batchsize {}'.format(len(shot_list),batch_size)) shot_sublists = shot_list.sublists(batch_size,equal_size=False) all_metrics = [] all_weights = [] for (i,shot_sublist) in enumerate(shot_sublists): batch_size = len(shot_sublist) model = specific_builder.build_model(True,custom_batch_size=batch_size) model.compile(optimizer=optimizer_class(),loss=conf['data']['target'].loss) specific_builder.load_model_weights(model) model.reset_states() X,y,shot_lengths,disr = loader.load_as_X_y_pred(shot_sublist,custom_batch_size=batch_size) #load data and fit on data all_metrics.append(model.evaluate(X,y,batch_size=batch_size,verbose=False)) all_weights.append(batch_size) model.reset_states() pbar.add(1.0*len(shot_sublist)) loader.verbose=False#True during the first iteration if len(all_metrics) > 1: print('evaluations all: {}'.format(all_metrics)) loss = np.average(all_metrics,weights = all_weights) print('Evaluation Loss: {}'.format(loss)) loader.set_inference_mode(False) return loss
def __init__(self, networks, depths, filters_exit, activation, n_filters): # Only allocate needed memory config = tf.ConfigProto() config.gpu_options.allow_growth=True session = tf.Session(config=config) ktf.set_session(session) self.networks = networks self.activation = activation self.n_filters = n_filters self.depth = depths self.filters_exit = filters_exit
def __init__(self, root, noise, option, depth, network_type, activation, lr, lr_multiplier, batch_size, nkernels): # Only allocate needed memory config = tf.ConfigProto() config.gpu_options.allow_growth=True session = tf.Session(config=config) ktf.set_session(session) self.root = root self.option = option self.noise = noise self.depth = depth self.n_filters = nkernels self.network_type = network_type self.activation = activation self.lr = lr self.lr_multiplier = lr_multiplier self.batch_size = batch_size tmp = np.loadtxt('/net/vena/scratch/Dropbox/GIT/DeepLearning/hmi_super/training/normalization.txt') self.median_HMI, self.median_SST = tmp[0], tmp[1] self.input_file_images_training = "/net/viga/scratch1/cdiazbas/DATABASE/database_training_x2_PSF2.h5" f = h5py.File(self.input_file_images_training, 'r') self.n_training_orig, self.nx, self.ny, _ = f['imHMI'].shape f.close() self.input_file_images_validation = "/net/viga/scratch1/cdiazbas/DATABASE/database_validation_x2_PSF2.h5" f = h5py.File(self.input_file_images_validation, 'r') self.n_validation_orig, self.nx, self.ny, _ = f['imHMI'].shape f.close() self.batchs_per_epoch_training = int(self.n_training_orig / self.batch_size) self.batchs_per_epoch_validation = int(self.n_validation_orig / self.batch_size) self.n_training = self.batchs_per_epoch_training * self.batch_size self.n_validation = self.batchs_per_epoch_validation * self.batch_size print("Original training set size: {0}".format(self.n_training_orig)) print(" - Final training set size: {0}".format(self.n_training)) print(" - Batch size: {0}".format(self.batch_size)) print(" - Batches per epoch: {0}".format(self.batchs_per_epoch_training)) print("Original validation set size: {0}".format(self.n_validation_orig)) print(" - Final validation set size: {0}".format(self.n_validation)) print(" - Batch size: {0}".format(self.batch_size)) print(" - Batches per epoch: {0}".format(self.batchs_per_epoch_validation))
def test(model_name, iter_num, gpu_id, vol_size=(160,192,224), nf_enc=[16,32,32,32], nf_dec=[32,32,32,32,32,16,16,3]): """ test nf_enc and nf_dec #nf_dec = [32,32,32,32,32,16,16,3] # This needs to be changed. Ideally, we could just call load_model, and we wont have to # specify the # of channels here, but the load_model is not working with the custom loss... """ gpu = '/gpu:' + str(gpu_id) # Anatomical labels we want to evaluate labels = sio.loadmat('../data/labels.mat')['labels'][0] atlas = np.load('../data/atlas_norm.npz') atlas_vol = atlas['vol'] atlas_seg = atlas['seg'] atlas_vol = np.reshape(atlas_vol, (1,)+atlas_vol.shape+(1,)) config = tf.ConfigProto() config.gpu_options.allow_growth = True config.allow_soft_placement = True set_session(tf.Session(config=config)) # load weights of model with tf.device(gpu): net = networks.unet(vol_size, nf_enc, nf_dec) net.load_weights('../models/' + model_name + '/' + str(iter_num) + '.h5') xx = np.arange(vol_size[1]) yy = np.arange(vol_size[0]) zz = np.arange(vol_size[2]) grid = np.rollaxis(np.array(np.meshgrid(xx, yy, zz)), 0, 4) X_vol, X_seg = datagenerators.load_example_by_name('../data/test_vol.npz', '../data/test_seg.npz') with tf.device(gpu): pred = net.predict([X_vol, atlas_vol]) # Warp segments with flow flow = pred[1][0, :, :, :, :] sample = flow+grid sample = np.stack((sample[:, :, :, 1], sample[:, :, :, 0], sample[:, :, :, 2]), 3) warp_seg = interpn((yy, xx, zz), X_seg[0, :, :, :, 0], sample, method='nearest', bounds_error=False, fill_value=0) vals, _ = dice(warp_seg, atlas_seg, labels=labels, nargout=2) print(np.mean(vals), np.std(vals))
def __init__(self, root, depth, model, activation, n_filters): # Only allocate needed memory config = tf.ConfigProto() config.gpu_options.allow_growth=True session = tf.Session(config=config) ktf.set_session(session) self.root = root self.depth = depth self.network_type = model self.activation = activation self.n_filters = n_filters self.input_file_images_training = "/net/duna/scratch1/aasensio/deepLearning/mfbd/database/validating_hypercube.fits"
def __init__(self, root, noise, option): # Only allocate needed memory config = tf.ConfigProto() config.gpu_options.allow_growth=True session = tf.Session(config=config) ktf.set_session(session) self.root = root self.option = option self.n_filters = 64 self.kernel_size = 3 self.batch_size = 32 self.n_conv_layers = 20 self.stride = 1 self.skip_frequency = 2 self.input_file_images_training = "/scratch1/aasensio/deepLearning/opticalFlow/database/database_images.h5" self.input_file_velocity_training = "/scratch1/aasensio/deepLearning/opticalFlow/database/database_velocity.h5" self.input_file_images_validation = "/scratch1/aasensio/deepLearning/opticalFlow/database/database_images_validation.h5" self.input_file_velocity_validation = "/scratch1/aasensio/deepLearning/opticalFlow/database/database_velocity_validation.h5" f = h5py.File(self.input_file_images_training, 'r') self.n_training_orig, self.nx, self.ny, self.n_times = f.get("intensity").shape f.close() f = h5py.File(self.input_file_images_validation, 'r') self.n_validation_orig, _, _, _ = f.get("intensity").shape f.close() self.batchs_per_epoch_training = int(self.n_training_orig / self.batch_size) self.batchs_per_epoch_validation = int(self.n_validation_orig / self.batch_size) self.n_training = self.batchs_per_epoch_training * self.batch_size self.n_validation = self.batchs_per_epoch_validation * self.batch_size print("Original training set size: {0}".format(self.n_training_orig)) print(" - Final training set size: {0}".format(self.n_training)) print(" - Batch size: {0}".format(self.batch_size)) print(" - Batches per epoch: {0}".format(self.batchs_per_epoch_training)) print("Original validation set size: {0}".format(self.n_validation_orig)) print(" - Final validation set size: {0}".format(self.n_validation)) print(" - Batch size: {0}".format(self.batch_size)) print(" - Batches per epoch: {0}".format(self.batchs_per_epoch_validation))
def __init__(self, root, depth, model, activation, n_filters): # Only allocate needed memory config = tf.ConfigProto() config.gpu_options.allow_growth=True session = tf.Session(config=config) ktf.set_session(session) self.root = root self.depth = depth self.network_type = model self.activation = activation self.n_filters = n_filters self.input_file_images = "/net/viga/scratch1/cdiazbas/DATABASE/database_validation_x2.h5" self.input_file_images_HMI = "/net/viga/scratch1/cdiazbas/DATABASE/database_prediction.h5"
def __init__(self, root, noise, option, depth, activation, lr, lr_multiplier, batch_size, nkernels): # Only allocate needed memory config = tf.ConfigProto() config.gpu_options.allow_growth=True session = tf.Session(config=config) ktf.set_session(session) self.root = root self.option = option self.noise = noise self.depth = depth self.n_filters = nkernels self.activation = activation self.lr = lr self.lr_multiplier = lr_multiplier self.batch_size = batch_size self.smooth = 1.0 self.input_file_images_training = "/net/viga/scratch1/deepLearning/mluna_segmentation/database/database.h5" f = h5py.File(self.input_file_images_training, 'r') self.n_training_orig, self.nx, self.ny, _ = f['halpha'].shape f.close() self.input_file_images_validation = "/net/viga/scratch1/deepLearning/mluna_segmentation/database/database_validation.h5" f = h5py.File(self.input_file_images_validation, 'r') self.n_validation_orig, self.nx, self.ny, _ = f['halpha'].shape f.close() self.batchs_per_epoch_training = int(self.n_training_orig / self.batch_size) self.batchs_per_epoch_validation = int(self.n_validation_orig / self.batch_size) self.n_training = self.batchs_per_epoch_training * self.batch_size self.n_validation = self.batchs_per_epoch_validation * self.batch_size print("Original training set size: {0}".format(self.n_training_orig)) print(" - Final training set size: {0}".format(self.n_training)) print(" - Batch size: {0}".format(self.batch_size)) print(" - Batches per epoch: {0}".format(self.batchs_per_epoch_training)) print("Original validation set size: {0}".format(self.n_validation_orig)) print(" - Final validation set size: {0}".format(self.n_validation)) print(" - Batch size: {0}".format(self.batch_size)) print(" - Batches per epoch: {0}".format(self.batchs_per_epoch_validation))
def make_predictions(conf,shot_list,loader): loader.set_inference_mode(True) use_cores = max(1,mp.cpu_count()-2) if backend == 'tf' or backend == 'tensorflow': first_time = "tensorflow" not in sys.modules if first_time: import tensorflow as tf os.environ['KERAS_BACKEND'] = 'tensorflow' from keras.backend.tensorflow_backend import set_session config = tf.ConfigProto(device_count={"CPU":use_cores}) set_session(tf.Session(config=config)) else: os.environ['THEANO_FLAGS'] = 'device=cpu' import theano from plasma.models.builder import ModelBuilder specific_builder = ModelBuilder(conf) y_prime = [] y_gold = [] disruptive = [] model = specific_builder.build_model(True) model.compile(optimizer=optimizer_class(),loss=conf['data']['target'].loss) specific_builder.load_model_weights(model) model_save_path = specific_builder.get_latest_save_path() start_time = time.time() pool = mp.Pool(use_cores) fn = partial(make_single_prediction,builder=specific_builder,loader=loader,model_save_path=model_save_path) print('running in parallel on {} processes'.format(pool._processes)) for (i,(y_p,y,is_disruptive)) in enumerate(pool.imap(fn,shot_list)): print('Shot {}/{}'.format(i,len(shot_list))) sys.stdout.flush() y_prime.append(y_p) y_gold.append(y) disruptive.append(is_disruptive) pool.close() pool.join() print('Finished Predictions in {} seconds'.format(time.time()-start_time)) loader.set_inference_mode(False) return y_prime,y_gold,disruptive
def __init__(self, root, output, name_of_variable): # Only allocate needed memory config = tf.ConfigProto() config.gpu_options.allow_growth=True session = tf.Session(config=config) ktf.set_session(session) self.root = root self.nx = 576 self.ny = 576 self.n_times = 2 self.n_filters = 64 self.batch_size = 1 self.n_conv_layers = 20 self.stride = 1 self.skip_frequency = 2 self.n_frames = 1 self.output = output self.name_of_variable = name_of_variable telescope_radius = 0.5 * 0.965 * u.meter pixel_size = 0.02759 * u.arcsec / u.pixel fov = 1152 * u.pixel lambda0 = 500 * u.nm imax = imax_degradation(telescope_radius, pixel_size, fov) imax.compute_psf(lambda0) res = io.readsav('/net/viga/scratch1/deepLearning/opticalFlow/mancha/c3d_1152_cont4_4bin_012000_continuum.sav')['continuum'] self.images = np.zeros((2,576,576), dtype='float32') # 576 pixels are obtained by resampling 1152 pixels of 0.02759 "/px to 0.0545 "/px for IMaX self.images[0,:,:] = congrid.resample(imax.apply_psf(res[0,:,:]), (576, 576)) self.images[1,:,:] = congrid.resample(imax.apply_psf(res[1,:,:]), (576, 576)) res = io.readsav('/net/viga/scratch1/deepLearning/opticalFlow/mancha/c3d_1152_cont4_4bin_012000.isotau.sav') self.vx = np.zeros((3,576,576), dtype='float32') self.vy = np.zeros((3,576,576), dtype='float32') for i in range(3): self.vx[i,:,:] = congrid.resample(imax.apply_psf(res['vx'][i,:,:]), (576, 576)) self.vy[i,:,:] = congrid.resample(imax.apply_psf(res['vy'][i,:,:]), (576, 576))
def __new__(cls): with cls._lock: # 初期設定(最初の呼び出しだけ) if cls._instance is None: cls.config = tf.ConfigProto() cls.config.gpu_options.per_process_gpu_memory_fraction = 0.45 set_session(tf.Session(config=cls.config)) cls.voc_classes = ['m1', 'm2', 'm3', 'm4', 'm5', 'm6', 'm7', 'm8', 'm9', \ 'p1', 'p2', 'p3', 'p4', 'p5', 'p6', 'p7', 'p8', 'p9', \ 's1', 's2', 's3', 's4', 's5', 's6', 's7', 's8', 's9', \ 'e', 's', 'w', 'n', 'haku', 'hatsu', 'chun'] cls.NUM_CLASSES = len(cls.voc_classes) + 1 cls.input_shape=(512, 512, 3) cls.model = SSD512(cls.input_shape, num_classes=cls.NUM_CLASSES) cls.model.load_weights(c['detection']['weight_path'], by_name=True) cls.bbox_util = BBoxUtility(cls.NUM_CLASSES) cls._instance = super().__new__(cls) return cls._instance
def train(model,save_name, gpu_id, lr, n_iterations, reg_param, model_save_iter): model_dir = '../models/' + save_name if not os.path.isdir(model_dir): os.mkdir(model_dir) gpu = '/gpu:' + str(gpu_id) os.environ["CUDA_VISIBLE_DEVICES"] = str(gpu_id) config = tf.ConfigProto() config.gpu_options.allow_growth = True config.allow_soft_placement = True set_session(tf.Session(config=config)) # UNET filters nf_enc = [16,32,32,32] if(model == 'vm1'): nf_dec = [32,32,32,32,8,8,3] else: nf_dec = [32,32,32,32,32,16,16,3] with tf.device(gpu): model = networks.unet(vol_size, nf_enc, nf_dec) model.compile(optimizer=Adam(lr=lr), loss=[ losses.cc3D(), losses.gradientLoss('l2')], loss_weights=[1.0, reg_param]) # model.load_weights('../models/udrnet2/udrnet1_1/120000.h5') train_example_gen = datagenerators.example_gen(train_vol_names) zero_flow = np.zeros((1, vol_size[0], vol_size[1], vol_size[2], 3)) for step in range(0, n_iterations): X = train_example_gen.__next__()[0] train_loss = model.train_on_batch( [X, atlas_vol], [atlas_vol, zero_flow]) if not isinstance(train_loss, list): train_loss = [train_loss] printLoss(step, 1, train_loss) if(step % model_save_iter == 0): model.save(model_dir + '/' + str(step) + '.h5')
def __init__(self, root): # Only allocate needed memory config = tf.ConfigProto() config.gpu_options.allow_growth=True session = tf.Session(config=config) ktf.set_session(session) self.root = root self.nx = 700 self.ny = 700 self.n_times = 2 self.n_filters = 64 self.batch_size = 1 self.n_conv_layers = 16 self.stride = 1 self.skip_frequency = 2 self.n_diversity = 2 self.input_file_images_validation = "/net/duna/scratch1/aasensio/deepLearning/phaseDiv/database/database_images_validation.h5"
def __init__(self, root, noise, option, depth): # Only allocate needed memory config = tf.ConfigProto() config.gpu_options.allow_growth=True session = tf.Session(config=config) ktf.set_session(session) self.root = root self.option = option self.noise = noise self.depth = depth self.batch_size = 128 self.input_file_images_training = "/net/duna/scratch1/aasensio/deepLearning/phaseDiv/database/database_images.h5" self.input_file_images_validation = "/net/duna/scratch1/aasensio/deepLearning/phaseDiv/database/database_images_validation.h5" f = h5py.File(self.input_file_images_training, 'r') self.n_training_orig, self.nx, self.ny, _ = f.get("intensity").shape f.close() f = h5py.File(self.input_file_images_validation, 'r') self.n_validation_orig, _, _, _ = f.get("intensity").shape f.close() self.batchs_per_epoch_training = int(self.n_training_orig / self.batch_size) self.batchs_per_epoch_validation = int(self.n_validation_orig / self.batch_size) self.n_training = self.batchs_per_epoch_training * self.batch_size self.n_validation = self.batchs_per_epoch_validation * self.batch_size print("Original training set size: {0}".format(self.n_training_orig)) print(" - Final training set size: {0}".format(self.n_training)) print(" - Batch size: {0}".format(self.batch_size)) print(" - Batches per epoch: {0}".format(self.batchs_per_epoch_training)) print("Original validation set size: {0}".format(self.n_validation_orig)) print(" - Final validation set size: {0}".format(self.n_validation)) print(" - Batch size: {0}".format(self.batch_size)) print(" - Batches per epoch: {0}".format(self.batchs_per_epoch_validation))
def __init__(self): # Only allocate needed memory config = tf.ConfigProto() config.gpu_options.allow_growth=True session = tf.Session(config=config) K.set_session(session) self.dropout = 0.4 self.depth = 64 self.batch_size = 32 self.zed = 100 (x_train, self.y_train), (self.x_test, self.y_test) = mnist.load_data(path='/net/vena/scratch/Dropbox/GIT/DeepLearning/GAN/mnist.dat') self.n_train, self.nx, self.ny = x_train.shape self.n_test, self.nx, self.ny = self.x_test.shape self.x_train = np.zeros((self.n_train, self.nx, self.ny, 1), dtype='float32') self.x_train[:,:,:,0] = x_train
def __init__(self): # Only allocate needed memory config = tf.ConfigProto() config.gpu_options.allow_growth=True session = tf.Session(config=config) ktf.set_session(session) self.root = '../training/cnns/test' self.batch_size = 32 self.fraction_training = 0.9 self.noise = 0.0 self.activation = 'relu' self.depth = 5 self.n_kernels = 64 self.n_mixture = 8 self.c = 9 # Number of variables self.l2_reg = 1e-7 self.lower = np.asarray([0.05, -5.0, 5.0, 0.0, 0.0, 0.0, -180.0, 0.0, -180.0]) self.upper = np.asarray([3.0, 5.0, 18.0, 0.5, 1000.0, 180.0, 180.0, 180.0, 180.0])
def __init__(self, root, observations, output, name_of_variable): # Only allocate needed memory config = tf.ConfigProto() config.gpu_options.allow_growth=True session = tf.Session(config=config) ktf.set_session(session) self.root = root self.nx = 800 self.ny = 800 self.n_times = 2 self.n_filters = 64 self.batch_size = 1 self.n_conv_layers = 20 self.stride = 1 self.skip_frequency = 2 self.n_frames = 57 self.observations = observations self.output = output self.name_of_variable = name_of_variable
def __init__(self, root): # Only allocate needed memory config = tf.ConfigProto() config.gpu_options.allow_growth=True session = tf.Session(config=config) ktf.set_session(session) self.root = root self.noise = 0.0 self.batch_size = 16 self.normalizations = np.load('../training/normalization.npy') self.input_validation = "/net/viga/scratch1/deepLearning/doppler_imaging/database/validation_stars.h5" f = h5py.File(self.input_validation, 'r') self.n_validation = len(f['modulus']) f.close() self.batchs_per_epoch_validation = int(self.n_validation / self.batch_size)
def __init__(self, root): # Only allocate needed memory config = tf.ConfigProto() config.gpu_options.allow_growth=True session = tf.Session(config=config) ktf.set_session(session) f = open('{0}_args.json'.format(root), 'r') tmp = json.load(f) f.close() self.noise = float(tmp['noise']) self.depth = int(tmp['depth']) self.n_filters = int(tmp['kernels']) self.activation = tmp['activation'] self.batch_size = int(tmp['batchsize']) self.l2_reg = float(tmp['l2_regularization']) self.root = root self.noise = 0.0 self.batch_size = 16 self.nx = 500 self.ny = 500
def __init__(self, parsed): # Only allocate needed memory config = tf.ConfigProto() config.gpu_options.allow_growth=True session = tf.Session(config=config) ktf.set_session(session) self.root = parsed['output'] self.batch_size = int(parsed['batchsize']) self.fraction_training = float(parsed['train_fraction']) self.noise = float(parsed['noise']) self.activation = parsed['activation'] self.depth = int(parsed['depth']) self.n_kernels = int(parsed['kernels']) self.lr = float(parsed['lr']) self.l2_reg = float(parsed['l2_regularization']) self.lr_multiplier = float(parsed['lr_multiplier']) self.n_classes = int(parsed['classes']) self.c = 9 # Number of variables self.method = parsed['method'] self.lower = np.asarray([0.05, -5.0, 5.0, 0.0, 0.0, 0.0, -180.0, 0.0, -180.0]) self.upper = np.asarray([3.0, 5.0, 18.0, 0.5, 1000.0, 180.0, 180.0, 180.0, 180.0])
import time import tensorflow as tf from keras_frcnn import config from keras import backend as K from keras.layers import Input from keras.models import Model from keras.backend.tensorflow_backend import set_session from keras_frcnn import roi_helpers sys.setrecursionlimit(40000) config = tf.ConfigProto() config.gpu_options.allow_growth = True config.log_device_placement = True sess = tf.Session(config=config) set_session(sess) parser = OptionParser() parser.add_option("-p", "--path", dest="test_path", help="Path to test data.") parser.add_option( "-n", "--num_rois", type="int", dest="num_rois", help="Number of ROIs per iteration. Higher means more memory use.", default=32) parser.add_option( "--config_filename", dest="config_filename", help=
config['system']['CUDA_VISIBLE_DEVICES'] = args.dev exp_name = args.run_name # Set path model_path = os.path.join(config['log']['model_dir'], exp_name) if not os.path.isdir(model_path): os.makedirs(model_path) result_basepath = os.path.join(config['log']['result_dir'], exp_name) if not os.path.isdir(result_basepath): os.makedirs(result_basepath) # Env settings os.environ["CUDA_VISIBLE_DEVICES"] = config['system']['CUDA_VISIBLE_DEVICES'] gpu_options = tf.compat.v1.GPUOptions(allow_growth=True) sess_config = tf.compat.v1.ConfigProto(gpu_options=gpu_options) set_session(tf.compat.v1.Session(config=sess_config)) # Split cases into train, val, test with open(args.json + '.json', 'r') as reader: case = json.loads(reader.read()) ntuh_partition, tcia_partition, msd_partition = case['partition'] tcia_partition['train'] = [] tcia_partition['validation'] = [] msd_partition['train'] = [] msd_partition['validation'] = [] # Get train patches train_X, train_y, train_idx = get_patches( config, [ntuh_partition, tcia_partition, msd_partition], mode='train') train_X = np.array(train_X)
def __init__(self, on_sim): ## to force CPU mode # import os # os.environ['CUDA_VISIBLE_DEVICES'] = '-1' from keras.backend.tensorflow_backend import set_session config = tf.ConfigProto() config.gpu_options.allow_growth = True set_session(tf.Session(config=config)) self.img_height = 600 # 396 #300 # Height of the input images self.img_width = 800 # 634 #480 # Width of the input images self.img_channels = 3 # Number of color channels of the input images intensity_mean = 127.5 # Set this to your preference (maybe `None`). The current settings transform the input pixel values to the interval `[-1,1]`. intensity_range = 127.5 # Set this to your preference (maybe `None`). The current settings transform the input pixel values to the interval `[-1,1]`. n_classes = 3 # 4 # Number of positive classes scales = [ 0.08, 0.16, 0.32, 0.64, 0.96 ] # An explicit list of anchor box scaling factors. If this is passed, it will override `min_scale` and `max_scale`. aspect_ratios = [0.5, 1.0, 2.0] # The list of aspect ratios for the anchor boxes two_boxes_for_ar1 = True # Whether or not you want to generate two anchor boxes for aspect ratio 1 steps = None # In case you'd like to set the step sizes for the anchor box grids manually; not recommended offsets = None # In case you'd like to set the offsets for the anchor box grids manually; not recommended clip_boxes = False # Whether or not to clip the anchor boxes to lie entirely within the image boundaries variances = [ 1.0, 1.0, 1.0, 1.0 ] # The list of variances by which the encoded target coordinates are scaled self.normalize_coords = True # Whether or not the model is supposed to use coordinates relative to the image size K.clear_session() # Clear previous self.model = build_model(image_size=(self.img_height, self.img_width, self.img_channels), n_classes=n_classes, mode='training', l2_regularization=0.0005, scales=scales, aspect_ratios_global=aspect_ratios, aspect_ratios_per_layer=None, two_boxes_for_ar1=two_boxes_for_ar1, steps=steps, offsets=offsets, clip_boxes=clip_boxes, variances=variances, normalize_coords=self.normalize_coords, subtract_mean=intensity_mean, divide_by_stddev=intensity_range) weights_path = 'ssd7_epoch-14_loss-1.0911_val_loss-0.5348.h5' if not on_sim: weights_path = 'ssd7_4_epoch-18_loss-2.8798_val_loss-2.7147.h5' rospy.loginfo("Loading SSD: %s", weights_path) self.model.load_weights(weights_path, by_name=True) global graph graph = tf.get_default_graph() adam = Adam(lr=0.001, beta_1=0.9, beta_2=0.999, epsilon=1e-08, decay=0.0) ssd_loss = SSDLoss(neg_pos_ratio=3, alpha=1.0) self.model.compile(optimizer=adam, loss=ssd_loss.compute_loss) pass
# Now train the NN to predict 2d dt = 1 # Time resolution From Alberto's TCAV axis conversion [um/pixel] dE = 1 # Energy resolution [MeV/pix] x = (np.arange(lpstrain.shape[1]) - np.round(lpstrain.shape[1] / 2)) * dt y = (np.arange(lpstrain.shape[2]) - np.round(lpstrain.shape[2] / 2)) * dE # Plot lps for a few random shots for i in range(1): plot_lps_vs_prediction_lucretia(lpstrain, lpstrain, x, y) #%%# Set up the Tensorflow environment to use only one thread. import tensorflow as tf from keras.backend.tensorflow_backend import set_session session_conf = tf.ConfigProto(intra_op_parallelism_threads=1, inter_op_parallelism_threads=1) set_session(tf.Session(config=session_conf)) # Build the MLP to train model = Sequential() model.add( Dense(500, activation='relu', input_shape=(X_train_scaled.shape[1], ))) model.add(Dense(200, activation='relu')) model.add(Dense(100, activation='relu')) #model.add(LeakyReLU(100,alpha=0.05)) #model.add(Dense(500, activation='relu',input_shape = (X_train_scaled.shape[1],))) #model.add(Dense(500, activation='relu')) #model.add(Dense(200, activation='relu')) #model.add(Dense(100, activation='relu')) model.add(Dense(lpstrainreshaped.shape[1], activation='linear')
def set_tf_allow_growth(): """ Allow TensorFlow to manage VRAM growth """ config = tf.ConfigProto() config.gpu_options.allow_growth = True config.gpu_options.visible_device_list = "0" set_session(tf.Session(config=config))
load = args.load train = args.train mode = args.mode suffix = args.suffix people_id = int(args.test_people_id) # filename prefix prefix = mode #if not train: if True: import tensorflow as tf import keras.backend.tensorflow_backend as KTF config = tf.ConfigProto() config.gpu_options.per_process_gpu_memory_fraction = 0.45 sess = tf.Session(config=config) KTF.set_session(sess) if people_id > 140: filename = 'test' else: filename = 'train' print('Config: Load pretrained<-{}, suffix<-{}, epoch<-{}, lr<-{}'.format( load, suffix, epoch, l_rate)) if mode == 'fusion_dr': # dim of input vector input_dim = 11510 * 9 # dim of per feature feature_dim = 9
import os import numpy as np import cv2 import tensorflow as tf from keras.utils import np_utils from cnn_model_with_keras.darknet19_model import get_model from keras.models import load_model from keras.backend import tensorflow_backend as k config = tf.ConfigProto() config.gpu_options.allow_growth = True sess = tf.Session(config=config) k.set_session(sess) width = 224 height = 224 channels = 3 def read_img(path): img = cv2.imread(path) img = cv2.resize(img, (width, height)) # img = np.array(img) return img # 类别文件夹 folder_list = os.listdir("../flower/") # 统计样本数 number = 0 for folder in folder_list: img_list = os.listdir("../flower/" + folder)
import sys, os sys.path.append(os.pardir) from data_helpers import BPE import numpy as np import pandas as pd import tensorflow as tf from keras.backend.tensorflow_backend import set_session config = tf.ConfigProto() # config.gpu_options.per_process_gpu_memory_fraction = 0.5 config.gpu_options.allow_growth = True # dynamically grow the memory used on the GPU sess = tf.Session(config=config) set_session(sess) # set this TensorFlow session as the default session for Keras. bpe = BPE("../pre-trained-model/en.wiki.bpe.op25000.vocab") # Build vocab, {token: index} vocab = {} for i, token in enumerate(bpe.words): vocab[token] = i + 1 # Load preprocessed data from npz file dataset = np.load('preprocessed_dataset.npz') x_train = dataset['x_train'] y_train = dataset['y_train'] x_test = dataset['x_test'] y_test = dataset['y_test'] from gensim.models import KeyedVectors
def processFile(image): aux = image.split("%") action = int(aux[0]) reward = float(aux[1].split(".")[0].replace("_",".")) return action, reward if __name__ == "__main__": if len(sys.argv) != 5: print("Usage: python3 ", sys.argv[0], " <batches> <n_epochs_per_batch> <frames_per_state> <reset_after>") print() print("Example: python3 ", sys.argv[0]," 15 10 1 50 # run 15 batches (with 10 instances per class in each) using 1 frame per state and reseting after 50 updatings") sys.exit(1) config = tf.ConfigProto(gpu_options=tf.GPUOptions(allow_growth=True)) session = tf.Session(config=config) tensorflow_backend.set_session(session) files = glob.glob("*.png") files.sort() batches = int(sys.argv[1]) n_epochs_per_batch = int(sys.argv[2]) frames_per_state = int(sys.argv[3]) reset_after = int(sys.argv[4]) test_inputs = getTestCNNInputs(files, frames_per_state) models = None for _ in range(batches): for i in range(len(files) - 1): right_images = [] left_images = [] for k in range(frames_per_state): if i + k >= len(files):
import tensorflow as tf from tensorflow import keras as tf_keras from packaging import version as package_version # prevent Keras from using up all gpu memory if tf.executing_eagerly(): gpus = tf.config.experimental.list_physical_devices("GPU") for gpu in gpus: tf.config.experimental.set_memory_growth(gpu, True) else: from keras.backend.tensorflow_backend import set_session config = tf.ConfigProto() config.gpu_options.per_process_gpu_memory_fraction = 0.5 set_session(tf.Session(config=config)) def pytest_generate_tests(metafunc): # This function generates the list of tests for pytest, based # on scenatios that will change the parameters in which the # tests use to run. # https://docs.pytest.org/en/latest/example/parametrize.html idlist = [] argvalues = [] for scenario in metafunc.cls.scenarios: idlist.append(scenario[0]) items = scenario[1].items() argnames = [x[0] for x in items] argvalues.append([x[1] for x in items]) metafunc.parametrize(argnames, argvalues, ids=idlist, scope="class")
def train(data_dir, model, model_dir, gpu_id, lr, nb_epochs, reg_param, steps_per_epoch, load_model_file, data_loss, window_size, batch_size): """ model training function :param data_dir: folder with npz files for each subject. :param model: either vm1 or vm2 (based on CVPR 2018 paper) :param model_dir: the model directory to save to :param gpu_id: integer specifying the gpu to use :param lr: learning rate :param reg_param: the smoothness/reconstruction tradeoff parameter (lambda in CVPR paper) :param steps_per_epoch: frequency with which to save models :param batch_size: Optional, default of 1. can be larger, depends on GPU memory and volume size :param load_model_file: optional h5 model file to initialize with :param data_loss: data_loss: 'mse' or 'ncc' """ vol_size = [256, 256, 144] # (width, height, depth) # set encoder, decoder feature number nf_enc = [16, 32, 32, 32] if model == 'vm1': nf_dec = [32, 32, 32, 32, 8, 8] elif model == 'vm2': nf_dec = [32, 32, 32, 32, 32, 16, 16] else: # 'vm2double': nf_enc = [f * 2 for f in nf_enc] nf_dec = [f * 2 for f in [32, 32, 32, 32, 32, 16, 16]] # set loss function # Mean Squared Error, Cross-Correlation, Negative Cross-Correlation assert data_loss in [ 'mse', 'cc', 'ncc' ], 'Loss should be one of mse or cc, found %s' % data_loss if data_loss in ['ncc', 'cc']: NCC = losses.NCC(win=window_size) data_loss = NCC.loss # prepare model folder if not os.path.isdir(model_dir): os.mkdir(model_dir) # GPU handling gpu = '/gpu:%d' % 0 # gpu_id os.environ["CUDA_VISIBLE_DEVICES"] = gpu_id config = tf.ConfigProto() config.gpu_options.allow_growth = True config.allow_soft_placement = True set_session(tf.Session(config=config)) # prepare the model with tf.device(gpu): # in the CVPR layout, the model takes in [moving image, fixed image] and outputs [warped image, flow] model = networks.cvpr2018_net(vol_size, nf_enc, nf_dec) if load_model_file is not None: model.load_weights(load_model_file) # save first iteration # model.save(os.path.join(model_dir, '%02d.h5' % initial_epoch)) # load data # path = "../../dataset/urinary" # vol_names = [filename for filename in os.listdir(data_dir) if (int(filename.split("_")[-1].split('.')[0]) < 206) and # (int(filename.split("_")[-1].split('.')[0]) not in except_list)] vol_names = [ filename for filename in os.listdir(data_dir) if int(filename.split("_")[-1].split(".")[0]) in normal ] # vol_names = [filename for filename in os.listdir(data_dir) if int(filename.split("_")[-1].split(".")[0]) in (9, 130, 128)] vol_names.sort() uro_gen = uro_generator(vol_names, data_dir, fixed='joyoungje') # test_path = os.path.join(data_dir, 'test') # test_vol_names = [filename for filename in os.listdir(test_path) if '.npz'] # test_gen = uro_generator(test_vol_names, test_path) # fit with tf.device(gpu): mg_model = model # compile mg_model.compile(optimizer=Adam(lr=lr), loss=[data_loss, losses.Grad('l2').loss], loss_weights=[1.0, reg_param]) # fit save_file_name = os.path.join(model_dir, '{epoch:04d}.h5') save_callback = ModelCheckpoint(save_file_name) mg_model.fit_generator(uro_gen, epochs=nb_epochs, verbose=1, callbacks=[save_callback], steps_per_epoch=steps_per_epoch)
monitor='val_acc', verbose=1, save_best_only=True, save_weights_only=False, mode='max', period=1) es_op = EarlyStopping(monitor='val_acc', patience=10, verbose=0, mode='max') def get_session(): tf_config = tf.ConfigProto(allow_soft_placement=True) #tf_config.gpu_options.allocator_type ='BFC' return tf.Session(config=tf_config) KTF.set_session(get_session()) from keras.engine import Input from keras.models import Model def get_model(embedding_matrix, max_length, n_class): inputs = Input(shape=(max_length, )) t_embed = K.placeholder(shape=(4, 300)) embedding_vec = Embedding(input_dim=embedding_matrix.shape[0], output_dim=embedding_matrix.shape[1], weights=[embedding_matrix], trainable=False)(inputs) logist = lea(embedding_vec, t_embed, n_class=n_class, l2_a=0) model = Model(inputs=inputs, outputs=logist)
from config import albert_model_path, config_path, checkpoint_path, dict_path, model_save_path, log_dir, TERM_FREQUENCY_FILE from bert4keras.bert import load_pretrained_model, set_gelu from bert4keras.utils import SimpleTokenizer, load_vocab from neo4j_search import search_entity set_gelu('tanh') # 切换gelu版本 # 禁用GPU os.environ["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID" # see issue #152 os.environ["CUDA_VISIBLE_DEVICES"] = "0,1" # "0,1" tf_config = tf.ConfigProto() tf_config.gpu_options.allocator_type = 'BFC' #A "Best-fit with coalescing" algorithm, simplified from a version of dlmalloc. tf_config.gpu_options.per_process_gpu_memory_fraction = 0.8 tf_config.gpu_options.allow_growth = True set_session(tf.Session(config=tf_config)) # TAG_SCHEME = 'BIO' TAG_SCHEME = 'BIOES' model_save_path = './models_ner_classify_albert_tiny20191101_1356' if TAG_SCHEME == 'BIO': TRAIN_DATA_PATH = "./data/ner_rel_train.txt" DEV_DATA_PATH = "./data/ner_rel_dev.txt" TEST_DATA_PATH = "./data/ner_rel_test.txt" # TRAIN_DATA_PATH = './data/ner_rel0_train_data.txt' # DEV_DATA_PATH = './data/ner_rel0_dev_data.txt' elif TAG_SCHEME == 'BIOES': TRAIN_DATA_PATH = "./data/ner_rel_train_BIOES.txt" DEV_DATA_PATH = "./data/ner_rel_dev_BIOES.txt"
# -*- coding:utf-8 -*- from keras.backend import tensorflow_backend as K import tensorflow as tf config = tf.compat.v1.ConfigProto() config.gpu_options.allow_growth = True K.set_session(tf.compat.v1.Session(config=config)) l2 = tf.keras.regularizers.l2 def part_model(h, num_classes, weight_decay): h = tf.keras.layers.ZeroPadding2D((3, 3))(h) h = tf.keras.layers.Conv2D(filters=32, kernel_size=7, strides=1, padding="valid", use_bias=False, kernel_regularizer=l2(weight_decay))(h) h = tf.keras.layers.BatchNormalization()(h) h = tf.keras.layers.LeakyReLU()(h) h = tf.keras.layers.MaxPooling2D(pool_size=(3, 3), strides=2, padding="same")(h) h = tf.keras.layers.ZeroPadding2D((3, 3))(h) h = tf.keras.layers.Conv2D(filters=64, kernel_size=7, strides=1, padding="valid",
import re from PIL import Image from dataset import data_gen from matplotlib import pyplot as plt import utils import math #from tensorflow.keras.preprocessing.image import ImageDataGenerator from keras import callbacks from tensorflow.python.keras.optimizers import Adam from keras import regularizers from metrics.miou import MeanIoU from keras.models import Model import keras.backend.tensorflow_backend as K cfg = K.tf.ConfigProto(gpu_options={'allow_growth': True}) K.set_session(K.tf.Session(config=cfg)) """ import tensorflow.keras.backend.tensorflow_backend as ktf def get_session(gpu_fraction=0.333): gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=gpu_fraction, allow_growth=True) return tf.Session(config=tf.ConfigProto(gpu_options=gpu_options)) ktf.set_session(get_session()) """ config = tf.ConfigProto(device_count={'GPU': 1, 'CPU': 32}) sess = tf.Session(config=config) keras.backend.set_session(sess) #flags = tf.compat.v1.app.flags
import os import time import keras.backend.tensorflow_backend as K import tensorflow as tf from keras.callbacks import ReduceLROnPlateau, ModelCheckpoint, TensorBoard, CSVLogger from keras.preprocessing.image import ImageDataGenerator import dense_net_121 import hyperparameters os.environ["CUDA_VISIBLE_DEVICES"] = "1" config = tf.ConfigProto() config.gpu_options.allow_growth = True sess = tf.Session(config=config) K.set_session(sess) h_params = hyperparameters.load_hyperparameters() data_train_path = '../MURA_trainval_keras/' data_valid_path = '../MURA_valid1_keras/' pretrained_weights_path = '../pretrain_network_weights/densenet121_weights_tf.h5' timestamp = int(time.time()) model_save_path = '../saved_models_' + str(timestamp) + '/' tensorboard_path = '../tensorboard_' + str(timestamp) + '/' os.makedirs(model_save_path, mode=0o700, exist_ok=True) os.makedirs(tensorboard_path, mode=0o700, exist_ok=True) data_train_generator = ImageDataGenerator( horizontal_flip=True,
def convert(self, video_file, swap_model = False, duration = None, start_time = None, use_gan = False, face_filter = False, photos = True, crop_x = None, width = None, side_by_side = False): # Magic incantation to not have tensorflow blow up with an out of memory error. import tensorflow as tf import keras.backend.tensorflow_backend as K config = tf.ConfigProto() config.gpu_options.allow_growth = True config.gpu_options.visible_device_list="0" K.set_session(tf.Session(config=config)) # Load model model_name = "Original" converter_name = "Masked" if use_gan: model_name = "GAN" converter_name = "GAN" model = PluginLoader.get_model(model_name)(Path(self._model_path(use_gan))) if not model.load(swap_model): print('model Not Found! A valid model must be provided to continue!') exit(1) # Load converter converter = PluginLoader.get_converter(converter_name) converter = converter(model.converter(False), blur_size=8, seamless_clone=True, mask_type="facehullandrect", erosion_kernel_size=None, smooth_mask=True, avg_color_adjust=True) # Load face filter filter_person = self._person_a if swap_model: filter_person = self._person_b filter = FaceFilter(self._people[filter_person]['faces']) # Define conversion method per frame def _convert_frame(frame, convert_colors = True): if convert_colors: frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB) # Swap RGB to BGR to work with OpenCV for face in detect_faces(frame, "cnn"): if (not face_filter) or (face_filter and filter.check(face)): frame = converter.patch_image(frame, face) frame = frame.astype(numpy.float32) if convert_colors: frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB) # Swap RGB to BGR to work with OpenCV return frame def _convert_helper(get_frame, t): return _convert_frame(get_frame(t)) media_path = self._video_path({ 'name' : video_file }) if not photos: # Process video; start loading the video clip video = VideoFileClip(media_path) # If a duration is set, trim clip if duration: video = video.subclip(start_time, start_time + duration) # Resize clip before processing if width: video = video.resize(width = width) # Crop clip if desired if crop_x: video = video.fx(crop, x2 = video.w / 2) # Kick off convert frames for each frame new_video = video.fl(_convert_helper) # Stack clips side by side if side_by_side: def add_caption(caption, clip): text = (TextClip(caption, font='Amiri-regular', color='white', fontsize=80). margin(40). set_duration(clip.duration). on_color(color=(0,0,0), col_opacity=0.6)) return CompositeVideoClip([clip, text]) video = add_caption("Original", video) new_video = add_caption("Swapped", new_video) final_video = clips_array([[video], [new_video]]) else: final_video = new_video # Resize clip after processing #final_video = final_video.resize(width = (480 * 2)) # Write video output_path = os.path.join(self.OUTPUT_PATH, video_file) final_video.write_videofile(output_path, rewrite_audio = True) # Clean up del video del new_video del final_video else: # Process a directory of photos for face_file in os.listdir(media_path): face_path = os.path.join(media_path, face_file) image = cv2.imread(face_path) image = _convert_frame(image, convert_colors = False) cv2.imwrite(os.path.join(self.OUTPUT_PATH, face_file), image)
num_classes = 1 save_model = True epochs = int(config[dataset]["attack_epochs"]) user_epochs = int(config[dataset]["user_epochs"]) attack_epochs = int(config[dataset]["attack_shallow_model_epochs"]) batch_size = int(config[dataset]["defense_batch_size"]) defense_train_testing_ratio = float(config[dataset]["defense_training_ratio"]) result_folder = config[dataset]["result_folder"] network_architecture = str(config[dataset]["network_architecture"]) fccnet = imp.load_source(str(config[dataset]["network_name"]), network_architecture) config_gpu = tf.ConfigProto() config_gpu.gpu_options.per_process_gpu_memory_fraction = 0.5 config_gpu.gpu_options.visible_device_list = "0" set_session(tf.Session(config=config_gpu)) #########loading defense data################### (x_evaluate, y_evaluate, l_evaluate) = input_data.input_data_attacker_evaluate() evaluation_noise_filepath = result_folder + "/attack/" + "noise_data_evaluation.npz" print(evaluation_noise_filepath) if not os.path.isfile(evaluation_noise_filepath): raise FileNotFoundError npz_defense = np.load(evaluation_noise_filepath) f_evaluate_noise = npz_defense['defense_output'] f_evaluate_origin = npz_defense['tc_output'] f_evaluate_defense = np.zeros(f_evaluate_noise.shape, dtype=np.float) np.random.seed(100) #one time randomness, fix the seed for i in np.arange(f_evaluate_defense.shape[0]):
0.1, # randomly shift images horizontally (fraction of total width) height_shift_range= 0.1, # randomly shift images vertically (fraction of total height) horizontal_flip=True, # randomly flip images vertical_flip=True) # randomly flip images #%% ------ CPU/GPU memory fix ------- import tensorflow as tf, keras.backend.tensorflow_backend as ktf def get_session(gpu_fraction=0.25): gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=gpu_fraction, allow_growth=True) return tf.Session(config=tf.ConfigProto(gpu_options=gpu_options)) ktf.set_session(get_session()) import psutil def check_memusage(): process = psutil.Process(os.getpid()) pctused = 100 * process.memory_info().rss / psutil.virtual_memory().total if pctused > 50: print('**** High memory usage detected***') print('Please check your code for memory leaks. Percent memory used: ', pctused) #%% Create demo data def makedata(basepath):
def api_segmentLungAndLesion(dirModelLung, dirModelLesion, series, ptrLogger=None, shape4Lung=(256, 256, 64), shape4Lesi=(512, 512, 256), gpuMemUsage=0.4): # (1) msg-helpers def msgInfo(msg): if ptrLogger is not None: ptrLogger.info(msg) else: print(msg) def msgErr(msg): if ptrLogger is not None: ptrLogger.error(msg) else: print(msg) # (2.1) check data if not series.isInitialized(): msgErr('Series is not initialized, skip .. [{0}]'.format(series)) return False # if not series.isDownloaded(): # msgErr('Series data is not downloaded, skip .. [{0}]'.format(series)) # return False if not series.isConverted(): msgErr( 'Series DICOM data is not converted to Nifti format, skip .. [{0}]' .format(series)) return False # (2.2) check existing files pathNii = series.pathConvertedNifti(isRelative=False) # pathSegmLungs = series.pathPostprocLungs(isRelative=False) pathSegmLungs = series.pathPostprocLungs(isRelative=False) pathSegmLesions = series.pathPostprocLesions2(isRelative=False) if os.path.isfile(pathSegmLungs) and os.path.isfile(pathSegmLesions): msgInfo('Series data is already segmented, skip task ... [{0}]'.format( series)) return False else: # (2.3.0) TF GPU memory usage constraints # FIXME:temporary fix, in future: apped memory usage parameter in application config import tensorflow as tf import keras.backend as K from keras.backend.tensorflow_backend import set_session if K.image_dim_ordering() == 'tf': config = tf.ConfigProto() config.gpu_options.per_process_gpu_memory_fraction = gpuMemUsage set_session(tf.Session(config=config)) # (2.3.1) load and resize try: dataNii = nib.load(pathNii) shapeOrig = dataNii.shape niiResiz4Lung = resizeNii(dataNii, shape4Lung) except Exception as err: msgErr( 'Cant load and resize input nifti file [{0}] : {1}, for series [{2}]' .format(pathNii, err, series)) return False # (2.3.2) segment lungs try: if not os.path.isfile(pathSegmLungs): K.clear_session() lungMask = segmentLungs25D( niiResiz4Lung, dirWithModel=dirModelLung, pathOutNii=None, outSize=shapeOrig, # outSize=shape4Lung, threshold=0.5) nib.save(lungMask, pathSegmLungs) else: pass # lungMask = nib.load(pathSegmLungs) except Exception as err: msgErr('Cant segment lungs for file [{0}] : {1}, for series [{2}]'. format(pathNii, err, series)) return False # (2.3.3) segment lesions try: if not os.path.isfile(pathSegmLesions): # lesionMask = segmentLesions3D(niiResiz4Lesi, # dirWithModel=dirModelLesion, # pathOutNii=None, # outSize=shapeOrig, # # outSize=shape4Lung, # threshold=None) K.clear_session() shape_lesions = [512, 512, 256] nii_lung_resiz4lesion = resizeNii(pathSegmLungs, shape_lesions, parOrder=0) nii_data_resiz4lesion = resizeNii(dataNii, shape_lesions, parOrder=1) lesionMask, lesionMaskVal = segmentLesions3Dv3( nii_data_resiz4lesion, dir_with_model=dirModelLesion, nii_lings=nii_lung_resiz4lesion, path_out_nii=None, # outSize=shapeOrig, # outSize=shape4Lung, threshold=None) # lesionMask = segmentLesions3Dv2(niiResiz4Lesi, # dirWithModel=dirModelLesion, # pathOutNii=None, # outSize=shapeOrig, # # outSize=shape4Lung, # threshold=None, # path_lungs=pathSegmLungs) # (2.3.4) save results try: lesionMask = resizeNii(lesionMask, shapeOrig, parOrder=0) nib.save(lesionMask, pathSegmLesions) except Exception as err: msgErr( 'Cant save segmentation results to file [{0}] : {1}, for series [{2}]' .format(pathSegmLesions, err, series)) return False except Exception as err: msgErr( 'Cant segment lesions for file [{0}] : {1}, for series [{2}]'. format(pathNii, err, series)) return False return True
"embeddings_size": 32, "use_full_tags": True, "callbacks": { "EarlyStopping": { "patience": 5, "monitor": "val_loss" } } } KEYS = ["labels", "feats", "reverse", "bigrams"] if __name__ == "__main__": config = tf.ConfigProto() config.gpu_options.per_process_gpu_memory_fraction = 0.3 kbt.set_session(tf.Session(config=config)) languages = ["belarusian"] corr_dir = os.path.join("conll2018", "task1", "all") use_label, use_bigram_loss, reverse = True, True, True for language in languages: infile = os.path.join(corr_dir, "{}-train-{}".format(language, mode)) data = read_infile(infile, for_lm=True) dev_file = os.path.join(corr_dir, "{}-dev".format(language, mode)) dev_data = read_infile(dev_file, for_lm=True) for (use_bigram_loss, use_feats) in itertools.product([False, True], [False, True]): model = NeuralLM(use_bigram_loss=use_bigram_loss, use_label=use_label, use_feats=use_feats, nepochs=30,
from keras import layers from keras import Model, Input from image_data import * import tensorflow as tf from keras import optimizers from keras.backend.tensorflow_backend import set_session config = tf.ConfigProto( gpu_options=tf.GPUOptions(per_process_gpu_memory_fraction=0.8) # device_count = {'GPU': 1} ) config.gpu_options.allow_growth = True session = tf.Session(config=config) set_session(session) import matplotlib.pyplot as plt # audio net input_image = Input(shape=(224, 224, 3), name='image') image_conv1_1 = layers.Conv2D(64, (3, 3))(input_image) x = layers.BatchNormalization()(image_conv1_1) x = layers.ReLU()(x) x = layers.ZeroPadding2D((1, 1))(x) x = layers.Conv2D(64, (3, 3))(x) x = layers.BatchNormalization()(x) x = layers.ReLU()(x) x = layers.ZeroPadding2D((1, 1))(x) image_pool1 = layers.MaxPooling2D((2, 2), strides=(2, 2))(x) x = layers.Conv2D(128, (3, 3))(image_pool1) x = layers.BatchNormalization()(x) x = layers.ReLU()(x)
PATH = '../' + KEYWORD ################# Parameter Setting ####################### ###########################Reproducible############################# import random np.random.seed(100) random.seed(100) os.environ['PYTHONHASHSEED'] = '0' # necessary for py3 import tensorflow as tf from keras.backend.tensorflow_backend import set_session tf.set_random_seed(100) config = tf.ConfigProto(intra_op_parallelism_threads=1, inter_op_parallelism_threads=1) config.gpu_options.per_process_gpu_memory_fraction = 0.3 config.gpu_options.visible_device_list = '2' set_session(tf.Session(graph=tf.get_default_graph(), config=config)) ################################################################### def main(): if not os.path.exists(PATH): os.makedirs(PATH) currentPython = sys.argv[0] shutil.copy2(currentPython, PATH) shutil.copy2('Param.py', PATH) shutil.copy2('Param_STResNet.py', PATH) data, timestamps = load_data(dataFile_lst, timeFile) data_norm = [x / MAX_FLOWIO for x in data] print(KEYWORD, 'training started', time.ctime()) trainModel(MODELNAME, data_norm, timestamps)
def handle_requests(ctx, socket): # Load SSD model ssd_model = load_SSD_model() # Load trained tensorflow car classifier and set tensorflow configs tf_config = K.tf.ConfigProto() tf_config.gpu_options.per_process_gpu_memory_fraction = CarConfig.crcl[ "classifier_gpu_memory_frac"] K.set_session(K.tf.Session(config=tf_config)) init = K.tf.global_variables_initializer() sess = K.get_session() sess.run(init) # Define a concurrent future and an executor to be used in case plate recognition is enabled future1 = None executor = None is_initialized = False use_plate_recognition = CarConfig.crcl["enable_plate_recognition"] if use_plate_recognition: # Initialize process executor once executor = ProcessPoolExecutor(max_workers=1) # Load model once car_classifier_model, car_classifier_loaded_model_json = load_model_and_json( ) print('Loaded both models successfully, ready to roll.') print('Server is started on:', tcp_address) while True: try: request = socket.recv() image = zmq_comm.decode_request(request) found_objects = [] with isess.as_default(): found_objects = extract_objects(ssd_model, image) clasifications = [] with sess.as_default(): for o in found_objects: # Crop image according to empirical margin values cropped, cropped_wo_margin = crop_image( image, o['topleft'], o['bottomright'], float(o['confidence'])) if cropped is None or cropped_wo_margin is None: continue found_plate = "" predictions = [] filtered_candidates = [] # Run plate recognition in parallel while the main thread continues # Note that, if you call 'future.result()' here, it just waits for process to end # Also notice that, we are sending the original cropped image to plate extraction (a.k.a. no margins) if use_plate_recognition: future1 = executor.submit(extract_plate, cropped_wo_margin, is_initialized) # Preprocess the image cropped = cropped * 1. / 255 cropped = cv2.resize(cropped, (299, 299)) cropped = cropped.reshape((1, ) + cropped.shape) # Feed image to classifier preds = car_classifier_model.predict(cropped)[0] predict_list = classifyIndices( car_classifier_loaded_model_json, preds, CarConfig.crcl["n"]) predictions = [] tags = ["model", "score"] for index, p in enumerate(predict_list): predictions.append( dict(zip(tags, [p.name, str(p.score)]))) # Wait for plate recognition to finish its job if use_plate_recognition and future1 is not None: try: found_plate, is_initialized = future1.result( timeout=CarConfig.crcl["plate_service_timeout"] ) except TimeoutError as e: print( "Could not get any respond from plate service. Timeout." ) cl = { 'label': o['label'], 'confidence': o['confidence'], 'topleft': o['topleft'], 'bottomright': o['bottomright'], 'predictions': predictions, 'plate': found_plate } clasifications.append(cl) result_dict = {} result_dict["result"] = clasifications result_dict["message"] = "OK" # print(result_dict) socket.send_json(result_dict) except Exception as e: result_dict = {} result_dict["result"] = [] result_dict["message"] = str(e) socket.send_json(result_dict) finally: #TODO: Implement this pass
from keras.layers import GRU, Input, Dense from keras import losses from keras.optimizers import Adam from keras.regularizers import l2 from keras.models import load_model #from keras.callbacks import EarlyStopping, ModelCheckpoint import tensorflow as tf import keras.backend.tensorflow_backend as KTF from sklearn.utils import class_weight from sklearn.metrics import classification_report KTF.set_session( tf.Session(config=tf.ConfigProto(device_count={ 'CPU': 1, 'GPU': 1 }))) datapath = '' modelpath = '' ############################# HYPERPARAMETERS ########################################## ntr = 3 nte = 3 nva = 5 #duration of one utterance 'in seconds' for - training, validation and testing files EPOCHS = 40 DROP = 0.2 REC_DROP = 0.2 BATCH_SIZE = 64 #VAL_SPLIT = 0.2
def train(data_dir, atlas_file, model_dir, model_dir_intermediate, gpu_id, lr, nb_epochs, prior_lambda, image_sigma, steps_per_epoch, batch_size, load_model_file, bidir, initial_epoch=0): """ model training function :param data_dir: folder with npz files for each subject. :param atlas_file: atlas filename. So far we support npz file with a 'vol' variable :param model_dir: model folder to save to :param gpu_id: integer specifying the gpu to use :param lr: learning rate :param nb_epochs: number of training iterations :param prior_lambda: the prior_lambda, the scalar in front of the smoothing laplacian, in MICCAI paper :param image_sigma: the image sigma in MICCAI paper :param steps_per_epoch: frequency with which to save models :param batch_size: Optional, default of 1. can be larger, depends on GPU memory and volume size :param load_model_file: optional h5 model file to initialize with :param bidir: logical whether to use bidirectional cost function """ # load atlas from provided files. The atlas we used is 160x192x224. atlas_vol = np.load(atlas_file)['vol'][np.newaxis, ..., np.newaxis] vol_size = atlas_vol.shape[1:-1] # prepare data files # for the CVPR and MICCAI papers, we have data arranged in train/validate/test folders # inside each folder is a /vols/ and a /asegs/ folder with the volumes # and segmentations. All of our papers use npz formated data. train_vol_names = glob.glob(os.path.join(data_dir, '*.npz')) random.shuffle(train_vol_names) # shuffle volume list assert len(train_vol_names) > 0, "Could not find any training data" # Diffeomorphic network architecture used in MICCAI 2018 paper nf_enc = [16, 32, 32, 32] nf_dec = [32, 32, 32, 32, 16, 3] # prepare model folder if not os.path.isdir(model_dir): os.mkdir(model_dir) # gpu handling debug_cpu = False if debug_cpu: # if you memory errors at home and just wanna test this before sending the job to DGX-1 print( '\n==============\nYou are now training with CPU in the debug mode! Was this desired?\n==============\n' ) dev = '/cpu:%d' % 0 # gpu_id os.environ["CUDA_VISIBLE_DEVICES"] = '' nb_gpus = 1 else: print('Training with GPU') dev = '/gpu:%d' % 0 # gpu_id os.environ["CUDA_VISIBLE_DEVICES"] = gpu_id nb_gpus = len(gpu_id.split(',')) assert np.mod(batch_size, nb_gpus) == 0, \ 'batch_size should be a multiple of the nr. of gpus. ' + \ 'Got batch_size %d, %d gpus' % (batch_size, nb_gpus) config = tf.ConfigProto() config.gpu_options.allow_growth = True config.allow_soft_placement = True set_session(tf.Session(config=config)) # prepare the model with tf.device(dev): # the MICCAI201 model takes in [image_1, image_2] and outputs [warped_image_1, velocity_stats] # in these experiments, we use image_2 as atlas model = networks.miccai2018_net(vol_size, nf_enc, nf_dec, bidir=bidir) # load initial weights if load_model_file is not None and load_model_file != "": print('Load initial weights from {}'.format(load_model_file)) model.load_weights(load_model_file) else: print('No pre-trained model given to load initial weights from') # save first iteration model.save(os.path.join(model_dir, '%02d.h5' % initial_epoch)) # compile # note: best to supply vol_shape here than to let tf figure it out. flow_vol_shape = model.outputs[-1].shape[1:-1] loss_class = losses.Miccai2018(image_sigma, prior_lambda, flow_vol_shape=flow_vol_shape) if bidir: model_losses = [ loss_class.recon_loss, loss_class.recon_loss, loss_class.kl_loss ] loss_weights = [0.5, 0.5, 1] else: model_losses = [loss_class.recon_loss, loss_class.kl_loss] loss_weights = [1, 1] # data generator train_example_gen = datagenerators.example_gen(train_vol_names, batch_size=batch_size) atlas_vol_bs = np.repeat(atlas_vol, batch_size, axis=0) miccai2018_gen = datagenerators.miccai2018_gen(train_example_gen, atlas_vol_bs, batch_size=batch_size, bidir=bidir) # prepare callbacks save_file_name = os.path.join(model_dir_intermediate, '{epoch:02d}.h5') # fit generator with tf.device(dev): # multi-gpu support if nb_gpus > 1: save_callback = nrn_gen.ModelCheckpointParallel(save_file_name) mg_model = multi_gpu_model(model, gpus=nb_gpus) # single gpu else: save_callback = ModelCheckpoint(save_file_name) mg_model = model # TODO! Add LR scheduler here model.compile(optimizer=Adam(lr=lr), loss=model_losses, loss_weights=loss_weights) model.fit_generator(miccai2018_gen, initial_epoch=initial_epoch, epochs=nb_epochs, callbacks=[save_callback], steps_per_epoch=steps_per_epoch, verbose=1)
def set_keras_gpu_use(percentage): config = tf.ConfigProto() config.gpu_options.allow_growth = True sess = tf.Session(config=config) KTF.set_session(sess)
from keras.models import load_model from keras.layers import Conv2D, MaxPooling2D, Dropout, Flatten, Dense from keras.callbacks import EarlyStopping, ModelCheckpoint from keras.optimizers import Adagrad from keras.backend import tensorflow_backend as tback from keras.utils import multi_gpu_model FLAGS = None config = tf.ConfigProto() config.gpu_options.allow_growth = True config.allow_soft_placement = True tback.set_session(tf.Session(config = config)) colorindex = np.array( [[0, 0, 0, 255], [0, 0, 255, 0], [0, 255, 0, 0], [255, 0, 0, 0]], dtype = np.float32) datafile = h5py.File('/home/pangaofeng/share/label/testn.mat', 'r'); data = datafile['testxdata']; label = datafile['testdata']; print(label.shape) with h5py.File('oauc.mat', 'r') as oauc_file:
from keras.layers import Conv2D, MaxPooling2D from keras.callbacks import TensorBoard import numpy as np import os import random import cv2 import time # Returns the TF session to be used by the backend. def get_session(gpu_fraction=0.85): gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=gpu_fraction) return tf.Session(config=tf.ConfigProto(gpu_options=gpu_options)) backend.set_session(get_session()) model = Sequential() model.add( Conv2D(32, (7, 7), padding='same', input_shape=(176, 200, 1), activation='relu')) model.add(Conv2D(32, (3, 3), activation='relu')) model.add(MaxPooling2D(pool_size=(2, 2))) model.add(Dropout(0.2)) model.add(Conv2D(64, (3, 3), padding='same', activation='relu')) model.add(Conv2D(64, (3, 3), activation='relu')) model.add(MaxPooling2D(pool_size=(2, 2))) model.add(Dropout(0.2))
import sys import random import keras import tensorflow as tf import keras.backend as K from keras.models import Model from keras.callbacks import EarlyStopping, ModelCheckpoint, ReduceLROnPlateau, TensorBoard from model import build_model from transform import trainAug, valAug from dataloader import DataGenerator from keras.losses import binary_crossentropy import keras.backend.tensorflow_backend as KTF from keras import optimizers KTF.set_session(tf.Session(config=tf.ConfigProto(device_count={'gpu': 0}))) class ViolenceNet: def __init__(self): self.epoch = 150 self.lr = 0.005 self.lr_drop = 5 self.batch_size = 1 self.model = build_model() self.trainLoader = DataGenerator('./data/rgb_train.txt', batch_size=self.batch_size, random_shift=True, transform=trainAug()) self.valLoader = DataGenerator('./data/rgb_val.txt',