def train(create_model, X, y, batch_size, epochs, gpu_count, parameter_server, method): if gpu_count > 1: ps_device = '/gpu:0' if parameter_server == 'gpu' else '/cpu:0' with tf.device(ps_device): serial_model = create_model() if method == 'kuza55': from keras_tf_multigpu.kuza55 import make_parallel model = make_parallel(serial_model, gpu_count=gpu_count, ps_device=ps_device) elif method == 'avolkov1': from keras_tf_multigpu.avolkov1 import make_parallel, get_available_gpus gpus_list = get_available_gpus(gpu_count) model = make_parallel(serial_model, gdev_list=gpus_list, ps_device=ps_device) elif method == 'fchollet': # requires Keras (2.0.9?) https://github.com/fchollet/keras/commit/3dd3e8331677e68e7dec6ed4a1cbf16b7ef19f7f from keras.utils import multi_gpu_model model = multi_gpu_model(serial_model, gpus=gpu_count) else: model = serial_model = create_model() print('Number of parameters:', serial_model.count_params()) model.compile(loss='categorical_crossentropy', optimizer='sgd', metrics=['accuracy']) gauge = SamplesPerSec(batch_size) model.fit(X, y, batch_size=batch_size, epochs=epochs, callbacks=[gauge]) gauge.print_results()
def main(argv=None): ''' ''' main.__doc__ = __doc__ argv = sys.argv if argv is None else sys.argv.extend(argv) desc = main.__doc__ # .format(os.path.basename(__file__)) # CLI parser args = parser_(desc) mgpu = 0 if getattr(args, 'mgpu', None) is None else args.mgpu checkpt = getattr(args, 'checkpt', None) checkpt_flag = False if checkpt is None else True filepath = checkpt # print('CHECKPT:', checkpt) gdev_list = get_available_gpus(mgpu or 1) ngpus = len(gdev_list) batch_size_1gpu = 32 batch_size = batch_size_1gpu * ngpus num_classes = 1000 epochs = args.epochs data_augmentation = args.aug logdevp = args.logdevp datadir = getattr(args, 'datadir', None) # The data, shuffled and split between train and test sets: (x_train, y_train), (x_test, y_test) = synthesize_imagenet_dataset(num_classes) train_samples = x_train.shape[0] test_samples = y_test.shape[0] steps_per_epoch = train_samples // batch_size print('train_samples:', train_samples) print('batch_size:', batch_size) print('steps_per_epoch:', steps_per_epoch) # validations_steps = test_samples // batch_size print(x_train.shape[0], 'train samples') print(x_test.shape[0], 'test samples') # The capacity variable controls the maximum queue size # allowed when prefetching data for training. capacity = 10000 # min_after_dequeue is the minimum number elements in the queue # after a dequeue, which ensures sufficient mixing of elements. # min_after_dequeue = 3000 # If `enqueue_many` is `False`, `tensors` is assumed to represent a # single example. An input tensor with shape `[x, y, z]` will be output # as a tensor with shape `[batch_size, x, y, z]`. # # If `enqueue_many` is `True`, `tensors` is assumed to represent a # batch of examples, where the first dimension is indexed by example, # and all members of `tensors` should have the same size in the # first dimension. If an input tensor has shape `[*, x, y, z]`, the # output will have shape `[batch_size, x, y, z]`. # enqueue_many = True # Force input pipeline to CPU:0 to avoid data operations ending up on GPU # and resulting in a slow down for multigpu case due to comm overhead. with tf.device('/cpu:0'): # if no augmentation can go directly from numpy arrays # x_train_batch, y_train_batch = tf.train.shuffle_batch( # tensors=[x_train, y_train], # # tensors=[x_train, y_train.astype(np.int32)], # batch_size=batch_size, # capacity=capacity, # min_after_dequeue=min_after_dequeue, # enqueue_many=enqueue_many, # num_threads=8) # NOTE: This bakes the whole dataset into the TF graph and for larger # datasets it fails on "ValueError: GraphDef cannot be larger than 2GB". # TODO: Load the a large dataset via queue from RAM/disk. input_images = tf.constant(x_train.reshape(train_samples, -1)) print('train_samples', train_samples) print('input_images', input_images.shape) image, label = tf.train.slice_input_producer([input_images, y_train], shuffle=True) # If using num_epochs=epochs have to: # sess.run(tf.local_variables_initializer()) # and maybe also: sess.run(tf.global_variables_initializer()) image = tf.reshape(image, x_train.shape[1:]) print('image', image.shape) test_images = tf.constant(x_test.reshape(test_samples, -1)) test_image, test_label = tf.train.slice_input_producer( [test_images, y_test], shuffle=False) test_image = tf.reshape(test_image, x_train.shape[1:]) if data_augmentation: print('Using real-time data augmentation.') # Randomly flip the image horizontally. distorted_image = tf.image.random_flip_left_right(image) # Because these operations are not commutative, consider # randomizing the order their operation. # NOTE: since per_image_standardization zeros the mean and # makes the stddev unit, this likely has no effect see # tensorflow#1458. distorted_image = tf.image.random_brightness(distorted_image, max_delta=63) distorted_image = tf.image.random_contrast(distorted_image, lower=0.2, upper=1.8) # Subtract off the mean and divide by the variance of the # pixels. image = tf.image.per_image_standardization(distorted_image) # Do this for testing as well if standardizing test_image = tf.image.per_image_standardization(test_image) # Use tf.train.batch if slice_input_producer shuffle=True, # otherwise use tf.train.shuffle_batch. Not sure which way is faster. x_train_batch, y_train_batch = tf.train.batch([image, label], batch_size=batch_size, capacity=capacity, num_threads=8) print('x_train_batch:', x_train_batch.shape) # x_train_batch, y_train_batch = tf.train.shuffle_batch( # tensors=[image, label], # batch_size=batch_size, # capacity=capacity, # min_after_dequeue=min_after_dequeue, # num_threads=8) x_test_batch, y_test_batch = tf.train.batch( [test_image, test_label], # TODO: shouldn't it be: batch_size=batch_size??? batch_size=train_samples, capacity=capacity, num_threads=8, name='test_batch', shared_name='test_batch') x_train_input = KL.Input(tensor=x_train_batch) print('x_train_input', x_train_input) gauge = SamplesPerSec(batch_size) callbacks = [gauge] if _DEVPROF or logdevp: # or True: # Setup Keras session using Tensorflow config = tf.ConfigProto(allow_soft_placement=True, log_device_placement=True) # config.gpu_options.allow_growth = True tfsess = tf.Session(config=config) KB.set_session(tfsess) model_init = make_model(x_train_input, num_classes) x_train_out = model_init.output # model_init.summary() lr = 0.0001 * ngpus if ngpus > 1: model = make_parallel(model_init, gdev_list) else: # Must re-instantiate model per API below otherwise doesn't work. model_init = Model(inputs=[x_train_input], outputs=[x_train_out]) model = model_init opt = RMSprop(lr=lr, decay=1e-6) # Let's train the model using RMSprop model.compile(loss='categorical_crossentropy', optimizer=opt, metrics=['accuracy'], target_tensors=[y_train_batch]) print_mgpu_modelsummary(model) # will print non-mgpu model as well if checkpt_flag: checkpoint = ModelCheckpoint(filepath, monitor='acc', verbose=1, save_best_only=True) callbacks = [checkpoint] # Start the queue runners. sess = KB.get_session() # sess.run([tf.local_variables_initializer(), # tf.global_variables_initializer()]) tf.train.start_queue_runners(sess=sess) # Fit the model using data from the TFRecord data tensors. coord = tf.train.Coordinator() threads = tf.train.start_queue_runners(sess, coord) val_in_train = False # not sure how the validation part works during fit. start_time = time.time() model.fit( # validation_data=(x_test_batch, y_test_batch) # if val_in_train else None, # validation data is not used??? # validation_steps=validations_steps if val_in_train else None, validation_steps=val_in_train, steps_per_epoch=steps_per_epoch, epochs=epochs, callbacks=callbacks) elapsed_time = time.time() - start_time print('[{}] finished in {} ms'.format('TRAINING', int(elapsed_time * 1000))) gauge.print_results() weights_file = checkptfile # './saved_cifar10_wt.h5' if not checkpt_flag: # empty list model.save_weights(checkptfile) # Clean up the TF session. coord.request_stop() coord.join(threads) KB.clear_session() # Second Session. Demonstrate that the model works # test_model = make_model(x_test.shape[1:], num_classes, # weights_file=weights_file) test_model = make_model(x_test.shape[1:], num_classes) test_model.load_weights(weights_file) test_model.compile(loss='categorical_crossentropy', optimizer=opt, metrics=['accuracy']) if data_augmentation: x_proccessed = sess.run(x_test_batch) y_proccessed = sess.run(y_test_batch) loss, acc = test_model.evaluate(x_proccessed, y_proccessed) else: loss, acc = test_model.evaluate(x_test, y_test) print('\nTest loss: {0}'.format(loss)) print('\nTest accuracy: {0}'.format(acc))
def main(): # user options batch_size = 128 val_in_train = False use_model_checkpt = False # demo processing sess = tf.Session() KB.set_session(sess) gdev_list = get_available_gpus() ngpus = len(gdev_list) batch_size = batch_size * ngpus data = mnist.load_mnist() X_train = data.train.images X_test = data.test.images train_samples = X_train.shape[0] # 60000 test_samples = X_test.shape[0] # 10000 height_nrows = 28 width_ncols = 28 batch_shape = [batch_size, height_nrows, width_ncols, 1] epochs = 5 steps_per_epoch = train_samples / batch_size validations_steps = test_samples / batch_size nclasses = 10 # The capacity variable controls the maximum queue size # allowed when prefetching data for training. capacity = 10000 # min_after_dequeue is the minimum number elements in the queue # after a dequeue, which ensures sufficient mixing of elements. min_after_dequeue = 3000 # If `enqueue_many` is `False`, `tensors` is assumed to represent a # single example. An input tensor with shape `[x, y, z]` will be output # as a tensor with shape `[batch_size, x, y, z]`. # # If `enqueue_many` is `True`, `tensors` is assumed to represent a # batch of examples, where the first dimension is indexed by example, # and all members of `tensors` should have the same size in the # first dimension. If an input tensor has shape `[*, x, y, z]`, the # output will have shape `[batch_size, x, y, z]`. enqueue_many = True x_train_batch, y_train_batch = tf.train.shuffle_batch( tensors=[data.train.images, data.train.labels.astype(np.int32)], batch_size=batch_size, capacity=capacity, min_after_dequeue=min_after_dequeue, enqueue_many=enqueue_many, num_threads=8) x_train_batch = tf.cast(x_train_batch, tf.float32) x_train_batch = tf.reshape(x_train_batch, shape=batch_shape) y_train_batch = tf.cast(y_train_batch, tf.int32) y_train_batch = tf.one_hot(y_train_batch, nclasses) x_train_input = Input(tensor=x_train_batch) x_test_batch, y_test_batch = tf.train.batch( tensors=[data.test.images, data.test.labels.astype(np.int32)], batch_size=batch_size, capacity=capacity, enqueue_many=enqueue_many, num_threads=8) # I like the non-functional definition of model more. # model_init = make_model(x_train_input, nclasses) # x_train_out = model_init.output # train_model = Model(inputs=[x_train_input], outputs=[x_train_out]) x_train_out = cnn_layers(x_train_input, nclasses) train_model = Model(inputs=[x_train_input], outputs=[x_train_out]) if ngpus > 1: train_model = make_parallel(train_model, gdev_list) lr = 2e-3 * ngpus train_model.compile(optimizer=RMSprop(lr=lr, decay=1e-5), loss='categorical_crossentropy', metrics=['accuracy'], target_tensors=[y_train_batch]) if ngpus > 1: print_mgpu_modelsummary(train_model) else: train_model.summary() # Callbacks if use_model_checkpt: mon = 'val_acc' if val_in_train else 'acc' checkpoint = ModelCheckpoint( 'saved_wt.h5', monitor=mon, verbose=0, save_best_only=True, save_weights_only=True) checkpoint = [checkpoint] else: checkpoint = [] callbacks = checkpoint # Training slower with callback. Multigpu slower with callback during # training than 1 GPU. Again, mnist is too trivial of a model and dataset # to benchmark or stress GPU compute capabilities. I set up this example # to illustrate potential for speedup of multigpu case trying to use mnist # as a stressor. # It's like comparing a 5 ft race between a person and a truck. A truck is # obviously faster than a person but in a 5 ft race the person will likely # win due to slower startup for the truck. # I will re-implement this with Cifar that should be a better benchmark. # Start the queue runners. tf.train.start_queue_runners(sess=sess) # Fit the model using data from the TFRecord data tensors. coord = tf.train.Coordinator() threads = tf.train.start_queue_runners(sess, coord) start_time = time.time() train_model.fit( validation_data=(x_test_batch, y_test_batch) if val_in_train else None, validation_steps=validations_steps if val_in_train else None, steps_per_epoch=steps_per_epoch, epochs=epochs, callbacks=callbacks) elapsed_time = time.time() - start_time print('[{}] finished in {} ms'.format('TRAINING', int(elapsed_time * 1000))) if not checkpoint: # empty list train_model.save_weights('./saved_wt.h5') # Clean up the TF session. coord.request_stop() coord.join(threads) KB.clear_session() # Second Session. Demonstrate that the model works and is independent of # the TFRecord pipeline, and to test loading trained model without tensors. x_test = np.reshape(data.validation.images, (data.validation.images.shape[0], 28, 28, 1)) y_test = data.validation.labels x_test_inp = KL.Input(shape=(x_test.shape[1:])) test_out = cnn_layers(x_test_inp, nclasses) test_model = Model(inputs=x_test_inp, outputs=test_out) test_model.load_weights('saved_wt.h5') test_model.compile(optimizer='rmsprop', loss='categorical_crossentropy', metrics=['accuracy']) test_model.summary() loss, acc = test_model.evaluate(x_test, to_categorical(y_test), nclasses) print('\nTest loss: {0}'.format(loss)) print('\nTest accuracy: {0}'.format(acc))
model_classifier = Model([img_input, roi_input], classifier) # this is a model that holds both the RPN and the classifier, used to # load/save weights for the models model_all = Model([img_input, roi_input], rpn[:2] + classifier) try: print('loading weights from {}'.format(C.base_net_weights)) model_rpn.load_weights(C.base_net_weights, by_name=True) model_classifier.load_weights(C.base_net_weights, by_name=True) except: print('Could not load pretrained model weights. Weights can be found in the keras application folder \ https://github.com/fchollet/keras/tree/master/keras/applications') # ------------------------------------------------------ User multi GPU support gpus = get_available_gpus() ngpus = len(gpus) print_mgpu_modelsummary(model_rpn) model_rpn = make_parallel(model_rpn, gpus) print_mgpu_modelsummary(model_rpn) optimizer = Adam(lr=1e-5) optimizer_classifier = Adam(lr=1e-5) model_rpn.compile(optimizer=optimizer, loss=[losses.rpn_loss_cls( num_anchors), losses.rpn_loss_regr(num_anchors)]) model_classifier.compile(optimizer=optimizer_classifier, loss=[losses.class_loss_cls, losses.class_loss_regr( len(classes_count) - 1)], metrics={'dense_class_{}'.format(len(classes_count)): 'accuracy'}) model_all.compile(optimizer='sgd', loss='mae') epoch_length = 1000 num_epochs = int(options.num_epochs)
def main(argv=None): ''' ''' main.__doc__ = __doc__ argv = sys.argv if argv is None else sys.argv.extend(argv) desc = main.__doc__ # .format(os.path.basename(__file__)) # CLI parser args = parser_(desc) mgpu = 0 if getattr(args, 'mgpu', None) is None else args.mgpu enqueue = args.enqueue usenccl = args.nccl syncopt = args.syncopt checkpt = getattr(args, 'checkpt', None) checkpt_flag = False if checkpt is None else True filepath = checkpt # print('CHECKPT:', checkpt) batch_size = args.batch_size num_classes = 10 epochs = args.epochs data_augmentation = args.aug logdevp = args.logdevp datadir = getattr(args, 'datadir', None) # The data, shuffled and split between train and test sets: # (x_train, y_train), (x_test, y_test) = cifar10.load_data() (x_train, y_train), (x_test, y_test) = cifar10_load_data(datadir) \ if datadir is not None else cifar10.load_data() print(x_train.shape[0], 'train samples') print(x_test.shape[0], 'test samples') # Convert class vectors to binary class matrices. y_train = to_categorical(y_train, num_classes) y_test = to_categorical(y_test, num_classes) x_train = x_train.astype('float32') x_test = x_test.astype('float32') x_train /= 255 x_test /= 255 callbacks = [] if _DEVPROF or logdevp: # or True: # Setup Keras session using Tensorflow config = tf.ConfigProto(allow_soft_placement=True, log_device_placement=True) # config.gpu_options.allow_growth = True tfsess = tf.Session(config=config) KB.set_session(tfsess) print(x_train.shape, 'train shape') # with tf.device('/cpu:0'): model_init = make_model(x_train.shape, num_classes, filepath if checkpt_flag else None) # model_init = partial(make_model, x_train.shape, num_classes, # filepath if checkpt_flag else None) if checkpt_flag: checkpoint = ModelCheckpoint(filepath, monitor='val_acc', verbose=1, save_best_only=True, mode='max') callbacks += [checkpoint] lr = 0.0001 if mgpu > 1 or mgpu == -1: gpus_list = get_available_gpus(mgpu) ngpus = len(gpus_list) print('Using GPUs: {}'.format(', '.join(gpus_list))) batch_size = batch_size * ngpus # lr = lr * ngpus # batch_size = 40000 # split over four devices works fine no grad avg # batch_size = 25000 # split over four devices works fine w/ grad avg # Data-Parallelize the model via function or class. model = make_parallel(model_init, gpus_list, usenccl=usenccl, syncopt=syncopt, enqueue=enqueue) # model = ModelMGPU(serial_model=model_init, gdev_list=gpus_list, # syncopt=syncopt, usenccl=usenccl, enqueue=enqueue) print_mgpu_modelsummary(model) if not syncopt: opt = RMSprop(lr=lr, decay=1e-6) else: opt = RMSPropMGPU(lr=lr, decay=1e-6, gdev_list=gpus_list) else: model = model_init # batch_size = batch_size * 3 # batch_size = 25000 # exhaust GPU memory. Crashes. print(model.summary()) # initiate RMSprop optimizer opt = RMSprop(lr=lr, decay=1e-6) gauge = SamplesPerSec(batch_size) callbacks += [gauge] # Let's train the model using RMSprop model.compile(loss='categorical_crossentropy', optimizer=opt, metrics=['accuracy']) nsamples = x_train.shape[0] steps_per_epoch = nsamples // batch_size if not data_augmentation: print('Not using data augmentation.') model.fit(x_train, y_train, batch_size=batch_size, epochs=epochs, validation_data=(x_test, y_test), shuffle=True, callbacks=callbacks) # Fit the model on the batches generated by datagen.flow(). # mygen = mygenerator(nsamples, batch_size, x_train, y_train) # model.fit_generator(mygen, # steps_per_epoch=steps_per_epoch, # epochs=epochs, # validation_data=(x_test, y_test), # callbacks=callbacks) else: print('Using real-time data augmentation.') # This will do preprocessing and realtime data augmentation: datagen = ImageDataGenerator( featurewise_center=False, # set input mean to 0 over the dataset samplewise_center=False, # set each sample mean to 0 # divide inputs by std of the dataset featurewise_std_normalization=False, samplewise_std_normalization=False, # divide each input by its std zca_whitening=False, # apply ZCA whitening # randomly rotate images in the range (degrees, 0 to 180) rotation_range=0, # randomly shift images horizontally (fraction of total width) width_shift_range=0.1, # randomly shift images vertically (fraction of total height) height_shift_range=0.1, horizontal_flip=True, # randomly flip images vertical_flip=False) # randomly flip images # Compute quantities required for feature-wise normalization # (std, mean, and principal components if ZCA whitening is applied). datagen.fit(x_train) # Fit the model on the batches generated by datagen.flow(). model.fit_generator(datagen.flow(x_train, y_train, batch_size=batch_size), steps_per_epoch=steps_per_epoch, epochs=epochs, validation_data=(x_test, y_test), callbacks=callbacks) gauge.print_results()