def fit_model(self, train_df, valid_df): # callbacks checkpointer = K.callbacks.ModelCheckpoint(filepath=self.model_filename, verbose=1, save_best_only=True) scheduler = K.callbacks.LearningRateScheduler(lambda epoch: self.learning_rate * pow(self.decay_rate, floor(epoch / self.decay_steps))) self.model.fit_generator( DataGenerator( train_df.index, train_df, self.batch_size, self.input_dims, self.train_images_dir ), epochs=self.num_epochs, verbose=self.verbose, validation_data=DataGenerator( valid_df.index, valid_df, self.batch_size, self.input_dims, self.train_images_dir ), use_multiprocessing=True, workers=4, callbacks=[PredictionCheckpoint(), scheduler, checkpointer] )
def train(): from data_trans import process process() from data_loader import train_data, SpoSearcher, dev_data, DataGenerator init_keras_config() train_model, subject_model, object_model = model() EMAer = ExponentialMovingAverage(train_model) EMAer.inject() spoer = SpoSearcher(train_data) train_D = DataGenerator(train_data) evaluator = Evaluate(train_model, EMAer=EMAer, dev_data=dev_data, spoer=spoer, subject_model=subject_model, object_model=object_model) train_model.fit_generator(train_D.__iter__(), steps_per_epoch=len(train_D), epochs=120, callbacks=[evaluator])
def get_stats(data_dir): reward_vals = [] files = DataGenerator.list_np_files(data_dir) for file in files: reward_vals.append(DataGenerator.extract_reward(file)) max_reward = np.max(reward_vals) mean_reward = np.average(reward_vals) return max_reward, mean_reward
def main(): config = args.parse_args() # Load pre-defined config if possible if config.config: config = load_config(config.config) config_str = " | ".join([ "{}={}".format(attr.upper(), value) for attr, value in vars(config).items() ]) print(config_str) # create the experiments dirs config = create_dirs(config) # create tensorflow session device_config = tf.ConfigProto() device_config.gpu_options.allow_growth = True sess = tf.Session(config=device_config) # build preprocessor preprocessor = DynamicPreprocessor(config) # load data, preprocess and generate data data = DataGenerator(preprocessor, config) # create tensorboard summary writer summary_writer = SummaryWriter(sess, config) # create trainer and pass all the previous components to it trainer = Seq2SeqTrainer(sess, preprocessor, data, config, summary_writer) # here you train your model trainer.train()
def main(): # create the experiments dirs create_dirs(config) # create tensorflow session sess = tf.Session() # build preprocessor preprocessor = Preprocessor(config) # load data, preprocess and generate data data = DataGenerator(preprocessor, config) # create an instance of the model you want model = TextCNN.TextCNN(preprocessor, config) # create tensorboard logger logger = Logger(sess, config) # create trainer and pass all the previous components to it trainer = Trainer(sess, model, data, config, logger) # load model if exists model.load(sess) # here you train your model trainer.train()
def _load_pretrained_model(self): base_dir = "/media/scatter/scatterdisk/reply_matching_model/runs/delstm_1024_nsrandom4_lr1e-3/" config_dir = base_dir + "config.json" best_model_dir = base_dir + "best_loss/best_loss.ckpt" model_config = load_config(config_dir) model_config.add_echo = False preprocessor = DynamicPreprocessor(model_config) preprocessor.build_preprocessor() infer_config = load_config(config_dir) setattr(infer_config, "tokenizer", "SentencePieceTokenizer") setattr( infer_config, "soynlp_scores", "/media/scatter/scatterdisk/tokenizer/soynlp_scores.sol.100M.txt") infer_preprocessor = DynamicPreprocessor(infer_config) infer_preprocessor.build_preprocessor() graph = tf.Graph() tf_config = tf.ConfigProto() tf_config.gpu_options.allow_growth = True with graph.as_default(): Model = get_model(model_config.model) data = DataGenerator(preprocessor, model_config) infer_model = Model(data, model_config) infer_sess = tf.Session(config=tf_config, graph=graph) infer_sess.run(tf.global_variables_initializer()) infer_sess.run(tf.local_variables_initializer()) infer_model.load(infer_sess, model_dir=best_model_dir) self.infer_preprocessor = infer_preprocessor return infer_model, infer_sess
def test_dataloader(self): test_set = DataGenerator(self.test_data_path, data_col=self.data_col) loader = torch.utils.data.DataLoader(test_set, batch_size=self.batch_size, shuffle=False, num_workers=6) return loader
def main(): test_df = read_testset(testset_filename) test_generator = DataGenerator(list_IDs=test_df.index, batch_size=batch_size, img_size=img_size, img_dir=test_images_dir) best_model = keras.models.load_model(MODEL_NAME, compile=False) create_submission(best_model, test_generator, test_df)
def train_on_train_test_split(): train_config = get_config() bert_config = get_bert_config(train_config) cased = train_config.BERT_DIR.split('/')[-1].startswith('cased') tokenizer = FullTokenizer(bert_config.vocab, do_lower_case=cased) with tf.device('/cpu:0'): model = get_bert_base_model(bert_config) text, label = load_data(os.path.join(train_config.DATA_DIR, 'train.csv')) train_text, val_text, train_label, val_label = train_test_split( text, label, test_size=0.055, random_state=59) train_gen = DataGenerator(train_text, train_label, tokenizer, batch_size=32) val_text = tokenize_examples(val_text, tokenizer, max_len=512) val_text = seq_padding(val_text) logger = Logger(model=model, val_text=val_text, val_label=(val_label > 0.5).astype(np.float32)) # OPTIMIZER PARAMs lr = 2e-5 weight_decay = 0.01 bsz = 32 decay_steps = 1 * len(train_gen) warmup_steps = int(0.1 * decay_steps) optimizer = AdamWarmup( decay_steps=decay_steps, warmup_steps=warmup_steps, lr=lr, weight_decay=weight_decay, ) parallel_model = multi_gpu_model(model, gpus=4) parallel_model.compile(loss='binary_crossentropy', optimizer=optimizer) parallel_model.fit_generator(train_gen.__iter__(), steps_per_epoch=len(train_gen), epochs=1, callbacks=[logger], max_queue_size=100)
def main(): nb_classes = 40 train_file = './ModelNet40/ply_data_train.h5' test_file = './ModelNet40/ply_data_test.h5' epochs = 100 batch_size = 32 train = DataGenerator(train_file, batch_size, nb_classes, train=True) val = DataGenerator(test_file, batch_size, nb_classes, train=False) model = PointNet(nb_classes) model.summary() lr = 0.0001 adam = Adam(lr=lr) model.compile(optimizer=adam, loss='categorical_crossentropy', metrics=['accuracy']) if not os.path.exists('./results/'): os.mkdir('./results/') checkpoint = ModelCheckpoint('./results/pointnet.h5', monitor='val_acc', save_weights_only=True, save_best_only=True, verbose=1) history = model.fit_generator(train.generator(), steps_per_epoch=9840 // batch_size, epochs=epochs, validation_data=val.generator(), validation_steps=2468 // batch_size, callbacks=[checkpoint, onetenth_50_75(lr)], verbose=1) plot_history(history, './results/') save_history(history, './results/') model.save_weights('./results/pointnet_weights.h5')
def train(): # create dataset dataGenerator = DataGenerator("../../data/train2dae.csv", batch_size) features_input = dataGenerator.getNFeatures() steps_per_epoch = dataGenerator.getSteps() #generator = dataGenerator.generate() m = model.get_model2(features_input, nhidden) decay_rate = learning_rate / NEPOCHS optimizer = optimizers.Adam(lr=learning_rate, decay=1 - 0.995) callbacks = [ ModelCheckpoint(filepath="./best_m", monitor='val_loss', save_best_only=True), EarlyStopping(monitor='val_loss', patience=2) ] m.compile(loss="mean_squared_error", optimizer=optimizer, metrics=["mse"]) m.fit_generator(generator=dataGenerator.generate(), steps_per_epoch=steps_per_epoch, epochs=NEPOCHS, callbacks=callbacks, validation_data=dataGenerator.generate(), validation_steps=steps_per_epoch)
def main(): # capture the config path from the run arguments # then process the json configuration file try: args = get_args() config = process_config(args.config) except: print("missing or invalid arguments") exit(0) # create the experiments dirs create_dirs([ config['result_dir'], config['checkpoint_dir'], config['checkpoint_dir_lstm'] ]) # save the config in a txt file save_config(config) sess_centralized = tf.Session(config=tf.ConfigProto()) data = DataGenerator(config) model_vae = VAEmodel(config, "Centralized") model_vae.load(sess_centralized) trainer_vae = vaeTrainer(sess_centralized, model_vae, data, config) # here you train your model if config['TRAIN_VAE']: if config['vae_epochs_per_comm_round'] > 0: trainer_vae.train() if config['TRAIN_LSTM']: # create a lstm model class instance lstm_model = lstmKerasModel("Centralized", config) # produce the embedding of all sequences for training of lstm model # process the windows in sequence to get their VAE embeddings lstm_model.produce_embeddings(model_vae, data, sess_centralized) # Create a basic model instance lstm_nn_model = lstm_model.lstm_nn_model lstm_nn_model.summary() # Display the model's architecture # checkpoint path checkpoint_path = lstm_model.config['checkpoint_dir_lstm']\ + "cp_{}.ckpt".format(lstm_model.name) # Create a callback that saves the model's weights cp_callback = tf.keras.callbacks.ModelCheckpoint( filepath=checkpoint_path, save_weights_only=True, verbose=1) # load weights if possible # lstm_model.load_model(lstm_nn_model, config, checkpoint_path) # start training if config['lstm_epochs_per_comm_round'] > 0: lstm_model.train(lstm_nn_model, cp_callback) sess_centralized.close()
def main(): nb_classes = 40 train_file = './ModelNet40/ply_data_train.h5' test_file = './ModelNet40/ply_data_test.h5' # train_num_points = 9840 # test_num_points = 2468 # below variables are for dev propose train_num_points = 1648 test_num_points = 420 epochs = 100 batch_size = 32 train = DataGenerator(train_file, batch_size, nb_classes, train=True) val = DataGenerator(test_file, batch_size, nb_classes, train=False) model = pointnet2(nb_classes) model.summary() lr = 0.0001 adam = Adam(lr=lr) model.compile(optimizer=adam, loss='categorical_crossentropy', metrics=['accuracy']) if not os.path.exists('./results/'): os.mkdir('./results/') last_epoch, last_meta = get_last_status(model) checkpoint = MetaCheckpoint('./results/pointnet.h5', monitor='val_acc', save_weights_only=True, save_best_only=True, verbose=1, meta=last_meta) history = model.fit_generator( train.generator(), steps_per_epoch=train_num_points // batch_size, epochs=epochs, validation_data=val.generator(), validation_steps=test_num_points // batch_size, callbacks=[checkpoint, onetenth_50_75(lr)], verbose=1, initial_epoch=last_epoch + 1) plot_history(history, './results/') save_history(history, './results/') model.save_weights('./results/pointnet_weights.h5')
def predict(model_points, predict_points): print( "Predicting with {} points on a model trained with {} points.".format( predict_points, model_points)) test_file = "./ModelNet40/ply_data_test.h5" test_file = h5py.File(test_file, mode="r") nb_classes = 40 val = DataGenerator(test_file, 32, predict_points, nb_classes, train=False) model = PointNet_cls(nb_classes, predict_points) model.load_weights("./results/pointnet-" + str(model_points) + ".h5") pred = np.argmax(model.predict(val), axis=1) labels = np.squeeze( [test_file["label"][x] for x in range(test_file["label"].shape[0])]) labels = np.array([int(x) for x in labels]) print("Accuracy: {:.5}%\n".format( 100 * sklearn.metrics.accuracy_score(labels[:pred.shape[0]], pred)))
def train(): # create dataset dataGenerator = DataGenerator( "../../data/train_woe.csv" , "../../data/labels_train.csv" , batch_size ) features_input = dataGenerator.getNFeatures() steps_per_epoch = dataGenerator.getSteps() #generator = dataGenerator.generate() m = model.get_model(features_input , nhidden) decay_rate = learning_rate / NEPOCHS optimizer = optimizers.Adam(lr = learning_rate , decay = decay_rate ) callbacks = [EarlyStopping(monitor='val_loss', patience=5), ModelCheckpoint(filepath= "./best_m", monitor='val_loss', save_best_only=True)] m.compile( loss = binary_crossentropy , optimizer = optimizer , metrics = [ binary_crossentropy , auc_roc ] ) m.fit_generator( generator = dataGenerator.generate(), steps_per_epoch = steps_per_epoch , epochs = NEPOCHS , callbacks = callbacks , validation_data = dataGenerator.generate() , validation_steps = steps_per_epoch ) y_train = m.predict( dataGenerator.getData() ) print( y_train )
model = Model(inputs=input_points, outputs=[prediction]) xx = np.random.rand(32,2048, 3) - 0.5 y = model.predict_on_batch(xx) nb_classes = 40 train_file = '/home/changetest/datasets/Modelnet40/ply_data_train.h5' test_file = '/home/changetest/datasets/Modelnet40/ply_data_test.h5' epochs = 100 batch_size = 32 train = DataGenerator(train_file, batch_size, nb_classes, train=True) val = DataGenerator(test_file, batch_size, nb_classes, train=False) model.summary() lr = 0.0001 adam = Adam(lr=lr) model.compile(optimizer=adam, loss='categorical_crossentropy', metrics=['accuracy']) if not os.path.exists('./results/'): os.mkdir('./results/') checkpoint = ModelCheckpoint('./results/pointnet.h5', monitor='val_acc', save_weights_only=True, save_best_only=True,verbose=1) model.fit_generator(train.generator(), steps_per_epoch=9840 // batch_size, epochs=epochs, validation_data=val.generator(), callbacks=[checkpoint, onetenth_50_75(lr)],
def trainer(train_dir_name, eval_dir_name, out_dir_name): ''' Train the model :param train_dir_name: path to training set directory :param eval_dir_name: path to evaluation set directory :param out_dir_name: output path to save model files :return: None ''' if not os.path.exists(out_dir_name): os.makedirs(out_dir_name) train_features, train_texts = load_data(train_dir_name) eval_features, eval_texts = load_data(eval_dir_name) steps_per_epoch = len(train_texts) / BATCH_SIZE print('Image file format is %s' % IMAGE_FILE_FORMAT) print('Keras backend file format is %s' % K.image_data_format()) print('Training images input shape: {}'.format(train_features.shape)) print('Evaluation images input shape: {}'.format(eval_features.shape)) print('Training texts shape: {}'.format(len(train_texts))) print('Evaluation texts input shape: {}'.format(len(eval_texts))) print('Epoch size: {}'.format(EPOCHS)) print('Batch size: {}'.format(BATCH_SIZE)) print('Steps per epoch: {}'.format(int(steps_per_epoch))) print('Kernel Initializer: {}'.format(KERNEL_INIT)) with open(os.path.join(out_dir_name, 'config.txt'), 'w') as fh: with redirect_stdout(fh): print('Image file format is %s' % IMAGE_FILE_FORMAT) print('Keras backend file format is %s' % K.image_data_format()) print('Training images input shape: {}'.format( train_features.shape)) print('Evaluation images input shape: {}'.format( eval_features.shape)) print('Training texts shape: {}'.format(len(train_texts))) print('Evaluation texts input shape: {}'.format(len(eval_texts))) print('Epoch size: {}'.format(EPOCHS)) print('Batch size: {}'.format(BATCH_SIZE)) print('Steps per epoch: {}'.format(int(steps_per_epoch))) print('Kernel Initializer: {}'.format(KERNEL_INIT)) # Prepare tokenizer to create the vocabulary tokenizer = Tokenizer(filters='', split=" ", lower=False) # Create the vocabulary tokenizer.fit_on_texts([load_doc('../data/code.vocab')]) # Initialize data generators for training and validation train_generator = DataGenerator(train_texts, train_features, batch_size=BATCH_SIZE, tokenizer=tokenizer, shuffle=True, image_data_format=IMAGE_FILE_FORMAT) validation_generator = DataGenerator(eval_texts, eval_features, batch_size=BATCH_SIZE, tokenizer=tokenizer, shuffle=True, image_data_format=IMAGE_FILE_FORMAT) # Initialize model model = CodeGeneratorModel(IMAGE_SIZE, out_dir_name, image_file_format=IMAGE_FILE_FORMAT, kernel_initializer=KERNEL_INIT) model.save_model() model.summarize() model.summarize_image_model() model.plot_model() if VALIDATE: model.fit_generator(generator=train_generator, steps_per_epoch=steps_per_epoch, callbacks=generate_callbacks(out_dir_name), validation_data=validation_generator) else: model.fit_generator(generator=train_generator, steps_per_epoch=steps_per_epoch, callbacks=generate_callbacks(out_dir_name))
import pandas as pd import numpy as np from data_loader import DataGenerator from model import unet, update_model, model_conv_4_colors, conv_net from utils import * from keras.models import load_model from keras.callbacks import ModelCheckpoint, EarlyStopping, ReduceLROnPlateau import tqdm import os saved_encoder_net = 'vgg_cells_last.hdf5' saved_detector_net = 'detector_net.hdf5' train_dataset = load_dataset() train_datagen = DataGenerator.create_train(train_dataset, 10) model = conv_net() image = DataGenerator.load_set_from_image(os.path.join(train_path, '000a6c98-bb9b-11e8-b2b9-ac1f6b6435d0')) #model.load_weights(os.path.join(data_path, saved_detector_net)) '''callbacks = [ ReduceLROnPlateau(monitor='val_categorical_accuracy', factor=0.5, patience=5, min_delta=0.005, mode='max', cooldown=3, verbose=1), ModelCheckpoint(filepath=os.path.join(data_path, saved_detector_net), monitor='loss', save_best_only=True, verbose=1) ]''' callbacks = [EarlyStopping(monitor='loss', patience=2), ModelCheckpoint(filepath=os.path.join(data_path, saved_detector_net), monitor='loss', save_best_only=True, verbose=1)]
def main(): # Check command line arguments. #if len(sys.argv) != 2 or sys.argv[1] not in model_names: # print("Must provide name of model.") # print("Options: " + " ".join(model_names)) # exit(0) #model_name = sys.argv[1] # Data preparation. nb_classes = 40 train_file = './ModelNet40/ply_data_train.h5' test_file = './ModelNet40/ply_data_test.h5' # Hyperparameters. number_of_points = 1024 epochs = 100 batch_size = 32 # Data generators for training and validation. train = DataGenerator(train_file, batch_size, number_of_points, nb_classes, train=True) val = DataGenerator(test_file, batch_size, number_of_points, nb_classes, train=False) # Create the model. if model_name == "pointnet": model = create_pointnet(number_of_points, nb_classes) elif model_name == "gapnet": model = GAPNet() model.summary() # Ensure output paths. output_path = "logs" if not os.path.exists(output_path): os.mkdir(output_path) output_path = os.path.join(output_path, model_name) if not os.path.exists(output_path): os.mkdir(output_path) output_path = os.path.join(output_path, training_name) if os.path.exists(output_path): shutil.rmtree(output_path) os.mkdir(output_path) # Compile the model. lr = 0.0001 adam = Adam(lr=lr) model.compile(optimizer=adam, loss='categorical_crossentropy', metrics=['accuracy']) # Checkpoint callback. checkpoint = ModelCheckpoint(os.path.join(output_path, "model.h5"), monitor="val_acc", save_weights_only=True, save_best_only=True, verbose=1) # Logging training progress with tensorboard. tensorboard_callback = tf.keras.callbacks.TensorBoard( log_dir=output_path, histogram_freq=0, batch_size=32, write_graph=True, write_grads=False, write_images=True, embeddings_freq=0, embeddings_layer_names=None, embeddings_metadata=None, embeddings_data=None, update_freq="epoch") callbacks = [] #callbacks.append(checkpoint) callbacks.append(onetenth_50_75(lr)) callbacks.append(tensorboard_callback) # Train the model. history = model.fit_generator(train.generator(), steps_per_epoch=9840 // batch_size, epochs=epochs, validation_data=val.generator(), validation_steps=2468 // batch_size, callbacks=callbacks, verbose=1) # Save history and model. plot_history(history, output_path) save_history(history, output_path) model.save_weights(os.path.join(output_path, "model_weights.h5"))
prediction = Flatten()(c) ''' model = Model(inputs=input_points, outputs=[out_0, prediction]) xx = np.random.rand(32,2048, 3) - 0.5 y = model.predict_on_batch(xx) ''' model = Model(inputs=input_points, outputs=[prediction]) nb_classes = 40 train_file = '/home/changetest/datasets/Modelnet40/ply_data_train.h5' test_file = '/home/changetest/datasets/Modelnet40/ply_data_test.h5' epochs = 100 batch_size = 32 train = DataGenerator(train_file, batch_size, nb_classes, train=True) val = DataGenerator(test_file, batch_size, nb_classes, train=False) model.summary() lr = 0.0001 adam = Adam(lr=lr) model.compile(optimizer=adam, loss='categorical_crossentropy', metrics=['accuracy']) if not os.path.exists('./results/'): os.mkdir('./results/') checkpoint = ModelCheckpoint('./results/pointnet.h5', monitor='val_acc', save_weights_only=True, save_best_only=True,
import os os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2' import time import numpy as np import tensorflow as tf from tensorflow import keras from tensorflow.keras.layers import Conv2D, Conv2DTranspose, MaxPooling2D, Flatten, Dense, Dropout, BatchNormalization, concatenate import matplotlib import matplotlib.pyplot as plt # Generate Dataset from data_loader import DataGenerator data = DataGenerator() data.print() x_train = data.x_train x_test = data.x_test y_train = data.y_train y_test = data.y_test # Training Parameters num_epochs = 10 display_step = 1 batch_size = 4 # Network Parameters WIDTH = data.WIDTH HEIGHT = data.HEIGHT CHANNELS = data.CHANNELS
index = config.cls_classes - i * 50 s = s[0:30 * index, :] l = l[0:30 * index, :] x_valid.extend(s) y_valid.extend(l) del s x_valid = np.array(x_valid) y_valid = np.array(y_valid) print("x_valid: {} | {:.2f} ~ {:.2f}".format(x_valid.shape, np.min(x_valid), np.max(x_valid))) print("y_valid: {} | {:.2f} ~ {:.2f}".format(y_valid.shape, np.min(y_valid), np.max(y_valid))) training_generator = DataGenerator(x_train, y_train, batch_size=config.batch_size, dim=(config.input_rows, config.input_cols, config.input_deps), nb_classes=config.cls_classes) validation_generator = DataGenerator(x_valid, y_valid, batch_size=config.batch_size, dim=(config.input_rows, config.input_cols, config.input_deps), nb_classes=config.cls_classes) if os.path.exists(os.path.join(config.model_path, config.exp_name + ".txt")): os.remove(os.path.join(config.model_path, config.exp_name + ".txt")) with open(os.path.join(config.model_path, config.exp_name + ".txt"), 'w') as fh: model.summary(positions=[.3, .55, .67, 1.], print_fn=lambda x: fh.write(x + '\n'))
import keras from model_cls import PointNet from data_loader import DataGenerator nb_classes = 40 test_file = './ModelNet40/ply_data_test.h5' epochs = 100 batch_size = 32 model = PointNet(nb_classes) val = DataGenerator(test_file, batch_size, nb_classes, train=False) model.load_weights("./results/pointnet.h5") loss, acc = model.evaluate_generator(val.generator(), verbose=1) print("loss is {}, and acc is {}".format(loss, acc))
''' from __future__ import absolute_import, division, print_function import os os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2' import numpy as np import tensorflow as tf from tensorflow import keras from tensorflow.keras.layers import Conv2D, Conv2DTranspose, MaxPooling2D, Flatten, Dense, Dropout, BatchNormalization, concatenate import matplotlib import matplotlib.pyplot as plt # Generate Dataset from data_loader import DataGenerator data = DataGenerator() data.print() # Training Parameters num_epochs = 10 display_step = 1 batch_size = 4 # Network Parameters WIDTH = data.WIDTH HEIGHT = data.HEIGHT CHANNELS = data.CHANNELS NUM_INPUTS = WIDTH * HEIGHT * CHANNELS NUM_OUTPUTS = 2
learning_rate=5e-4, num_epochs=num_epochs, decay_rate=0.8, decay_steps=1, weights="imagenet", verbose=1, train_image_dir=train_images_dir, model_filename=model_filename) model.load("epoch2.hdf5") #model.load(model_filename) # Use previous checkpoint if (TRAINING == True): df = read_trainset(trainset_filename) ss = ShuffleSplit(n_splits=10, test_size=0.1, random_state=816).split(df.index) # lets go for the first fold only train_idx, valid_idx = next(ss) # Train the model model.fit_model(df.iloc[train_idx], df.iloc[valid_idx]) test_df = read_testset(testset_filename) test_generator = DataGenerator(test_df.index, None, 1, img_shape, test_images_dir) best_model = K.models.load_model(model.model_filename, compile=False) prediction_df = create_submission(best_model, test_generator, test_df)
infer_config = load_config(config_dir) setattr(infer_config, "tokenizer", TOKENIZER) setattr(infer_config, "soynlp_scores", "/media/scatter/scatterdisk/tokenizer/soynlp_scores.sol.100M.txt") infer_preprocessor = DynamicPreprocessor(infer_config) infer_preprocessor.build_preprocessor() model_config.add_echo = False graph = tf.Graph() tf_config = tf.ConfigProto() tf_config.gpu_options.allow_growth = True with graph.as_default(): Model = get_model(model_config.model) data = DataGenerator(preprocessor, model_config) infer_model = Model(data, model_config) infer_sess = tf.Session(config=tf_config, graph=graph) infer_sess.run(tf.global_variables_initializer()) infer_sess.run(tf.local_variables_initializer()) infer_model.load(infer_sess, model_dir=best_model_dir) with open("../reply_matching_model/data/reply_set_new.txt", "r") as f: reply_set = [line.strip() for line in f if line] indexed_reply_set, reply_set_lengths = zip( *[infer_preprocessor.preprocess(r) for r in reply_set]) def get_result(query, reply): preprocessed_query, query_length = infer_preprocessor.preprocess(query) preprocessed_reply, reply_length = infer_preprocessor.preprocess(reply)
'batch_size': config.batch_size, 'n_classes': 2, 'n_channels': 3, 'shuffle': False, 'aug': False } datalist = pd.read_csv('dataset/train/train.txt', header=None, delimiter=' ') X_train, X_test, y_train, y_test = train_test_split(datalist[0].values, datalist[1].values, stratify=datalist[1], test_size=0.2) train_generator = DataGenerator(X_train, y_train, **train_params) val_generator = DataGenerator(X_test, y_test, **val_params) print("Training size =", len(train_generator)) print("Validation size =", len(val_generator)) # models # add a global spatial average pooling layer base_model = get_base_model() x = base_model.output x = GlobalAveragePooling2D()(x) # let's add a fully-connected layer x = Dense(1024, activation='relu')(x) # and a logistic layer -- let's say we have 2 classes predictions = Dense(2, activation='softmax')(x)
Tensorflow Code for a color segmentation network ''' from __future__ import absolute_import from __future__ import division from __future__ import print_function import tensorflow as tf import matplotlib import matplotlib.pyplot as plt import os os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2' # Import Dataset from data_loader import DataGenerator data = DataGenerator() # Import Models from .model import unet # Training Parameters learning_rate = 0.0001 num_steps = 1000 batch_size = 32 display_step = 100 # Network Parameters WIDTH = 256 HEIGHT = 256 CHANNELS = 2 NUM_INPUTS = WIDTH * HEIGHT * CHANNELS
def get_available_gpus(): local_device_protos = device_lib.list_local_devices() return [x.name for x in local_device_protos if x.device_type == 'GPU'] print(get_available_gpus()) # load VAE model config = process_config('PX4_config.json') # create the experiments dirs create_dirs([config['result_dir'], config['checkpoint_dir']]) # create tensorflow session sess = tf.Session(config=tf.ConfigProto(log_device_placement=True)) # create your data generator data = DataGenerator(config) # create a CNN model model_vae = VAEmodel(config) # create a CNN model trainer_vae = vaeTrainer(sess, model_vae, data, config) model_vae.load(sess) # here you train your model if config['TRAIN_VAE']: if config['num_epochs_vae'] > 0: trainer_vae.train() # load LSTM model lstm_model = lstmKerasModel(data) lstm_model.produce_embeddings(config, model_vae, data, sess) lstm_nn_model = lstm_model.create_lstm_model(config)
Tensorflow Code for a color segmentation network ''' from __future__ import absolute_import from __future__ import division from __future__ import print_function import tensorflow as tf import matplotlib import matplotlib.pyplot as plt import os os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2' # Import Dataset from data_loader import DataGenerator data = DataGenerator() data.print() # Training Parameters learning_rate = 0.0001 num_steps = 10000 batch_size = 32 display_step = 100 # Network Parameters WIDTH = 128; HEIGHT = 128; CHANNELS = 3 NUM_INPUTS = WIDTH * HEIGHT * CHANNELS NUM_OUTPUTS = 2 # Network Varibles and placeholders X = tf.placeholder(tf.float32, [None, HEIGHT, WIDTH, CHANNELS]) # Input