def train_model(data_dir): dr = data_reader.DataReader(data_dir) data, label, _ = dr.parse() data_label = list(zip(data, label)) random.shuffle(data_label) data, label = zip(*data_label) data = numpy.array(data) label = numpy.array(label) rows = len(data) rows_training = int(rows * TRAINING_PERCENT) data_training = data[:rows_training] label_training = label[:rows_training] data_validation = data[rows_training:] label_validation = label[rows_training:] estimator = get_estimator() train_input_fn = tf.estimator.inputs.numpy_input_fn(x={"x": data_training}, y=label_training, shuffle=True, num_epochs=None) train_spec = tf.estimator.TrainSpec(input_fn=train_input_fn, max_steps=5000000) evaluate_input_fn = tf.estimator.inputs.numpy_input_fn( x={"x": data_validation}, y=label_validation, shuffle=False) eval_spec = tf.estimator.EvalSpec(input_fn=evaluate_input_fn, throttle_secs=30) tf.estimator.train_and_evaluate(estimator, train_spec, eval_spec)
def train(config): # load Vocab src_vocab = data_reader.Vocab(vocab_limits=config['src_vocab_size']) src_vocab.load_metadata(config['metadata']['src']) config['src_vocab_size'] = src_vocab.vocab_size() tgt_vocab = data_reader.Vocab(vocab_limits=config['tgt_vocab_size']) tgt_vocab.load_metadata(config['metadata']['tgt']) config['tgt_vocab_size'] = tgt_vocab.vocab_size() tf.logging.info(config) initializer = tf.random_uniform_initializer(-config['init_scale'], config['init_scale']) # create models with tf.name_scope('Train'): opt, lr = optimizer.get_optimizer(config['optimizer'], config['learning_rate']) with tf.variable_scope("Model", reuse=None, initializer=initializer): train_model = model.Model(is_training=True, config=config, seq_length=config['tgt_length'] - 1, optimizer=opt, lr=lr) with tf.name_scope('Test'): with tf.variable_scope("Model", reuse=True): test_model = model.Model(is_training=False, config=config, seq_length=1) sv = tf.train.Supervisor(logdir=config['logdir']) sess_config = tf.ConfigProto(allow_soft_placement=True, log_device_placement=True) sess_config.gpu_options.allow_growth = True sess_config.gpu_options.per_process_gpu_memory_fraction = 0.9 # load Data train_data = data_reader.DataReader( src_data=config['train_data']['src'][0], tgt_data=config['train_data']['tgt'][0], src_vocab=src_vocab, tgt_vocab=tgt_vocab, src_length=config['src_length'], tgt_length=config['tgt_length'], batch_size=config['batch_size']) tf.logging.info('Start Sess') with sv.managed_session(config=sess_config) as sess: for i in range(config['n_epoch']): lr_decay = config['lr_decay']**max(i + 1 - config['decay_epoch'], 0) train_model.assign_lr(sess, config['learning_rate'] * lr_decay) tf.logging.info('Iter %d Start, Learning_rate: %.4f' % (i, sess.run(train_model.lr))) loss = run_epoch(sess, train_model, train_data, is_training = True, \ t_model = test_model, src_vocab = src_vocab, tgt_vocab = tgt_vocab) tf.logging.info('Iter %d: training_loss: %.4f' % (i, np.power(2, loss)))
def main(): # get the parameters datafile = sys.argv[1] neurons = int(sys.argv[2]) learningRate = float(sys.argv[3]) threshold = float(sys.argv[4]) trainPerc = float(sys.argv[5]) seed = int(sys.argv[6]) title = sys.argv[7] # set up the RNG random.seed(seed) outputFile = open( "results_nn_" + title + "_" + sys.argv[2] + "_" + sys.argv[3] + "_" + sys.argv[4] + "_" + sys.argv[5] + "_" + sys.argv[6] + ".txt", "w") # read in the data, we edited parts of this to get rid of columns that were irrelevant reader = data_reader.DataReader(datafile, trainPerc, outputFile) numAttributes, numLabels = reader.readData() possibleLabels = reader.discreteValues[data_reader.DataReader.LABEL_COLUMN] # get the randomized training and test sets train, valid, test = reader.splitData() # train the network runNetwork(train, valid, test, neurons, learningRate, threshold, numAttributes, possibleLabels, outputFile) outputFile.close()
def save_processed_data(path): dr = data_reader.DataReader() dp = data_preprocessor.DataPreprocessor() choices, dynamics, stim_viewing = dr.get_data(path=path + 'merged_raw/', stim_viewing=True) dynamics = dp.preprocess_data(choices, dynamics) stim_viewing = dp.preprocess_data(choices, stim_viewing) choices = dp.get_hand_and_gaze_measures(choices, dynamics, stim_viewing) choices, dynamics, stim_viewing = dp.exclude_trials( choices, dynamics, stim_viewing) processed_path = path + 'processed/' if not os.path.exists(processed_path): os.makedirs(processed_path) choices.to_csv(processed_path + 'choices.txt', sep='\t', na_rep='nan', float_format='%.4f') dynamics.to_csv(processed_path + 'dynamics.txt', sep='\t', na_rep='nan', float_format='%.4f') stim_viewing.to_csv(processed_path + 'stim_viewing.txt', sep='\t', na_rep='nan', float_format='%.4f')
def learnig_pipeline(): # 0. load parameters params = LearningParams() # 1. load the data paths_list = [ 'data/clockwise/', 'data/clockwise_1/', 'data/counterclockwise/', 'data/clockcouterwise_1/', 'data/counterclockwise_recovery/', 'data/clockwise_recovery/' ] reader = dread.DataReader() # compile and train the model using the generator function [train_generator, validation_generator ] = reader.read_using_generator(paths_list, batch_size=params.batch_size, debug=False) input_shape = (reader.image_shape[2], reader.image_shape[0], reader.image_shape[1]) # 2. Create the model clone_model = model.BehavioralCloneModel(reader.image_shape) for layer in clone_model.model.layers: print(layer.output_shape) # 2.2 Compile the model clone_model.model.compile(loss='mse', optimizer='adam') steps_per_epoch = math.ceil(reader.train_samples_size / params.batch_size) validation_steps = math.ceil(reader.validation_samples_size / params.batch_size) save_path = "model_checkpoints/saved-model-{epoch:02d}-{val_loss:.2f}.hdf5" checkpoint = keras.callbacks.ModelCheckpoint(filepath=save_path, monitor='val_loss', save_best_only=True) history_object = clone_model.model.fit_generator( train_generator, steps_per_epoch=steps_per_epoch, callbacks=[checkpoint], validation_data=validation_generator, validation_steps=validation_steps, epochs=params.num_epochs, verbose=1) # saving model clone_model.model.save('model.h5') # 3. Model visualization. Print the keys contained in the history object print(history_object.history.keys()) # 3.1 Plot the training and validation loss for each epoch plt.plot(history_object.history['loss']) plt.plot(history_object.history['val_loss']) plt.title('model mean squared error loss') plt.ylabel('mean squared error loss') plt.xlabel('epoch') plt.legend(['training set', 'validation set'], loc='upper right') plt.show()
def readData(self): print 'Read data manually.' data_start = 0 data_finish = 1000 #'end' model = 'bed' subject = 'sub6_shaver' print 'Starting to convert data!' self.runData = dr.DataReader(subject=subject, data_start=data_start, data_finish=data_finish, model=model)
def main(flags): # initialize data reader if len(flags.tconv_dims) == 0: output_size = flags.fc_filters[-1] else: output_size = flags.tconv_dims[-1] reader = data_reader.DataReader(input_size=flags.input_size, output_size=output_size, x_range=flags.x_range, y_range=flags.y_range, cross_val=flags.cross_val, val_fold=flags.val_fold, batch_size=flags.batch_size, shuffle_size=flags.shuffle_size) features, labels, train_init_op, valid_init_op = reader.get_data_holder_and_init_op( (flags.train_file, flags.valid_file)) # make network ntwk = network_maker.CnnNetwork(features, labels, utils.my_model_fn, flags.batch_size, fc_filters=flags.fc_filters, tconv_dims=flags.tconv_dims, tconv_filters=flags.tconv_filters, learn_rate=flags.learn_rate, decay_step=flags.decay_step, decay_rate=flags.decay_rate) # define hooks for monitoring training train_hook = network_helper.TrainValueHook(flags.verb_step, ntwk.loss, ckpt_dir=ntwk.ckpt_dir, write_summary=True) lr_hook = network_helper.TrainValueHook(flags.verb_step, ntwk.learn_rate, ckpt_dir=ntwk.ckpt_dir, write_summary=True, value_name='learning_rate') valid_hook = network_helper.ValidationHook(flags.eval_step, valid_init_op, ntwk.labels, ntwk.logits, ntwk.loss, ckpt_dir=ntwk.ckpt_dir, write_summary=True) # train the network ntwk.train(train_init_op, flags.train_step, [train_hook, valid_hook, lr_hook], write_summary=True)
def __init__(self, filename): ''' self.data : Contains the parsed data as a dictionary self. self.theta_nb : Contains the weights for the naive bayes model which is nothing but log_pxy and log_py concatenated for each label. Dimension of 1x129 - 1x128 for log_pxy ans 1x1 for log_py. ''' self.dt = data_reader.DataReader(filename) self.data = self.dt.parseFile() self.theta_nb = defaultdict(float) self.labels = [] self.START_LETTER = '**START**' self.END_LETTER = '**END**' self.smoothing = 0.1 self.smooth_hmm = 0.1
def run_epoch(sess, model, datapath, config, is_training=False, gen_model=None, vocab=None): costs = 0 iters = 0 times = 0 start_time = time.time() for line in open(datapath): data = data_reader.DataReader(line.strip(), vocab=vocab, batch_size=config['batch_size'], seq_length=config['seq_length']) data.reset() while data.has_next_batch(): x_batch, y_batch, mask = data.next_batch() feed_dict = { model.input_data: x_batch, model.target_data: y_batch, model.mask: mask } fetches = {"costs": model._cost} if is_training: fetches["train_op"] = model.train_op vals = sess.run(fetches, feed_dict) costs += vals["costs"] iters += data.seq_length times += 1 if times % 2000 == 100: tf.logging.info('step {}: training_loss:{:4f}'.format( times, np.exp(costs / iters))) if times % 20000 == 0 and gen_model != None and vocab != None: sample(sess, model=gen_model, vocab=vocab) if gen_model != None and vocab != None: sample(sess, model=gen_model, vocab=vocab) return costs / iters, times * data.batch_size * data.seq_length / ( time.time() - start_time)
def compute_mean(self, folder): year = int(folder.split("_")[-1]) datar = data_reader.DataReader(year, torch.device('cpu')) n_days = datar.n_test_batches values = [] for i in range(n_days): x, y = datar.get_test_batch(i, cumulative=True) if self.args.conditional: y_step = y[:, self.args.step_prediction - 1] else: y_step = y[:, self.args.steps - 1] values.append(utils.convert_from_tensor(y_step)) values = np.array(values) mean = values.mean() return mean
def main(): # get the parameters datafile = sys.argv[1] neurons = int(sys.argv[2]) learningRate = float(sys.argv[3]) threshold = float(sys.argv[4]) trainPerc = float(sys.argv[5]) seed = int(sys.argv[6]) # set up the RNG random.seed(seed) # read in the data reader = data_reader.DataReader(datafile, trainPerc) numAttributes, numLabels = reader.readData() possibleLabels = reader.discreteValues[data_reader.DataReader.LABEL_COLUMN] # get the randomized training and test sets train, valid, test = reader.splitData() # train the network runNetwork(train, valid, test, neurons, learningRate, threshold, numAttributes, possibleLabels)
def read_concat_write(indir, file_name, max_n, mb_sz, side, sigreg): reader = dr.DataReader(indir) cuts = {'mJJ': 1100.} if side: cuts['sideband'] = 1.4 if sigreg: cuts['signalregion'] = 1.4 keys = [ l.encode('utf-8') for l in [ 'jetConstituentsList', 'particleFeatureNames', 'eventFeatures', 'eventFeatureNames' ] ] particle_feature_names, dijet_feature_names = encode_uf8( reader.read_labels_from_dir()) # write multiple file parts if mb_sz: for part_n, (constituents_concat, features_concat) in enumerate( reader.generate_event_parts_from_dir(min_sz_mb=mb_sz, **cuts)): write_single_file_part([ constituents_concat, particle_feature_names, features_concat, dijet_feature_names ], keys=keys, file_name=file_name, part_n=part_n) # write single concat file else: constituents_concat, _, features_concat, _ = reader.read_events_from_dir( max_n=max_n, **cuts) write_file([ constituents_concat, particle_feature_names, features_concat, dijet_feature_names ], keys=keys, file_name=file_name)
def main(flags): ckpt_dir = os.path.join(os.path.dirname(__file__), 'models', flags.model_name) fc_filters, tconv_dims, tconv_filters = network_helper.get_parameters( ckpt_dir) # initialize data reader if len(tconv_dims) == 0: output_size = fc_filters[-1] else: output_size = tconv_dims[-1] reader = data_reader.DataReader(input_size=flags.input_size, output_size=output_size, x_range=flags.x_range, y_range=flags.y_range, cross_val=flags.cross_val, val_fold=flags.val_fold, batch_size=flags.batch_size, shuffle_size=flags.shuffle_size) features, labels, train_init_op, valid_init_op = reader.get_data_holder_and_init_op( (flags.train_file, flags.valid_file)) # make network ntwk = network_maker.CnnNetwork(features, labels, utils.my_model_fn, flags.batch_size, fc_filters=fc_filters, tconv_dims=tconv_dims, tconv_filters=tconv_filters, learn_rate=flags.learn_rate, decay_step=flags.decay_step, decay_rate=flags.decay_rate, make_folder=False) # evaluate the results if the results does not exist or user force to re-run evaluation save_file = os.path.join(os.path.dirname(__file__), 'data', 'test_pred_{}.csv'.format(flags.model_name)) if FORCE_RUN or (not os.path.exists(save_file)): print('Evaluating the model ...') pred_file, truth_file = ntwk.evaluate(valid_init_op, ckpt_dir=ckpt_dir, model_name=flags.model_name) else: pred_file = save_file truth_file = os.path.join(os.path.dirname(__file__), 'data', 'test_truth.csv') mae, mse = compare_truth_pred(pred_file, truth_file) plt.figure(figsize=(12, 6)) plt.hist(mse, bins=100) plt.xlabel('Mean Squared Error') plt.ylabel('cnt') plt.suptitle('FC + TCONV (Avg MSE={:.4e}'.format(np.mean(mse))) plt.savefig( os.path.join( os.path.dirname(__file__), 'data', 'fc_tconv_single_channel_result_cmp_{}.png'.format( flags.model_name))) plt.show()
""" Author : Byunghyun Ban Date : 2020.07.24. """ from tensorflow import keras import data_reader # 몇 에포크 만큼 학습을 시킬 것인지 결정합니다. EPOCHS = 20 # 예제 기본값은 20입니다. # 데이터를 읽어옵니다. dr = data_reader.DataReader() # 인공신경망을 제작합니다. model = keras.Sequential([ keras.layers.Dense(3), keras.layers.Dense(128, activation="relu"), keras.layers.Dense(3, activation='softmax') ]) # 인공신경망을 컴파일합니다. model.compile(optimizer="adam", metrics=["accuracy"], loss="sparse_categorical_crossentropy") # 인공신경망을 학습시킵니다. print("************ TRAINING START ************") early_stop = keras.callbacks.EarlyStopping(monitor='val_loss', patience=10) history = model.fit(dr.train_X, dr.train_Y, epochs=EPOCHS,
def cluster_train(config): # init for cluster cluster_conf = config["cluster_conf"] cluster = tf.train.ClusterSpec(cluster_conf) server = tf.train.Server(cluster, job_name=FLAGS.job_name, task_index=FLAGS.task_id) n_ps = len(cluster_conf['ps']) n_workers = len(cluster_conf['worker']) if FLAGS.job_name == "ps": server.join() is_chief = (FLAGS.task_id == 0) # load Vocab vocab = data_reader.Vocab(vocab_limits=config['vocab_size']) vocab.load_metadata(config['metadata']) config['vocab_size'] = vocab.vocab_size() tf.logging.info(config) # load Data train_data = data_reader.DataReader(config['train_data'][FLAGS.task_id], vocab=vocab, batch_size=config['batch_size'], seq_length=config['seq_length']) initializer = tf.random_uniform_initializer(-config['init_scale'], config['init_scale']) # create models with tf.device( tf.train.replica_device_setter( worker_device="/job:worker/task:%d" % FLAGS.task_id, cluster=cluster)): with tf.name_scope('Train'): opt, lr = optimizer.get_optimizer("sgd", config['learning_rate']) sync_opt = tf.train.SyncReplicasOptimizer( opt, replicas_to_aggregate=n_workers * config['sync_freq'], total_num_replicas=n_workers) with tf.variable_scope("Model", reuse=None, initializer=initializer): train_model = model.Model(is_training = True, \ config = config, \ optimizer = sync_opt, lr = lr) with tf.name_scope('Generate'): generate_config = copy.deepcopy(config) generate_config['batch_size'] = 1 generate_config['seq_length'] = 1 with tf.variable_scope("Model", reuse=True, initializer=initializer): gen_model = model.Model(is_training=False, config=generate_config) chief_queue_runner = train_model.optimizer.get_chief_queue_runner() init_tokens_op = train_model.optimizer.get_init_tokens_op() sv = tf.train.Supervisor(is_chief=is_chief, logdir=config['logdir']) sess_config = tf.ConfigProto( allow_soft_placement=True, log_device_placement=True, device_filters=["/job:ps", "/job:worker/task:%d" % FLAGS.task_id]) sess_config.gpu_options.allow_growth = True sess_config.gpu_options.per_process_gpu_memory_fraction = 0.5 tf.logging.info('Start Sess') with sv.prepare_or_wait_for_session(server.target, config=sess_config) as sess: if is_chief: sv.start_queue_runners(sess, [chief_queue_runner]) sess.run(init_tokens_op) for i in range(config['n_epoch']): lr_decay = config['lr_decay']**max(i + 1 - config['decay_epoch'], 0) train_model.assign_lr(sess, config['learning_rate'] * lr_decay) tf.logging.info('Iter %d Start, Learning_rate: %.4f' % (i, sess.run(train_model.lr))) costs, speed = run_epoch(sess, train_model, train_data, is_training=True, gen_model=gen_model, vocab=vocab) tf.logging.info( 'Iter %d: training_loss:%.4f, speed %.4f words/sec' % (i, np.exp(costs), speed))
import random import math from sklearn.utils import shuffle import pdb import re import data_reader as reader import networks as nets import utils import params SHOW_IMAGES = False IS_RESTORE = tf.train.latest_checkpoint(params.folder_data) is not None params.show_params() data_reader = reader.DataReader('./../../../../images-training/t1w', './data/validation', './data/test') # training batch_size = 128 stride = 0 input = tf.placeholder(tf.float32, (batch_size, data_reader.dim_patch_in, data_reader.dim_patch_in, params.num_channels), name='input') target = tf.placeholder(tf.float32, (batch_size, data_reader.dim_patch_gt - 2*stride, data_reader.dim_patch_gt - 2*stride, params.num_channels), name='target') output_PS, output = params.network_architecture(input) print('output shape is ', output.shape, target.shape) if params.LOSS == params.L1_LOSS: loss = tf.reduce_mean(tf.reduce_mean(tf.abs(output - target)) + tf.reduce_mean(tf.abs(output_PS - target))) if params.LOSS == params.L2_LOSS: loss = tf.reduce_mean(tf.reduce_mean(tf.square(output - target)) + tf.reduce_mean(tf.square(output_PS - target)))
from sklearn.utils import shuffle import pdb import re import data_reader as reader import networks as nets import utils import params SHOW_IMAGES = False IS_RESTORE = tf.train.latest_checkpoint(params.folder_data) != None params.show_params() data_reader = reader.DataReader( 'C:\\Research\\SR\\medical images\\namic\\images-training\\t1w', './data/validation', './data/test', SHOW_IMAGES=False) # training batch_size = 128 input = tf.placeholder(tf.float32, (batch_size, data_reader.dim_patch_in_rows, data_reader.dim_patch_in_cols, params.num_channels), name='input') target = tf.placeholder(tf.float32, (batch_size, data_reader.dim_patch_gt_rows, data_reader.dim_patch_gt_cols, params.num_channels), name='target') output_PS, output = params.network_architecture(input)
def train_and_sample(minibatch_iterations, restore): tf.reset_default_graph() batch_size = 64 lstm_sizes = [512, 512] batch_len = 100 learning_rate = 2e-3 filepath = './wap.txt' data_feed = data_reader.DataReader( filepath, batch_len, batch_size) labels = data_feed.char_list print('labels: ', labels) save_path = './model.tf' model = Model( batch_size, batch_len, lstm_sizes, 0.8, labels, save_path) model.init_graph() optimizer = tf.train.AdamOptimizer(learning_rate) model.init_train_op(optimizer) init_op = tf.initialize_all_variables() with tf.Session() as sess: sess.run(init_op) if restore: print('Restoring model') model.restore(sess) model.reset_state(sess) start_time = time.time() for i in range(minibatch_iterations): input_batch, target_batch = next(iter(data_feed)) loss, _ = sess.run( [model.loss, model.train_op], feed_dict={ model.inputs: input_batch, model.targets: target_batch}) if i % 50 == 0 and i != 0: print('i: ', i) duration = time.time() - start_time print('loss: {} ({} sec.)'.format(loss, duration)) start_time = time.time() if i % 1000 == 0 and i != 0: model.save(sess) if i % 100 == 0 and i != 0: print('Reset initial state') model.reset_state(sess) if i % 1000 == 0 and i != 0: print('Reset minibatch feeder') data_feed.reset_indices() model.save(sess) print('\n sampling after {} iterations'.format(minibatch_iterations)) tf.reset_default_graph() model = Model( 1, None, lstm_sizes, 1.0, labels, save_path) model.init_graph() init_op = tf.initialize_all_variables() with tf.Session() as sess: sess.run(init_op) model.restore(sess) print('\nSample 1:') sample = model.sample( sess, prime_string=u'\n\nThis feeling was ', sample_length=500) print(u'sample: \n{}'.format(sample)) print('\nSample 2:') sample = model.sample( sess, prime_string=u'She was born in the year ', sample_length=500) print(u'sample: \n{}'.format(sample)) print('\nSample 3:') sample = model.sample( sess, prime_string=u'The meaning of this all is ', sample_length=500) print(u'sample: \n{}'.format(sample)) print('\nSample 4:') sample = model.sample( sess, prime_string=u'In the midst of a conversation on political matters Anna Pávlovna burst out:,', sample_length=500) print(u'sample: \n{}'.format(sample)) print('\nSample 5:') sample = model.sample( sess, prime_string=u'\n\nCHAPTER X\n\n', sample_length=500) print(u'sample: \n{}'.format(sample)) print('\nSample 5:') sample = model.sample( sess, prime_string=u'"If only you knew,"', sample_length=500) print(u'sample: \n{}'.format(sample))
# from this i conclude, that there are 15 views on one tetris object # ptest[0..64:tetris_block][0:64x64 image->, 1:locational data->][0..15:views or locdata][64x64xrgb data] ptest = torch.load("./our_2d_data/train/Pilt1_not0.pt") print(len(ptest), np.shape(ptest[0])) lookadoodle(ptest[0][0], ptest[0][1]) print("\nfirst image loc values:", ptest[1][0], "\nsecond image loc values:", ptest[1][1]) # Lets look ad the data_reader.py here with the shepard metzler data print('\nmetzler 5 parts in DataReader:') _NUM_CHANNELS = 3 # this is the camera dimension parameter length modifier - let's keep it at 5 :) _NUM_RAW_CAMERA_PARAMS = 5 dat = dr.DataReader("shepard_metzler_5_parts", 3, "./shepard_metzler_5_parts", mode='train') file_names = dr._get_dataset_files(dat._dataset_info, 'train', "./shepard_metzler_5_parts") filename_queue = tf.train.string_input_producer(file_names, seed=99) reader = tf.TFRecordReader() _, raw_data = reader.read_up_to(filename_queue, num_records=16) feature_map = { 'frames': tf.FixedLenFeature(shape=dat._dataset_info.sequence_size, dtype=tf.string), 'cameras': tf.FixedLenFeature( shape=[dat._dataset_info.sequence_size * _NUM_RAW_CAMERA_PARAMS], dtype=tf.float32) }
) val_acc.append(val_acc_) val_loss.append(val_loss_) val_mask1_loss.append(val_mask1_loss_) val_mask2_loss.append(val_mask2_loss_) val_class_loss.append(val_class_loss_) val_acc_m = np.mean(val_acc) val_loss_m = np.mean(val_loss) val_mask1_loss_m = np.mean(val_mask1_loss) val_mask2_loss_m = np.mean(val_mask2_loss) val_class_loss_m = np.mean(val_class_loss) sum_val_ = tf.Summary(value=[ tf.Summary.Value(tag="b1-val_acc", simple_value=val_acc_m), tf.Summary.Value(tag="b2-val_loss", simple_value=val_loss_m), tf.Summary.Value(tag="b3-val_mask1_loss", simple_value=val_mask1_loss_m), tf.Summary.Value(tag="b4-val_mask2_loss", simple_value=val_mask2_loss_m), tf.Summary.Value(tag="b5-val_class_loss", simple_value=val_class_loss_m) ]) writer.add_summary(sum_val_, counter) pool.close() pool.join() if (e + 1) % 100 == 0: saver.save(sess, r'./weights/model_epoch_{}.ckpt'.format(e)) if __name__ == '__main__': import data_reader datasets_path = r'./datasets' reader = data_reader.DataReader(datasets_path, is_training=True) main(reader)
if cosineSimilarity > max_cosineSimilarity: max_cosineSimilarity = cosineSimilarity max_index = i return answer_list[max_index] # Launch the graph with tf.Session() as sess: # sess = tf.Session('', tf.Graph()) # with sess.graph.as_default(): # Initializing the variables sess.run(tf.global_variables_initializer()) # import data dr = data_reader.DataReader(is_training=True) valid_x, valid_y = dr.validation_batch(validation_size) valid_x = valid_x.reshape((validation_size, n_steps, n_input)) valid_y = valid_y.reshape((validation_size, n_input)) epoch_size = dr.epoch_size(batch_size) init_learning_rate = learning_rate # training for epoch in range(epoches): ts = time.time() learning_rate = init_learning_rate / (epoch + 1) for batch_num in range(epoch_size): batch_x, batch_y = dr.next_batch(batch_size) # Reshape data to get n_steps seq of n_input elements batch_x = batch_x.reshape((batch_size, n_steps, n_input))
""" Author : Byunghyun Ban Date : 2020.07.24. """ import tensorflow_hub as hub import data_reader # 데이터를 불러옵니다. dr = data_reader.DataReader("content.jpg", "style.jpg") # Hub로부터 style transfer 모듈을 불러옵니다. hub_module = hub.load( 'https://tfhub.dev/google/magenta/arbitrary-image-stylization-v1-256/1') # 모듈에 이미지를 삽입해 Style Transfer를 실시합니다. stylized_image = hub_module(dr.content, dr.style)[0] # 결과를 출력합니다. result = data_reader.tensor_to_image(stylized_image) # 결과를 저장합니다. result.save("result.jpg")
def read_file(self, path): event_reader = dr.DataReader(path) events = event_reader.read_jet_constituents() dijet_features = event_reader.read_jet_features() labels = event_reader.read_labels() return [events[:, 0, :, :], events[:, 1, :, :], dijet_features, labels]
if test_case[key] in word_vector: midContextVector = word_vector[test_case[key]] else: midContextVector = np.zeros(WORD_VECTOR_SIZE) embedded_test_X.append([preContextVector, midContextVector]) embedded_test_Y.append(postContextVector) pickle.dump((np.array(embedded_test_X), np.array(embedded_test_Y), len(embedded_test_X)), open('data/embedded_test_data', 'wb'), True) saver = tf.train.Saver() with tf.Session() as sess: saver.restore(sess, pretrained_model) print("Model restored.") # testing ts = time.time() dr = data_reader.DataReader(is_training=False) with open(sys.argv[2], 'w') as out: out.write('id,answer\n') test_case = 1 while dr.check_test_index() == True: test_x, test_y = dr.next_test(batch_size) tp = sess.run(predict, feed_dict={x: test_x}) predict_answer = test_predictAnswer(tp[0:5], test_y) out.write("%s,%s\n" % (test_case, predict_answer)) test_case += 1
"Log placement of ops on devices") FLAGS = tf.flags.FLAGS args = parser.parse_args() print("dc", args.doc_len) max_sen_length = args.sen_len max_doc_length = args.doc_len logging.info('generate config') word_vocab, word_tensors, max_doc_length, label_tensors = \ dr.load_data(args.train_file, max_doc_length, max_sen_length) batch_size = 1 time1 = time.time() test_reader = dr.DataReader(word_tensors['test'], label_tensors['test'], batch_size) graph = tf.Graph() with graph.as_default(): session_conf = tf.ConfigProto( allow_soft_placement=FLAGS.allow_soft_placement, log_device_placement=FLAGS.log_device_placement) sess = tf.Session(config=session_conf) with sess.as_default(): # Checkpoint directory. Tensorflow assumes this directory already exists so we need to create it ''' with tf.variable_scope("Model"): #m = build_model(word_vocab) global_step = tf.Variable(0, dtype=tf.int32, name='global_step') ''' #saver = tf.train.Saver() saver = tf.train.import_meta_graph(
stock_outputs = self.outputs[stock] split_percent = .80 split_pos = int(len(stock_data) * split_percent) print split_pos train_x = stock_data[:split_pos] self.test_x = stock_data[split_pos:] train_y = stock_outputs[:split_pos] self.test_y = stock_outputs[split_pos:] self.clf.fit(train_x, train_y) def predict(self): return self.clf.predict(self.test_x) def evaluate(self): return self.predict() - self.test_y def score(self): return self.clf.score(self.test_x, self.test_y) if __name__ == "__main__": dr = data_reader.DataReader("data/all_stocks_5yr.csv") dr.gen_outputs() trainer = Trainer(dr.data, dr.outputs, linear_model.LinearRegression()) for stock in dr.data: trainer.train(stock) print stock, trainer.score()
def train(w2v_model): # Training # ================================================== max_sen_length = 40 max_doc_length = 90 word_vocab, word_tensors, max_doc_length, label_tensors = \ \ dr.load_data(FLAGS.train_data_file, max_doc_length, max_sen_length) train_reader = dr.DataReader(word_tensors['train'], label_tensors['train'], 1) valid_reader = dr.DataReader(word_tensors['valid'], label_tensors['valid'], 1) test_reader = dr.DataReader(word_tensors['test'], label_tensors['test'], 1) pretrained_embedding = dr.get_embed(word_vocab) print("ppp", pretrained_embedding.shape) #x_train, x_dev, y_train, y_dev ,vocab_siz, pretrained_embedding= load_data(w2v_model) embedding_size = 150 with tf.Graph().as_default(): session_conf = tf.ConfigProto( allow_soft_placement=FLAGS.allow_soft_placement, log_device_placement=FLAGS.log_device_placement) sess = tf.Session(config=session_conf) with sess.as_default(): #需修改 ''' cnn = TextCNN( w2v_model, sequence_length=x_train.shape[1], num_classes=y_train.shape[1], vocab_size=vocab_size, embedding_size=FLAGS.embedding_dim, filter_sizes=list(map(int, FLAGS.filter_sizes.split(","))), num_filters=FLAGS.num_filters, l2_reg_lambda=FLAGS.l2_reg_lambda) ''' # Checkpoint directory. Tensorflow assumes this directory already exists so we need to create it print word_vocab.size Summa = SummaRuNNer(word_vocab.size, embedding_size, pretrained_embedding) #loss_sum = tf.Variable(initial_value=0, dtype=tf.float32) global_step = tf.Variable(0, name="global_step", trainable=False) ''' optimizer = tf.train.AdamOptimizer(1e-3) grads_and_vars = optimizer.compute_gradients(Summa.loss) train_op = optimizer.apply_gradients(grads_and_vars, global_step=global_step) ''' train_params = tf.trainable_variables() train_op = tf.train.AdamOptimizer(learning_rate=0.0001).minimize( Summa.loss, var_list=train_params) timestamp = str(int(time.time())) out_dir = os.path.abspath( os.path.join(os.path.curdir, "runs", timestamp)) checkpoint_dir = os.path.abspath( os.path.join(out_dir, "checkpoints")) checkpoint_prefix = os.path.join(checkpoint_dir, "model") if not os.path.exists(checkpoint_dir): os.makedirs(checkpoint_dir) saver = tf.train.Saver(tf.global_variables(), max_to_keep=FLAGS.num_checkpoints) checkpoint_dir = os.path.abspath( os.path.join(out_dir, "checkpoints")) checkpoint_prefix = os.path.join(checkpoint_dir, "model") batches = train_reader valid_batches = valid_reader sess.run(tf.global_variables_initializer()) #step = 0 min_eval_loss = float('Inf') #fetch_list = [Summa.xks, Summa.data_c, Summa.result] for epoch in range(FLAGS.num_epochs): step = 0 loss_sum = 0 #loss_sum.assign(value=0) #value_sum = 0 for x_batch, y_batch in batches.iter(): step += 1 feed_dict = { Summa.x: x_batch[0], Summa.y: y_batch[0], } ''' lucky_boy, lucky_girl, data_cc = sess.run(fetch_list, feed_dict) print ("lucky_boy, ", lucky_boy) print ("lucky_girl, ", lucky_girl) print ("data_cc:", data_cc) ''' sess.run(train_op, feed_dict) loss = sess.run([Summa.loss], feed_dict) predict = sess.run([Summa.y_], feed_dict) loss_sum += loss[0] #print predict #grads_and_vars = optimizer.compute_gradients(Summa.loss) #train_op = optimizer.apply_gradients(grads_and_vars, global_step=global_step) #print step #print len(y_batch[0]) #print len(predict[0]) #print step % 128 #print step if step % 128 == 0 and step != 0: #print ("here") #logging.info('Epoch ' + str(epoch) + ' Loss: ' + str(loss_sum / 128.0)) print('Epoch ' + str(epoch) + ' Loss: ' + str(loss_sum / 128.0)) loss_sum = 0 if step % 512 == 0 and step != 0: eval_loss = 0 for x_batch, y_batch in valid_batches.iter(): feed_dict = { Summa.x: x_batch[0], Summa.y: y_batch[0], } loss = sess.run([Summa.loss], feed_dict) eval_loss += loss[0] print('epoch ' + str(epoch) + ' Loss in validation: ' + str(eval_loss * 1.0 / valid_reader.length)) if eval_loss < min_eval_loss: min_eval_loss = eval_loss path = saver.save(sess, checkpoint_prefix, global_step=step) print( "Saved model checkpoint to {}\n".format(path))
import os # from mpl_toolkits.mplot3d import Axes3D # noqa: F401 unused import # from importlib import reload # Python 3.4+ only. # reload(plotter_bayesian_data) # plter = plotter_bayesian_data.Plotter('./', save_figs=False) # choose which data should be used no_spatial_conv = True n_draws = 200 plter = plotter_bayesian_data.Plotter('./', save_figs=False) ############### READ DATA ############### reader = data_reader.DataReader( not no_spatial_conv, path= '/media/oesst/TOSHIBA EXT/Backup PHD Stuff/Code/Python/msi_network_model') # get the means and variances of the results means, vars = reader.get_means_variances() intensities = reader.get_s_intensities() s_locations = reader.get_s_locations() s_variances = reader.get_s_variance() # caluclate the percentage of fused event in a simulation fusion = reader.get_fusion() fusion_fb_on = fusion[:, :, 0] fusion_fb_off = fusion[:, :, 1] # # %% Save the data for faster excess # exp_dir = os.path.join(os.getcwd(), 'Combined_Results')
def train(w2v_model): # Training # ================================================== max_sen_length = 40 max_doc_length = 90 word_vocab, word_tensors, max_doc_length, label_tensors = \ \ dr.load_data(FLAGS.train_data_file, max_doc_length, max_sen_length) train_reader = dr.DataReader(word_tensors['train'], label_tensors['train'], 1) valid_reader = dr.DataReader(word_tensors['valid'], label_tensors['valid'], 1) test_reader = dr.DataReader(word_tensors['test'], label_tensors['test'], 1) pretrained_embedding = dr.get_embed(word_vocab) embedding_size = 150 with tf.Graph().as_default(): session_conf = tf.ConfigProto( allow_soft_placement=FLAGS.allow_soft_placement, log_device_placement=FLAGS.log_device_placement) sess = tf.Session(config=session_conf) with sess.as_default(): print word_vocab.size Summa = SummaRuNNer(word_vocab.size, embedding_size, pretrained_embedding) #loss_sum = tf.Variable(initial_value=0, dtype=tf.float32) global_step = tf.Variable(0, name="global_step", trainable=False) train_params = tf.trainable_variables() train_op = tf.train.AdamOptimizer(learning_rate=0.0001).minimize( Summa.loss, var_list=train_params) timestamp = str(int(time.time())) out_dir = os.path.abspath( os.path.join(os.path.curdir, "runs", timestamp)) checkpoint_dir = os.path.abspath( os.path.join(out_dir, "checkpoints")) checkpoint_prefix = os.path.join(checkpoint_dir, "model") if not os.path.exists(checkpoint_dir): os.makedirs(checkpoint_dir) saver = tf.train.Saver(tf.global_variables(), max_to_keep=FLAGS.num_checkpoints) checkpoint_dir = os.path.abspath( os.path.join(out_dir, "checkpoints")) checkpoint_prefix = os.path.join(checkpoint_dir, "model") batches = train_reader valid_batches = valid_reader sess.run(tf.global_variables_initializer()) #step = 0 min_eval_loss = float('Inf') for epoch in range(FLAGS.num_epochs): step = 0 loss_sum = 0 for x_batch, y_batch in batches.iter(): step += 1 feed_dict = { Summa.x: x_batch[0], Summa.y: y_batch[0], } sess.run(train_op, feed_dict) loss = sess.run([Summa.loss], feed_dict) predict = sess.run([Summa.y_], feed_dict) loss_sum += loss[0] if step % 128 == 0 and step != 0: print('Epoch ' + str(epoch) + ' Loss: ' + str(loss_sum / 128.0)) loss_sum = 0 if step % 512 == 0 and step != 0: eval_loss = 0 for x_batch, y_batch in valid_batches.iter(): feed_dict = { Summa.x: x_batch[0], Summa.y: y_batch[0], } loss = sess.run([Summa.loss], feed_dict) eval_loss += loss[0] print('epoch ' + str(epoch) + ' Loss in validation: ' + str(eval_loss * 1.0 / valid_reader.length)) if eval_loss < min_eval_loss: min_eval_loss = eval_loss path = saver.save(sess, checkpoint_prefix, global_step=step) print( "Saved model checkpoint to {}\n".format(path))