def train_model(data_dir):
dr = data_reader.DataReader(data_dir)
data, label, _ = dr.parse()
data_label = list(zip(data, label))
random.shuffle(data_label)
data, label = zip(*data_label)
data = numpy.array(data)
label = numpy.array(label)
rows = len(data)
rows_training = int(rows * TRAINING_PERCENT)
data_training = data[:rows_training]
label_training = label[:rows_training]
data_validation = data[rows_training:]
label_validation = label[rows_training:]
estimator = get_estimator()
train_input_fn = tf.estimator.inputs.numpy_input_fn(x={"x": data_training},
y=label_training,
shuffle=True,
num_epochs=None)
train_spec = tf.estimator.TrainSpec(input_fn=train_input_fn,
max_steps=5000000)
evaluate_input_fn = tf.estimator.inputs.numpy_input_fn(
x={"x": data_validation}, y=label_validation, shuffle=False)
eval_spec = tf.estimator.EvalSpec(input_fn=evaluate_input_fn,
throttle_secs=30)
tf.estimator.train_and_evaluate(estimator, train_spec, eval_spec)
def train(config):
# load Vocab
src_vocab = data_reader.Vocab(vocab_limits=config['src_vocab_size'])
src_vocab.load_metadata(config['metadata']['src'])
config['src_vocab_size'] = src_vocab.vocab_size()
tgt_vocab = data_reader.Vocab(vocab_limits=config['tgt_vocab_size'])
tgt_vocab.load_metadata(config['metadata']['tgt'])
config['tgt_vocab_size'] = tgt_vocab.vocab_size()
tf.logging.info(config)
initializer = tf.random_uniform_initializer(-config['init_scale'],
config['init_scale'])
# create models
with tf.name_scope('Train'):
opt, lr = optimizer.get_optimizer(config['optimizer'],
config['learning_rate'])
with tf.variable_scope("Model", reuse=None, initializer=initializer):
train_model = model.Model(is_training=True,
config=config,
seq_length=config['tgt_length'] - 1,
optimizer=opt,
lr=lr)
with tf.name_scope('Test'):
with tf.variable_scope("Model", reuse=True):
test_model = model.Model(is_training=False,
config=config,
seq_length=1)
sv = tf.train.Supervisor(logdir=config['logdir'])
sess_config = tf.ConfigProto(allow_soft_placement=True,
log_device_placement=True)
sess_config.gpu_options.allow_growth = True
sess_config.gpu_options.per_process_gpu_memory_fraction = 0.9
# load Data
train_data = data_reader.DataReader(
src_data=config['train_data']['src'][0],
tgt_data=config['train_data']['tgt'][0],
src_vocab=src_vocab,
tgt_vocab=tgt_vocab,
src_length=config['src_length'],
tgt_length=config['tgt_length'],
batch_size=config['batch_size'])
tf.logging.info('Start Sess')
with sv.managed_session(config=sess_config) as sess:
for i in range(config['n_epoch']):
lr_decay = config['lr_decay']**max(i + 1 - config['decay_epoch'],
0)
train_model.assign_lr(sess, config['learning_rate'] * lr_decay)
tf.logging.info('Iter %d Start, Learning_rate: %.4f' %
(i, sess.run(train_model.lr)))
loss = run_epoch(sess, train_model, train_data, is_training = True, \
t_model = test_model, src_vocab = src_vocab, tgt_vocab = tgt_vocab)
tf.logging.info('Iter %d: training_loss: %.4f' %
(i, np.power(2, loss)))
def main():
# get the parameters
datafile = sys.argv[1]
neurons = int(sys.argv[2])
learningRate = float(sys.argv[3])
threshold = float(sys.argv[4])
trainPerc = float(sys.argv[5])
seed = int(sys.argv[6])
title = sys.argv[7]
# set up the RNG
random.seed(seed)
outputFile = open(
"results_nn_" + title + "_" + sys.argv[2] + "_" + sys.argv[3] + "_" +
sys.argv[4] + "_" + sys.argv[5] + "_" + sys.argv[6] + ".txt", "w")
# read in the data, we edited parts of this to get rid of columns that were irrelevant
reader = data_reader.DataReader(datafile, trainPerc, outputFile)
numAttributes, numLabels = reader.readData()
possibleLabels = reader.discreteValues[data_reader.DataReader.LABEL_COLUMN]
# get the randomized training and test sets
train, valid, test = reader.splitData()
# train the network
runNetwork(train, valid, test, neurons, learningRate, threshold,
numAttributes, possibleLabels, outputFile)
outputFile.close()
def save_processed_data(path):
dr = data_reader.DataReader()
dp = data_preprocessor.DataPreprocessor()
choices, dynamics, stim_viewing = dr.get_data(path=path + 'merged_raw/',
stim_viewing=True)
dynamics = dp.preprocess_data(choices, dynamics)
stim_viewing = dp.preprocess_data(choices, stim_viewing)
choices = dp.get_hand_and_gaze_measures(choices, dynamics, stim_viewing)
choices, dynamics, stim_viewing = dp.exclude_trials(
choices, dynamics, stim_viewing)
processed_path = path + 'processed/'
if not os.path.exists(processed_path):
os.makedirs(processed_path)
choices.to_csv(processed_path + 'choices.txt',
sep='\t',
na_rep='nan',
float_format='%.4f')
dynamics.to_csv(processed_path + 'dynamics.txt',
sep='\t',
na_rep='nan',
float_format='%.4f')
stim_viewing.to_csv(processed_path + 'stim_viewing.txt',
sep='\t',
na_rep='nan',
float_format='%.4f')
def learnig_pipeline():
# 0. load parameters
params = LearningParams()
# 1. load the data
paths_list = [
'data/clockwise/', 'data/clockwise_1/', 'data/counterclockwise/',
'data/clockcouterwise_1/', 'data/counterclockwise_recovery/',
'data/clockwise_recovery/'
]
reader = dread.DataReader()
# compile and train the model using the generator function
[train_generator, validation_generator
] = reader.read_using_generator(paths_list,
batch_size=params.batch_size,
debug=False)
input_shape = (reader.image_shape[2], reader.image_shape[0],
reader.image_shape[1])
# 2. Create the model
clone_model = model.BehavioralCloneModel(reader.image_shape)
for layer in clone_model.model.layers:
print(layer.output_shape)
# 2.2 Compile the model
clone_model.model.compile(loss='mse', optimizer='adam')
steps_per_epoch = math.ceil(reader.train_samples_size / params.batch_size)
validation_steps = math.ceil(reader.validation_samples_size /
params.batch_size)
save_path = "model_checkpoints/saved-model-{epoch:02d}-{val_loss:.2f}.hdf5"
checkpoint = keras.callbacks.ModelCheckpoint(filepath=save_path,
monitor='val_loss',
save_best_only=True)
history_object = clone_model.model.fit_generator(
train_generator,
steps_per_epoch=steps_per_epoch,
callbacks=[checkpoint],
validation_data=validation_generator,
validation_steps=validation_steps,
epochs=params.num_epochs,
verbose=1)
# saving model
clone_model.model.save('model.h5')
# 3. Model visualization. Print the keys contained in the history object
print(history_object.history.keys())
# 3.1 Plot the training and validation loss for each epoch
plt.plot(history_object.history['loss'])
plt.plot(history_object.history['val_loss'])
plt.title('model mean squared error loss')
plt.ylabel('mean squared error loss')
plt.xlabel('epoch')
plt.legend(['training set', 'validation set'], loc='upper right')
plt.show()
def readData(self):
print 'Read data manually.'
data_start = 0
data_finish = 1000 #'end'
model = 'bed'
subject = 'sub6_shaver'
print 'Starting to convert data!'
self.runData = dr.DataReader(subject=subject,
data_start=data_start,
data_finish=data_finish,
model=model)
def main(flags):
# initialize data reader
if len(flags.tconv_dims) == 0:
output_size = flags.fc_filters[-1]
else:
output_size = flags.tconv_dims[-1]
reader = data_reader.DataReader(input_size=flags.input_size,
output_size=output_size,
x_range=flags.x_range,
y_range=flags.y_range,
cross_val=flags.cross_val,
val_fold=flags.val_fold,
batch_size=flags.batch_size,
shuffle_size=flags.shuffle_size)
features, labels, train_init_op, valid_init_op = reader.get_data_holder_and_init_op(
(flags.train_file, flags.valid_file))
# make network
ntwk = network_maker.CnnNetwork(features,
labels,
utils.my_model_fn,
flags.batch_size,
fc_filters=flags.fc_filters,
tconv_dims=flags.tconv_dims,
tconv_filters=flags.tconv_filters,
learn_rate=flags.learn_rate,
decay_step=flags.decay_step,
decay_rate=flags.decay_rate)
# define hooks for monitoring training
train_hook = network_helper.TrainValueHook(flags.verb_step,
ntwk.loss,
ckpt_dir=ntwk.ckpt_dir,
write_summary=True)
lr_hook = network_helper.TrainValueHook(flags.verb_step,
ntwk.learn_rate,
ckpt_dir=ntwk.ckpt_dir,
write_summary=True,
value_name='learning_rate')
valid_hook = network_helper.ValidationHook(flags.eval_step,
valid_init_op,
ntwk.labels,
ntwk.logits,
ntwk.loss,
ckpt_dir=ntwk.ckpt_dir,
write_summary=True)
# train the network
ntwk.train(train_init_op,
flags.train_step, [train_hook, valid_hook, lr_hook],
write_summary=True)
def __init__(self, filename):
'''
self.data : Contains the parsed data as a dictionary
self.
self.theta_nb : Contains the weights for the naive bayes model which is nothing but log_pxy and log_py concatenated
for each label. Dimension of 1x129 - 1x128 for log_pxy ans 1x1 for log_py.
'''
self.dt = data_reader.DataReader(filename)
self.data = self.dt.parseFile()
self.theta_nb = defaultdict(float)
self.labels = []
self.START_LETTER = '**START**'
self.END_LETTER = '**END**'
self.smoothing = 0.1
self.smooth_hmm = 0.1
def run_epoch(sess,
model,
datapath,
config,
is_training=False,
gen_model=None,
vocab=None):
costs = 0
iters = 0
times = 0
start_time = time.time()
for line in open(datapath):
data = data_reader.DataReader(line.strip(),
vocab=vocab,
batch_size=config['batch_size'],
seq_length=config['seq_length'])
data.reset()
while data.has_next_batch():
x_batch, y_batch, mask = data.next_batch()
feed_dict = {
model.input_data: x_batch,
model.target_data: y_batch,
model.mask: mask
}
fetches = {"costs": model._cost}
if is_training: fetches["train_op"] = model.train_op
vals = sess.run(fetches, feed_dict)
costs += vals["costs"]
iters += data.seq_length
times += 1
if times % 2000 == 100:
tf.logging.info('step {}: training_loss:{:4f}'.format(
times, np.exp(costs / iters)))
if times % 20000 == 0 and gen_model != None and vocab != None:
sample(sess, model=gen_model, vocab=vocab)
if gen_model != None and vocab != None:
sample(sess, model=gen_model, vocab=vocab)
return costs / iters, times * data.batch_size * data.seq_length / (
time.time() - start_time)
def compute_mean(self, folder):
year = int(folder.split("_")[-1])
datar = data_reader.DataReader(year, torch.device('cpu'))
n_days = datar.n_test_batches
values = []
for i in range(n_days):
x, y = datar.get_test_batch(i, cumulative=True)
if self.args.conditional:
y_step = y[:, self.args.step_prediction - 1]
else:
y_step = y[:, self.args.steps - 1]
values.append(utils.convert_from_tensor(y_step))
values = np.array(values)
mean = values.mean()
return mean
def main():
# get the parameters
datafile = sys.argv[1]
neurons = int(sys.argv[2])
learningRate = float(sys.argv[3])
threshold = float(sys.argv[4])
trainPerc = float(sys.argv[5])
seed = int(sys.argv[6])
# set up the RNG
random.seed(seed)
# read in the data
reader = data_reader.DataReader(datafile, trainPerc)
numAttributes, numLabels = reader.readData()
possibleLabels = reader.discreteValues[data_reader.DataReader.LABEL_COLUMN]
# get the randomized training and test sets
train, valid, test = reader.splitData()
# train the network
runNetwork(train, valid, test, neurons, learningRate, threshold,
numAttributes, possibleLabels)
def read_concat_write(indir, file_name, max_n, mb_sz, side, sigreg):
reader = dr.DataReader(indir)
cuts = {'mJJ': 1100.}
if side:
cuts['sideband'] = 1.4
if sigreg:
cuts['signalregion'] = 1.4
keys = [
l.encode('utf-8') for l in [
'jetConstituentsList', 'particleFeatureNames', 'eventFeatures',
'eventFeatureNames'
]
]
particle_feature_names, dijet_feature_names = encode_uf8(
reader.read_labels_from_dir())
# write multiple file parts
if mb_sz:
for part_n, (constituents_concat, features_concat) in enumerate(
reader.generate_event_parts_from_dir(min_sz_mb=mb_sz, **cuts)):
write_single_file_part([
constituents_concat, particle_feature_names, features_concat,
dijet_feature_names
],
keys=keys,
file_name=file_name,
part_n=part_n)
# write single concat file
else:
constituents_concat, _, features_concat, _ = reader.read_events_from_dir(
max_n=max_n, **cuts)
write_file([
constituents_concat, particle_feature_names, features_concat,
dijet_feature_names
],
keys=keys,
file_name=file_name)
def main(flags):
ckpt_dir = os.path.join(os.path.dirname(__file__), 'models',
flags.model_name)
fc_filters, tconv_dims, tconv_filters = network_helper.get_parameters(
ckpt_dir)
# initialize data reader
if len(tconv_dims) == 0:
output_size = fc_filters[-1]
else:
output_size = tconv_dims[-1]
reader = data_reader.DataReader(input_size=flags.input_size,
output_size=output_size,
x_range=flags.x_range,
y_range=flags.y_range,
cross_val=flags.cross_val,
val_fold=flags.val_fold,
batch_size=flags.batch_size,
shuffle_size=flags.shuffle_size)
features, labels, train_init_op, valid_init_op = reader.get_data_holder_and_init_op(
(flags.train_file, flags.valid_file))
# make network
ntwk = network_maker.CnnNetwork(features,
labels,
utils.my_model_fn,
flags.batch_size,
fc_filters=fc_filters,
tconv_dims=tconv_dims,
tconv_filters=tconv_filters,
learn_rate=flags.learn_rate,
decay_step=flags.decay_step,
decay_rate=flags.decay_rate,
make_folder=False)
# evaluate the results if the results does not exist or user force to re-run evaluation
save_file = os.path.join(os.path.dirname(__file__), 'data',
'test_pred_{}.csv'.format(flags.model_name))
if FORCE_RUN or (not os.path.exists(save_file)):
print('Evaluating the model ...')
pred_file, truth_file = ntwk.evaluate(valid_init_op,
ckpt_dir=ckpt_dir,
model_name=flags.model_name)
else:
pred_file = save_file
truth_file = os.path.join(os.path.dirname(__file__), 'data',
'test_truth.csv')
mae, mse = compare_truth_pred(pred_file, truth_file)
plt.figure(figsize=(12, 6))
plt.hist(mse, bins=100)
plt.xlabel('Mean Squared Error')
plt.ylabel('cnt')
plt.suptitle('FC + TCONV (Avg MSE={:.4e}'.format(np.mean(mse)))
plt.savefig(
os.path.join(
os.path.dirname(__file__), 'data',
'fc_tconv_single_channel_result_cmp_{}.png'.format(
flags.model_name)))
plt.show()
"""
Author : Byunghyun Ban
Date : 2020.07.24.
"""
from tensorflow import keras
import data_reader
# 몇 에포크 만큼 학습을 시킬 것인지 결정합니다.
EPOCHS = 20 # 예제 기본값은 20입니다.
# 데이터를 읽어옵니다.
dr = data_reader.DataReader()
# 인공신경망을 제작합니다.
model = keras.Sequential([
keras.layers.Dense(3),
keras.layers.Dense(128, activation="relu"),
keras.layers.Dense(3, activation='softmax')
])
# 인공신경망을 컴파일합니다.
model.compile(optimizer="adam",
metrics=["accuracy"],
loss="sparse_categorical_crossentropy")
# 인공신경망을 학습시킵니다.
print("************ TRAINING START ************")
early_stop = keras.callbacks.EarlyStopping(monitor='val_loss', patience=10)
history = model.fit(dr.train_X,
dr.train_Y,
epochs=EPOCHS,
def cluster_train(config):
# init for cluster
cluster_conf = config["cluster_conf"]
cluster = tf.train.ClusterSpec(cluster_conf)
server = tf.train.Server(cluster,
job_name=FLAGS.job_name,
task_index=FLAGS.task_id)
n_ps = len(cluster_conf['ps'])
n_workers = len(cluster_conf['worker'])
if FLAGS.job_name == "ps": server.join()
is_chief = (FLAGS.task_id == 0)
# load Vocab
vocab = data_reader.Vocab(vocab_limits=config['vocab_size'])
vocab.load_metadata(config['metadata'])
config['vocab_size'] = vocab.vocab_size()
tf.logging.info(config)
# load Data
train_data = data_reader.DataReader(config['train_data'][FLAGS.task_id],
vocab=vocab,
batch_size=config['batch_size'],
seq_length=config['seq_length'])
initializer = tf.random_uniform_initializer(-config['init_scale'],
config['init_scale'])
# create models
with tf.device(
tf.train.replica_device_setter(
worker_device="/job:worker/task:%d" % FLAGS.task_id,
cluster=cluster)):
with tf.name_scope('Train'):
opt, lr = optimizer.get_optimizer("sgd", config['learning_rate'])
sync_opt = tf.train.SyncReplicasOptimizer(
opt,
replicas_to_aggregate=n_workers * config['sync_freq'],
total_num_replicas=n_workers)
with tf.variable_scope("Model",
reuse=None,
initializer=initializer):
train_model = model.Model(is_training = True, \
config = config, \
optimizer = sync_opt,
lr = lr)
with tf.name_scope('Generate'):
generate_config = copy.deepcopy(config)
generate_config['batch_size'] = 1
generate_config['seq_length'] = 1
with tf.variable_scope("Model",
reuse=True,
initializer=initializer):
gen_model = model.Model(is_training=False,
config=generate_config)
chief_queue_runner = train_model.optimizer.get_chief_queue_runner()
init_tokens_op = train_model.optimizer.get_init_tokens_op()
sv = tf.train.Supervisor(is_chief=is_chief, logdir=config['logdir'])
sess_config = tf.ConfigProto(
allow_soft_placement=True,
log_device_placement=True,
device_filters=["/job:ps",
"/job:worker/task:%d" % FLAGS.task_id])
sess_config.gpu_options.allow_growth = True
sess_config.gpu_options.per_process_gpu_memory_fraction = 0.5
tf.logging.info('Start Sess')
with sv.prepare_or_wait_for_session(server.target,
config=sess_config) as sess:
if is_chief:
sv.start_queue_runners(sess, [chief_queue_runner])
sess.run(init_tokens_op)
for i in range(config['n_epoch']):
lr_decay = config['lr_decay']**max(i + 1 - config['decay_epoch'],
0)
train_model.assign_lr(sess, config['learning_rate'] * lr_decay)
tf.logging.info('Iter %d Start, Learning_rate: %.4f' %
(i, sess.run(train_model.lr)))
costs, speed = run_epoch(sess,
train_model,
train_data,
is_training=True,
gen_model=gen_model,
vocab=vocab)
tf.logging.info(
'Iter %d: training_loss:%.4f, speed %.4f words/sec' %
(i, np.exp(costs), speed))
import random
import math
from sklearn.utils import shuffle
import pdb
import re
import data_reader as reader
import networks as nets
import utils
import params
SHOW_IMAGES = False
IS_RESTORE = tf.train.latest_checkpoint(params.folder_data) is not None
params.show_params()
data_reader = reader.DataReader('./../../../../images-training/t1w', './data/validation', './data/test')
# training
batch_size = 128
stride = 0
input = tf.placeholder(tf.float32, (batch_size, data_reader.dim_patch_in, data_reader.dim_patch_in, params.num_channels), name='input')
target = tf.placeholder(tf.float32, (batch_size, data_reader.dim_patch_gt - 2*stride, data_reader.dim_patch_gt - 2*stride, params.num_channels), name='target')
output_PS, output = params.network_architecture(input)
print('output shape is ', output.shape, target.shape)
if params.LOSS == params.L1_LOSS:
loss = tf.reduce_mean(tf.reduce_mean(tf.abs(output - target)) + tf.reduce_mean(tf.abs(output_PS - target)))
if params.LOSS == params.L2_LOSS:
loss = tf.reduce_mean(tf.reduce_mean(tf.square(output - target)) + tf.reduce_mean(tf.square(output_PS - target)))
from sklearn.utils import shuffle
import pdb
import re
import data_reader as reader
import networks as nets
import utils
import params
SHOW_IMAGES = False
IS_RESTORE = tf.train.latest_checkpoint(params.folder_data) != None
params.show_params()
data_reader = reader.DataReader(
'C:\\Research\\SR\\medical images\\namic\\images-training\\t1w',
'./data/validation',
'./data/test',
SHOW_IMAGES=False)
# training
batch_size = 128
input = tf.placeholder(tf.float32,
(batch_size, data_reader.dim_patch_in_rows,
data_reader.dim_patch_in_cols, params.num_channels),
name='input')
target = tf.placeholder(tf.float32,
(batch_size, data_reader.dim_patch_gt_rows,
data_reader.dim_patch_gt_cols, params.num_channels),
name='target')
output_PS, output = params.network_architecture(input)
def train_and_sample(minibatch_iterations, restore):
tf.reset_default_graph()
batch_size = 64
lstm_sizes = [512, 512]
batch_len = 100
learning_rate = 2e-3
filepath = './wap.txt'
data_feed = data_reader.DataReader(
filepath, batch_len, batch_size)
labels = data_feed.char_list
print('labels: ', labels)
save_path = './model.tf'
model = Model(
batch_size, batch_len, lstm_sizes, 0.8, labels,
save_path)
model.init_graph()
optimizer = tf.train.AdamOptimizer(learning_rate)
model.init_train_op(optimizer)
init_op = tf.initialize_all_variables()
with tf.Session() as sess:
sess.run(init_op)
if restore:
print('Restoring model')
model.restore(sess)
model.reset_state(sess)
start_time = time.time()
for i in range(minibatch_iterations):
input_batch, target_batch = next(iter(data_feed))
loss, _ = sess.run(
[model.loss, model.train_op],
feed_dict={
model.inputs: input_batch, model.targets: target_batch})
if i % 50 == 0 and i != 0:
print('i: ', i)
duration = time.time() - start_time
print('loss: {} ({} sec.)'.format(loss, duration))
start_time = time.time()
if i % 1000 == 0 and i != 0:
model.save(sess)
if i % 100 == 0 and i != 0:
print('Reset initial state')
model.reset_state(sess)
if i % 1000 == 0 and i != 0:
print('Reset minibatch feeder')
data_feed.reset_indices()
model.save(sess)
print('\n sampling after {} iterations'.format(minibatch_iterations))
tf.reset_default_graph()
model = Model(
1, None, lstm_sizes, 1.0, labels, save_path)
model.init_graph()
init_op = tf.initialize_all_variables()
with tf.Session() as sess:
sess.run(init_op)
model.restore(sess)
print('\nSample 1:')
sample = model.sample(
sess, prime_string=u'\n\nThis feeling was ', sample_length=500)
print(u'sample: \n{}'.format(sample))
print('\nSample 2:')
sample = model.sample(
sess, prime_string=u'She was born in the year ', sample_length=500)
print(u'sample: \n{}'.format(sample))
print('\nSample 3:')
sample = model.sample(
sess, prime_string=u'The meaning of this all is ',
sample_length=500)
print(u'sample: \n{}'.format(sample))
print('\nSample 4:')
sample = model.sample(
sess,
prime_string=u'In the midst of a conversation on political matters Anna Pávlovna burst out:,',
sample_length=500)
print(u'sample: \n{}'.format(sample))
print('\nSample 5:')
sample = model.sample(
sess, prime_string=u'\n\nCHAPTER X\n\n',
sample_length=500)
print(u'sample: \n{}'.format(sample))
print('\nSample 5:')
sample = model.sample(
sess, prime_string=u'"If only you knew,"',
sample_length=500)
print(u'sample: \n{}'.format(sample))
# from this i conclude, that there are 15 views on one tetris object
# ptest[0..64:tetris_block][0:64x64 image->, 1:locational data->][0..15:views or locdata][64x64xrgb data]
ptest = torch.load("./our_2d_data/train/Pilt1_not0.pt")
print(len(ptest), np.shape(ptest[0]))
lookadoodle(ptest[0][0], ptest[0][1])
print("\nfirst image loc values:", ptest[1][0], "\nsecond image loc values:",
ptest[1][1])
# Lets look ad the data_reader.py here with the shepard metzler data
print('\nmetzler 5 parts in DataReader:')
_NUM_CHANNELS = 3
# this is the camera dimension parameter length modifier - let's keep it at 5 :)
_NUM_RAW_CAMERA_PARAMS = 5
dat = dr.DataReader("shepard_metzler_5_parts",
3,
"./shepard_metzler_5_parts",
mode='train')
file_names = dr._get_dataset_files(dat._dataset_info, 'train',
"./shepard_metzler_5_parts")
filename_queue = tf.train.string_input_producer(file_names, seed=99)
reader = tf.TFRecordReader()
_, raw_data = reader.read_up_to(filename_queue, num_records=16)
feature_map = {
'frames':
tf.FixedLenFeature(shape=dat._dataset_info.sequence_size, dtype=tf.string),
'cameras':
tf.FixedLenFeature(
shape=[dat._dataset_info.sequence_size * _NUM_RAW_CAMERA_PARAMS],
dtype=tf.float32)
}
)
val_acc.append(val_acc_)
val_loss.append(val_loss_)
val_mask1_loss.append(val_mask1_loss_)
val_mask2_loss.append(val_mask2_loss_)
val_class_loss.append(val_class_loss_)
val_acc_m = np.mean(val_acc)
val_loss_m = np.mean(val_loss)
val_mask1_loss_m = np.mean(val_mask1_loss)
val_mask2_loss_m = np.mean(val_mask2_loss)
val_class_loss_m = np.mean(val_class_loss)
sum_val_ = tf.Summary(value=[
tf.Summary.Value(tag="b1-val_acc", simple_value=val_acc_m),
tf.Summary.Value(tag="b2-val_loss", simple_value=val_loss_m),
tf.Summary.Value(tag="b3-val_mask1_loss", simple_value=val_mask1_loss_m),
tf.Summary.Value(tag="b4-val_mask2_loss", simple_value=val_mask2_loss_m),
tf.Summary.Value(tag="b5-val_class_loss", simple_value=val_class_loss_m)
])
writer.add_summary(sum_val_, counter)
pool.close()
pool.join()
if (e + 1) % 100 == 0:
saver.save(sess, r'./weights/model_epoch_{}.ckpt'.format(e))
if __name__ == '__main__':
import data_reader
datasets_path = r'./datasets'
reader = data_reader.DataReader(datasets_path, is_training=True)
main(reader)
if cosineSimilarity > max_cosineSimilarity:
max_cosineSimilarity = cosineSimilarity
max_index = i
return answer_list[max_index]
# Launch the graph
with tf.Session() as sess:
# sess = tf.Session('', tf.Graph())
# with sess.graph.as_default():
# Initializing the variables
sess.run(tf.global_variables_initializer())
# import data
dr = data_reader.DataReader(is_training=True)
valid_x, valid_y = dr.validation_batch(validation_size)
valid_x = valid_x.reshape((validation_size, n_steps, n_input))
valid_y = valid_y.reshape((validation_size, n_input))
epoch_size = dr.epoch_size(batch_size)
init_learning_rate = learning_rate
# training
for epoch in range(epoches):
ts = time.time()
learning_rate = init_learning_rate / (epoch + 1)
for batch_num in range(epoch_size):
batch_x, batch_y = dr.next_batch(batch_size)
# Reshape data to get n_steps seq of n_input elements
batch_x = batch_x.reshape((batch_size, n_steps, n_input))
"""
Author : Byunghyun Ban
Date : 2020.07.24.
"""
import tensorflow_hub as hub
import data_reader
# 데이터를 불러옵니다.
dr = data_reader.DataReader("content.jpg", "style.jpg")
# Hub로부터 style transfer 모듈을 불러옵니다.
hub_module = hub.load(
'https://tfhub.dev/google/magenta/arbitrary-image-stylization-v1-256/1')
# 모듈에 이미지를 삽입해 Style Transfer를 실시합니다.
stylized_image = hub_module(dr.content, dr.style)[0]
# 결과를 출력합니다.
result = data_reader.tensor_to_image(stylized_image)
# 결과를 저장합니다.
result.save("result.jpg")
def read_file(self, path):
event_reader = dr.DataReader(path)
events = event_reader.read_jet_constituents()
dijet_features = event_reader.read_jet_features()
labels = event_reader.read_labels()
return [events[:, 0, :, :], events[:, 1, :, :], dijet_features, labels]
if test_case[key] in word_vector:
midContextVector = word_vector[test_case[key]]
else:
midContextVector = np.zeros(WORD_VECTOR_SIZE)
embedded_test_X.append([preContextVector, midContextVector])
embedded_test_Y.append(postContextVector)
pickle.dump((np.array(embedded_test_X), np.array(embedded_test_Y),
len(embedded_test_X)), open('data/embedded_test_data', 'wb'),
True)
saver = tf.train.Saver()
with tf.Session() as sess:
saver.restore(sess, pretrained_model)
print("Model restored.")
# testing
ts = time.time()
dr = data_reader.DataReader(is_training=False)
with open(sys.argv[2], 'w') as out:
out.write('id,answer\n')
test_case = 1
while dr.check_test_index() == True:
test_x, test_y = dr.next_test(batch_size)
tp = sess.run(predict, feed_dict={x: test_x})
predict_answer = test_predictAnswer(tp[0:5], test_y)
out.write("%s,%s\n" % (test_case, predict_answer))
test_case += 1
"Log placement of ops on devices")
FLAGS = tf.flags.FLAGS
args = parser.parse_args()
print("dc", args.doc_len)
max_sen_length = args.sen_len
max_doc_length = args.doc_len
logging.info('generate config')
word_vocab, word_tensors, max_doc_length, label_tensors = \
dr.load_data(args.train_file, max_doc_length, max_sen_length)
batch_size = 1
time1 = time.time()
test_reader = dr.DataReader(word_tensors['test'], label_tensors['test'],
batch_size)
graph = tf.Graph()
with graph.as_default():
session_conf = tf.ConfigProto(
allow_soft_placement=FLAGS.allow_soft_placement,
log_device_placement=FLAGS.log_device_placement)
sess = tf.Session(config=session_conf)
with sess.as_default():
# Checkpoint directory. Tensorflow assumes this directory already exists so we need to create it
'''
with tf.variable_scope("Model"):
#m = build_model(word_vocab)
global_step = tf.Variable(0, dtype=tf.int32, name='global_step')
'''
#saver = tf.train.Saver()
saver = tf.train.import_meta_graph(
stock_outputs = self.outputs[stock]
split_percent = .80
split_pos = int(len(stock_data) * split_percent)
print split_pos
train_x = stock_data[:split_pos]
self.test_x = stock_data[split_pos:]
train_y = stock_outputs[:split_pos]
self.test_y = stock_outputs[split_pos:]
self.clf.fit(train_x, train_y)
def predict(self):
return self.clf.predict(self.test_x)
def evaluate(self):
return self.predict() - self.test_y
def score(self):
return self.clf.score(self.test_x, self.test_y)
if __name__ == "__main__":
dr = data_reader.DataReader("data/all_stocks_5yr.csv")
dr.gen_outputs()
trainer = Trainer(dr.data, dr.outputs, linear_model.LinearRegression())
for stock in dr.data:
trainer.train(stock)
print stock, trainer.score()
def train(w2v_model):
# Training
# ==================================================
max_sen_length = 40
max_doc_length = 90
word_vocab, word_tensors, max_doc_length, label_tensors = \
\
dr.load_data(FLAGS.train_data_file, max_doc_length, max_sen_length)
train_reader = dr.DataReader(word_tensors['train'], label_tensors['train'],
1)
valid_reader = dr.DataReader(word_tensors['valid'], label_tensors['valid'],
1)
test_reader = dr.DataReader(word_tensors['test'], label_tensors['test'], 1)
pretrained_embedding = dr.get_embed(word_vocab)
print("ppp", pretrained_embedding.shape)
#x_train, x_dev, y_train, y_dev ,vocab_siz, pretrained_embedding= load_data(w2v_model)
embedding_size = 150
with tf.Graph().as_default():
session_conf = tf.ConfigProto(
allow_soft_placement=FLAGS.allow_soft_placement,
log_device_placement=FLAGS.log_device_placement)
sess = tf.Session(config=session_conf)
with sess.as_default():
#需修改
'''
cnn = TextCNN(
w2v_model,
sequence_length=x_train.shape[1],
num_classes=y_train.shape[1],
vocab_size=vocab_size,
embedding_size=FLAGS.embedding_dim,
filter_sizes=list(map(int, FLAGS.filter_sizes.split(","))),
num_filters=FLAGS.num_filters,
l2_reg_lambda=FLAGS.l2_reg_lambda)
'''
# Checkpoint directory. Tensorflow assumes this directory already exists so we need to create it
print word_vocab.size
Summa = SummaRuNNer(word_vocab.size, embedding_size,
pretrained_embedding)
#loss_sum = tf.Variable(initial_value=0, dtype=tf.float32)
global_step = tf.Variable(0, name="global_step", trainable=False)
'''
optimizer = tf.train.AdamOptimizer(1e-3)
grads_and_vars = optimizer.compute_gradients(Summa.loss)
train_op = optimizer.apply_gradients(grads_and_vars, global_step=global_step)
'''
train_params = tf.trainable_variables()
train_op = tf.train.AdamOptimizer(learning_rate=0.0001).minimize(
Summa.loss, var_list=train_params)
timestamp = str(int(time.time()))
out_dir = os.path.abspath(
os.path.join(os.path.curdir, "runs", timestamp))
checkpoint_dir = os.path.abspath(
os.path.join(out_dir, "checkpoints"))
checkpoint_prefix = os.path.join(checkpoint_dir, "model")
if not os.path.exists(checkpoint_dir):
os.makedirs(checkpoint_dir)
saver = tf.train.Saver(tf.global_variables(),
max_to_keep=FLAGS.num_checkpoints)
checkpoint_dir = os.path.abspath(
os.path.join(out_dir, "checkpoints"))
checkpoint_prefix = os.path.join(checkpoint_dir, "model")
batches = train_reader
valid_batches = valid_reader
sess.run(tf.global_variables_initializer())
#step = 0
min_eval_loss = float('Inf')
#fetch_list = [Summa.xks, Summa.data_c, Summa.result]
for epoch in range(FLAGS.num_epochs):
step = 0
loss_sum = 0
#loss_sum.assign(value=0)
#value_sum = 0
for x_batch, y_batch in batches.iter():
step += 1
feed_dict = {
Summa.x: x_batch[0],
Summa.y: y_batch[0],
}
'''
lucky_boy, lucky_girl, data_cc = sess.run(fetch_list, feed_dict)
print ("lucky_boy, ", lucky_boy)
print ("lucky_girl, ", lucky_girl)
print ("data_cc:", data_cc)
'''
sess.run(train_op, feed_dict)
loss = sess.run([Summa.loss], feed_dict)
predict = sess.run([Summa.y_], feed_dict)
loss_sum += loss[0]
#print predict
#grads_and_vars = optimizer.compute_gradients(Summa.loss)
#train_op = optimizer.apply_gradients(grads_and_vars, global_step=global_step)
#print step
#print len(y_batch[0])
#print len(predict[0])
#print step % 128
#print step
if step % 128 == 0 and step != 0:
#print ("here")
#logging.info('Epoch ' + str(epoch) + ' Loss: ' + str(loss_sum / 128.0))
print('Epoch ' + str(epoch) + ' Loss: ' +
str(loss_sum / 128.0))
loss_sum = 0
if step % 512 == 0 and step != 0:
eval_loss = 0
for x_batch, y_batch in valid_batches.iter():
feed_dict = {
Summa.x: x_batch[0],
Summa.y: y_batch[0],
}
loss = sess.run([Summa.loss], feed_dict)
eval_loss += loss[0]
print('epoch ' + str(epoch) + ' Loss in validation: ' +
str(eval_loss * 1.0 / valid_reader.length))
if eval_loss < min_eval_loss:
min_eval_loss = eval_loss
path = saver.save(sess,
checkpoint_prefix,
global_step=step)
print(
"Saved model checkpoint to {}\n".format(path))
import os
# from mpl_toolkits.mplot3d import Axes3D # noqa: F401 unused import
# from importlib import reload # Python 3.4+ only.
# reload(plotter_bayesian_data)
# plter = plotter_bayesian_data.Plotter('./', save_figs=False)
# choose which data should be used
no_spatial_conv = True
n_draws = 200
plter = plotter_bayesian_data.Plotter('./', save_figs=False)
############### READ DATA ###############
reader = data_reader.DataReader(
not no_spatial_conv,
path=
'/media/oesst/TOSHIBA EXT/Backup PHD Stuff/Code/Python/msi_network_model')
# get the means and variances of the results
means, vars = reader.get_means_variances()
intensities = reader.get_s_intensities()
s_locations = reader.get_s_locations()
s_variances = reader.get_s_variance()
# caluclate the percentage of fused event in a simulation
fusion = reader.get_fusion()
fusion_fb_on = fusion[:, :, 0]
fusion_fb_off = fusion[:, :, 1]
# # %% Save the data for faster excess
# exp_dir = os.path.join(os.getcwd(), 'Combined_Results')
def train(w2v_model):
# Training
# ==================================================
max_sen_length = 40
max_doc_length = 90
word_vocab, word_tensors, max_doc_length, label_tensors = \
\
dr.load_data(FLAGS.train_data_file, max_doc_length, max_sen_length)
train_reader = dr.DataReader(word_tensors['train'], label_tensors['train'],
1)
valid_reader = dr.DataReader(word_tensors['valid'], label_tensors['valid'],
1)
test_reader = dr.DataReader(word_tensors['test'], label_tensors['test'], 1)
pretrained_embedding = dr.get_embed(word_vocab)
embedding_size = 150
with tf.Graph().as_default():
session_conf = tf.ConfigProto(
allow_soft_placement=FLAGS.allow_soft_placement,
log_device_placement=FLAGS.log_device_placement)
sess = tf.Session(config=session_conf)
with sess.as_default():
print word_vocab.size
Summa = SummaRuNNer(word_vocab.size, embedding_size,
pretrained_embedding)
#loss_sum = tf.Variable(initial_value=0, dtype=tf.float32)
global_step = tf.Variable(0, name="global_step", trainable=False)
train_params = tf.trainable_variables()
train_op = tf.train.AdamOptimizer(learning_rate=0.0001).minimize(
Summa.loss, var_list=train_params)
timestamp = str(int(time.time()))
out_dir = os.path.abspath(
os.path.join(os.path.curdir, "runs", timestamp))
checkpoint_dir = os.path.abspath(
os.path.join(out_dir, "checkpoints"))
checkpoint_prefix = os.path.join(checkpoint_dir, "model")
if not os.path.exists(checkpoint_dir):
os.makedirs(checkpoint_dir)
saver = tf.train.Saver(tf.global_variables(),
max_to_keep=FLAGS.num_checkpoints)
checkpoint_dir = os.path.abspath(
os.path.join(out_dir, "checkpoints"))
checkpoint_prefix = os.path.join(checkpoint_dir, "model")
batches = train_reader
valid_batches = valid_reader
sess.run(tf.global_variables_initializer())
#step = 0
min_eval_loss = float('Inf')
for epoch in range(FLAGS.num_epochs):
step = 0
loss_sum = 0
for x_batch, y_batch in batches.iter():
step += 1
feed_dict = {
Summa.x: x_batch[0],
Summa.y: y_batch[0],
}
sess.run(train_op, feed_dict)
loss = sess.run([Summa.loss], feed_dict)
predict = sess.run([Summa.y_], feed_dict)
loss_sum += loss[0]
if step % 128 == 0 and step != 0:
print('Epoch ' + str(epoch) + ' Loss: ' +
str(loss_sum / 128.0))
loss_sum = 0
if step % 512 == 0 and step != 0:
eval_loss = 0
for x_batch, y_batch in valid_batches.iter():
feed_dict = {
Summa.x: x_batch[0],
Summa.y: y_batch[0],
}
loss = sess.run([Summa.loss], feed_dict)
eval_loss += loss[0]
print('epoch ' + str(epoch) + ' Loss in validation: ' +
str(eval_loss * 1.0 / valid_reader.length))
if eval_loss < min_eval_loss:
min_eval_loss = eval_loss
path = saver.save(sess,
checkpoint_prefix,
global_step=step)
print(
"Saved model checkpoint to {}\n".format(path))
