class Evaluator(TrainEvalBase):
def __init__(self, model, loss_fn, graph, input_reader):
"""Initialize an `Evaluator` object.
Args:
model: an instance of a subclass of the `ModelBase` class (defined in
`model_base.py`).
loss_fn: a tensorflow op, a loss function for training a model. See:
https://www.tensorflow.org/code/tensorflow/contrib/losses/python/losses/loss_ops.py
for a list of available loss functions.
graph: a tensorflow computation graph.
input_reader: an instance of a subclass of the `InputReaderBase` class
(defined in `input_reader_base.py`).
"""
self._config = HParams(batch_size=128,
num_batches=400,
eval_interval_secs=100)
if FLAGS.eval_config:
self._config.parse(FLAGS.eval_config)
super(Evaluator, self).__init__(model, loss_fn, FLAGS.data_path, FLAGS.logdir, graph, input_reader)
# TODO: make the evaluating metrics customizable.
def _compute_loss_and_other_metrics(self):
"""Compute loss function and other evaluating metrics."""
self._compute_loss()
probabilities = tf.sigmoid(self._outputs)
predictions = tf.argmax(self._outputs, dimension=1)
truths = tf.argmax(self._labels, dimension=1)
metrics_to_values, self._metrics_to_updates = slim.metrics.aggregate_metric_map(
{
"Accuracy" : slim.metrics.streaming_accuracy(predictions, truths),
"Loss_Eval" : slim.metrics.streaming_mean(self._loss),
})
for metric_name, metric_value in metrics_to_values.iteritems():
self._summary_ops.append(tf.scalar_summary(metric_name, metric_value))
def run(self):
"""Run evaluation."""
# Create logging directory if not exists.
if not os.path.isdir(self._eval_log_dir):
os.makedirs(self._eval_log_dir)
# Compute loss function and other evaluating metrics.
self._initialize()
# Visualize input images in Tensorboard.
self._summary_ops.append(tf.image_summary("Eval_Image", self._observations, max_images=5))
# Use `slim.evaluation.evaluation_loop` to evaluate the model periodically.
slim.evaluation.evaluation_loop(
master='',
checkpoint_dir=self._train_log_dir,
logdir=self._eval_log_dir,
num_evals=self._config.num_batches,
eval_op=self._metrics_to_updates.values(),
summary_op=tf.merge_summary(self._summary_ops),
eval_interval_secs=self._config.eval_interval_secs)
def _default_hparams():
"""Returns default or overridden user-specified hyperparameters."""
hparams = HParams(learning_rate=1.0e-3)
if FLAGS.hparams:
hparams = hparams.parse(FLAGS.hparams)
return hparams
class Trainer(TrainEvalBase):
def __init__(self, model, loss_fn, graph, input_reader):
"""Initialize a `Trainer` object.
Args:
model: an instance of a subclass of the `ModelBase` class (defined in
`model_base.py`).
loss_fn: a tensorflow op, a loss function for training a model. See:
https://www.tensorflow.org/code/tensorflow/contrib/losses/python/losses/loss_ops.py
for a list of available loss functions.
graph: a tensorflow computation graph.
input_reader: an instance of a subclass of the `InputReaderBase` class
(defined in `input_reader_base.py`).
"""
self._config = HParams(learning_rate=0.1,
batch_size=16,
train_steps=10000,
save_summaries_secs=100,
save_interval_secs=100)
if FLAGS.train_config:
self._config.parse(FLAGS.train_config)
super(Trainer, self).__init__(model, loss_fn, FLAGS.data_path,
FLAGS.logdir, graph, input_reader)
def _compute_loss_and_other_metrics(self):
"""Compute loss function."""
self._compute_loss()
self._summary_ops.append(tf.scalar_summary('Loss_Train', self._loss))
def run(self):
"""Run training."""
# Create logging directory if not exists.
if not os.path.isdir(self._train_log_dir):
os.makedirs(self._train_log_dir)
# Load data and compute loss function
self._initialize()
# Visualize input images in Tensorboard.
self._summary_ops.append(tf.image_summary("Image_Train", self._observations, max_images=5))
# Initialize optimizer.
optimizer = tf.train.AdadeltaOptimizer(self._config.learning_rate)
train_op = slim.learning.create_train_op(self._loss, optimizer)
# Use `slim.learning.train` to manage training.
slim.learning.train(train_op=train_op,
logdir=self._train_log_dir,
graph=self._graph,
number_of_steps=self._config.train_steps,
summary_op=tf.merge_summary(self._summary_ops),
save_summaries_secs=self._config.save_summaries_secs,
save_interval_secs=self._config.save_interval_secs)
def __init__(self, session, training, hparams = None):
self.session = session
self.training = training
print("# Prepare dataset placeholder and hyper parameters ...")
#Load Hyper-parameters file
if hparams is None:
self.hparams = HParams(SYSTEM_ROOT).hparams
else:
self.hparams = hparams
# Initializer
initializer = self.get_initializer(self.hparams.init_op,
self.hparams.random_seed,
self.hparams.init_weight)
tf.get_variable_scope().set_initializer(initializer)
self.tokenized_data = TokenizedData(hparams = self.hparams, training = self.training)
self.vocab_list = self.tokenized_data.vocab_list
self.vocab_size = self.tokenized_data.vocab_size
self.vocab_table = self.tokenized_data.vocab_table
self.reverse_vocab_table = self.tokenized_data.reverse_vocab_table
if self.training:
self.build_train_model()
#tensorboard
self.train_summary_writer = tf.summary.FileWriter(TRAIN_LOG_DIR+self.train_mode, self.session.graph)
self.test_summary_writer = tf.summary.FileWriter(TEST_LOG_DIR+self.train_mode)
else:
self.build_predict_model()
#Tensorboard
tf.summary.FileWriter(INFER_LOG_DIR, self.session.graph)
def __init__(self, training, hparams = None):
self.training = training
print("# Prepare dataset placeholder and hyper parameters ...")
#Load Hyper-parameters file
self.hparams = hparams if hparams else HParams().hparams
if self.hparams.use_gpu:
self.config = tf.ConfigProto(allow_soft_placement=True)
tf.GPUOptions(per_process_gpu_memory_fraction=0.7)
self.config.gpu_options.allow_growth = True
else:
self.config = None
# Enable cluster server
self.enable_cluster = (self.hparams.job_name != None)
self.is_chief = (self.hparams.task_index == 0)
if self.enable_cluster:
self.build_server()
else:
with tf.Graph().as_default() as graph:
self.session1 = tf.Session(config=self.config)
# self.session2 = tf.Session(config=self.config)
self.graph = graph
self.build_env()
self.train()
def get_default_hparams():
return HParams(batch_size=128,
num_steps=20,
num_shards=8,
num_layers=1,
learning_rate=0.2,
max_grad_norm=10.0,
num_delayed_steps=150,
keep_prob=0.9,
optimizer=0,
vocab_size=793470,
emb_size=512,
state_size=2048,
projected_size=512,
num_sampled=8192,
num_gpus=8,
float16_rnn=False,
float16_non_rnn=False,
average_params=True,
run_profiler=False,
do_summaries=False,
max_time=180,
fact_size=None,
fnon_linearity="none",
num_of_groups=0,
save_model_every_min=30,
save_summary_every_min=16,
do_sharing=False,
use_residual=False)
def default_hparams():
return HParams(
n_vocab= 0,
n_ctx= 1024,
n_embd= 768,
n_head= 12,
n_layer= 12,
)
def __init__(self, is_training):
"""Initialize a `ModelCifar10` object.
Args:
is_training: a bool, whether the model is used for training or
evaluation.
"""
self._hparams = HParams(nums_conv_filters=[64, 64],
conv_filter_sizes=[3, 3],
pooling_size=2,
pooling_stride=2,
dropout_prob=0.5,
regularize_constant=0.004,
init_stddev=5e-2)
if FLAGS.model_hparams:
self._hparams.parse(FLAGS.model_hparams)
super(ModelCifar10, self).__init__(is_training)
def test_basic(self):
hps = HParams(int_value=13,
float_value=17.5,
bool_value=True,
str_value="test")
self.assertEqual(hps.int_value, 13)
self.assertEqual(hps.float_value, 17.5)
self.assertEqual(hps.bool_value, True)
self.assertEqual(hps.str_value, "test")
def test_parse(self):
hps = HParams(int_value=13,
float_value=17.5,
bool_value=True,
str_value="test")
self.assertEqual(hps.parse("int_value=10").int_value, 10)
self.assertEqual(hps.parse("float_value=10").float_value, 10)
self.assertEqual(hps.parse("float_value=10.3").float_value, 10.3)
self.assertEqual(hps.parse("bool_value=true").bool_value, True)
self.assertEqual(hps.parse("bool_value=True").bool_value, True)
self.assertEqual(hps.parse("bool_value=false").bool_value, False)
self.assertEqual(hps.parse("str_value=value").str_value, "value")
def get_default_hparams():
return HParams(
batch_size=100,
num_steps=128,
learning_rate=0.01,
num_delayed_steps=150,
keep_prob=0.9,
vocab_size=1562,
emb_size=50,
)
def main():
hparams = HParams().hparams
params_server = hparams.ps_hosts.split(",")
worker_server = hparams.worker_hosts.split(",")
# create cluster
cluster = tf.train.ClusterSpec({
"ps": params_server,
"worker": worker_server
})
# create the server
server = tf.train.Server(cluster,
job_name=hparams.job_name,
task_index=hparams.task_index)
server.join()
with tf.device('/job:ps/task:0/cpu:0'):
input_data = tf.Variable(
[[1., 2., 3.], [4., 5., 6.], [7., 8., 9.], [10., 11., 12.]],
name="input_data")
b = tf.Variable([[1.], [1.], [2.]], name="w")
inputs = tf.split(input_data, 2)
outputs = []
# Track statistics of the run using Timeline
run_options = tf.RunOptions(trace_level=tf.RunOptions.FULL_TRACE)
run_metadata = tf.RunMetadata()
if hparams.job_name == 'ps':
server.join()
else:
# 图内复制，只在worker0上创建client
with tf.Session("grpc://localhost:2223") as sess:
init_op = tf.global_variables_initializer()
sess.run(init_op)
for i in range(2):
with tf.device("/job:worker/task:%d/gpu:0" % i):
print("now is worker %d: " % i)
print(sess.run(inputs[i]))
outputs.append(tf.matmul(inputs[i], b))
with tf.device('/job:ps/task:0/cpu:0'):
output = tf.concat(outputs, axis=0)
print(
sess.run(output,
options=run_options,
run_metadata=run_metadata))
# for tensorboard
tf.summary.FileWriter("logs/", sess.graph)
# Create timeline and write it to a json file
tl = timeline.Timeline(step_stats=run_metadata.step_stats)
ctf = tl.generate_chrome_trace_format()
with open('timeline_client.json', 'w') as f:
f.write(ctf)
def test_parse(self):
hps = HParams(int_value=13, float_value=17.5, bool_value=True, str_value="test")
self.assertEqual(hps.parse("int_value=10").int_value, 10)
self.assertEqual(hps.parse("float_value=10").float_value, 10)
self.assertEqual(hps.parse("float_value=10.3").float_value, 10.3)
self.assertEqual(hps.parse("bool_value=true").bool_value, True)
self.assertEqual(hps.parse("bool_value=True").bool_value, True)
self.assertEqual(hps.parse("bool_value=false").bool_value, False)
self.assertEqual(hps.parse("str_value=value").str_value, "value")
def __init__(self, model, loss_fn, graph, input_reader):
"""Initialize an `Evaluator` object.
Args:
model: an instance of a subclass of the `ModelBase` class (defined in
`model_base.py`).
loss_fn: a tensorflow op, a loss function for training a model. See:
https://www.tensorflow.org/code/tensorflow/contrib/losses/python/losses/loss_ops.py
for a list of available loss functions.
graph: a tensorflow computation graph.
input_reader: an instance of a subclass of the `InputReaderBase` class
(defined in `input_reader_base.py`).
"""
self._config = HParams(batch_size=128,
num_batches=400,
eval_interval_secs=100)
if FLAGS.eval_config:
self._config.parse(FLAGS.eval_config)
super(Evaluator, self).__init__(model, loss_fn, FLAGS.data_path, FLAGS.logdir, graph, input_reader)
def load_data_test():
hp = HParams(
data_filename='date.txt',
vocab_filename = 'vocab.pickle',
model_save_dir = './saved_model',
checkpoint_name = 'model_ckpt',
MAX_LEN = 29,
batch_size = 32,
num_epoch = 10,
num_layers = 6,
hidden_dim = 32,
dff = 128,
num_heads = 8,
drop_rate = 0.1,
)
train_flag = True
if train_flag:
inputs, targets, word_to_index, index_to_word, VOCAB_SIZE, INPUT_LENGTH, OUTPUT_LENGTH = load_data(hp,train_flag=train_flag) # (50000, 29), (50000, 12)
print(word_to_index)
hp['vocab_size'] = VOCAB_SIZE # set_hparam도 있음.
train_input, test_input, train_target, test_target = train_test_split(inputs, targets, test_size=0.1, random_state=13371447)
print('train_input: {},test_input: {},train_target: {},test_target: {}'.format(train_input.shape,test_input.shape,train_target.shape,test_target.shape))
dataset = tf.data.Dataset.from_tensor_slices((train_input, train_target)) # 여기의 argument가 mapping_fn의 argument가 된다.
dataset = dataset.shuffle(buffer_size=hp['batch_size']*10)
dataset = dataset.batch(hp['batch_size'],drop_remainder=False)
for i,(x,y) in enumerate(dataset):
if i% 300 == 0:
print(x.shape,y.shape)
print('='*10)
for i,(x,y) in enumerate(dataset):
if i% 1000 == 0:
print(x.shape,y.shape)
else:
inputs, word_to_index, index_to_word, VOCAB_SIZE, INPUT_LENGTH = load_data(hp,'test_data.txt',train_flag=train_flag)
print(word_to_index)
hp['vocab_size'] = VOCAB_SIZE # set_hparam도 있음.
def hparams_tagging_base():
return HParams(
batch_size=100,
embedding_size_word=300,
embedding_size_char=0, # No char embeddings
embedding_size_char_per_word=100,
embedding_size_tags=100,
hidden_size=128,
learning_rate=0.2,
learning_rate_decay=None,
max_length=256,
num_hidden_layers=1,
dropout=0.2,
optimizer_adam_beta1=0.9,
optimizer_adam_beta2=0.98,
use_crf=False)
def main():
hparams = HParams()
model = Model(hparams, mode="infer")
# placeholder
video = model.video
video_mask = model.video_mask
activation_map = model.activation_map # dec_max_len, enc_max_len
# probs.shape=(batch_size, time_step, vocab_size)
probs = tf.transpose(model.probs, [0, 2, 1])
print video, video_mask, probs
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
session = tf.Session(config=config)
# saver
saver = tf.train.Saver()
print "reload weight from file"
saver.restore(session, "/dl_data/video_caption/s2vc-ckpt-35000")
for filename in glob.glob(
"/dl_data/video_caption/s2vc/data_preprocess/vgg/train_data_30/*.avi"
):
print filename
video_data, video_mask_data = get_data(hparams, filename)
heatmap_filename = filename[len(os.path.dirname(filename)) + 1:][:-4]
prediction, activation_map_data = session.run([probs, activation_map],
feed_dict={
video:
video_data,
video_mask:
video_mask_data
})
prediction = prediction[0]
prediction = np.argmax(prediction, axis=1)
id2word = hparams.id2word
p = []
for item in prediction:
p.append(id2word[int(item)])
# activation_map_data.shape=(dec_max_len, enc_max_len)
attention_map(activation_map_data,
x_ticks=None,
y_ticks=p,
store_path=os.path.join("attention_maps",
heatmap_filename))
print " ".join(p)
def get_test_hparams():
return HParams(
batch_size=21,
num_steps=12,
num_shards=2,
num_layers=1,
learning_rate=0.2,
max_grad_norm=1.0,
vocab_size=1000,
emb_size=14,
state_size=17,
projected_size=15,
num_sampled=500,
num_gpus=1,
average_params=True,
run_profiler=False,
)
def __init__(self,
corpus_dir,
hparams=None,
training=True,
buffer_size=8192):
"""
Args:
corpus_dir: Name of the folder storing corpus files for training.
hparams: The object containing the loaded hyper parameters. If None, it will be
initialized here.
training: Whether to use this object for training.
buffer_size: The buffer size used for mapping process during data processing.
"""
if hparams is None:
self.hparams = HParams(corpus_dir).hparams
else:
self.hparams = hparams
self.src_max_len = self.hparams.src_max_len
self.tgt_max_len = self.hparams.tgt_max_len
self.cant_max_len = self.hparams.cant_max_len
self.training = training
self.text_set = None
self.id_set = None
vocab_file = os.path.join(corpus_dir, VOCAB_FILE)
self.vocab_size, _ = check_vocab(vocab_file)
self.vocab_table = lookup_ops.index_table_from_file(
vocab_file, default_value=self.hparams.unk_id)
print("vocab_size = {}".format(self.vocab_size))
if training:
self.case_table = prepare_case_table()
self.reverse_vocab_table = None
self.load_corpus(corpus_dir)
# self._load_corpus(corpus_dir)
self._convert_to_tokens(buffer_size)
else:
self.case_table = None
self.reverse_vocab_table = \
lookup_ops.index_to_string_table_from_file(vocab_file,
default_value=self.hparams.unk_token)
def get_default_hparams():
return HParams(
batch_size=128,
num_steps=20,
num_shards=8,
num_layers=1,
learning_rate=0.2,
max_grad_norm=10.0,
num_delayed_steps=150,
keep_prob=0.9,
vocab_size=793470,
emb_size=512,
state_size=2048,
projected_size=512,
num_sampled=8192,
num_gpus=1,
average_params=True,
run_profiler=False,
)
def __init__(self, hparams=None, tokenizer=None):
self.hparams = hparams if hparams else HParams().hparams
self.tokenizer = tokenizer if tokenizer else Tokenizer(
hparams, VOCAB_FILE)
self.bert_config = modeling.BertConfig.from_json_file(BERT_CONFIG_FILE)
self.graph = tf.Graph()
with self.graph.as_default():
with tf.variable_scope('placeholder'):
self.input_ids = tf.placeholder(shape=[None, None],
dtype=tf.int32,
name="input_ids")
self.input_mask = tf.placeholder(shape=[None, None],
dtype=tf.int32,
name="input_mask")
self.segment_ids = tf.placeholder(shape=[None, None],
dtype=tf.int32,
name="segment_ids")
self.model = modeling.BertModel(config=self.bert_config,
is_training=False,
input_ids=self.input_ids,
input_mask=self.input_mask,
token_type_ids=self.segment_ids,
use_one_hot_embeddings=False,
scope="bert",
reuse=tf.AUTO_REUSE)
tvars = tf.trainable_variables()
(assignment_map, initialized_variable_names
) = modeling.get_assignment_map_from_checkpoint(
tvars, BERT_PARAMS_FILE)
tf.train.init_from_checkpoint(BERT_PARAMS_FILE, assignment_map)
self.variables = [v for v in tf.global_variables()]
self.context_vector = tf.identity(self.model.get_sequence_output(),
name="context_vector")
self.finetuning_vector = tf.identity(
self.model.get_pooled_output(), name="finetuning_vector")
def get_default_hparams():
# return HParams(
# batch_size=128,
# num_steps=20,
# num_shards=1,
# num_layers=1,
# learning_rate=0.2,
# max_grad_norm=10.0,
# num_delayed_steps=150,
# keep_prob=0.9,
#
# vocab_size=42466,
# emb_size=512,
# state_size=1024,
# projected_size=512,
# num_sampled=1000,
# num_gpus=1,
#
# average_params=True,
# run_profiler=False,
# )
return HParams(
batch_size=128,
num_steps=20,
num_shards=1,
num_layers=1,
learning_rate=0.2,
max_grad_norm=10.0,
num_delayed_steps=150,
keep_prob=0.9,
vocab_size=42466,
emb_size=100,
state_size=100,
projected_size=100,
num_sampled=2000,
num_gpus=1,
average_params=True,
run_profiler=False,
)
class ModelCifar10(ModelBase):
def __init__(self, is_training):
"""Initialize a `ModelCifar10` object.
Args:
is_training: a bool, whether the model is used for training or
evaluation.
"""
self._hparams = HParams(nums_conv_filters=[64, 64],
conv_filter_sizes=[3, 3],
pooling_size=2,
pooling_stride=2,
dropout_prob=0.5,
regularize_constant=0.004,
init_stddev=5e-2)
if FLAGS.model_hparams:
self._hparams.parse(FLAGS.model_hparams)
super(ModelCifar10, self).__init__(is_training)
def arg_scope(self):
"""Configure the neural network's layers."""
batch_norm_params = {
"is_training": self.is_training,
"decay": 0.9997,
"epsilon": 0.001,
"variables_collections": {
"beta": None,
"gamma": None,
"moving_mean": ["moving_vars"],
"moving_variance": ["moving_vars"]
}
}
with slim.arg_scope(
[slim.conv2d, slim.fully_connected],
weights_initializer=tf.truncated_normal_initializer(
stddev=self._hparams.init_stddev),
weights_regularizer=slim.l2_regularizer(
self._hparams.regularize_constant),
activation_fn=tf.nn.relu,
normalizer_fn=slim.batch_norm,
normalizer_params=batch_norm_params) as sc:
return sc
def compute(self, inputs):
"""Compute a batch of outputs of the neural network from a batch of inputs.
Args:
inputs: a tensorflow tensor, a batch of input images. Each image is of
size InputReaderCifar10.IMAGE_SIZE x InputReaderCifar10.IMAGE_SIZE x
InputReaderCifar10.NUM_CHANNELS.
Returns:
net: a tensorflow op, output of the network.
embedding: a tensorflow op, output of the embedding layer (the second
last fully connected layer).
"""
hparams = self._hparams
net = None
num_pool_conv_layers = len(hparams.nums_conv_filters)
for i in xrange(num_pool_conv_layers):
net = slim.conv2d(
inputs if i == 0 else net,
hparams.nums_conv_filters[i],
[hparams.conv_filter_sizes[i], hparams.conv_filter_sizes[i]],
padding="SAME",
biases_initializer=tf.constant_initializer(0.1 * i),
scope="conv_{0}".format(i))
net = slim.max_pool2d(net,
[hparams.pooling_size, hparams.pooling_size],
hparams.pooling_stride,
scope="pool_{0}".format(i))
net = slim.flatten(net, scope="flatten")
net = slim.fully_connected(
net,
384,
biases_initializer=tf.constant_initializer(0.1),
scope="fc_{0}".format(num_pool_conv_layers))
net = slim.dropout(net,
hparams.dropout_prob,
scope="dropout_{0}".format(num_pool_conv_layers))
embedding = slim.fully_connected(
net,
192,
biases_initializer=tf.constant_initializer(0.1),
scope="fc_{0}".format(num_pool_conv_layers + 1))
net = slim.fully_connected(
embedding,
InputReaderCifar10.NUM_CLASSES,
activation_fn=None,
biases_initializer=tf.constant_initializer(0.0),
scope="fc_{0}".format(num_pool_conv_layers + 2))
return net, embedding
import random
from hparams import HParams
#from utils.train_utils import run_epoch, get_gap_lr_bs
from PIL import Image
from tqdm import tqdm
#from torch.utils.tensorboard import SummaryWriter
import torch.nn as nn
import torch.optim as optim
from model import GMM, ConditionalGMM
parser = argparse.ArgumentParser()
parser.add_argument('--cfg_file', type=str)
args = parser.parse_args()
params = HParams(args.cfg_file)
pprint(params.dict)
np.random.seed(params.seed)
torch.manual_seed(params.seed)
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
torch.autograd.set_detect_anomaly(True)
# creat exp dir
if not os.path.exists(params.exp_dir):
os.mkdir(params.exp_dir)
if not os.path.exists(os.path.join(params.exp_dir, 'gen')):
os.mkdir(os.path.join(params.exp_dir, 'gen'))
if not os.path.exists(os.path.join(params.exp_dir, 'ckpt')):
os.mkdir(os.path.join(params.exp_dir, 'ckpt'))
train_data = np.load('/playpen1/scribble/ssy/data/Levine_32_matrix_train.npy')
from transformer_torch import Transformer
print('torch version: {}, cuda vertion: {}'.format(torch.__version__,
torch.version.cuda))
device = torch.device(
'cuda' if torch.cuda.is_available() else 'cpu') # --> device(type='cuda')
print('device: ', device, 'available: ', torch.cuda.is_available())
hp = HParams(
data_filename='date.txt',
vocab_filename='vocab2.pickle',
model_save_dir='./saved_model2',
batch_size=128,
num_epoch=10,
INPUT_LENGTH=29,
OUTPUT_LENGTH=11,
lr=0.001,
drop_rate=0.025,
d_model=32,
nhead=8,
num_encoder_layers=6,
num_decoder_layers=6,
dim_feedforward=128,
)
if not os.path.exists(hp['model_save_dir']):
os.makedirs(hp['model_save_dir'])
def train():
TEXT = torchtext.data.Field(sequential=True,
tokenize=list,
def main():
# Model configuration for regression
features = [['CRIM', 'numeric'],
['ZN', 'numeric'],
['INDUS', 'numeric'],
['CHAS', 'numeric'],
['NOX', 'numeric'],
['RM', 'numeric'],
['AGE', 'numeric'],
['DIS', 'numeric'],
['RAD', 'numeric'],
['TAX', 'numeric'],
['PT', 'numeric'],
['B', 'numeric'],
['LSTAT', 'numeric']]
label = 'MV'
model_dir = 'data/output/test/housing/3'
csv_dir = 'data/test/housing.csv'
hparams = HParams(batch_size=10, train_steps=3000, model_type='Linear')
# Predicting data
# House with MV 24
predict_data = {'CRIM': [0.00632],
'ZN': [18],
'INDUS': [2.309999943],
'CHAS': [0],
'NOX': [0.537999988],
'RM': [6.574999809],
'AGE': [65.19999695],
'DIS': [4.090000153],
'RAD': [1],
'TAX': [296],
'PT': [15.30000019],
'B': [396.8999939],
'LSTAT': [4.980000019]}
predict_df = pd.DataFrame.from_dict(predict_data)
predictions = predict_tf_model(model_dir, hparams, False, csv_dir, label, features, predict_data)
print(list(predictions))
# Model configuration for classification
features = [['sepal_length', 'numeric'],
['sepal_width', 'numeric'],
['petal_length', 'numeric'],
['petal_width', 'numeric']]
label = 'species'
model_dir = 'data/output/test/flowers/2'
csv_dir = 'data/test/iris.csv'
hparams = HParams(batch_size=50, train_steps=100, model_type='Linear')
# Predicting data
# Iris-virginica
predict_data = {'sepal_length': [5.7],
'sepal_width': [2.5],
'petal_length': [5.0],
'petal_width': [2.0]}
predict_df = pd.DataFrame.from_dict(predict_data) # Iris virginica sample
predictions = predict_tf_model(model_dir, hparams, True, csv_dir, label, features, predict_data)
print(list(predictions))
import sys
from tree_utils import *
from hparams import HParams
piece_file = "data/orm_data/set0-trainunfilt.tok.piece.eng"
tree_file = "data/orm_data/set0-trainunfilt.tok.eng.dep"
rule_vocab_file = "data/orm_data/vocab.dep_rule.eng"
word_vocab_file = "data/orm_data/vocab.dep_word.eng"
hp = HParams()
rule_vocab = RuleVocab(hparams=hp, frozen=False)
word_vocab = Vocab(hparams=hp, frozen=False)
piece_file = open(piece_file, 'r', encoding='utf-8')
tree_file = open(tree_file, 'r', encoding='utf-8')
for piece_line, tree_line in zip(piece_file, tree_file):
tree = Tree(parse_root(tokenize(tree_line)))
#remove_preterminal_POS(tree.root)
#merge_depth(tree.root, 4, 0)
pieces = sent_piece_segs(piece_line)
#print(pieces)
#print(tree.root.to_string())
split_sent_piece(tree.root, pieces, 0)
add_preterminal_wordswitch(tree.root, add_eos=True)
#remove_lhs(tree.root, 'ROOT')
tree.root.label = "XXX"
tree.reset_timestep()
tree.get_data_root(rule_vocab, word_vocab)
binarize = False
del_preterm_POS = True
from transformer_tf import loss_function, Transformer, create_masks
from utils_tf import load_data, plot_attention_weights, CustomSchedule
GO_TOKEN = '<sos>'
END_TOKEN = '<eos>'
PAD = '<pad>'
UNK = '<unk>'
hp = HParams(
data_filename='date.txt',
vocab_filename='vocab.pickle',
model_save_dir='./saved_model',
checkpoint_name='model_ckpt',
MAX_LEN=29,
batch_size=32,
num_epoch=10,
num_layers=6,
hidden_dim=32,
dff=128,
num_heads=8,
drop_rate=0.1,
)
def train():
############ DATA
inputs, targets, word_to_index, index_to_word, VOCAB_SIZE, INPUT_LENGTH, OUTPUT_LENGTH = load_data(
hp) # (50000, 29), (50000, 12)
print(word_to_index)
from transformer_tf import loss_function, Transformer, create_masks
from utils_tf import load_data, CustomSchedule, SequenceAccuracy
GO_TOKEN = '<sos>'
END_TOKEN = '<eos>'
PAD = '<pad>'
UNK = '<unk>'
hp = HParams(
data_filename='date.txt',
vocab_filename='vocab.pickle',
model_save_dir='./saved_model_keras',
MAX_LEN=29,
batch_size=32,
num_epoch=20,
# model parameters
num_layers=6,
hidden_dim=32,
dff=128,
num_heads=8,
drop_rate=0.1,
)
def train():
############ DATA
inputs, targets, word_to_index, index_to_word, VOCAB_SIZE, INPUT_LENGTH, OUTPUT_LENGTH = load_data(
hp) # (50000, 29), (50000, 12)
print(word_to_index)
type=int,
default=100,
help='gradient clipping')
p.add_argument('--optimizer', type=str, default='adam', help='optimizer')
p.add_argument('--init', type=str, default='random', help='initialization')
# switches
p.add_argument('--test-only',
type=int,
default=0,
help='just test model contained in model file')
args = p.parse_args()
print("ARGS:")
print(args)
hyparams = HParams()
hyparams.parse_args(args)
print hyparams
if args.test_only:
test_model(args.model_file,
train_path=args.tweet_file,
vocab_file=args.vocab,
test_path=args.test_file,
output_file=args.results_file,
label_file=args.label_file)
else:
learn_model(hyparams,
args.tweet_file,
vocab_file=args.vocab,
test_path=args.test_file,
print("File : {}, Batch_size : {}".format(file_name, len(batch_list)))
senin_id, emoin_id, senin_length = search_id(hparams, tokenizer,
batch_list, 'senin',
'emoin')
senout_id, emoout_id, senout_length = search_id(
hparams, tokenizer, batch_list, 'senout', 'emoout')
element_values = (senin_id, senout_id, senin_length, senout_length,
emoin_id, emoout_id)
for element, values in zip(ELEMENT_LIST, element_values):
element.update({'values': values})
file_name = 'trainingdata' if conbine else file_name
save_dataset(ELEMENT_LIST, file_name)
if __name__ == "__main__":
hparams = HParams().hparams
# generate_dataset(hparams, CORPUS_DATA_QA_DIR, dialogue_type = 'QA', types = 'txt', conbine = False)
# generate_dataset(hparams, CORPUS_DATA_MTEM_DIR, dialogue_type = 'MTEM', types = 'json', conbine = False)
# Load tfrecord
# loader = DataLoader(hparams, training=True, mode = 'ddpg')
#
# batch_input = loader.get_training_batch(loader.train_dataset)
# initializer = tf.random_uniform_initializer(-0.1, 0.1, seed= 0.1)
# tf.get_variable_scope().set_initializer(initializer)
#
# with tf.Session() as sess:
# sess.run(tf.global_variables_initializer())
# sess.run(tf.tables_initializer())
p.add_argument('--learning-rate', type=float, default=0.1, help='learning rate')
p.add_argument('--bidirectional', type=int, default=1, help='bidirectional LSTM?')
p.add_argument('--nhidden', type=int, default=256, help='num hidden units')
p.add_argument('--embedding-dim', type=int, default=50, help='embedding size')
p.add_argument('--pool', type=str, default='mean', help='pooling strategy')
p.add_argument('--grad-clip', type=int, default=100, help='gradient clipping')
p.add_argument('--optimizer', type=str, default='adam', help='optimizer')
p.add_argument('--init', type=str, default='random', help='initialization')
# switches
p.add_argument('--test-only', type=int, default=0, help='just test model contained in model file')
args = p.parse_args()
print("ARGS:")
print(args)
hyparams = HParams()
hyparams.parse_args(args)
print hyparams
if args.test_only:
test_model(args.model_file,
train_path=args.tweet_file,
vocab_file=args.vocab,
test_path=args.test_file,
output_file=args.results_file,
label_file=args.label_file
)
else:
learn_model(hyparams,
args.tweet_file,
vocab_file=args.vocab,
