def main(argv=None): # pylint: disable=unused-argument
if gfile.Exists(FLAGS.eval_dir):
gfile.DeleteRecursively(FLAGS.eval_dir)
gfile.MakeDirs(FLAGS.eval_dir)
evaluate()
def main(argv=None): # pylint: disable=unused-argument
cifar10.maybe_download_and_extract()
if gfile.Exists(FLAGS.eval_dir):
gfile.DeleteRecursively(FLAGS.eval_dir)
gfile.MakeDirs(FLAGS.eval_dir)
evaluate()
def train():
if FLAGS.data_type == 1 or FLAGS.data_type == 2:
# datadir, org_height, org_width, org_depth=3, batch_size=32, threads_num=4
datas = dataset.Dataset(FLAGS.data_dir,
FLAGS.image_height_org,
FLAGS.image_width_org,
FLAGS.image_depth_org,
FLAGS.batch_size,
FLAGS.num_threads,
type=FLAGS.data_type,
crop=FLAGS.is_crop)
else:
print("invalid data type.")
return
z = tf.placeholder(tf.float32, [None, FLAGS.z_dim], name='z')
dcgan = DCGAN(FLAGS.model_name, FLAGS.checkpoint_dir)
R1_logits, R_sum, z_sum = dcgan.step(z)
r_loss, r_loss_sum = dcgan.cost(R1_logits, z)
# trainable variables
t_vars = tf.trainable_variables()
# g_vars = [var for var in t_vars if 'g_' in var.name]
r_vars = [
var for var in t_vars if ('e_' in var.name) or ('d_fc1' in var.name)
]
# train operations
r_optim = R_train_op(r_loss, r_vars, FLAGS.learning_rate, FLAGS.beta1)
#
all_vars = tf.global_variables()
dg_vars = [
var for var in all_vars if ('d_' in var.name) or ('g_' in var.name)
]
# saver of d and g
saver = tf.train.Saver(dg_vars)
# saver of e_
saver_e = tf.train.Saver(r_vars)
# saver of all variables
saver_all = tf.train.Saver()
# initialization
init_op = tf.global_variables_initializer()
gpu_options = tf.GPUOptions(
per_process_gpu_memory_fraction=FLAGS.gpu_memory_fraction)
sess = tf.Session(config=tf.ConfigProto(allow_soft_placement=True,
gpu_options=gpu_options))
writer = tf.summary.FileWriter("./logs", sess.graph_def)
# run
sess.run(init_op)
# load parameters
print("G and D Model.")
model_dir = os.path.join(FLAGS.model_name, FLAGS.checkpoint_dir)
ckpt = tf.train.get_checkpoint_state(model_dir)
if ckpt and ckpt.model_checkpoint_path:
print("Model: %s" % (ckpt.model_checkpoint_path))
saver.restore(sess, ckpt.model_checkpoint_path)
else:
print("G and D Model: No checkpoint file found")
exit()
print("G and D Model: restored.")
# load e parameters
print("E Model.")
model_e_dir = os.path.join(FLAGS.reverser_model_name, FLAGS.checkpoint_dir)
ckpt_e = tf.train.get_checkpoint_state(model_e_dir)
if ckpt_e and ckpt_e.model_checkpoint_path:
print("Model: %s" % (ckpt_e.model_checkpoint_path))
saver_e.restore(sess, ckpt_e.model_checkpoint_path)
print("E Model: restored.")
else:
print("E model: No checkpoint file found")
# summary
r_sum = tf.summary.merge([z_sum, R_sum, r_loss_sum])
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(sess=sess, coord=coord)
counter = 1
start_time = time.time()
for epoch in xrange(FLAGS.epochs):
for idx in xrange(0, int(datas.batch_idxs)):
batch_z = np.random.uniform(
-1, 1, [FLAGS.batch_size, FLAGS.z_dim]).astype(np.float32)
# R optimization
_, summary_str = sess.run([r_optim, r_sum], {z: batch_z})
writer.add_summary(summary_str, counter)
errR = sess.run(r_loss, {z: batch_z})
print("epochs: %02d %04d/%04d time: %4.4f, r_loss: %.8f" %
(epoch, idx, FLAGS.steps, time.time() - start_time, errR))
# if np.mod(counter, 100) == 1:
# print("generate samples.")
# generated_image_eval = sess.run(generate_images, {z: batch_z})
# out_dir = os.path.join(FLAGS.model_name, FLAGS.sample_dir)
# if not gfile.Exists(out_dir):
# gfile.MakeDirs(out_dir)
# filename = os.path.join(out_dir, 'out_%05d.png' % counter)
# with open(filename, 'wb') as f:
# f.write(generated_image_eval)
counter += 1
if np.mod(epoch, 2) == 0:
out_dir = os.path.join(FLAGS.reverser_model_name,
FLAGS.checkpoint_dir)
if not gfile.Exists(out_dir):
gfile.MakeDirs(out_dir)
out_path = os.path.join(out_dir, 'model.ckpt')
saver_all.save(sess, out_path, global_step=epoch)
coord.request_stop()
coord.join(threads)
sess.close()
def sampling(z_eval, org_image=None):
print("-----------")
print(org_image.shape)
gpu_options = tf.GPUOptions(
per_process_gpu_memory_fraction=FLAGS.gpu_memory_fraction_v)
with tf.Session(config=tf.ConfigProto(allow_soft_placement=True,
gpu_options=gpu_options)) as sess:
dcgan = DCGAN_S(FLAGS.model_name_v, FLAGS.checkpoint_dir_v)
z = tf.placeholder(tf.float32, [None, FLAGS.z_dim_v], name='z')
# build model
dcgan.step(z)
generate_images = dcgan.generate_images(z, 1, 1, png=False)
# saver
t_vars = tf.trainable_variables()
g_vars = [var for var in t_vars if 'g_' in var.name]
saver = tf.train.Saver(g_vars)
# create session
sess.run(tf.global_variables_initializer())
# load parameters
model_dir = os.path.join(FLAGS.model_name_v, FLAGS.checkpoint_dir_v)
ckpt = tf.train.get_checkpoint_state(model_dir)
if ckpt and ckpt.model_checkpoint_path:
print("Model: %s" % ckpt.model_checkpoint_path)
saver.restore(sess, ckpt.model_checkpoint_path)
else:
print("No checkpoint file found")
exit()
print("Model restored.")
generated_image_eval = sess.run(generate_images, {z: z_eval})
out_dir = os.path.join(FLAGS.model_name_v, FLAGS.sample_dir_v)
if not gfile.Exists(out_dir):
gfile.MakeDirs(out_dir)
filename = os.path.join(out_dir,
'sampling_%s.png' % (FLAGS.sample_type_v))
if org_image is None:
with open(filename, 'wb') as f:
f.write(generated_image_eval)
else:
fig = plt.figure()
a = fig.add_subplot(1, 2, 1)
lum_img = org_image
imgplot = plt.imshow(lum_img)
a.set_title('Real')
a = fig.add_subplot(1, 2, 2)
lum2_img = generated_image_eval
imgplot = plt.imshow(lum2_img)
a.set_title('Sampleing from reverse z')
now = datetime.datetime.now()
utime = now.strftime("%s")
out_dir = os.path.join(FLAGS.model_name_v, FLAGS.sample_dir_v)
if not gfile.Exists(out_dir):
gfile.MakeDirs(out_dir)
out_path = os.path.join(out_dir, "sampling_%s.png" % (utime))
plt.savefig(out_path)
def create_image_lists(image_dir, testing_percentage, validation_percentage):
"""
Brief:
Builds a list of training images from the file system.
Analyzes the sub folders in the image directory, splits them into stable
training, testing, and validation sets, and returns a data structure
describing the lists of images for each label and their paths.
Args:
image_dir: String path to a folder containing subfolders of images.
testing_percentage: Integer percentage of the images to reserve for tests.
validation_percentage: Integer percentage of images reserved for validation.
Returns:
A dictionary containing an entry for each label subfolder, with images split
into training, testing, and validation sets within each label.
"""
if not gfile.Exists(image_dir):
print("Image directory '" + image_dir + "' not found.")
return None
result = {}
sub_dirs = [x[0] for x in gfile.Walk(image_dir)]
# The root directory comes first, so skip it.
is_root_dir = True
for sub_dir in sub_dirs:
if is_root_dir:
is_root_dir = False
continue
extensions = ['jpg', 'jpeg', 'JPG', 'JPEG']
file_list = []
dir_name = os.path.basename(sub_dir)
if dir_name == image_dir:
continue
print("Looking for images in '" + dir_name + "'")
for extension in extensions:
file_glob = os.path.join(image_dir, dir_name, '*.' + extension)
file_list.extend(gfile.Glob(file_glob))
if not file_list:
print('No files found')
continue
if len(file_list) < 20:
print('WARNING: Folder has less than 20 images, which may cause issues.')
elif len(file_list) > MAX_NUM_IMAGES_PER_CLASS:
print('WARNING: Folder {} has more than {} images. Some images will '
'never be selected.'.format(dir_name, MAX_NUM_IMAGES_PER_CLASS))
label_name = re.sub(r'[^a-z0-9]+', ' ', dir_name.lower())
training_images = []
testing_images = []
validation_images = []
for file_name in file_list:
base_name = os.path.basename(file_name)
# We want to ignore anything after '_nohash_' in the file name when
# deciding which set to put an image in, the data set creator has a way of
# grouping photos that are close variations of each other. For example
# this is used in the plant disease data set to group multiple pictures of
# the same leaf.
hash_name = re.sub(r'_nohash_.*$', '', file_name)
# This looks a bit magical, but we need to decide whether this file should
# go into the training, testing, or validation sets, and we want to keep
# existing files in the same set even if more files are subsequently
# added.
# To do that, we need a stable way of deciding based on just the file name
# itself, so we do a hash of that and then use that to generate a
# probability value that we use to assign it.
hash_name_hashed = hashlib.sha1(compat.as_bytes(hash_name)).hexdigest()
percentage_hash = ((int(hash_name_hashed, 16) %
(MAX_NUM_IMAGES_PER_CLASS + 1)) *
(100.0 / MAX_NUM_IMAGES_PER_CLASS))
if percentage_hash < validation_percentage:
validation_images.append(base_name)
elif percentage_hash < (testing_percentage + validation_percentage):
testing_images.append(base_name)
else:
training_images.append(base_name)
result[label_name] = {
'dir': dir_name,
'training': training_images,
'testing': testing_images,
'validation': validation_images,
}
return result
import requests, zipfile, regex
from collections import Counter
from urllib.parse import urlparse
from tensorflow.python.platform import gfile
data_path = './data/de-en/train.tags.de-en.en'
def download_dataset():
os.system(
'wget -qO- https://wit3.fbk.eu/archive/2016-01//texts/de/en/de-en.tgz | tar xz'
)
os.system('mv de-en data')
if not (gfile.Exists(data_path)):
download_dataset()
def make_vocab(fpath, fname):
'''Constructs vocabulary.
Args:
fpath: A string. Input file path.
fname: A string. Output file name.
Writes vocabulary line by line to `preprocessed/fname`
'''
text = codecs.open(fpath, 'r', 'utf-8').read()
text = regex.sub("[^\s\p{Latin}']", "", text)
words = text.split()
def prepare_multi_task_data(data_dir, in_vocab_size, out_vocab_size):
sentence_file = os.path.join(data_dir, "source_sentence.txt")
slot_file = os.path.join(data_dir, "source_slot.txt")
intent_file = os.path.join(data_dir, "source_intent.txt")
if not gfile.Exists(sentence_file) and not gfile.Exists(slot_file):
import training_data as gen_data
gen_data.main(data_dir)
train_path = data_dir + '/train/train'
dev_path = data_dir + '/valid/valid'
test_path = data_dir + '/test/test'
split_data(sentence_file, slot_file, intent_file, train_path + ".seq.in",
dev_path + ".seq.in", test_path + ".seq.in",
train_path + ".seq.out", dev_path + ".seq.out",
test_path + ".seq.out", train_path + ".label",
dev_path + ".label",
test_path + ".label") # If no data set pairs
# Create vocabularies of the appropriate sizes.
in_vocab_path = os.path.join(data_dir, "in_vocab_%d.txt" % in_vocab_size)
out_vocab_path = os.path.join(data_dir,
"out_vocab_%d.txt" % out_vocab_size)
label_path = os.path.join(data_dir, "label.txt")
create_vocabulary(in_vocab_path,
train_path + ".seq.in",
in_vocab_size,
tokenizer=naive_tokenizer)
create_vocabulary(out_vocab_path,
train_path + ".seq.out",
out_vocab_size,
tokenizer=naive_tokenizer)
create_label_vocab(label_path, train_path + ".label")
# Create token ids for the training data.
in_seq_train_ids_path = train_path + (".ids%d.seq.in" % in_vocab_size)
out_seq_train_ids_path = train_path + (".ids%d.seq.out" % out_vocab_size)
label_train_ids_path = train_path + (".ids.label")
data_to_token_ids(train_path + ".seq.in",
in_seq_train_ids_path,
in_vocab_path,
tokenizer=naive_tokenizer)
data_to_token_ids(train_path + ".seq.out",
out_seq_train_ids_path,
out_vocab_path,
tokenizer=naive_tokenizer)
data_to_token_ids(train_path + ".label",
label_train_ids_path,
label_path,
normalize_digits=False,
use_padding=False)
# Create token ids for the development data.
in_seq_dev_ids_path = dev_path + (".ids%d.seq.in" % in_vocab_size)
out_seq_dev_ids_path = dev_path + (".ids%d.seq.out" % out_vocab_size)
label_dev_ids_path = dev_path + (".ids.label")
data_to_token_ids(dev_path + ".seq.in",
in_seq_dev_ids_path,
in_vocab_path,
tokenizer=naive_tokenizer)
data_to_token_ids(dev_path + ".seq.out",
out_seq_dev_ids_path,
out_vocab_path,
tokenizer=naive_tokenizer)
data_to_token_ids(dev_path + ".label",
label_dev_ids_path,
label_path,
normalize_digits=False,
use_padding=False)
# Create token ids for the test data.
in_seq_test_ids_path = test_path + (".ids%d.seq.in" % in_vocab_size)
out_seq_test_ids_path = test_path + (".ids%d.seq.out" % out_vocab_size)
label_test_ids_path = test_path + (".ids.label")
data_to_token_ids(test_path + ".seq.in",
in_seq_test_ids_path,
in_vocab_path,
tokenizer=naive_tokenizer)
data_to_token_ids(test_path + ".seq.out",
out_seq_test_ids_path,
out_vocab_path,
tokenizer=naive_tokenizer)
data_to_token_ids(test_path + ".label",
label_test_ids_path,
label_path,
normalize_digits=False,
use_padding=False)
return (in_seq_train_ids_path, out_seq_train_ids_path,
label_train_ids_path, in_seq_dev_ids_path, out_seq_dev_ids_path,
label_dev_ids_path, in_seq_test_ids_path, out_seq_test_ids_path,
label_test_ids_path, in_vocab_path, out_vocab_path, label_path)
def main(unused_argv):
tf.logging.set_verbosity(tf.logging.INFO)
# Read hyperparams and master spec.
hyperparam_config = spec_pb2.GridPoint()
text_format.Parse(FLAGS.hyperparams, hyperparam_config)
print hyperparam_config
master_spec = spec_pb2.MasterSpec()
with gfile.GFile(FLAGS.master_spec, 'r') as fin:
text_format.Parse(fin.read(), master_spec)
# Make output folder
if not gfile.Exists(FLAGS.output_folder):
gfile.MakeDirs(FLAGS.output_folder)
# Construct TF Graph.
graph = tf.Graph()
with graph.as_default():
builder = graph_builder.MasterBuilder(master_spec, hyperparam_config)
# Construct default per-component targets.
default_targets = [
spec_pb2.TrainTarget(name=component.name,
max_index=idx + 1,
unroll_using_oracle=[False] * idx + [True])
for idx, component in enumerate(master_spec.component)
if (component.transition_system.registered_name != 'shift-only')
]
# Add default and manually specified targets.
trainers = []
for target in default_targets:
trainers += [builder.add_training_from_config(target)]
check.Eq(len(trainers), 1,
"Expected only one training target (FF unit)")
# Construct annotation and saves. Will use moving average if enabled.
annotator = builder.add_annotation()
builder.add_saver()
# Add backwards compatible training summary.
summaries = []
for component in builder.components:
summaries += component.get_summaries()
merged_summaries = tf.summary.merge_all()
# Construct target to initialize variables.
tf.group(tf.global_variables_initializer(), name='inits')
# Prepare tensorboard dir.
events_dir = os.path.join(FLAGS.output_folder, "tensorboard")
empty_dir(events_dir)
summary_writer = tf.summary.FileWriter(events_dir, graph)
print "Wrote events (incl. graph) for Tensorboard to folder:", events_dir
print "The graph can be viewed via"
print " tensorboard --logdir=" + events_dir
print " then navigating to http://localhost:6006 and clicking on 'GRAPHS'"
with graph.as_default():
tf.set_random_seed(hyperparam_config.seed)
# Read train and dev corpora.
print "Reading corpora..."
train_corpus = read_corpus(FLAGS.train_corpus)
dev_corpus = read_corpus(FLAGS.dev_corpus)
# Prepare checkpoint folder.
checkpoint_path = os.path.join(FLAGS.output_folder, 'checkpoints/best')
checkpoint_dir = os.path.dirname(checkpoint_path)
empty_dir(checkpoint_dir)
with tf.Session(FLAGS.tf_master, graph=graph) as sess:
# Make sure to re-initialize all underlying state.
sess.run(tf.global_variables_initializer())
# Run training.
trainer_lib.run_training(
sess,
trainers,
annotator,
evaluator,
[0], # pretrain_steps
[FLAGS.train_steps],
train_corpus,
dev_corpus,
dev_corpus,
FLAGS.batch_size,
summary_writer,
FLAGS.report_every,
builder.saver,
checkpoint_path)
# Convert model to a Myelin flow.
if len(FLAGS.flow) != 0:
tf.logging.info('Saving flow to %s', FLAGS.flow)
flow = convert_model(master_spec, sess)
flow.save(FLAGS.flow)
tf.logging.info('Best checkpoint written to %s', checkpoint_path)
def doBasicsOneExportPath(self,
export_path,
clear_devices=False,
global_step=GLOBAL_STEP,
sharded=True,
export_count=1):
# Build a graph with 2 parameter nodes on different devices.
tf.reset_default_graph()
with tf.Session(target="",
config=config_pb2.ConfigProto(
device_count={"CPU": 2})) as sess:
# v2 is an unsaved variable derived from v0 and v1. It is used to
# exercise the ability to run an init op when restoring a graph.
with sess.graph.device("/cpu:0"):
v0 = tf.Variable(10, name="v0")
with sess.graph.device("/cpu:1"):
v1 = tf.Variable(20, name="v1")
v2 = tf.Variable(1, name="v2", trainable=False, collections=[])
assign_v2 = tf.assign(v2, tf.add(v0, v1))
init_op = tf.group(assign_v2, name="init_op")
tf.add_to_collection("v", v0)
tf.add_to_collection("v", v1)
tf.add_to_collection("v", v2)
named_tensor_bindings = {
"logical_input_A": v0,
"logical_input_B": v1
}
signatures = {
"foo":
exporter.regression_signature(input_tensor=v0,
output_tensor=v1),
"generic":
exporter.generic_signature(named_tensor_bindings)
}
asset_filepath_orig = os.path.join(tf.test.get_temp_dir(),
"hello42.txt")
asset_file = tf.constant(asset_filepath_orig, name="filename42")
tf.add_to_collection(tf.GraphKeys.ASSET_FILEPATHS, asset_file)
with gfile.FastGFile(asset_filepath_orig, "w") as f:
f.write("your data here")
assets_collection = tf.get_collection(tf.GraphKeys.ASSET_FILEPATHS)
ignored_asset = os.path.join(tf.test.get_temp_dir(), "ignored.txt")
with gfile.FastGFile(ignored_asset, "w") as f:
f.write("additional data here")
tf.initialize_all_variables().run()
# Run an export.
save = tf.train.Saver({
"v0": v0,
"v1": v1
},
restore_sequentially=True,
sharded=sharded)
export = exporter.Exporter(save)
compare_def = tf.get_default_graph().as_graph_def()
export.init(
compare_def,
init_op=init_op,
clear_devices=clear_devices,
default_graph_signature=exporter.classification_signature(
input_tensor=v0),
named_graph_signatures=signatures,
assets_collection=assets_collection)
for x in range(export_count):
export.export(export_path,
tf.constant(global_step + x),
sess,
exports_to_keep=gc.largest_export_versions(2))
# Set global_step to the last exported version, as the rest of the test
# uses it to construct model export path, loads model from it, and does
# verifications. We want to make sure to always use the last exported
# version, as old ones may have be garbage-collected.
global_step += export_count - 1
# Restore graph.
tf.reset_default_graph()
with tf.Session(target="",
config=config_pb2.ConfigProto(
device_count={"CPU": 2})) as sess:
save = tf.train.import_meta_graph(
os.path.join(export_path,
constants.VERSION_FORMAT_SPECIFIER % global_step,
constants.META_GRAPH_DEF_FILENAME))
self.assertIsNotNone(save)
meta_graph_def = save.export_meta_graph()
collection_def = meta_graph_def.collection_def
# Validate custom graph_def.
graph_def_any = collection_def[constants.GRAPH_KEY].any_list.value
self.assertEquals(len(graph_def_any), 1)
graph_def = tf.GraphDef()
graph_def_any[0].Unpack(graph_def)
if clear_devices:
for node in compare_def.node:
node.device = ""
self.assertProtoEquals(compare_def, graph_def)
# Validate init_op.
init_ops = collection_def[constants.INIT_OP_KEY].node_list.value
self.assertEquals(len(init_ops), 1)
self.assertEquals(init_ops[0], "init_op")
# Validate signatures.
signatures_any = collection_def[
constants.SIGNATURES_KEY].any_list.value
self.assertEquals(len(signatures_any), 1)
signatures = manifest_pb2.Signatures()
signatures_any[0].Unpack(signatures)
default_signature = signatures.default_signature
self.assertEqual(
default_signature.classification_signature.input.tensor_name,
"v0:0")
bindings = signatures.named_signatures[
"generic"].generic_signature.map
self.assertEquals(bindings["logical_input_A"].tensor_name, "v0:0")
self.assertEquals(bindings["logical_input_B"].tensor_name, "v1:0")
read_foo_signature = (
signatures.named_signatures["foo"].regression_signature)
self.assertEquals(read_foo_signature.input.tensor_name, "v0:0")
self.assertEquals(read_foo_signature.output.tensor_name, "v1:0")
# Validate the assets.
assets_any = collection_def[constants.ASSETS_KEY].any_list.value
self.assertEquals(len(assets_any), 1)
asset = manifest_pb2.AssetFile()
assets_any[0].Unpack(asset)
assets_path = os.path.join(
export_path, constants.VERSION_FORMAT_SPECIFIER % global_step,
constants.ASSETS_DIRECTORY, "hello42.txt")
asset_contents = gfile.GFile(assets_path).read()
self.assertEqual(asset_contents, b"your data here")
self.assertEquals("hello42.txt", asset.filename)
self.assertEquals("filename42:0", asset.tensor_binding.tensor_name)
ignored_asset_path = os.path.join(
export_path, constants.VERSION_FORMAT_SPECIFIER % global_step,
constants.ASSETS_DIRECTORY, "ignored.txt")
self.assertFalse(gfile.Exists(ignored_asset_path))
# Validate graph restoration.
if sharded:
save.restore(
sess,
os.path.join(
export_path,
constants.VERSION_FORMAT_SPECIFIER % global_step,
constants.VARIABLES_FILENAME_PATTERN))
else:
save.restore(
sess,
os.path.join(
export_path,
constants.VERSION_FORMAT_SPECIFIER % global_step,
constants.VARIABLES_FILENAME))
self.assertEqual(10, tf.get_collection("v")[0].eval())
self.assertEqual(20, tf.get_collection("v")[1].eval())
tf.get_collection(constants.INIT_OP_KEY)[0].run()
self.assertEqual(30, tf.get_collection("v")[2].eval())
def split_data(sentence_file, slot_file, intent_file, sentence_training_file,
sentence_developing_file, sentence_testing_file,
slot_training_file, slot_developing_file, slot_testing_file,
intent_training_file, intent_developing_file,
intent_testing_file):
if not gfile.Exists(sentence_training_file):
with open(sentence_file, 'r', encoding='utf-8') as infile_sentence:
with open(slot_file, 'r', encoding='utf-8') as infile_slot:
with open(intent_file, 'r', encoding='utf-8') as infile_intent:
sentence, slot, intent = infile_sentence.readlines(),\
infile_slot.readlines(), infile_intent.readlines()
len_sentence = len(sentence)
len_slot = len(slot)
if len_sentence != len_slot:
raise Exception('Length is not matched!')
print("Processing line %d" % len_sentence)
num_dev = round(len_sentence / 400)
num_test = round(len_sentence / 400)
# Index for random sampling
idx = random.sample(range(len_sentence), len_sentence)
sentence_training = [sentence[i] for i in idx[:-(num_dev + num_test)]]
with open(sentence_training_file, "w") as outfile:
for line in sentence_training:
outfile.write(str(line))
sentence_developing = [
sentence[i] for i in idx[-(num_dev + num_test):-num_dev]
]
with open(sentence_developing_file, "w") as outfile:
for line in sentence_developing:
outfile.write(str(line))
sentence_testing = [sentence[i] for i in idx[-num_test:]]
with open(sentence_testing_file, "w") as outfile:
for line in sentence_testing:
outfile.write(str(line))
slot_training = [slot[i] for i in idx[:-(num_dev + num_test)]]
with open(slot_training_file, "w") as outfile:
for line in slot_training:
outfile.write(str(line))
slot_developing = [
slot[i] for i in idx[-(num_dev + num_test):-num_dev]
]
with open(slot_developing_file, "w") as outfile:
for line in slot_developing:
outfile.write(str(line))
slot_testing = [slot[i] for i in idx[-num_test:]]
with open(slot_testing_file, "w") as outfile:
for line in slot_testing:
outfile.write(str(line))
intent_training = [intent[i] for i in idx[:-(num_dev + num_test)]]
with open(intent_training_file, "w") as outfile:
for line in intent_training:
outfile.write(str(line))
intent_developing = [
intent[i] for i in idx[-(num_dev + num_test):-num_dev]
]
with open(intent_developing_file, "w") as outfile:
for line in intent_developing:
outfile.write(str(line))
intent_testing = [intent[i] for i in idx[-num_test:]]
with open(intent_testing_file, "w") as outfile:
for line in intent_testing:
outfile.write(str(line))
def freeze_graph(input_graph,
input_saver,
input_binary,
input_checkpoint,
output_node_names,
restore_op_name,
filename_tensor_name,
output_graph,
clear_devices,
initializer_nodes,
variable_names_blacklist=""):
"""Converts all variables in a graph and checkpoint into constants."""
del restore_op_name, filename_tensor_name # Unused by updated loading code.
if not gfile.Exists(input_graph):
print("Input graph file '" + input_graph + "' does not exist!")
return -1
if input_saver and not gfile.Exists(input_saver):
print("Input saver file '" + input_saver + "' does not exist!")
return -1
# 'input_checkpoint' may be a prefix if we're using Saver V2 format
if not saver_lib.checkpoint_exists(input_checkpoint):
print("Input checkpoint '" + input_checkpoint + "' doesn't exist!")
return -1
if not output_node_names:
print("You need to supply the name of a node to --output_node_names.")
return -1
input_graph_def = graph_pb2.GraphDef()
mode = "rb" if input_binary else "r"
with gfile.FastGFile(input_graph, mode) as f:
if input_binary:
input_graph_def.ParseFromString(f.read())
else:
text_format.Merge(f.read().decode("utf-8"), input_graph_def)
# Remove all the explicit device specifications for this node. This helps to
# make the graph more portable.
if clear_devices:
for node in input_graph_def.node:
node.device = ""
_ = importer.import_graph_def(input_graph_def, name="tensorName")
with session.Session() as sess:
if input_saver:
with gfile.FastGFile(input_saver, mode) as f:
saver_def = saver_pb2.SaverDef()
if input_binary:
saver_def.ParseFromString(f.read())
else:
text_format.Merge(f.read(), saver_def)
saver = saver_lib.Saver(saver_def=saver_def)
saver.restore(sess, input_checkpoint)
else:
var_list = {}
reader = pywrap_tensorflow.NewCheckpointReader(input_checkpoint)
var_to_shape_map = reader.get_variable_to_shape_map()
for key in var_to_shape_map:
try:
tensor = sess.graph.get_tensor_by_name(key + ":0")
except KeyError:
# This tensor doesn't exist in the graph (for example it's
# 'global_step' or a similar housekeeping element) so skip it.
continue
var_list[key] = tensor
saver = saver_lib.Saver(var_list=var_list)
saver.restore(sess, input_checkpoint)
if initializer_nodes:
sess.run(initializer_nodes)
variable_names_blacklist = (variable_names_blacklist.split(",")
if variable_names_blacklist else None)
output_graph_def = graph_util.convert_variables_to_constants(
sess,
input_graph_def,
output_node_names.split(","),
variable_names_blacklist=variable_names_blacklist)
with gfile.GFile(output_graph, "wb") as f:
f.write(output_graph_def.SerializeToString())
print("%d ops in the final graph." % len(output_graph_def.node))
def fit(self, train_data, train_label):
np.random.seed(1337)
tf.set_random_seed(1337)
n_samples, self.channels, self.time, self.img_size_x, self.img_size_y = train_data.shape[
0:5]
train_label_mat = []
for i in train_label:
train_label_mat.append([i])
train_label_mat = np.array(train_label_mat)
self.n_classes = 1
Hidden_layer, Pro, self.Location_out = self.build_model()
assert len(Hidden_layer) == self.time
assert len(Pro) == self.time - 1
self.y_conv = []
self.yy_multi = []
for i in range(self.time):
self.y_conv.append(
tf.divide(
1.0,
tf.minimum(
tf.add(
1.0,
tf.exp(-(tf.matmul(Hidden_layer[i],
self.action_layer_w) +
self.action_layer_b))), 1000.0)))
self.yy_multi.append(self.yy)
with tf.name_scope("kkkkkkkkkkkkkkk"):
self.y_conv = tf.stack(self.y_conv)
self.yy_multi = tf.stack(self.yy_multi)
with tf.Session() as sess:
#shape=(time, batch , 1)
cross_entropy = (
tf.multiply(self.yy_multi, tf.log(self.y_conv)) + tf.multiply(
(1.0 - self.yy_multi), tf.log(
(1.0 - self.y_conv + 0.0001))))
correct_prediction = tf.abs(self.y_conv - self.yy_multi) < 0.5
r_pre = tf.reshape(tf.to_float(correct_prediction),
(self.time, self.batch_size, 1))
r = r_pre[1:r_pre.get_shape()[0], :, :]
Rew_eve = np.zeros([(self.time - 1), self.batch_size, 1])
for i in range(self.batch_size):
'''
for j in range((self.time-1)):
if r[j,i,0]==1.0:
for m in range((j+1)):
Rew_eve[m,i,0]=Rew_eve[m,i,0]+1
'''
if r[(self.time - 2), i, 0] == 1.0:
for m in range((self, time - 1)):
Rew_eve[m, i, 0] = Rew_eve[m, i, 0] + 1
Rew_eve = tf.stack(Rew_eve)
Rew_eve = tf.cast(Rew_eve, tf.float32)
assert Rew_eve.get_shape()[0] == self.time - 1
assert Rew_eve.get_shape()[1] == self.batch_size
assert Rew_eve.get_shape()[2] == 1
'''
Decay_reward=np.ones(((self.time-1), self.batch_size,1))
for i in range(self.time):
for j in range(self.batch_size):
Decay_reward[i,j]=[np.exp(-(self.time-i-1.0)/self.time)]
Decay_reward=tf.stack(Decay_reward)
Decay_reward=tf.cast(Decay_reward, tf.float32)
assert Decay_reward.get_shape()[0]==self.time-1
assert Decay_reward.get_shape()[1]==self.batch_size
assert Decay_reward.get_shape()[2]==1
r=tf.multiply(r,Decay_reward)
print (tf.stack(Pro).get_shape())
'''
log_p = tf.reshape(tf.log(tf.stack(Pro) + 1e-10),
((self.time - 1), self.batch_size, 1))
#log_p = tf.transpose(log_p, perm=[1,0,2])
#log_p=tf.reduce_sum(log_p,axis=2)
# combine
BB = tf.transpose(tf.multiply(Rew_eve, log_p), perm=[1, 0, 2])
'''
B=tf.reduce_mean(BB,axis=0)
B_aver=[]
for i in range(self.batch_size):
B_aver.append(B)
B=tf.stack(B_aver)
#B=tf.reshape(B,(self.time,self.batch_size,1))
#shape=(batch ,time, 1)
Rew=BB-B
'''
Rew = BB
'''
Reward_decay=[]
for i in range(self.time-1):
Reward_decay_line=[]
num_count=0
for j in range(i,(self.time-1)):
num_count=num_count+1.0
Reward_decay_line.append(Rew[:,j,:]*0.99**(j-i))
Reward_decay_line=tf.reduce_sum(tf.stack(Reward_decay_line),axis=0)
assert Reward_decay_line.get_shape()[0]==self.batch_size
assert Reward_decay_line.get_shape()[1]==1
Reward_decay.append(Reward_decay_line/num_count*self.time)
Reward_decay=tf.transpose(tf.stack(Reward_decay),[1,0,2])
print (Reward_decay.get_shape())
'''
cross_entropy = cross_entropy[1:cross_entropy.get_shape()[0], :, :]
cross_eve_before = np.zeros([(self.time - 2), self.batch_size, 1])
cross_eve_last = np.ones([1, self.batch_size, 1])
cross_eve = np.concatenate((cross_eve_before, cross_eve_last),
axis=0)
cross_eve = tf.stack(cross_eve)
cross_eve = tf.cast(cross_eve, tf.float32)
cross_entropy = tf.multiply(cross_entropy, cross_eve)
cross_entropy = tf.transpose(cross_entropy, perm=[1, 0, 2])
J = cross_entropy + Rew
J = tf.squeeze(J)
J = tf.reduce_sum(J, axis=1)
J = tf.reduce_mean(J, axis=0)
cost = -J
var_train = tf.trainable_variables()
lossL2 = tf.add_n(
[tf.nn.l2_loss(v)
for v in var_train if 'bias' not in v.name]) * 0.0001
cost = cost + lossL2
self.lr = tf.placeholder("float", None)
trainer = tf.train.AdamOptimizer(self.lr)
gvs = trainer.compute_gradients(cost)
def ClipIfNotNone(grad):
if grad is None:
return grad
return tf.clip_by_value(grad, -10, 10)
clipped_gradients = [(ClipIfNotNone(grad), var)
for grad, var in gvs]
train_step = trainer.apply_gradients(clipped_gradients)
correct_prediction = tf.abs(self.y_conv - self.yy_multi) < 0.5
correct_prediction = tf.squeeze(correct_prediction)
accurac = tf.reduce_mean(tf.cast(correct_prediction, tf.float32),
axis=1)
print(accurac.get_shape())
accuracy = accurac[self.time - 1]
sess.run(tf.global_variables_initializer())
train_data = train_data.tolist()
train_label = train_label.tolist()
train_label_mat = train_label_mat.tolist()
if len(train_data) % self.batch_size != 0:
for i in range(self.batch_size -
len(train_data) % self.batch_size):
index = random.randint(0, len(train_data) - 1)
train_data.append(train_data[index])
train_label.append(train_label[index])
train_label_mat.append(train_label_mat[index])
assert len(train_data) % self.batch_size == 0
assert len(train_label_mat) == len(train_data)
saver = tf.train.Saver()
Index = [i for i in range(len(train_data))]
for i in range(self.epoch):
'''
if i%10==0 and i>50:
self.learning_rate=self.learning_rate*0.9
print (self.learning_rate)
if self.learning_rate<0.0000004:
self.learning_rate=0.0000004
'''
Total_acc = 0.0
random.shuffle(Index)
for j in range(int(len(train_data) / self.batch_size)):
eve_train_data = []
eve_train_label_mat = []
for mm in Index[j * self.batch_size:(1 + j) *
self.batch_size]:
eve_train_data.append(train_data[mm])
eve_train_label_mat.append(train_label_mat[mm])
train_step.run(
feed_dict={
self.lr: self.learning_rate,
self.img: eve_train_data,
self.yy: eve_train_label_mat
})
train_accuracy = (accuracy.eval(
feed_dict={
self.img: eve_train_data,
self.yy: eve_train_label_mat
}))
Total_acc = Total_acc + train_accuracy
'''
CE=sess.run(cross_entropy,feed_dict={self.img: eve_train_data, self.yy: eve_train_label_mat})
print (CE*30)
Re=sess.run(Rew,feed_dict={self.img: eve_train_data, self.yy: eve_train_label_mat})
print (Re)
'''
print("Epoch %d,training accuracy " % (i + 1))
print(Total_acc / (float(len(train_data) / self.batch_size)))
del eve_train_data, eve_train_label_mat
gc.collect()
saver = tf.train.Saver()
folder = os.path.dirname(self.checkpoint_path)
if not gfile.Exists(folder):
gfile.MakeDirs(folder)
#if (i%100==0):
saver.save(sess, self.checkpoint_path)
def check_lockfile(filepath):
lockfile_path = filepath + '.lock'
return gfile.Exists(lockfile_path)
def create_output_directory(output_dir):
if not directory_handler.Exists(output_dir):
directory_handler.MakeDirs(output_dir)
def prepare_pretrained_data(data_dir,
in_vocab_size,
word_embedding_size,
vocabulary_path,
normalize_digits=False):
# Create token ids for the training data form pre-trained vocab
train_path = data_dir + '/train/'
dev_path = data_dir + '/valid/'
test_path = data_dir + '/test/'
pretrained_path = 'word_vector'
train_path_ = pretrained_path + '/train/'
if not gfile.Exists(train_path_):
os.makedirs(train_path_)
dev_path_ = pretrained_path + '/valid/'
if not gfile.Exists(dev_path_):
os.makedirs(dev_path_)
test_path_ = pretrained_path + '/test/'
if not gfile.Exists(test_path_):
os.makedirs(test_path_)
in_vocab_path = os.path.join(pretrained_path,
"in_vocab_%d.txt" % in_vocab_size)
vocab_, embedding = load_fasttext('word_vector/wiki.zh_classical.vec',
in_vocab_size, word_embedding_size)
if not gfile.Exists(in_vocab_path):
print("Creating vocabulary %s from data %s" %
(in_vocab_path, train_path))
n_word = len(vocab_)
n1_word = 0
# Concatenate original vocabulary
_, rev_vocab = initialize_vocabulary(vocabulary_path)
for i, w in enumerate(rev_vocab):
if i > len(START_VOCAB_dict['with_padding']) - 1:
if w not in vocab_:
vocab_.append(w)
n1_word += 1
vocab_list = START_VOCAB_dict['with_padding'] + vocab_
print(vocab_list) # Debug
print(len(vocab_list))
print(n_word)
print(n1_word)
temp = in_vocab_size + n1_word + len(START_VOCAB_dict['with_padding'])
if len(vocab_list) > temp:
vocab_list = vocab_list[:temp]
n_new = len(vocab_list) - n_word - len(
START_VOCAB_dict['with_padding'])
print("Total vocabulary size: {}".format(len(vocab_list)))
print("Supplemental words: {}".format(n1_word))
print("Prefix vocabulary:{}".format(
len(START_VOCAB_dict['with_padding'])))
with gfile.GFile(in_vocab_path, mode="w") as vocab_file:
for w in vocab_list:
vocab_file.write(w + "\n")
else:
_, rev_vocab = initialize_vocabulary(in_vocab_path)
print(len(rev_vocab))
n_new = len(rev_vocab) - len(embedding) - len(
START_VOCAB_dict['with_padding'])
# Expand embedding
new_embedding = np.matlib.repmat(np.zeros(word_embedding_size),
len(START_VOCAB_dict['with_padding']), 1)
embedding = np.concatenate((embedding, new_embedding), axis=0)
new_embedding = np.matlib.repmat(np.zeros(word_embedding_size), n_new, 1)
embedding = np.concatenate((embedding, new_embedding), axis=0)
in_seq_train_ids_path = pretrained_path + ("/train/train.ids%d.seq.in" %
in_vocab_size)
data_to_token_ids(train_path + "train.seq.in",
in_seq_train_ids_path,
in_vocab_path,
tokenizer=naive_tokenizer)
# Create token ids for the development data.
in_seq_dev_ids_path = pretrained_path + ("/valid/valid.ids%d.seq.in" %
in_vocab_size)
data_to_token_ids(dev_path + "valid.seq.in",
in_seq_dev_ids_path,
in_vocab_path,
tokenizer=naive_tokenizer)
# Create token ids for the test data.
in_seq_test_ids_path = pretrained_path + ("/test/test.ids%d.seq.in" %
in_vocab_size)
data_to_token_ids(test_path + "test.seq.in",
in_seq_test_ids_path,
in_vocab_path,
tokenizer=naive_tokenizer)
return (in_seq_train_ids_path, in_seq_dev_ids_path, in_seq_test_ids_path,
in_vocab_path, embedding)
def empty_dir(folder):
if gfile.IsDirectory(folder):
gfile.DeleteRecursively(folder)
elif gfile.Exists(folder):
gfile.Remove(folder)
gfile.MakeDirs(folder)
def test_dir_operations(self):
""" Test directory operations"""
d = get_oss_path("d1/d2/d3/d4")
gfile.MakeDirs(d)
self.assertTrue(gfile.Stat(d).is_directory)
# Test listing bucket directory with and without trailing '/'
content = gfile.ListDirectory(
"oss://%s\x01id=%s\x02key=%s\x02host=%s"
% (bucket, access_id, access_key, host)
)
content_s = gfile.ListDirectory(
"oss://%s\x01id=%s\x02key=%s\x02host=%s/"
% (bucket, access_id, access_key, host)
)
self.assertEqual(content, content_s)
self.assertIn("oss_fs_test", content)
self.assertIn("oss_fs_test/d1", content)
self.assertIn("oss_fs_test/d1/d2", content)
# Test listing test directory with and without trailing '/'
content = gfile.ListDirectory(
"oss://%s\x01id=%s\x02key=%s\x02host=%s"
% (bucket, access_id, access_key, host)
+ "/oss_fs_test"
)
content_s = gfile.ListDirectory(
"oss://%s\x01id=%s\x02key=%s\x02host=%s"
% (bucket, access_id, access_key, host)
+ "/oss_fs_test/"
)
self.assertEqual(content, content_s)
self.assertIn("d1", content)
self.assertIn("d1/d2", content)
# Test listing sub directories.
content = gfile.ListDirectory(get_oss_path("d1"))
content_s = gfile.ListDirectory(get_oss_path("d1/"))
self.assertEqual(content, content_s)
self.assertIn("d2", content)
content = gfile.ListDirectory(get_oss_path("d1/d2/d3/d4"))
content_s = gfile.ListDirectory(get_oss_path("d1/d2/d3/d4"))
self.assertEqual(content, content_s)
self.assertEqual([], content)
# Test Rename directories
self.assertTrue(gfile.Exists(get_oss_path("d1")))
gfile.Rename(get_oss_path("d1"), get_oss_path("rename_d1"), overwrite=True)
self.assertTrue(gfile.Exists(get_oss_path("rename_d1")))
self.assertFalse(gfile.Exists(get_oss_path("d1")))
content = gfile.ListDirectory(get_oss_path("rename_d1"))
content_s = gfile.ListDirectory(get_oss_path("rename_d1/"))
self.assertEqual(content, content_s)
self.assertIn("d2", content)
# Test Rename non-empty directories
not_empty_dir = get_oss_path("not_empty_dir/")
rename_not_empty_dir = get_oss_path("rename_not_empty_dir/")
gfile.MakeDirs(not_empty_dir)
not_empty_file = get_oss_path("not_empty_dir/not_empty_file")
rename_not_empty_file = get_oss_path("rename_not_empty_dir/not_empty_file")
with gfile.Open(not_empty_file, mode="w") as fh:
content = "file content"
fh.write(content)
self.assertTrue(gfile.Exists(not_empty_dir))
self.assertTrue(gfile.Exists(not_empty_file))
gfile.Rename(not_empty_dir, rename_not_empty_dir, overwrite=True)
self.assertFalse(gfile.Exists(not_empty_dir))
self.assertFalse(gfile.Exists(not_empty_file))
self.assertTrue(gfile.Exists(rename_not_empty_dir))
self.assertTrue(gfile.Exists(rename_not_empty_file))
def main(_):
args = config_distill.get_args_for_config(FLAGS.config_name)
args.logdir = FLAGS.logdir
args.solver.num_workers = FLAGS.num_workers
args.solver.task = FLAGS.task
args.solver.ps_tasks = FLAGS.ps_tasks
args.solver.master = FLAGS.master
args.buildinger.env_class = nav_env.MeshMapper
fu.makedirs(args.logdir)
args.buildinger.logdir = args.logdir
R = nav_env.get_multiplexor_class(args.buildinger, args.solver.task)
if False:
pr = cProfile.Profile()
pr.enable()
rng = np.random.RandomState(0)
for i in range(1):
b, instances_perturbs = R.sample_building(rng)
inputs = b.worker(*(instances_perturbs))
for j in range(inputs['imgs'].shape[0]):
p = os.path.join('tmp', '{:d}.png'.format(j))
img = inputs['imgs'][j, 0, :, :, :3] * 1
img = (img).astype(np.uint8)
fu.write_image(p, img)
print(inputs['imgs'].shape)
inputs = R.pre(inputs)
pr.disable()
pr.print_stats(2)
if args.control.train:
if not gfile.Exists(args.logdir):
gfile.MakeDirs(args.logdir)
m = utils.Foo()
m.tf_graph = tf.Graph()
config = tf.ConfigProto()
config.device_count['GPU'] = 1
config.gpu_options.allow_growth = True
config.gpu_options.per_process_gpu_memory_fraction = 0.8
with m.tf_graph.as_default():
with tf.device(tf.train.replica_device_setter(
args.solver.ps_tasks)):
m = distill.setup_to_run(m,
args,
is_training=True,
batch_norm_is_training=True)
train_step_kwargs = distill.setup_train_step_kwargs_mesh(
m,
R,
os.path.join(args.logdir, 'train'),
rng_seed=args.solver.task,
is_chief=args.solver.task == 0,
iters=1,
train_display_interval=args.summary.display_interval)
final_loss = slim.learning.train(
train_op=m.train_op,
logdir=args.logdir,
master=args.solver.master,
is_chief=args.solver.task == 0,
number_of_steps=args.solver.max_steps,
train_step_fn=tf_utils.train_step_custom,
train_step_kwargs=train_step_kwargs,
global_step=m.global_step_op,
init_op=m.init_op,
init_fn=m.init_fn,
sync_optimizer=m.sync_optimizer,
saver=m.saver_op,
summary_op=None,
session_config=config)
if args.control.test:
m = utils.Foo()
m.tf_graph = tf.Graph()
checkpoint_dir = os.path.join(format(args.logdir))
with m.tf_graph.as_default():
m = distill.setup_to_run(m,
args,
is_training=False,
batch_norm_is_training=args.control.
force_batchnorm_is_training_at_test)
train_step_kwargs = distill.setup_train_step_kwargs_mesh(
m,
R,
os.path.join(args.logdir, args.control.test_name),
rng_seed=args.solver.task + 1,
is_chief=args.solver.task == 0,
iters=args.summary.test_iters,
train_display_interval=None)
sv = slim.learning.supervisor.Supervisor(
graph=ops.get_default_graph(),
logdir=None,
init_op=m.init_op,
summary_op=None,
summary_writer=None,
global_step=None,
saver=m.saver_op)
last_checkpoint = None
while True:
last_checkpoint = slim.evaluation.wait_for_new_checkpoint(
checkpoint_dir, last_checkpoint)
checkpoint_iter = int(
os.path.basename(last_checkpoint).split('-')[1])
start = time.time()
LOGGING.info(
'Starting evaluation at %s using checkpoint %s.',
time.strftime('%Y-%m-%d-%H:%M:%S', time.localtime()),
last_checkpoint)
config = tf.ConfigProto()
config.device_count['GPU'] = 1
config.gpu_options.allow_growth = True
config.gpu_options.per_process_gpu_memory_fraction = 0.8
with sv.managed_session(args.solver.master,
config=config,
start_standard_services=False) as sess:
sess.run(m.init_op)
sv.saver.restore(sess, last_checkpoint)
sv.start_queue_runners(sess)
vals, _ = tf_utils.train_step_custom(sess,
None,
m.global_step_op,
train_step_kwargs,
mode='val')
if checkpoint_iter >= args.solver.max_steps:
break
def train(gp=True):
if FLAGS.data_type == 1:
# datadir, org_height, org_width, org_depth=3, batch_size=32, threads_num=4
datas = dataset.Dataset(FLAGS.data_dir,
FLAGS.image_height_org,
FLAGS.image_width_org,
FLAGS.image_depth_org,
FLAGS.batch_size,
FLAGS.num_threads,
type=FLAGS.data_type,
crop=FLAGS.is_crop)
elif FLAGS.data_type == 2:
datas = dataset.Dataset(FLAGS.data_dir,
FLAGS.image_height_org,
FLAGS.image_width_org,
FLAGS.image_depth_org,
FLAGS.batch_size,
FLAGS.num_threads,
type=FLAGS.data_type,
crop=FLAGS.is_crop)
else:
print("invalid data type.")
return
images = datas.get_inputs(FLAGS.image_height, FLAGS.image_width)
z = tf.placeholder(tf.float32, [None, FLAGS.z_dim], name='z')
if gp:
dcgan = DCGAN_GP(FLAGS.model_name, FLAGS.checkpoint_dir)
images_inf, generates, logits1, logits2, inter1, inter2, G_sum, z_sum, d1_sum, d2_sum = dcgan.step(
images, z)
d_loss_real, d_loss_fake, d_loss_real_sum, d_loss_fake_sum, d_loss_sum, g_loss_sum, d_loss, g_loss, fm_loss = dcgan.cost(
images, generates, logits1, logits2, inter1, inter2)
# trainable variables
t_vars = tf.trainable_variables()
d_vars = [var for var in t_vars if 'd_' in var.name]
g_vars = [var for var in t_vars if 'g_' in var.name]
# train operations
d_optim = D_train_op(d_loss, d_vars, FLAGS.learning_rate, FLAGS.beta1)
g_optim = G_train_op(g_loss, g_vars, FLAGS.learning_rate, FLAGS.beta1)
# clip d parameters
#clip_updates = [tf.assign(var, tf.clip_by_value(var, -FLAGS.c_param, FLAGS.c_param)) for var in d_vars]
else:
dcgan = DCGAN(FLAGS.model_name, FLAGS.checkpoint_dir)
images_inf, logits1, logits2, inter1, inter2, G_sum, z_sum, d1_sum, d2_sum = dcgan.step(
images, z)
d_loss_real, d_loss_fake, d_loss_real_sum, d_loss_fake_sum, d_loss_sum, g_loss_sum, d_loss, g_loss, fm_loss = dcgan.cost(
logits1, logits2, inter1, inter2)
# trainable variables
t_vars = tf.trainable_variables()
d_vars = [var for var in t_vars if 'd_' in var.name]
g_vars = [var for var in t_vars if 'g_' in var.name]
# train operations
d_optim = D_train_op(-d_loss, d_vars, FLAGS.learning_rate, FLAGS.beta1)
g_optim = G_train_op(g_loss + fm_loss, g_vars, FLAGS.learning_rate,
FLAGS.beta1)
# clip d parameters
clip_updates = [
tf.assign(var, tf.clip_by_value(var, -FLAGS.c_param,
FLAGS.c_param)) for var in d_vars
]
# saver
saver = tf.train.Saver()
# sampling from z
generate_images = dcgan.generate_images(z, 8, 8)
# initialization
init_op = tf.global_variables_initializer()
gpu_options = tf.GPUOptions(
per_process_gpu_memory_fraction=FLAGS.gpu_memory_fraction)
sess = tf.Session(config=tf.ConfigProto(allow_soft_placement=True,
gpu_options=gpu_options))
writer = tf.summary.FileWriter("./logs", sess.graph_def)
# run
sess.run(init_op)
# load trained parameters
model_dir = os.path.join(FLAGS.model_name, FLAGS.checkpoint_dir)
ckpt = tf.train.get_checkpoint_state(model_dir)
if ckpt and ckpt.model_checkpoint_path:
saver.restore(sess, ckpt.model_checkpoint_path)
print("Model restored.")
else:
print("No checkpoint file found")
# summary
g_sum = tf.summary.merge(
[z_sum, d2_sum, G_sum, d_loss_fake_sum, g_loss_sum])
d_sum = tf.summary.merge([z_sum, d1_sum, d_loss_real_sum, d_loss_sum])
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(sess=sess, coord=coord)
counter = 1
start_time = time.time()
for epoch in xrange(FLAGS.epochs):
for idx in xrange(0, int(datas.batch_idxs)):
batch_z = np.random.uniform(
-1, 1, [FLAGS.batch_size, FLAGS.z_dim]).astype(np.float32)
# D optimization
for _ in xrange(5):
if not gp:
sess.run([clip_updates], {z: batch_z})
images_inf_eval, _, summary_str = sess.run(
[images_inf, d_optim, d_sum], {z: batch_z})
writer.add_summary(summary_str, counter)
# twice G optimization
# _, summary_str = sess.run([g_optim, g_sum], {z: batch_z})
# writer.add_summary(summary_str, counter)
_, summary_str = sess.run([g_optim, g_sum], {z: batch_z})
writer.add_summary(summary_str, counter)
if gp:
errD = sess.run(d_loss, {z: batch_z})
errG = sess.run(g_loss, {z: batch_z})
print(
"epochs: %02d %04d/%04d time: %4.4f, d_loss: %.8f, g_loss: %.8f: sample count: %d"
% (epoch, idx, FLAGS.steps, time.time() - start_time, errD,
errG, counter))
else:
errD_fake = sess.run(d_loss_fake, {z: batch_z})
errD_real = sess.run(d_loss_real)
errG = sess.run(g_loss, {z: batch_z})
print(
"epochs: %02d %04d/%04d time: %4.4f, d_loss: %.8f (F:%.8f, R:%.8f), g_loss: %.8f: : sample count: %d"
% (epoch, idx, FLAGS.steps, time.time() - start_time,
errD_fake + errD_real, errD_fake, errD_real, errG,
counter))
if np.mod(counter, 100) == 1:
print("generate samples.")
generated_image_eval = sess.run(generate_images, {z: batch_z})
out_dir = os.path.join(FLAGS.model_name, FLAGS.sample_dir)
if not gfile.Exists(out_dir):
gfile.MakeDirs(out_dir)
filename = os.path.join(out_dir, 'out_%05d.png' % counter)
with open(filename, 'wb') as f:
f.write(generated_image_eval)
counter += 1
if np.mod(epoch, 10) == 0:
out_dir = os.path.join(FLAGS.model_name, FLAGS.checkpoint_dir)
if not gfile.Exists(out_dir):
gfile.MakeDirs(out_dir)
out_path = os.path.join(out_dir, 'model.ckpt')
saver.save(sess, out_path, global_step=epoch)
coord.request_stop()
coord.join(threads)
sess.close()
def create_image_lists(image_dir, testing_percentage, validation_percentage):
"""file system으로부터 training 이미지들의 list를 만든다.
이미지 디렉토리로부터 sub folder들을 분석하고, 그들을 training, testing, validation sets으로 나눈다.
그리고 각각의 label을 위한 이미지 list와 그들의 경로(path)를 나타내는 자료구조(data structure)를 반환한다.
인수들(Args):
image_dir: 이미지들의 subfolder들을 포함한 folder의 String path.
testing_percentage: 전체 이미지중 테스트를 위해 사용되는 비율을 나타내는 Integer.
validation_percentage: 전체 이미지중 validation을 위해 사용되는 비율을 나타내는 Integer.
반환값들(Returns):
각각의 label subfolder를 위한 enrtry를 포함한 dictionary A dictionary
(각각의 label에서 이미지드릉ㄴ training, testing, validation sets으로 나뉘어져 있다.)
"""
if not gfile.Exists(image_dir):
print("Image directory '" + image_dir + "' not found.")
return None
result = {}
sub_dirs = [x[0] for x in gfile.Walk(image_dir)]
# root directory는 처음에 온다. 따라서 이를 skip한다.
is_root_dir = True
for sub_dir in sub_dirs:
if is_root_dir:
is_root_dir = False
continue
extensions = ['jpg', 'jpeg', 'JPG', 'JPEG']
file_list = []
dir_name = os.path.basename(sub_dir)
if dir_name == image_dir:
continue
print("Looking for images in '" + dir_name + "'")
for extension in extensions:
file_glob = os.path.join(image_dir, dir_name, '*.' + extension)
file_list.extend(gfile.Glob(file_glob))
if not file_list:
print('No files found')
continue
if len(file_list) < 20:
print(
'WARNING: Folder has less than 20 images, which may cause issues.'
)
elif len(file_list) > MAX_NUM_IMAGES_PER_CLASS:
print(
'WARNING: Folder {} has more than {} images. Some images will '
'never be selected.'.format(dir_name,
MAX_NUM_IMAGES_PER_CLASS))
label_name = re.sub(r'[^a-z0-9]+', ' ', dir_name.lower())
training_images = []
testing_images = []
validation_images = []
for file_name in file_list:
base_name = os.path.basename(file_name)
# 어떤 이미지로 리스트를 만들지 결정할때 파일 이름에 "_nohash_"가 포함되어 있으면 이를 무시할 수 있다.
# 이를 이용해서, 데이터셋을 만드는 사람은 서로 비슷한 사진들을 grouping할 수있다.
# 예를 들어, plant disease를 데이터셋을 만들기 위해서, 여러 장의 같은 잎사귀(leaf)를 grouping할 수 있다.
hash_name = re.sub(r'_nohash_.*$', '', file_name)
# 이는 일종의 마법처럼 보일 수 있다. 하지만, 우리는 이 파일이 training sets로 갈지, testing sets로 갈지, validation sets로 갈지 결정해야만 한다.
# 그리고 우리는 더많은 파일들이 추가되더라도, 같은 set에 이미 존재하는 파일들이 유지되길 원한다.
# 그렇게 하기 위해서는, 우리는 파일 이름 그자체로부터 결정하는 안정적인 방법이 있어야만 한다.
# 따라서, 우리는 파일 이름을 hash하고, 이를 이를 할당하는데 사용하는 확률을 결정하는데 사용한다.
hash_name_hashed = hashlib.sha1(
compat.as_bytes(hash_name)).hexdigest()
percentage_hash = ((int(hash_name_hashed, 16) %
(MAX_NUM_IMAGES_PER_CLASS + 1)) *
(100.0 / MAX_NUM_IMAGES_PER_CLASS))
if percentage_hash < validation_percentage:
validation_images.append(base_name)
elif percentage_hash < (testing_percentage +
validation_percentage):
testing_images.append(base_name)
else:
training_images.append(base_name)
result[label_name] = {
'dir': dir_name,
'training': training_images,
'testing': testing_images,
'validation': validation_images,
}
return result
def get_random_decoded_images(sess, image_lists, how_many, category, image_dir,
jpeg_data_tensor, decoded_image_tensor):
"""Retrieves bottleneck values for cached images.
If no distortions are being applied, this function can retrieve the cached
bottleneck values directly from disk for images. It picks a random set of
images from the specified category.
Args:
sess: Current TensorFlow Session.
image_lists: Dictionary of training images for each label.
how_many: If positive, a random sample of this size will be chosen.
If negative, all bottlenecks will be retrieved.
category: Name string of which set to pull from - training, testing, or
validation.
bottleneck_dir: Folder string holding cached files of bottleneck values.
image_dir: Root folder string of the subfolders containing the training
images.
jpeg_data_tensor: The layer to feed jpeg image data into.
decoded_image_tensor: The output of decoding and resizing the image.
resized_input_tensor: The input node of the recognition graph.
bottleneck_tensor: The bottleneck output layer of the CNN graph.
architecture: The name of the model architecture.
Returns:
List of bottleneck arrays, their corresponding ground truths, and the
relevant filenames.
"""
class_count = len(image_lists.keys())
bottlenecks = []
ground_truths = []
filenames = []
decoded_image_datas = []
if how_many >= 0:
# Retrieve a random sample of bottlenecks.
for unused_i in range(how_many):
label_index = random.randrange(class_count)
label_name = list(image_lists.keys())[label_index]
image_index = random.randrange(MAX_NUM_IMAGES_PER_CLASS + 1)
image_path = get_image_path(image_lists, label_name, image_index,
image_dir, category)
if not gfile.Exists(image_path):
tf.logging.fatal('File does not exist %s', image_path)
image_data = gfile.FastGFile(image_path, 'rb').read()
decoded_image_data = get_decoded_image(sess, image_data,
jpeg_data_tensor,
decoded_image_tensor)
# bottleneck = get_or_create_bottleneck(
# sess, image_lists, label_name, image_index, image_dir, category,
# bottleneck_dir, jpeg_data_tensor, decoded_image_tensor,
# resized_input_tensor, bottleneck_tensor, architecture)
ground_truth = np.zeros(class_count, dtype=np.float32)
ground_truth[label_index] = 1.0
# bottlenecks.append(bottleneck)
ground_truths.append(ground_truth)
filenames.append(image_path)
decoded_image_datas.append(decoded_image_data[0])
else:
# Retrieve all bottlenecks.
for label_index, label_name in enumerate(image_lists.keys()):
for image_index, image_name in enumerate(
image_lists[label_name][category]):
image_path = get_image_path(image_lists, label_name,
image_index, image_dir, category)
if not gfile.Exists(image_path):
tf.logging.fatal('File does not exist %s', image_path)
image_data = gfile.FastGFile(image_path, 'rb').read()
# bottleneck = get_or_create_bottleneck(
# sess, image_lists, label_name, image_index, image_dir, category,
# bottleneck_dir, jpeg_data_tensor, decoded_image_tensor,
# resized_input_tensor, bottleneck_tensor, architecture)
ground_truth = np.zeros(class_count, dtype=np.float32)
ground_truth[label_index] = 1.0
# bottlenecks.append(bottleneck)
ground_truths.append(ground_truth)
filenames.append(image_path)
decoded_image_data = get_decoded_image(sess, image_data,
jpeg_data_tensor,
decoded_image_tensor)
decoded_image_datas.append(decoded_image_data[0])
# return bottlenecks, ground_truths, filenames
return np.array(decoded_image_datas), np.array(ground_truths), filenames
def create_vocabulary(vocabulary_path,
data_path,
max_vocabulary_size,
tokenizer=None,
normalize_digits=True,
_DIGIT_RE=re.compile(br"\d"),
_START_VOCAB=None):
"""Create vocabulary file (if it does not exist yet) from data file.
Data file is assumed to contain one sentence per line. Each sentence is
tokenized and digits are normalized (if normalize_digits is set).
Vocabulary contains the most-frequent tokens up to max_vocabulary_size.
We write it to vocabulary_path in a one-token-per-line format, so that later
token in the first line gets id=0, second line gets id=1, and so on.
Parameters
-----------
vocabulary_path : str
Path where the vocabulary will be created.
data_path : str
Data file that will be used to create vocabulary.
max_vocabulary_size : int
Limit on the size of the created vocabulary.
tokenizer : function
A function to use to tokenize each data sentence. If None, basic_tokenizer will be used.
normalize_digits : boolean
If true, all digits are replaced by `0`.
_DIGIT_RE : regular expression function
Default is ``re.compile(br"\d")``.
_START_VOCAB : list of str
The pad, go, eos and unk token, default is ``[b"_PAD", b"_GO", b"_EOS", b"_UNK"]``.
References
----------
- Code from ``/tensorflow/models/rnn/translation/data_utils.py``
"""
if _START_VOCAB is None:
_START_VOCAB = [b"_PAD", b"_GO", b"_EOS", b"_UNK"]
if not gfile.Exists(vocabulary_path):
logging.info("Creating vocabulary %s from data %s" %
(vocabulary_path, data_path))
vocab = {}
with gfile.GFile(data_path, mode="rb") as f:
counter = 0
for line in f:
counter += 1
if counter % 100000 == 0:
logging.info(" processing line %d" % counter)
tokens = tokenizer(line) if tokenizer else basic_tokenizer(
line)
for w in tokens:
word = re.sub(_DIGIT_RE, b"0",
w) if normalize_digits else w
if word in vocab:
vocab[word] += 1
else:
vocab[word] = 1
vocab_list = _START_VOCAB + sorted(
vocab, key=vocab.get, reverse=True)
if len(vocab_list) > max_vocabulary_size:
vocab_list = vocab_list[:max_vocabulary_size]
with gfile.GFile(vocabulary_path, mode="wb") as vocab_file:
for w in vocab_list:
vocab_file.write(w + b"\n")
else:
logging.info("Vocabulary %s from data %s exists" %
(vocabulary_path, data_path))
def timeGraph(gdef,
batch_size,
num_loops,
input_name,
outputs,
timelineName=None):
tf.logging.info("Starting execution")
gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=0.50)
tf.reset_default_graph()
g = tf.Graph()
outlist = []
with g.as_default():
rand = tf.random_normal(shape=(batch_size, 64))
out = tf.import_graph_def(graph_def=gdef,
input_map={input_name: rand},
return_elements=outputs)
out = out[0].outputs[0]
outlist.append(out)
timings = []
with tf.Session(graph=g,
config=tf.ConfigProto(gpu_options=gpu_options)) as sess:
run_options = tf.RunOptions(trace_level=tf.RunOptions.FULL_TRACE)
run_metadata = tf.RunMetadata()
tf.logging.info("Starting Warmup cycle")
def mergeTraceStr(mdarr):
tl = timeline.Timeline(mdarr[0][0].step_stats)
ctf = tl.generate_chrome_trace_format()
Gtf = json.loads(ctf)
deltat = mdarr[0][1][1]
for md in mdarr[1:]:
tl = timeline.Timeline(md[0].step_stats)
ctf = tl.generate_chrome_trace_format()
tmp = json.loads(ctf)
deltat = 0
Gtf["traceEvents"].extend(tmp["traceEvents"])
deltat = md[1][1]
return json.dumps(Gtf, indent=2)
rmArr = [[tf.RunMetadata(), 0] for x in range(20)]
if timelineName:
if gfile.Exists(timelineName):
gfile.Remove(timelineName)
ttot = int(0)
tend = time.time()
for i in range(20):
tstart = time.time()
valt = sess.run(outlist,
options=run_options,
run_metadata=rmArr[i][0])
tend = time.time()
rmArr[i][1] = (int(tstart * 1.e6), int(tend * 1.e6))
with gfile.FastGFile(timelineName, "a") as tlf:
tlf.write(mergeTraceStr(rmArr))
else:
for i in range(20):
valt = sess.run(outlist)
tf.logging.info("Warmup done. Starting real timing")
num_iters = 50
for i in range(num_loops):
tstart = time.time()
for k in range(num_iters):
val = sess.run(outlist)
timings.append((time.time() - tstart) / float(num_iters))
print("iter ", i, " ", timings[-1])
comp = sess.run(tf.reduce_all(tf.equal(val[0], valt[0])))
print("Comparison=", comp)
sess.close()
tf.logging.info("Timing loop done!")
return timings, comp, val[0], None
