def detection(image_path, verbose=False):
# input noize to generator
z = tf.placeholder(tf.float32, [None, FLAGS.z_dim], name='z')
# input image to reverser
samples = tf.placeholder(
tf.float32,
[FLAGS.sample_num, FLAGS.image_height, FLAGS.image_width, FLAGS.c_dim],
name='sample_inputs')
# base model class
dcgan = DCGAN_SR(FLAGS.model_name, FLAGS.checkpoint_dir)
# generate vector and discriminator output
d_logits = dcgan.step(samples)
# saver
saver = tf.train.Saver()
# create session
gpu_options = tf.GPUOptions(
per_process_gpu_memory_fraction=FLAGS.gpu_memory_fraction)
sess = tf.Session(config=tf.ConfigProto(allow_soft_placement=True,
gpu_options=gpu_options))
sess.run(tf.global_variables_initializer())
# load parameters
model_dir = os.path.join(FLAGS.model_name, FLAGS.checkpoint_dir)
ckpt = tf.train.get_checkpoint_state(model_dir)
if ckpt and ckpt.model_checkpoint_path:
print("Model: %s" % (ckpt.model_checkpoint_path))
saver.restore(sess, ckpt.model_checkpoint_path)
else:
print("No checkpoint file found")
exit()
print("Model restored.")
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(sess=sess, coord=coord)
path, ext = os.path.splitext(os.path.basename(image_path))
if ext == '.csv':
images = load_csv(image_path)
vectors_evals = []
for i, image in enumerate(images):
# temporaly
if i == FLAGS.db_size:
break
print("No.%d %s" % (i, image[0]))
pil_img = Image.open(image[0])
pil_img = pil_img.resize(
(FLAGS.image_height_org, FLAGS.image_width_org))
img_array = np.asarray(pil_img)
if img_array.size != FLAGS.image_height_org * FLAGS.image_width_org * FLAGS.c_dim:
continue
height_diff = FLAGS.image_height_org - FLAGS.image_height
width_diff = FLAGS.image_width_org - FLAGS.image_width
img_array = img_array[int(height_diff / 2):int(height_diff / 2) +
FLAGS.image_height,
int(width_diff / 2):int(width_diff / 2) +
FLAGS.image_width, :]
# input for reverser image = tf.subtract(tf.div(image, 127.5), 1.0)
img_array = img_array / 127.5 - 1.0
img_array = img_array[None, ...]
d_logits_eval = sess.run([d_logits], {samples: img_array})
if verbose:
print("discriminator confidence:")
print(d_logits_eval[0])
#vectors_evals.append(vectors_eval[0])
if d_logits_eval[0] < FLAGS.anomaly_threshold:
print(d_logits_eval)
print("anomaly: %s: %f" % (image[0], d_logits_eval[0]))
fig = plt.figure()
a = fig.add_subplot(1, 1, 1)
lum2_img = pil_img
imgplot = plt.imshow(lum2_img)
a.set_title('discriminator detection')
a.set_xlabel("confidence: %f" % d_logits_eval[0])
out_dir = os.path.join(FLAGS.model_name, FLAGS.anomaly_dir)
if not gfile.Exists(out_dir):
gfile.MakeDirs(out_dir)
out_path = os.path.join(out_dir, "anom_%d.png" % (i))
plt.savefig(out_path)
if FLAGS.mode == 'sampling':
#features_obj = features.Features(images, vectors_evals)
pass
# TODO save features object
else:
# visualization
print("Calculate NearestNeighbors:")
X = np.array(vectors_evals)
print(X.shape)
nbrs = NearestNeighbors(n_neighbors=2, algorithm='auto').fit(X)
distances, indices = nbrs.kneighbors(X)
print("10 ramdom samples")
sample_index = np.random.randint(FLAGS.db_size, size=10000)
for i, index in enumerate(sample_index):
nbrs_sample = indices[index]
nbrs_distance = distances[index]
sample_relate_image = images[nbrs_sample[0]][0]
top_1_index = nbrs_sample[1]
top_1_nbrs_distance = nbrs_distance[1]
if top_1_nbrs_distance >= 3.5:
continue
nn_image = images[top_1_index][0]
print("No.%d sample similarity." % i)
print(sample_relate_image)
print(nn_image)
sample_relate_image_mat = mpimg.imread(sample_relate_image)
nn_image_mat = mpimg.imread(nn_image)
fig = plt.figure()
a = fig.add_subplot(1, 2, 1)
lum_img = sample_relate_image_mat
imgplot = plt.imshow(lum_img)
a.set_title('Sample')
a = fig.add_subplot(1, 2, 2)
lum2_img = nn_image_mat
imgplot = plt.imshow(lum2_img)
a.set_title('NearestNeighbors Top-1')
a.set_xlabel("distance: %f" % top_1_nbrs_distance)
now = datetime.datetime.now()
utime = now.strftime("%s")
out_dir = os.path.join(FLAGS.model_name, FLAGS.sample_dir)
if not gfile.Exists(out_dir):
gfile.MakeDirs(out_dir)
out_path = os.path.join(out_dir, "%d_%s.png" % (i, utime))
plt.savefig(out_path)
else:
pil_img = Image.open(image_path)
pil_img = pil_img.resize((64, 64))
img_array = np.asarray(pil_img)
#input for reverser image = tf.subtract(tf.div(image, 127.5), 1.0)
img_array = img_array / 127.5 - 1.0
img_array = img_array[None, ...]
d_logits_eval = sess.run([d_logits], {samples: img_array})
print(d_logits_eval)
# regenerate_sample = sess.run(regenerate, {z: input_vector})
# out_dir = os.path.join(FLAGS.model_name, FLAGS.sample_dir)
# now = datetime.datetime.now()
# utime = now.strftime("%s")
# if not gfile.Exists(out_dir):
# gfile.MakeDirs(out_dir)
# filename = os.path.join(out_dir, "%s.png" % (utime))
# with open(filename, 'wb') as f:
# f.write(regenerate_sample)
# fig = plt.figure()
# a = fig.add_subplot(1, 2, 1)
# lum_img = mpimg.imread(image_path)
# imgplot = plt.imshow(lum_img)
# a.set_title('Original')
#
# a = fig.add_subplot(1, 2, 2)
# lum2_img = regenerate_sample
# imgplot = plt.imshow(lum2_img)
# a.set_title('Re Sampling')
#
# out_dir = os.path.join(FLAGS.model_name, FLAGS.sample_dir)
# if not gfile.Exists(out_dir):
# gfile.MakeDirs(out_dir)
# now = datetime.datetime.now()
# utime = now.strftime("%s")
# out_path = os.path.join(out_dir, "%s.png" % (utime))
# plt.savefig(out_path)
print("finish to predict.")
coord.request_stop()
coord.join(threads)
sess.close()
def _TestDir(self, test_name):
test_dir = os.path.join(self.get_temp_dir(), test_name)
if os.path.exists(test_dir):
shutil.rmtree(test_dir)
gfile.MakeDirs(test_dir)
return test_dir
def main(unused_argv):
logging.set_verbosity(logging.INFO)
if not gfile.IsDirectory(FLAGS.resource_path):
gfile.MakeDirs(FLAGS.resource_path)
# Constructs lexical resources for SyntaxNet in the given resource path, from
# the training data.
if FLAGS.compute_lexicon:
logging.info('Computing lexicon...')
lexicon.build_lexicon(FLAGS.resource_path, FLAGS.training_corpus_path)
# Construct the "lookahead" ComponentSpec. This is a simple right-to-left RNN
# sequence model, which encodes the context to the right of each token. It has
# no loss except for the downstream components.
char2word = spec_builder.ComponentSpecBuilder('char_lstm')
char2word.set_network_unit(name='wrapped_units.LayerNormBasicLSTMNetwork',
hidden_layer_sizes='256')
char2word.set_transition_system(name='char-shift-only',
left_to_right='true')
char2word.add_fixed_feature(name='chars',
fml='char-input.text-char',
embedding_dim=16)
char2word.fill_from_resources(FLAGS.resource_path, FLAGS.tf_master)
lookahead = spec_builder.ComponentSpecBuilder('lookahead')
lookahead.set_network_unit(name='wrapped_units.LayerNormBasicLSTMNetwork',
hidden_layer_sizes='256')
lookahead.set_transition_system(name='shift-only', left_to_right='false')
lookahead.add_link(source=char2word,
fml='input.last-char-focus',
embedding_dim=32)
lookahead.fill_from_resources(FLAGS.resource_path, FLAGS.tf_master)
# Construct the ComponentSpec for tagging. This is a simple left-to-right RNN
# sequence tagger.
tagger = spec_builder.ComponentSpecBuilder('tagger')
tagger.set_network_unit(name='wrapped_units.LayerNormBasicLSTMNetwork',
hidden_layer_sizes='256')
tagger.set_transition_system(name='tagger')
tagger.add_token_link(source=lookahead,
fml='input.focus',
embedding_dim=32)
tagger.fill_from_resources(FLAGS.resource_path, FLAGS.tf_master)
# Construct the ComponentSpec for parsing.
parser = spec_builder.ComponentSpecBuilder('parser')
parser.set_network_unit(name='FeedForwardNetwork',
hidden_layer_sizes='256',
layer_norm_hidden='True')
parser.set_transition_system(name='arc-standard')
parser.add_token_link(source=lookahead,
fml='input.focus',
embedding_dim=32)
parser.add_token_link(source=tagger,
fml='input.focus stack.focus stack(1).focus',
embedding_dim=32)
# Recurrent connection for the arc-standard parser. For both tokens on the
# stack, we connect to the last time step to either SHIFT or REDUCE that
# token. This allows the parser to build up compositional representations of
# phrases.
parser.add_link(
source=parser, # recurrent connection
name='rnn-stack', # unique identifier
fml='stack.focus stack(1).focus', # look for both stack tokens
source_translator='shift-reduce-step', # maps token indices -> step
embedding_dim=32) # project down to 32 dims
parser.fill_from_resources(FLAGS.resource_path, FLAGS.tf_master)
master_spec = spec_pb2.MasterSpec()
master_spec.component.extend(
[char2word.spec, lookahead.spec, tagger.spec, parser.spec])
logging.info('Constructed master spec: %s', str(master_spec))
hyperparam_config = spec_pb2.GridPoint()
hyperparam_config.decay_steps = 128000
hyperparam_config.learning_rate = 0.001
hyperparam_config.learning_method = 'adam'
hyperparam_config.adam_beta1 = 0.9
hyperparam_config.adam_beta2 = 0.9
hyperparam_config.adam_eps = 0.0001
hyperparam_config.gradient_clip_norm = 1
hyperparam_config.self_norm_alpha = 1.0
hyperparam_config.use_moving_average = True
hyperparam_config.dropout_rate = 0.7
hyperparam_config.seed = 1
# Build the TensorFlow graph.
graph = tf.Graph()
with graph.as_default():
builder = graph_builder.MasterBuilder(master_spec, hyperparam_config)
component_targets = spec_builder.default_targets_from_spec(master_spec)
trainers = [
builder.add_training_from_config(target)
for target in component_targets
]
assert len(trainers) == 2
annotator = builder.add_annotation()
builder.add_saver()
# Read in serialized protos from training data.
training_set = sentence_io.ConllSentenceReader(
FLAGS.training_corpus_path,
projectivize=FLAGS.projectivize_training_set).corpus()
dev_set = sentence_io.ConllSentenceReader(FLAGS.dev_corpus_path,
projectivize=False).corpus()
# Ready to train!
logging.info('Training on %d sentences.', len(training_set))
logging.info('Tuning on %d sentences.', len(dev_set))
pretrain_steps = [100, 0]
tagger_steps = 1000
train_steps = [tagger_steps, 8 * tagger_steps]
tf.logging.info('Creating TensorFlow checkpoint dir...')
gfile.MakeDirs(os.path.dirname(FLAGS.checkpoint_filename))
summary_writer = trainer_lib.get_summary_writer(FLAGS.tensorboard_dir)
with tf.Session(FLAGS.tf_master, graph=graph) as sess:
# Make sure to re-initialize all underlying state.
sess.run(tf.global_variables_initializer())
trainer_lib.run_training(sess, trainers, annotator,
evaluation.parser_summaries, pretrain_steps,
train_steps, training_set, dev_set, dev_set,
FLAGS.batch_size, summary_writer,
FLAGS.report_every, builder.saver,
FLAGS.checkpoint_filename)
def export_savedmodel(self,
export_dir_base,
serving_input_receiver_fn,
assets_extra=None,
as_text=False,
checkpoint_path=None):
"""Exports inference graph as a SavedModel into given dir.
This method builds a new graph by first calling the
serving_input_receiver_fn to obtain feature `Tensor`s, and then calling
this `Estimator`'s model_fn to generate the model graph based on those
features. It restores the given checkpoint (or, lacking that, the most
recent checkpoint) into this graph in a fresh session. Finally it creates
a timestamped export directory below the given export_dir_base, and writes
a `SavedModel` into it containing a single `MetaGraphDef` saved from this
session.
The exported `MetaGraphDef` will provide one `SignatureDef` for each
element of the export_outputs dict returned from the model_fn, named using
the same keys. One of these keys is always
signature_constants.DEFAULT_SERVING_SIGNATURE_DEF_KEY, indicating which
signature will be served when a serving request does not specify one.
For each signature, the outputs are provided by the corresponding
`ExportOutput`s, and the inputs are always the input receivers provided by
the serving_input_receiver_fn.
Extra assets may be written into the SavedModel via the extra_assets
argument. This should be a dict, where each key gives a destination path
(including the filename) relative to the assets.extra directory. The
corresponding value gives the full path of the source file to be copied.
For example, the simple case of copying a single file without renaming it
is specified as `{'my_asset_file.txt': '/path/to/my_asset_file.txt'}`.
Args:
export_dir_base: A string containing a directory in which to create
timestamped subdirectories containing exported SavedModels.
serving_input_receiver_fn: A function that takes no argument and
returns a `ServingInputReceiver`.
assets_extra: A dict specifying how to populate the assets.extra directory
within the exported SavedModel, or `None` if no extra assets are needed.
as_text: whether to write the SavedModel proto in text format.
checkpoint_path: The checkpoint path to export. If `None` (the default),
the most recent checkpoint found within the model directory is chosen.
Returns:
The string path to the exported directory.
Raises:
ValueError: if no serving_input_receiver_fn is provided, no export_outputs
are provided, or no checkpoint can be found.
"""
if serving_input_receiver_fn is None:
raise ValueError('serving_input_receiver_fn must be defined.')
with ops.Graph().as_default() as g:
self._create_and_assert_global_step(g)
random_seed.set_random_seed(self._config.tf_random_seed)
serving_input_receiver = serving_input_receiver_fn()
# Call the model_fn and collect the export_outputs.
estimator_spec = self._call_model_fn(
features=serving_input_receiver.features,
labels=None,
mode=model_fn_lib.ModeKeys.PREDICT)
# Build the SignatureDefs from receivers and all outputs
signature_def_map = build_all_signature_defs(
serving_input_receiver.receiver_tensors,
estimator_spec.export_outputs)
if not checkpoint_path:
# Locate the latest checkpoint
checkpoint_path = saver.latest_checkpoint(self._model_dir)
if not checkpoint_path:
raise ValueError("Couldn't find trained model at %s." %
self._model_dir)
export_dir = get_timestamped_export_dir(export_dir_base)
temp_export_dir = get_temp_export_dir(export_dir)
# TODO(soergel): Consider whether MonitoredSession makes sense here
with tf_session.Session() as session:
saver_for_restore = estimator_spec.scaffold.saver or saver.Saver(
sharded=True)
saver_for_restore.restore(session, checkpoint_path)
# TODO(b/36111876): replace legacy_init_op with main_op mechanism
# pylint: disable=protected-access
local_init_op = (
estimator_spec.scaffold.local_init_op
or monitored_session.Scaffold._default_local_init_op())
# pylint: enable=protected-access
# Perform the export
builder = saved_model_builder.SavedModelBuilder(
temp_export_dir)
builder.add_meta_graph_and_variables(
session, [tag_constants.SERVING],
signature_def_map=signature_def_map,
assets_collection=ops.get_collection(
ops.GraphKeys.ASSET_FILEPATHS),
legacy_init_op=local_init_op)
builder.save(as_text)
# Add the extra assets
if assets_extra:
assets_extra_path = os.path.join(
compat.as_bytes(temp_export_dir),
compat.as_bytes('assets.extra'))
for dest_relative, source in assets_extra.items():
dest_absolute = os.path.join(
compat.as_bytes(assets_extra_path),
compat.as_bytes(dest_relative))
dest_path = os.path.dirname(dest_absolute)
gfile.MakeDirs(dest_path)
gfile.Copy(source, dest_absolute)
gfile.Rename(temp_export_dir, export_dir)
return export_dir
def write_report_proto(self, report_proto, tt_parameters):
"""Writes the given report proto under trace_dir."""
gfile.MakeDirs(tt_parameters.trace_dir)
report_path = report_proto_path(tt_parameters.trace_dir)
with gfile.GFile(report_path, 'wb') as f:
f.write(report_proto.SerializeToString())
def _dump_file(self, pb, basename):
if not gfile.Exists(self._profiler_dir):
gfile.MakeDirs(self._profiler_dir)
with gfile.Open(os.path.join(self._profiler_dir, basename), 'w') as f:
f.write('%s' % pb)
def main(argv=None): # pylint: disable=unused-argument
cifar10.maybe_download_and_extract()
if gfile.Exists(FLAGS.train_dir):
gfile.DeleteRecursively(FLAGS.train_dir)
gfile.MakeDirs(FLAGS.train_dir)
train()
def setUp(self):
self._base_dir = os.path.join(self.get_temp_dir(), "saver_utils_test")
gfile.MakeDirs(self._base_dir)
def setUp(self):
self._base_dir = os.path.join(self.get_temp_dir(),
'keras_mirrored_strategy_test')
gfile.MakeDirs(self._base_dir)
self._config = run_config_lib.RunConfig(tf_random_seed=_RANDOM_SEED,
model_dir=self._base_dir)
def mytrain_main(argv=None):
if not FLAGS.restore_bool:
if gfile.Exists(FLAGS.train_dir):
gfile.DeleteRecursively(FLAGS.train_dir)
gfile.MakeDirs(FLAGS.train_dir)
train()
def _get_test_dir(self, dirname):
test_dir = os.path.join(self.get_temp_dir(), dirname)
gfile.MakeDirs(test_dir)
return test_dir
def train_in_batches(self,
train_dataset,
val_dataset,
learning_rate,
layers,
losses=None,
epochs_to_run=1,
batch_size=0,
steps_per_epoch=0):
'''
Train the model.
train_dataset,
val_dataset: Training and validation Dataset objects.
learning_rate: The learning rate to train with
epochs: Number of training epochs. Note that previous training epochs
are considered to be done already, so this actually determines
the epochs to train in total rather than in this particaular
call.
layers: Allows selecting wich layers to train. It can be:
- A regular expression to match layer names to train
- One of these predefined values:
heads: The RPN, classifier and mask heads of the network
all: All the layers
3+: Train Resnet stage 3 and up
4+: Train Resnet stage 4 and up
5+: Train Resnet stage 5 and up
'''
assert self.mode == "training", "Create model in training mode."
# Use Pre-defined layer regular expressions
# if layers in self.layer_regex.keys():
# layers = self.layer_regex[layers]
print(layers)
train_regex_list = [self.layer_regex[x] for x in layers]
print(train_regex_list)
layers = '|'.join(train_regex_list)
print('layers regex :', layers)
if batch_size == 0:
batch_size = self.config.BATCH_SIZE
if steps_per_epoch == 0:
steps_per_epoch = self.config.STEPS_PER_EPOCH
# Data generators
train_generator = data_generator(train_dataset,
self.config,
shuffle=True,
batch_size=batch_size)
val_generator = data_generator(val_dataset,
self.config,
shuffle=True,
batch_size=batch_size,
augment=False)
log(" Last epoch completed : {} ".format(self.epoch))
log(" Starting from epoch : {} for {} epochs".format(
self.epoch, epochs_to_run))
log(" Learning Rate : {} ".format(learning_rate))
log(" Steps per epoch : {} ".format(steps_per_epoch))
log(" Batchsize : {} ".format(batch_size))
log(" Checkpoint Folder : {} ".format(self.checkpoint_path))
epochs = self.epoch + epochs_to_run
from tensorflow.python.platform import gfile
if not gfile.IsDirectory(self.log_dir):
log('Creating checkpoint folder')
gfile.MakeDirs(self.log_dir)
else:
log('Checkpoint folder already exists')
self.set_trainable(layers)
self.compile(learning_rate, self.config.LEARNING_MOMENTUM, losses)
# copied from \keras\engine\training.py
# def _get_deduped_metrics_names(self):
## get metrics from keras_model.metrics_names
out_labels = self.get_deduped_metrics_names()
print(' ====> out_labels : ', out_labels)
## setup Progress Bar callback
callback_metrics = out_labels + ['val_' + n for n in out_labels]
print(' Callback metrics monitored by progbar')
pp.pprint(callback_metrics)
progbar = keras.callbacks.ProgbarLogger(count_mode='steps')
progbar.set_model(self.keras_model)
progbar.set_params({
'epochs': epochs,
'steps': steps_per_epoch,
'verbose': 1,
'do_validation': False,
'metrics': callback_metrics,
})
progbar.set_model(self.keras_model)
## setup Checkpoint callback
chkpoint = keras.callbacks.ModelCheckpoint(self.checkpoint_path,
monitor='val_loss',
verbose=1,
save_best_only=True,
save_weights_only=True)
chkpoint.set_model(self.keras_model)
progbar.on_train_begin()
epoch_idx = self.epoch
if epoch_idx >= epochs:
print(
'Final epoch {} has already completed - Training will not proceed'
.format(epochs))
else:
while epoch_idx < epochs:
progbar.on_epoch_begin(epoch_idx)
for steps_index in range(steps_per_epoch):
batch_logs = {}
# print(' self.epoch {} epochs {} step {} '.format(self.epoch, epochs, steps_index))
batch_logs['batch'] = steps_index
batch_logs['size'] = batch_size
progbar.on_batch_begin(steps_index, batch_logs)
train_batch_x, train_batch_y = next(train_generator)
outs = self.keras_model.train_on_batch(
train_batch_x, train_batch_y)
if not isinstance(outs, list):
outs = [outs]
for l, o in zip(out_labels, outs):
batch_logs[l] = o
progbar.on_batch_end(steps_index, batch_logs)
# print(outs)
progbar.on_epoch_end(epoch_idx, {})
# if (epoch_idx % 10) == 0:
chkpoint.on_epoch_end(epoch_idx, batch_logs)
epoch_idx += 1
# if epoch_idx != self.epoch:
# chkpoint.on_epoch_end(epoch_idx -1, batch_logs)
self.epoch = max(epoch_idx - 1, epochs)
print('Final : self.epoch {} epochs {}'.format(
self.epoch, epochs))
def prepare_encode_decode_data(source_path, save_path,
encode_decode_window, encode_decode_gap, encode_decode_step,
if_sample=False, sample_number=0):
"""
为预测模型准备<编码, 解码, 标签>样本
Args:
source_path: 原始文件输入路径
save_path: 保存文件路径
label: 错误/正常类型
encode_decode_window: 编解码对窗口大小
encode_decode_gap: 编解码对间距
encode_decode_step: 编解码对窗口每次滑动步长
if_sample: 是否采样取
sample_number： 采样时多少行取一次
Returns:
"""
# 先判断是否已经将原始样本切分为normal/error/recovery三部分
if not gfile.Exists(pjoin(save_path, "error")):
print("cut data to normal/error/recovery...")
cut_data(save_path)
print("get <encode, decode, label> data...")
if_err = []
count = 0
this_save_path = pjoin(save_path, "_".join([str(encode_decode_window), str(encode_decode_gap), str(encode_decode_step)]))
gfile.MakeDirs(pjoin(this_save_path, "train"))
gfile.MakeDirs(pjoin(this_save_path, "test"))
print("this_save_path = ", this_save_path)
f_en = gfile.GFile(pjoin(this_save_path, "encode.txt"), mode="wb")
f_de = gfile.GFile(pjoin(this_save_path, "decode.txt"), mode="wb")
for num in range(1, len(gfile.ListDirectory(pjoin(save_path, "error")))):
with gfile.GFile(pjoin(save_path, "normal", str(num)+".txt"), mode="rb") as normal_f, \
gfile.GFile(pjoin(save_path, "error", str(num)+".txt"), mode="rb") as error_f:
normal_c = normal_f.readlines()
error_c = error_f.readlines()
contxt = normal_c + error_c
# print("len(contxt) = %d"%len(contxt), "len(normal_c) = %d"%len(normal_c), "len(error_c) = %d"%len(error_c))
encode_s = 0
encode_e = encode_s + encode_decode_window
decode_s = encode_e + encode_decode_gap
decode_e = decode_s + encode_decode_window
while (decode_e < len(normal_c)):
if not if_sample:
this_line = " ".join([s.strip() for s in contxt[encode_s: encode_e]])
f_en.write(this_line + "\n")
this_line = " ".join([s.strip() for s in contxt[decode_s: decode_e]])
f_de.write(this_line + "\n")
else:
this_line = " ".join([s.strip() for s in contxt[encode_s: encode_e: sample_number]])
f_en.write(this_line + "\n")
this_line = " ".join([s.strip() for s in contxt[decode_s: decode_e: sample_number]])
f_de.write(this_line + "\n")
if_err.append("Normal")
encode_s += encode_decode_step
encode_e = encode_s + encode_decode_window
decode_s = encode_e + encode_decode_gap
decode_e = decode_s + encode_decode_window
while (encode_e < len(normal_c) and decode_e < len(contxt)):
if not if_sample:
this_line = " ".join([s.strip() for s in contxt[encode_s: encode_e]])
else:
this_line = " ".join([s.strip() for s in contxt[encode_s: encode_e: sample_number]])
f_en.write(this_line + "\n")
this_line = ""
if not if_sample:
for line in contxt[decode_s: decode_e]:
arr = line.split()
write = True
for word in ERRORNAME:
if word in arr:
# print(line, "ERRORNAME")
write = False
break
if write:
this_line = " ".join([this_line, line.strip()])
else:
for line in contxt[decode_s: decode_e: sample_number]:
arr = line.split()
write = True
for word in ERRORNAME:
if word in arr:
# print(line, "ERRORNAME")
write = False
break
if write:
this_line = " ".join([this_line, line.strip()])
f_de.write(this_line + "\n")
# outf.write(line.strip()+" SEN_END\n")
# outf.write(line.strip() + "\n")
if_err.append("Error")
count += 1
encode_s += encode_decode_step
encode_e = encode_s + encode_decode_window
decode_s = encode_e + encode_decode_gap
decode_e = decode_s + encode_decode_window
#print("len(if_err)", len(if_err))
with gfile.GFile(pjoin(this_save_path, "labels.txt"), "wb") as wf:
for state in if_err:
wf.write(state + "\n")
def main(argv=None): # pylint: disable=unused-argument
if gfile.Exists(FLAGS.eval_dir):
gfile.DeleteRecursively(FLAGS.eval_dir)
gfile.MakeDirs(FLAGS.eval_dir)
evaluate()
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Utilities for manipulating files.
"""
import os
import PIL
from tensorflow.python.platform import gfile
import cv2
exists = lambda path: gfile.Exists(path)
fopen = lambda path, mode: gfile.Open(path, mode)
makedirs = lambda path: gfile.MakeDirs(path)
listdir = lambda path: gfile.ListDir(path)
copyfile = lambda a, b, o: gfile.Copy(a, b, o)
def write_image(image_path, rgb):
ext = os.path.splitext(image_path)[1]
with gfile.GFile(image_path, 'w') as f:
img_str = cv2.imencode(ext, rgb[:, :, ::-1])[1].tostring()
f.write(img_str)
def read_image(image_path, type='rgb'):
with fopen(file_name, 'r') as f:
I = PIL.Image.open(f)
II = np.array(I)
def setUp(self):
self._base_dir = os.path.join(self.get_temp_dir(), 'keras_estimator_test')
gfile.MakeDirs(self._base_dir)
self._config = run_config_lib.RunConfig(
tf_random_seed=_RANDOM_SEED, model_dir=self._base_dir)
super(TestKerasEstimator, self).setUp()
# Calculate predictions.
top_k_op = tf.nn.in_top_k(logits, labels, 1)
# Restore the moving average version of the learned variables for eval.
variable_averages = tf.train.ExponentialMovingAverage(
cifar10.MOVING_AVERAGE_DECAY)
variables_to_restore = variable_averages.variables_to_restore()
saver = tf.train.Saver(variables_to_restore)
# Build the summary operation based on the TF collection of Summaries.
summary_op = tf.summary.merge_all()
graph_def = tf.get_default_graph().as_graph_def()
summary_writer = tf.summary.FileWriter(FLAGS.eval_dir,
graph_def=graph_def)
while True:
eval_once(saver, summary_writer, top_k_op, summary_op)
if FLAGS.run_once:
break
time.sleep(FLAGS.eval_interval_secs)
if __name__ == '__main__':
cifar10.maybe_download_and_extract()
if gfile.Exists(FLAGS.eval_dir):
gfile.DeleteRecursively(FLAGS.eval_dir)
gfile.MakeDirs(FLAGS.eval_dir)
evaluate()
exit('testing finished')
def main(_argv):
"""Program entrypoint.
"""
predictions, _, _ = create_inference_graph(model_dir=FLAGS.model_dir,
input_file=FLAGS.source,
batch_size=FLAGS.batch_size,
beam_width=FLAGS.beam_width)
# Filter fetched predictions to save memory
prediction_keys = set([
"predicted_tokens", "features.source_len", "features.source_tokens",
"attention_scores"
])
# Optional UNK token replacement
unk_replace_fn = None
if FLAGS.unk_replace:
if "attention_scores" not in predictions.keys():
raise ValueError("""To perform UNK replacement you must use a model
class that outputs attention scores.""")
prediction_keys.add("attention_scores")
mapping = None
if FLAGS.unk_mapping is not None:
mapping = get_unk_mapping(FLAGS.unk_mapping)
if FLAGS.unk_replace:
unk_replace_fn = functools.partial(unk_replace, mapping=mapping)
predictions = {
k: v
for k, v in predictions.items() if k in prediction_keys
}
saver = tf.train.Saver()
checkpoint_path = FLAGS.checkpoint_path
if not checkpoint_path:
checkpoint_path = tf.train.latest_checkpoint(FLAGS.model_dir)
with tf.Session() as sess:
# Initialize variables
sess.run(tf.global_variables_initializer())
sess.run(tf.local_variables_initializer())
sess.run(tf.tables_initializer())
# Restore checkpoint
saver.restore(sess, checkpoint_path)
tf.logging.info("Restored model from %s", checkpoint_path)
# Accumulate attention scores in this array.
# Shape: [num_examples, target_length, input_length]
attention_scores_accum = []
if FLAGS.dump_attention_dir is not None:
gfile.MakeDirs(FLAGS.dump_attention_dir)
# Output predictions
predictions_iter = create_predictions_iter(predictions, sess)
for idx, predictions_dict in enumerate(predictions_iter):
# Convert to unicode
predictions_dict["predicted_tokens"] = np.char.decode(
predictions_dict["predicted_tokens"].astype("S"), "utf-8")
predicted_tokens = predictions_dict["predicted_tokens"]
if FLAGS.beam_width is not None:
# If we're using beam search we take the first beam
predicted_tokens = predicted_tokens[:, 0]
predictions_dict["features.source_tokens"] = np.char.decode(
predictions_dict["features.source_tokens"].astype("S"),
"utf-8")
source_tokens = predictions_dict["features.source_tokens"]
source_len = predictions_dict["features.source_len"]
if unk_replace_fn is not None:
# We slice the attention scores so that we do not
# accidentially replace UNK with a SEQUENCE_END token
attention_scores = predictions_dict["attention_scores"]
attention_scores = attention_scores[:, :source_len - 1]
predicted_tokens = unk_replace_fn(
source_tokens=source_tokens,
predicted_tokens=predicted_tokens,
attention_scores=attention_scores)
# Optionally Dump attention
if FLAGS.dump_attention_dir is not None:
if not FLAGS.dump_attention_no_plot:
output_path = os.path.join(FLAGS.dump_attention_dir,
"{:05d}.png".format(idx))
create_figure(predictions_dict)
plt.savefig(output_path)
plt.close()
tf.logging.info("Wrote %s", output_path)
attention_scores_accum.append(get_scores(predictions_dict))
sent = FLAGS.delimiter.join(predicted_tokens).split(
"SEQUENCE_END")[0]
# Replace special BPE tokens
sent = sent.replace("@@ ", "")
sent = sent.strip()
print(sent)
# Write attention scores
if FLAGS.dump_attention_dir is not None:
scores_path = os.path.join(FLAGS.dump_attention_dir,
"attention_scores.npz")
np.savez(scores_path, *attention_scores_accum)
def save_model(self, output_dir):
if not gfile.Exists(output_dir):
gfile.MakeDirs(output_dir)
checkpoint_path = output_dir + '/model'
self.saver.save(self.sess, checkpoint_path)
def set_log_dir(self, model_path=None, new_folder=False):
'''
Sets the model log directory and epoch counter.
model_path: If None, or a format different from what this code uses
then set a new log directory and start epochs from 0. Otherwise,
extract the log directory and the epoch counter from the file
name.
'''
# Set date and epoch counter as if starting a new model
# print('>>> Set_log_dir() -- model dir is ', self.model_dir)
# print(' model_path : ', model_path)
# print(' config.LAST_EPOCH_RAN: ', self.config.LAST_EPOCH_RAN)
self.tb_dir = os.path.join(self.model_dir, 'tensorboard')
self.epoch = 0
regex_match = False
last_checkpoint_epoch = 0
now = datetime.datetime.now()
# If we have a model path with date and epochs use them
if model_path:
# Continue from we left off. Get epoch and date from the file name
# A sample model path might look like:
# /path/to/logs/coco20171029T2315/mask_rcnn_coco_0001.h5
model_path = model_path.replace('\\', "/")
# print(' set_log_dir: model_path (input) is : {} '.format(model_path))
regex = r".*/\w+(\d{4})(\d{2})(\d{2})T(\d{2})(\d{2})/fcn\w+(\d{4})\.h5"
regex_match = re.match(regex, model_path)
if regex_match:
now = datetime.datetime(int(regex_match.group(1)),
int(regex_match.group(2)),
int(regex_match.group(3)),
int(regex_match.group(4)),
int(regex_match.group(5)))
last_checkpoint_epoch = int(regex_match.group(6)) + 1
# print(' set_log_dir: self.epoch set to {} (Next epoch to run)'.format(self.epoch))
# print(' set_log_dir: tensorboard path: {}'.format(self.tb_dir))
if last_checkpoint_epoch > 0 and self.config.LAST_EPOCH_RAN > last_checkpoint_epoch:
self.epoch = self.config.LAST_EPOCH_RAN
else:
self.epoch = last_checkpoint_epoch
# Set directory for training logs
# if new_folder = True or appropriate checkpoint filename was not found, generate new folder
if new_folder or self.config.NEW_LOG_FOLDER:
now = datetime.datetime.now()
self.log_dir = os.path.join(
self.model_dir, "{}{:%Y%m%dT%H%M}".format(self.config.NAME.lower(),
now))
##--------------------------------------------------------------------------------
## Create checkpoint folder if it doesn't exists
##--------------------------------------------------------------------------------
from tensorflow.python.platform import gfile
print(' set_log_dir(): self.log_dir : {} '.format(self.log_dir),
file=sys.__stdout__)
if not gfile.IsDirectory(self.log_dir):
print(' Creating checkpoint folder : {}'.format(self.log_dir),
file=sys.__stdout__)
gfile.MakeDirs(self.log_dir)
else:
print(' Checkpoint folder already exists: {}'.format(
self.log_dir),
file=sys.__stdout__)
# Path to save after each epoch. Include placeholders that get filled by Keras.
self.checkpoint_path = os.path.join(
self.log_dir, "{}_*epoch*.h5".format(self.config.NAME.lower()))
self.checkpoint_path = self.checkpoint_path.replace(
"*epoch*", "{epoch:04d}")
log(' set_log_dir(): self.Checkpoint_path: {} '.format(
self.checkpoint_path))
log(' set_log_dir(): self.log_dir : {} '.format(self.log_dir))
log(' set_log_dir(): Last completed epoch (self.epoch): {} '.format(
self.epoch))
return
}
model_dir = getcwd()
filename = 'retrained_graph.pb'
model_filename = os.path.join(model_dir, filename)
model_info['pbfile_path'] = model_filename
base_dir = model_dir + '/tf_logs'
now = datetime.utcnow().strftime("%Y%m%d%H%M%S")
logdir = "{}/run-{}/".format(base_dir, now)
tflite_dir = logdir + 'tflite/'
tflite_path = tflite_dir + filename.split('.')[0] + '.tflite'
if not gfile.Exists(logdir):
gfile.MakeDirs(logdir)
if not gfile.Exists(tflite_dir):
gfile.MakeDirs(tflite_dir)
# load TF computational graph from a pb file
tf.reset_default_graph()
tf.logging.info('[convert_tflite_from_pb] Frozen graph is loading from %s' %
model_filename)
with gfile.FastGFile(model_filename, 'rb') as f:
graph_def = tf.GraphDef()
graph_def.ParseFromString(f.read())
# Import the graph from "graph_def" into the current default graph
_ = tf.import_graph_def(graph_def=graph_def, name='')
def main(argv=None):
cifar10.maybe_download_and_extract()
if gfile.Exists(FLAGS.train_dir):
gfile.DeleteRecursively(FLAGS.train_dir)
gfile.MakeDirs(FLAGS.train_dir)
train()
def _AddEvents(path):
if not gfile.IsDirectory(path):
gfile.MakeDirs(path)
fpath = os.path.join(path, 'hypothetical.tfevents.out')
with gfile.GFile(fpath, 'w'):
return fpath
def export(self,
export_dir_base,
global_step_tensor,
sess=None,
exports_to_keep=None):
"""Exports the model.
Args:
export_dir_base: A string path to the base export dir.
global_step_tensor: An Tensor or tensor name providing the
global step counter to append to the export directory path and set
in the manifest version.
sess: A Session to use to save the parameters.
exports_to_keep: a gc.Path filter function used to determine the set of
exports to keep. If set to None, all versions will be kept.
Returns:
The string path to the exported directory.
Raises:
RuntimeError: if init is not called.
RuntimeError: if the export would overwrite an existing directory.
"""
if not self._has_init:
raise RuntimeError("init must be called first")
# Export dir must not end with / or it will break exports to keep. Strip /.
if export_dir_base.endswith("/"):
export_dir_base = export_dir_base[:-1]
global_step = training_util.global_step(sess, global_step_tensor)
export_dir = os.path.join(
compat.as_bytes(export_dir_base),
compat.as_bytes(constants.VERSION_FORMAT_SPECIFIER % global_step))
# Prevent overwriting on existing exports which could lead to bad/corrupt
# storage and loading of models. This is an important check that must be
# done before any output files or directories are created.
if gfile.Exists(export_dir):
raise RuntimeError("Overwriting exports can cause corruption and are "
"not allowed. Duplicate export dir: %s" % export_dir)
# Output to a temporary directory which is atomically renamed to the final
# directory when complete.
tmp_export_dir = compat.as_text(export_dir) + "-tmp"
gfile.MakeDirs(tmp_export_dir)
self._saver.save(sess,
os.path.join(
compat.as_text(tmp_export_dir),
compat.as_text(constants.EXPORT_BASE_NAME)),
meta_graph_suffix=constants.EXPORT_SUFFIX_NAME)
# Run the asset callback.
if self._assets_callback and self._assets_to_copy:
assets_dir = os.path.join(
compat.as_bytes(tmp_export_dir),
compat.as_bytes(constants.ASSETS_DIRECTORY))
gfile.MakeDirs(assets_dir)
self._assets_callback(self._assets_to_copy, assets_dir)
# TODO(b/27794910): Delete *checkpoint* file before rename.
gfile.Rename(tmp_export_dir, export_dir)
if exports_to_keep:
# create a simple parser that pulls the export_version from the directory.
def parser(path):
match = re.match("^" + export_dir_base + "/(\\d{8})$", path.path)
if not match:
return None
return path._replace(export_version=int(match.group(1)))
paths_to_delete = gc.negation(exports_to_keep)
for p in paths_to_delete(gc.get_paths(export_dir_base, parser=parser)):
gfile.DeleteRecursively(p.path)
return export_dir
def testNonSharded(self):
save_dir = os.path.join(self.get_temp_dir(), "max_to_keep_non_sharded")
try:
gfile.DeleteRecursively(save_dir)
except OSError:
pass # Ignore
gfile.MakeDirs(save_dir)
with self.test_session() as sess:
v = tf.Variable(10.0, name="v")
save = tf.train.Saver({"v": v}, max_to_keep=2)
tf.initialize_all_variables().run()
self.assertEqual([], save.last_checkpoints)
s1 = save.save(sess, os.path.join(save_dir, "s1"))
self.assertEqual([s1], save.last_checkpoints)
self.assertTrue(gfile.Exists(s1))
s2 = save.save(sess, os.path.join(save_dir, "s2"))
self.assertEqual([s1, s2], save.last_checkpoints)
self.assertTrue(gfile.Exists(s1))
self.assertTrue(gfile.Exists(s2))
s3 = save.save(sess, os.path.join(save_dir, "s3"))
self.assertEqual([s2, s3], save.last_checkpoints)
self.assertFalse(gfile.Exists(s1))
self.assertTrue(gfile.Exists(s2))
self.assertTrue(gfile.Exists(s3))
# Create a second helper, identical to the first.
save2 = tf.train.Saver(saver_def=save.as_saver_def())
save2.set_last_checkpoints(save.last_checkpoints)
# Create a third helper, with the same configuration but no knowledge of
# previous checkpoints.
save3 = tf.train.Saver(saver_def=save.as_saver_def())
# Exercise the first helper.
# Adding s2 again (old s2 is removed first, then new s2 appended)
s2 = save.save(sess, os.path.join(save_dir, "s2"))
self.assertEqual([s3, s2], save.last_checkpoints)
self.assertFalse(gfile.Exists(s1))
self.assertTrue(gfile.Exists(s3))
self.assertTrue(gfile.Exists(s2))
# Adding s1 (s3 should now be deleted as oldest in list)
s1 = save.save(sess, os.path.join(save_dir, "s1"))
self.assertEqual([s2, s1], save.last_checkpoints)
self.assertFalse(gfile.Exists(s3))
self.assertTrue(gfile.Exists(s2))
self.assertTrue(gfile.Exists(s1))
# Exercise the second helper.
# Adding s2 again (old s2 is removed first, then new s2 appended)
s2 = save2.save(sess, os.path.join(save_dir, "s2"))
self.assertEqual([s3, s2], save2.last_checkpoints)
# Created by the first helper.
self.assertTrue(gfile.Exists(s1))
# Deleted by the first helper.
self.assertFalse(gfile.Exists(s3))
self.assertTrue(gfile.Exists(s2))
# Adding s1 (s3 should now be deleted as oldest in list)
s1 = save2.save(sess, os.path.join(save_dir, "s1"))
self.assertEqual([s2, s1], save2.last_checkpoints)
self.assertFalse(gfile.Exists(s3))
self.assertTrue(gfile.Exists(s2))
self.assertTrue(gfile.Exists(s1))
# Exercise the third helper.
# Adding s2 again (but helper is unaware of previous s2)
s2 = save3.save(sess, os.path.join(save_dir, "s2"))
self.assertEqual([s2], save3.last_checkpoints)
# Created by the first helper.
self.assertTrue(gfile.Exists(s1))
# Deleted by the first helper.
self.assertFalse(gfile.Exists(s3))
self.assertTrue(gfile.Exists(s2))
# Adding s1 (s3 should not be deleted because helper is unaware of it)
s1 = save3.save(sess, os.path.join(save_dir, "s1"))
self.assertEqual([s2, s1], save3.last_checkpoints)
self.assertFalse(gfile.Exists(s3))
self.assertTrue(gfile.Exists(s2))
self.assertTrue(gfile.Exists(s1))
def testReportingBenchmark(self):
tempdir = test.get_temp_dir()
try:
gfile.MakeDirs(tempdir)
except OSError as e:
# It's OK if the directory already exists.
if " exists:" not in str(e):
raise e
prefix = os.path.join(
tempdir, "reporting_bench_%016x_" % random.getrandbits(64))
expected_output_file = "%s%s" % (
prefix, "TestReportingBenchmark.benchmarkReport1")
expected_output_file_2 = "%s%s" % (
prefix, "TestReportingBenchmark.custom_benchmark_name")
expected_output_file_3 = "%s%s" % (
prefix, "TestReportingBenchmark.op_benchmark")
try:
self.assertFalse(gfile.Exists(expected_output_file))
# Run benchmark but without env, shouldn't write anything
if benchmark.TEST_REPORTER_TEST_ENV in os.environ:
del os.environ[benchmark.TEST_REPORTER_TEST_ENV]
reporting = TestReportingBenchmark()
reporting.benchmarkReport1(
) # This should run without writing anything
self.assertFalse(gfile.Exists(expected_output_file))
# Runbenchmark with env, should write
os.environ[benchmark.TEST_REPORTER_TEST_ENV] = prefix
reporting = TestReportingBenchmark()
reporting.benchmarkReport1() # This should write
reporting.benchmarkReport2() # This should write
benchmark_values3 = reporting.benchmark_times_an_op(
) # This should write
# Check the files were written
self.assertTrue(gfile.Exists(expected_output_file))
self.assertTrue(gfile.Exists(expected_output_file_2))
self.assertTrue(gfile.Exists(expected_output_file_3))
# Check the contents are correct
expected_1 = test_log_pb2.BenchmarkEntry()
expected_1.name = "TestReportingBenchmark.benchmarkReport1"
expected_1.iters = 1
expected_2 = test_log_pb2.BenchmarkEntry()
expected_2.name = "TestReportingBenchmark.custom_benchmark_name"
expected_2.iters = 2
expected_2.extras["number_key"].double_value = 3
expected_2.extras["other_key"].string_value = "string"
expected_3 = test_log_pb2.BenchmarkEntry()
expected_3.name = "TestReportingBenchmark.op_benchmark"
expected_3.iters = 1000
def read_benchmark_entry(f):
s = gfile.GFile(f, "rb").read()
entries = test_log_pb2.BenchmarkEntries.FromString(s)
self.assertEqual(1, len(entries.entry))
return entries.entry[0]
read_benchmark_1 = read_benchmark_entry(expected_output_file)
self.assertProtoEquals(expected_1, read_benchmark_1)
read_benchmark_2 = read_benchmark_entry(expected_output_file_2)
self.assertProtoEquals(expected_2, read_benchmark_2)
read_benchmark_3 = read_benchmark_entry(expected_output_file_3)
self.assertEqual(expected_3.name, read_benchmark_3.name)
self.assertEqual(expected_3.iters, read_benchmark_3.iters)
self.assertGreater(read_benchmark_3.wall_time, 0)
# Trace is not stored in benchmark entry. Instead we get it from
# return value of `run_op_benchmark` call.
full_trace = benchmark_values3["extras"][
"full_trace_chrome_format"]
json_trace = json.loads(full_trace)
self.assertTrue(isinstance(json_trace, dict))
self.assertTrue("traceEvents" in json_trace.keys())
allocator_keys = [
k for k in read_benchmark_3.extras.keys()
if k.startswith("allocator_maximum_num_bytes_")
]
self.assertGreater(len(allocator_keys), 0)
for k in allocator_keys:
self.assertGreater(read_benchmark_3.extras[k].double_value, 0)
finally:
gfile.DeleteRecursively(tempdir)
def train():
if FLAGS.data_type == 1 or FLAGS.data_type == 2:
# datadir, org_height, org_width, org_depth=3, batch_size=32, threads_num=4
datas = dataset.Dataset(FLAGS.data_dir, FLAGS.image_height_org, FLAGS.image_width_org,
FLAGS.image_depth_org, FLAGS.batch_size, FLAGS.num_threads, type=FLAGS.data_type,
crop=FLAGS.is_crop)
else:
print("invalid data type.")
return
images = datas.get_inputs(FLAGS.image_height, FLAGS.image_width)
#z = tf.placeholder(tf.float32, [None, FLAGS.z_dim], name='z')
dcgan = ANOGAN(FLAGS.model_name, FLAGS.checkpoint_dir)
generates, R1_inter, D2_logits, D2_inter = dcgan.step(images)
d_loss_fake, r_loss, fm_loss, d_loss_fake_sum, r_loss_sum, fm_loss_sum = dcgan.cost(images, generates, R1_inter, D2_logits, D2_inter)
#r_loss, r_loss_sum = dcgan.cost(, generates, R1_inter, D2_logits, D2_inter)
# trainable variables
t_vars = tf.trainable_variables()
# g_vars = [var for var in t_vars if 'g_' in var.name]
r_vars = [var for var in t_vars if ('e_' in var.name) or ('d_fc1' in var.name)]
# train operations
r_optim = R_train_op(r_loss + fm_loss, r_vars, FLAGS.learning_rate, FLAGS.beta1)
# for saver variables
all_vars = tf.global_variables()
dg_vars = [var for var in all_vars if ('d_' in var.name) or ('g_' in var.name)]
# saver of d and g
saver = tf.train.Saver(dg_vars)
# saver of e_
saver_e = tf.train.Saver(r_vars)
# saver of all variables
saver_all = tf.train.Saver()
# initialization
init_op = tf.global_variables_initializer()
gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=FLAGS.gpu_memory_fraction)
sess = tf.Session(config=tf.ConfigProto(
allow_soft_placement=True,
gpu_options=gpu_options))
writer = tf.summary.FileWriter("./logs", sess.graph_def)
# run
sess.run(init_op)
# load parameters
print("G and D Model.")
model_dir = os.path.join(FLAGS.model_name, FLAGS.checkpoint_dir)
ckpt = tf.train.get_checkpoint_state(model_dir)
if ckpt and ckpt.model_checkpoint_path:
print("Model: %s" % (ckpt.model_checkpoint_path))
saver.restore(sess, ckpt.model_checkpoint_path)
else:
print("G and D Model: No checkpoint file found")
exit()
print("G and D Model: restored.")
# load e parameters
print("E Model.")
model_e_dir = os.path.join(FLAGS.reverser_model_name, FLAGS.checkpoint_dir)
ckpt_e = tf.train.get_checkpoint_state(model_e_dir)
if ckpt_e and ckpt_e.model_checkpoint_path:
print("Model: %s" % (ckpt_e.model_checkpoint_path))
saver_e.restore(sess, ckpt_e.model_checkpoint_path)
print("E Model: restored.")
else:
print("E model: No checkpoint file found")
# summary
r_sum = tf.summary.merge([r_loss_sum, fm_loss_sum])
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(sess=sess, coord=coord)
counter = 1
start_time = time.time()
for epoch in xrange(FLAGS.epochs):
for idx in xrange(0, int(datas.batch_idxs)):
batch_z = np.random.uniform(-1, 1, [FLAGS.batch_size, FLAGS.z_dim]).astype(np.float32)
# R optimization
_, summary_str = sess.run([r_optim, r_sum])
writer.add_summary(summary_str, counter)
errR = sess.run(r_loss)
print("epochs: %02d %04d/%04d time: %4.4f, r_loss: %.8f" % (
epoch, idx, FLAGS.steps, time.time() - start_time, errR))
# if np.mod(counter, 100) == 1:
# print("generate samples.")
# generated_image_eval = sess.run(generate_images, {z: batch_z})
# out_dir = os.path.join(FLAGS.model_name, FLAGS.sample_dir)
# if not gfile.Exists(out_dir):
# gfile.MakeDirs(out_dir)
# filename = os.path.join(out_dir, 'out_%05d.png' % counter)
# with open(filename, 'wb') as f:
# f.write(generated_image_eval)
counter += 1
if np.mod(epoch, 2) == 0:
out_dir = os.path.join(FLAGS.reverser_model_name, FLAGS.checkpoint_dir)
if not gfile.Exists(out_dir):
gfile.MakeDirs(out_dir)
out_path = os.path.join(out_dir, 'model.ckpt')
saver_all.save(sess, out_path, global_step=epoch)
coord.request_stop()
coord.join(threads)
sess.close()
def fit(self, train_data, train_label):
np.random.seed(1337)
tf.set_random_seed(1337)
n_samples, self.channels, self.time, self.img_size_x, self.img_size_y = train_data.shape[
0:5]
'''
self.lb = LabelBinarizer()
train_label_mat = np.array(self.lb.fit_transform(train_label))
self.n_classes = train_label_mat.shape[1]
Hidden_layer=self.build_model()
self.y_conv =tf.nn.softmax(tf.matmul(Hidden_layer,self.action_layer_w)+self.action_layer_b)
'''
'''
with tf.Session() as sess:
cross_entropy = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=self.y_conv, labels=self.yy))
correct_prediction = tf.equal(tf.argmax(self.y_conv,1), tf.argmax(self.yy,1))
trainer=tf.train.AdamOptimizer(self.learning_rate)
Grads_and_Vars = trainer.compute_gradients(cross_entropy)
train_step=trainer.apply_gradients(grads_and_vars=Grads_and_Vars)
correct_prediction = tf.equal(tf.argmax(self.y_conv,1), tf.argmax(self.yy,1))
'''
train_label_mat = []
for i in train_label:
train_label_mat.append([i])
train_label_mat = np.array(train_label_mat)
print(tf.shape(train_label_mat))
self.n_classes = 1
Hidden_layer = self.build_model()
#with tf.name_scope("kkkkkkkkkkkkkkk"):
self.y_conv = tf.divide(
1.0,
tf.add(
1.0,
tf.exp(-(tf.matmul(Hidden_layer, self.action_layer_w) +
self.action_layer_b))))
with tf.Session() as sess:
cross_entropy = tf.reduce_mean(
tf.multiply(self.yy, tf.log(self.y_conv + 0.0001)) +
tf.multiply((1.0 -
self.yy), tf.log((0.0001 + 1.0 - self.y_conv))))
var_train = tf.trainable_variables()
lossL2 = tf.add_n(
[tf.nn.l2_loss(v)
for v in var_train if 'bias' not in v.name]) * 0.0001
cross_entropy = cross_entropy + lossL2
#self.lr=tf.placeholder("float", None)
trainer = tf.train.AdamOptimizer(self.learning_rate)
gvs = trainer.compute_gradients(cross_entropy)
def ClipIfNotNone(grad):
if grad is None:
return grad
return tf.clip_by_value(grad, -1, 1)
clipped_gradients = [(ClipIfNotNone(grad), var)
for grad, var in gvs]
train_step = trainer.apply_gradients(clipped_gradients)
correct_prediction = tf.abs(self.y_conv - self.yy) < 0.5
accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))
sess.run(tf.global_variables_initializer())
train_data = train_data.tolist()
train_label = train_label.tolist()
train_label_mat = train_label_mat.tolist()
if len(train_data) % self.batch_size != 0:
for i in range(len(train_data) % self.batch_size):
index = random.randint(0, len(train_data) - 1)
train_data.append(train_data[index])
train_label.append(train_label[index])
train_label_mat.append(train_label_mat[index])
assert len(train_data) % self.batch_size == 0
assert len(train_label_mat) == len(train_data)
Index = [i for i in range(len(train_data))]
for i in range(self.epoch):
Total_acc = 0.0
random.shuffle(Index)
for j in range(int(len(train_data) / self.batch_size)):
eve_train_data = []
eve_train_label_mat = []
for mm in Index[j * self.batch_size:(1 + j) *
self.batch_size]:
eve_train_data.append(train_data[mm])
eve_train_label_mat.append(train_label_mat[mm])
train_step.run(feed_dict={
self.img: eve_train_data,
self.yy: eve_train_label_mat
})
train_accuracy = (accuracy.eval(
feed_dict={
self.img: eve_train_data,
self.yy: eve_train_label_mat
}))
Output = self.y_conv.eval(feed_dict={
self.img: eve_train_data,
self.yy: eve_train_label_mat
})
Total_acc = Total_acc + train_accuracy
print("Epoch %d,training accuracy %g" %
(i + 1, Total_acc /
(float(len(train_data) / self.batch_size))))
saver = tf.train.Saver()
folder = os.path.dirname(self.checkpoint_path)
if not gfile.Exists(folder):
gfile.MakeDirs(folder)
saver.save(sess, self.checkpoint_path)
def main(_):
args = config_distill.get_args_for_config(FLAGS.config_name)
args.logdir = FLAGS.logdir
args.solver.num_workers = FLAGS.num_workers
args.solver.task = FLAGS.task
args.solver.ps_tasks = FLAGS.ps_tasks
args.solver.master = FLAGS.master
args.buildinger.env_class = nav_env.MeshMapper
fu.makedirs(args.logdir)
args.buildinger.logdir = args.logdir
R = nav_env.get_multiplexor_class(args.buildinger, args.solver.task)
if False:
pr = cProfile.Profile()
pr.enable()
rng = np.random.RandomState(0)
for i in range(1):
b, instances_perturbs = R.sample_building(rng)
inputs = b.worker(*(instances_perturbs))
for j in range(inputs['imgs'].shape[0]):
p = os.path.join('tmp', '{:d}.png'.format(j))
img = inputs['imgs'][j, 0, :, :, :3] * 1
img = (img).astype(np.uint8)
fu.write_image(p, img)
print(inputs['imgs'].shape)
inputs = R.pre(inputs)
pr.disable()
pr.print_stats(2)
if args.control.train:
if not gfile.Exists(args.logdir):
gfile.MakeDirs(args.logdir)
m = utils.Foo()
m.tf_graph = tf.Graph()
config = tf.ConfigProto()
config.device_count['GPU'] = 1
config.gpu_options.allow_growth = True
config.gpu_options.per_process_gpu_memory_fraction = 0.8
with m.tf_graph.as_default():
with tf.device(tf.train.replica_device_setter(
args.solver.ps_tasks)):
m = distill.setup_to_run(m,
args,
is_training=True,
batch_norm_is_training=True)
train_step_kwargs = distill.setup_train_step_kwargs_mesh(
m,
R,
os.path.join(args.logdir, 'train'),
rng_seed=args.solver.task,
is_chief=args.solver.task == 0,
iters=1,
train_display_interval=args.summary.display_interval)
final_loss = slim.learning.train(
train_op=m.train_op,
logdir=args.logdir,
master=args.solver.master,
is_chief=args.solver.task == 0,
number_of_steps=args.solver.max_steps,
train_step_fn=tf_utils.train_step_custom,
train_step_kwargs=train_step_kwargs,
global_step=m.global_step_op,
init_op=m.init_op,
init_fn=m.init_fn,
sync_optimizer=m.sync_optimizer,
saver=m.saver_op,
summary_op=None,
session_config=config)
if args.control.test:
m = utils.Foo()
m.tf_graph = tf.Graph()
checkpoint_dir = os.path.join(format(args.logdir))
with m.tf_graph.as_default():
m = distill.setup_to_run(m,
args,
is_training=False,
batch_norm_is_training=args.control.
force_batchnorm_is_training_at_test)
train_step_kwargs = distill.setup_train_step_kwargs_mesh(
m,
R,
os.path.join(args.logdir, args.control.test_name),
rng_seed=args.solver.task + 1,
is_chief=args.solver.task == 0,
iters=args.summary.test_iters,
train_display_interval=None)
sv = slim.learning.supervisor.Supervisor(
graph=ops.get_default_graph(),
logdir=None,
init_op=m.init_op,
summary_op=None,
summary_writer=None,
global_step=None,
saver=m.saver_op)
last_checkpoint = None
while True:
last_checkpoint = slim.evaluation.wait_for_new_checkpoint(
checkpoint_dir, last_checkpoint)
checkpoint_iter = int(
os.path.basename(last_checkpoint).split('-')[1])
start = time.time()
tf.logging.info(
'Starting evaluation at %s using checkpoint %s.',
time.strftime('%Y-%m-%d-%H:%M:%S', time.localtime()),
last_checkpoint)
config = tf.ConfigProto()
config.device_count['GPU'] = 1
config.gpu_options.allow_growth = True
config.gpu_options.per_process_gpu_memory_fraction = 0.8
with sv.managed_session(args.solver.master,
config=config,
start_standard_services=False) as sess:
sess.run(m.init_op)
sv.saver.restore(sess, last_checkpoint)
sv.start_queue_runners(sess)
vals, _ = tf_utils.train_step_custom(sess,
None,
m.global_step_op,
train_step_kwargs,
mode='val')
if checkpoint_iter >= args.solver.max_steps:
break
self.assertTrue(gfile.Exists(s2))
# Adding s1 (s3 should not be deleted because helper is unaware of it)
s1 = save3.save(sess, os.path.join(save_dir, "s1"))
self.assertEqual([s2, s1], save3.last_checkpoints)
self.assertFalse(gfile.Exists(s3))
self.assertTrue(gfile.Exists(s2))
self.assertTrue(gfile.Exists(s1))
def testSharded(self):
save_dir = os.path.join(self.get_temp_dir(), "max_to_keep_sharded")
try:
gfile.DeleteRecursively(save_dir)
except gfile.GOSError, _:
pass # Ignore
gfile.MakeDirs(save_dir)
with tf.Session(
target="",
config=tf.ConfigProto(device_count={"CPU": 2})) as sess:
with sess.graph.device("/cpu:0"):
v0 = tf.Variable(111, name="v0")
with sess.graph.device("/cpu:1"):
v1 = tf.Variable(222, name="v1")
save = tf.train.Saver({"v0": v0, "v1": v1}, sharded=True, max_to_keep=2)
tf.initialize_all_variables().run()
self.assertEqual([], save.last_checkpoints)
s1 = save.save(sess, os.path.join(save_dir, "s1"))
self.assertEqual([s1], save.last_checkpoints)
self.assertEquals(2, len(gfile.Glob(s1)))
