def test_dropblock(self):
with self.test_session() as sess:
data = tf.ones(shape=[1,32,32,1],dtype=tf.float32)
data =wnnl.dropblock(data,keep_prob=0.8,block_size=4,is_training=True)
data = tf.reshape(data,shape=[32,32])
data = tf.cast(data,tf.int32)
wmlu.show_list(sess.run(data).tolist())
def get_data(data_dir,
batch_size=4,
num_samples=1,
num_classes=3,
id_to_label={}):
dataset = get_database(dataset_dir=data_dir,
num_classes=num_classes,
num_samples=num_samples)
files = wmlu.recurse_get_filepath_in_dir(data_dir, suffix=".tfrecord")
print("tfrecords:")
wmlu.show_list(files)
with tf.name_scope('data_provider'):
provider = slim.dataset_data_provider.DatasetDataProvider(
dataset,
num_readers=2,
common_queue_capacity=20 * batch_size,
common_queue_min=10 * batch_size,
seed=int(time.time()),
shuffle=True)
[image, glabels,
bboxes] = provider.get(["image", "object/label", "object/bbox"])
label = 1
if len(id_to_label) == 0:
for i in range(1, 80):
id_to_label[i] = label
label += 1
table = tf.contrib.lookup.HashTable(
tf.contrib.lookup.KeyValueTensorInitializer(
np.array(list(id_to_label.keys()), dtype=np.int64),
np.array(list(id_to_label.values()), dtype=np.int64)), -1)
glabels = table.lookup(glabels)
wmlt.add_to_hash_table_collection(table.init)
return image, glabels, bboxes
def multi_thread_to_tfrecords_by_files(self,files, output_dir,shuffling=False,fidx=0):
wmlu.create_empty_dir(output_dir,remove_if_exists=True,yes_to_all=True)
if shuffling:
random.seed(time.time())
random.shuffle(files)
wmlu.show_list(files[:100])
if len(files)>100:
print("...")
print(f"Total {len(files)} files.")
sys.stdout.flush()
files = wmlu.list_to_2dlist(files,SAMPLES_PER_FILES)
files_data = list(enumerate(files))
if fidx != 0:
_files_data = []
for fid,file_d in files_data:
_files_data.append([fid+fidx,file_d])
files_data = _files_data
sys.stdout.flush()
pool = Pool(13)
pool.map(functools.partial(self.make_tfrecord,output_dir=output_dir),files_data)
#list(map(functools.partial(self.make_tfrecord,output_dir=output_dir),files_data))
pool.close()
pool.join()
print('\nFinished converting the dataset total %d examples.!'%(len(files)))
def build_inference_net(self):
try:
if not os.path.exists(self.log_dir):
wmlu.create_empty_dir(self.log_dir)
if not os.path.exists(self.ckpt_dir):
wmlu.create_empty_dir(self.ckpt_dir)
except:
pass
'''
When inference, self.data is just a tensor
'''
data = {IMAGE: self.data}
DataLoader.detection_image_summary(data, name="data_source")
self.input_data = data
'''if self.cfg.GLOBAL.DEBUG:
data[IMAGE] = tf.Print(data[IMAGE],[tf.shape(data[IMAGE]),data[ORG_HEIGHT],data[ORG_WIDTH],data[HEIGHT],data[WIDTH]],summarize=100,
name="XXXXX")'''
self.res_data, loss_dict = self.model.forward(data)
config = tf.ConfigProto(allow_soft_placement=True)
config.gpu_options.allow_growth = True
self.top_variable_name_scope = "Model"
print("batch_norm_ops.")
wmlu.show_list(
[x.name for x in tf.get_collection(tf.GraphKeys.UPDATE_OPS)])
def all_reduce_norm(module):
"""
All reduce norm statistics in different devices.
"""
states = get_async_norm_states(module)
print("Reduce keys:")
wmlu.show_list(list(states.keys()))
states = all_reduce(states, op="mean")
module.load_state_dict(states, strict=False)
def test_orthogonal_regularizerv1(self):
with self.test_session() as sess:
print("test_orthogonal_regularizerv1")
fn = wnnl.orthogonal_regularizer(1)
weight = np.array(list(range(3)),np.float32)
wmlu.show_list(weight)
v = fn(weight)
v = sess.run(v)
self.assertAllClose(v,9,atol=1e-4)
def multithread_create_tf_record(data_dir, output_dir, img_suffix="jpg", name="train", shuffling=True, fidx=0,
label_text_to_id=None):
files = get_files(data_dir, img_suffix=img_suffix)
if os.path.exists(output_dir) and (data_dir != output_dir):
shutil.rmtree(output_dir)
print("删除文件夹%s" % (output_dir))
wmlu.show_list(files)
print(f"Find {len(files)} files.")
return multithread_create_tf_record_by_files(files, output_dir,
name, shuffling, fidx,
label_text_to_id)
def get_database(dataset_dir, num_parallel=1, file_pattern='*.record'):
file_pattern = os.path.join(dataset_dir, file_pattern)
files = glob.glob(file_pattern)
if len(files) == 0:
logging.error(f'No files found in {file_pattern}')
else:
print(f"Total {len(files)} files.")
wmlu.show_list(files)
dataset = tf.data.TFRecordDataset(files, num_parallel_reads=num_parallel)
dataset = dataset.map(__parse_func, num_parallel_calls=num_parallel)
return dataset
def forward(self,inputs):
if self.tmp_path.startswith("/tmp"):
wmlu.create_empty_dir(self.tmp_path,remove_if_exists=True,yes_to_all=True)
print("inputs shape:",inputs.shape)
raw_path = os.path.join(self.tmp_path,"input.raw")
input_list = os.path.join(self.tmp_path,"file_list.txt")
output_dir = os.path.join(self.tmp_path,"output")
if not os.path.isdir(output_dir):
os.makedirs(output_dir)
with open(input_list, "w") as f:
if self.output_layers is not None:
v = f"#{self.output_layers[0]}"
for x in self.output_layers[1:]:
v += f" {x}"
v += "\n"
f.write(v)
f.write(raw_path)
self.to_snpe_raw(inputs,raw_path)
cmd = "{} --container {} --input_list {} --output_dir {}".format(self.bin, self.model_path,
input_list,
output_dir)
print(f"CMD:{cmd}")
print(f"All output files.")
os.system(cmd)
all_files = wmlu.recurse_get_filepath_in_dir(output_dir,suffix=".raw")
wmlu.show_list(all_files)
print("-------------------------------")
res_data = []
output_dir = "/home/wj/0day/output" #for DEBUG
if self.output_names is not None:
for name,shape in zip(self.output_names,self.output_shapes):
path = os.path.join(output_dir,"Result_0",name+self.output_suffix)
if not os.path.exists(path):
print(f"{path} not exits")
res_data.append(None)
else:
print(f"Read from {path}")
td = np.fromfile(path,dtype=np.float32)
if shape is not None:
td = np.reshape(td,shape)
res_data.append(td)
else:
path = all_files[0]
print(f"Use ")
td = np.fromfile(path, dtype=np.float32)
res_data.append(td)
return res_data
def get_database(dataset_dir, num_parallel=1, file_pattern='*_train.record'):
file_pattern = os.path.join(dataset_dir, file_pattern)
files = glob.glob(file_pattern)
if len(files) == 0:
logging.error(f'No files found in {file_pattern}')
a = input(f"No files found in {file_pattern}, continue?(y/n)")
if a != 'y':
exit(-1)
else:
wmlu.show_list(files)
dataset = tf.data.TFRecordDataset(files, num_parallel_reads=num_parallel)
dataset = dataset.map(__parse_func, num_parallel_calls=num_parallel)
return dataset
def get_regularization_losses(scopes=None,re_pattern=None,reduction="sum"):
with tf.name_scope("regularization_losses"):
col = get_variables_of_collection(tf.GraphKeys.REGULARIZATION_LOSSES,scopes=scopes,re_pattern=re_pattern)
if len(col)>0:
print("wr_loss")
wmlu.show_list(col)
print("end wr_loss")
if reduction == "mean":
return tf.reduce_mean(col)
elif reduction == "sum":
return tf.reduce_sum(col)
else:
return col
else:
return None
def test_channel_upsampling(self):
with self.test_session() as sess:
print("Test channel upsample")
data_in = list(range(16))
data_in = np.array(data_in)
data_in = np.reshape(data_in,[1,2,2,4])
expected_data = np.array([[0,1,4,5],[2,3,6,7],[8,9,12,13],[10,11,14,15]])
wmlu.show_list(data_in)
data_in = tf.constant(data_in,dtype=tf.float32)
data_in = wmlt.channel_upsample(data_in,scale=2)
data_in = tf.squeeze(data_in,axis=-1)
data_in = tf.squeeze(data_in,axis=0)
data_out = sess.run(data_in)
wmlu.show_list(data_out)
self.assertAllClose(expected_data,data_out,atol=1e-4)
def sample_in_dir(dir_path, nr, split_nr=None):
'''
sample data in dir_path's sub dirs
sample nr images in each sub dir, if split_nr is not None, sampled nr images will be split
to split_nr part and saved in different dir
'''
res = {}
dirs = wmlu.get_subdir_in_dir(dir_path, absolute_path=True)
print(f"Find dirs in {dir_path}")
wmlu.show_list(dirs)
for dir in dirs:
data = sample_in_one_dir(dir, nr)
if split_nr is None:
append_to_dict(res, 0, data)
else:
data = wmlu.list_to_2dlistv2(data, split_nr)
for i, d in enumerate(data):
append_to_dict(res, i, d)
return res
def __call__(self, dir_path):
dir_path = os.path.abspath(dir_path)
while True:
files = wmlu.recurse_get_filepath_in_dir(dir_path,
suffix=".tar.gz")
wmlu.show_list(files)
process_nr = 0
for file in files:
if file in self.history:
continue
ckpt_file = self.extract_data(file)
if ckpt_file is None:
continue
print("process file {}.".format(file))
self.history.append(file)
if self.evaler is not None:
result, info = self.evaler(ckpt_file)
else:
result, info = self.eval(ckpt_file)
if result < 0.01:
print("Unnormal result {}, ignored.".format(result))
continue
if result < self.best_result:
print(
"{} not the best result, best result is {}/{}, achieved at {}, skip backup."
.format(file, self.best_result, self.best_file,
self.best_result_time))
continue
print("RESULT:", self.best_result, result)
print("New best result {}, {}.".format(file, info))
self.best_file = file
self.best_result = result
self.best_result_time = time.strftime("%m-%d %H:%M:%S",
time.localtime())
self.best_result_t = time.time()
if process_nr == 0:
print("sleep for 30 seconds.")
sys.stdout.flush()
time.sleep(30)
continue
new_mask = odm.dense_mask_to_sparse_mask(binary_mask,category_ids,default_label=255)
base_name = wmlu.base_name(full_path)+".png"
save_path = os.path.join(save_dir,base_name)
new_mask = new_mask.astype(np.uint8)
if os.path.exists(save_path):
print(f"WARNING: File {save_path} exists.")
cv2.imwrite(save_path,new_mask)
sys.stdout.write(f"\r{i}")
if __name__ == "__main__":
data_dir ="/home/wj/ai/mldata/mapillary_vistas/"
save_dir = os.path.join(data_dir,'boe_labels_validation')
name_to_id_dict = update_name_to_id(name_to_id_dict,data_dir)
idxs = list(range(0,18049,50))
r_idxs = []
for i in range(len(idxs)-1):
r_idxs.append([idxs[i],idxs[i+1]])
wmlu.show_list(r_idxs)
pool = Pool(10)
def fun(d):
trans_data(data_dir,save_dir,d[0],d[1])
res = list(pool.map(fun,r_idxs))
pool.close()
pool.join()
print(res)
#list(map(fun,r_idxs))
ddir = args.out_dir
tdir = wmlu.base_name(sd, process_suffix=False)
move = False
if args.reverse:
if tdir not in base_names:
move = True
else:
if tdir in base_names:
move = True
if move:
dirs_need_to_move.append(sd)
move_data.append((rdir, ddir))
else:
dirs_dont_move.append(sd)
print(f"dirs don't need move:")
wmlu.show_list(dirs_dont_move)
print(f"Total don't move dir {len(dirs_dont_move)}")
print(f"dirs need move:")
wmlu.show_list(dirs_need_to_move)
print(f"Total dirs need to move {len(dirs_need_to_move)}")
ans = input("Move dirs?[y/n]")
if ans == 'y':
wmlu.create_empty_dir(args.out_dir, remove_if_exists=False)
for rdir, ddir in move_data:
cmd = f"mv \"{rdir}\" \"{ddir}\""
print(cmd)
os.system(cmd)
def restore_variables(sess,path,exclude=None,only_scope=None,silent=False,restore_evckp=True,value_key=None,exclude_var=None,extend_vars=None,global_scopes=None,verbose=False):
#if restore_evckp and os.path.isdir(path):
# evt.WEvalModel.restore_ckp(FLAGS.check_point_dir)
bn_ops = tf.get_collection(key=tf.GraphKeys.UPDATE_OPS)
bn_ops = [x.name for x in bn_ops]
print("batch_norm_ops.")
wmlu.show_list(bn_ops)
if exclude is None and exclude_var is not None:
exclude = exclude_var
file_path = wmlt.get_ckpt_file_path(path)
if file_path is None:
return False
print(f"resotre from {file_path}.")
for v in tf.global_variables():
if "moving_mean" in v.name or "moving_variance" in v.name:
tf.add_to_collection(tf.GraphKeys.MOVING_AVERAGE_VARIABLES,v)
variables = []
variables_restored_in_ckpt = []
variables0 = restore_variables_by_key(sess,file_path,exclude,only_scope,key=tf.GraphKeys.TRAINABLE_VARIABLES,name="train",silent=silent,value_key=value_key)
if len(variables0)>1:
if isinstance(variables0[1],list):
for v in variables0[1]:
variables.append(v.name)
variables_restored_in_ckpt.append(v.name)
else:
for k,v in variables0[1].items():
variables.append(v.name)
variables_restored_in_ckpt.append(k)
variables1 = restore_variables_by_key(sess,file_path,exclude,only_scope,key=tf.GraphKeys.MOVING_AVERAGE_VARIABLES,name='moving',silent=silent,value_key=value_key)
if len(variables1)>1:
if isinstance(variables1[1],list):
for v in variables1[1]:
variables.append(v.name)
variables_restored_in_ckpt.append(v.name)
else:
for k,v in variables1[1].items():
variables.append(v.name)
variables_restored_in_ckpt.append(k)
if global_scopes is not None:
variables2 = restore_variables_by_key(sess,file_path,None,global_scopes,key=tf.GraphKeys.GLOBAL_VARIABLES,name='global_variables',silent=silent,value_key=value_key)
if len(variables2)>1:
if isinstance(variables2[1],list):
for v in variables2[1]:
variables.append(v.name)
variables_restored_in_ckpt.append(v.name)
else:
for k,v in variables2[1].items():
variables.append(v.name)
variables_restored_in_ckpt.append(k)
if extend_vars is not None:
variables2 = restore_variables_by_var_list(sess,file_path,extend_vars)
if len(variables2)>1:
if isinstance(variables2[1],list):
for v in variables2[1]:
variables.append(v.name)
variables_restored_in_ckpt.append(v.name)
else:
for k,v in variables2[1].items():
variables.append(v.name)
variables_restored_in_ckpt.append(k)
for i,v in enumerate(variables):
index = v.find(':')
if index>0:
variables[i] = variables[i][:index]
for i,v in enumerate(variables_restored_in_ckpt):
index = v.find(':')
if index>0:
variables_restored_in_ckpt[i] = variables_restored_in_ckpt[i][:index]
unrestored_variables = wmlt.get_variables_unrestored(variables_restored_in_ckpt,file_path,
exclude_var="Adam")
unrestored_variables0 = wmlt.get_variables_unrestoredv1(variables,exclude_var="Adam")
if not verbose:
def v_filter(x:str):
#return (not x.endswith("ExponentialMovingAverage")) and (not x.endswith("/u"))
return (not x.endswith("ExponentialMovingAverage")) and (not x.endswith("Momentum"))
unrestored_variables = filter(v_filter,unrestored_variables)
unrestored_variables0 = filter(v_filter,unrestored_variables0)
show_values(unrestored_variables, "Unrestored variables in ckpt")
show_values(unrestored_variables0, "Unrestored variables of global variables")
return True
exclude_dir = args.exc_dir
suffix = args.ext
files0 = wmlu.recurse_get_filepath_in_dir(src_dir,suffix=suffix)
files1 = wmlu.recurse_get_filepath_in_dir(exclude_dir,suffix=suffix)
files1 = [os.path.basename(file) for file in files1]
total_skip = 0
total_remove = 0
files_to_remove = []
for file in files0:
base_name = os.path.basename(file)
if base_name not in files1:
print(f"Skip {base_name}")
total_skip += 1
else:
print(f"Remove {file}")
total_remove += 1
files_to_remove.append(file)
print(f"Files need to remove:")
wmlu.show_list(files_to_remove)
res = input("remove [y/n]")
if res != 'y':
print(f"Cancel.")
exit(0)
for file in files_to_remove:
remove_file(file)
print(f"Total files in src dir {len(files0)}, total files in exclude dir {len(files1)}.")
print(f"Total skip {total_skip}, total remove {total_remove}, total process {total_skip+total_remove}")
def build_net(self):
if not os.path.exists(self.log_dir):
wmlu.create_empty_dir(self.log_dir)
if not os.path.exists(self.ckpt_dir):
wmlu.create_empty_dir(self.ckpt_dir)
with tf.device(":/cpu:0"):
data = self.data.get_next()
DataLoader.detection_image_summary(data, name="data_source")
self.input_data = data
'''if self.cfg.GLOBAL.DEBUG:
data[IMAGE] = tf.Print(data[IMAGE],[tf.shape(data[IMAGE]),data[ORG_HEIGHT],data[ORG_WIDTH],data[HEIGHT],data[WIDTH]],summarize=100,
name="XXXXX")'''
self.res_data, loss_dict = self.model.forward(data)
if self.model.is_training:
for k, v in loss_dict.items():
tf.summary.scalar(f"loss/{k}", v)
v = tf.cond(tf.logical_or(tf.is_nan(v), tf.is_inf(v)),
lambda: tf.zeros_like(v), lambda: v)
tf.losses.add_loss(v)
elif self.cfg.GLOBAL.SUMMARY_LEVEL <= SummaryLevel.RESEARCH:
research = self.cfg.GLOBAL.RESEARCH
if 'result_classes' in research:
print("replace labels with gtlabels.")
labels = odt.replace_with_gtlabels(
bboxes=self.res_data[RD_BOXES],
labels=self.res_data[RD_LABELS],
length=self.res_data[RD_LENGTH],
gtbboxes=data[GT_BOXES],
gtlabels=data[GT_LABELS],
gtlength=data[GT_LENGTH])
self.res_data[RD_LABELS] = labels
if 'result_bboxes' in research:
print("replace bboxes with gtbboxes.")
bboxes = odt.replace_with_gtbboxes(
bboxes=self.res_data[RD_BOXES],
labels=self.res_data[RD_LABELS],
length=self.res_data[RD_LENGTH],
gtbboxes=data[GT_BOXES],
gtlabels=data[GT_LABELS],
gtlength=data[GT_LENGTH])
self.res_data[RD_BOXES] = bboxes
self.loss_dict = loss_dict
if not self.model.is_training and self.cfg.GLOBAL.GPU_MEM_FRACTION > 0.1:
gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=self.
cfg.GLOBAL.GPU_MEM_FRACTION)
config = tf.ConfigProto(allow_soft_placement=True,
gpu_options=gpu_options)
else:
config = tf.ConfigProto(allow_soft_placement=True)
if not self.model.is_training and self.cfg.GLOBAL.GPU_MEM_FRACTION <= 0.1:
config.gpu_options.allow_growth = True
self.sess = tf.Session(config=config)
self.top_variable_name_scope = "Model"
if self.model.is_training:
steps = self.cfg.SOLVER.STEPS
print("Train steps:", steps)
lr = wnn.build_learning_rate(
self.cfg.SOLVER.BASE_LR,
global_step=self.global_step,
lr_decay_type=self.cfg.SOLVER.LR_DECAY_TYPE,
steps=steps,
decay_factor=self.cfg.SOLVER.LR_DECAY_FACTOR,
total_steps=steps[-1],
warmup_steps=self.cfg.SOLVER.WARMUP_ITERS)
tf.summary.scalar("lr", lr)
opt = wnn.str2optimizer("Momentum", lr, momentum=0.9)
self.max_train_step = steps[-1]
self.train_op, self.total_loss, self.variables_to_train = wnn.nget_train_op(
self.global_step,
optimizer=opt,
clip_norm=self.cfg.SOLVER.CLIP_NORM)
print("variables to train:")
wmlu.show_list(self.variables_to_train)
for v in self.variables_to_train:
wsummary.histogram_or_scalar(v, v.name[:-2])
wnn.log_moving_variable()
self.saver = tf.train.Saver(max_to_keep=100)
tf.summary.scalar(self.cfg.GLOBAL.PROJ_NAME + "_total_loss",
self.total_loss)
self.summary = tf.summary.merge_all()
self.summary_writer = tf.summary.FileWriter(self.log_dir,
self.sess.graph)
init = tf.global_variables_initializer()
self.sess.run(init)
print("batch_norm_ops.")
wmlu.show_list(
[x.name for x in tf.get_collection(tf.GraphKeys.UPDATE_OPS)])
def build_net_run_on_multi_gpus_nccl(self):
if not os.path.exists(self.log_dir):
wmlu.create_empty_dir(self.log_dir)
if not os.path.exists(self.ckpt_dir):
wmlu.create_empty_dir(self.ckpt_dir)
'''if self.cfg.GLOBAL.DEBUG:
data[IMAGE] = tf.Print(data[IMAGE],[tf.shape(data[IMAGE]),data[ORG_HEIGHT],data[ORG_WIDTH],data[HEIGHT],data[WIDTH]],summarize=100,
name="XXXXX")'''
all_loss_dict = {}
steps = self.cfg.SOLVER.STEPS
print("Train steps:", steps)
lr = wnn.build_learning_rate(
self.cfg.SOLVER.BASE_LR,
global_step=self.global_step,
lr_decay_type=self.cfg.SOLVER.LR_DECAY_TYPE,
steps=steps,
decay_factor=self.cfg.SOLVER.LR_DECAY_FACTOR,
total_steps=steps[-1],
min_lr=1e-6,
warmup_steps=self.cfg.SOLVER.WARMUP_ITERS)
tf.summary.scalar("lr", lr)
self.max_train_step = steps[-1]
if self.cfg.SOLVER.OPTIMIZER == "Momentum":
opt = wnn.str2optimizer(
"Momentum", lr, momentum=self.cfg.SOLVER.OPTIMIZER_momentum)
else:
opt = wnn.str2optimizer(self.cfg.SOLVER.OPTIMIZER, lr)
tower_grads = []
if len(self.gpus) == 0:
self.gpus = [0]
if len(self.cfg.SOLVER.TRAIN_SCOPES) > 1:
train_scopes = self.cfg.SOLVER.TRAIN_SCOPES
else:
train_scopes = None
if len(self.cfg.SOLVER.TRAIN_REPATTERN) > 1:
train_repattern = self.cfg.SOLVER.TRAIN_REPATTERN
else:
train_repattern = None
for i in range(len(self.gpus)):
scope = tf.get_variable_scope()
if i > 0:
#scope._reuse = tf.AUTO_REUSE
scope.reuse_variables()
with tf.device(f"/gpu:{i}"):
with tf.device(":/cpu:0"):
data = self.data.get_next()
self.input_data = data
with tf.name_scope(f"GPU{self.gpus[i]}"):
with tf.device(":/cpu:0"):
DataLoader.detection_image_summary(
data, name=f"data_source{i}")
self.res_data, loss_dict = self.model.forward(data)
loss_values = []
for k, v in loss_dict.items():
all_loss_dict[k + f"_stage{i}"] = v
tf.summary.scalar(f"loss/{k}", v)
##
#v = tf.Print(v,[k,tf.is_nan(v), tf.is_inf(v)])
##
v = tf.cond(tf.logical_or(tf.is_nan(v), tf.is_inf(v)),
lambda: tf.zeros_like(v), lambda: v)
loss_values.append(v)
scope._reuse = tf.AUTO_REUSE
'''if (i==0) and len(tf.get_collection(GRADIENT_DEBUG_COLLECTION))>0:
total_loss_sum = tf.add_n(loss_values)
xs = tf.get_collection(GRADIENT_DEBUG_COLLECTION)
grads = tf.gradients(total_loss_sum,xs)
grads = [tf.reduce_sum(tf.abs(x)) for x in grads]
loss_values[0] = tf.Print(loss_values[0],grads+["grads"],summarize=100)'''
grads, total_loss, variables_to_train = wnn.nget_train_opv3(
optimizer=opt,
loss=loss_values,
scopes=train_scopes,
re_pattern=train_repattern)
#
if self.cfg.SOLVER.FILTER_NAN_AND_INF_GRADS:
grads = [list(x) for x in grads]
for i, (g, v) in enumerate(grads):
try:
if g is not None:
g = tf.where(
tf.logical_or(tf.is_nan(g), tf.is_inf(g)),
tf.random_normal(
shape=wmlt.
combined_static_and_dynamic_shape(g),
stddev=1e-5), g)
except:
print(f"Error {g}/{v}")
raise Exception("Error")
grads[i][0] = g
#
tower_grads.append(grads)
########################
'''tower_grads[0] = [list(x) for x in tower_grads[0]]
for i,(g,v) in enumerate(tower_grads[0]):
tower_grads[0][i][0] = tf.Print(g,["B_"+v.name,tf.reduce_min(g),tf.reduce_mean(g),tf.reduce_max(g)])'''
########################
if self.cfg.SOLVER.CLIP_NORM > 1:
avg_grads = wnn.average_grads_nccl(
tower_grads, clip_norm=self.cfg.SOLVER.CLIP_NORM)
else:
avg_grads = wnn.average_grads_nccl(tower_grads, clip_norm=None)
'''avg_grads = [list(x) for x in avg_grads]
for i,(g,v) in enumerate(avg_grads):
avg_grads[i][0] = tf.Print(g,[v.name,tf.reduce_min(g),tf.reduce_mean(g),tf.reduce_max(g)])'''
opt0 = wnn.apply_gradientsv3(avg_grads, self.global_step, opt)
opt1 = wnn.get_batch_norm_ops()
self.train_op = tf.group(opt0, opt1)
self.total_loss, self.variables_to_train = total_loss, variables_to_train
self.loss_dict = all_loss_dict
config = tf.ConfigProto(allow_soft_placement=True)
#config.gpu_options.allow_growth = True
self.sess = tf.Session(config=config)
if self.debug_tf:
self.sess = tfdbg.LocalCLIDebugWrapperSession(self.sess)
print("variables to train:")
wmlu.show_list(self.variables_to_train)
for v in self.variables_to_train:
wsummary.histogram_or_scalar(v, v.name[:-2])
wnn.log_moving_variable()
self.saver = tf.train.Saver(max_to_keep=100)
tf.summary.scalar("total_loss", self.total_loss)
self.summary = tf.summary.merge_all()
self.summary_writer = tf.summary.FileWriter(self.log_dir,
self.sess.graph)
init = tf.global_variables_initializer()
self.sess.run(init)
print("batch_norm_ops.")
wmlu.show_list(
[x.name for x in tf.get_collection(tf.GraphKeys.UPDATE_OPS)])