def save_selected_activations(self):
if not self.input:
self.log_window(title='Error!',
Msg='Please indicate a input image')
else:
save_act_path = file_dialog(
title='choose a folder to save the images',
type='/').openFolderDialog()
selected_idx = self.actList.selectionModel().selectedIndexes()
for idx in selected_idx:
layer_name = self.actList.item(idx.row()).text()
rlt = np.squeeze(self.activations[layer_name])
if rlt.ndim == 3:
for i in range(rlt.shape[-1]):
check_N_mkdir(save_act_path +
layer_name.replace('/', '_'))
Image.fromarray(
rlt[:, :, i]).save(save_act_path +
layer_name.replace('/', '_') +
'/{}.tif'.format(i))
elif rlt.ndim == 2:
check_N_mkdir(save_act_path + layer_name.replace('/', '_'))
Image.fromarray(
rlt[:, :]).save(save_act_path +
layer_name.replace('/', '_') +
'/act.tif')
else:
logger.debug(
'got an unexpected ndim of the activations: {}'.format(
rlt.ndim))
def tsne_on_bias(params=None, mode='2D'):
assert params != None, 'please define the dictionary of paths'
assert isinstance(params, dict), 'paths should be a dictionary containning path'
# get bias
_, bn_init, _, bs_init, _, dnn_bn_init, _, dnn_bs_init = get_all_trainable_variables(params['ckpt_path_init'])
_, bn, _, bs, _, dnn_bn, _, dnn_bs = get_all_trainable_variables(params['ckpt_path'])
shapes = [b.shape[0] for b in bs + dnn_bs]
max_shape = 0
for _shape in shapes:
if _shape >= max_shape:
max_shape = _shape
new_bn = []
new_bs = []
grps = []
which = []
# preparation: unify the b shape by padding
# for first ckpt
for _bn, _b in zip(bn_init + dnn_bn_init, bs_init + dnn_bs_init):
new_bn.append(_bn.split(':')[0])
grps.append(_bn.split('/')[0])
which.append(0)
# pad
if _b.shape[0] < max_shape:
_b = np.pad(_b, (0, max_shape - _b.shape[0]), constant_values=0)
new_bs.append(_b)
# for second ckpt
for _bn, _b in zip(bn + dnn_bn, bs + dnn_bs):
new_bn.append(_bn.split(':')[0])
grps.append(_bn.split('/')[0])
which.append(1)
# pad
if _b.shape[0] < max_shape:
_b = np.pad(_b, (0, max_shape - _b.shape[0]), constant_values=0)
new_bs.append(_b)
# inject into t-SNE
res = tsne(
np.asarray(new_bs).reshape(len(new_bs), -1),
perplexity=params['perplexity'],
niter=params['niter'],
mode=mode,
)
# mkdir
check_N_mkdir(params['rlt_dir'])
# visualize the tsne
if mode == '2D':
compare_tsne_2D(res, new_bn, grps, which=which, rlt_dir=params['tsne_dir'], preffix='Bias', fst=paths['ckpt_path_init'].split('step')[1], sec=paths['ckpt_path'].split('step')[1])
elif mode == '3D':
compare_tsne_3D(res, new_bn, grps, which=which, rlt_dir=params['tsne_dir'], suffix=params['step'])
else:
raise NotImplementedError('please choose 2D or 3D mode')
def visualize_weights(params=None, mode='copy', write_rlt=True):
assert isinstance(params, dict)
if write_rlt:
dir = params['rlt_dir'] + 'weights/step{}/'.format(params['step'])
wn, _, ws, _, _, _, _, _ = get_all_trainable_variables(params['ckpt_path'])
if write_rlt:
for _wn, _w in zip(wn, ws):
for i in range(_w.shape[3]):
if mode == 'interpolation':
# interpolation and enlarge to a bigger matrix (instead of repeating)
x = np.linspace(-1, 1, _w.shape[0])
y = np.linspace(-1, 1, _w.shape[1])
f = interpolate.interp2d(x, y, np.sum(_w[:, :, :, i], axis=2), kind='cubic')
x = np.linspace(-1, 1, _w.shape[0] * 30)
y = np.linspace(-1, 1, _w.shape[1] * 30)
tmp = f(x, y)
elif mode == 'copy':
tmp = np.repeat(np.repeat(np.sum(_w[:, :, :, i], axis=2), 30, axis=0), 30, axis=1)
else:
raise NotImplementedError('mode??')
# save
if write_rlt:
check_N_mkdir(dir + '{}/'.format(_wn.split('/')[0]))
Image.fromarray(tmp).save(
dir + '{}/{}.tif'.format(_wn.split('/')[0], i))
return wn, ws
def tsne_on_weights(params=None, mode='2D'):
"""
input:
-------
ckptpath: (string) path to the checkpoint that we convert to .pb. e.g. './logs/YYYY_MM_DD_.../hourHH/ckpt/step{}'
return:
-------
None
"""
assert params!=None, 'please define the dictionary of paths'
assert isinstance(params, dict), 'paths should be a dictionary containning path'
# run tsne on wieghts
# get weights from checkpoint
wns_init, _, ws_init, _, _, _, _, _ = get_all_trainable_variables(params['ckpt_path_init'])
wns, _, ws, _, _, _, _, _ = get_all_trainable_variables(params['ckpt_path'])
# arange label and kernel
new_wn = []
new_ws = []
grps = []
which = []
# for 1st ckpt
for wn, w in zip(wns_init, ws_init): # w.shape = [c_w, c_h, c_in, nb_conv]
for i in range(w.shape[3]):
new_wn.append(wn + '_{}'.format(i)) # e.g. conv4bis_96
grps.append(wn.split('/')[0])
which.append(0)
#note: associativity: a x b + a x c = a x (b + c)
# "...a kernel is the sum of all the dimensions in the previous layer..."
# https://stackoverflow.com/questions/42712219/dimensions-in-convolutional-neural-network
new_ws.append(np.sum(w[:, :, :, i], axis=2)) # e.g. (3, 3, 12, 24) [w, h, in, nb_conv] --> (3, 3, 24)
# for 2nd ckpt
for wn, w in zip(wns, ws): # w.shape = [c_w, c_h, c_in, nb_conv]
for i in range(w.shape[3]):
new_wn.append(wn + '_{}'.format(i)) # e.g. conv4bis_96
grps.append(wn.split('/')[0])
which.append(1)
new_ws.append(np.sum(w[:, :, :, i], axis=2)) # e.g. (3, 3, 12, 24) [w, h, in, nb_conv] --> (3, 3, 24)
# inject into t-SNE
res = tsne(np.array(new_ws).transpose((1, 2, 0)).reshape(len(new_ws), -1),
perplexity=params['perplexity'],
niter=params['niter'], mode=mode) # e.g. (3, 3, x) --> (9, x) --> (x, 2) or (x, 3)
# mkdir
check_N_mkdir(params['rlt_dir'])
# visualize the tsne
if mode == '2D':
compare_tsne_2D(res, new_wn, grps, which, rlt_dir=params['tsne_dir'], fst=paths['ckpt_path_init'].split('step')[1], sec=paths['ckpt_path'].split('step')[1])
elif mode == '3D':
compare_tsne_3D(res, new_wn, grps, which, rlt_dir=params['tsne_dir'], suffix=params['step'])
else:
raise NotImplementedError('please choose 2D or 3D mode')
def optimize_pb_for_inference(paths=None, conserve_nodes=None):
assert isinstance(
paths, dict
), 'The paths parameter expected a dictionnay but other type is provided'
# clean graph first
tf.reset_default_graph()
check_N_mkdir(paths['save_pb_dir'])
# load protobuff
with tf.gfile.FastGFile(paths['save_pb_path'], "rb") as f:
graph_def = tf.GraphDef()
graph_def.ParseFromString(f.read())
# optimize pb
try:
optimize_graph_def = optimize_for_inference(
input_graph_def=graph_def,
input_node_names=['new_input', 'new_BN', 'new_dropout'],
output_node_names=conserve_nodes,
placeholder_type_enum=[
dtypes.float32.as_datatype_enum,
dtypes.bool.as_datatype_enum,
dtypes.float32.as_datatype_enum,
])
except:
optimize_graph_def = optimize_for_inference(
input_graph_def=graph_def,
input_node_names=['new_input', 'new_BN'],
output_node_names=conserve_nodes,
placeholder_type_enum=[
dtypes.float32.as_datatype_enum,
dtypes.bool.as_datatype_enum,
dtypes.float32.as_datatype_enum,
])
with tf.gfile.GFile(paths['optimized_pb_path'], 'wb') as f:
f.write(optimize_graph_def.SerializeToString())
def gradient_extractor(event_dir: str, write_rlt=True):
if not event_dir.endswith('train/'):
_dir = os.path.join(event_dir, 'train/')
else:
_dir = event_dir
accumulator = event_accumulator.EventAccumulator(_dir,
size_guidance={
event_accumulator.SCALARS: 0,
event_accumulator.HISTOGRAMS: 0,
})
accumulator.Reload()
tags = accumulator.Tags()
l_grad_tag = []
for param_name in tags['histograms']:
if 'grad' in param_name:
l_grad_tag.append(param_name)
logger.info(l_grad_tag)
# stack params
mapping = []
gamma = []
gamman = []
beta = []
betan = []
w = []
wn = []
layer = {}
step = np.asarray(get_sum(accumulator, l_grad_tag[0])[1])
for grad in l_grad_tag:
mapping.append(np.asarray(get_sum(accumulator, grad)[0]))
if 'gamma' in grad:
gamma.append(np.asarray(get_sum(accumulator, grad)[0]))
gamman.append(grad)
elif ('beta' in grad) or ('b_0' in grad):
beta.append(np.asarray(get_sum(accumulator, grad)[0]))
betan.append(grad)
elif 'w_0' in grad:
w.append(np.asarray(get_sum(accumulator, grad)[0]))
wn.append(grad)
try:
layer_name = re.search('conv(\d+b?)', grad).group(1)
try:
layer[layer_name].append(np.asarray(get_sum(accumulator, grad)[0]))
except Exception as e:
logger.error(e)
layer[layer_name] = [np.asarray(get_sum(accumulator, grad)[0])]
except Exception as e:
logger.error(grad)
pass
block_mapping = []
layer_mapping = []
for k, v in layer.items():
# take absolute value
layer_mapping.append(np.sum(np.abs(elt) for elt in v))
if 'b' in k:
block_mapping.append(np.sum(np.abs(elt) for elt in v))
block_mapping = np.stack(block_mapping).transpose()
layer_mapping = np.stack(layer_mapping).transpose()
full_mapping = np.stack(mapping, axis=1)
# take the absolute values of the gradients
gamma = np.abs(np.stack(gamma, axis=1))
beta = np.abs(np.stack(np.abs(beta), axis=1))
w = np.abs(np.stack(np.abs(w), axis=1))
# fold N times then sum
N = 50
M = 1
# solution: repeat N times is easier
block_mapping = np.repeat(np.repeat(block_mapping, N, axis=1), M, axis=0)
layer_mapping = np.repeat(np.repeat(layer_mapping, N, axis=1), M, axis=0)
full_mapping = np.repeat(np.repeat(full_mapping, N, axis=1), M, axis=0)
gamma = np.repeat(np.repeat(gamma, N, axis=1), M, axis=0)
beta = np.repeat(np.repeat(beta, N, axis=1), M, axis=0)
w = np.repeat(np.repeat(w, N, axis=1), M, axis=0)
if write_rlt:
check_N_mkdir(event_dir + 'grad/')
# save gradient mappings
Image.fromarray(block_mapping).save(os.path.join(event_dir, 'grad', 'each_block_mapping.tif'))
Image.fromarray(layer_mapping).save(os.path.join(event_dir, 'grad', 'each_layer_mapping.tif'))
Image.fromarray(full_mapping).save(os.path.join(event_dir, 'grad', 'all_param_mapping.tif'))
Image.fromarray(gamma).save(os.path.join(event_dir, 'grad', ' gamma.tif'))
Image.fromarray(beta).save(os.path.join(event_dir, 'grad', 'beta.tif'))
Image.fromarray(w).save(os.path.join(event_dir, 'grad', 'w.tif'))
# save gradient as .csv
np.savetxt(os.path.join(event_dir, "grad", "each_layer_mapping.csv"), block_mapping, delimiter=",")
np.savetxt(os.path.join(event_dir, "grad", "all_param_mapping.csv"), full_mapping, delimiter=",")
np.savetxt(os.path.join(event_dir, "grad", "gamma.csv"), gamma, delimiter=",")
np.savetxt(os.path.join(event_dir, "grad", "beta.csv"), beta, delimiter=",")
np.savetxt(os.path.join(event_dir, "grad", "w.csv"), w, delimiter=",")
_gamma = {}
_betaOrBias = {}
_w = {}
for i, n in enumerate(gamman):
#gamma.shape: (step, N*nb_layer), gamman.shape: (nb_layer)
_gamma[n] = gamma[::M, i * N]
for i, n in enumerate(betan):
_betaOrBias[n] = beta[::M, i * N]
for i, n in enumerate(wn):
_w[n] = w[::M, i * N]
return block_mapping, full_mapping, _gamma, _betaOrBias, _w, step
def _weighttWriter(tensor, layer_name='', path=None):
'''
tensor: images(numpy array or list of image) to save of (Height, Width, nth_Conv)
path: path(string)
layer_name: name of the layer(string)
'''
# mkdir
check_N_mkdir(path + layer_name)
# for writting inference partial rlt
if isinstance(tensor, list):
for i, elt in enumerate(tensor):
logger.debug('layer_name: {}'.format(layer_name))
if (True in np.isnan(elt)) or (True in np.isinf(elt)):
try:
elt = clean(elt)
except Exception as e:
logger.debug(e)
logger.warning('Clean function cannot clean the nan value from the layer {}! '.format(layer_name))
break
# scope: conv p_infer: (ps, ps, nc)
if elt.ndim == 3:
# note: save only the first
for j in range(elt.shape[-1]):
Image.fromarray(np.asarray(elt[:, :, j])).save(path + '{}/{}.tif'.format(layer_name, j))
elif elt.ndim == 2:
Image.fromarray(np.asarray(elt)).save(path + '{}/{}.tif'.format(layer_name, i))
# scope: dnn p_infer: (nb_post neuron)
elif elt.ndim == 1:
# reshape the 1D array
ceil = int(np.ceil(np.sqrt(elt.size)))
tmp = np.zeros((ceil ** 2), np.float32).ravel()
tmp[:elt.size] = elt
tmp = tmp.reshape(ceil, ceil)
Image.fromarray(tmp).save(path + '{}/dnn.tif'.format(layer_name))
else:
# treat dnn weights
if tensor.ndim == 1:
# reshape the 1D array
ceil = int(np.ceil(np.sqrt(tensor.size)))
tmp = np.zeros((ceil ** 2), np.float32).ravel()
tmp[:tensor.size] = tensor
tmp = tmp.reshape(ceil, ceil)
Image.fromarray(tmp).save(path + '{}/dnn.tif'.format(layer_name))
# scope: not I_1_hoted tensor: (B, H, W, C)
elif tensor.ndim == 4:
tensor = np.squeeze(tensor.astype(np.float32))
if 'diff' in layer_name:
for i in range(tensor.shape[0]):
Image.fromarray(tensor[i]).save(path + '{}/{}.tif'.format(layer_name, i))
# for cnn ndim=3
elif tensor.ndim == 3:
for i in range(tensor.shape[2]):
Image.fromarray(np.asarray(tensor[:, :, i], dtype=np.float)).save(path + '{}/{}.tif'.format(layer_name, i))
else:
logger.warn('Not implement writer for this kind of data in layer :{}'.format(layer_name))
raise NotImplementedError('Ouppss {}'.format(layer_name))
def weights_angularity(ckpt_dir=None, rlt_dir=None):
"""
inputs:
-------
path: (string) path to get the checkpoint e.g. './logs/YYYY_MM_DD_.../hourHH/ckpt/'
return:
-------
None
"""
# construct dataframe
# header sheet_name weight: [step0, step20, ...]
# header sheet_name bias: [step0, step20, ...]
check_N_mkdir(rlt_dir)
lnames = []
for step in os.listdir(ckpt_dir):
if step.endswith('.meta'):
lnames.append(ckpt_dir + step.split('.')[0])
lnames = sorted(lnames)
# get weights-bias values at step0
wn, bn, ws_init, bs_init, dnn_wn, dnn_bn, dnn_ws_init, dnn_bs_init = get_all_trainable_variables(lnames[0])
print('\n ********* processing angularity for each checkpoint')
l_total_w_avg = [0]
l_total_b_avg = [0]
l_total_w_std = [0]
l_total_b_std = [0]
dic_w = {'step': [0]}
dic_b = {'step': [0]}
for key in wn + dnn_wn:
dic_w[key.split('/')[0] + '_avg'] = [0]
dic_w[key.split('/')[0] + '_std'] = [0]
dic_b[key.split('/')[0] + '_avg'] = [0]
dic_b[key.split('/')[0] + '_std'] = [0]
for ckpt_path in lnames[1:]:
# insert step
step = int(ckpt_path.split('step')[1].split('.')[0])
print(step)
dic_w['step'].append(step)
dic_b['step'].append(step)
total_ang_w = []
total_ang_b = []
# get ws values at stepX
wn, bn, ws_, bs_, dnn_wn, dnn_bn, dnn_ws_, dnn_bs_ = get_all_trainable_variables(ckpt_path)
# program cosine alpha
# for w
for _wn, w_init, w_ in zip(wn + dnn_wn, ws_init + dnn_ws_init, ws_ + dnn_ws_):
l_ang_w = []
try:
# for CNN
# retrive the filters
w_init, w_ = np.sum(w_init, axis=2), np.sum(w_, axis=2)
# write w
for i in range(w_init.shape[2]):
# note: need to flatten the kernel
angle_w = np.dot(w_init[:, :, i].ravel(), w_[:, :, i].ravel()) / (np.linalg.norm(w_init[:, :, i].ravel()) * np.linalg.norm(w_[:, :, i].ravel()))
l_ang_w.append(angle_w)
total_ang_w.append(angle_w)
except Exception as e:
# for DNN
# Retrieve weights
w_init, w_ = np.sum(w_init, axis=1), np.sum(w_, axis=1)
angle_w = np.dot(w_init.T, w_) / (np.linalg.norm(w_init) * np.linalg.norm(w_))
l_ang_w.append(angle_w)
total_ang_w.append(angle_w)
# save w into dfs
dic_w[_wn.split('/')[0] + '_avg'].append(np.asarray(l_ang_w).mean())
dic_w[_wn.split('/')[0] + '_std'].append(np.asarray(l_ang_w).std())
# for b
for _bn, b_init, b_ in zip(bn + dnn_bn, bs_init + dnn_bs_init, bs_ + dnn_bs_):
l_ang_b = []
ang_b = np.dot(b_init.ravel(), b_.ravel()) / (np.linalg.norm(b_init) * np.linalg.norm(b_))
l_ang_b.append(ang_b)
total_ang_b.append(ang_b)
# write b into dfs
dic_b[_bn.split('/')[0] + '_avg'].append(np.asarray(l_ang_b).mean())
dic_b[_bn.split('/')[0] + '_std'].append(np.asarray(l_ang_b).std())
l_total_w_avg.append(np.asarray(total_ang_w).mean())
l_total_w_std.append(np.asarray(total_ang_w).std())
l_total_b_avg.append(np.asarray(total_ang_b).mean())
l_total_b_std.append(np.asarray(total_ang_b).std())
dic_w['total_avg'] = l_total_w_avg
dic_w['total_std'] = l_total_w_std
dic_b['total_avg'] = l_total_b_avg
dic_b['total_std'] = l_total_b_std
# create df
try:
dfs = {'weight': pd.DataFrame(dic_w), 'bias': pd.DataFrame(dic_b)}
except Exception as e:
#note: in a BN network, there are less bias
logger.info(e)
dfs = {'weight': pd.DataFrame(dic_w)}
# write into excel
with pd.ExcelWriter(rlt_dir + 'angularity.xlsx', engine='xlsxwriter') as writer:
for sheet_name in dfs.keys():
dfs[sheet_name].sort_values('step').to_excel(writer, sheet_name=sheet_name, index=False)
def weights_hists_2excel(ckpt_dir=None, rlt_dir=None):
"""
inputs:
-------
path: (string) path to get the checkpoint e.g. './logs/YYYY_MM_DD_.../hourHH/ckpt/'
return:
-------
None
"""
# note
# construct dataframe
# header sheet_name conv1: [step0, step20, ...]
# header sheet_name conv1bis: [step0, step20, ...]
#construct list [step0, step100, step200...]
#ckpt name convention: step{}.meta
check_N_mkdir(rlt_dir)
lnames = []
for step in os.listdir(ckpt_dir):
if step.endswith('.meta'):
lnames.append(ckpt_dir + step.split('.')[0])
assert len(lnames) > 1, 'The ckpt directory should have at least 2 ckpts!'
lnames = sorted(lnames)
# fixme: ValueError: This sheet is too large! Your sheet size is: 1280000, 1 Max sheet size is: 1048576, 16384
bins = 1000
step = []
_min = {} # [conv1w, conv1b...]
_max = {} # [conv1w, conv1b...]
df_w = {} # {conv1_w: pd.DataFrame({0:..., 1000:...}), conv1bis_w: pd.DataFrame({0:..., 1000:..., ...})}
df_b = {} # {conv1_b: pd.DataFrame({0:..., 1000:...}), conv1bis_b: pd.DataFrame({0:..., 1000:..., ...})}
hist_w = {} # {conv1_w: pd.DataFrame({x:..., 0:..., 1000:...}), conv1bis_w: pd.DataFrame({x:..., 0:..., 1000:..., ...})}
hist_b = {} # {conv1_b: pd.DataFrame({x:..., 0:..., 1000:...}), conv1bis_b: pd.DataFrame({x:..., 0:..., 1000:..., ...})}
# step 0
wn, bn, ws, bs, dnn_wn, dnn_bn, dnn_ws, dnn_bs = get_all_trainable_variables(lnames[0])
_ws = ws + dnn_ws
_bs = bs + dnn_bs
step.append(lnames[0].split('step')[1].split('.')[0])
# init dataframes
for i, layer_name in enumerate(wn + dnn_wn):
df_w[layer_name.split(':')[0].replace('/', '_')] = pd.DataFrame({'0': _ws[i].flatten()})
for i, layer_name in enumerate(bn + dnn_bn):
df_b[layer_name.split(':')[0].replace('/', '_')] = pd.DataFrame({'0': _bs[i].flatten()})
# add more step to layers params
for i, ckpt_path in enumerate(lnames[1:]):
step.append(ckpt_path.split('step')[1].split('.')[0])
# get weights-bias names and values
wn, bn, ws, bs, dnn_wn, dnn_bn, dnn_ws, dnn_bs = get_all_trainable_variables(ckpt_path)
_ws = ws + dnn_ws
_bs = bs + dnn_bs
# insert values
for j, layer_name in enumerate(wn + dnn_wn):
df_w[layer_name.split(':')[0].replace('/', '_')].insert(i + 1, step[i + 1], _ws[j].flatten())
for j, layer_name in enumerate(bn + dnn_bn):
df_b[layer_name.split(':')[0].replace('/', '_')].insert(i + 1, step[i + 1], _bs[j].flatten())
# calculate histogram
# find min and max of w/b of each layer
for j, layer_name in enumerate(wn + dnn_wn):
_min[layer_name.split(':')[0].replace('/', '_')] = df_w[layer_name.split(':')[0].replace('/', '_')].min()
_max[layer_name.split(':')[0].replace('/', '_')] = df_w[layer_name.split(':')[0].replace('/', '_')].max()
for j, layer_name in enumerate(bn + dnn_bn):
_min[layer_name.split(':')[0].replace('/', '_')] = df_b[layer_name.split(':')[0].replace('/', '_')].min()
_max[layer_name.split(':')[0].replace('/', '_')] = df_b[layer_name.split(':')[0].replace('/', '_')].max()
# get histogram of W
for layer_name in wn + dnn_wn:
_, _edge = np.histogram(
np.asarray(df_w[layer_name.split(':')[0].replace('/', '_')]),
bins=np.linspace(
_min[layer_name.split(':')[0].replace('/', '_')][0],
_max[layer_name.split(':')[0].replace('/', '_')][0],
bins
)
)
hist_w[layer_name.split(':')[0].replace('/', '_')] = pd.DataFrame({'x': _edge[1:]})
i = 0
for _step, params in df_w[layer_name.split(':')[0].replace('/', '_')].iteritems():
_hist, _ = np.histogram(
np.asarray(params),
bins=np.linspace(_min[layer_name.split(':')[0].replace('/', '_')][_step],
_max[layer_name.split(':')[0].replace('/', '_')][_step],
num=bins
)
)
hist_w[layer_name.split(':')[0].replace('/', '_')].insert(i + 1, _step, _hist)
i += 1
# clean instance
del df_w
# get histogram of b
for layer_name in bn + dnn_bn:
_hist, _edge = np.histogram(
np.asarray(df_b[layer_name.split(':')[0].replace('/', '_')]),
bins=np.linspace(
_min[layer_name.split(':')[0].replace('/', '_')][0],
_max[layer_name.split(':')[0].replace('/', '_')][0],
bins
)
)
hist_b[layer_name.split(':')[0].replace('/', '_')] = pd.DataFrame({'x': _edge[1:]})
i = 0
for _step, params in df_b[layer_name.split(':')[0].replace('/', '_')].iteritems():
_hist, _edge = np.histogram(
np.asarray(params),
bins=np.linspace(
_min[layer_name.split(':')[0].replace('/', '_')][_step],
_max[layer_name.split(':')[0].replace('/', '_')][_step],
bins)
)
hist_b[layer_name.split(':')[0].replace('/', '_')].insert(i + 1, _step, _hist)
i += 1
# clean instance
del df_b
# write into excel
check_N_mkdir(rlt_dir + 'weight_hist/')
for xlsx_name in hist_w.keys():
with pd.ExcelWriter(rlt_dir + 'weight_hist/{}.xlsx'.format(xlsx_name), engine='xlsxwriter') as writer:
hist_w[xlsx_name].to_excel(writer, index=False)
for xlsx_name in hist_b.keys():
with pd.ExcelWriter(rlt_dir + 'weight_hist/{}.xlsx'.format(xlsx_name), engine='xlsxwriter') as writer:
hist_b[xlsx_name].to_excel(writer, index=False)
def inference_and_save_partial_res(g_main, ops_dict, conserve_nodes, hyper=None, input_dir=None, rlt_dir=None, feature_map=False, norm=1e3, write_rlt=True):
"""
Parameters
----------
g_combined: (tf.Graph())
ops_dict: (list of operations)
conserve_nodes: (list of string)
Returns
-------
None
"""
config_params = {}
if hyper['device_option'] == 'cpu':
config_params['config'] = tf.ConfigProto(device_count={'GPU': 0})
elif 'specific' in hyper['device_option']:
print('using GPU:{}'.format(hyper['device_option'].split(':')[-1]))
config_params['config'] = tf.ConfigProto(
gpu_options=tf.GPUOptions(visible_device_list=hyper['device_option'].split(':')[-1]),
allow_soft_placement=True,
log_device_placement=False,
)
with g_main.as_default() as g_main:
# init a writer class
plt_illd = plot_input_logit_label_diff()
new_input = g_main.get_tensor_by_name('new_input:0')
# write firstly input and output images
# todo: following is useless
l = []
if os.path.isdir(input_dir):
for f in os.listdir(input_dir):
if '_label' not in f:
l.append(input_dir + f)
else:
l.append(input_dir)
# todo: above is useless
img_path = l[0]
logger.debug('img_path: %s' % img_path)
if hyper['model'] in ['LRCS', 'LRCS2', 'Xlearn']:
img = load_img(img_path)[:hyper['window_size'], :hyper['window_size']]
else:
img = dimension_regulator(load_img(img_path), maxp_times=4 if hyper['model'] in ['Unet', 'Segnet', 'Unet5', 'Unet6'] else 3)
# note: the following try to normalize the input img e.g. 32IDC FBP-CUDA --> ~range(0, 0.0012) *1000 ~ (0 ~ 1)
if norm:
img = img * norm
img_size = img.shape
logger.info('input shape: {}'.format(img_size))
if feature_map:
# weka like input
l_func = [
Gaussian_Blur,
Sobel,
Hessian,
DoG,
Gabor,
# 'membrane_proj': Membrane_proj,
Anisotropic_Diffusion1,
Anisotropic_Diffusion2,
Bilateral,
Median,
]
imgs = [img]
for func in l_func:
imgs.append(func(imgs[0]))
imgs = np.stack(imgs, axis=2).astype(np.float32)
labels = [dimension_regulator(load_img(img_path.replace('.tif', '_label.tif')),
maxp_times=4 if hyper['model'] in ['Unet', 'Segnet', 'Unet5', 'Unet6'] else 3)]
logger.info('label shape: {}'.format(labels[0].shape))
else:
imgs = [
img
]
if hyper['model'] in ['LRCS', 'LRCS2', 'Xlearn']:
labels = [dimension_regulator(load_img(img_path.replace('.tif', '_label.tif'))[:hyper['window_size'], :hyper['window_size']])]
else:
labels = [dimension_regulator(load_img(img_path.replace('.tif', '_label.tif')),
maxp_times=4 if hyper['model'] in ['Unet', 'Segnet', 'Unet5', 'Unet6'] else 3)]
logger.info('label shape: {}'.format(labels[0].shape))
# save imgs
plt_illd.add_input(np.asarray(imgs))
if write_rlt:
_resultWriter(imgs, 'input', path=rlt_dir, contrast=False)
plt_illd.add_label(np.asarray(labels))
if write_rlt:
_resultWriter(labels, 'label', path=rlt_dir)
# prepare feed_dict
feed_dict = {
new_input: np.array(imgs).reshape((-1, img_size[0], img_size[1], 10 if hyper['feature_map'] else 1)),
}
if hyper['batch_normalization']:
new_BN_phase = g_main.get_tensor_by_name('new_BN:0')
feed_dict[new_BN_phase] = False
try:
dropout_input = g_main.get_tensor_by_name('new_dropout:0')
feed_dict[dropout_input] = 1.0
except Exception as e:
logger.error(e)
pass
# run inference
with tf.Session(graph=g_main, **config_params) as sess:
print_nodes_name_shape(sess.graph)
# run partial results operations and diff block
res = sess.run(ops_dict['ops'], feed_dict=feed_dict)
activations = {}
# note: save partial/final inferences of the first image
for layer_name, tensors in zip(conserve_nodes, res):
try:
if tensors.ndim == 4 or 2:
if 'logit' in layer_name:
tensors = customized_softmax_np(tensors)
tensors = _inverse_one_hot(tensors)
plt_illd.add_logit(tensors)
else:
tensors = [np.squeeze(tensors[i]) for i in range(tensors.shape[0])]
except Exception as e:
logger.error(e)
pass
if layer_name == 'add':
if write_rlt:
_resultWriter(tensors, layer_name=layer_name,
path=rlt_dir,
batch_or_channel='channel' if hyper['feature_map'] else 'batch') # for cnn outputs shape: [batch, w, h, nb_conv]
else:
if write_rlt:
_resultWriter(tensors, layer_name=layer_name.split('/')[-2],
path=rlt_dir,
batch_or_channel='channel' if hyper['feature_map'] else 'batch') # for cnn outputs shape: [batch, w, h, nb_conv]
activations[layer_name] = tensors
# calculate diff by numpy
# res[-1] final result
if hyper['mode'] == 'regression':
res_diff = np.equal(np.asarray(np.squeeze(res[-1]), dtype=np.int), np.asarray(labels))
res_diff = np.asarray(res_diff, dtype=np.int)
plt_illd.add_diff(np.asarray(res_diff))
if write_rlt:
_resultWriter(np.transpose(res_diff, (1, 2, 0)), 'diff',
path=rlt_dir) # for diff output shape: [batch, w, h, 1]
else:
# one-hot the label
labels = np.expand_dims(np.asarray(labels), axis=3) # list --> array --> (B, H, W, 1)
logits = customized_softmax_np(np.asarray(res[-1], dtype=np.int)) # (B, H, W, 3)
res_diff = np.equal(_inverse_one_hot(clean(logits)), labels) #(B, H, W)
plt_illd.add_diff(res_diff.astype(int))
if write_rlt:
_resultWriter(res_diff.astype(int), 'diff', path=rlt_dir) # for diff output shape: [batch, w, h, 3]
if write_rlt:
check_N_mkdir(rlt_dir + 'illd/')
plt_illd.plot(out_path=rlt_dir + 'illd/illd.tif')
# return
return activations
def freeze_ckpt_for_inference(paths=None, hyper=None, conserve_nodes=None):
assert isinstance(
paths, dict
), 'The paths parameter expected a dictionnay but other type is provided'
assert isinstance(
hyper, dict
), 'The hyper parameter expected a dictionnay but other type is provided'
assert isinstance(
conserve_nodes,
list), 'The name of the conserve node should be in a list'
# clean graph first
tf.reset_default_graph()
# freeze ckpt then convert to pb
new_input = tf.placeholder(
tf.float32,
shape=[None, None, None, 10 if hyper['feature_map'] else 1],
name='new_input') # note: resize the input while inferencing
new_BN = tf.placeholder_with_default(
False, [],
name='new_BN') #note: it seems like T/F after freezing isn't important
# load meta graph
input_map = {
'input_pipeline_train/IteratorGetNext': new_input,
}
try:
new_dropout = tf.placeholder_with_default(1.0, [], name='new_dropout')
input_map['dropout_prob'] = new_dropout
if hyper['batch_normalization']:
input_map['BN_phase'] = new_BN
restorer = tf.train.import_meta_graph(
paths['ckpt_path'] + '.meta' if
not paths['ckpt_path'].endswith('.meta') else paths['ckpt_path'],
input_map=input_map,
clear_devices=True,
)
except Exception as e:
if hyper['batch_normalization']:
input_map['BN_phase'] = new_BN
logger.warning('Error(msg):', e)
restorer = tf.train.import_meta_graph(
paths['ckpt_path'] + '.meta' if
not paths['ckpt_path'].endswith('.meta') else paths['ckpt_path'],
input_map=input_map,
clear_devices=True,
)
input_graph_def = tf.get_default_graph().as_graph_def()
check_N_mkdir(paths['save_pb_dir'])
# use cpu or gpu
config_params = {}
if hyper['device_option'] == 'cpu':
config_params['config'] = tf.ConfigProto(device_count={'GPU': 0})
elif 'specific' in hyper['device_option']:
print('using GPU:{}'.format(hyper['device_option'].split(':')[-1]))
config_params['config'] = tf.ConfigProto(
gpu_options=tf.GPUOptions(
visible_device_list=hyper['device_option'].split(':')[-1]),
allow_soft_placement=True,
log_device_placement=False,
)
# freeze to pb
with tf.Session(**config_params) as sess:
# restore variables
restorer.restore(sess, paths['ckpt_path'])
# convert variable to constant
# todo: verify this convert_variables_to_constants() function if it's correctly working for batch norm
output_graph_def = tf.graph_util.convert_variables_to_constants(
sess=sess,
input_graph_def=input_graph_def,
output_node_names=conserve_nodes,
)
# save to pb
tf.summary.FileWriter(paths['working_dir'] + 'tb/after_freeze',
sess.graph)
with tf.gfile.GFile(paths['save_pb_path'],
'wb') as f: # 'wb' stands for write binary
f.write(output_graph_def.SerializeToString())
def inference_recursive_V3(l_input_path=None,
conserve_nodes=None,
paths=None,
hyper=None,
norm=1e-3):
assert isinstance(conserve_nodes, list), 'conserve nodes should be a list'
assert isinstance(
l_input_path, list
), 'inputs is expected to be a list of images for heterogeneous image size!'
assert isinstance(paths, dict), 'paths should be a dict'
assert isinstance(hyper, dict), 'hyper should be a dict'
from mpi4py import MPI
# prevent GPU
os.environ["CUDA_VISIBLE_DEVICES"] = "-1"
communicator = MPI.COMM_WORLD
rank = communicator.Get_rank()
nb_process = communicator.Get_size()
# optimize ckpt to pb for inference
if rank == 0:
for img in l_img_path:
logger.debug(img)
check_N_mkdir(paths['inference_dir'])
freeze_ckpt_for_inference(paths=paths,
hyper=hyper,
conserve_nodes=conserve_nodes)
optimize_pb_for_inference(paths=paths, conserve_nodes=conserve_nodes)
reconstructor = reconstructor_V3_cls(
image_size=load_img(l_input_path[0]).shape,
z_len=len(l_input_path),
nb_class=hyper['nb_classes'],
maxp_times=hyperparams['maxp_times'])
pbar1 = tqdm(total=len(l_input_path))
# ************************************************************************************************ I'm a Barrier
communicator.Barrier()
# reconstruct volumn
remaining = len(l_input_path)
nb_img_per_rank = remaining // (nb_process - 1)
rest_img = remaining % (nb_process - 1)
print(nb_img_per_rank, rest_img, nb_process)
if rank == 0:
# start gathering batches from other rank
s = MPI.Status()
communicator.Probe(status=s)
while remaining > 0:
if s.tag == tag_compute:
# receive outputs
slice_id, out_batch = communicator.recv(tag=tag_compute)
logger.debug(slice_id)
reconstructor.write_slice(out_batch, slice_id)
# progress
remaining -= 1
pbar1.update(1)
else:
try:
if (rank - 1) < rest_img:
start_id = (rank - 1) * (nb_img_per_rank + 1)
id_list = np.arange(start_id, start_id + nb_img_per_rank + 1,
1)
else:
start_id = (rank - 1) * nb_img_per_rank + rest_img
id_list = np.arange(start_id, start_id + nb_img_per_rank, 1)
logger.debug('{}: {}'.format(rank, id_list))
_inference_recursive_V3(l_input_path=l_input_path,
id_list=id_list,
pb_path=paths['optimized_pb_path'],
conserve_nodes=conserve_nodes,
hyper=hyper,
comm=communicator,
maxp_times=hyperparams['maxp_times'],
normalization=norm)
except Exception as e:
logger.error(e)
MPI.MPI_abort(communicator)
# ************************************************************************************************ I'm a Barrier
communicator.Barrier()
# save recon
if rank == 0:
recon = reconstructor.get_volume()
for i in tqdm(range(len(l_input_path)), desc='writing data'):
Image.fromarray(
recon[i]).save(paths['inference_dir'] +
'step{}_{}.tif'.format(paths['step'], i))
def compare_tsne_2D(embedded_tensor,
labels,
grps,
which,
figsize=(90, 90),
rlt_dir=None,
preffix='Weights',
fst=0,
sec=0):
"""
inputs:
-------
embedded_tensor: (numpy ndarray)
labels: (numpy ndarray?)
grps: (pandas column)
figsize: (tuple of int)
suffix: (str)
return:
-------
None
"""
assert rlt_dir != None, "enter a rlt_dir"
assert embedded_tensor.shape[0] >= len(
labels), 'You should have more embeddings then labels'
df = pd.DataFrame(
zip(embedded_tensor[:, 0], embedded_tensor[:, 1], labels, grps, which))
df.columns = ['coordX', 'coordY', 'labels', 'layers', 'which']
df_init = df.loc[df['which'] == 0]
df_evolv = df.loc[df['which'] == 1]
# convert column groups to categories int
df_init['colors'] = pd.Categorical(df_init['layers']).codes
df_evolv['colors'] = pd.Categorical(df_evolv['layers']).codes
df['colors'] = pd.Categorical(df['layers']).codes
# 2D scatter plots
fig1, ((ax1, ax2), (ax3, ax4)) = plt.subplots(2, 2, figsize=figsize)
scat1 = ax1.scatter(df_init['coordX'],
df_init['coordY'],
c=df_init['colors'],
cmap='coolwarm',
alpha=0.5)
scat2 = ax2.scatter(df_evolv['coordX'],
df_evolv['coordY'],
c=df_evolv['colors'],
cmap='coolwarm',
alpha=0.5)
scat3 = ax3.scatter(df_init['coordX'], df_init['coordY'], c='black')
ax3.scatter(df_evolv['coordX'],
df_evolv['coordY'],
c=df_evolv['colors'],
cmap='coolwarm',
alpha=0.5)
scat4 = ax4.quiver(
np.asarray(df_init['coordX']),
np.asarray(df_init['coordY']),
np.asarray(df_evolv['coordX']) - np.asarray(df_init['coordX']),
np.asarray(df_evolv['coordY']) - np.asarray(df_init['coordY']),
scale_units='xy',
angles='xy',
scale=1,
)
scat4 = ax4.scatter(df_init['coordX'], df_init['coordY'], c='black')
ax4.scatter(df_evolv['coordX'],
df_evolv['coordY'],
c=df_evolv['colors'],
cmap='coolwarm',
alpha=0.5)
# set titles
ax1.set_title('Init weights')
ax2.set_title('Evolved weights')
ax3.set_title('Compare weights')
ax4.set_title('Trajectory')
# set legends
leg1 = ax1.legend(scat1.legend_elements()[0],
df_init['layers'].unique(),
title='Init Layers') #note: unique() might change order
ax1.add_artist(leg1)
leg2 = ax2.legend(
scat2.legend_elements()[0],
df_evolv['layers'].unique(),
title='Evolved Layers') #note: unique() might change order
ax2.add_artist(leg2)
# leg3 = ax3.legend(scat3.legend_elements()[0], df['which'].unique(), title='Init vs Evolve') #note: unique() might change order
# ax3.add_artist(leg3)
# ax3.legend(loc='center left', bbox_to_anchor=(1.04, 0.5))
check_N_mkdir(rlt_dir)
plt.savefig(
rlt_dir +
'{}_2D_plot_step{}_vs_step{}_trajectory.png'.format(preffix, fst, sec))
pd.DataFrame(df_init).to_csv(rlt_dir +
'{}_2D_plot_step{}.csv'.format(preffix, fst))
pd.DataFrame(df_evolv).to_csv(rlt_dir +
'{}_2D_plot_step{}.csv'.format(preffix, sec))
plt.show()
def compare_tsne_3D(embedded_tensor,
labels,
grps,
which,
figsize=(90, 90),
rlt_dir=None,
suffix=0):
"""
inputs:
-------
embedded_tensor: (numpy ndarray)
labels: (numpy ndarray?)
grps: (pandas column)
figsize: (tuple of int)
suffix: (str)
return:
-------
None
"""
assert rlt_dir != None, "enter a rlt_dir"
assert embedded_tensor.shape[0] >= len(
labels), 'You should have more embeddings then labels'
# group data with pandas
df = pd.DataFrame(
zip(embedded_tensor[:, 0], embedded_tensor[:, 1],
embedded_tensor[:, 2], labels, grps, which))
df.columns = ['coordX', 'coordY', 'coordZ', 'labels', 'layers', 'which']
df_init = df.loc[df['which'] == 0]
df_evolv = df.loc[~df['which'] == 1]
# convert colume groups to categories int
df_init['colors'] = pd.Categorical(df_init['layers']).codes
df_evolv['colors'] = pd.Categorical(df_evolv['layers']).codes
df['colors'] = pd.Categorical(df['layers']).codes
# plots conv
fig = plt.figure(figsize=figsize)
ax = Axes3D(fig)
ax.set_title('Weights from encoder')
ax.scatter(df_init['coordX'],
df_init['coordY'],
df_init['coordZ'],
c=df_init['colors'],
cmap=plt.get_cmap('Spectral'),
marker='o')
ax.set_xlabel('X')
ax.set_ylabel('Y')
ax.set_zlabel('Z')
check_N_mkdir(rlt_dir)
# plt.savefig(rlt_dir + 'conv_weights_3Dplot_step{}.png'.format(suffix))
# plots deconv
fig2 = plt.figure(figsize=figsize)
ax2 = Axes3D(fig2)
ax2.set_title('Weights from decoder')
ax2.scatter(df_evolv['coordX'],
df_evolv['coordY'],
df_evolv['coordZ'],
c=df_evolv['colors'],
cmap=plt.get_cmap('Spectral'),
marker='o')
ax2.set_xlabel('X')
ax2.set_ylabel('Y')
ax2.set_zlabel('Z')
# plots conv
fig3 = plt.figure(figsize=figsize)
ax3 = Axes3D(fig3)
ax3.set_title('Weights from all layers')
ax3.scatter(df_init['coordX'],
df_init['coordY'],
df_init['coordZ'],
c=df_init['colors'],
cmap=plt.get_cmap('Spectral'),
marker='o')
ax3.scatter(df_evolv['coordX'],
df_evolv['coordY'],
df_evolv['coordZ'],
c=df_evolv['colors'],
cmap=plt.get_cmap('Spectral'),
marker='o')
ax3.set_xlabel('X')
ax3.set_ylabel('Y')
ax3.set_zlabel('Z')
plt.show()
