def EditModels(args):
# Common changes in all models
model_files = glob.glob("models/*.pbtxt")
for model_file in model_files:
model = util.ReadModel(model_file)
model.hyperparams.base_epsilon = args.base_epsilon
model.hyperparams.epsilon_decay = deepnet_pb2.Hyperparams.INVERSE_T \
if args.epsilon_decay else deepnet_pb2.Hyperparams.NONE
model.hyperparams.sparsity = args.sparsity
model.hyperparams.dropout = args.dropout
model.hyperparams.l2_decay = args.l2_decay
model.hyperparams.initial_momentum = args.initial_momentum
model.hyperparams.final_momentum = args.final_momentum
with open(model_file, 'w') as f:
text_format.PrintMessage(model, f)
# Specific changes to rbm1
model_file = os.path.join('models', 'rbm1.pbtxt')
model = util.ReadModel(model_file)
for layer in model.layer:
if layer.name == 'input_layer':
layer.dimensions = args.input_width
layer.numlabels = args.input_numlabels
if layer.name == 'hidden1':
layer.dimensions = args.hidden1_width
with open(model_file, 'w') as f:
text_format.PrintMessage(model, f)
# Specific changes to rbm2
model_file = os.path.join('models', 'rbm2.pbtxt')
model = util.ReadModel(model_file)
for layer in model.layer:
if layer.name == 'hidden1':
layer.dimensions = args.hidden1_width
if layer.name == 'hidden2':
layer.dimensions = args.hidden2_width
with open(model_file, 'w') as f:
text_format.PrintMessage(model, f)
# Specific changes in joint
model_file = os.path.join('models', 'joint.pbtxt')
model = util.ReadModel(model_file)
model.prefix = args.model_dir
for layer in model.layer:
if layer.name == 'input_layer':
layer.dimensions = args.input_width
layer.numlabels = args.input_numlabels
if layer.name == 'hidden1':
layer.dimensions = args.hidden1_width
if layer.name == 'hidden2':
layer.dimensions = args.hidden2_width
with open(model_file, 'w') as f:
text_format.PrintMessage(model, f)
def EditModelsDBM(args):
""" DBM specific only """
if args.model not in ['dbm', 'lcdbm']:
raise ValueError('Unknown model {}'.format(args.model))
# Common changes in all models
model_files = glob.glob("models/*.pbtxt")
for model_file in model_files:
model = util.ReadModel(model_file)
model.hyperparams.base_epsilon = args.base_epsilon
model.hyperparams.sparsity = args.sparsity
model.hyperparams.dropout = args.dropout
model.hyperparams.l2_decay = args.l2_decay
model.hyperparams.initial_momentum = args.initial_momentum
model.hyperparams.final_momentum = args.final_momentum
with open(model_file, 'w') as f:
text_format.PrintMessage(model, f)
# Specific changes to rbm2
model_file = os.path.join('models', 'rbm2.pbtxt')
model = util.ReadModel(model_file)
for layer in model.layer:
if layer.name == 'hidden1' or layer.name == 'bernoulli_hidden1':
layer.dimensions = args.hidden1_width
if layer.name == 'hidden2':
layer.dimensions = args.hidden2_width
with open(model_file, 'w') as f:
text_format.PrintMessage(model, f)
# Specific changes to joint
model_file = os.path.join('models', 'joint.pbtxt')
model = util.ReadModel(model_file)
model.prefix = args.model_dir
for layer in model.layer:
if layer.name == 'input_layer':
layer.dimensions = args.input_width
layer.numlabels = args.input_numlabels
if layer.name == 'bernoulli_hidden1' or layer.name == 'hidden1':
layer.dimensions = args.hidden1_width
if layer.name == 'hidden2':
layer.dimensions = args.hidden2_width
# Sparsity mask
if args.model in ['lcdbm']:
edge = next(e for e in model.edge if e.node1 == 'input_layer' and \
e.node2 == 'bernoulli_hidden1')
param = next(p for p in edge.param if p.name == 'weight')
sparsity_mask_file = param.sparsity_mask
param.sparsity_mask = os.path.join(args.data_dir, sparsity_mask_file)
with open(model_file, 'w') as f:
text_format.PrintMessage(model, f)
def main():
path = sys.argv[1]
output_file = sys.argv[2]
#import pdb
#pdb.set_trace()
layers = ['ZCR', 'MFCC', 'Spectral', 'Energy',
'Chroma', 'PLP']
MS = ['withMS', 'withoutMS']
predsMap = {}
f = open(output_file, 'w')
f.write('\\begin{tabular}{|c|c|c|c|c|c|c} \\hline \n')
f.write('MS &ZCR &MFCC &Spectral &Energy &Chroma &PLP \\\\ \\hline\n')
for ms in MS:
for layer in layers:
mfile = os.path.join(path, ms, '%s_dnn_BEST' % layer)
if not os.path.exists(mfile):
if ms not in predsMap:
predsMap[ms] = []
predsMap[ms].append(0.0)
continue
model = util.ReadModel(mfile)
preds = model.test_stat_es.correct_preds/model.test_stat_es.count
if ms not in predsMap:
predsMap[ms] = []
predsMap[ms].append(preds)
for ms in MS:
f.write(ms)
for item in predsMap[ms]:
f.write('&%.4f'% item)
f.write('\\hline\n')
f.write('\\end{tabular}\n')
f.close()
def MakeModels(model_file, output_path):
model = util.ReadModel(model_file)
for l in model.layer:
for p in l.param:
EditPretrainedModels(p, output_path)
for e in model.edge:
for p in e.param:
EditPretrainedModels(p, output_path)
util.WritePbtxt(model_file, model)
def Convert(model_file, output_file):
model = util.ReadModel(model_file)
params = {}
for l in model.layer:
for p in l.param:
params['%s_%s' % (l.name, p.name)] = util.ParameterAsNumpy(p)
for e in model.edge:
for p in e.param:
params['%s_%s_%s' %
(e.node1, e.node2, p.name)] = util.ParameterAsNumpy(p)
scipy.io.savemat(output_file, params, oned_as='column')
def GetPredictions(model_file, train_op_file, output_dir, dataset='test'):
board = tr.LockGPU()
model = util.ReadModel(model_file)
model.layer[0].data_field.test = '%s_data' % dataset
train_op = util.ReadOperation(train_op_file)
train_op.verbose = False
train_op.get_last_piece = True
train_op.randomize = False
layernames = ['output_layer']
ex.ExtractRepresentations(model_file, train_op, layernames, output_dir)
tr.FreeGPU(board)
def main():
filePath = sys.argv[1]
outputPath = sys.argv[2]
classifiers = sorted(glob.glob(os.path.join(filePath, "*BEST")))
f = open(outputPath, 'w')
pdb.set_trace()
for classifier in classifiers:
baseName = os.path.basename(classifier)
if "op" in baseName:
continue
model = util.ReadModel(classifier)
testPreds = model.test_stat_es.correct_preds / model.test_stat_es.count
trainPreds = model.train_stat_es.correct_preds / model.train_stat_es.count
validPreds = model.best_valid_stat.correct_preds / model.best_valid_stat.count
f.write(
'%s train: %.5f valid: %.5f test: %.5f \n' %
(os.path.basename(classifier), testPreds, trainPreds, validPreds))
f.close()
def change_model(proto, layers=None):
model = util.ReadModel(proto)
if layers is None:
layers = ['image_hidden1', 'image_hidden2', 'image_hidden3',
'text_hidden1', 'text_hidden2', 'text_hidden3',
'image_layer', 'text_layer', 'joint_layer',
'image_tied_hidden', 'text_tied_hidden',
'image_hidden2_recon', 'text_hidden2_recon',
'cross_image_hidden2_recon', 'cross_text_hidden2_recon']
for layer in layers:
try:
layer_proto = next(lay for lay in model.layer if lay.name == layer)
layer_proto.dimensions = dimensions
except StopIteration:
pass
with open(proto, 'w') as f:
text_format.PrintMessage(model, f)
def EditTrainers(data_dir, model_dir, rep_dir, numsplits):
tnames = [
'train_CD_image_layer1.pbtxt', 'train_CD_image_layer2.pbtxt',
'train_CD_text_layer1.pbtxt', 'train_CD_text_layer2.pbtxt',
'train_CD_joint_layer.pbtxt'
]
for tname in tnames:
t_op_file = os.path.join('trainers', 'dbn', tname)
t_op = util.ReadOperation(t_op_file)
if 'layer1' in tname:
t_op.data_proto_prefix = data_dir
else:
t_op.data_proto_prefix = rep_dir
t_op.checkpoint_directory = model_dir
with open(t_op_file, 'w') as f:
text_format.PrintMessage(t_op, f)
t_op_file = os.path.join('trainers', 'classifiers', 'baseclassifier.pbtxt')
t_op = util.ReadOperation(t_op_file)
for i in range(1, numsplits + 1):
t_op_file = os.path.join('trainers', 'classifiers',
'split_%d.pbtxt' % i)
t_op.data_proto_prefix = rep_dir
t_op.data_proto = os.path.join('split_%d' % i, 'data.pbtxt')
t_op.checkpoint_prefix = model_dir
t_op.checkpoint_directory = os.path.join('classifiers', 'split_%d' % i)
with open(t_op_file, 'w') as f:
text_format.PrintMessage(t_op, f)
# Change prefix in multimodal dbn model
mnames = ['multimodal_dbn.pbtxt']
for mname in mnames:
model_file = os.path.join('models', mname)
model = util.ReadModel(model_file)
model.prefix = model_dir
with open(model_file, 'w') as f:
text_format.PrintMessage(model, f)
def main():
path = sys.argv[1]
numsplits = int(sys.argv[2])
output_file = sys.argv[3]
layers = [
'RNA1seq_input', 'RNA1seq_hidden1', 'RNA1seq_hidden2', 'joint_hidden',
'RNA2seq_hidden2', 'RNA2seq_hidden1', 'RNA2seq_input'
]
maps = {}
precs = {}
for i in range(1, numsplits + 1):
for layer in layers:
mfile = os.path.join(path, 'split_%d' % i,
'%s_classifier_BEST' % layer)
model = util.ReadModel(mfile)
MAP = model.test_stat_es.MAP
prec50 = model.test_stat_es.prec50
if layer not in maps:
maps[layer] = []
if layer not in precs:
precs[layer] = []
maps[layer].append(MAP)
precs[layer].append(prec50)
f = open(output_file, 'w')
f.write('\\begin{tabular}{|l|c|c|} \\hline \n')
f.write('Layer & MAP & Prec@50 \\\\ \\hline\n')
for layer in layers:
lmap = np.array(maps[layer])
lprec = np.array(precs[layer])
f.write('%s & %.3f $\\pm$ %.3f & %.3f $\\pm$ %.3f \\\\ \n' %
(layer, lmap.mean(), lmap.std(), lprec.mean(), lprec.std()))
f.write('\\hline\n')
f.write('\\end{tabular}\n')
f.close()
def Convert(args,
dirpath,
mat_file,
dump_npy=False,
out_file='rbm_mrf',
model_file=None):
""" Create the necesarry things"""
matfile = sio.loadmat(mat_file)
if args.minfill:
# get the weight matrix
weight = np.asarray(matfile['minL'], dtype='float32')
Pmat = matfile['Pmat']
weight = weight.dot(Pmat)
weight = weight.T
elif args.random:
# get the weight matrix
weight = np.asarray(matfile['L'].T, dtype='float32')
weight[np.abs(weight) < 1e-8] = 0.0
nnz = np.count_nonzero(weight)
nNodes = weight.shape[0]
weight = np.random.randn(nNodes**2, 1)
rangeIdx = np.arange(nNodes**2)
np.random.shuffle(rangeIdx)
weight[rangeIdx[nnz:]] = 0.
weight = weight.reshape(nNodes, nNodes)
print("nnz: {}".format(nnz))
elif args.thresh:
# get both the weight materices
if not -0.00001 < args.thresh < 100.000001:
raise ValueError("Threshold should be b/w 0 and 1")
weight = np.asarray(matfile['L'].T, dtype='float32')
weight_minfill = np.asarray(matfile['minL'], dtype='float32')
nnz = np.sum(np.abs(weight) > 1e-10)
nnz_minfill = np.sum(np.abs(weight_minfill) > 1e-10)
num_to_delete = int((nnz - nnz_minfill) * args.thresh / 100.)
weight_nnz = np.abs(weight[np.abs(weight) > 1e-10])
threshold = np.sort(weight_nnz)[num_to_delete]
weight[np.abs(weight) < threshold] = 0.0
else:
weight = np.asarray(matfile['L'].T, dtype='float32')
nFeats, _ = weight.shape
diag = np.ones([nFeats, 1]) * matfile['min_eig'] * (1 + matfile['alpha'])
diag = np.asarray(diag, dtype='float32')
if dump_npy:
if args.edge_input_file:
edge_file = os.path.join(dirpath, args.edge_input_file)
else:
edge_file = os.path.join(dirpath, 'edge_input_to_gaussian.npy')
diag_file = os.path.join(dirpath, 'diag_gaussian.npy')
np.save(edge_file, weight)
np.save(diag_file, diag)
else:
model = util.ReadModel(model_file)
proto_weight = next(param for param in model.edge[0].param
if param.name == 'weight')
proto_weight.mat = util.NumpyAsParameter(weight)
proto_weight.dimensions.extend(weight.shape)
input_layer = next(l for l in model.layer if l.name == 'input_layer')
proto_diag = next(param for param in input_layer.param
if param.name == 'diag')
proto_diag.mat = util.NumpyAsParameter(diag)
proto_diag.dimensions.extend(diag.shape)
out_file = os.path.join(dirpath, out_file)
f = gzip.open(out_file, 'wb')
f.write(model.SerializeToString())
f.close()
def main():
model_file = sys.argv[1]
base_output_dir = sys.argv[2]
rep_dir = sys.argv[3]
prefix = sys.argv[4]
gpu_mem = sys.argv[5]
main_mem = sys.argv[6]
model = util.ReadModel(model_file)
data_pb = deepnet_pb2.Dataset()
data_pb.name = model.name
data_pb.gpu_memory = gpu_mem
data_pb.main_memory = main_mem
output_dir = os.path.join(base_output_dir, 'validation')
if not os.path.isdir(output_dir):
os.makedirs(output_dir)
output_proto_file = os.path.join(base_output_dir, 'data.pbtxt')
# IMAGE PATHWAY
img_input_pbtxt = os.path.join(prefix, 'flickr.pbtxt')
img_hidden1_pbtxt = os.path.join(rep_dir, 'image_rbm1_LAST', 'data.pbtxt')
img_hidden2_pbtxt = os.path.join(rep_dir, 'image_rbm2_LAST', 'data.pbtxt')
# TEXT PATHWAY
text_input_pbtxt = os.path.join(prefix, 'flickr_nnz.pbtxt')
text_hidden1_pbtxt = os.path.join(rep_dir, 'text_rbm1_LAST', 'data.pbtxt')
text_hidden2_pbtxt = os.path.join(rep_dir, 'text_rbm2_LAST', 'data.pbtxt')
text_pbtxt_z = os.path.join(rep_dir, 'generated_text', 'data.pbtxt')
joint_pbtxt = os.path.join(rep_dir, 'joint_rbm_LAST', 'data.pbtxt')
img_input_pb = util.ReadData(img_input_pbtxt)
data = next(d for d in img_input_pb.data if d.name == 'image_labelled')
data.file_pattern = os.path.join(img_input_pb.prefix, data.file_pattern)
data.stats_file = os.path.join(img_input_pb.prefix, data.stats_file)
data.name = 'image_input'
data_pb.data.extend([data])
img_hidden1_pb = util.ReadData(img_hidden1_pbtxt)
data = next(d for d in img_hidden1_pb.data
if d.name == 'image_hidden1_validation')
data.file_pattern = os.path.join(img_hidden1_pb.prefix, data.file_pattern)
data.name = 'image_hidden1'
data_pb.data.extend([data])
img_hidden2_pb = util.ReadData(img_hidden2_pbtxt)
data = next(d for d in img_hidden2_pb.data
if d.name == 'image_hidden2_validation')
data.file_pattern = os.path.join(img_hidden2_pb.prefix, data.file_pattern)
data.name = 'image_hidden2'
data_pb.data.extend([data])
indices_file = os.path.join(prefix, 'text', 'indices_labelled.npz')
indices = np.load(indices_file)
nnz_indices = indices['nnz_indices']
z_indices = indices['z_indices']
text_pb_z = util.ReadData(text_pbtxt_z)
text_input_pb = util.ReadData(text_input_pbtxt)
data_nnz = next(d for d in text_input_pb.data if d.name == 'text_labelled')
data_z = next(d for d in text_pb_z.data
if d.name == 'text_input_layer_validation')
output_file = os.path.join(output_dir, 'text_input-00001-of-00001.npy')
data = Merge(data_nnz, data_z, nnz_indices, z_indices, text_pb_z.prefix,
text_input_pb.prefix, 'text_input', output_file)
data_pb.data.extend([data])
text_hidden1_pb = util.ReadData(text_hidden1_pbtxt)
data_nnz = next(d for d in text_hidden1_pb.data
if d.name == 'text_hidden1_validation')
data_z = next(d for d in text_pb_z.data
if d.name == 'text_hidden1_validation')
output_file = os.path.join(output_dir, 'text_hidden1-00001-of-00001.npy')
data = Merge(data_nnz, data_z, nnz_indices, z_indices, text_pb_z.prefix,
text_hidden1_pb.prefix, 'text_hidden1', output_file)
data_pb.data.extend([data])
text_hidden2_pb = util.ReadData(text_hidden2_pbtxt)
data_nnz = next(d for d in text_hidden2_pb.data
if d.name == 'text_hidden2_validation')
data_z = next(d for d in text_pb_z.data
if d.name == 'text_hidden2_validation')
output_file = os.path.join(output_dir, 'text_hidden2-00001-of-00001.npy')
data = Merge(data_nnz, data_z, nnz_indices, z_indices, text_pb_z.prefix,
text_hidden2_pb.prefix, 'text_hidden2', output_file)
data_pb.data.extend([data])
joint_pb = util.ReadData(joint_pbtxt)
data_nnz = next(d for d in joint_pb.data
if d.name == 'joint_hidden_validation')
data_z = next(d for d in text_pb_z.data
if d.name == 'joint_hidden_validation')
output_file = os.path.join(output_dir, 'joint_hidden-00001-of-00001.npy')
data = Merge(data_nnz, data_z, nnz_indices, z_indices, text_pb_z.prefix,
joint_pb.prefix, 'joint_hidden', output_file)
data_pb.data.extend([data])
with open(output_proto_file, 'w') as f:
text_format.PrintMessage(data_pb, f)
from deepnet import util
from deepnet import visualize
import sys
m = util.ReadModel(sys.argv[1])
w = util.ParameterAsNumpy(m.edge[0].param[0])
pvh = visualize.display_convw2(w, 5, 8, 8, 1)
raw_input('Press enter.')
def main():
model_file = sys.argv[1]
base_output_dir = sys.argv[2]
rep_dir = sys.argv[3]
prefix = sys.argv[4]
gpu_mem = sys.argv[5]
main_mem = sys.argv[6]
model = util.ReadModel(model_file)
data_pb = deepnet_pb2.Dataset()
data_pb.name = model.name
data_pb.gpu_memory = gpu_mem
data_pb.main_memory = main_mem
output_dir = os.path.join(base_output_dir, 'validation')
if not os.path.isdir(output_dir):
os.makedirs(output_dir)
output_proto_file = os.path.join(base_output_dir, 'data.pbtxt')
# IMAGE PATHWAY
img_input_pbtxt = os.path.join(prefix, 'RNAseq.pbtxt')
img_hidden1_pbtxt = os.path.join(rep_dir, 'RNA1seq_rbm1_LAST', 'data.pbtxt')
#img_hidden2_pbtxt = os.path.join(rep_dir, 'RNA1seq_rbm2_LAST', 'data.pbtxt')
# TEXT PATHWAY
text_input_pbtxt = os.path.join(prefix, 'RNAseq.pbtxt')
text_hidden1_pbtxt = os.path.join(rep_dir, 'RNA2seq_rbm1_LAST', 'data.pbtxt')
#text_hidden2_pbtxt = os.path.join(rep_dir, 'RNA2seq_rbm2_LAST', 'data.pbtxt')
#text_pbtxt_z = os.path.join(rep_dir, 'generated_text', 'data.pbtxt')
joint_pbtxt = os.path.join(rep_dir, 'joint_rbm_LAST', 'data.pbtxt')
joint2_pbtxt = os.path.join(rep_dir, 'joint_rbm2_LAST', 'data.pbtxt')
img_input_pb = util.ReadData(img_input_pbtxt)
data = next(d for d in img_input_pb.data if d.name == 'RNA1seq_train')
data.file_pattern = os.path.join(img_input_pb.prefix, data.file_pattern)
#data.stats_file = os.path.join(img_input_pb.prefix, data.stats_file)
data.name = 'RNA1seq_input'
data_pb.data.extend([data])
img_hidden1_pb = util.ReadData(img_hidden1_pbtxt)
data = next(d for d in img_hidden1_pb.data if d.name == 'RNA1seq_hidden1_train')
data.file_pattern = os.path.join(img_hidden1_pb.prefix, data.file_pattern)
data.name = 'RNA1seq_hidden1'
data_pb.data.extend([data])
#img_hidden2_pb = util.ReadData(img_hidden2_pbtxt)
#data = next(d for d in img_hidden2_pb.data if d.name == 'RNA1seq_hidden2_train')
#data.file_pattern = os.path.join(img_hidden2_pb.prefix, data.file_pattern)
#data.name = 'RNA1seq_hidden2'
#data_pb.data.extend([data])
#indices_file = os.path.join(prefix, 'text', 'indices_labelled.npz')
#indices = np.load(indices_file)
#nnz_indices = indices['nnz_indices']
#z_indices = indices['z_indices']
#text_pb_z = util.ReadData(text_pbtxt_z)
text_input_pb = util.ReadData(text_input_pbtxt)
data = next(d for d in text_input_pb.data if d.name == 'RNA2seq_train')
data.file_pattern = os.path.join(text_input_pb.prefix, data.file_pattern)
data.name = 'RNA2seq_input'
data_pb.data.extend([data])
text_hidden1_pb = util.ReadData(text_hidden1_pbtxt)
data = next(d for d in text_hidden1_pb.data if d.name == 'RNA2seq_hidden1_train')
data.file_pattern = os.path.join(text_hidden1_pb.prefix, data.file_pattern)
data.name = 'RNA2seq_hidden1'
data_pb.data.extend([data])
#text_hidden2_pb = util.ReadData(text_hidden2_pbtxt)
#data = next(d for d in text_hidden2_pb.data if d.name == 'RNA2seq_hidden2_train')
#data.file_pattern = os.path.join(text_hidden2_pb.prefix, data.file_pattern)
#data.name = 'RNA2seq_hidden2'
#data_pb.data.extend([data])
joint_pb = util.ReadData(joint_pbtxt)
data = next(d for d in joint_pb.data if d.name == 'joint_hidden_train')
data.file_pattern = os.path.join(joint_pb.prefix, data.file_pattern)
data.name = 'joint_hidden'
data_pb.data.extend([data])
joint2_pb = util.ReadData(joint2_pbtxt)
data = next(d for d in joint2_pb.data if d.name == 'joint_hidden2_train')
data.file_pattern = os.path.join(joint2_pb.prefix, data.file_pattern)
data.name = 'joint_hidden2'
data_pb.data.extend([data])
with open(output_proto_file, 'w') as f:
text_format.PrintMessage(data_pb, f)
def EditModels(args):
# Common changes in all models
model_files = glob.glob("models/*.pbtxt")
for model_file in model_files:
model = util.ReadModel(model_file)
model.hyperparams.base_epsilon = args.base_epsilon
model.hyperparams.sparsity = args.sparsity
model.hyperparams.dropout = args.dropout
model.hyperparams.l2_decay = args.l2_decay
model.hyperparams.initial_momentum = args.initial_momentum
model.hyperparams.final_momentum = args.final_momentum
with open(model_file, 'w') as f:
text_format.PrintMessage(model, f)
if args.model == 'sprbm':
model.hyperparams.sparsity = True
if args.model == 'droprbm':
model.hyperparams.dropout = True
# Specific changes to rbm
model_file = os.path.join('models', 'rbm.pbtxt')
model = util.ReadModel(model_file)
for layer in model.layer:
if layer.name == 'input_layer':
layer.dimensions = args.input_width
layer.numlabels = args.input_numlabels
if args.model in ('rbm', 'sprbm', 'droprbm'):
if layer.name == 'bernoulli_hidden1':
layer.dimensions = args.hidden1_width
elif args.model in ('lcrbm', 'ghrbm'):
if layer.name == 'gaussian_hidden1':
layer.dimensions = args.input_width * 21
elif args.model in ('warmrbm', 'warmlcrbm'):
if layer.name == 'bernoulli_hidden1':
layer.dimensions = args.input_width * 21
elif args.model in ('warmslcrbm'):
if layer.name == 'bernoulli_hidden1':
layer.dimensions = args.input_width * 21
if layer.name == 'bernoulli2_hidden1':
layer.dimensions = args.bernoulli2_hidden1_width
elif args.model in ('slcrbm', 'srbm'):
if layer.name == 'bernoulli_hidden1':
layer.dimensions = args.hidden1_width
if layer.name == 'gaussian_hidden1':
layer.dimensions = args.input_width * 21
else:
raise ValueError('Unknown model {}'.format(args.model))
# Add in the datafiles
if args.model in ('lcrbm', 'ghrbm', 'slcrbm', 'srbm'):
edge = next(e for e in model.edge if e.node1 == 'input_layer' and \
e.node2 == 'gaussian_hidden1')
param = next(p for p in edge.param if p.name == 'weight')
pretrained_model_file = param.pretrained_model[0]
param.pretrained_model[0] = os.path.join(args.data_dir,
pretrained_model_file)
layer = next(l for l in model.layer if l.name == 'input_layer')
param = next(p for p in layer.param if p.name == 'diag')
pretrained_model_file = param.pretrained_model[0]
param.pretrained_model[0] = os.path.join(args.data_dir,
pretrained_model_file)
if args.model in ('warmrbm', 'warmlcrbm', 'warmslcrbm'):
edge = next(e for e in model.edge if e.node1 == 'input_layer' and \
e.node2 == 'bernoulli_hidden1')
param = next(p for p in edge.param if p.name == 'weight')
pretrained_model_file = param.pretrained_model[0]
param.pretrained_model[0] = os.path.join(args.data_dir,
pretrained_model_file)
layer = next(l for l in model.layer if l.name == 'input_layer')
param = next(p for p in layer.param if p.name == 'diag')
pretrained_model_file = param.pretrained_model[0]
param.pretrained_model[0] = os.path.join(args.data_dir,
pretrained_model_file)
if args.model in ('lcrbm', 'slcrbm'):
edge = next(e for e in model.edge if e.node1 == 'input_layer' and \
e.node2 == 'gaussian_hidden1')
param = next(p for p in edge.param if p.name == 'weight')
sparsity_mask_file = param.sparsity_mask
param.sparsity_mask = os.path.join(args.data_dir, sparsity_mask_file)
if args.model in ['warmlcrbm', 'warmslcrbm']:
edge = next(e for e in model.edge if e.node1 == 'input_layer' and \
e.node2 == 'bernoulli_hidden1')
param = next(p for p in edge.param if p.name == 'weight')
sparsity_mask_file = param.sparsity_mask
param.sparsity_mask = os.path.join(args.data_dir, sparsity_mask_file)
with open(model_file, 'w') as f:
text_format.PrintMessage(model, f)
with open(model_file, 'w') as f:
text_format.PrintMessage(model, f)
import glob
import copy
rep_tied_lambda = float(sys.argv[1])
loss_factor = 1 - rep_tied_lambda
pre_dir = sys.argv[2]
prefix = os.path.join(pre_dir, 'ae_models')
for model_file in [
'corr_ae.pbtxt', 'cross_corr_ae.pbtxt', 'full_corr_ae.pbtxt'
]:
proto = os.path.join(prefix, model_file)
model = util.ReadModel(proto)
try:
text_tied_hidden_proto = next(lay for lay in model.layer
if lay.name == 'text_tied_hidden')
text_tied_hidden_proto.rep_tied_lambda = rep_tied_lambda
except StopIteration:
pass
try:
image_tied_hidden_proto = next(lay for lay in model.layer
if lay.name == 'image_tied_hidden')
image_tied_hidden_proto.rep_tied_lambda = rep_tied_lambda
except StopIteration:
pass
try:
for record in records:
writer.writerow(record)
if __name__ == '__main__':
from argparse import ArgumentParser
parser = ArgumentParser(
description='Parses results by walking directories')
parser.add_argument("--outf", type=str, help="Output file")
parser.add_argument("--mode", type=str, help="html/csv")
args = parser.parse_args()
model_paths = walk_dir('.')
exp_paths = defaultdict(list)
get_expid = lambda f: f.split("/")[1]
get_model = lambda f: util.ReadModel(f)
get_op = lambda f: util.ReadOperation(f)
no_match = lambda path: all(bool(s not in path) for s in drop_string_list)
model_paths['BEST'] = filter(no_match, model_paths['BEST'])
for path in model_paths['BEST']:
exp_paths[get_expid(path)].append(path)
exp_paths = dict(exp_paths)
rows = []
for exp in exp_paths:
models = defaultdict(list)
for f in exp_paths[exp]:
model_name = os.path.basename(f).split('_')[0]
models[model_name].append(f)