def GetPredictions(model_file, train_op_file, output_dir, dataset='test'):
board = tr.LockGPU()
model = util.ReadModel(model_file)
model.layer[0].data_field.test = '%s_data' % dataset
train_op = util.ReadOperation(train_op_file)
train_op.verbose = False
train_op.get_last_piece = True
train_op.randomize = False
layernames = ['output_layer']
ex.ExtractRepresentations(model_file, train_op, layernames, output_dir)
tr.FreeGPU(board)
def main():
model_file = sys.argv[1]
op_file = sys.argv[2]
output_dir = sys.argv[3]
data_proto = sys.argv[4]
if len(sys.argv) > 5:
gpu_mem = sys.argv[5]
else:
gpu_mem = '2G'
if len(sys.argv) > 6:
main_mem = sys.argv[6]
else:
main_mem = '30G'
board = tr.LockGPU()
SampleText(model_file, op_file, output_dir, data_proto, gpu_mem, main_mem)
tr.FreeGPU(board)
def main(job_id, params):
board = trainer.LockGPU()
prefix = os.getcwd()
model_file = os.path.join(prefix, 'deepnet_base.pbtxt')
train_op_file = os.path.join(prefix, 'train.pbtxt')
eval_op_file = os.path.join(prefix, 'eval.pbtxt')
model, train_op, eval_op = trainer.LoadExperiment(model_file,
train_op_file,
eval_op_file)
model.name = 'deepnet_%d' % job_id
ModifyHyperparameters(model, params)
net = trainer.CreateDeepnet(model, train_op, eval_op)
net.Train()
trainer.FreeGPU(board)
value = GetLastValidationError(net)
return value
def main():
patternfile = sys.argv[1]
targetfile = sys.argv[2]
label_file = sys.argv[3]
output_dir = sys.argv[4]
statiticsFile = '/data1/ningzhang/flickr/flickr_stats.npz'
batchsize = 128
K = 5
if len(sys.argv) > 5:
K = sys.argv[5]
if len(sys.argv) > 6:
batchsize = sys.argv[6]
else:
gpu_mem = '2G'
if len(sys.argv) > 6:
main_mem = sys.argv[6]
else:
main_mem = '30G'
import pdb
pdb.set_trace()
board = tr.LockGPU()
targets = np.load(targetfile)
patterns = np.load(patternfile)
labels = np.load(label_file)
stats = np.load(statiticsFile)
dist, minDist_indices, neibor_labels = Knn(patterns, targets, batchsize, K,
labels, stats)
dist_dir = os.path.join(output_dir, 'distance')
indices_dir = os.path.join(output_dir, 'indices')
labels_dir = os.path.join(output_dir, 'labels')
np.save(dist_dir, dist)
np.save(indices_dir, minDist_indices)
np.save(labels_dir, neibor_labels)
sio.savemat(os.path.join(output_dir, 'distance_mat'), {'distance': dist})
sio.savemat(os.path.join(output_dir, 'indices_mat'),
{'indices': minDist_indices})
sio.savemat(os.path.join(output_dir, 'labels_mat'),
{'labels': neibor_labels})
tr.FreeGPU(board)
action='store_true',
help="Calculate blosum90 scores")
parser.add_argument("--ncols", type=int, help="Number of multiple columns")
parser.add_argument("--multmode",
type=str,
help="Multicol mode",
default='rand')
args = parser.parse_args()
if not args.outf:
raise ValueError('Output file not defined')
if not args.train_file or not args.model_file:
raise ValueError('Models and data missing')
board = tr.LockGPU()
model_file = args.model_file
train_file = args.train_file
model = dbm.DBM(model_file, train_file)
trainer_pb = util.ReadOperation(train_file)
dataset = os.path.basename(trainer_pb.data_proto_prefix)
# Fix paths
dirname = os.path.split(model.t_op.data_proto_prefix)[1]
model.t_op.data_proto_prefix = os.path.join('datasets/',\
dirname)
model.t_op.skip_last_piece = False
model.t_op.get_last_piece = True
model.t_op.randomize = False
def main():
patternFilePattern = sys.argv[1]
targetFilePattern = sys.argv[2]
output_dir = sys.argv[3]
if len(sys.argv) > 4:
label_file = sys.argv[4]
statiticsFile = '/data1/ningzhang/flickr/flickr_stats.npz'
batchsize = 1000
K = 5
if len(sys.argv) > 5:
K = sys.argv[5]
if len(sys.argv) > 6:
batchsize = sys.argv[6]
else:
gpu_mem = '2G'
if len(sys.argv) > 6:
main_mem = sys.argv[6]
else:
main_mem = '30G'
import pdb
pdb.set_trace()
board = tr.LockGPU()
patternFiles = sorted(glob.glob(patternFilePattern))
targetFiles = sorted(glob.glob(targetFilePattern))
stats = np.load(statiticsFile)
patternlist = []
m = 0
for i, patternFile in enumerate(patternFiles):
patternlist.append(np.load(patternFile))
m += patternlist[i].shape[0]
patterns = np.zeros((m, patternlist[0].shape[1]))
pos = 0
for patternShark in patternlist:
patterns[pos:pos + patternShark.shape[0], :] = patternShark
pos = pos + patternShark.shape[0]
pos = 0
dist_pool = np.zeros((1, 2 * K))
if len(sys.argv) > 4:
labels = np.load(label_file)
for targetFile in targetFiles:
targets = np.load(targetFile)
if len(sys.argv) > 4:
dist_interm, minDist_indices_interm, neibor_labels_interm = Knn(
patterns, targets, batchsize, K, labels, stats)
else:
dist_interm, minDist_indices_interm = Knn(patterns, targets,
batchsize,
K) #, stats = stats)
if pos == 0:
dist = np.copy(dist_interm)
minDist_indices = np.copy(minDist_indices_interm)
if len(sys.argv) > 4:
neibor_labels = np.copy(neibor_labels_interm)
else:
K_new = K if K <= targets.shape[0] else targets.shape[0]
if K_new < K:
dist_pool = np.zeros((1, K + K_new))
for ind_1 in range(m):
dist_pool[0, 0:K] = dist[ind_1, 0:K]
dist_pool[0, K:K + K_new] = dist_interm[ind_1, 0:K_new]
internal_compare = dist_pool.argsort().flatten()
dist[ind_1, :] = dist_pool[0, internal_compare[0:K]]
for j in range(K):
minDist_indices[ind_1, j] = minDist_indices[
ind_1, j] if internal_compare[
j] < K else minDist_indices_interm[ind_1, j] + pos
if len(sys.argv) > 4:
neibor_labels[ind_1, j] = labels[minDist_indices[ind_1,
j], :]
pos = pos + targets.shape[0]
dist_dir = os.path.join(output_dir, 'distance')
indices_dir = os.path.join(output_dir, 'indices')
labels_dir = os.path.join(output_dir, 'labels')
np.save(dist_dir, dist)
np.save(indices_dir, minDist_indices)
np.save(labels_dir, neibor_labels)
sio.savemat(os.path.join(output_dir, 'distance_mat'), {'distance': dist})
sio.savemat(os.path.join(output_dir, 'indices_mat'),
{'indices': minDist_indices})
sio.savemat(os.path.join(output_dir, 'labels_mat'),
{'labels': neibor_labels})
tr.FreeGPU(board)
