def GetPredictions(model_file, train_op_file, output_dir, dataset='test'): board = tr.LockGPU() model = util.ReadModel(model_file) model.layer[0].data_field.test = '%s_data' % dataset train_op = util.ReadOperation(train_op_file) train_op.verbose = False train_op.get_last_piece = True train_op.randomize = False layernames = ['output_layer'] ex.ExtractRepresentations(model_file, train_op, layernames, output_dir) tr.FreeGPU(board)
def main(): model_file = sys.argv[1] op_file = sys.argv[2] output_dir = sys.argv[3] data_proto = sys.argv[4] if len(sys.argv) > 5: gpu_mem = sys.argv[5] else: gpu_mem = '2G' if len(sys.argv) > 6: main_mem = sys.argv[6] else: main_mem = '30G' board = tr.LockGPU() SampleText(model_file, op_file, output_dir, data_proto, gpu_mem, main_mem) tr.FreeGPU(board)
def main(job_id, params): board = trainer.LockGPU() prefix = os.getcwd() model_file = os.path.join(prefix, 'deepnet_base.pbtxt') train_op_file = os.path.join(prefix, 'train.pbtxt') eval_op_file = os.path.join(prefix, 'eval.pbtxt') model, train_op, eval_op = trainer.LoadExperiment(model_file, train_op_file, eval_op_file) model.name = 'deepnet_%d' % job_id ModifyHyperparameters(model, params) net = trainer.CreateDeepnet(model, train_op, eval_op) net.Train() trainer.FreeGPU(board) value = GetLastValidationError(net) return value
def main(): patternfile = sys.argv[1] targetfile = sys.argv[2] label_file = sys.argv[3] output_dir = sys.argv[4] statiticsFile = '/data1/ningzhang/flickr/flickr_stats.npz' batchsize = 128 K = 5 if len(sys.argv) > 5: K = sys.argv[5] if len(sys.argv) > 6: batchsize = sys.argv[6] else: gpu_mem = '2G' if len(sys.argv) > 6: main_mem = sys.argv[6] else: main_mem = '30G' import pdb pdb.set_trace() board = tr.LockGPU() targets = np.load(targetfile) patterns = np.load(patternfile) labels = np.load(label_file) stats = np.load(statiticsFile) dist, minDist_indices, neibor_labels = Knn(patterns, targets, batchsize, K, labels, stats) dist_dir = os.path.join(output_dir, 'distance') indices_dir = os.path.join(output_dir, 'indices') labels_dir = os.path.join(output_dir, 'labels') np.save(dist_dir, dist) np.save(indices_dir, minDist_indices) np.save(labels_dir, neibor_labels) sio.savemat(os.path.join(output_dir, 'distance_mat'), {'distance': dist}) sio.savemat(os.path.join(output_dir, 'indices_mat'), {'indices': minDist_indices}) sio.savemat(os.path.join(output_dir, 'labels_mat'), {'labels': neibor_labels}) tr.FreeGPU(board)
action='store_true', help="Calculate blosum90 scores") parser.add_argument("--ncols", type=int, help="Number of multiple columns") parser.add_argument("--multmode", type=str, help="Multicol mode", default='rand') args = parser.parse_args() if not args.outf: raise ValueError('Output file not defined') if not args.train_file or not args.model_file: raise ValueError('Models and data missing') board = tr.LockGPU() model_file = args.model_file train_file = args.train_file model = dbm.DBM(model_file, train_file) trainer_pb = util.ReadOperation(train_file) dataset = os.path.basename(trainer_pb.data_proto_prefix) # Fix paths dirname = os.path.split(model.t_op.data_proto_prefix)[1] model.t_op.data_proto_prefix = os.path.join('datasets/',\ dirname) model.t_op.skip_last_piece = False model.t_op.get_last_piece = True model.t_op.randomize = False
def main(): patternFilePattern = sys.argv[1] targetFilePattern = sys.argv[2] output_dir = sys.argv[3] if len(sys.argv) > 4: label_file = sys.argv[4] statiticsFile = '/data1/ningzhang/flickr/flickr_stats.npz' batchsize = 1000 K = 5 if len(sys.argv) > 5: K = sys.argv[5] if len(sys.argv) > 6: batchsize = sys.argv[6] else: gpu_mem = '2G' if len(sys.argv) > 6: main_mem = sys.argv[6] else: main_mem = '30G' import pdb pdb.set_trace() board = tr.LockGPU() patternFiles = sorted(glob.glob(patternFilePattern)) targetFiles = sorted(glob.glob(targetFilePattern)) stats = np.load(statiticsFile) patternlist = [] m = 0 for i, patternFile in enumerate(patternFiles): patternlist.append(np.load(patternFile)) m += patternlist[i].shape[0] patterns = np.zeros((m, patternlist[0].shape[1])) pos = 0 for patternShark in patternlist: patterns[pos:pos + patternShark.shape[0], :] = patternShark pos = pos + patternShark.shape[0] pos = 0 dist_pool = np.zeros((1, 2 * K)) if len(sys.argv) > 4: labels = np.load(label_file) for targetFile in targetFiles: targets = np.load(targetFile) if len(sys.argv) > 4: dist_interm, minDist_indices_interm, neibor_labels_interm = Knn( patterns, targets, batchsize, K, labels, stats) else: dist_interm, minDist_indices_interm = Knn(patterns, targets, batchsize, K) #, stats = stats) if pos == 0: dist = np.copy(dist_interm) minDist_indices = np.copy(minDist_indices_interm) if len(sys.argv) > 4: neibor_labels = np.copy(neibor_labels_interm) else: K_new = K if K <= targets.shape[0] else targets.shape[0] if K_new < K: dist_pool = np.zeros((1, K + K_new)) for ind_1 in range(m): dist_pool[0, 0:K] = dist[ind_1, 0:K] dist_pool[0, K:K + K_new] = dist_interm[ind_1, 0:K_new] internal_compare = dist_pool.argsort().flatten() dist[ind_1, :] = dist_pool[0, internal_compare[0:K]] for j in range(K): minDist_indices[ind_1, j] = minDist_indices[ ind_1, j] if internal_compare[ j] < K else minDist_indices_interm[ind_1, j] + pos if len(sys.argv) > 4: neibor_labels[ind_1, j] = labels[minDist_indices[ind_1, j], :] pos = pos + targets.shape[0] dist_dir = os.path.join(output_dir, 'distance') indices_dir = os.path.join(output_dir, 'indices') labels_dir = os.path.join(output_dir, 'labels') np.save(dist_dir, dist) np.save(indices_dir, minDist_indices) np.save(labels_dir, neibor_labels) sio.savemat(os.path.join(output_dir, 'distance_mat'), {'distance': dist}) sio.savemat(os.path.join(output_dir, 'indices_mat'), {'indices': minDist_indices}) sio.savemat(os.path.join(output_dir, 'labels_mat'), {'labels': neibor_labels}) tr.FreeGPU(board)