def run(self):
try:
input_files = get_input_files(self.input_path)
input_files.insert(0, get_pioneer_file(self.ds_name))
gfs_file_path = get_supported_datasets()[self.ds_name]
num_processes = 2
cur = get_db_cur(self.con_details)
for schema in [self.schema, self.schema + '_tmp']:
if not create_schema(cur, schema):
print 'Failed to create schema %s' % schema
self.quit(1)
return
pg_source = 'PG:dbname=\'%s\' host=\'%s\' port=\'%d\' active_schema=%s user=\'%s\'' % \
(self.con_details['database'], self.con_details['host'], self.con_details['port'],
self.schema + '_tmp', self.con_details['user'])
for arg in build_args(input_files, gfs_file_path, pg_source):
self.im.add(arg)
print 'Importing...'
self.im.start(num_processes)
except:
print
print 'Translation failed:'
print
print '%s\n\n' % traceback.format_exc()
print
self.quit(1)
return
def main():
"""Main workflow"""
args = utils.build_args(argparse.ArgumentParser())
utils.init_logger(args.model_file)
assert torch.cuda.is_available()
torch.cuda.set_device(args.gpuid)
utils.init_random(args.seed)
utils.set_params(args)
logger.info("Config:\n%s", pformat(vars(args)))
fields = utils.build_fields()
logger.info("Fields: %s", fields.keys())
logger.info("Load %s", args.train_file)
train_data = LMDataset(fields, args.train_file, args.sent_length_trunc)
logger.info("Training sentences: %d", len(train_data))
logger.info("Load %s", args.valid_file)
val_data = LMDataset(fields, args.valid_file, args.sent_length_trunc)
logger.info("Validation sentences: %d", len(val_data))
fields["sent"].build_vocab(train_data)
train_iter = utils.build_dataset_iter(train_data, args)
val_iter = utils.build_dataset_iter(val_data, args, train=False)
if args.resume and os.path.isfile(args.checkpoint_file):
logger.info("Resume training")
logger.info("Load checkpoint %s", args.checkpoint_file)
checkpoint = torch.load(args.checkpoint_file,
map_location=lambda storage, loc: storage)
es_stats = checkpoint["es_stats"]
args = utils.set_args(args, checkpoint)
else:
checkpoint = None
es_stats = ESStatistics(args)
model = utils.build_model(fields, args, checkpoint)
logger.info("Model:\n%s", model)
optimizer = utils.build_optimizer(model, args, checkpoint)
try_train_val(fields, model, optimizer, train_iter, val_iter, es_stats,
args)
def build(json_payload):
args = build_args(json_payload)
docker_exec = find_executable()
cmd = '{} {}'.format(docker_exec, args)
out, err = docker_run(cmd)
return out
import argparse
import os.path as ops
import torch
import torch.utils.data
import torch.nn as nn
import torch.optim as optim
from torch.autograd import Variable
from models_lite import fusion_VAE
from sklearn import preprocessing
import numpy as np
from utils import cox, visualize, build_args
import random
import pandas as pd
import os
args = build_args()
args.cuda = not args.no_cuda and torch.cuda.is_available()
random.seed(0)
torch.manual_seed(args.seed)
if args.cuda:
torch.cuda.manual_seed(args.seed)
kwargs = {'num_workers': 1, 'pin_memory': True} if args.cuda else {}
def tcga_load(disease):
def read(fp):
print(fp)
tmp = pd.read_csv(fp, sep='\t')
#return np.loadtxt(fp, delimiter='\t', skiprows=1, dtype=bytes)[:, 1:-1].T.astype(float)
#print(tmp)