def run(self): try: input_files = get_input_files(self.input_path) input_files.insert(0, get_pioneer_file(self.ds_name)) gfs_file_path = get_supported_datasets()[self.ds_name] num_processes = 2 cur = get_db_cur(self.con_details) for schema in [self.schema, self.schema + '_tmp']: if not create_schema(cur, schema): print 'Failed to create schema %s' % schema self.quit(1) return pg_source = 'PG:dbname=\'%s\' host=\'%s\' port=\'%d\' active_schema=%s user=\'%s\'' % \ (self.con_details['database'], self.con_details['host'], self.con_details['port'], self.schema + '_tmp', self.con_details['user']) for arg in build_args(input_files, gfs_file_path, pg_source): self.im.add(arg) print 'Importing...' self.im.start(num_processes) except: print print 'Translation failed:' print print '%s\n\n' % traceback.format_exc() print self.quit(1) return
def main(): """Main workflow""" args = utils.build_args(argparse.ArgumentParser()) utils.init_logger(args.model_file) assert torch.cuda.is_available() torch.cuda.set_device(args.gpuid) utils.init_random(args.seed) utils.set_params(args) logger.info("Config:\n%s", pformat(vars(args))) fields = utils.build_fields() logger.info("Fields: %s", fields.keys()) logger.info("Load %s", args.train_file) train_data = LMDataset(fields, args.train_file, args.sent_length_trunc) logger.info("Training sentences: %d", len(train_data)) logger.info("Load %s", args.valid_file) val_data = LMDataset(fields, args.valid_file, args.sent_length_trunc) logger.info("Validation sentences: %d", len(val_data)) fields["sent"].build_vocab(train_data) train_iter = utils.build_dataset_iter(train_data, args) val_iter = utils.build_dataset_iter(val_data, args, train=False) if args.resume and os.path.isfile(args.checkpoint_file): logger.info("Resume training") logger.info("Load checkpoint %s", args.checkpoint_file) checkpoint = torch.load(args.checkpoint_file, map_location=lambda storage, loc: storage) es_stats = checkpoint["es_stats"] args = utils.set_args(args, checkpoint) else: checkpoint = None es_stats = ESStatistics(args) model = utils.build_model(fields, args, checkpoint) logger.info("Model:\n%s", model) optimizer = utils.build_optimizer(model, args, checkpoint) try_train_val(fields, model, optimizer, train_iter, val_iter, es_stats, args)
def build(json_payload): args = build_args(json_payload) docker_exec = find_executable() cmd = '{} {}'.format(docker_exec, args) out, err = docker_run(cmd) return out
import argparse import os.path as ops import torch import torch.utils.data import torch.nn as nn import torch.optim as optim from torch.autograd import Variable from models_lite import fusion_VAE from sklearn import preprocessing import numpy as np from utils import cox, visualize, build_args import random import pandas as pd import os args = build_args() args.cuda = not args.no_cuda and torch.cuda.is_available() random.seed(0) torch.manual_seed(args.seed) if args.cuda: torch.cuda.manual_seed(args.seed) kwargs = {'num_workers': 1, 'pin_memory': True} if args.cuda else {} def tcga_load(disease): def read(fp): print(fp) tmp = pd.read_csv(fp, sep='\t') #return np.loadtxt(fp, delimiter='\t', skiprows=1, dtype=bytes)[:, 1:-1].T.astype(float) #print(tmp)