def gen_y4m(net, vid, device): lr_fns = sorted( glob.glob(os.path.join(args.infer_path, '*{:05d}*.npy'.format(vid)))) lr_fns = lr_fns[::25] if (vid >= mid and len(lr_fns) > 4) else lr_fns message = 'generating vid {}, total frame {}'.format(vid, len(lr_fns)) print(message) logging(logger, message) frames = [] for lr_fn in lr_fns: lr_tensor = read_as_tensor(lr_fn).to(device) sr_tensor = net(lr_tensor).squeeze().float().permute(1, 2, 0) sr_tensor = torch.clamp((sr_tensor + mean_torch) * 255, 0, 255).round() # 这个存的位数和numpy似乎有点不一样 sr_np = sr_tensor.detach().cpu().numpy().astype( np.uint8) # ycbcr for np.load, rgb for misc.imread frames.append(sr_np) today = time.strftime('%m%d', time.localtime(time.time())) spcified_path = os.path.join(args.output_path, '{}_{}'.format(today, args.model.lower())) if not os.path.exists(spcified_path): os.makedirs(spcified_path) if vid < mid: whole_save_path = os.path.join(spcified_path, 'Youku_{:05d}_h_Res.y4m'.format(vid)) else: whole_save_path = os.path.join( spcified_path, 'Youku_{:05d}_h_Sub25_Res.y4m'.format(vid)) frames2y4m(frames, whole_save_path) return vid
def run_assembler(assembler_name, read_alignment, res_gtf, params=None): """This function calls assembler (e.g. stringtie etc). Args: assembler_name: which assembler to use read_alignment: absolute path of read alignment file res_gtf: absolute path to store assembly res params: a dictionary of key as param string (e.g. -h etc) and val as the param value. Default is None. Returns: assembler output in gtf format stored in res_gtf """ cmd = get_default_cmds(assembler_name, read_alignment, res_gtf) if params is not None: cmd = cmd + util.params_dic2str(params) # pdb.set_trace() util.run_cmd(cmd) util.logging('%s written'%(res_gtf)) return res_gtf
def extract_stat(eval_res_prefix): """Extract stat from eval_res_prefix.stats file. Args: eval_res_prefix: prefix of a set of files generated from evaluate() Returns: a dic w/ key as metric (e.g. Intron) and val as [sens, precision] possible keys: Base, Exon, Intron, Transcript etc """ stat_path = eval_res_prefix + '.stats' stat = {} # key: metric val: [sens, prec] with open(stat_path, 'r') as fi: for line in fi: if line[0] == '#': continue tokens = [t for t in line.split() if t != ''] if len(tokens) < 5: continue if tokens[1] == 'level:': stat[tokens[0]] = [float(tokens[2]), float(tokens[4])] util.logging('extracted stats:') util.logging(stat) return stat
def main(*args, **kwargs): """ """ if not HAS_MATPLOTLIB: raise ImportError( "Please install matplotlib to generate a plot from this example.") ############################################ # Step 1: Define parameters for the models # ############################################ util.mkdir_p(FLAGS.run_dir) util.logging(FLAGS) tf.logging.info('[Step 1]: Define parameters for the models') util.report_param(FLAGS) ################################ # Step 2: Define input dataset # ################################ tf.logging.info('[Step 2]: Define input dataset') if FLAGS.data_type == 'random_signal': x, y = util.random_signal(length=FLAGS.data_length) # Save the data into numpy arra np.savez_compressed(os.path.join(FLAGS.run_dir, 'input-data'), x=x, y=y) plt.plot(x, y) plt.savefig(os.path.join(FLAGS.run_dir, 'input-timeseries.jpg')) for i in tqdm(range(FLAGS.iterations)): # train the model iterations pass print("learning rate", FLAGS.learning_rate)
def extract_stat_shannon(eval_res_dir): """Extract stat from eval_res_dir/summary.log file. Args: eval_res_dir: dir of a set of files generated from evaluate_shannon() Returns: a dic w/ key as metric (e.g. aligned, detected, ref) and val int """ #TODO(bowen): better summary log stat_path = eval_res_dir + '/summary.log' stat = {} with open(stat_path, 'r') as fi: for line in fi: if line[0] == '#': continue tokens = [t for t in line.split() if t != ''] if len(tokens) == 0: continue if tokens[0] == 'total' and tokens[1] == 'seq': stat['aligned'] = int(tokens[2]) stat['detected'] = int(tokens[4]) elif tokens[0] == 'num_reference_seq': stat['ref'] = int(tokens[1]) util.logging('extracted stats:') util.logging(stat) return stat
def check_bo_tune(assembler_name, metric_type, paths): print '----- CHECK BO TUNE -----' param_default, param_range, param2str = config.parse_params_bo( paths['param_config_path']) bo = BayesianOptimization( call_and_eval_assembler(assembler_name, metric_type, paths, param2str), param_range) gp_params = {'kernel': None, 'alpha': 1e-3} bo.maximize(init_points=5, n_iter=50, kappa=5, **gp_params) msg = 'BO-tuned metric (%s) for %s:\n' % (metric_type, assembler_name) msg += 'bo.res[max]:\n' msg += str(bo.res['max']) util.logging(msg, paths['log']) msg += '\nbo tune done\n' msg += '%s written\n' % paths['log'] print msg return
def calc_metric(stat_dic, metric_type, lam=0.5): """Calibrated metric. Args: stat_dic: a dic w/ key as e.g. Intron etc and val as [sens, precision] obtained from extract_stat metric_type: indicates the way to calculated the metric. possible types: 'tr-sum' 'tr-f1' lam: a weight to adjust F1 score Returns: a fload of metric """ if metric_type == 'tr-sum': sens, prec = stat_dic['Transcript'] return float(sens + prec) elif metric_type == 'tr-f1': sens, prec = stat_dic['Transcript'] sens = (1 - lam) * sens prec = lam * prec sum_val = sens + prec if sum_val == 0: return 0 else: return float(2 * sens * prec / float(sens + prec)) else: util.logging('unknown metric_type; None to be returned.') return None
def convert(v, tp): if tp == 'float': return float(v) elif tp == 'int': return int(v) else: util.logging('unknown type: ' + tp) return None
def start(self): try: logger = open('log', 'a') for attachId in range(self.initId + 1, self.endId): attachment = get_attach(self.rootUrl + str(attachId)) if attachment: save_file(attachment) logging(logger, attachId) finally: if logger: logger.close()
def get_default_cmds(assembler_name, read_alignment, res_gtf): """default commands for a particular assembler""" if assembler_name == 'stringtie': cmd = 'stringtie %s -o %s -p 25 '%(read_alignment, res_gtf) elif assembler_name == 'cufflinks': res_dir, _ = util.parent_dir(res_gtf) cmd = 'cufflinks -o %s %s '%(res_dir, read_alignment) else: util.logging('unknown assembler: %s'%assembler_name) cmd = None return cmd
def q_learning(mode, filename=None): if mode == 'test': TOTAL_OBSERVATION = 1_000 else: TOTAL_OBSERVATION = 3_200 observe = TOTAL_OBSERVATION epsilon = INITIAL_EPSILON # init network network = init_network(observe, epsilon, mode, filename) # open up a game state to communicate with emulator game_state = game.GameState() # store the previous observations in replay memory queue = deque(maxlen=REPLAY_MEMORY) s_t0 = get_init_stack(game_state) t = 0 time0 = time.time() total_loss = 0 while (True): action_index, r_t = 0, 0 a_t = np.zeros([ACTIONS]) action_index = chose_action(network, s_t0, a_t, t, epsilon) a_t[action_index] = 1 # We reduced the epsilon gradually if epsilon > FINAL_EPSILON and t > observe: epsilon -= (INITIAL_EPSILON - FINAL_EPSILON) / TOTAL_EXPLORE s_t1, r_t, terminal = get_next_stack(game_state, a_t, s_t0) queue.append((s_t0, action_index, r_t, s_t1, terminal)) if t > observe: # only train if done observing loss, q_sa = train_network(queue, network) else: loss, q_sa = 0, 0 total_loss += loss s_t0, t = s_t1, t + 1 logging(mode, t, time0, network, observe, epsilon, action_index, r_t, q_sa, loss, total_loss, TOTAL_EXPLORE) print("Episode finished!") print("************************")
def _call_and_eval_assembler(**kwargs): params = util.kwargs_to_params_dic(kwargs, param2str) if params is not None: lam = params.get('-lam', 0.5) params.pop('-lam', None) else: lam = 0.5 assembler.run_assembler(assembler_name, paths['read_alignment'], paths['assembly_gtf'], params) evaluator.evaluate(paths['ref_gtf'], paths['assembly_gtf'], paths['eval_res_prefix']) metric_stat = evaluator.extract_stat(paths['eval_res_prefix']) #TODO(shunfu): find a good metric for BO if metric_stat == {}: metric = 0 else: metric = evaluator.calc_metric(metric_stat, metric_type, lam) # res logs util.logging('assembler:' + assembler_name, paths['log']) util.logging( 'params: default' if params is None else 'params:\n' + str(params), paths['log']) util.logging('metric stat is: %s' % str(metric_stat), paths['log']) util.logging( 'metric (%s, lam=%.2f) is %f\n' % (metric_type, lam, metric), paths['log']) return metric
def check_baseline(assembler_name, metric_type, paths): print '----- CHECK ASSEMBLY BASELINE -----' param_default, param_range, param2str = config.parse_params_bo( paths['param_config_path']) fn = call_and_eval_assembler(assembler_name, metric_type, paths) metric = fn() msg = 'Baseline metric (%s) for %s: %f\n\n' % (metric_type, assembler_name, metric) util.logging(msg, paths['log']) print msg return
def __init__(self, train_path): super(TrainDataset, self).__init__() if args.debug: args.dN = 2 self.list_hr = sorted( glob.glob(os.path.join(train_path, '*_h_GT_*.npy')))[:-args.dN:args.dR] self.list_lr = [hr.replace('h_GT', 'l') for hr in self.list_hr] print('1', self.list_lr) if args.debug: self.list_lr = self.list_lr[:32] self.list_hr = self.list_hr[:32] print('data lenght: ', len(self.list_lr) * args.repeat) logging(logger, 'data lenght: {}'.format(len(self.list_lr))) if args.load_mem == 'all': self.lr_np = [read_img(lr) for lr in self.list_lr] self.hr_np = [read_img(hr) for hr in self.list_hr] elif args.load_mem == 'lr': self.lr_np = [read_img(lr) for lr in self.list_lr] self.hr_np = []
def calc_metric_shannon(stat_dic, metric_type): """Calibrated metric. Args: stat_dic: a dic w/ key as aligned, detected and ref metric_type: indicates the way to calculated the metric. possible types: 'tr-sum' 'tr-f1' Returns: a fload of metric """ sens = float(stat_dic['aligned']) / stat_dic['ref'] prec = float(stat_dic['aligned']) / stat_dic['detected'] if metric_type == 'tr-sum': return float(sens + prec) elif metric_type == 'tr-f1': return float(2 * sens * prec / float(sens + prec)) else: util.logging('unknown metric_type; None to be returned.') return None
def test_params_dic2str(): """Convert a params_dic to string. For example, params_dic={'-a', 1} returns '-a 1' params_dic={'-b', 0.1} returns '-b 0.1' Currently only support int/float params. Args: params_dic: a dic of params Returns: params_str: a str representing params_dic """ params_dic = {'-a': 1, '-b': 0.1} params_str = util.params_dic2str(params_dic) util.logging('params_dic:') util.logging(params_dic) util.logging('params_str:') util.logging(params_str) return
] # 10-20-30-50 -> [10, 20, 30, 50] scheduler = lrs.MultiStepLR( optimizer, # 固定迭代次数,达到每次乘下面的gamma # hdnet用小迭代,dbcn用大迭代 milestones=milestones, gamma=0.5) if args.ckpt: net.load(args.ckpt) for i in range(args.repoch): scheduler.step() print('restored ckpt from {} at step: {}'.format( args.ckpt if args.ckpt else ckpt, args.repoch)) now = time.strftime('%Y.%m.%d %H:%M:%S\n', time.localtime(time.time())) logging(logger, args.message + now) psnr_best = 0 print('begin training') for epoch in range(args.repoch, args.epochs): scheduler.step() start_train = time.time() total_loss = 0 cnt = 0 for iteration, batch in enumerate(train_loader): lr = batch[0] label = batch[1] label_down4 = F.interpolate(label, scale_factor=0.25, mode="bicubic",
whole_save_path = os.path.join(spcified_path, 'Youku_{:05d}_h_Res.y4m'.format(vid)) else: whole_save_path = os.path.join( spcified_path, 'Youku_{:05d}_h_Sub25_Res.y4m'.format(vid)) frames2y4m(frames, whole_save_path) return vid if __name__ == '__main__': # logger = 'log/{}_infer.txt'.format(args.model.lower()) logger = 's4_rcan_ps32_bs16_lossYUV.txt' now = time.strftime('%Y.%m.%d %H:%M:%S\n', time.localtime(time.time())) logging(logger, args.message + now) logging(logger, 'using gpu: {}\n'.format(torch.cuda.is_available())) if args.output_path: mkdirs(args.output_path) device = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu') mean_numpy = np.array(args.mean).reshape((1, 1, -1)) mean_torch = torch.from_numpy(mean_numpy).float().to(device) #### vid setting #### begin = args.begin_id mid = args.mid_id end = args.end assert mid - begin == 5, 'full video must be 5' assert end - begin == 50, 'end id must be 50 more than actual given id' vids = [i for i in range(begin, end)] vids = [i for i in range(begin, end)]
def test_logging(): msg = 'hello world!' util.logging(msg, log='tmp.txt')
def test_kwargs_to_params_dic(): kwargs_dic = {'a': 1, 'b': 2.0} param2str = {'a': ['-a', 'int'], 'b': ['--b', 'float']} util.logging(util.kwargs_to_params_dic(kwargs_dic, param2str))
def _test_run_assembler(assembler_name, read_alignment, res_gtf): assembler.run_assembler(assembler_name, read_alignment, res_gtf) util.logging(res_gtf + ' written') return
def test_calc_metric_shannon(): stat_dic = {'aligned': 2429, 'detected': 10713, 'ref': 7703} metric_types = ['tr-sum', 'tr-f1'] for metric_type in metric_types: util.logging(str(metric_type)) util.logging(str(evaluator.calc_metric_shannon(stat_dic, metric_type)))
def parse_params_old(param_config_path): """parse params configs. The expected format: - param type default range - skip # as comments - e.g. -f float 0.1 (0.0,1.0) # bla bla bla Args: param_config_path: absolute path of param config file. Returns: a dic w/ key as param str (e.g. f) and val as [default_val] a dic w/ key as param str (e.g. f) and val as (range_min, range_max) a dic w/ key as param str (e.g. f) and val as [aug param str (e.g. -f), tp] Note: these are mained to be used by BO framework """ def convert(v, tp): if tp == 'float': return float(v) elif tp == 'int': return int(v) else: util.logging('unknown type: ' + tp) return None res_default = {} res_range = {} res_param2str = {} res_range_lis = [] param_str_list = [] with open(param_config_path, 'r') as fi: for line in fi: if line[0] == '#': continue tokens = [t for t in line.split() if t != ''] if len(tokens) < 4: continue if tokens[0][0] == '-': param_str = tokens[0] #TODO(shunfu) init '-' is not asc-ii #if param_str=='-min-frags-per-transfrag': # pdb.set_trace() #param = param_str.split('-')[-1] param = param_str[1:] tp = tokens[1] res_param2str[param] = [param_str, tp] val_default = convert(tokens[2], tp) val_range = tokens[3][1:-1].split(',') val_min = convert(val_range[0], tp) val_max = convert(val_range[1], tp) res_default[param] = [val_default] if tp == 'float': res_range[param] = tuple([val_min, val_max]) else: res_range[param] = tuple( [i for i in range(val_min, val_max + 1)]) # Creating a list of dicts; a dict per parameter res_range_lis.append({ 'name': str(param), 'type': str('continuous' if tp == 'float' else 'discrete'), 'domain': tuple([val_min, val_max]) }) param_str_list.append(param_str) util.logging('%s parsed as:' % param_config_path) util.logging('default:') util.logging(str(res_default)) util.logging('range:') util.logging(str(res_range)) util.logging('param2str:') util.logging(str(res_param2str)) #return res_default, res_range, res_param2str #pdb.set_trace() return res_range_lis, res_param2str, param_str_list, res_default, res_range
import warnings warnings.filterwarnings("ignore") import torch import torch.nn as nn # config config = util.read_config('./configs/predict.yaml') torch.manual_seed(config.seed) # log log_path = './data/results/' # log_path = config.log + '2018-10-20-12:17:22/' if not os.path.exists(log_path): os.mkdir(log_path) logging = util.logging(log_path+'res_log.txt') # 记录本次运行的记录 # checkpoint if config.checkpoint_restore: print('loading checkpoint from {}...'.format(config.checkpoint_restore)) checkpoints = torch.load(config.checkpoint_restore) # cuda use_cuda = torch.cuda.is_available() and len(config.gpus) > 0 #use_cuda = True if use_cuda: torch.cuda.set_device(config.gpus[0]) torch.cuda.manual_seed(config.seed) print('can use cuda: {}'.format(use_cuda)) # data
def test_calc_metric(): stat_dic = {'Transcript': [0.3, 0.4]} metric_types = ['tr-sum', 'tr-f1'] for metric_type in metric_types: util.logging(str(metric_type)) util.logging(str(evaluator.calc_metric(stat_dic, metric_type)))