def run_command(command, verbose=False): log = utils.buffer() out = log error = utils.quotedFile(log, "\033[31m", "\033[0m") message='' if verbose: message = "Running: '%s'"%command out = utils.tee(log, sys.stdout) error = utils.tee(error, sys.stderr) ok = utils.run(command, log=out, err=error, fatal=False, message=message) return log.output(), ok
def run_command(command, verbose=False): log = utils.buffer() out = log error = utils.quotedFile(log, "\033[31m", "\033[0m") message = '' if verbose: message = "Running: '%s'" % command out = utils.tee(log, sys.stdout) error = utils.tee(error, sys.stderr) ok = utils.run(command, log=out, err=error, fatal=False, message=message) return log.output(), ok
def run(summ_path, ref_path, rouge_args=None, verbose=False, saveto=None, eos=".", ignore_empty=False, stemming=True): s = settings.Settings() s._load() stime = time() with tempfile.TemporaryDirectory() as dirpath: sys_root, model_root = [ os.path.join(dirpath, _) for _ in ["system", "model"] ] print("Preparing documents...", end=" ") utils.mkdirs([sys_root, model_root]) ignored = utils.split_files(model_file=ref_path, system_file=summ_path, model_dir=model_root, system_dir=sys_root, eos=eos, ignore_empty=ignore_empty) print("%d line(s) ignored" % len(ignored)) print("Running ROUGE...") log_level = logging.ERROR if not verbose else None r = pyrouge.Rouge155(rouge_dir=os.path.dirname(s.data['ROUGE_path']), log_level=log_level, stemming=stemming) r.system_dir = sys_root r.model_dir = model_root r.system_filename_pattern = r's.(\d+).txt' r.model_filename_pattern = 'm.[A-Z].#ID#.txt' data_arg = "-e %s" % s.data['ROUGE_data'] if not rouge_args: rouge_args = ['-c', 95, '-r', 1000, '-n', 2, '-a'] rouge_args_str = " ".join([str(_) for _ in rouge_args]) else: rouge_args_str = rouge_args rouge_args_str = "%s %s" % (data_arg, rouge_args_str) output = r.convert_and_evaluate(rouge_args=rouge_args_str) if saveto is not None: saveto = open(saveto, 'w') utils.tee(saveto, output) print("Elapsed time: %.3f seconds" % (time() - stime))
def main(): parser = argparse.ArgumentParser( description="Calculating ROUGE score between two files (line-by-line)") parser.add_argument("summary", help="Path of summary file") parser.add_argument("reference", help="Path of references file") parser.add_argument('-v', '--verbose', action="store_true", help="""Prints ROUGE logs""") parser.add_argument('-a', '--args', help="ROUGE Arguments") parser.add_argument('-s', '--saveto', dest="saveto", help="File to save scores") parser.add_argument('-e', '--eos', dest="eos", default='.', help="""End of sentence separator (for multisentence). Default: \".\"""") args = parser.parse_args() ref_path = args.reference summ_path = args.summary rouge_args = args.args verbose = args.verbose saveto = args.saveto if saveto is not None: saveto = open(saveto, 'w') s = settings.Settings() s._load() stime = time() dirpath = tempfile.mkdtemp() sys_root, model_root = [ os.path.join(dirpath, _) for _ in ["system", "model"] ] print("Preparing documents...") utils.mkdirs([sys_root, model_root]) utils.split_files(model_file=ref_path, system_file=summ_path, model_dir=model_root, system_dir=sys_root, eos=args.eos) print("Running ROUGE...") log_level = logging.ERROR if not verbose else None r = pyrouge.Rouge155(rouge_dir=os.path.dirname(s.data['ROUGE_path']), log_level=log_level) r.system_dir = sys_root r.model_dir = model_root r.system_filename_pattern = 's.(\d+).txt' r.model_filename_pattern = 'm.[A-Z].#ID#.txt' data_arg = "-e %s" % s.data['ROUGE_data'] if not rouge_args: rouge_args = ['-c', 95, '-r', 1000, '-n', 2, '-a'] rouge_args_str = " ".join([str(_) for _ in rouge_args]) else: rouge_args_str = rouge_args rouge_args_str = "%s %s" % (data_arg, rouge_args_str) output = r.convert_and_evaluate(rouge_args=rouge_args_str) utils.tee(saveto, output) print("Elapsed time: %.3f secondes" % (time() - stime))
def train_network(cfg): try: os.mkdir(cfg.NAME) except Exception as e: print 'cannot create dir ', cfg.NAME print e if (cfg.TRAIN.EPOCH < 0): epoch = np.array([ int(i[:-len('.npz')]) for i in os.listdir(cfg.NAME + "/models") ]).max() cfg.TRAIN.EPOCH = int(epoch) print 'start epoch ', int(epoch) logger_name = cfg.NAME + '/train.log' train_loader = TieLoader('data/train_ties%d' % cfg.DATASET.T_SIZE, cfg.DATASET.TRAIN_MINR, cfg.DATASET.TRAIN_MAXR, t_size=cfg.DATASET.T_SIZE, mask_size=cfg.OUT_SIZE, sample_size=cfg.TILE_SIZE, cache_samples=cfg.DATASET.CASHE_SAMPLES) test_loader = TieLoader('data/test_ties%d' % cfg.DATASET.T_SIZE, cfg.DATASET.TEST_MINR, cfg.DATASET.TEST_MAXR, t_size=cfg.DATASET.T_SIZE, mask_size=cfg.OUT_SIZE, sample_size=cfg.TILE_SIZE, cache_samples=cfg.DATASET.CASHE_SAMPLES) data = T.tensor4(name='data') label = T.tensor3(name='label') net = make_FCN(cfg.NETWORK, data, ndim=cfg.NDIM, model_name='%s/models/%03d' % (cfg.NAME, cfg.TRAIN.EPOCH) if cfg.TRAIN.EPOCH > 0 else '', input_shape=(None, 3, cfg.TILE_SIZE, cfg.TILE_SIZE), pad='same') non_learn_params = { 'min_cov': theano.shared(1e-8), 'lr': theano.shared(np.array(1e-2, dtype=theano.config.floatX)), 'width': theano.shared(4.), 'total_grad_constraint': 10, 'histogram_bins': 100, 'use_approx_grad': True, 'ndim': cfg.NDIM, 'gm_num': cfg.GM_NUM } train_fn = make_train(net, data, label, non_learn_params) print 'train_fn compiled' test_fn = make_test(net, data, label, non_learn_params) print 'test_fn compiled' metrix = { 'aps': average_precision_score, 'int_pp_pn': lambda l, pred: get_pp_pn(l, pred) } def update_params(epoch, params): if (epoch == 0): params['lr'].set_value(5e-2) if (epoch == 4): params['lr'].set_value(5e-3) if (epoch == 10): params['lr'].set_value(1e-3) logger = open(logger_name, 'a') tee( '\n################### train network ' + cfg.NAME + ' ' + str(datetime.datetime.now()) + '################\n', logger) tee('config\n' + str(cfg), logger) tee('non_learn_params\n' + str(non_learn_params), logger) fit(cfg.NAME, net, train_fn, test_fn, train_loader, test_loader, non_learn_params, update_params=update_params, metrix=metrix, logger=logger, train_esize=cfg.TRAIN.TRAIN_EPOCH_SIZE, test_esize=cfg.TRAIN.TEST_EPOCH_SIZE, epochs=cfg.TRAIN.EPOCH_NUM, start_epoch=cfg.TRAIN.EPOCH) tee( '\n################### test network ' + cfg.NAME + ' ' + str(datetime.datetime.now()) + '################\n', logger) test_network(cfg.NAME, cfg.NETWORK, cfg.NDIM, cfg.TRAIN.EPOCH_NUM - 1, cfg.GM_NUM, im_size=(320, 240), train_size=100, test_size=400) tee('\n################### done #######################\n', logger)
def main(**kwargs): defaults = { "time": 30, "difficulty": 1, "initial_move": "child", "warmup": True, "cooldown": True, "aerobics": 0, "strength": 0, "target": "plank", "verbose":1, "memory":5, "outfile": None, } defaults.update(kwargs) if defaults["outfile"]: f = open(defaults["outfile"], "a") f.write(str(datetime.datetime.now()) + "\n") f.write(str(defaults) + "\n") else: f = None utils.speak('Beginning in') utils.countdown(3) if defaults["verbose"] >= 2: print(utils.wrapper.fill(str(defaults))) elif defaults["verbose"] >= 1: print("Workout length:", defaults['time'], "minutes.", "Beginning in:") # setup total_time = defaults['time']*60 movesGraph = moves.generateMoves(difficulty=defaults["difficulty"]) stretches.defineStretches(movesGraph, difficulty=defaults["difficulty"]) start = time.time() end = start + total_time class Times(Enum): warmup_end = start + min(max(45,total_time//15),300) halfway = start + total_time//2.4 - 30 cooldown_begin = (end - max(60, total_time//5)) if defaults['cooldown'] else end imbalance = [] prev = collections.deque([],defaults["memory"]) try: pose = movesGraph[defaults['initial_move']] except KeyError: pose = movesGraph['child'] try: #warmup if defaults["warmup"]: pose = pose(time=min(30,max(15, total_time//120+7)), imbalance=imbalance, prev=prev, verbose=defaults["verbose"], f=f) while time.time() < Times.warmup_end.value: pose = pose(imbalance=imbalance, extended=True, early=True, prev=prev, verbose=defaults["verbose"], f=f) #start slower #get me to my target: moves.linkMain(movesGraph, defaults['difficulty']) if defaults['aerobics']: strengthaerobics.linkAerobics(movesGraph, defaults["difficulty"], defaults["aerobics"]) if defaults['strength']: strengthaerobics.linkStrength(movesGraph, defaults["difficulty"], defaults["strength"]) if defaults['aerobics']: strengthaerobics.linkStrengthAerobics(movesGraph, defaults["difficulty"], defaults["strength"], defaults["aerobics"]) if defaults['warmup']: pose = fixImbalance(pose, imbalance, maxTime=max(45,total_time//12.5), prev=prev, verbose=defaults['verbose'], f=f) imbalance = moves.unlinkWarmup(movesGraph, imbalance=imbalance, difficulty=defaults["difficulty"]) try: target = movesGraph[defaults['target']] except KeyError: target = movesGraph['plank'] pose = get_me_to(pose, target, imbalance=imbalance, playLast=False, prev=prev, verbose=defaults["verbose"], f=f) if defaults["warmup"]: utils.tee("Warmup Over: " + utils.prettyTime(time.time() - start), f, say="Alright, warmup over.") pose = pose(imbalance=imbalance, prev=prev, verbose=defaults["verbose"], f=f) #starting main part of workout while time.time() < Times.halfway.value: pose = fixImbalance(pose, imbalance, maxImbalance=10 + total_time//600, maxTime=max(60,total_time//12), prev=prev, verbose=defaults["verbose"], f=f) pose = pose(imbalance=imbalance, prev=prev, verbose=defaults["verbose"], f=f) #add harder poses in here if defaults["difficulty"] >= 1: moves.linkHarder(movesGraph, defaults["difficulty"]) if defaults["strength"]: strengthaerobics.linkStrengthHarder(movesGraph, defaults["difficulty"], defaults["strength"]) pose = fixImbalance(pose, imbalance, maxTime=max(60, total_time//10), prev=prev, verbose=defaults["verbose"], f=f) try: pose = get_me_to(pose, movesGraph[defaults['target']], imbalance=imbalance, prev=prev, verbose=defaults["verbose"], f=f) except KeyError: pass if defaults["verbose"] >= 1: utils.tee("Halfway point: " + utils.prettyTime(time.time()-start), f, say="We have reached the halway point") #end adding harder poses harder = defaults["difficulty"] >= 1 while time.time() < Times.cooldown_begin.value: extendedChance = (time.time() - start)/total_time extended = random.random() < extendedChance pose = fixImbalance(pose, imbalance, maxImbalance=8+total_time//800, maxTime=max(110,total_time//10), prev=prev, verbose=defaults["verbose"], \ f=f, harder=harder) pose = pose(harder=harder, imbalance = imbalance, extended=extended, prev=prev, verbose=defaults["verbose"], f=f) moves.linkEnding(movesGraph) while time.time() < (end - max(60, total_time//10)): pose = fixImbalance(pose, imbalance, maxImbalance=max(1,total_time//800), maxTime=max(120, total_time//8), prev=prev, verbose=defaults["verbose"], f=f) pose = pose(imbalance = imbalance, prev=prev, verbose=defaults["verbose"], f=f) if defaults["cooldown"]: pose = fixImbalance(pose, imbalance, maxImbalance=1, maxTime=max(75, total_time//10+15), prev=prev, verbose=defaults["verbose"], f=f) utils.tee("Cooldown begins: " + utils.prettyTime(time.time() - start), f, say="Cooldown begins") stretches.linkCooldown(movesGraph, difficulty=defaults["difficulty"]) if defaults["strength"]: strengthaerobics.linkStrengthCooldown(movesGraph,difficulty=defaults["difficulty"], strength = defaults["strength"]) if defaults["aerobics"]: strengthaerobics.linkAerobicsCooldown(movesGraph,difficulty=defaults["difficulty"], aerobics = defaults["aerobics"]) pose = get_me_to(pose, movesGraph['wheel'], imbalance=imbalance, prev=prev, verbose=defaults['verbose'], f=f) pose = fixImbalance(pose, imbalance, maxImbalance=1, maxTime=max(60, total_time//10), prev=prev, verbose=defaults['verbose'], f=f) while time.time() < (end-max(30, total_time//10)) if defaults["cooldown"] else end: #pose = pose(imbalance=imbalance, extended=True, prev=prev, verbose=defaults['verbose'], f=f) pose = fixImbalance(pose, imbalance, maxImbalance=1, maxTime=max(30, total_time//10), prev=prev, verbose=defaults['verbose'], f=f) if defaults['cooldown']: moves.linkSavasana(movesGraph, difficulty=defaults['difficulty']) pose = fixImbalance(pose, imbalance, maxImbalance=1, maxTime=max(30, total_time//10), prev=prev, verbose=defaults['verbose'], f=f) pose = get_me_to(pose, movesGraph['savasana'], imbalance=imbalance, prev=prev, verbose=defaults['verbose'], f=f) except (KeyboardInterrupt, BrokenPipeError): moves.linkSavasana(movesGraph, difficulty=defaults['difficulty']) pose = get_me_to(pose, movesGraph['savasana'], imbalance=imbalance, prev=prev, verbose=defaults['verbose'], f=f) finally: final_time = utils.prettyTime(time.time() - start) utils.tee('\nTotal Time: %s' % final_time, f=f, say='Done! Total time was %s' % final_time.replace('(','').replace(')','')) if f: f.close() sys.stdout.write(colorama.Style.RESET_ALL) print(utils.wrapper.fill(str(imbalance))) return imbalance