def main(): args = arguments.parse() if args is None: return set_seed(args) logger = initialize_logger(args) logger.info(f'Arguments:\n{pformat(vars(args))}') # 调用vars(args)的format函数,得到字符串? field, save_dict = None, None # tuple unpacking if args.load is not None: logger.info(f'Loading field from {os.path.join(args.save, args.load)}') save_dict = torch.load(os.path.join(args.save, args.load)) field = save_dict['field'] # field is the value in the 'field' key of the data field, train_sets, val_sets = prepare_data(args, field, logger) run_args = (field, train_sets, val_sets, save_dict) if len(args.gpus) > 1: logger.info(f'Multiprocessing') # 多gpu mp = Multiprocess(run, args) mp.run(run_args) else: logger.info(f'Processing') # f string of python 3.6 run(args, run_args, world_size=args.world_size)
def test_do_tasks_after_error_raises_exc(): def f(): raise ValueError('error') m = Multiprocess() m.add_tasks(f, [()]) with pytest.raises(MultiprocessProcessException): m.do_tasks() with pytest.raises(MultiprocessClosed): m.do_tasks() m.close()
def test_do_tasks_twice(): def f(q): q.push(1) q = Queue() m = Multiprocess() m.add_tasks(f, [(q, )]) m.do_tasks() assert (q.qsize() == 1) m.add_tasks(f, [(q, )]) m.do_tasks() assert (q.qsize() == 2)
def test_do_tasks_after_close_raises_exc(): m = Multiprocess() f = lambda: 1 m.add_tasks(f, [()]) m.close() with pytest.raises(MultiprocessClosed): m.do_tasks()
def test_empty_fn(): def f(): pass m = Multiprocess() m.add_tasks(f, [()]) m.do_tasks() m.close()
def test_lambda_pickling(): f = lambda q: q.push(1) q = Queue() m = Multiprocess() m.add_tasks(f, [(q, )]) m.do_tasks() m.close() assert (q.qsize() == 1) assert (q.pop() == 1) assert (q.qsize() == 0)
def test_fn_raises_exc_is_caught(): m = Multiprocess() def f(): raise ValueError('unique') m.add_tasks(f, [()]) with pytest.raises(MultiprocessProcessException) as excinfo: m.do_tasks() assert 'unique' in str(excinfo.value) m.close()
def test_queue(): q = Queue() m = Multiprocess() def f(q): q.push(1) m.add_tasks(f, [(q, )]) m.do_tasks() m.close() assert (q.qsize() == 1) assert (q.pop() == 1) assert (q.qsize() == 0)
def test_stdout(capsys): m = Multiprocess() sys.stderr.write("\r\n") m.add_tasks(lambda: 1, [()]) m.do_tasks() m.close() out, err = capsys.readouterr() assert (out == '') spl = [v.split('\r')[-1] for v in err.split('\n')] while len(spl) and spl[0] == '': spl = spl[1:] assert (len(spl) == 2) assert (spl[0].startswith('100%')) assert (spl[1] == '')
def test_disable_load_bar_works(capsys): m = Multiprocess(show_loading_bar=False) m.add_tasks(lambda: 1, [()]) m.close() out, err = capsys.readouterr() assert (out == '') assert (err == '')
def main(): args = arguments.parse() if args is None: return set_seed(args) logger = initialize_logger(args) logger.info(f'Arguments:\n{pformat(vars(args))}') field, save_dict = None, None if args.load is not None: logger.info(f'Loading field from {os.path.join(args.save, args.load)}') save_dict = torch.load(os.path.join(args.save, args.load)) field = save_dict['field'] field, train_sets, val_sets = prepare_data(args, field, logger) run_args = (field, train_sets, val_sets, save_dict) if len(args.gpus) > 1: logger.info(f'Multiprocessing') mp = Multiprocess(run, args) mp.run(run_args) else: logger.info(f'Processing') run(args, run_args, world_size=args.world_size)
def main(): args = arguments.parse() if args is None: return set_seed(args) logger = initialize_logger(args) logger.info(f'Arguments:\n{pformat(vars(args))}') field, save_dict = None, None if args.load is not None: logger.info(f'Loading field from {os.path.join(args.save, args.load)}') save_dict = torch.load(os.path.join(args.save, args.load)) field = save_dict['field'] field, train_sets, val_sets = prepare_data(args, field, logger) run_args = (field, train_sets, val_sets, save_dict) if len(args.gpus) > 1: logger.info(f'Multiprocessing') mp = Multiprocess(run, args) mp.run(run_args) else: logger.info(f'Processing') run(args, run_args, world_size=args.world_size)
def test_multiple_tasks(): def f(q, num): q.push(num) q = Queue() m = Multiprocess() arr = range(100) m.add_tasks(f, [( q, num, ) for num in arr]) m.do_tasks() m.close() assert (q.qsize() == len(arr)) l = [q.pop() for v in range(q.qsize())] assert (set(l) == set(arr))
def test_close_does_calls_do_task(): def f(q): q.push(1) q = Queue() m = Multiprocess() m.add_tasks(f, [(q, )]) m.close() assert (q.qsize() == 1) assert (q.pop() == 1) assert (q.qsize() == 0)
def multiprocess(fn, arr_of_args, **kwargs): """Execute several tasks in parallel. Requires a function `fn` and an array of argument tuples `arr_of_args`, each representing a call to the function. Additionally, you can provide arguments the same as you would with `Multiprocess` Example >>> # exec f(x) and f(y) in parallel >>> multiprocess(f, [(x,), (y,)]) If you don't want a loading bar >>> multiprocess(f, [(x,), (y,)], show_loading_bar=False) """ m = Multiprocess(**kwargs) m.add_tasks(fn, arr_of_args) m.do_tasks() m.close()
def init_opt(args, model): opt = None if args.transformer_lr: opt = torch.optim.Adam(model.params, betas=(0.9, 0.98), eps=1e-9) else: opt = torch.optim.Adam(model.params, betas=(args.beta0, 0.999)) return opt if __name__ == '__main__': args = arguments.parse() set_seed(args) logger = initialize_logger(args) logger.info(f'Arguments:\n{pformat(vars(args))}') field, save_dict = None, None if args.load is not None: logger.info(f'Loading field from {os.path.join(args.save, args.load)}') save_dict = torch.load(os.path.join(args.save, args.load)) field = save_dict['field'] field, train_sets, val_sets = prepare_data(args, field, logger) run_args = (field, train_sets, val_sets, save_dict) if len(args.gpus) > 1: logger.info(f'Multiprocessing') mp = Multiprocess(run, args) mp.run(run_args) else: logger.info(f'Processing') run(args, run_args, world_size=args.world_size)
def test_close_twice_doesnt_raise_exc(): m = Multiprocess() m.close() m.close()
def test_no_loading_bar(): m = Multiprocess(show_loading_bar=False) f = lambda: 1 m.add_tasks(f, [()]) m.close()
def init_opt(args, model): opt = None if args.transformer_lr: opt = torch.optim.Adam(model.params, betas=(0.9, 0.98), eps=1e-9) else: opt = torch.optim.Adam(model.params, betas=(args.beta0, 0.999)) return opt if __name__ == '__main__': args = arguments.parse() set_seed(args) logger = initialize_logger(args) logger.info(f'Arguments:\n{pformat(vars(args))}') field, save_dict = None, None if args.load is not None: logger.info(f'Loading field from {os.path.join(args.save, args.load)}') save_dict = torch.load(os.path.join(args.save, args.load)) field = save_dict['field'] field, train_sets, val_sets = prepare_data(args, field, logger) run_args = (field, train_sets, val_sets, save_dict) if len(args.gpus) > 1: logger.info(f'Multiprocessing') mp = Multiprocess(run, args) mp.run(run_args) else: logger.info(f'Processing') run(args, run_args, world_size=args.world_size)