def test_exception_handler(): from minghu6.algs.decorator import exception_handler def myhandler(e): print('Caught exception!', e) # Examples # Specify exceptions in order, first one is handled first # last one last. @exception_handler(myhandler, (ZeroDivisionError,)) def f1(): 1 / 0 @exception_handler() def f3(*pargs): l = pargs return l.index(10) buff = StringIO() with redirect_stdout(buff): f1() assert buff.getvalue() == 'Caught exception! division by zero\n' buff = StringIO() with redirect_stdout(buff): f3() assert buff.getvalue() == 'ValueError : tuple.index(x): x not in tuple\n'
def test_pants_fitter(self): with io.StringIO() as buf, redirect_stdout(buf): with patch("builtins.input", side_effect=["Ziggy", 34, 2, "fancy"]): self.assertIsNone(pants_fitter()) self.assertEquals( "Greetings Ziggy welcome to Pants-R-Us\n" + "2 pairs of large fancy pants: $ 200\n", buf.getvalue() ) with io.StringIO() as buf, redirect_stdout(buf): with patch("builtins.input", side_effect=["Elmer", 31, 10, "regular"]): pants_fitter() self.assertEquals( "Greetings Elmer welcome to Pants-R-Us\n" + "10 pairs of medium regular pants: $ 400\n", buf.getvalue(), ) with io.StringIO() as buf, redirect_stdout(buf): with patch("builtins.input", side_effect=["Minnie", 12, 1, "fancy"]): pants_fitter() self.assertEquals( "Greetings Minnie welcome to Pants-R-Us\n" + "1 pairs of small fancy pants: $ 100\n", buf.getvalue() ) with io.StringIO() as buf, redirect_stdout(buf): with patch("builtins.input", side_effect=["Henry", 30, 4, "regular"]): pants_fitter() self.assertEquals( "Greetings Henry welcome to Pants-R-Us\n" + "4 pairs of medium regular pants: $ 160\n", buf.getvalue(), ) with io.StringIO() as buf, redirect_stdout(buf): with patch("builtins.input", side_effect=["Ali Smith", -10, 90, "fancy"]): pants_fitter() self.assertEquals( "Greetings Ali Smith welcome to Pants-R-Us\n" + "90 pairs of small fancy pants: $ 9000\n", buf.getvalue(), )
def test_create_thumbnails(product_with_image, settings): sizeset = settings.VERSATILEIMAGEFIELD_RENDITION_KEY_SETS["products"] product_image = product_with_image.images.first() # There's no way to list images created by versatile prewarmer # So we delete all created thumbnails/crops and count them log_deleted_images = io.StringIO() with redirect_stdout(log_deleted_images): product_image.image.delete_all_created_images() log_deleted_images = log_deleted_images.getvalue() # Image didn't have any thumbnails/crops created, so there's no log assert not log_deleted_images create_thumbnails(product_image.pk, ProductImage, "products") log_deleted_images = io.StringIO() with redirect_stdout(log_deleted_images): product_image.image.delete_all_created_images() log_deleted_images = log_deleted_images.getvalue() for image_name, method_size in sizeset: method, size = method_size.split("__") if method == "crop": assert product_image.image.crop[size].name in log_deleted_images elif method == "thumbnail": assert ( product_image.image.thumbnail[size].name in log_deleted_images ) # noqa
def test_try_catch(self): # 69 self.assertEval('(try* 123 (catch* e 456))', self.env, '123') f = StringIO() with redirect_stdout(f): res = pymal.rep('(try* (abc 1 2)' ' (catch* exc (prn "exc is:" exc)))', self.env) self.assertEqual(res, 'nil') self.assertEqual(f.getvalue(), '"exc is:" Symbol value is void: \'abc\'\n') g = StringIO() with redirect_stdout(g): res = pymal.rep('(try* (throw (list 1 2 3))' ' (catch* exc (do (prn "err:" exc) 7)))', self.env) self.assertEqual(res, '7') self.assertEqual(g.getvalue(), '"err:" (1 2 3)\n') h = StringIO() with redirect_stdout(h): res = pymal.rep('(try* (throw "my exception")' ' (catch* exc (do (prn "err:" exc) 7)))', self.env) self.assertEqual(res, '7') self.assertEqual(h.getvalue(), '"err:" my exception\n')
def test_get_F_cm_Guo_continuous_and_discrete(self): dcmt = DiscreteCMTransforms(e_D2Q9, u2D, F2D, rho) F_cm_Guo_disc = get_mom_vector_from_discrete_def(dcmt.get_force_Guo, discrete_transform=dcmt.get_cm, moments_order=moments_dict['D2Q9'], serial_run=True) from SymbolicCollisions.core.ContinousCMTransforms import \ ContinousCMTransforms, get_mom_vector_from_continuous_def from SymbolicCollisions.core.cm_symbols import \ F3D, dzeta3D, u3D ccmt = ContinousCMTransforms(dzeta3D, u3D, F3D, rho) F_cm_Guo_cont = get_mom_vector_from_continuous_def(ccmt.get_force_Guo, continuous_transformation=ccmt.get_cm, moments_order=moments_dict['D2Q9'], serial_run=True) # print_as_vector(F_cm_Guo_cont, 'F_cm') results = [F_cm_Guo_disc, F_cm_Guo_cont] f = io.StringIO() with redirect_stdout(f): print_as_vector(hardcoded_F_cm_Guo_hydro_LB_velocity_based_D2Q9, 'F_cm') expected_result = f.getvalue() for result in results: f = io.StringIO() with redirect_stdout(f): print_as_vector(result, 'F_cm') out = f.getvalue() assert out == expected_result
def test_timer(): from minghu6.algs.decorator import timer # Test on functions @timer(trace=True, label='[CCC]==>') def listcomp(N): # Like listcomp = timer(...)(listcomp) return [x * 2 for x in range(N)] # listcomp(...) triggers onCall @timer(trace=True, label='[MMM]==>', unit='s') def mapcall(N): return list(map((lambda x: x * 2), range(N))) # list() for 3.0 views for func in (listcomp, mapcall): buff = StringIO() with redirect_stdout(buff): result = func(5) # Time for this call, all calls, return value if func is listcomp: assert buff.getvalue().startswith('[CCC]==>listcomp') elif func is mapcall: assert buff.getvalue().startswith('[MMM]==>mapcall') assert result == [0, 2, 4, 6, 8] assert isinstance(func.alltime, (float, int)) # Total time for all calls # Test on methods class Person: def __init__(self, name, pay): self.name = name self.pay = pay @timer() def give_raise(self, percent): # giveRaise = timer()(giveRaise) self.pay *= (1.0 + percent) # tracer remembers giveRaise @timer(label='**') def last_name(self): # lastName = timer(...)(lastName) return self.name.split()[-1] bob = Person('Bob Smith', 50000) sue = Person('Sue Jones', 100000) buff = StringIO() with redirect_stdout(buff): bob.give_raise(.10) assert buff.getvalue().startswith('give_raise') print(buff.getvalue()) with redirect_stdout(buff): sue.give_raise(.20) # runs onCall(sue, .10) assert (int(bob.pay), int(sue.pay)) == (55000, 120000) buff = StringIO() with redirect_stdout(buff): assert ((bob.last_name(), sue.last_name()) == ('Smith', 'Jones')) assert buff.getvalue().startswith('**last_name')
def run_online_mode(args): if args.n == 1: registry = look_up(args.registry)() finish = None if args.msgpack: newin = os.fdopen(sys.stdin.fileno(), "rb", buffering=0) input_stream = MsgPackObjectReader(registry, newin, deref=True) output_stream = MsgPackObjectWriter(registry, sys.stdout.buffer, host=args.name) else: input_stream = JSONObjectReader(registry, sys.stdin, deref=True) output_stream = JSONObjectWriter(registry, sys.stdout, host=args.name) # run the init function if it is given if args.init: with redirect_stdout(sys.stderr): look_up(args.init)() if args.finish: finish = look_up(args.finish) for msg in input_stream: if isinstance(msg, JobMessage): key, job = msg elif isinstance(msg, tuple): key, job = msg else: continue if args.jobdirs: # make a directory os.mkdir("noodles-{0}".format(key.hex)) # enter it os.chdir("noodles-{0}".format(key.hex)) if args.verbose: print( "worker: ", job.foo.__name__, job.bound_args.args, job.bound_args.kwargs, file=sys.stderr, flush=True, ) with redirect_stdout(sys.stderr): result = run_job(key, job) if args.verbose: print("result: ", result, file=sys.stderr, flush=True) if args.jobdirs: # parent directory os.chdir("..") output_stream.send(result) if finish: finish()
def test_take_damage(self): with io.StringIO() as buf, redirect_stdout(buf): self.assertIsNone(self.rich.take_damage(5)) self.assertEqual(5, self.rich.hit_points) self.assertEqual('\tRich has 5 hit points remaining.\n', buf.getvalue()) with io.StringIO() as buf, redirect_stdout(buf): self.rich.take_damage(5) self.assertEqual(0, self.rich.hit_points) self.assertEqual('\tAlas, Rich has fallen!\n', buf.getvalue())
def test_check_for_winner(self): self.assertIsNotNone(check_for_winner("Elmer", 30)) with io.StringIO() as buf, redirect_stdout(buf): self.assertTrue(check_for_winner("Ziggy",50)) self.assertEqual("THE WINNER IS: Ziggy!!!!!\n", buf.getvalue()) with io.StringIO() as buf, redirect_stdout(buf): self.assertTrue(check_for_winner("Ziggy",51)) self.assertEqual("THE WINNER IS: Ziggy!!!!!\n", buf.getvalue()) with io.StringIO() as buf, redirect_stdout(buf): self.assertFalse(check_for_winner("Elmer", 49)) self.assertEqual("", buf.getvalue())
def test_process_standard_options_for_setup_help(self): f = StringIO() with redirect_stdout(f): process_standard_options_for_setup_help('--help-commands') self.assertIn('Commands processed by pyquickhelper:', f.getvalue()) f = StringIO() with redirect_stdout(f): process_standard_options_for_setup_help(['--help', 'unittests']) self.assertIn('-f file', f.getvalue()) f = StringIO() with redirect_stdout(f): process_standard_options_for_setup_help(['--help', 'clean_space']) self.assertIn('clean unnecessary spaces', f.getvalue())
def test_doctor(self): with io.StringIO() as buf, redirect_stdout(buf): with patch("builtins.input", side_effect=[119, 79]): self.assertIsNone(doctor()) self.assertEquals("Your blood pressure is normal.\n", buf.getvalue()) with io.StringIO() as buf, redirect_stdout(buf): with patch("builtins.input", side_effect=[133, 79]): doctor() self.assertEquals("Your blood pressure is high.\n", buf.getvalue()) with io.StringIO() as buf, redirect_stdout(buf): with patch("builtins.input", side_effect=[120, 80]): doctor() self.assertEquals("Your blood pressure is high.\n", buf.getvalue())
def test_word_length(self): with io.StringIO() as buf, redirect_stdout(buf): self.assertIsNone(word_length("liversnaps", 7)) self.assertEqual("Longer than 7 characters: liversnaps\n", buf.getvalue()) with io.StringIO() as buf, redirect_stdout(buf): word_length("earwax", 5) self.assertEqual("Longer than 5 characters: earwax\n", buf.getvalue()) with io.StringIO() as buf, redirect_stdout(buf): word_length("chickenfat", 10) self.assertEqual("Exactly 10 characters: chickenfat\n", buf.getvalue()) with io.StringIO() as buf, redirect_stdout(buf): word_length("Gross!", 13) self.assertEqual("Shorter than 13 characters: Gross!\n", buf.getvalue())
def test_play_turn(self): random.seed(87) #[2, 6, 2, 5, 1, 3, 3, 6, 5, 6, 3, 6, 1] with io.StringIO() as buf, redirect_stdout(buf): with patch('builtins.input', side_effect = ['y','y', 'n']): self.assertEquals(10, play_turn("Henry")) self.assertEquals("---------- Henry's turn ----------\n" + \ "\t<<< Henry rolls a 2 >>>\n" + \ "\tPoints: 2\n\t<<< Henry rolls a 6 >>>\n\tPoints: 8\n" + \ "\t<<< Henry rolls a 2 >>>\n\tPoints: 10\n", buf.getvalue()) with io.StringIO() as buf, redirect_stdout(buf): with patch('builtins.input', side_effect = ['y','y']): self.assertEquals(0, play_turn("Johnny")) self.assertEquals("---------- Johnny's turn ----------\n" + \ "\t<<< Johnny rolls a 5 >>>\n" + \ "\tPoints: 5\n\t<<< Johnny rolls a 1 >>>\n" + \ "\t!!! PIG! No points earned, sorry Johnny !!!\n", buf.getvalue()) with io.StringIO() as buf, redirect_stdout(buf): with patch('builtins.input', side_effect = ['n']): self.assertEquals(3, play_turn("Henry")) self.assertEquals("---------- Henry's turn ----------\n" + \ "\t<<< Henry rolls a 3 >>>\n\tPoints: 3\n", buf.getvalue()) # testing prompts sys.stdin = open('play_turn_in.txt') with io.StringIO() as buf, redirect_stdout(buf): play_turn("Johnny") out_text = "---------- Johnny's turn ----------\n" + \ "\t<<< Johnny rolls a 3 >>>\n" + \ "\tPoints: 3\n" + \ "Roll again, Johnny? (Y/N) " + \ "\t<<< Johnny rolls a 6 >>>\n" + \ "\tPoints: 9\n" + \ "Roll again, Johnny? (Y/N) " + \ "\t<<< Johnny rolls a 5 >>>\n" + \ "\tPoints: 14\n" + \ "Roll again, Johnny? (Y/N) " + \ "\t<<< Johnny rolls a 6 >>>\n" + \ "\tPoints: 20\n" + \ "Roll again, Johnny? (Y/N) " + \ "\t<<< Johnny rolls a 3 >>>\n" + \ "\tPoints: 23\n" + \ "Roll again, Johnny? (Y/N) " + \ "\t<<< Johnny rolls a 6 >>>\n" + \ "\tPoints: 29\n" + \ "Roll again, Johnny? (Y/N) " + \ "\t<<< Johnny rolls a 1 >>>\n" + \ "\t!!! PIG! No points earned, sorry Johnny !!!\n" + \ "(enter to continue)" self.assertEqual(out_text, buf.getvalue()) sys.stdin = sys.__stdin__
def test_print_l(self): from contextlib import redirect_stdout f = io.StringIO() with redirect_stdout(f): print_l(["lines"]) s = f.getvalue() self.assertEqual(s, "lines\n") f = io.StringIO() with redirect_stdout(f): print_l(["xx","xx","xx"]) s = f.getvalue() self.assertEqual(s, "xx\nxx\nxx\n")
def test_stop_light(self): with io.StringIO() as buf, redirect_stdout(buf): self.assertIsNone(stop_light("green", 61)) self.assertEqual("yellow\n", buf.getvalue()) with io.StringIO() as buf, redirect_stdout(buf): stop_light("green", 60) self.assertEqual("green\n", buf.getvalue()) with io.StringIO() as buf, redirect_stdout(buf): stop_light("green", 59) self.assertEqual("green\n", buf.getvalue()) with io.StringIO() as buf, redirect_stdout(buf): stop_light("yellow", 5) self.assertEqual("yellow\n", buf.getvalue()) with io.StringIO() as buf, redirect_stdout(buf): stop_light("yellow", 6) self.assertEqual("red\n", buf.getvalue()) with io.StringIO() as buf, redirect_stdout(buf): stop_light("red", 12) self.assertEqual("red\n", buf.getvalue()) with io.StringIO() as buf, redirect_stdout(buf): stop_light("red", 56) self.assertEqual("green\n", buf.getvalue()) with io.StringIO() as buf, redirect_stdout(buf): stop_light("red", 54) self.assertEqual("red\n", buf.getvalue())
def test_cm_vector_from_continuous_def(self): # this test runs long without output and CI may consider it as a timeout :/ ccmt = ContinousCMTransforms(dzeta3D, u3D, F3D, rho) lattices = [ 'D2Q9', 'D3Q19', 'D2Q9', 'D2Q9', 'D3Q19', ] functions = [ ccmt.get_Maxwellian_DF, ccmt.get_Maxwellian_DF, ccmt.get_force_Guo, ccmt.get_force_He_MB, ccmt.get_force_He_MB, ] expected_results = [ hardcoded_cm_eq_compressible_D2Q9, hardcoded_cm_eq_compressible_D3Q19, hardcoded_F_cm_Guo_hydro_LB_velocity_based_D2Q9, hardcoded_F_cm_hydro_density_based_D2Q9, hardcoded_F_cm_hydro_density_based_D3Q19, ] for fun, lattice, expected_result in zip(functions, lattices, expected_results): cm_eq = get_mom_vector_from_continuous_def(fun, continuous_transformation=ccmt.get_cm, moments_order=moments_dict[lattice], serial_run=True ) # print("------------\n\n") # print_as_vector(cm_eq, 'CM') # print_as_vector(expected_result, 'CM_expected') # print("------------\n\n") f = io.StringIO() with redirect_stdout(f): print_as_vector(cm_eq, 'cm_eq') out = f.getvalue() f2 = io.StringIO() with redirect_stdout(f2): print_as_vector(expected_result, 'cm_eq') ccode_expected_result = f2.getvalue() assert ccode_expected_result == out
def test(self): stdout = StringIO() with redirect_stdout(stdout): self.progress.send(self.input1) self.assertEqual(stdout.getvalue(), self.expected1) stdout = StringIO() with redirect_stdout(stdout): self.progress.send(self.input2) self.assertEqual(stdout.getvalue(), self.expected2) stdout = StringIO() with redirect_stdout(stdout): self.progress.send(self.input3) self.assertEqual(stdout.getvalue(), self.expected3)
def test_apply_with_core_functions(self): # 72 self.assertEval('(apply + (list 2 3))', self.env, '5') self.assertEval('(apply + 4 (list 5))', self.env, '9') f = StringIO() with redirect_stdout(f): res = pymal.rep('(apply prn (list 1 2 "3" (list)))', self.env) self.assertEqual(res, 'nil') self.assertEqual(f.getvalue(), '1 2 "3" ()\n') g = StringIO() with redirect_stdout(g): res = pymal.rep('(apply prn 1 2 (list "3" (list)))', self.env) self.assertEqual(res, 'nil') self.assertEqual(g.getvalue(), '1 2 "3" ()\n')
def handle_worker(args): """usage: {program} worker [options] <module-path> <operator> <occurrence> [<config-file>] Run a worker process which performs a single mutation and test run. Each worker does a minimal, isolated chunk of work: it mutates the <occurence>-th instance of <operator> in <module-path>, runs the test suite defined in the configuration, prints the results, and exits. Normally you won't run this directly. Rather, it will be launched by an execution engine. However, it can be useful to run this on its own for testing and debugging purposes. options: --keep-stdout Do not squelch stdout """ config = load_config(args.get('<config-file>')) with open(os.devnull, 'w') as devnull: with redirect_stdout(sys.stdout if args['--keep-stdout'] else devnull): work_item = cosmic_ray.worker.worker( Path(args['<module-path>']), config.python_version, args['<operator>'], int(args['<occurrence>']), config.test_command, None) sys.stdout.write(json.dumps(work_item, cls=WorkItemJsonEncoder)) return ExitCode.OK
def __call__(self, filename): global ROBOT_FILE ROBOT_FILE = '<string>' fp = super(RobotCode, self).__call__(filename) code = compile(fp.read(), ROBOT_FILE, 'exec') if not code.co_names: raise argparse.ArgumentTypeError('The supplied code is empty') robot_ns = {} with contextlib.redirect_stdout(open(os.devnull, 'w')): exec(code, robot_ns) for key, val in robot_ns.items(): if not key.startswith('_'): if is_robot_class(val): class_ = val break else: raise argparse.ArgumentTypeError('The code does not contain a RobotSnake-based class') globals().update(robot_ns) return class_
def defineNetwork(self): print("Setting up network...") inputs = Input(shape=(self.nx, self.ny, self.n_times)) conv = Convolution2D(self.n_filters, 3, 3, activation='relu', border_mode='same', init='he_normal')(inputs) x = self.residual(conv) for i in range(self.n_conv_layers): x = self.residual(x) x = Convolution2D(self.n_filters, 3, 3, border_mode='same', init='he_normal')(x) x = BatchNormalization()(x) x = merge([x, conv], 'sum') final = Convolution2D(6, 1, 1, activation='linear', border_mode='same', init='he_normal')(x) self.model = Model(input=inputs, output=final) json_string = self.model.to_json() f = open('{0}_model.json'.format(self.root), 'w') f.write(json_string) f.close() with open('{0}_summary.txt'.format(self.root), 'w') as f: with redirect_stdout(f): self.model.summary() kerasPlot(self.model, to_file='{0}_model.png'.format(self.root), show_shapes=True)
def run(filename, runid): print('file:', filename, 'runid:', runid) filename = os.path.abspath(filename) basename = os.path.basename(filename) dirpath = '{}.{}'.format(os.path.basename(filename), runid) dirpath = os.path.join(result_dir, dirpath) if not os.path.exists(dirpath): os.mkdir(dirpath) try: rootdir = os.path.abspath('.') os.chdir(dirpath) with open('report.txt', 'w') as file, \ redirect_stdout(CompositeStream(stdout, file)): wrapper = newwrapper(filename) np.random.seed(1) run_experiment(time_to_opt, wrapper) np.random.seed(1) run_experiment(result_over_time, wrapper, 1) np.random.seed(1) if basename in best_penalty: run_experiment(result_over_time, wrapper, best_penalty[basename], filename='accuracy_over_time') finally: os.chdir(rootdir)
def defineNetwork(self): print("Setting up network...") inputs = Input(shape=(self.nx, self.ny, self.n_diversity)) x = GaussianNoise(self.noise)(inputs) conv = Convolution2D(self.n_filters, (3, 3), activation='relu', padding='same', kernel_initializer='he_normal')(x) x = self.residual(conv) for i in range(self.n_conv_layers): x = self.residual(x) x = Convolution2D(self.n_filters, (3, 3), padding='same', kernel_initializer='he_normal')(x) x = BatchNormalization()(x) x = add([x, conv]) final = Convolution2D(1, (1, 1), activation='relu', padding='same', kernel_initializer='he_normal')(x) self.model = Model(inputs=inputs, outputs=final) json_string = self.model.to_json() f = open('{0}_model.json'.format(self.root), 'w') f.write(json_string) f.close() with open('{0}_summary.txt'.format(self.root), 'w') as f: with redirect_stdout(f): self.model.summary() plot_model(self.model, to_file='{0}_model.png'.format(self.root), show_shapes=True)
def reverse_file(filename, symbol, options): gctx = GlobalContext() gctx.sectionsname = False gctx.color = False gctx.filename = filename gctx.entry = symbol gctx.quiet = True for o in options: if o == "--raw x86": gctx.raw_type = "x86" elif o == "--raw x64": gctx.raw_type = "x64" elif o.startswith("--rawbase"): gctx.raw_base = int(o.split(" ")[1], 16) if not gctx.load_file(): die() sio = StringIO() with redirect_stdout(sio): o = gctx.get_addr_context(gctx.entry).decompile() if o is not None: o.print() postfix = '{0}.rev'.format('' if symbol is None else '_' + symbol) with open(filename.replace('.bin', postfix)) as f: assert_equal(sio.getvalue(), f.read())
def test_get_cumulants(self): from SymbolicCollisions.core.cumulants import get_cumulant from SymbolicCollisions.core.printers import print_as_vector from sympy.matrices import Matrix order = [ (1, 0, 0), (0, 1, 0), (0, 0, 1), (2, 0, 0), (0, 2, 0), (0, 0, 2), (1, 1, 0)] result = [ '= m100;', '= m010;', '= m001;', '= m200 - m100**2/m000;', '= m020 - m010**2/m000', '= m002 - m001**2/m000;', '= m110 - m010*m100/m000;'] for o, r in zip(order, result): cumulant, _ = get_cumulant(*o) f = io.StringIO() with redirect_stdout(f): mc = Matrix([cumulant]) # printer works using Matrix format print_as_vector(mc, 'c') out = f.getvalue() assert r in out
async def _eval(ctx, *, body): """Evaluates python code""" env = { 'ctx': ctx, 'channel': ctx.channel, 'author': ctx.author, 'guild': ctx.guild, 'message': ctx.message, '_': bot._last_result, 'source': inspect.getsource, 'session': bot.session } env.update(globals()) body = utils.cleanup_code(body) stdout = io.StringIO() err = out = None to_compile = f'async def func(): \n{textwrap.indent(body, " ")}' try: exec(to_compile, env) except Exception as e: err = await ctx.send(f'```py\n{e.__class__.__name__}: {e}\n```') return await ctx.message.add_reaction('\u2049') func = env['func'] try: with redirect_stdout(stdout): ret = await func() except Exception as e: value = stdout.getvalue() err = await ctx.send(f'```py\n{value}{traceback.format_exc()}\n```') else: value = stdout.getvalue() if ret is None: if value: try: out = await ctx.send(f'```py\n{value}\n```') except: paginated_text = utils.paginate(value) for page in paginated_text: if page == paginated_text[-1]: out = await ctx.send(f'```py\n{page}\n```') break await ctx.send(f'```py\n{page}\n```') else: bot._last_result = ret try: out = await ctx.send(f'```py\n{value}{ret}\n```') except: paginated_text = utils.paginate(f"{value}{ret}") for page in paginated_text: if page == paginated_text[-1]: out = await ctx.send(f'```py\n{page}\n```') break await ctx.send(f'```py\n{page}\n```') if out: await ctx.message.add_reaction('\u2705') # tick elif err: await ctx.message.add_reaction('\u2049') # x else: await ctx.message.add_reaction('\u2705')
def _run_complexity_analysis(on_ci): """Generates cyclomatic complexity reports for the package :param bool on_ci: Indicates whether an automated tool is running this operation. Output will be customized for machine readability """ modlog.debug("Running complexity analysis") # generate cyclomatic complexities for source files in XML format for integration with external tools pyjen_path = os.path.join(os.getcwd(), "pyjen") from radon.cli import cc # TODO: output in XML format when running on CI standard_output = StringIO() with redirect_stdout(standard_output): modlog.debug("Calling radon.cc") cc(paths=[pyjen_path], show_complexity=True, show_closures=True, total_average=True, xml=on_ci) modlog.debug("Writing report to disk") cc_report = os.path.join(log_folder, "radon_complexity.xml") with open(cc_report, "w") as fh: fh.write(standard_output.getvalue()) standard_output.close() modlog.info("Cyclomatic complexity analysis complete. See " + os.path.relpath(cc_report))
def runWith(fcn, args, stdin): ''' Run a function with given arguments and stdin, and return the output and stdout. ''' # Copy arguments in case the function modifies its inputs args2 = copy.deepcopy(args) # Set the stdin with SetStdin(stdin): # Redirect the stdout f = io.StringIO() with redirect_stdout(f): # Try the function with the arguments, catch any exceptions try: # *args expands the 'args' list into separate arguments output = fcn(*args2) except Exception: output = None # Provide a warning if the function modifies its inputs if args != args2: print('\tWARNING: THIS FUNCTION MODIFIES ITS INPUTS.') # Extract a string for the stdout stdout = f.getvalue() return output, stdout
def next_batch(self, validation=False): programs = [self.generate_program(hash_mod=0 if validation else 1) for _ in range(self.batch_size)] # Execute the programs to get the targets results = [] for program in programs: with io.StringIO() as buf, redirect_stdout(buf): exec(program) results.append(buf.getvalue()[:-1]) input_sequences = encode_sequences(programs, symbol_to_idx=SYMBOL_TO_IDX, sequence_len=INPUT_SEQ_LEN, #go_symbol=GO_SYMBOL, pad_symbol=PAD_SYMBOL, pad_beginning=True, reverse=True) input_sequences = dense_to_one_hot(input_sequences, num_classes=len(SYMBOL_TO_IDX)) target_sequences = encode_sequences(results, symbol_to_idx=SYMBOL_TO_IDX, sequence_len=OUTPUT_SEQ_LEN, go_symbol=None, pad_beginning=False, pad_symbol=PAD_SYMBOL) target_sequences = dense_to_one_hot(target_sequences, num_classes=len(SYMBOL_TO_IDX)) return input_sequences, target_sequences
def test_process_streams_redirect(self): # This won't work for asyncio implementation of subprocess async def test(): prog = bR''' import sys print('out', flush=True) print('err', file=sys.stderr, flush=True) ''' proc = await asyncio.create_subprocess_exec( sys.executable, '-c', prog, loop=self.loop) out, err = await proc.communicate() self.assertIsNone(out) self.assertIsNone(err) with tempfile.NamedTemporaryFile('w') as stdout: with tempfile.NamedTemporaryFile('w') as stderr: with contextlib.redirect_stdout(stdout): with contextlib.redirect_stderr(stderr): self.loop.run_until_complete(test()) stdout.flush() stderr.flush() with open(stdout.name, 'rb') as so: self.assertEqual(so.read(), b'out\n') with open(stderr.name, 'rb') as se: self.assertEqual(se.read(), b'err\n')
def load_batch_ko_data(): """ Load all simulations, """ def _load_exp_data(): long_df = pd.read_csv(data_path / "datasets" / "long2019_tidy.csv") long_df["sample_id"] = long_df.Genotype long_df = long_df.assign(author="Long").rename( { "Measurement_ID": "BiGG_ID", "Original_Value": "normalized_flux", "Value": "flux", "Original_ID": "ID", }, axis=1, ) long_df = long_df[long_df["Measurement_Type"] == "flux"] long_df = long_df[~long_df["BiGG_ID"].isna()] # the same reaction as rpe long_df = long_df[long_df["sample_id"] != "sgcE"] long_df = long_df[[ "BiGG_ID", "ID", "flux", "author", "sample_id", "normalized_flux" ]] """ douglas_flux_data = pd.read_csv("../../../DataAnalysis/DouglasKineticData/data/flux_data_processed.csv", index_col=0) douglas_sample_names = { "Evo04": "WT", "Evo04gnd": "gnd", "Evo04pgi": "pgi", "Evo04sdhCB": "sdh", "Evo04tpiA": "tpi", } douglas_exp_df = douglas_flux_data.query( "sample_name in @douglas_sample_names.keys()" ) douglas_exp_df["sample_id"] = douglas_exp_df.sample_name.apply( lambda x: douglas_sample_names[x] ) douglas_exp_df["author"] = "McCloskey" douglas_exp_df = douglas_exp_df.rename( {"rxn_id": "BiGG_ID", "sampling_median": "flux"}, axis=1 ).drop(["sampling_var", "sampling_min", "sampling_max", "sample_name"], axis=1) x_doug = douglas_exp_df.set_index(["author", "sample_id", "BiGG_ID"]).to_xarray() x_doug['normalized_flux'] = 100*(x_doug.flux / x_doug.sel(BiGG_ID="GLCptspp").flux) mccloskey_results = x_doug.to_dataframe().reset_index() mccloskey_results.head() """ return long_df def _load_kinetic_ko_sims(): """ Load the data from kinetic models simulations """ khodayari_results = load_khodayari( sample_names="all", load_path=(path_to_results / "Khodayari"), id_df=khod_idf, files=get_khodayari_batch_kos(), ) kurata_results = load_kurata( sample_names="all", load_path=(path_to_results / "Kurata" / "batch_knockouts"), id_df=kurata_idf, files=get_kurata_batch_kos(), mode="batch", ) millard_results = load_millard( sample_names="all", load_path=(path_to_results / "Millard" / "batch_knockouts"), id_df=millard_idf, files=get_millard_batch_kos(), ) chassagnole_results = load_chassagnole( sample_names="all", load_path=(path_to_results / "Chassagnole" / "batch_knockouts"), id_df=chassagnole_idf, files=get_chassagnole_kos(), ) simulation_results = pd.concat([ khodayari_results, kurata_results, millard_results, chassagnole_results ], sort=False) return simulation_results def _load_cobra_ko_sims(): """ Load simulations from iML1515, ECC2 and iML1515 and ECC2 conditioned on experimental data """ iml_results = pd.read_csv( path_to_results / "COBRA" / "iML1515" / "batch_knockouts" / "knockouts_all.csv", index_col=0, ) # ecc_results = pd.read_csv( # path_to_results # / "COBRA" # / "ECC2" # / "batch_knockouts" # / "knockouts_all.csv", # index_col=0, # ) exp_iml_results = pd.read_csv( path_to_results / "COBRA" / "Exp_iML1515" / "batch_knockouts" / "knockouts_all.csv", index_col=0, ) # exp_ecc_results = pd.read_csv( # path_to_results # / "COBRA" # / "Exp_ECC2" # / "batch_knockouts" # / "knockouts_all.csv", # index_col=0, # ) df = pd.concat([iml_results, exp_iml_results]) # Fix direction to match experimental data # Only for iML1515 iml_reversed = ["PGM", "PGK", "SUCOAS"] # fix PGM direction df.loc[df["ID"].isin(iml_reversed), "flux"] = (-1 * df.loc[df["ID"].isin(iml_reversed), "flux"]) df.loc[df["ID"].isin(iml_reversed), "normalized_flux"] = ( -1 * df.loc[df["ID"].isin(iml_reversed), "normalized_flux"]) # df = pd.concat([df, ecc_results, exp_ecc_results]) # For both iML1515 and ECC2 # fix RPI direction ecc_reversed = ["RPI"] df.loc[df["ID"].isin(ecc_reversed), "flux"] = (-1 * df.loc[df["ID"].isin(ecc_reversed), "flux"]) df.loc[df["ID"].isin(ecc_reversed), "normalized_flux"] = ( -1 * df.loc[df["ID"].isin(ecc_reversed), "normalized_flux"]) return df with io.StringIO() as buf, redirect_stdout(buf): simulation_data = _load_kinetic_ko_sims() cobra_data = _load_cobra_ko_sims() exp_data = _load_exp_data() file_info = buf.getvalue() return pd.concat([simulation_data, cobra_data, exp_data], sort=False), file_info
def load_ko_data(): """ Load all simulations of knockout phenotypes, """ def _load_experimental_ko_data(): """ Load Ishii data """ df = pd.read_csv("../data/datasets/ishii2007_tidy.csv") # this regexp matches deletions starting with d like dpgi df["sample_id"] = df.Genotype.str.extract(r"d(\w+)") df.loc[df.Genotype == "WT", "sample_id"] = "WT" df = df.assign(author="Ishii") df = df.rename( { "Measurement_ID": "BiGG_ID", "Original_Value": "normalized_flux", "Value": "flux", "Original_ID": "ID", }, axis=1, ) df = df[df["Measurement_Type"] == "flux"] df.loc[df["BiGG_ID"] == "PYKF", "BiGG_ID"] = "PYK" df = df[[ "flux", "ID", "BiGG_ID", "author", "sample_id", "normalized_flux" ]] return df def _load_cobra_ko_sims(): """ Load simulations from iML1515, ECC2 and iML1515 and ECC2 conditioned on experimental data """ iml_results = pd.read_csv(path_to_results / "COBRA" / "iML1515" / "chemostat_knockouts" / "knockouts_all.csv", index_col=0) # ecc_results = pd.read_csv( # path_to_results / "COBRA" / "ECC2" / "knockouts_all.csv", index_col=0 # ) exp_iml_results = pd.read_csv(path_to_results / "COBRA" / "Exp_iML1515" / "chemostat_knockouts" / "knockouts_all.csv", index_col=0) # exp_ecc_results = pd.read_csv( # path_to_results / "COBRA" / "Exp_ECC2" / "knockouts_all.csv", index_col=0 # ) df = pd.concat([iml_results, exp_iml_results]) # Fix direction to match experimental data # Only for iML1515 # fix PGM direction iml_reversed = ["PGM", "PGK", "SUCOAS"] df.loc[df["ID"].isin(iml_reversed), "flux"] = (-1 * df.loc[df["ID"].isin(iml_reversed), "flux"]) df.loc[df["ID"].isin(iml_reversed), "normalized_flux"] = ( -1 * df.loc[df["ID"].isin(iml_reversed), "normalized_flux"]) # df = pd.concat([df, ecc_results, exp_ecc_results]) # For both iML1515 and ECC2 # fix RPI direction ecc_reversed = ["RPI"] df.loc[df["ID"].isin(ecc_reversed), "flux"] = (-1 * df.loc[df["ID"].isin(ecc_reversed), "flux"]) df.loc[df["ID"].isin(ecc_reversed), "normalized_flux"] = ( -1 * df.loc[df["ID"].isin(ecc_reversed), "normalized_flux"]) return df def _load_kinetic_ko_sims(): """ Load the data from kinetic models simulations """ khodayari_results = load_khodayari( sample_names="all", load_path=(path_to_results / "Khodayari"), id_df=khod_idf, files=get_khodayari_kos(), ) kurata_results = load_kurata( sample_names="all", load_path=(path_to_results / "Kurata"), id_df=kurata_idf, files=get_kurata_kos(), ) millard_results = load_millard( sample_names="all", load_path=(path_to_results / "Millard"), id_df=millard_idf, files=get_millard_kos(), ) chassagnole_results = load_chassagnole( sample_names="all", load_path=(path_to_results / "Chassagnole" / "chemostat_knockouts"), id_df=chassagnole_idf, files=get_chassagnole_kos(), ) simulation_results = pd.concat([ khodayari_results, kurata_results, millard_results, chassagnole_results ], sort=False) return simulation_results with io.StringIO() as buf, redirect_stdout(buf): simulation_data = _load_kinetic_ko_sims() cobra_data = _load_cobra_ko_sims() exp_data = _load_experimental_ko_data() file_info = buf.getvalue() return pd.concat([simulation_data, cobra_data, exp_data], sort=False), file_info
def test_missing_problem(self): f = io.StringIO() with contextlib.redirect_stdout(f): with self.assertRaises(lib50.InvalidSlugError): lib50.connect("cs50/lib50/tests/i_do_not_exist", self.loader)
def test_no_tool_in_config(self): f = io.StringIO() loader = lib50.config.Loader("i_do_not_exist") with contextlib.redirect_stdout(f): with self.assertRaises(lib50.InvalidSlugError): lib50.connect("cs50/lib50/tests/bar", loader)
def handle(self, event: Event) -> None: transaction_inst = Transaction.from_cloudevents_model(event) transaction_key_value_insts = TransactionKeyValue.from_cloudevents_model(event) file_insts = File.from_cloudevents_model(event) config_by_config_id = _to_proxymod_config_by_config_id(transaction_key_values=transaction_key_value_insts) for config_id in ['config_1', 'config_2', 'config_3']: if config_id not in config_by_config_id: raise ConfigNotFoundProxEventHandlerError(event, config_id) config = config_by_config_id[config_id] if not _is_valid_proxymod_config(config): raise InvalidConfigProxEventHandlerError(event, config_id, config) input_file_insts = [] model_file_insts = [] _in_dir = config_by_config_id.get('config_1', {}).get('INPUTS', {}).get('in_dir', None) _in_file_one = config_by_config_id.get('config_1', {}).get('INPUTS', {}).get('in_file_one', None) _in_file_two = config_by_config_id.get('config_1', {}).get('INPUTS', {}).get('in_file_two', None) for file_inst in file_insts: if ('text/csv' == file_inst.mimetype) and (file_inst.subdir is not None) and (_in_dir == file_inst.subdir) and (file_inst.name is not None) and ((_in_file_one == file_inst.name) or (_in_file_two == file_inst.name)): input_file_insts.append(file_inst) elif ('text/x-python' == file_inst.mimetype) and ('models/' == file_inst.subdir): model_file_insts.append(file_inst) else: # NOTE Ignore other files. pass with tempfile.TemporaryDirectory() as downloader_tempdir_name: with tempfile.TemporaryDirectory() as uploader_tempdir_name: # model_file_openers = self.downloader_runner.download(downloader_tempdir_name, model_file_insts) with open(os.path.join(uploader_tempdir_name, 'download-stdout.log'), mode='w') as downloader_stdout_file: with open(os.path.join(uploader_tempdir_name, 'download-stderr.log'), mode='w') as downloader_stderr_file: with contextlib.redirect_stdout(downloader_stdout_file): with contextlib.redirect_stderr(downloader_stderr_file): model_file_openers = self.downloader_runner.download(downloader_tempdir_name, model_file_insts) model_file_funcs = [] for model_file_inst, model_file_opener in zip(model_file_insts, model_file_openers): with model_file_opener() as file: try: name = os.path.splitext(model_file_inst.name)[0] spec = importlib.util.spec_from_file_location(name, file.name) module = importlib.util.module_from_spec(spec) spec.loader.exec_module(module) # NOTE Deliberately raise `AttributeError` if `name` does not exist. func = getattr(module, name) if callable(func): model_file_funcs.append(func) else: # NOTE Deliberately raise `TypeError` by calling an uncallable. func() except Exception as reason: raise InvalidModelProxEventHandlerError(event, model_file_inst, reason) # input_file_openers = self.downloader_runner.download(downloader_tempdir_name, input_file_insts) with open(os.path.join(uploader_tempdir_name, 'download-stdout.log'), mode='a') as downloader_stdout_file: with open(os.path.join(uploader_tempdir_name, 'download-stderr.log'), mode='a') as downloader_stderr_file: with contextlib.redirect_stdout(downloader_stdout_file): with contextlib.redirect_stderr(downloader_stderr_file): input_file_openers = self.downloader_runner.download(downloader_tempdir_name, input_file_insts) abspath_config_by_config_id = copy.deepcopy(config_by_config_id) for config_id, config in abspath_config_by_config_id.items(): if 'INPUTS' in config: if 'in_dir' in config['INPUTS']: for input_file_inst, opener in zip(input_file_insts, input_file_openers): with opener() as file: config['INPUTS']['in_dir'] = os.path.abspath(os.path.dirname(file.name)) break if 'OUTPUTS' in config: if 'out_dir' in config['OUTPUTS']: config['OUTPUTS']['out_dir'] = os.path.abspath(os.path.join(uploader_tempdir_name, config['OUTPUTS']['out_dir'])) for config_id, config in config_by_config_id.items(): with open(os.path.join(uploader_tempdir_name, '{0}.ini'.format(config_id)), mode='w') as config_file: config_file.write(_format_proxymod_config(config)) with tempfile.NamedTemporaryFile(suffix='.ini') as config_1_file: config_1_file.write(bytes(_format_proxymod_config(abspath_config_by_config_id['config_1']), 'utf-8')) config_1_file.seek(0) with tempfile.NamedTemporaryFile(suffix='.ini') as config_2_file: config_2_file.write(bytes(_format_proxymod_config(abspath_config_by_config_id['config_2']), 'utf-8')) config_2_file.seek(0) with tempfile.NamedTemporaryFile(suffix='.ini') as config_3_file: config_3_file.write(bytes(_format_proxymod_config(abspath_config_by_config_id['config_3']), 'utf-8')) config_3_file.seek(0) # for model_file_inst, model_file_func in zip(model_file_insts, model_file_funcs): # try: # model_file_func(config_1_file.name, config_2_file.name, config_3_file.name) # except Exception as reason: # raise InvalidModelProxEventHandlerError(event, model_file_inst, reason) with open(os.path.join(uploader_tempdir_name, 'stdout.log'), mode='w') as stdout_file: with open(os.path.join(uploader_tempdir_name, 'stderr.log'), mode='w') as stderr_file: with contextlib.redirect_stdout(stdout_file): with contextlib.redirect_stderr(stderr_file): for model_file_inst, model_file_func in zip(model_file_insts, model_file_funcs): try: model_file_func(config_1_file.name, config_2_file.name, config_3_file.name) except Exception as reason: raise InvalidModelProxEventHandlerError(event, model_file_inst, reason) # (bundle, job_id, state) = self.uploader_runner.upload(uploader_tempdir_name, transaction=Transaction(submitter=transaction_inst.submitter, instrument=transaction_inst.instrument, proposal=transaction_inst.proposal), transaction_key_values=[TransactionKeyValue(key='Transactions._id', value=transaction_inst._id)]) with open(os.path.join(uploader_tempdir_name, 'upload-stdout.log'), mode='w') as uploader_stdout_file: with open(os.path.join(uploader_tempdir_name, 'upload-stderr.log'), mode='w') as uploader_stderr_file: with contextlib.redirect_stdout(uploader_stdout_file): with contextlib.redirect_stderr(uploader_stderr_file): (bundle, job_id, state) = self.uploader_runner.upload(uploader_tempdir_name, transaction=Transaction(submitter=transaction_inst.submitter, instrument=transaction_inst.instrument, proposal=transaction_inst.proposal), transaction_key_values=[TransactionKeyValue(key='Transactions._id', value=transaction_inst._id)]) pass pass pass
def draw(callback, update): with contextlib.ExitStack() as stack: # create virtual terminal fdread, fdwrite = getattr(os, 'openpty', os.pipe)() stack.callback( os.close, fdread) # fdwrite is closed separately to signal to the thread # save original output file descriptor fileno = os.dup(sys.stdout.fileno()) stack.callback(os.close, fileno) # create thread t = StickyBar(fdread, fileno, callback, sys.stdout.encoding, update) stack.callback(t.join) # replace stdout by virtual terminal os.dup2(fdwrite, sys.stdout.fileno()) stack.callback( os.dup2, fileno, sys.stdout.fileno()) # restore stdout and signal to thread os.close(fdwrite) if platform.system() != 'Windows': os.write(fileno, b'\033[?7l') # disable line wrap stack.callback(os.write, fileno, b'\033[?7h') # enable line wrap else: # set console mode import ctypes kernel32 = ctypes.WinDLL('kernel32') handle = kernel32.GetStdHandle( -11 ) # https://docs.microsoft.com/en-us/windows/console/getstdhandle orig_mode = ctypes.c_uint32( ) # https://docs.microsoft.com/en-us/windows/desktop/WinProg/windows-data-types#lpdword kernel32.GetConsoleMode( handle, ctypes.byref(orig_mode) ) # https://docs.microsoft.com/en-us/windows/console/getconsolemode new_mode = orig_mode.value new_mode |= 4 # check ENABLE_VIRTUAL_TERMINAL_PROCESSING new_mode &= ~2 # uncheck ENABLE_WRAP_AT_EOL_OUTPUT if new_mode != orig_mode.value: kernel32.SetConsoleMode( handle, ctypes.c_uint32(new_mode) ) # https://docs.microsoft.com/en-us/windows/console/setconsolemode stack.callback(kernel32.SetConsoleMode, handle, orig_mode) # In Windows, `sys.stdout` becomes unusable after # `os.dup2(..,sys.stdout.fileno())`, hence we recreate `sys.stdout` here. # Because a pipe is not a tty, `fdopen` defaults to buffering with fixed # size chunks. `buffering=1` enforces lines buffering. The new # `sys.stdout` answers `False` to `.isatty()` for the same reason. stack.enter_context( contextlib.redirect_stdout( os.fdopen(sys.stdout.fileno(), 'w', encoding=sys.stdout.encoding, buffering=1))) # start bar-drawing thread t.start() yield
def test_no_config(self): f = io.StringIO() with contextlib.redirect_stdout(f): with self.assertRaises(lib50.InvalidSlugError): lib50.connect("cs50/lib50/tests/no_config", self.loader)
def load_dilution_data(): def _load_cobra_dilution_sims(): """ Load simulations from iML1515, ECC2 """ iml_results = pd.read_csv(path_to_results / "COBRA" / "iML1515" / "dilutions" / "all.csv", index_col=0) # ecc_results = pd.read_csv( # path_to_results / "COBRA" / "ECC2" / "dilutions" / "all.csv", index_col=0 # ) df = iml_results # Fix direction to match experimental data # Only for iML1515 # fix PGM direction iml_reversed = ["PGM", "PGK", "SUCOAS"] df.loc[df["ID"].isin(iml_reversed), "flux"] = (-1 * df.loc[df["ID"].isin(iml_reversed), "flux"]) df.loc[df["ID"].isin(iml_reversed), "normalized_flux"] = ( -1 * df.loc[df["ID"].isin(iml_reversed), "normalized_flux"]) # df = pd.concat([df, ecc_results]) # For both iML1515 and ECC2 # fix RPI direction ecc_reversed = ["RPI"] df.loc[df["ID"].isin(ecc_reversed), "flux"] = (-1 * df.loc[df["ID"].isin(ecc_reversed), "flux"]) df.loc[df["ID"].isin(ecc_reversed), "normalized_flux"] = ( -1 * df.loc[df["ID"].isin(ecc_reversed), "normalized_flux"]) return df def _load_experimental_dilution_data(): yao_df = pd.read_csv(data_path / "datasets" / "yao2011_tidy.csv") consumption_rates = yao_df.query( 'Measurement_Type == "consumption_rate"') yao_fluxes = yao_df.query('Measurement_Type == "flux"') def normalize_to_uptake(group): consumption_rate = consumption_rates.loc[ consumption_rates.Dilution == group.name, "Value"].values[0] print(f"Consumption rate for D {group.name} is {consumption_rate}") group = group.assign( normalized_flux=lambda x: x.Value / consumption_rate * 100) return group df = (yao_fluxes.groupby("Dilution").apply( normalize_to_uptake).reset_index(drop=True)) df = df.assign(author="Yao") df = df.rename( { "Measurement_ID": "BiGG_ID", "Value": "flux", "Original_ID": "ID", "Dilution": "sample_id", }, axis=1, ) df = df[df["Measurement_Type"] == "flux"] df = df[[ "flux", "ID", "BiGG_ID", "author", "sample_id", "normalized_flux" ]] df.sample_id = df.sample_id.apply(str) return df def _load_kinetic_dilution_sims(): khodayari_dil = load_khodayari( sample_names="all", load_path=(path_to_results / "Khodayari" / "dilutions"), id_df=khod_idf, files=get_khodayari_dilutions(), ) kurata_dil = load_kurata( sample_names="all", load_path=(path_to_results / "Kurata" / "dilutions"), id_df=kurata_idf, files=get_kurata_dilutions(), ) millard_dil = load_millard( sample_names="all", load_path=(path_to_results / "Millard" / "dilutions"), id_df=millard_idf, files=get_millard_dilutions(), ) chassagnole_dil = load_chassagnole( sample_names="all", load_path=(path_to_results / "Chassagnole" / "dilutions"), id_df=chassagnole_idf, files=get_chassagnole_dilutions(), ) return pd.concat( [khodayari_dil, kurata_dil, millard_dil, chassagnole_dil], sort=False) with io.StringIO() as buf, redirect_stdout(buf): simulation_data = _load_kinetic_dilution_sims() exp_data = _load_experimental_dilution_data() cobra_data = _load_cobra_dilution_sims() file_info = buf.getvalue() return pd.concat([simulation_data, exp_data, cobra_data], sort=False), file_info
def create_archipelago( unknowns: list, optimizers: list, optimizers_kwargs: list, pg_problem: pygmo.problem, rel_pop_size: float, archipelago_kwargs: dict, log_each_nth_gen: int, report_level: int, ) -> PyfoombArchipelago: """ Helper method for parallelized estimation using the generalized island model. Creates the archipelago object for running several rounds of evolutions. Arguments --------- unknowns : list The unknowns, sorted alphabetically and case-insensitive. optimizers : list A list of optimizers to be used on individual islands. optimizers_kwargs : list A list of corresponding kwargs. pg_problem : pygmo.problem An pygmo problem instance. archipelago_kwargs : dict Additional kwargs for archipelago creation. log_each_nth_gen : int Specifies at which each n-th generation the algorithm stores logs. report_level : int Prints information on the archipelago creation for values >= 1. Returns ------- archipelago : PyfoombArchipelago """ _cpus = joblib.cpu_count() # There is one optimizer with a set of kwargs if len(optimizers) == 1 and len(optimizers_kwargs) == 1: optimizers = optimizers * _cpus optimizers_kwargs = optimizers_kwargs * _cpus # Several optimizers with the same kwargs elif len(optimizers) > 1 and len(optimizers_kwargs) == 1: optimizers_kwargs = optimizers_kwargs * len(optimizers) # Several kwargs for the same optimizer elif len(optimizers) == 1 and len(optimizers_kwargs) > 1: optimizers = optimizers * len(optimizers_kwargs) elif len(optimizers) != len(optimizers_kwargs): raise ValueError( 'Number of optimizers does not match number of corresponding kwarg dicts' ) # Get the optimizer intances algos = [ PygmoOptimizers.get_optimizer_algo_instance( name=_optimizers, kwargs=_optimizers_kwargs) for _optimizers, _optimizers_kwargs in zip(optimizers, optimizers_kwargs) ] # Update number of islands n_islands = len(algos) if report_level >= 1: print( f'Creating archipelago with {n_islands} islands. May take some time...' ) pop_size = int(numpy.ceil(rel_pop_size * len(unknowns))) prop_create_args = ((pg_problem, pop_size, seed * numpy.random.randint(0, 1e4)) for seed, pop_size in enumerate([pop_size] * n_islands)) try: parallel_verbose = 0 if report_level == 0 else 1 with joblib.parallel_backend('loky', n_jobs=n_islands): pops = joblib.Parallel(verbose=parallel_verbose)(map( joblib.delayed( ArchipelagoHelpers.parallel_create_population), prop_create_args)) except Exception as ex: print( f'Parallelized archipelago creation failed, falling back to sequential\n{ex}' ) pops = ( ArchipelagoHelpers.parallel_create_population(prop_create_arg) for prop_create_arg in prop_create_args) # Now create the empyty archipelago if not 't' in archipelago_kwargs.keys(): archipelago_kwargs['t'] = pygmo.fully_connected() archi = PyfoombArchipelago(**archipelago_kwargs) archi.set_migrant_handling(pygmo.migrant_handling.preserve) # Add the populations to the archipelago and wait for its construction with contextlib.redirect_stdout(io.StringIO()): for _pop, _algo in zip(pops, algos): if log_each_nth_gen is not None: _algo.set_verbosity(int(log_each_nth_gen)) _island = pygmo.island(algo=_algo, pop=_pop, udi=pygmo.mp_island()) archi.push_back(_island) archi.wait_check() return archi
def __init__(self, name="root", level="INFO"): self.logger = logging.getLogger(name) self.source = self.logger.name self.level = getattr(logging, level) self._redirector = contextlib.redirect_stdout(self)
import contextlib with contextlib.redirect_stdout(None): import pygame pygame.font.init() # you have to call this at the start, # if you want to use this module. TEXT_FONT = pygame.font.SysFont('Comic Sans MS', 16) TEXT_FONT_SMALL = pygame.font.SysFont('Comic Sans MS', 8) ALIGN_TOP = 0 ALIGN_CENTER = 1 ALIGN_BOTTOM = 2 ALIGN_LEFT = 0 ALIGN_RIGHT = 2 class PyGameComponent(object): def __init__(self): self.visible = True self.x = 0 self.y = 0 self.width = 0 self.height = 0 self.rect = {"left": 0, "top": 0, "right": 0, "bottom": 0} self.screen_rect = {"left": 0, "top": 0, "right": 0, "bottom": 0} self.anchors = {"min_x": 0, "min_y": 0, "max_x": 1, "max_y": 1} self.pivot = {"x": 0, "y": 0} self.mouse_state = { "left_down": False, "middle_down": False, "right_down": False,
"PE20"] metric_dictionary1 = {metric_names[i]: line1[i] for i in range(len( metric_names))} metric_dictionary2 = {metric_names[i]: line2[i] for i in range(len( metric_names))} # metric_dictionary3 = {metric_names[i]: line3[i] for i in range(len(metric_names))} # metric_dictionary4 = {metric_names[i]: line4[i] for i in range(len(metric_names))} print("Error metrics for call options") for key, value in metric_dictionary1.items(): print(f"{key}:", value) from contextlib import redirect_stdout with open('MLP1_call_error_metrics.txt', 'w') as f: with redirect_stdout(f): print("Error metrics for call options") for key, value in metric_dictionary1.items(): print(f"{key}:", value) print("\nError metrics for put options") for key, value in metric_dictionary2.items(): print(f"{key}:", value) from contextlib import redirect_stdout with open('MLP1_put_error_metrics.txt', 'w') as f: with redirect_stdout(f): print("Error metrics for put options") for key, value in metric_dictionary2.items(): print(f"{key}:", value)
logfile = snakemake.log[0] fnames = snakemake.input.cts assembly = snakemake.wildcards.assembly fa = snakemake.input.fa[0] gtf = snakemake.input.gtf[0] from_gtf = snakemake.params.from_gtf out_counts = snakemake.output.counts[0] out_tpms = snakemake.output.tpms[0] out_lengths = snakemake.output.lengths[0] sample_names = snakemake.params.names # redirect all messages to a logfile open(logfile, "w") # start a clean log with open(logfile, "a") as log: # appending because we mix streams (tqdm, logger & std) with contextlib.redirect_stdout(log), contextlib.redirect_stderr(log): pytxi.logger.remove() pytxi.logger.add(logfile) outdir = os.path.dirname(out_counts) os.makedirs(outdir, exist_ok=True) tx2gene = None if from_gtf: ann = Annotation(gtf) # check if we can convert transcript ids to symbols with the GTF file test = ann.gtf["attribute"].head(100) attribute = None if any(test.str.contains("gene_name")): attribute = "gene_name"
def load_sensitivity_data(): def _load_kinetic_sensitivity_sims(): khodayari_zwf = load_khodayari( sample_names="all", load_path=(path_to_results / "Khodayari" / "zwf_sensitivity"), id_df=khod_idf, files=get_khodayari_zwf(), ) khodayari_pgi = load_khodayari( sample_names="all", load_path=(path_to_results / "Khodayari" / "pgi_sensitivity"), id_df=khod_idf, files=get_khodayari_pgi(), ) khodayari_eno = load_khodayari( sample_names="all", load_path=(path_to_results / "Khodayari" / "eno_sensitivity"), id_df=khod_idf, files=get_khodayari_eno(), ) kurata_zwf = load_kurata( sample_names="all", load_path=(path_to_results / "Kurata" / "zwf_sensitivity"), id_df=kurata_idf, files=get_kurata_zwf(), ) kurata_pgi = load_kurata( sample_names="all", load_path=(path_to_results / "Kurata" / "pgi_sensitivity"), id_df=kurata_idf, files=get_kurata_pgi(), ) kurata_eno = load_kurata( sample_names="all", load_path=(path_to_results / "Kurata" / "eno_sensitivity"), id_df=kurata_idf, files=get_kurata_eno(), ) millard_zwf = load_millard( sample_names="all", load_path=(path_to_results / "Millard" / "zwf_sensitivity"), id_df=millard_idf, files=get_millard_zwf(), ) millard_pgi = load_millard( sample_names="all", load_path=(path_to_results / "Millard" / "pgi_sensitivity"), id_df=millard_idf, files=get_millard_pgi(), ) millard_eno = load_millard( sample_names="all", load_path=(path_to_results / "Millard" / "eno_sensitivity"), id_df=millard_idf, files=get_millard_eno(), ) chassagnole_zwf = load_chassagnole( sample_names="all", load_path=(path_to_results / "Chassagnole" / "zwf_pgi_eno_sensitivity"), id_df=chassagnole_idf, files=get_chassagnole_zwf(), ) chassagnole_pgi = load_chassagnole( sample_names="all", load_path=(path_to_results / "Chassagnole" / "zwf_pgi_eno_sensitivity"), id_df=chassagnole_idf, files=get_chassagnole_pgi(), ) chassagnole_eno = load_chassagnole( sample_names="all", load_path=(path_to_results / "Chassagnole" / "zwf_pgi_eno_sensitivity"), id_df=chassagnole_idf, files=get_chassagnole_eno(), ) simulation_zwf = pd.concat( [khodayari_zwf, kurata_zwf, millard_zwf, chassagnole_zwf], sort=False) simulation_pgi = pd.concat( [khodayari_pgi, kurata_pgi, millard_pgi, chassagnole_pgi], sort=False) simulation_eno = pd.concat( [khodayari_eno, kurata_eno, millard_eno, chassagnole_eno], sort=False) return (simulation_zwf, simulation_pgi, simulation_eno) def _load_experimental_sensitivity_data(): """ Load experimental results and return a tuple of 3 dataframes each corresponding to genes zwf, pgi and eno """ """ Nicloas, 2007 data, for zwf knockout """ df = pd.read_csv("../data/datasets/nicolas2007_tidy.csv") df = df.assign(author="Nicolas") df = df.rename( { "Measurement_ID": "BiGG_ID", "Original_Value": "normalized_flux", "Value": "flux", "Original_ID": "ID", "Genotype": "sample_id", }, axis=1, ) df = df[df["Measurement_Type"] == "flux"] df = df[[ "flux", "ID", "BiGG_ID", "author", "sample_id", "normalized_flux" ]] exp_results_zwf = df """ Usui, 2012 data for pgi and eno data """ df = pd.read_csv("../data/datasets/usui2012_tidy.csv") df = df.assign(author="Usui") df = df.rename( { "Measurement_ID": "BiGG_ID", "Original_Value": "normalized_flux", "Value": "flux", "Original_ID": "ID", "Genotype": "sample_id", }, axis=1, ) df = df[df["Measurement_Type"] == "flux"] df = df[[ "flux", "ID", "BiGG_ID", "author", "sample_id", "normalized_flux" ]] exp_results_pgi = df exp_results_eno = df return (exp_results_zwf, exp_results_pgi, exp_results_eno) with io.StringIO() as buf, redirect_stdout(buf): simulation_data_zwf, simulation_data_pgi, simulation_data_eno = ( _load_kinetic_sensitivity_sims()) exp_data_zwf, exp_data_pgi, exp_data_eno = _load_experimental_sensitivity_data( ) file_info = buf.getvalue() return ( ( pd.concat([simulation_data_zwf, exp_data_zwf], sort=False), pd.concat([simulation_data_pgi, exp_data_pgi], sort=False), pd.concat([simulation_data_eno, exp_data_eno], sort=False), ), file_info, )
def test_rf_classification(small_clf, datatype, split_algo, max_samples, max_features, use_experimental_backend): use_handle = True X, y = small_clf X = X.astype(datatype) y = y.astype(np.int32) X_train, X_test, y_train, y_test = train_test_split(X, y, train_size=0.8, random_state=0) # Create a handle for the cuml model handle, stream = get_handle(use_handle, n_streams=1) # Initialize, fit and predict using cuML's # random forest classification model cuml_model = curfc(max_features=max_features, max_samples=max_samples, n_bins=16, split_algo=split_algo, split_criterion=0, min_samples_leaf=2, random_state=123, n_streams=1, n_estimators=40, handle=handle, max_leaves=-1, max_depth=16, use_experimental_backend=use_experimental_backend) f = io.StringIO() with redirect_stdout(f): cuml_model.fit(X_train, y_train) captured_stdout = f.getvalue() if use_experimental_backend: is_fallback_used = False if max_features != 1.0: assert ('Experimental backend does not yet support feature ' + 'sub-sampling' in captured_stdout) is_fallback_used = True if split_algo != 1: assert ('Experimental backend does not yet support histogram ' + 'split algorithm' in captured_stdout) is_fallback_used = True if is_fallback_used: assert ('Not using the experimental backend due to above ' + 'mentioned reason(s)' in captured_stdout) else: assert ('Using experimental backend for growing trees' in captured_stdout) else: assert captured_stdout == '' fil_preds = cuml_model.predict(X_test, predict_model="GPU", output_class=True, threshold=0.5, algo='auto') cu_preds = cuml_model.predict(X_test, predict_model="CPU") fil_preds = np.reshape(fil_preds, np.shape(cu_preds)) cuml_acc = accuracy_score(y_test, cu_preds) fil_acc = accuracy_score(y_test, fil_preds) if X.shape[0] < 500000: sk_model = skrfc(n_estimators=40, max_depth=16, min_samples_split=2, max_features=max_features, random_state=10) sk_model.fit(X_train, y_train) sk_preds = sk_model.predict(X_test) sk_acc = accuracy_score(y_test, sk_preds) assert fil_acc >= (sk_acc - 0.07) assert fil_acc >= (cuml_acc - 0.02)
def main(): # Training settings parser = argparse.ArgumentParser(description='PyTorch MNIST Example') parser.add_argument('--type', default='linear', choices=['linear', 'conv'], help='model architecture type: ' + ' | '.join(['linear', 'conv']) + ' (default: linear)') parser.add_argument( '--model', default='', type=str, metavar='MODEL_PATH', help= 'path to model file to load both its architecture and weights (default: none)' ) parser.add_argument( '--weights', default='', type=str, metavar='WEIGHTS_PATH', help='path to file to load its weights (default: none)') parser.add_argument('--shift-depth', type=int, default=0, help='how many layers to convert to shift') parser.add_argument( '-st', '--shift-type', default='PS', choices=['Q', 'PS'], help= 'type of DeepShift method for training and representing weights (default: PS)' ) parser.add_argument('-r', '--rounding', default='deterministic', choices=['deterministic', 'stochastic'], help='type of rounding (default: deterministic)') parser.add_argument('-wb', '--weight-bits', type=int, default=5, help='number of bits to represent the shift weights') parser.add_argument('-j', '--workers', default=1, type=int, metavar='N', help='number of data loading workers (default: 1)') parser.add_argument('--batch-size', type=int, default=64, metavar='N', help='input batch size for training (default: 64)') parser.add_argument('--test-batch-size', type=int, default=1000, metavar='N', help='input batch size for testing (default: 1000)') parser.add_argument('--epochs', type=int, default=10, metavar='N', help='number of epochs to train (default: 10)') parser.add_argument('-opt', '--optimizer', metavar='OPT', default="SGD", help='optimizer algorithm') parser.add_argument('--lr', type=float, default=0.01, metavar='LR', help='learning rate (default: 0.01)') parser.add_argument('--momentum', type=float, default=0.0, metavar='M', help='SGD momentum (default: 0.0)') parser.add_argument('--resume', default='', type=str, metavar='CHECKPOINT_PATH', help='path to latest checkpoint (default: none)') parser.add_argument('-e', '--evaluate', dest='evaluate', action='store_true', help='only evaluate model on validation set') parser.add_argument('--no-cuda', action='store_true', default=False, help='disables CUDA training') parser.add_argument('--seed', type=int, default=1, metavar='S', help='random seed (default: 1)') parser.add_argument( '--log-interval', type=int, default=10, metavar='N', help='how many batches to wait before logging training status') parser.add_argument('--pretrained', dest='pretrained', default=False, type=lambda x: bool(distutils.util.strtobool(x)), help='use pre-trained model of full conv or fc model') parser.add_argument('--save-model', default=True, type=lambda x: bool(distutils.util.strtobool(x)), help='For Saving the current Model (default: True)') parser.add_argument( '--print-weights', default=True, type=lambda x: bool(distutils.util.strtobool(x)), help='For printing the weights of Model (default: True)') parser.add_argument('--desc', type=str, default=None, help='description to append to model directory name') parser.add_argument('--use-kernel', type=lambda x: bool(distutils.util.strtobool(x)), default=False, help='whether using custom shift kernel') parser.add_argument('-sb', '--shift-base', type=int, default=2, help='base of the wegiht representation') args = parser.parse_args() use_cuda = not args.no_cuda and torch.cuda.is_available() if (args.evaluate is False and args.use_kernel is True): raise ValueError( 'Our custom kernel currently supports inference only, not training.' ) torch.manual_seed(args.seed) device = torch.device("cuda" if use_cuda else "cpu") kwargs = { 'num_workers': args.workers, 'pin_memory': True } if use_cuda else {} # Load training MNIST data train_loader = torch.utils.data.DataLoader( datasets.MNIST( '../data', train=True, download=True, transform=transforms.Compose([ transforms.ToTensor(), transforms.Normalize( (0.1307, ), (0.3081, )) # transforms.Normalize((0,), (255,)) ])), batch_size=args.batch_size, shuffle=True, **kwargs) # Load testing MNIST data test_loader = torch.utils.data.DataLoader( datasets.MNIST( '../data', train=False, transform=transforms.Compose([ transforms.ToTensor(), transforms.Normalize( (0.1307, ), (0.3081, )) # transforms.Normalize((0,), (255,)) ])), batch_size=args.test_batch_size, shuffle=True, **kwargs) # Use an existing model (directory path provided) if args.model: if args.type or args.pretrained: print( "WARNING: Ignoring arguments \"type\" and \"pretrained\" when creating model..." ) model = None saved_checkpoint = torch.load(args.model) if isinstance(saved_checkpoint, nn.Module): model = saved_checkpoint elif "model" in saved_checkpoint: model = saved_checkpoint["model"] else: raise Exception("Unable to load model from " + args.model) # Generate new model (linear or convolution) else: if args.type == 'linear': model = LinearMNIST().to(device) elif args.type == 'conv': model = ConvMNIST().to(device) if args.pretrained: model.load_state_dict( torch.load("./models/mnist/simple_" + args.type + "/shift_0/weights.pth")) model = model.to(device) model_rounded = None if args.weights: saved_weights = torch.load(args.weights) if isinstance(saved_weights, nn.Module): state_dict = saved_weights.state_dict() elif "state_dict" in saved_weights: state_dict = saved_weights["state_dict"] else: state_dict = saved_weights model.load_state_dict(state_dict) if args.shift_depth > 0: model, _ = convert_to_shift(model, args.shift_depth, args.shift_type, args.shift_base, convert_all_linear=(args.type != 'linear'), convert_weights=True, use_kernel=args.use_kernel, use_cuda=use_cuda, rounding=args.rounding, weight_bits=args.weight_bits) model = model.to(device) elif args.use_kernel and args.shift_depth == 0: model = convert_to_unoptimized(model) model = model.to(device) elif args.use_kernel and args.shift_depth == 0: model = convert_to_unoptimized(model) model = model.to(device) loss_fn = F.cross_entropy # F.nll_loss # define optimizer optimizer = None if (args.optimizer.lower() == "sgd"): optimizer = torch.optim.SGD(model.parameters(), args.lr, momentum=args.momentum) elif (args.optimizer.lower() == "adadelta"): optimizer = torch.optim.Adadelta(model.parameters(), args.lr) elif (args.optimizer.lower() == "adagrad"): optimizer = torch.optim.Adagrad(model.parameters(), args.lr) elif (args.optimizer.lower() == "adam"): optimizer = torch.optim.Adam(model.parameters(), args.lr) elif (args.optimizer.lower() == "rmsprop"): optimizer = torch.optim.RMSprop(model.parameters(), args.lr) elif (args.optimizer.lower() == "radam"): optimizer = optim.RAdam(model.parameters(), args.lr) elif (args.optimizer.lower() == "ranger"): optimizer = optim.Ranger(model.parameters(), args.lr) else: raise ValueError("Optimizer type: ", args.optimizer, " is not supported or known") # optionally resume from a checkpoint if args.resume: if os.path.isfile(args.resume): print("=> loading checkpoint '{}'".format(args.resume)) checkpoint = torch.load(args.resume) if 'state_dict' in checkpoint: model.load_state_dict(checkpoint['state_dict']) else: model.load_state_dict(checkpoint) print("=> loaded checkpoint '{}'".format(args.resume)) else: print("=> no checkpoint found at '{}'".format(args.resume)) # name model sub-directory "shift_all" if all layers are converted to shift layers conv2d_layers_count = count_layer_type( model, nn.Conv2d) + count_layer_type(model, unoptimized.UnoptimizedConv2d) linear_layers_count = count_layer_type( model, nn.Linear) + count_layer_type(model, unoptimized.UnoptimizedLinear) if (args.shift_depth > 0): if (args.shift_type == 'Q'): shift_label = "shift_q" else: shift_label = "shift_ps" else: shift_label = "shift" # name model sub-directory "shift_all" if all layers are converted to shift layers conv2d_layers_count = count_layer_type(model, nn.Conv2d) linear_layers_count = count_layer_type(model, nn.Linear) if (conv2d_layers_count == 0 and linear_layers_count == 0): shift_label += "_all" else: shift_label += "_%s" % (args.shift_depth) if (args.shift_depth > 0): shift_label += "_wb_%s" % (args.weight_bits) if (args.shift_base > 0): shift_base = args.shift_base shift_label += "_sb_%s" % (args.shift_base) if (args.desc is not None and len(args.desc) > 0): desc_label = "_%s" % (args.desc) else: desc_label = "" global model_name # MZ addition model_name = 'simple_%s/%s%s' % (args.type, shift_label, desc_label) # if evaluating round weights to ensure that the results are due to powers of 2 weights if (args.evaluate): model = round_shift_weights(model, shift_base) model_summary = None try: model_summary, model_params_info = torchsummary.summary_string( model, input_size=(1, 28, 28)) print(model_summary) print( "WARNING: The summary function reports duplicate parameters for multi-GPU case" ) except: print("WARNING: Unable to obtain summary of model") model_dir = os.path.join( os.path.join(os.path.join(os.getcwd(), "models"), "mnist"), model_name) if not os.path.isdir(model_dir): os.makedirs(model_dir, exist_ok=True) if (args.save_model): with open(os.path.join(model_dir, 'command_args.txt'), 'w') as command_args_file: for arg, value in sorted(vars(args).items()): command_args_file.write(arg + ": " + str(value) + "\n") with open(os.path.join(model_dir, 'model_summary.txt'), 'w') as summary_file: with redirect_stdout(summary_file): if (model_summary is not None): print(model_summary) print( "WARNING: The summary function reports duplicate parameters for multi-GPU case" ) else: print("WARNING: Unable to obtain summary of model") # del model_tmp_copy start_time = time.time() if args.evaluate: test_loss, correct = test(args, model, device, test_loader, loss_fn) test_log = [(test_loss, correct / 1e4)] with open(os.path.join(model_dir, "test_log.csv"), "w") as test_log_file: test_log_csv = csv.writer(test_log_file) test_log_csv.writerow(['test_loss', 'correct']) test_log_csv.writerows(test_log) else: ################################################################################################################################### # Start recording training usage metrics global is_training is_training = True t = report_usage_training() ################################################################################################################################### train_log = [] ################################################################################################################################### test_start = None test_end = None ################################################################################################################################### for epoch in range(1, args.epochs + 1): train_loss = train(args, model, device, train_loader, loss_fn, optimizer, epoch) test_start = time.time() test_loss, correct = test(args, model, device, test_loader, loss_fn) test_end = time.time() ################################################################################################################################### # Record test-specific metrics test_set_size = len(test_loader.dataset) eval_time = test_end - test_start global testing_perf testing_perf = testing_perf.append( { 'TestSetSize': test_set_size, 'EvaluationTime': eval_time, 'Loss': test_loss, 'Correct%': correct / test_set_size, 'Epoch': epoch }, ignore_index=True) ################################################################################################################################### if (args.print_weights): with open( os.path.join(model_dir, 'weights_log_' + str(epoch) + '.txt'), 'w') as weights_log_file: with redirect_stdout(weights_log_file): # Log model's state_dict print("Model's state_dict:") # TODO: Use checkpoint above for param_tensor in model.state_dict(): print(param_tensor, "\t", model.state_dict()[param_tensor].size()) print(model.state_dict()[param_tensor]) print("") train_log.append((epoch, train_loss, test_loss, correct / 1e4)) ################################################################################################################################### # Stop recording training usage metrics is_training = False t.join() global training_perf training_perf.to_excel(model_dir + '\\train_performance.xlsx', index=False) organize_performance_profile_training() organize_performance_accuracy_testing() ################################################################################################################################### with open(os.path.join(model_dir, "train_log.csv"), "w") as train_log_file: train_log_csv = csv.writer(train_log_file) train_log_csv.writerow( ['epoch', 'train_loss', 'test_loss', 'test_accuracy']) train_log_csv.writerows(train_log) if (args.save_model): model_rounded = round_shift_weights(model, shift_base, clone=True) torch.save(model_rounded, os.path.join(model_dir, "model.pth")) torch.save(model_rounded.state_dict(), os.path.join(model_dir, "weights.pth")) end_time = time.time() print("Total Time:", end_time - start_time) if (args.print_weights): if (model_rounded is None): model_rounded = round_shift_weights(model, shift_base, clone=True) with open(os.path.join(model_dir, 'weights_log.txt'), 'w') as weights_log_file: with redirect_stdout(weights_log_file): # Log model's state_dict print("Model's state_dict:") # TODO: Use checkpoint above for param_tensor in model_rounded.state_dict(): print(param_tensor, "\t", model_rounded.state_dict()[param_tensor].size()) print(model_rounded.state_dict()[param_tensor]) print("")
def test_case_0(self, input_mock=None): text_trap = io.StringIO() with redirect_stdout(text_trap): import solution self.assertEqual(text_trap.getvalue(), '19\n')
def _run_file_processor( result_channel: MultiprocessingConnection, parent_channel: MultiprocessingConnection, file_path: str, pickle_dags: bool, dag_ids: Optional[List[str]], thread_name: str, callback_requests: List[CallbackRequest], ) -> None: """ Process the given file. :param result_channel: the connection to use for passing back the result :type result_channel: multiprocessing.Connection :param parent_channel: the parent end of the channel to close in the child :type parent_channel: multiprocessing.Connection :param file_path: the file to process :type file_path: str :param pickle_dags: whether to pickle the DAGs found in the file and save them to the DB :type pickle_dags: bool :param dag_ids: if specified, only examine DAG ID's that are in this list :type dag_ids: list[str] :param thread_name: the name to use for the process that is launched :type thread_name: str :param callback_requests: failure callback to execute :type callback_requests: List[airflow.utils.callback_requests.CallbackRequest] :return: the process that was launched :rtype: multiprocessing.Process """ # This helper runs in the newly created process log: logging.Logger = logging.getLogger("airflow.processor") # Since we share all open FDs from the parent, we need to close the parent side of the pipe here in # the child, else it won't get closed properly until we exit. log.info("Closing parent pipe") parent_channel.close() del parent_channel set_context(log, file_path) setproctitle(f"airflow scheduler - DagFileProcessor {file_path}") try: # redirect stdout/stderr to log with redirect_stdout(StreamLogWriter( log, logging.INFO)), redirect_stderr( StreamLogWriter(log, logging.WARN)), Stats.timer() as timer: # Re-configure the ORM engine as there are issues with multiple processes settings.configure_orm() # Change the thread name to differentiate log lines. This is # really a separate process, but changing the name of the # process doesn't work, so changing the thread name instead. threading.current_thread().name = thread_name log.info("Started process (PID=%s) to work on %s", os.getpid(), file_path) dag_file_processor = DagFileProcessor(dag_ids=dag_ids, log=log) result: Tuple[int, int] = dag_file_processor.process_file( file_path=file_path, pickle_dags=pickle_dags, callback_requests=callback_requests, ) result_channel.send(result) log.info("Processing %s took %.3f seconds", file_path, timer.duration) except Exception: # Log exceptions through the logging framework. log.exception("Got an exception! Propagating...") raise finally: # We re-initialized the ORM within this Process above so we need to # tear it down manually here settings.dispose_orm() result_channel.close()
def _run_smoketest(): print_step('Starting Raiden') config = deepcopy(App.DEFAULT_CONFIG) if args.get('extra_config', dict()): merge_dict(config, args['extra_config']) del args['extra_config'] args['config'] = config raiden_stdout = StringIO() with contextlib.redirect_stdout(raiden_stdout): app = run_app(**args) try: raiden_api = RaidenAPI(app.raiden) rest_api = RestAPI(raiden_api) (api_host, api_port) = split_endpoint(args['api_address']) api_server = APIServer(rest_api, config={'host': api_host, 'port': api_port}) api_server.start() raiden_api.channel_open( registry_address=contract_addresses[CONTRACT_TOKEN_NETWORK_REGISTRY], token_address=to_canonical_address(token.contract.address), partner_address=to_canonical_address(TEST_PARTNER_ADDRESS), ) raiden_api.set_total_channel_deposit( contract_addresses[CONTRACT_TOKEN_NETWORK_REGISTRY], to_canonical_address(token.contract.address), to_canonical_address(TEST_PARTNER_ADDRESS), TEST_DEPOSIT_AMOUNT, ) token_addresses = [to_checksum_address(token.contract.address)] success = False print_step('Running smoketest') error = run_smoketests( app.raiden, args['transport'], token_addresses, contract_addresses[CONTRACT_ENDPOINT_REGISTRY], debug=debug, ) if error is not None: append_report('Smoketest assertion error', error) else: success = True finally: app.stop() app.raiden.get() node = ethereum[0] node.send_signal(2) err, out = node.communicate() append_report('Ethereum stdout', out) append_report('Ethereum stderr', err) append_report('Raiden Node stdout', raiden_stdout.getvalue()) if success: print_step(f'Smoketest successful') else: print_step(f'Smoketest had errors', error=True) return success
async def ev(self, ctx: utils.Context, *, content: str): """Evaluates some Python code Gracefully stolen from Rapptz -> https://github.com/Rapptz/RoboDanny/blob/rewrite/cogs/admin.py#L72-L117""" # Make the environment env = { 'bot': self.bot, 'ctx': ctx, 'channel': ctx.channel, 'author': ctx.author, 'guild': ctx.guild, 'message': ctx.message, 'self': self, } env.update(globals()) # Make code and output string content = self._cleanup_code(content) code = f'async def func():\n{textwrap.indent(content, " ")}' # Make the function into existence stdout = io.StringIO() try: exec(code, env) except Exception as e: return await ctx.send(f'```py\n{e.__class__.__name__}: {e}\n```') # Grab the function we just made and run it func = env['func'] try: # Shove stdout into StringIO with contextlib.redirect_stdout(stdout): ret = await func() except Exception: # Oh no it caused an error value = stdout.getvalue() await ctx.send(f'```py\n{value}{traceback.format_exc()}\n```') else: # Oh no it didn't cause an error value = stdout.getvalue() # Give reaction just to show that it ran await ctx.message.add_reaction("\N{OK HAND SIGN}") # If the function returned nothing if ret is None: # It might have printed something if value: await ctx.send(f'```py\n{value}\n```') # If the function did return a value else: self._last_result = ret result_raw = ret or value result = str(result_raw) if type(result_raw) == dict: try: result = json.dumps(result_raw, indent=4) except Exception: pass if type(result_raw) == dict and type(result) == str: text = f'```json\n{result}\n```' else: text = f'```py\n{result}\n```' if len(text) > 2000: await ctx.send(file=discord.File(io.StringIO(result), filename='ev.txt')) else: await ctx.send(text)
# -t exon # -f bam ############################################ print("\tUsing BAM file `" + bamFile + "`") if not os.path.exists(bamFile): print( "\t\tThe system couldn't locate the file, trying the next one ..." ) badFileList.append([bamFile, "Does Not Exist"]) continue samPass = [bamFile] # We need to redirect stdout to a buffer to write a string # See: # https://stackoverflow.com/a/22434594/6860368 outputString = "" with io.StringIO() as buf, redirect_stdout(buf): try: count.count_reads_in_features( samPass, # BAM file gff3File, # GFF3 file "bam", # -f flag "name", # default 30000000, # default "no", # -s flag "union", # -m flag "all", # --nonunique flag "score", # default "score", # default "exon", # -t flag "gene_id", # -i flag, default (), # Default
def set_coco_api(args, dataset_name, seqs): if args.num != -1 and not args.direct_avg: # create temp json file if not using all sequences save_file = osp.join( 'datasets/waymo_labels/subsets', '{}_{}seqs.json'.format(dataset_name, args.num), ) if not osp.exists(save_file): new_imgs = [] new_annos = [] for seq in seqs[0]: metadata = MetadataCatalog.get(seq) json_file = PathManager.get_local_path(metadata.json_file) data = json.load(open(json_file)) new_imgs.extend(data['images']) new_annos.extend(data['annotations']) new_data = { 'videos': data['videos'], 'images': new_imgs, 'categories': data['categories'], 'annotations': new_annos, } if 'cameras' in data: new_data['cameras'] = data['cameras'] with open(save_file, 'w') as fp: json.dump(new_data, fp) else: metadata = MetadataCatalog.get(dataset_name) new_data = json.load(open(save_file)) img_ids = list(set([img['id'] for img in new_data['images']])) json_file = save_file else: metadata = MetadataCatalog.get(dataset_name) json_file = PathManager.get_local_path(metadata.json_file) img_ids = None if args.direct_avg and args.full_data: data = json.load(open(json_file)) img_ids = list(set([img['id'] for img in data['images']])) # set up coco_api with contextlib.redirect_stdout(io.StringIO()): coco_api = COCO(json_file) sub_apis = [] img_split = None basename = osp.splitext(json_file)[0].replace('_full', '') try: with contextlib.redirect_stdout(io.StringIO()): for i in range(args.range): name = basename + '_{}.json'.format(i) api = COCO(name) sub_apis.append(api) # print(basename+'_img_ids.json') with open(basename + '_img_ids.json') as fp: img_split = json.load(fp) except: pass # thing_classes thing_clases = metadata.get("thing_classes") return coco_api, thing_clases, sub_apis, img_split, img_ids
def check_model_representation(model, X, y=None, convs=None, output_names=None, only_float=True, verbose=False, suffix="", fLOG=None): """ Checks that a trained model can be exported in a specific list of formats and produces the same outputs if the representation can be used to predict. @param model model (a class or an instance of a model but not trained) @param X features @param y targets @param convs list of format to check, all possible by default ``['json', 'c']`` @param output_names list of output columns (can be None, a default value is infered based on scikit-learn output then) @param verbose print some information @param suffix add this to disambiguate module @param fLOG logging function @return function to call to run the prediction """ if not only_float: raise NotImplementedError( # pragma: no cover "Only float are allowed.") if isinstance(X, list): X = pandas.DataFrame(X) if len(X.shape) != 2: raise ValueError( # pragma: no cover "X cannot be converted into a proper DataFrame. It has shape {0}." "".format(X.shape)) if only_float: X = X.as_matrix() if isinstance(y, list): y = numpy.array(y) if convs is None: convs = ['json', 'c'] # sklearn if not hasattr(model.__class__, "fit"): # It is a class object and not an instance. We use the default values. model = model() model.fit(X, y) h = random.randint(0, X.shape[0] - 1) if isinstance(X, pandas.DataFrame): oneX = X.iloc[h, :].astype(numpy.float32) else: oneX = X[h, :].ravel().astype(numpy.float32) # model or transform moneX = numpy.resize(oneX, (1, len(oneX))) if hasattr(model, "predict"): ske = model.predict(moneX) else: ske = model.transform(moneX) if verbose and fLOG: fLOG("---------------------") fLOG(type(oneX), oneX.dtype) fLOG(model) for k, v in sorted(model.__dict__.items()): if k[-1] == '_': fLOG(" {0}={1}".format(k, v)) fLOG("---------------------") # grammar gr = sklearn2graph(model, output_names=output_names) lot = gr.execute(Features=oneX) if verbose and fLOG: fLOG(gr.graph_execution()) # verification check_is_almost_equal(lot, ske) # default for output_names if output_names is None: if len(ske.shape) == 1: output_names = ["Prediction"] elif len(ske.shape) == 2: output_names = ["p%d" % i for i in range(ske.shape[1])] else: raise ValueError( # pragma: no cover "Cannot guess default values for output_names.") for lang in convs: if lang in ('c', ): code_c = gr.export(lang=lang)['code'] if code_c is None: raise ValueError("cannot be None") # pragma: no cover compile_fct = compile_c_function from contextlib import redirect_stdout, redirect_stderr from io import StringIO fout = StringIO() ferr = StringIO() with redirect_stdout(fout): with redirect_stderr(ferr): try: fct = compile_fct( code_c, len(output_names), suffix=suffix, fLOG=lambda s: fout.write(s + "\n")) except Exception as e: # pragma: no cover raise RuntimeError( "Unable to compile a code\n-OUT-\n{0}\n-ERR-\n{1}\n-CODE-" "\n{2}".format(fout.getvalue(), ferr.getvalue(), code_c)) from e if verbose and fLOG: fLOG("-----------------") fLOG(output_names) fLOG("-----------------") fLOG(code_c) fLOG("-----------------") fLOG("h=", h, "oneX=", oneX) fLOG("-----------------") lotc = fct(oneX) check_is_almost_equal( lotc, ske, message="Issue with lang='{0}'".format(lang)) lotc_exp = lotc.copy() lotc2 = fct(oneX, lotc) if not numpy.array_equal(lotc_exp, lotc2): raise ValueError( # pragma: no cover "Second call returns different results.\n{0}\n{1}".format( lotc_exp, lotc2)) else: ser = gr.export(lang="json", hook={'array': lambda v: v.tolist()}) if ser is None: raise ValueError( # pragma: no cover "No output for long='{0}'".format(lang))
def save_params(self,filename): with open(filename, 'w') as f: with redirect_stdout(f): #Write error report lm.report_fit(out1.params)
def _explain(args, base_dir, workflows_dir, config_path): """ Run a complete dryrun workflow, then return the explanations of each rule used. """ if not os.path.exists(config_path): sys.stdout.write( f"The config file: {config_path} does not exist.\nProvide a path to the config file with " f"--config or if you do not have a config file run:\n" f"seq2science init {args.workflow}\n") sys.exit(1) # parse the args parsed_args = { "snakefile": os.path.join(workflows_dir, args.workflow.replace("-", "_"), "Snakefile"), "dryrun": True, "forceall": True, "quiet": False } # get the additional snakemake options snakemake_options = args.snakemakeOptions if args.snakemakeOptions is not None else dict( ) snakemake_options.setdefault("config", {}).update( {"rule_dir": os.path.join(base_dir, "rules")}) snakemake_options["configfiles"] = [config_path] parsed_args.update(snakemake_options) # parse the profile if args.profile is not None: profile_file = snakemake.get_profile_file(args.profile, "config.yaml") if profile_file is None: subjectively_prettier_error( profile_arg, "profile given but no config.yaml found.") add_profile_args(profile_file, parsed_args) # cores parsed_args["cores"] = 999 # starting message rules_used = { "start": f"\nPreprocessing of reads was done automatically with workflow tool " f"seq2science v{seq2science.__version__} (https://doi.org/10.5281/zenodo.3921913)." } def log_handler(log): if log["level"] == "job_info" and "msg" in log and log[ "msg"] is not None and log["name"] not in rules_used: rules_used[log["name"]] = log["msg"] parsed_args["log_handler"] = [log_handler] parsed_args["config"]["explain_rule"] = True # run snakemake (silently) with open(os.devnull, "w") as null: with contextlib.redirect_stdout(null), contextlib.redirect_stderr( null): exit_code = snakemake.snakemake(**parsed_args) print(" ".join(rules_used.values())) sys.exit(0) if exit_code else sys.exit(1)
def test_initialize_logging_with_debug(self): with redirect_stdout(io.StringIO()): initialize_logging(debug=True) self.assertEqual(logging.getLogger().level, logging.DEBUG) self.assertEqual(logging.getLogger('limis').level, logging.DEBUG)
async def repl(self, ctx): msg = ctx.message variables = { 'ctx': ctx, 'bot': self.bot, 'message': msg, 'server': msg.guild, 'channel': msg.channel, 'author': msg.author, 'last': None, } if msg.channel.id in self.sessions: await ctx.send( 'Already running a REPL session in this channel. Exit it with `quit`.' ) return self.sessions.add(msg.channel.id) await ctx.send( 'Enter code to execute or evaluate. `exit()` or `quit` to exit.') while True: response = await self.bot.wait_for( 'message', check=lambda m: m.content.startswith('`') and m.author == msg. author and m.channel == msg.channel) cleaned = self.cleanup_code(response.content) if cleaned in ('quit', 'exit', 'exit()'): await ctx.send('Exiting.') self.sessions.remove(msg.channel.id) return executor = exec if cleaned.count('\n') == 0: # single statement, potentially 'eval' try: code = compile(cleaned, '<repl session>', 'eval') except SyntaxError: pass else: executor = eval if executor is exec: try: code = compile(cleaned, '<repl session>', 'exec') except SyntaxError as e: await ctx.send(self.get_syntax_error(e)) continue variables['message'] = response fmt = None stdout = io.StringIO() try: with redirect_stdout(stdout): result = executor(code, variables) if inspect.isawaitable(result): result = await result except Exception as e: value = stdout.getvalue() fmt = '```py\n{}{}\n```'.format(value, traceback.format_exc()) else: value = stdout.getvalue() if result is not None: fmt = '```py\n{}{}\n```'.format(value, result) variables['last'] = result elif value: fmt = '```py\n{}\n```'.format(value) try: if fmt is not None: if len(fmt) > 2000: await msg.channel.send('Content too big to be printed.' ) else: await msg.channel.send(fmt) except discord.Forbidden: pass except discord.HTTPException as e: await msg.channel.send('Unexpected error: `{}`'.format(e))
def setUp(self): json_file = MetadataCatalog.get("coco_2017_val_100").json_file if not os.path.isfile(json_file): raise unittest.SkipTest("{} not found".format(json_file)) with contextlib.redirect_stdout(io.StringIO()): self.coco = COCO(json_file)
def test_display(self): """Method to test display output of square with # char""" r1 = Rectangle(2, 3, 0, 0) meow = "##\n##\n##\n" r1.display() assert re.match(r'^##\n##\n##\n$', meow) r1 = Rectangle(2, 3, 3, 0) meow = " ##\n ##\n ##\n" assert re.match(r'^ ##\n ##\n ##\n$', meow) """output = sys.stdout r1 = Rectangle(3,2, 0, 0) """ r1 = Rectangle(2, 3, x=0, y=0) outputStr = io.StringIO() with contextlib.redirect_stdout(outputStr): r1.display() self.assertEqual("##\n##\n##\n", outputStr.getvalue()) """ self.assertEqual(r1.display(), meow) suite = unittest.TestLoader().loadTestsFromModule(__Rectangle__) with io.StringIO() as buf: with contextlib.redirect_stdout(buf): unittest.TextTestRunner(stream=buf).run(suite) """ r1 = Rectangle(2, 3, 1, 1) meow = "\n ##\n ##\n ##\n" outputStr = io.StringIO() with contextlib.redirect_stdout(outputStr): r1.display() self.assertEqual("\n ##\n ##\n ##\n", outputStr.getvalue()) r2 = Rectangle(3, 2) outputStr = io.StringIO() with contextlib.redirect_stdout(outputStr): r2.display() self.assertEqual("###\n###\n", outputStr.getvalue()) r2 = Rectangle(3, 2, 2) outputStr = io.StringIO() with contextlib.redirect_stdout(outputStr): r2.display() self.assertEqual(" ###\n ###\n", outputStr.getvalue()) r2 = Rectangle(3, 2, 2, 0) outputStr = io.StringIO() with contextlib.redirect_stdout(outputStr): r2.display() self.assertEqual(" ###\n ###\n", outputStr.getvalue()) r2 = Rectangle(3, 2, 0, 1, 1) outputStr = io.StringIO() with contextlib.redirect_stdout(outputStr): r2.display() self.assertEqual("\n###\n###\n", outputStr.getvalue()) r2 = Rectangle(3, 2, 0, 0, 1) outputStr = io.StringIO() with contextlib.redirect_stdout(outputStr): r2.display() self.assertEqual("###\n###\n", outputStr.getvalue())
def __init__(self, clargs, data_instance, vae_latent_dim, vae_hidden_dims, dnn_hidden_dims, generationID=0, chromosomeID=0, vae_kl_weight=1.0, dnn_weight=1.0, dnn_kl_weight=1.0, verbose=False): self.verbose = verbose self.clargs = clargs self.data_instance = data_instance self.generationID = generationID self.chromosomeID = chromosomeID self.time_stamp = clargs.time_stamp self.vae_latent_dim = vae_latent_dim self.vae_hidden_dims = vae_hidden_dims self.dnn_hidden_dims = dnn_hidden_dims self.vae_kl_weight = vae_kl_weight self.dnn_weight = dnn_weight self.dnn_kl_weight = dnn_kl_weight self.params_dict = {} for k, layer_size in enumerate(self.vae_hidden_dims): self.params_dict['size_vae_hidden{}'.format(k)] = layer_size self.params_dict['vae_latent_dim'] = self.vae_latent_dim for k, layer_size in enumerate(self.dnn_hidden_dims): self.params_dict['size_dnn_hidden{}'.format(k)] = layer_size self.model_dir = clargs.model_dir self.run_name = clargs.run_name self.predictor_type = clargs.predictor_type self.original_dim = clargs.original_dim self.dnn_weight = clargs.dnn_weight self.optimizer = clargs.optimizer self.batch_size = clargs.batch_size self.use_prev_input = False self.dnn_out_dim = clargs.n_labels self.dnn_latent_dim = clargs.n_labels - 1 self.get_model() self.neural_net = self.model self.fitness = 0 assert (os.path.exists(self.model_dir)), "{} does not exist.".format( self.model_dir) self.model_topology_savefile = '{}/{}_{}_{}_model_topology_savefile_{}.save' self.model_topology_savefile = self.model_topology_savefile.format( self.model_dir, self.run_name, self.generationID, self.chromosomeID, self.time_stamp) with open(self.model_topology_savefile, 'w') as f: with redirect_stdout(f): self.neural_net.summary() yaml_filename = self.model_topology_savefile.replace('.save', '.yaml') with open(yaml_filename, 'w') as yaml_fileout: yaml_fileout.write(self.neural_net.to_yaml()) # save model args json_filename = self.model_topology_savefile.replace('.save', '.json') with open(json_filename, 'w') as json_fileout: json_fileout.write(self.neural_net.to_json()) if verbose: self.neural_net.summary()