def test_exception_handler():
from minghu6.algs.decorator import exception_handler
def myhandler(e):
print('Caught exception!', e)
# Examples
# Specify exceptions in order, first one is handled first
# last one last.
@exception_handler(myhandler, (ZeroDivisionError,))
def f1():
1 / 0
@exception_handler()
def f3(*pargs):
l = pargs
return l.index(10)
buff = StringIO()
with redirect_stdout(buff):
f1()
assert buff.getvalue() == 'Caught exception! division by zero\n'
buff = StringIO()
with redirect_stdout(buff):
f3()
assert buff.getvalue() == 'ValueError : tuple.index(x): x not in tuple\n'
def test_pants_fitter(self):
with io.StringIO() as buf, redirect_stdout(buf):
with patch("builtins.input", side_effect=["Ziggy", 34, 2, "fancy"]):
self.assertIsNone(pants_fitter())
self.assertEquals(
"Greetings Ziggy welcome to Pants-R-Us\n" + "2 pairs of large fancy pants: $ 200\n", buf.getvalue()
)
with io.StringIO() as buf, redirect_stdout(buf):
with patch("builtins.input", side_effect=["Elmer", 31, 10, "regular"]):
pants_fitter()
self.assertEquals(
"Greetings Elmer welcome to Pants-R-Us\n" + "10 pairs of medium regular pants: $ 400\n",
buf.getvalue(),
)
with io.StringIO() as buf, redirect_stdout(buf):
with patch("builtins.input", side_effect=["Minnie", 12, 1, "fancy"]):
pants_fitter()
self.assertEquals(
"Greetings Minnie welcome to Pants-R-Us\n" + "1 pairs of small fancy pants: $ 100\n", buf.getvalue()
)
with io.StringIO() as buf, redirect_stdout(buf):
with patch("builtins.input", side_effect=["Henry", 30, 4, "regular"]):
pants_fitter()
self.assertEquals(
"Greetings Henry welcome to Pants-R-Us\n" + "4 pairs of medium regular pants: $ 160\n",
buf.getvalue(),
)
with io.StringIO() as buf, redirect_stdout(buf):
with patch("builtins.input", side_effect=["Ali Smith", -10, 90, "fancy"]):
pants_fitter()
self.assertEquals(
"Greetings Ali Smith welcome to Pants-R-Us\n" + "90 pairs of small fancy pants: $ 9000\n",
buf.getvalue(),
)
def test_create_thumbnails(product_with_image, settings):
sizeset = settings.VERSATILEIMAGEFIELD_RENDITION_KEY_SETS["products"]
product_image = product_with_image.images.first()
# There's no way to list images created by versatile prewarmer
# So we delete all created thumbnails/crops and count them
log_deleted_images = io.StringIO()
with redirect_stdout(log_deleted_images):
product_image.image.delete_all_created_images()
log_deleted_images = log_deleted_images.getvalue()
# Image didn't have any thumbnails/crops created, so there's no log
assert not log_deleted_images
create_thumbnails(product_image.pk, ProductImage, "products")
log_deleted_images = io.StringIO()
with redirect_stdout(log_deleted_images):
product_image.image.delete_all_created_images()
log_deleted_images = log_deleted_images.getvalue()
for image_name, method_size in sizeset:
method, size = method_size.split("__")
if method == "crop":
assert product_image.image.crop[size].name in log_deleted_images
elif method == "thumbnail":
assert (
product_image.image.thumbnail[size].name in log_deleted_images
) # noqa
def test_try_catch(self): # 69
self.assertEval('(try* 123 (catch* e 456))', self.env, '123')
f = StringIO()
with redirect_stdout(f):
res = pymal.rep('(try* (abc 1 2)'
' (catch* exc (prn "exc is:" exc)))', self.env)
self.assertEqual(res, 'nil')
self.assertEqual(f.getvalue(),
'"exc is:" Symbol value is void: \'abc\'\n')
g = StringIO()
with redirect_stdout(g):
res = pymal.rep('(try* (throw (list 1 2 3))'
' (catch* exc (do (prn "err:" exc) 7)))',
self.env)
self.assertEqual(res, '7')
self.assertEqual(g.getvalue(),
'"err:" (1 2 3)\n')
h = StringIO()
with redirect_stdout(h):
res = pymal.rep('(try* (throw "my exception")'
' (catch* exc (do (prn "err:" exc) 7)))',
self.env)
self.assertEqual(res, '7')
self.assertEqual(h.getvalue(),
'"err:" my exception\n')
def test_get_F_cm_Guo_continuous_and_discrete(self):
dcmt = DiscreteCMTransforms(e_D2Q9, u2D, F2D, rho)
F_cm_Guo_disc = get_mom_vector_from_discrete_def(dcmt.get_force_Guo,
discrete_transform=dcmt.get_cm,
moments_order=moments_dict['D2Q9'],
serial_run=True)
from SymbolicCollisions.core.ContinousCMTransforms import \
ContinousCMTransforms, get_mom_vector_from_continuous_def
from SymbolicCollisions.core.cm_symbols import \
F3D, dzeta3D, u3D
ccmt = ContinousCMTransforms(dzeta3D, u3D, F3D, rho)
F_cm_Guo_cont = get_mom_vector_from_continuous_def(ccmt.get_force_Guo,
continuous_transformation=ccmt.get_cm,
moments_order=moments_dict['D2Q9'],
serial_run=True)
# print_as_vector(F_cm_Guo_cont, 'F_cm')
results = [F_cm_Guo_disc, F_cm_Guo_cont]
f = io.StringIO()
with redirect_stdout(f):
print_as_vector(hardcoded_F_cm_Guo_hydro_LB_velocity_based_D2Q9, 'F_cm')
expected_result = f.getvalue()
for result in results:
f = io.StringIO()
with redirect_stdout(f):
print_as_vector(result, 'F_cm')
out = f.getvalue()
assert out == expected_result
def test_timer():
from minghu6.algs.decorator import timer
# Test on functions
@timer(trace=True, label='[CCC]==>')
def listcomp(N): # Like listcomp = timer(...)(listcomp)
return [x * 2 for x in range(N)] # listcomp(...) triggers onCall
@timer(trace=True, label='[MMM]==>', unit='s')
def mapcall(N):
return list(map((lambda x: x * 2), range(N))) # list() for 3.0 views
for func in (listcomp, mapcall):
buff = StringIO()
with redirect_stdout(buff):
result = func(5) # Time for this call, all calls, return value
if func is listcomp:
assert buff.getvalue().startswith('[CCC]==>listcomp')
elif func is mapcall:
assert buff.getvalue().startswith('[MMM]==>mapcall')
assert result == [0, 2, 4, 6, 8]
assert isinstance(func.alltime, (float, int)) # Total time for all calls
# Test on methods
class Person:
def __init__(self, name, pay):
self.name = name
self.pay = pay
@timer()
def give_raise(self, percent): # giveRaise = timer()(giveRaise)
self.pay *= (1.0 + percent) # tracer remembers giveRaise
@timer(label='**')
def last_name(self): # lastName = timer(...)(lastName)
return self.name.split()[-1]
bob = Person('Bob Smith', 50000)
sue = Person('Sue Jones', 100000)
buff = StringIO()
with redirect_stdout(buff):
bob.give_raise(.10)
assert buff.getvalue().startswith('give_raise')
print(buff.getvalue())
with redirect_stdout(buff):
sue.give_raise(.20) # runs onCall(sue, .10)
assert (int(bob.pay), int(sue.pay)) == (55000, 120000)
buff = StringIO()
with redirect_stdout(buff):
assert ((bob.last_name(), sue.last_name()) == ('Smith', 'Jones'))
assert buff.getvalue().startswith('**last_name')
def run_online_mode(args):
if args.n == 1:
registry = look_up(args.registry)()
finish = None
if args.msgpack:
newin = os.fdopen(sys.stdin.fileno(), "rb", buffering=0)
input_stream = MsgPackObjectReader(registry, newin, deref=True)
output_stream = MsgPackObjectWriter(registry, sys.stdout.buffer, host=args.name)
else:
input_stream = JSONObjectReader(registry, sys.stdin, deref=True)
output_stream = JSONObjectWriter(registry, sys.stdout, host=args.name)
# run the init function if it is given
if args.init:
with redirect_stdout(sys.stderr):
look_up(args.init)()
if args.finish:
finish = look_up(args.finish)
for msg in input_stream:
if isinstance(msg, JobMessage):
key, job = msg
elif isinstance(msg, tuple):
key, job = msg
else:
continue
if args.jobdirs:
# make a directory
os.mkdir("noodles-{0}".format(key.hex))
# enter it
os.chdir("noodles-{0}".format(key.hex))
if args.verbose:
print(
"worker: ",
job.foo.__name__,
job.bound_args.args,
job.bound_args.kwargs,
file=sys.stderr,
flush=True,
)
with redirect_stdout(sys.stderr):
result = run_job(key, job)
if args.verbose:
print("result: ", result, file=sys.stderr, flush=True)
if args.jobdirs:
# parent directory
os.chdir("..")
output_stream.send(result)
if finish:
finish()
def test_take_damage(self):
with io.StringIO() as buf, redirect_stdout(buf):
self.assertIsNone(self.rich.take_damage(5))
self.assertEqual(5, self.rich.hit_points)
self.assertEqual('\tRich has 5 hit points remaining.\n', buf.getvalue())
with io.StringIO() as buf, redirect_stdout(buf):
self.rich.take_damage(5)
self.assertEqual(0, self.rich.hit_points)
self.assertEqual('\tAlas, Rich has fallen!\n', buf.getvalue())
def test_check_for_winner(self):
self.assertIsNotNone(check_for_winner("Elmer", 30))
with io.StringIO() as buf, redirect_stdout(buf):
self.assertTrue(check_for_winner("Ziggy",50))
self.assertEqual("THE WINNER IS: Ziggy!!!!!\n", buf.getvalue())
with io.StringIO() as buf, redirect_stdout(buf):
self.assertTrue(check_for_winner("Ziggy",51))
self.assertEqual("THE WINNER IS: Ziggy!!!!!\n", buf.getvalue())
with io.StringIO() as buf, redirect_stdout(buf):
self.assertFalse(check_for_winner("Elmer", 49))
self.assertEqual("", buf.getvalue())
def test_process_standard_options_for_setup_help(self):
f = StringIO()
with redirect_stdout(f):
process_standard_options_for_setup_help('--help-commands')
self.assertIn('Commands processed by pyquickhelper:', f.getvalue())
f = StringIO()
with redirect_stdout(f):
process_standard_options_for_setup_help(['--help', 'unittests'])
self.assertIn('-f file', f.getvalue())
f = StringIO()
with redirect_stdout(f):
process_standard_options_for_setup_help(['--help', 'clean_space'])
self.assertIn('clean unnecessary spaces', f.getvalue())
def test_doctor(self):
with io.StringIO() as buf, redirect_stdout(buf):
with patch("builtins.input", side_effect=[119, 79]):
self.assertIsNone(doctor())
self.assertEquals("Your blood pressure is normal.\n", buf.getvalue())
with io.StringIO() as buf, redirect_stdout(buf):
with patch("builtins.input", side_effect=[133, 79]):
doctor()
self.assertEquals("Your blood pressure is high.\n", buf.getvalue())
with io.StringIO() as buf, redirect_stdout(buf):
with patch("builtins.input", side_effect=[120, 80]):
doctor()
self.assertEquals("Your blood pressure is high.\n", buf.getvalue())
def test_word_length(self):
with io.StringIO() as buf, redirect_stdout(buf):
self.assertIsNone(word_length("liversnaps", 7))
self.assertEqual("Longer than 7 characters: liversnaps\n", buf.getvalue())
with io.StringIO() as buf, redirect_stdout(buf):
word_length("earwax", 5)
self.assertEqual("Longer than 5 characters: earwax\n", buf.getvalue())
with io.StringIO() as buf, redirect_stdout(buf):
word_length("chickenfat", 10)
self.assertEqual("Exactly 10 characters: chickenfat\n", buf.getvalue())
with io.StringIO() as buf, redirect_stdout(buf):
word_length("Gross!", 13)
self.assertEqual("Shorter than 13 characters: Gross!\n", buf.getvalue())
def test_play_turn(self):
random.seed(87) #[2, 6, 2, 5, 1, 3, 3, 6, 5, 6, 3, 6, 1]
with io.StringIO() as buf, redirect_stdout(buf):
with patch('builtins.input', side_effect = ['y','y', 'n']):
self.assertEquals(10, play_turn("Henry"))
self.assertEquals("---------- Henry's turn ----------\n" + \
"\t<<< Henry rolls a 2 >>>\n" + \
"\tPoints: 2\n\t<<< Henry rolls a 6 >>>\n\tPoints: 8\n" + \
"\t<<< Henry rolls a 2 >>>\n\tPoints: 10\n", buf.getvalue())
with io.StringIO() as buf, redirect_stdout(buf):
with patch('builtins.input', side_effect = ['y','y']):
self.assertEquals(0, play_turn("Johnny"))
self.assertEquals("---------- Johnny's turn ----------\n" + \
"\t<<< Johnny rolls a 5 >>>\n" + \
"\tPoints: 5\n\t<<< Johnny rolls a 1 >>>\n" + \
"\t!!! PIG! No points earned, sorry Johnny !!!\n", buf.getvalue())
with io.StringIO() as buf, redirect_stdout(buf):
with patch('builtins.input', side_effect = ['n']):
self.assertEquals(3, play_turn("Henry"))
self.assertEquals("---------- Henry's turn ----------\n" + \
"\t<<< Henry rolls a 3 >>>\n\tPoints: 3\n", buf.getvalue())
# testing prompts
sys.stdin = open('play_turn_in.txt')
with io.StringIO() as buf, redirect_stdout(buf):
play_turn("Johnny")
out_text = "---------- Johnny's turn ----------\n" + \
"\t<<< Johnny rolls a 3 >>>\n" + \
"\tPoints: 3\n" + \
"Roll again, Johnny? (Y/N) " + \
"\t<<< Johnny rolls a 6 >>>\n" + \
"\tPoints: 9\n" + \
"Roll again, Johnny? (Y/N) " + \
"\t<<< Johnny rolls a 5 >>>\n" + \
"\tPoints: 14\n" + \
"Roll again, Johnny? (Y/N) " + \
"\t<<< Johnny rolls a 6 >>>\n" + \
"\tPoints: 20\n" + \
"Roll again, Johnny? (Y/N) " + \
"\t<<< Johnny rolls a 3 >>>\n" + \
"\tPoints: 23\n" + \
"Roll again, Johnny? (Y/N) " + \
"\t<<< Johnny rolls a 6 >>>\n" + \
"\tPoints: 29\n" + \
"Roll again, Johnny? (Y/N) " + \
"\t<<< Johnny rolls a 1 >>>\n" + \
"\t!!! PIG! No points earned, sorry Johnny !!!\n" + \
"(enter to continue)"
self.assertEqual(out_text, buf.getvalue())
sys.stdin = sys.__stdin__
def test_print_l(self):
from contextlib import redirect_stdout
f = io.StringIO()
with redirect_stdout(f):
print_l(["lines"])
s = f.getvalue()
self.assertEqual(s, "lines\n")
f = io.StringIO()
with redirect_stdout(f):
print_l(["xx","xx","xx"])
s = f.getvalue()
self.assertEqual(s, "xx\nxx\nxx\n")
def test_stop_light(self):
with io.StringIO() as buf, redirect_stdout(buf):
self.assertIsNone(stop_light("green", 61))
self.assertEqual("yellow\n", buf.getvalue())
with io.StringIO() as buf, redirect_stdout(buf):
stop_light("green", 60)
self.assertEqual("green\n", buf.getvalue())
with io.StringIO() as buf, redirect_stdout(buf):
stop_light("green", 59)
self.assertEqual("green\n", buf.getvalue())
with io.StringIO() as buf, redirect_stdout(buf):
stop_light("yellow", 5)
self.assertEqual("yellow\n", buf.getvalue())
with io.StringIO() as buf, redirect_stdout(buf):
stop_light("yellow", 6)
self.assertEqual("red\n", buf.getvalue())
with io.StringIO() as buf, redirect_stdout(buf):
stop_light("red", 12)
self.assertEqual("red\n", buf.getvalue())
with io.StringIO() as buf, redirect_stdout(buf):
stop_light("red", 56)
self.assertEqual("green\n", buf.getvalue())
with io.StringIO() as buf, redirect_stdout(buf):
stop_light("red", 54)
self.assertEqual("red\n", buf.getvalue())
def test_cm_vector_from_continuous_def(self):
# this test runs long without output and CI may consider it as a timeout :/
ccmt = ContinousCMTransforms(dzeta3D, u3D, F3D, rho)
lattices = [
'D2Q9',
'D3Q19',
'D2Q9',
'D2Q9',
'D3Q19',
]
functions = [
ccmt.get_Maxwellian_DF,
ccmt.get_Maxwellian_DF,
ccmt.get_force_Guo,
ccmt.get_force_He_MB,
ccmt.get_force_He_MB,
]
expected_results = [
hardcoded_cm_eq_compressible_D2Q9,
hardcoded_cm_eq_compressible_D3Q19,
hardcoded_F_cm_Guo_hydro_LB_velocity_based_D2Q9,
hardcoded_F_cm_hydro_density_based_D2Q9,
hardcoded_F_cm_hydro_density_based_D3Q19,
]
for fun, lattice, expected_result in zip(functions, lattices, expected_results):
cm_eq = get_mom_vector_from_continuous_def(fun,
continuous_transformation=ccmt.get_cm,
moments_order=moments_dict[lattice],
serial_run=True
)
# print("------------\n\n")
# print_as_vector(cm_eq, 'CM')
# print_as_vector(expected_result, 'CM_expected')
# print("------------\n\n")
f = io.StringIO()
with redirect_stdout(f):
print_as_vector(cm_eq, 'cm_eq')
out = f.getvalue()
f2 = io.StringIO()
with redirect_stdout(f2):
print_as_vector(expected_result, 'cm_eq')
ccode_expected_result = f2.getvalue()
assert ccode_expected_result == out
def test(self):
stdout = StringIO()
with redirect_stdout(stdout):
self.progress.send(self.input1)
self.assertEqual(stdout.getvalue(), self.expected1)
stdout = StringIO()
with redirect_stdout(stdout):
self.progress.send(self.input2)
self.assertEqual(stdout.getvalue(), self.expected2)
stdout = StringIO()
with redirect_stdout(stdout):
self.progress.send(self.input3)
self.assertEqual(stdout.getvalue(), self.expected3)
def test_apply_with_core_functions(self): # 72
self.assertEval('(apply + (list 2 3))', self.env, '5')
self.assertEval('(apply + 4 (list 5))', self.env, '9')
f = StringIO()
with redirect_stdout(f):
res = pymal.rep('(apply prn (list 1 2 "3" (list)))', self.env)
self.assertEqual(res, 'nil')
self.assertEqual(f.getvalue(), '1 2 "3" ()\n')
g = StringIO()
with redirect_stdout(g):
res = pymal.rep('(apply prn 1 2 (list "3" (list)))', self.env)
self.assertEqual(res, 'nil')
self.assertEqual(g.getvalue(), '1 2 "3" ()\n')
def handle_worker(args):
"""usage: {program} worker [options] <module-path> <operator> <occurrence> [<config-file>]
Run a worker process which performs a single mutation and test run.
Each worker does a minimal, isolated chunk of work: it mutates the
<occurence>-th instance of <operator> in <module-path>, runs the test
suite defined in the configuration, prints the results, and exits.
Normally you won't run this directly. Rather, it will be launched
by an execution engine. However, it can be useful to run this on
its own for testing and debugging purposes.
options:
--keep-stdout Do not squelch stdout
"""
config = load_config(args.get('<config-file>'))
with open(os.devnull, 'w') as devnull:
with redirect_stdout(sys.stdout if args['--keep-stdout'] else devnull):
work_item = cosmic_ray.worker.worker(
Path(args['<module-path>']),
config.python_version, args['<operator>'],
int(args['<occurrence>']),
config.test_command,
None)
sys.stdout.write(json.dumps(work_item, cls=WorkItemJsonEncoder))
return ExitCode.OK
def __call__(self, filename):
global ROBOT_FILE
ROBOT_FILE = '<string>'
fp = super(RobotCode, self).__call__(filename)
code = compile(fp.read(), ROBOT_FILE, 'exec')
if not code.co_names:
raise argparse.ArgumentTypeError('The supplied code is empty')
robot_ns = {}
with contextlib.redirect_stdout(open(os.devnull, 'w')):
exec(code, robot_ns)
for key, val in robot_ns.items():
if not key.startswith('_'):
if is_robot_class(val):
class_ = val
break
else:
raise argparse.ArgumentTypeError('The code does not contain a RobotSnake-based class')
globals().update(robot_ns)
return class_
def defineNetwork(self):
print("Setting up network...")
inputs = Input(shape=(self.nx, self.ny, self.n_times))
conv = Convolution2D(self.n_filters, 3, 3, activation='relu', border_mode='same', init='he_normal')(inputs)
x = self.residual(conv)
for i in range(self.n_conv_layers):
x = self.residual(x)
x = Convolution2D(self.n_filters, 3, 3, border_mode='same', init='he_normal')(x)
x = BatchNormalization()(x)
x = merge([x, conv], 'sum')
final = Convolution2D(6, 1, 1, activation='linear', border_mode='same', init='he_normal')(x)
self.model = Model(input=inputs, output=final)
json_string = self.model.to_json()
f = open('{0}_model.json'.format(self.root), 'w')
f.write(json_string)
f.close()
with open('{0}_summary.txt'.format(self.root), 'w') as f:
with redirect_stdout(f):
self.model.summary()
kerasPlot(self.model, to_file='{0}_model.png'.format(self.root), show_shapes=True)
def run(filename, runid):
print('file:', filename, 'runid:', runid)
filename = os.path.abspath(filename)
basename = os.path.basename(filename)
dirpath = '{}.{}'.format(os.path.basename(filename), runid)
dirpath = os.path.join(result_dir, dirpath)
if not os.path.exists(dirpath):
os.mkdir(dirpath)
try:
rootdir = os.path.abspath('.')
os.chdir(dirpath)
with open('report.txt', 'w') as file, \
redirect_stdout(CompositeStream(stdout, file)):
wrapper = newwrapper(filename)
np.random.seed(1)
run_experiment(time_to_opt, wrapper)
np.random.seed(1)
run_experiment(result_over_time, wrapper, 1)
np.random.seed(1)
if basename in best_penalty:
run_experiment(result_over_time, wrapper, best_penalty[basename], filename='accuracy_over_time')
finally:
os.chdir(rootdir)
def defineNetwork(self):
print("Setting up network...")
inputs = Input(shape=(self.nx, self.ny, self.n_diversity))
x = GaussianNoise(self.noise)(inputs)
conv = Convolution2D(self.n_filters, (3, 3), activation='relu', padding='same', kernel_initializer='he_normal')(x)
x = self.residual(conv)
for i in range(self.n_conv_layers):
x = self.residual(x)
x = Convolution2D(self.n_filters, (3, 3), padding='same', kernel_initializer='he_normal')(x)
x = BatchNormalization()(x)
x = add([x, conv])
final = Convolution2D(1, (1, 1), activation='relu', padding='same', kernel_initializer='he_normal')(x)
self.model = Model(inputs=inputs, outputs=final)
json_string = self.model.to_json()
f = open('{0}_model.json'.format(self.root), 'w')
f.write(json_string)
f.close()
with open('{0}_summary.txt'.format(self.root), 'w') as f:
with redirect_stdout(f):
self.model.summary()
plot_model(self.model, to_file='{0}_model.png'.format(self.root), show_shapes=True)
def reverse_file(filename, symbol, options):
gctx = GlobalContext()
gctx.sectionsname = False
gctx.color = False
gctx.filename = filename
gctx.entry = symbol
gctx.quiet = True
for o in options:
if o == "--raw x86":
gctx.raw_type = "x86"
elif o == "--raw x64":
gctx.raw_type = "x64"
elif o.startswith("--rawbase"):
gctx.raw_base = int(o.split(" ")[1], 16)
if not gctx.load_file():
die()
sio = StringIO()
with redirect_stdout(sio):
o = gctx.get_addr_context(gctx.entry).decompile()
if o is not None:
o.print()
postfix = '{0}.rev'.format('' if symbol is None else '_' + symbol)
with open(filename.replace('.bin', postfix)) as f:
assert_equal(sio.getvalue(), f.read())
def test_get_cumulants(self):
from SymbolicCollisions.core.cumulants import get_cumulant
from SymbolicCollisions.core.printers import print_as_vector
from sympy.matrices import Matrix
order = [
(1, 0, 0),
(0, 1, 0),
(0, 0, 1),
(2, 0, 0),
(0, 2, 0),
(0, 0, 2),
(1, 1, 0)]
result = [
'= m100;',
'= m010;',
'= m001;',
'= m200 - m100**2/m000;',
'= m020 - m010**2/m000',
'= m002 - m001**2/m000;',
'= m110 - m010*m100/m000;']
for o, r in zip(order, result):
cumulant, _ = get_cumulant(*o)
f = io.StringIO()
with redirect_stdout(f):
mc = Matrix([cumulant]) # printer works using Matrix format
print_as_vector(mc, 'c')
out = f.getvalue()
assert r in out
async def _eval(ctx, *, body):
"""Evaluates python code"""
env = {
'ctx': ctx,
'channel': ctx.channel,
'author': ctx.author,
'guild': ctx.guild,
'message': ctx.message,
'_': bot._last_result,
'source': inspect.getsource,
'session': bot.session
}
env.update(globals())
body = utils.cleanup_code(body)
stdout = io.StringIO()
err = out = None
to_compile = f'async def func(): \n{textwrap.indent(body, " ")}'
try:
exec(to_compile, env)
except Exception as e:
err = await ctx.send(f'```py\n{e.__class__.__name__}: {e}\n```')
return await ctx.message.add_reaction('\u2049')
func = env['func']
try:
with redirect_stdout(stdout):
ret = await func()
except Exception as e:
value = stdout.getvalue()
err = await ctx.send(f'```py\n{value}{traceback.format_exc()}\n```')
else:
value = stdout.getvalue()
if ret is None:
if value:
try:
out = await ctx.send(f'```py\n{value}\n```')
except:
paginated_text = utils.paginate(value)
for page in paginated_text:
if page == paginated_text[-1]:
out = await ctx.send(f'```py\n{page}\n```')
break
await ctx.send(f'```py\n{page}\n```')
else:
bot._last_result = ret
try:
out = await ctx.send(f'```py\n{value}{ret}\n```')
except:
paginated_text = utils.paginate(f"{value}{ret}")
for page in paginated_text:
if page == paginated_text[-1]:
out = await ctx.send(f'```py\n{page}\n```')
break
await ctx.send(f'```py\n{page}\n```')
if out:
await ctx.message.add_reaction('\u2705') # tick
elif err:
await ctx.message.add_reaction('\u2049') # x
else:
await ctx.message.add_reaction('\u2705')
def _run_complexity_analysis(on_ci):
"""Generates cyclomatic complexity reports for the package
:param bool on_ci: Indicates whether an automated tool is running this operation. Output will be customized for
machine readability
"""
modlog.debug("Running complexity analysis")
# generate cyclomatic complexities for source files in XML format for integration with external tools
pyjen_path = os.path.join(os.getcwd(), "pyjen")
from radon.cli import cc
# TODO: output in XML format when running on CI
standard_output = StringIO()
with redirect_stdout(standard_output):
modlog.debug("Calling radon.cc")
cc(paths=[pyjen_path], show_complexity=True, show_closures=True, total_average=True, xml=on_ci)
modlog.debug("Writing report to disk")
cc_report = os.path.join(log_folder, "radon_complexity.xml")
with open(cc_report, "w") as fh:
fh.write(standard_output.getvalue())
standard_output.close()
modlog.info("Cyclomatic complexity analysis complete. See " + os.path.relpath(cc_report))
def runWith(fcn, args, stdin):
''' Run a function with given arguments and stdin,
and return the output and stdout. '''
# Copy arguments in case the function modifies its inputs
args2 = copy.deepcopy(args)
# Set the stdin
with SetStdin(stdin):
# Redirect the stdout
f = io.StringIO()
with redirect_stdout(f):
# Try the function with the arguments, catch any exceptions
try:
# *args expands the 'args' list into separate arguments
output = fcn(*args2)
except Exception:
output = None
# Provide a warning if the function modifies its inputs
if args != args2:
print('\tWARNING: THIS FUNCTION MODIFIES ITS INPUTS.')
# Extract a string for the stdout
stdout = f.getvalue()
return output, stdout
def next_batch(self, validation=False):
programs = [self.generate_program(hash_mod=0 if validation else 1)
for _ in range(self.batch_size)]
# Execute the programs to get the targets
results = []
for program in programs:
with io.StringIO() as buf, redirect_stdout(buf):
exec(program)
results.append(buf.getvalue()[:-1])
input_sequences = encode_sequences(programs,
symbol_to_idx=SYMBOL_TO_IDX,
sequence_len=INPUT_SEQ_LEN,
#go_symbol=GO_SYMBOL,
pad_symbol=PAD_SYMBOL,
pad_beginning=True,
reverse=True)
input_sequences = dense_to_one_hot(input_sequences,
num_classes=len(SYMBOL_TO_IDX))
target_sequences = encode_sequences(results,
symbol_to_idx=SYMBOL_TO_IDX,
sequence_len=OUTPUT_SEQ_LEN,
go_symbol=None,
pad_beginning=False,
pad_symbol=PAD_SYMBOL)
target_sequences = dense_to_one_hot(target_sequences,
num_classes=len(SYMBOL_TO_IDX))
return input_sequences, target_sequences
def test_process_streams_redirect(self):
# This won't work for asyncio implementation of subprocess
async def test():
prog = bR'''
import sys
print('out', flush=True)
print('err', file=sys.stderr, flush=True)
'''
proc = await asyncio.create_subprocess_exec(
sys.executable, '-c', prog,
loop=self.loop)
out, err = await proc.communicate()
self.assertIsNone(out)
self.assertIsNone(err)
with tempfile.NamedTemporaryFile('w') as stdout:
with tempfile.NamedTemporaryFile('w') as stderr:
with contextlib.redirect_stdout(stdout):
with contextlib.redirect_stderr(stderr):
self.loop.run_until_complete(test())
stdout.flush()
stderr.flush()
with open(stdout.name, 'rb') as so:
self.assertEqual(so.read(), b'out\n')
with open(stderr.name, 'rb') as se:
self.assertEqual(se.read(), b'err\n')
def load_batch_ko_data():
"""
Load all simulations,
"""
def _load_exp_data():
long_df = pd.read_csv(data_path / "datasets" / "long2019_tidy.csv")
long_df["sample_id"] = long_df.Genotype
long_df = long_df.assign(author="Long").rename(
{
"Measurement_ID": "BiGG_ID",
"Original_Value": "normalized_flux",
"Value": "flux",
"Original_ID": "ID",
},
axis=1,
)
long_df = long_df[long_df["Measurement_Type"] == "flux"]
long_df = long_df[~long_df["BiGG_ID"].isna()]
# the same reaction as rpe
long_df = long_df[long_df["sample_id"] != "sgcE"]
long_df = long_df[[
"BiGG_ID", "ID", "flux", "author", "sample_id", "normalized_flux"
]]
"""
douglas_flux_data = pd.read_csv("../../../DataAnalysis/DouglasKineticData/data/flux_data_processed.csv", index_col=0)
douglas_sample_names = {
"Evo04": "WT",
"Evo04gnd": "gnd",
"Evo04pgi": "pgi",
"Evo04sdhCB": "sdh",
"Evo04tpiA": "tpi",
}
douglas_exp_df = douglas_flux_data.query(
"sample_name in @douglas_sample_names.keys()"
)
douglas_exp_df["sample_id"] = douglas_exp_df.sample_name.apply(
lambda x: douglas_sample_names[x]
)
douglas_exp_df["author"] = "McCloskey"
douglas_exp_df = douglas_exp_df.rename(
{"rxn_id": "BiGG_ID", "sampling_median": "flux"}, axis=1
).drop(["sampling_var", "sampling_min", "sampling_max", "sample_name"], axis=1)
x_doug = douglas_exp_df.set_index(["author", "sample_id", "BiGG_ID"]).to_xarray()
x_doug['normalized_flux'] = 100*(x_doug.flux / x_doug.sel(BiGG_ID="GLCptspp").flux)
mccloskey_results = x_doug.to_dataframe().reset_index()
mccloskey_results.head()
"""
return long_df
def _load_kinetic_ko_sims():
"""
Load the data from kinetic models simulations
"""
khodayari_results = load_khodayari(
sample_names="all",
load_path=(path_to_results / "Khodayari"),
id_df=khod_idf,
files=get_khodayari_batch_kos(),
)
kurata_results = load_kurata(
sample_names="all",
load_path=(path_to_results / "Kurata" / "batch_knockouts"),
id_df=kurata_idf,
files=get_kurata_batch_kos(),
mode="batch",
)
millard_results = load_millard(
sample_names="all",
load_path=(path_to_results / "Millard" / "batch_knockouts"),
id_df=millard_idf,
files=get_millard_batch_kos(),
)
chassagnole_results = load_chassagnole(
sample_names="all",
load_path=(path_to_results / "Chassagnole" / "batch_knockouts"),
id_df=chassagnole_idf,
files=get_chassagnole_kos(),
)
simulation_results = pd.concat([
khodayari_results, kurata_results, millard_results,
chassagnole_results
],
sort=False)
return simulation_results
def _load_cobra_ko_sims():
"""
Load simulations from iML1515, ECC2 and iML1515 and ECC2 conditioned on experimental data
"""
iml_results = pd.read_csv(
path_to_results / "COBRA" / "iML1515" / "batch_knockouts" /
"knockouts_all.csv",
index_col=0,
)
# ecc_results = pd.read_csv(
# path_to_results
# / "COBRA"
# / "ECC2"
# / "batch_knockouts"
# / "knockouts_all.csv",
# index_col=0,
# )
exp_iml_results = pd.read_csv(
path_to_results / "COBRA" / "Exp_iML1515" / "batch_knockouts" /
"knockouts_all.csv",
index_col=0,
)
# exp_ecc_results = pd.read_csv(
# path_to_results
# / "COBRA"
# / "Exp_ECC2"
# / "batch_knockouts"
# / "knockouts_all.csv",
# index_col=0,
# )
df = pd.concat([iml_results, exp_iml_results])
# Fix direction to match experimental data
# Only for iML1515
iml_reversed = ["PGM", "PGK", "SUCOAS"]
# fix PGM direction
df.loc[df["ID"].isin(iml_reversed),
"flux"] = (-1 * df.loc[df["ID"].isin(iml_reversed), "flux"])
df.loc[df["ID"].isin(iml_reversed), "normalized_flux"] = (
-1 * df.loc[df["ID"].isin(iml_reversed), "normalized_flux"])
# df = pd.concat([df, ecc_results, exp_ecc_results])
# For both iML1515 and ECC2
# fix RPI direction
ecc_reversed = ["RPI"]
df.loc[df["ID"].isin(ecc_reversed),
"flux"] = (-1 * df.loc[df["ID"].isin(ecc_reversed), "flux"])
df.loc[df["ID"].isin(ecc_reversed), "normalized_flux"] = (
-1 * df.loc[df["ID"].isin(ecc_reversed), "normalized_flux"])
return df
with io.StringIO() as buf, redirect_stdout(buf):
simulation_data = _load_kinetic_ko_sims()
cobra_data = _load_cobra_ko_sims()
exp_data = _load_exp_data()
file_info = buf.getvalue()
return pd.concat([simulation_data, cobra_data, exp_data],
sort=False), file_info
def load_ko_data():
"""
Load all simulations of knockout phenotypes,
"""
def _load_experimental_ko_data():
"""
Load Ishii data
"""
df = pd.read_csv("../data/datasets/ishii2007_tidy.csv")
# this regexp matches deletions starting with d like dpgi
df["sample_id"] = df.Genotype.str.extract(r"d(\w+)")
df.loc[df.Genotype == "WT", "sample_id"] = "WT"
df = df.assign(author="Ishii")
df = df.rename(
{
"Measurement_ID": "BiGG_ID",
"Original_Value": "normalized_flux",
"Value": "flux",
"Original_ID": "ID",
},
axis=1,
)
df = df[df["Measurement_Type"] == "flux"]
df.loc[df["BiGG_ID"] == "PYKF", "BiGG_ID"] = "PYK"
df = df[[
"flux", "ID", "BiGG_ID", "author", "sample_id", "normalized_flux"
]]
return df
def _load_cobra_ko_sims():
"""
Load simulations from iML1515, ECC2 and iML1515 and ECC2 conditioned on experimental data
"""
iml_results = pd.read_csv(path_to_results / "COBRA" / "iML1515" /
"chemostat_knockouts" / "knockouts_all.csv",
index_col=0)
# ecc_results = pd.read_csv(
# path_to_results / "COBRA" / "ECC2" / "knockouts_all.csv", index_col=0
# )
exp_iml_results = pd.read_csv(path_to_results / "COBRA" /
"Exp_iML1515" / "chemostat_knockouts" /
"knockouts_all.csv",
index_col=0)
# exp_ecc_results = pd.read_csv(
# path_to_results / "COBRA" / "Exp_ECC2" / "knockouts_all.csv", index_col=0
# )
df = pd.concat([iml_results, exp_iml_results])
# Fix direction to match experimental data
# Only for iML1515
# fix PGM direction
iml_reversed = ["PGM", "PGK", "SUCOAS"]
df.loc[df["ID"].isin(iml_reversed),
"flux"] = (-1 * df.loc[df["ID"].isin(iml_reversed), "flux"])
df.loc[df["ID"].isin(iml_reversed), "normalized_flux"] = (
-1 * df.loc[df["ID"].isin(iml_reversed), "normalized_flux"])
# df = pd.concat([df, ecc_results, exp_ecc_results])
# For both iML1515 and ECC2
# fix RPI direction
ecc_reversed = ["RPI"]
df.loc[df["ID"].isin(ecc_reversed),
"flux"] = (-1 * df.loc[df["ID"].isin(ecc_reversed), "flux"])
df.loc[df["ID"].isin(ecc_reversed), "normalized_flux"] = (
-1 * df.loc[df["ID"].isin(ecc_reversed), "normalized_flux"])
return df
def _load_kinetic_ko_sims():
"""
Load the data from kinetic models simulations
"""
khodayari_results = load_khodayari(
sample_names="all",
load_path=(path_to_results / "Khodayari"),
id_df=khod_idf,
files=get_khodayari_kos(),
)
kurata_results = load_kurata(
sample_names="all",
load_path=(path_to_results / "Kurata"),
id_df=kurata_idf,
files=get_kurata_kos(),
)
millard_results = load_millard(
sample_names="all",
load_path=(path_to_results / "Millard"),
id_df=millard_idf,
files=get_millard_kos(),
)
chassagnole_results = load_chassagnole(
sample_names="all",
load_path=(path_to_results / "Chassagnole" /
"chemostat_knockouts"),
id_df=chassagnole_idf,
files=get_chassagnole_kos(),
)
simulation_results = pd.concat([
khodayari_results, kurata_results, millard_results,
chassagnole_results
],
sort=False)
return simulation_results
with io.StringIO() as buf, redirect_stdout(buf):
simulation_data = _load_kinetic_ko_sims()
cobra_data = _load_cobra_ko_sims()
exp_data = _load_experimental_ko_data()
file_info = buf.getvalue()
return pd.concat([simulation_data, cobra_data, exp_data],
sort=False), file_info
def test_missing_problem(self):
f = io.StringIO()
with contextlib.redirect_stdout(f):
with self.assertRaises(lib50.InvalidSlugError):
lib50.connect("cs50/lib50/tests/i_do_not_exist", self.loader)
def test_no_tool_in_config(self):
f = io.StringIO()
loader = lib50.config.Loader("i_do_not_exist")
with contextlib.redirect_stdout(f):
with self.assertRaises(lib50.InvalidSlugError):
lib50.connect("cs50/lib50/tests/bar", loader)
def handle(self, event: Event) -> None:
transaction_inst = Transaction.from_cloudevents_model(event)
transaction_key_value_insts = TransactionKeyValue.from_cloudevents_model(event)
file_insts = File.from_cloudevents_model(event)
config_by_config_id = _to_proxymod_config_by_config_id(transaction_key_values=transaction_key_value_insts)
for config_id in ['config_1', 'config_2', 'config_3']:
if config_id not in config_by_config_id:
raise ConfigNotFoundProxEventHandlerError(event, config_id)
config = config_by_config_id[config_id]
if not _is_valid_proxymod_config(config):
raise InvalidConfigProxEventHandlerError(event, config_id, config)
input_file_insts = []
model_file_insts = []
_in_dir = config_by_config_id.get('config_1', {}).get('INPUTS', {}).get('in_dir', None)
_in_file_one = config_by_config_id.get('config_1', {}).get('INPUTS', {}).get('in_file_one', None)
_in_file_two = config_by_config_id.get('config_1', {}).get('INPUTS', {}).get('in_file_two', None)
for file_inst in file_insts:
if ('text/csv' == file_inst.mimetype) and (file_inst.subdir is not None) and (_in_dir == file_inst.subdir) and (file_inst.name is not None) and ((_in_file_one == file_inst.name) or (_in_file_two == file_inst.name)):
input_file_insts.append(file_inst)
elif ('text/x-python' == file_inst.mimetype) and ('models/' == file_inst.subdir):
model_file_insts.append(file_inst)
else:
# NOTE Ignore other files.
pass
with tempfile.TemporaryDirectory() as downloader_tempdir_name:
with tempfile.TemporaryDirectory() as uploader_tempdir_name:
# model_file_openers = self.downloader_runner.download(downloader_tempdir_name, model_file_insts)
with open(os.path.join(uploader_tempdir_name, 'download-stdout.log'), mode='w') as downloader_stdout_file:
with open(os.path.join(uploader_tempdir_name, 'download-stderr.log'), mode='w') as downloader_stderr_file:
with contextlib.redirect_stdout(downloader_stdout_file):
with contextlib.redirect_stderr(downloader_stderr_file):
model_file_openers = self.downloader_runner.download(downloader_tempdir_name, model_file_insts)
model_file_funcs = []
for model_file_inst, model_file_opener in zip(model_file_insts, model_file_openers):
with model_file_opener() as file:
try:
name = os.path.splitext(model_file_inst.name)[0]
spec = importlib.util.spec_from_file_location(name, file.name)
module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(module)
# NOTE Deliberately raise `AttributeError` if `name` does not exist.
func = getattr(module, name)
if callable(func):
model_file_funcs.append(func)
else:
# NOTE Deliberately raise `TypeError` by calling an uncallable.
func()
except Exception as reason:
raise InvalidModelProxEventHandlerError(event, model_file_inst, reason)
# input_file_openers = self.downloader_runner.download(downloader_tempdir_name, input_file_insts)
with open(os.path.join(uploader_tempdir_name, 'download-stdout.log'), mode='a') as downloader_stdout_file:
with open(os.path.join(uploader_tempdir_name, 'download-stderr.log'), mode='a') as downloader_stderr_file:
with contextlib.redirect_stdout(downloader_stdout_file):
with contextlib.redirect_stderr(downloader_stderr_file):
input_file_openers = self.downloader_runner.download(downloader_tempdir_name, input_file_insts)
abspath_config_by_config_id = copy.deepcopy(config_by_config_id)
for config_id, config in abspath_config_by_config_id.items():
if 'INPUTS' in config:
if 'in_dir' in config['INPUTS']:
for input_file_inst, opener in zip(input_file_insts, input_file_openers):
with opener() as file:
config['INPUTS']['in_dir'] = os.path.abspath(os.path.dirname(file.name))
break
if 'OUTPUTS' in config:
if 'out_dir' in config['OUTPUTS']:
config['OUTPUTS']['out_dir'] = os.path.abspath(os.path.join(uploader_tempdir_name, config['OUTPUTS']['out_dir']))
for config_id, config in config_by_config_id.items():
with open(os.path.join(uploader_tempdir_name, '{0}.ini'.format(config_id)), mode='w') as config_file:
config_file.write(_format_proxymod_config(config))
with tempfile.NamedTemporaryFile(suffix='.ini') as config_1_file:
config_1_file.write(bytes(_format_proxymod_config(abspath_config_by_config_id['config_1']), 'utf-8'))
config_1_file.seek(0)
with tempfile.NamedTemporaryFile(suffix='.ini') as config_2_file:
config_2_file.write(bytes(_format_proxymod_config(abspath_config_by_config_id['config_2']), 'utf-8'))
config_2_file.seek(0)
with tempfile.NamedTemporaryFile(suffix='.ini') as config_3_file:
config_3_file.write(bytes(_format_proxymod_config(abspath_config_by_config_id['config_3']), 'utf-8'))
config_3_file.seek(0)
# for model_file_inst, model_file_func in zip(model_file_insts, model_file_funcs):
# try:
# model_file_func(config_1_file.name, config_2_file.name, config_3_file.name)
# except Exception as reason:
# raise InvalidModelProxEventHandlerError(event, model_file_inst, reason)
with open(os.path.join(uploader_tempdir_name, 'stdout.log'), mode='w') as stdout_file:
with open(os.path.join(uploader_tempdir_name, 'stderr.log'), mode='w') as stderr_file:
with contextlib.redirect_stdout(stdout_file):
with contextlib.redirect_stderr(stderr_file):
for model_file_inst, model_file_func in zip(model_file_insts, model_file_funcs):
try:
model_file_func(config_1_file.name, config_2_file.name, config_3_file.name)
except Exception as reason:
raise InvalidModelProxEventHandlerError(event, model_file_inst, reason)
# (bundle, job_id, state) = self.uploader_runner.upload(uploader_tempdir_name, transaction=Transaction(submitter=transaction_inst.submitter, instrument=transaction_inst.instrument, proposal=transaction_inst.proposal), transaction_key_values=[TransactionKeyValue(key='Transactions._id', value=transaction_inst._id)])
with open(os.path.join(uploader_tempdir_name, 'upload-stdout.log'), mode='w') as uploader_stdout_file:
with open(os.path.join(uploader_tempdir_name, 'upload-stderr.log'), mode='w') as uploader_stderr_file:
with contextlib.redirect_stdout(uploader_stdout_file):
with contextlib.redirect_stderr(uploader_stderr_file):
(bundle, job_id, state) = self.uploader_runner.upload(uploader_tempdir_name, transaction=Transaction(submitter=transaction_inst.submitter, instrument=transaction_inst.instrument, proposal=transaction_inst.proposal), transaction_key_values=[TransactionKeyValue(key='Transactions._id', value=transaction_inst._id)])
pass
pass
pass
def draw(callback, update):
with contextlib.ExitStack() as stack:
# create virtual terminal
fdread, fdwrite = getattr(os, 'openpty', os.pipe)()
stack.callback(
os.close,
fdread) # fdwrite is closed separately to signal to the thread
# save original output file descriptor
fileno = os.dup(sys.stdout.fileno())
stack.callback(os.close, fileno)
# create thread
t = StickyBar(fdread, fileno, callback, sys.stdout.encoding, update)
stack.callback(t.join)
# replace stdout by virtual terminal
os.dup2(fdwrite, sys.stdout.fileno())
stack.callback(
os.dup2, fileno,
sys.stdout.fileno()) # restore stdout and signal to thread
os.close(fdwrite)
if platform.system() != 'Windows':
os.write(fileno, b'\033[?7l') # disable line wrap
stack.callback(os.write, fileno, b'\033[?7h') # enable line wrap
else:
# set console mode
import ctypes
kernel32 = ctypes.WinDLL('kernel32')
handle = kernel32.GetStdHandle(
-11
) # https://docs.microsoft.com/en-us/windows/console/getstdhandle
orig_mode = ctypes.c_uint32(
) # https://docs.microsoft.com/en-us/windows/desktop/WinProg/windows-data-types#lpdword
kernel32.GetConsoleMode(
handle, ctypes.byref(orig_mode)
) # https://docs.microsoft.com/en-us/windows/console/getconsolemode
new_mode = orig_mode.value
new_mode |= 4 # check ENABLE_VIRTUAL_TERMINAL_PROCESSING
new_mode &= ~2 # uncheck ENABLE_WRAP_AT_EOL_OUTPUT
if new_mode != orig_mode.value:
kernel32.SetConsoleMode(
handle, ctypes.c_uint32(new_mode)
) # https://docs.microsoft.com/en-us/windows/console/setconsolemode
stack.callback(kernel32.SetConsoleMode, handle, orig_mode)
# In Windows, `sys.stdout` becomes unusable after
# `os.dup2(..,sys.stdout.fileno())`, hence we recreate `sys.stdout` here.
# Because a pipe is not a tty, `fdopen` defaults to buffering with fixed
# size chunks. `buffering=1` enforces lines buffering. The new
# `sys.stdout` answers `False` to `.isatty()` for the same reason.
stack.enter_context(
contextlib.redirect_stdout(
os.fdopen(sys.stdout.fileno(),
'w',
encoding=sys.stdout.encoding,
buffering=1)))
# start bar-drawing thread
t.start()
yield
def test_no_config(self):
f = io.StringIO()
with contextlib.redirect_stdout(f):
with self.assertRaises(lib50.InvalidSlugError):
lib50.connect("cs50/lib50/tests/no_config", self.loader)
def load_dilution_data():
def _load_cobra_dilution_sims():
"""
Load simulations from iML1515, ECC2
"""
iml_results = pd.read_csv(path_to_results / "COBRA" / "iML1515" /
"dilutions" / "all.csv",
index_col=0)
# ecc_results = pd.read_csv(
# path_to_results / "COBRA" / "ECC2" / "dilutions" / "all.csv", index_col=0
# )
df = iml_results
# Fix direction to match experimental data
# Only for iML1515
# fix PGM direction
iml_reversed = ["PGM", "PGK", "SUCOAS"]
df.loc[df["ID"].isin(iml_reversed),
"flux"] = (-1 * df.loc[df["ID"].isin(iml_reversed), "flux"])
df.loc[df["ID"].isin(iml_reversed), "normalized_flux"] = (
-1 * df.loc[df["ID"].isin(iml_reversed), "normalized_flux"])
# df = pd.concat([df, ecc_results])
# For both iML1515 and ECC2
# fix RPI direction
ecc_reversed = ["RPI"]
df.loc[df["ID"].isin(ecc_reversed),
"flux"] = (-1 * df.loc[df["ID"].isin(ecc_reversed), "flux"])
df.loc[df["ID"].isin(ecc_reversed), "normalized_flux"] = (
-1 * df.loc[df["ID"].isin(ecc_reversed), "normalized_flux"])
return df
def _load_experimental_dilution_data():
yao_df = pd.read_csv(data_path / "datasets" / "yao2011_tidy.csv")
consumption_rates = yao_df.query(
'Measurement_Type == "consumption_rate"')
yao_fluxes = yao_df.query('Measurement_Type == "flux"')
def normalize_to_uptake(group):
consumption_rate = consumption_rates.loc[
consumption_rates.Dilution == group.name, "Value"].values[0]
print(f"Consumption rate for D {group.name} is {consumption_rate}")
group = group.assign(
normalized_flux=lambda x: x.Value / consumption_rate * 100)
return group
df = (yao_fluxes.groupby("Dilution").apply(
normalize_to_uptake).reset_index(drop=True))
df = df.assign(author="Yao")
df = df.rename(
{
"Measurement_ID": "BiGG_ID",
"Value": "flux",
"Original_ID": "ID",
"Dilution": "sample_id",
},
axis=1,
)
df = df[df["Measurement_Type"] == "flux"]
df = df[[
"flux", "ID", "BiGG_ID", "author", "sample_id", "normalized_flux"
]]
df.sample_id = df.sample_id.apply(str)
return df
def _load_kinetic_dilution_sims():
khodayari_dil = load_khodayari(
sample_names="all",
load_path=(path_to_results / "Khodayari" / "dilutions"),
id_df=khod_idf,
files=get_khodayari_dilutions(),
)
kurata_dil = load_kurata(
sample_names="all",
load_path=(path_to_results / "Kurata" / "dilutions"),
id_df=kurata_idf,
files=get_kurata_dilutions(),
)
millard_dil = load_millard(
sample_names="all",
load_path=(path_to_results / "Millard" / "dilutions"),
id_df=millard_idf,
files=get_millard_dilutions(),
)
chassagnole_dil = load_chassagnole(
sample_names="all",
load_path=(path_to_results / "Chassagnole" / "dilutions"),
id_df=chassagnole_idf,
files=get_chassagnole_dilutions(),
)
return pd.concat(
[khodayari_dil, kurata_dil, millard_dil, chassagnole_dil],
sort=False)
with io.StringIO() as buf, redirect_stdout(buf):
simulation_data = _load_kinetic_dilution_sims()
exp_data = _load_experimental_dilution_data()
cobra_data = _load_cobra_dilution_sims()
file_info = buf.getvalue()
return pd.concat([simulation_data, exp_data, cobra_data],
sort=False), file_info
def create_archipelago(
unknowns: list,
optimizers: list,
optimizers_kwargs: list,
pg_problem: pygmo.problem,
rel_pop_size: float,
archipelago_kwargs: dict,
log_each_nth_gen: int,
report_level: int,
) -> PyfoombArchipelago:
"""
Helper method for parallelized estimation using the generalized island model.
Creates the archipelago object for running several rounds of evolutions.
Arguments
---------
unknowns : list
The unknowns, sorted alphabetically and case-insensitive.
optimizers : list
A list of optimizers to be used on individual islands.
optimizers_kwargs : list
A list of corresponding kwargs.
pg_problem : pygmo.problem
An pygmo problem instance.
archipelago_kwargs : dict
Additional kwargs for archipelago creation.
log_each_nth_gen : int
Specifies at which each n-th generation the algorithm stores logs.
report_level : int
Prints information on the archipelago creation for values >= 1.
Returns
-------
archipelago : PyfoombArchipelago
"""
_cpus = joblib.cpu_count()
# There is one optimizer with a set of kwargs
if len(optimizers) == 1 and len(optimizers_kwargs) == 1:
optimizers = optimizers * _cpus
optimizers_kwargs = optimizers_kwargs * _cpus
# Several optimizers with the same kwargs
elif len(optimizers) > 1 and len(optimizers_kwargs) == 1:
optimizers_kwargs = optimizers_kwargs * len(optimizers)
# Several kwargs for the same optimizer
elif len(optimizers) == 1 and len(optimizers_kwargs) > 1:
optimizers = optimizers * len(optimizers_kwargs)
elif len(optimizers) != len(optimizers_kwargs):
raise ValueError(
'Number of optimizers does not match number of corresponding kwarg dicts'
)
# Get the optimizer intances
algos = [
PygmoOptimizers.get_optimizer_algo_instance(
name=_optimizers, kwargs=_optimizers_kwargs) for _optimizers,
_optimizers_kwargs in zip(optimizers, optimizers_kwargs)
]
# Update number of islands
n_islands = len(algos)
if report_level >= 1:
print(
f'Creating archipelago with {n_islands} islands. May take some time...'
)
pop_size = int(numpy.ceil(rel_pop_size * len(unknowns)))
prop_create_args = ((pg_problem, pop_size,
seed * numpy.random.randint(0, 1e4))
for seed, pop_size in enumerate([pop_size] *
n_islands))
try:
parallel_verbose = 0 if report_level == 0 else 1
with joblib.parallel_backend('loky', n_jobs=n_islands):
pops = joblib.Parallel(verbose=parallel_verbose)(map(
joblib.delayed(
ArchipelagoHelpers.parallel_create_population),
prop_create_args))
except Exception as ex:
print(
f'Parallelized archipelago creation failed, falling back to sequential\n{ex}'
)
pops = (
ArchipelagoHelpers.parallel_create_population(prop_create_arg)
for prop_create_arg in prop_create_args)
# Now create the empyty archipelago
if not 't' in archipelago_kwargs.keys():
archipelago_kwargs['t'] = pygmo.fully_connected()
archi = PyfoombArchipelago(**archipelago_kwargs)
archi.set_migrant_handling(pygmo.migrant_handling.preserve)
# Add the populations to the archipelago and wait for its construction
with contextlib.redirect_stdout(io.StringIO()):
for _pop, _algo in zip(pops, algos):
if log_each_nth_gen is not None:
_algo.set_verbosity(int(log_each_nth_gen))
_island = pygmo.island(algo=_algo,
pop=_pop,
udi=pygmo.mp_island())
archi.push_back(_island)
archi.wait_check()
return archi
def __init__(self, name="root", level="INFO"):
self.logger = logging.getLogger(name)
self.source = self.logger.name
self.level = getattr(logging, level)
self._redirector = contextlib.redirect_stdout(self)
import contextlib
with contextlib.redirect_stdout(None):
import pygame
pygame.font.init() # you have to call this at the start,
# if you want to use this module.
TEXT_FONT = pygame.font.SysFont('Comic Sans MS', 16)
TEXT_FONT_SMALL = pygame.font.SysFont('Comic Sans MS', 8)
ALIGN_TOP = 0
ALIGN_CENTER = 1
ALIGN_BOTTOM = 2
ALIGN_LEFT = 0
ALIGN_RIGHT = 2
class PyGameComponent(object):
def __init__(self):
self.visible = True
self.x = 0
self.y = 0
self.width = 0
self.height = 0
self.rect = {"left": 0, "top": 0, "right": 0, "bottom": 0}
self.screen_rect = {"left": 0, "top": 0, "right": 0, "bottom": 0}
self.anchors = {"min_x": 0, "min_y": 0, "max_x": 1, "max_y": 1}
self.pivot = {"x": 0, "y": 0}
self.mouse_state = {
"left_down": False,
"middle_down": False,
"right_down": False,
"PE20"]
metric_dictionary1 = {metric_names[i]: line1[i] for i in range(len(
metric_names))}
metric_dictionary2 = {metric_names[i]: line2[i] for i in range(len(
metric_names))}
# metric_dictionary3 = {metric_names[i]: line3[i] for i in range(len(metric_names))}
# metric_dictionary4 = {metric_names[i]: line4[i] for i in range(len(metric_names))}
print("Error metrics for call options")
for key, value in metric_dictionary1.items():
print(f"{key}:", value)
from contextlib import redirect_stdout
with open('MLP1_call_error_metrics.txt', 'w') as f:
with redirect_stdout(f):
print("Error metrics for call options")
for key, value in metric_dictionary1.items():
print(f"{key}:", value)
print("\nError metrics for put options")
for key, value in metric_dictionary2.items():
print(f"{key}:", value)
from contextlib import redirect_stdout
with open('MLP1_put_error_metrics.txt', 'w') as f:
with redirect_stdout(f):
print("Error metrics for put options")
for key, value in metric_dictionary2.items():
print(f"{key}:", value)
logfile = snakemake.log[0]
fnames = snakemake.input.cts
assembly = snakemake.wildcards.assembly
fa = snakemake.input.fa[0]
gtf = snakemake.input.gtf[0]
from_gtf = snakemake.params.from_gtf
out_counts = snakemake.output.counts[0]
out_tpms = snakemake.output.tpms[0]
out_lengths = snakemake.output.lengths[0]
sample_names = snakemake.params.names
# redirect all messages to a logfile
open(logfile, "w") # start a clean log
with open(logfile,
"a") as log: # appending because we mix streams (tqdm, logger & std)
with contextlib.redirect_stdout(log), contextlib.redirect_stderr(log):
pytxi.logger.remove()
pytxi.logger.add(logfile)
outdir = os.path.dirname(out_counts)
os.makedirs(outdir, exist_ok=True)
tx2gene = None
if from_gtf:
ann = Annotation(gtf)
# check if we can convert transcript ids to symbols with the GTF file
test = ann.gtf["attribute"].head(100)
attribute = None
if any(test.str.contains("gene_name")):
attribute = "gene_name"
def load_sensitivity_data():
def _load_kinetic_sensitivity_sims():
khodayari_zwf = load_khodayari(
sample_names="all",
load_path=(path_to_results / "Khodayari" / "zwf_sensitivity"),
id_df=khod_idf,
files=get_khodayari_zwf(),
)
khodayari_pgi = load_khodayari(
sample_names="all",
load_path=(path_to_results / "Khodayari" / "pgi_sensitivity"),
id_df=khod_idf,
files=get_khodayari_pgi(),
)
khodayari_eno = load_khodayari(
sample_names="all",
load_path=(path_to_results / "Khodayari" / "eno_sensitivity"),
id_df=khod_idf,
files=get_khodayari_eno(),
)
kurata_zwf = load_kurata(
sample_names="all",
load_path=(path_to_results / "Kurata" / "zwf_sensitivity"),
id_df=kurata_idf,
files=get_kurata_zwf(),
)
kurata_pgi = load_kurata(
sample_names="all",
load_path=(path_to_results / "Kurata" / "pgi_sensitivity"),
id_df=kurata_idf,
files=get_kurata_pgi(),
)
kurata_eno = load_kurata(
sample_names="all",
load_path=(path_to_results / "Kurata" / "eno_sensitivity"),
id_df=kurata_idf,
files=get_kurata_eno(),
)
millard_zwf = load_millard(
sample_names="all",
load_path=(path_to_results / "Millard" / "zwf_sensitivity"),
id_df=millard_idf,
files=get_millard_zwf(),
)
millard_pgi = load_millard(
sample_names="all",
load_path=(path_to_results / "Millard" / "pgi_sensitivity"),
id_df=millard_idf,
files=get_millard_pgi(),
)
millard_eno = load_millard(
sample_names="all",
load_path=(path_to_results / "Millard" / "eno_sensitivity"),
id_df=millard_idf,
files=get_millard_eno(),
)
chassagnole_zwf = load_chassagnole(
sample_names="all",
load_path=(path_to_results / "Chassagnole" /
"zwf_pgi_eno_sensitivity"),
id_df=chassagnole_idf,
files=get_chassagnole_zwf(),
)
chassagnole_pgi = load_chassagnole(
sample_names="all",
load_path=(path_to_results / "Chassagnole" /
"zwf_pgi_eno_sensitivity"),
id_df=chassagnole_idf,
files=get_chassagnole_pgi(),
)
chassagnole_eno = load_chassagnole(
sample_names="all",
load_path=(path_to_results / "Chassagnole" /
"zwf_pgi_eno_sensitivity"),
id_df=chassagnole_idf,
files=get_chassagnole_eno(),
)
simulation_zwf = pd.concat(
[khodayari_zwf, kurata_zwf, millard_zwf, chassagnole_zwf],
sort=False)
simulation_pgi = pd.concat(
[khodayari_pgi, kurata_pgi, millard_pgi, chassagnole_pgi],
sort=False)
simulation_eno = pd.concat(
[khodayari_eno, kurata_eno, millard_eno, chassagnole_eno],
sort=False)
return (simulation_zwf, simulation_pgi, simulation_eno)
def _load_experimental_sensitivity_data():
"""
Load experimental results and return a tuple of 3 dataframes each corresponding to
genes zwf, pgi and eno
"""
"""
Nicloas, 2007 data, for zwf knockout
"""
df = pd.read_csv("../data/datasets/nicolas2007_tidy.csv")
df = df.assign(author="Nicolas")
df = df.rename(
{
"Measurement_ID": "BiGG_ID",
"Original_Value": "normalized_flux",
"Value": "flux",
"Original_ID": "ID",
"Genotype": "sample_id",
},
axis=1,
)
df = df[df["Measurement_Type"] == "flux"]
df = df[[
"flux", "ID", "BiGG_ID", "author", "sample_id", "normalized_flux"
]]
exp_results_zwf = df
"""
Usui, 2012 data for pgi and eno data
"""
df = pd.read_csv("../data/datasets/usui2012_tidy.csv")
df = df.assign(author="Usui")
df = df.rename(
{
"Measurement_ID": "BiGG_ID",
"Original_Value": "normalized_flux",
"Value": "flux",
"Original_ID": "ID",
"Genotype": "sample_id",
},
axis=1,
)
df = df[df["Measurement_Type"] == "flux"]
df = df[[
"flux", "ID", "BiGG_ID", "author", "sample_id", "normalized_flux"
]]
exp_results_pgi = df
exp_results_eno = df
return (exp_results_zwf, exp_results_pgi, exp_results_eno)
with io.StringIO() as buf, redirect_stdout(buf):
simulation_data_zwf, simulation_data_pgi, simulation_data_eno = (
_load_kinetic_sensitivity_sims())
exp_data_zwf, exp_data_pgi, exp_data_eno = _load_experimental_sensitivity_data(
)
file_info = buf.getvalue()
return (
(
pd.concat([simulation_data_zwf, exp_data_zwf], sort=False),
pd.concat([simulation_data_pgi, exp_data_pgi], sort=False),
pd.concat([simulation_data_eno, exp_data_eno], sort=False),
),
file_info,
)
def test_rf_classification(small_clf, datatype, split_algo, max_samples,
max_features, use_experimental_backend):
use_handle = True
X, y = small_clf
X = X.astype(datatype)
y = y.astype(np.int32)
X_train, X_test, y_train, y_test = train_test_split(X,
y,
train_size=0.8,
random_state=0)
# Create a handle for the cuml model
handle, stream = get_handle(use_handle, n_streams=1)
# Initialize, fit and predict using cuML's
# random forest classification model
cuml_model = curfc(max_features=max_features,
max_samples=max_samples,
n_bins=16,
split_algo=split_algo,
split_criterion=0,
min_samples_leaf=2,
random_state=123,
n_streams=1,
n_estimators=40,
handle=handle,
max_leaves=-1,
max_depth=16,
use_experimental_backend=use_experimental_backend)
f = io.StringIO()
with redirect_stdout(f):
cuml_model.fit(X_train, y_train)
captured_stdout = f.getvalue()
if use_experimental_backend:
is_fallback_used = False
if max_features != 1.0:
assert ('Experimental backend does not yet support feature ' +
'sub-sampling' in captured_stdout)
is_fallback_used = True
if split_algo != 1:
assert ('Experimental backend does not yet support histogram ' +
'split algorithm' in captured_stdout)
is_fallback_used = True
if is_fallback_used:
assert ('Not using the experimental backend due to above ' +
'mentioned reason(s)' in captured_stdout)
else:
assert ('Using experimental backend for growing trees'
in captured_stdout)
else:
assert captured_stdout == ''
fil_preds = cuml_model.predict(X_test,
predict_model="GPU",
output_class=True,
threshold=0.5,
algo='auto')
cu_preds = cuml_model.predict(X_test, predict_model="CPU")
fil_preds = np.reshape(fil_preds, np.shape(cu_preds))
cuml_acc = accuracy_score(y_test, cu_preds)
fil_acc = accuracy_score(y_test, fil_preds)
if X.shape[0] < 500000:
sk_model = skrfc(n_estimators=40,
max_depth=16,
min_samples_split=2,
max_features=max_features,
random_state=10)
sk_model.fit(X_train, y_train)
sk_preds = sk_model.predict(X_test)
sk_acc = accuracy_score(y_test, sk_preds)
assert fil_acc >= (sk_acc - 0.07)
assert fil_acc >= (cuml_acc - 0.02)
def main():
# Training settings
parser = argparse.ArgumentParser(description='PyTorch MNIST Example')
parser.add_argument('--type',
default='linear',
choices=['linear', 'conv'],
help='model architecture type: ' +
' | '.join(['linear', 'conv']) + ' (default: linear)')
parser.add_argument(
'--model',
default='',
type=str,
metavar='MODEL_PATH',
help=
'path to model file to load both its architecture and weights (default: none)'
)
parser.add_argument(
'--weights',
default='',
type=str,
metavar='WEIGHTS_PATH',
help='path to file to load its weights (default: none)')
parser.add_argument('--shift-depth',
type=int,
default=0,
help='how many layers to convert to shift')
parser.add_argument(
'-st',
'--shift-type',
default='PS',
choices=['Q', 'PS'],
help=
'type of DeepShift method for training and representing weights (default: PS)'
)
parser.add_argument('-r',
'--rounding',
default='deterministic',
choices=['deterministic', 'stochastic'],
help='type of rounding (default: deterministic)')
parser.add_argument('-wb',
'--weight-bits',
type=int,
default=5,
help='number of bits to represent the shift weights')
parser.add_argument('-j',
'--workers',
default=1,
type=int,
metavar='N',
help='number of data loading workers (default: 1)')
parser.add_argument('--batch-size',
type=int,
default=64,
metavar='N',
help='input batch size for training (default: 64)')
parser.add_argument('--test-batch-size',
type=int,
default=1000,
metavar='N',
help='input batch size for testing (default: 1000)')
parser.add_argument('--epochs',
type=int,
default=10,
metavar='N',
help='number of epochs to train (default: 10)')
parser.add_argument('-opt',
'--optimizer',
metavar='OPT',
default="SGD",
help='optimizer algorithm')
parser.add_argument('--lr',
type=float,
default=0.01,
metavar='LR',
help='learning rate (default: 0.01)')
parser.add_argument('--momentum',
type=float,
default=0.0,
metavar='M',
help='SGD momentum (default: 0.0)')
parser.add_argument('--resume',
default='',
type=str,
metavar='CHECKPOINT_PATH',
help='path to latest checkpoint (default: none)')
parser.add_argument('-e',
'--evaluate',
dest='evaluate',
action='store_true',
help='only evaluate model on validation set')
parser.add_argument('--no-cuda',
action='store_true',
default=False,
help='disables CUDA training')
parser.add_argument('--seed',
type=int,
default=1,
metavar='S',
help='random seed (default: 1)')
parser.add_argument(
'--log-interval',
type=int,
default=10,
metavar='N',
help='how many batches to wait before logging training status')
parser.add_argument('--pretrained',
dest='pretrained',
default=False,
type=lambda x: bool(distutils.util.strtobool(x)),
help='use pre-trained model of full conv or fc model')
parser.add_argument('--save-model',
default=True,
type=lambda x: bool(distutils.util.strtobool(x)),
help='For Saving the current Model (default: True)')
parser.add_argument(
'--print-weights',
default=True,
type=lambda x: bool(distutils.util.strtobool(x)),
help='For printing the weights of Model (default: True)')
parser.add_argument('--desc',
type=str,
default=None,
help='description to append to model directory name')
parser.add_argument('--use-kernel',
type=lambda x: bool(distutils.util.strtobool(x)),
default=False,
help='whether using custom shift kernel')
parser.add_argument('-sb',
'--shift-base',
type=int,
default=2,
help='base of the wegiht representation')
args = parser.parse_args()
use_cuda = not args.no_cuda and torch.cuda.is_available()
if (args.evaluate is False and args.use_kernel is True):
raise ValueError(
'Our custom kernel currently supports inference only, not training.'
)
torch.manual_seed(args.seed)
device = torch.device("cuda" if use_cuda else "cpu")
kwargs = {
'num_workers': args.workers,
'pin_memory': True
} if use_cuda else {}
# Load training MNIST data
train_loader = torch.utils.data.DataLoader(
datasets.MNIST(
'../data',
train=True,
download=True,
transform=transforms.Compose([
transforms.ToTensor(),
transforms.Normalize(
(0.1307, ),
(0.3081, )) # transforms.Normalize((0,), (255,))
])),
batch_size=args.batch_size,
shuffle=True,
**kwargs)
# Load testing MNIST data
test_loader = torch.utils.data.DataLoader(
datasets.MNIST(
'../data',
train=False,
transform=transforms.Compose([
transforms.ToTensor(),
transforms.Normalize(
(0.1307, ),
(0.3081, )) # transforms.Normalize((0,), (255,))
])),
batch_size=args.test_batch_size,
shuffle=True,
**kwargs)
# Use an existing model (directory path provided)
if args.model:
if args.type or args.pretrained:
print(
"WARNING: Ignoring arguments \"type\" and \"pretrained\" when creating model..."
)
model = None
saved_checkpoint = torch.load(args.model)
if isinstance(saved_checkpoint, nn.Module):
model = saved_checkpoint
elif "model" in saved_checkpoint:
model = saved_checkpoint["model"]
else:
raise Exception("Unable to load model from " + args.model)
# Generate new model (linear or convolution)
else:
if args.type == 'linear':
model = LinearMNIST().to(device)
elif args.type == 'conv':
model = ConvMNIST().to(device)
if args.pretrained:
model.load_state_dict(
torch.load("./models/mnist/simple_" + args.type +
"/shift_0/weights.pth"))
model = model.to(device)
model_rounded = None
if args.weights:
saved_weights = torch.load(args.weights)
if isinstance(saved_weights, nn.Module):
state_dict = saved_weights.state_dict()
elif "state_dict" in saved_weights:
state_dict = saved_weights["state_dict"]
else:
state_dict = saved_weights
model.load_state_dict(state_dict)
if args.shift_depth > 0:
model, _ = convert_to_shift(model,
args.shift_depth,
args.shift_type,
args.shift_base,
convert_all_linear=(args.type != 'linear'),
convert_weights=True,
use_kernel=args.use_kernel,
use_cuda=use_cuda,
rounding=args.rounding,
weight_bits=args.weight_bits)
model = model.to(device)
elif args.use_kernel and args.shift_depth == 0:
model = convert_to_unoptimized(model)
model = model.to(device)
elif args.use_kernel and args.shift_depth == 0:
model = convert_to_unoptimized(model)
model = model.to(device)
loss_fn = F.cross_entropy # F.nll_loss
# define optimizer
optimizer = None
if (args.optimizer.lower() == "sgd"):
optimizer = torch.optim.SGD(model.parameters(),
args.lr,
momentum=args.momentum)
elif (args.optimizer.lower() == "adadelta"):
optimizer = torch.optim.Adadelta(model.parameters(), args.lr)
elif (args.optimizer.lower() == "adagrad"):
optimizer = torch.optim.Adagrad(model.parameters(), args.lr)
elif (args.optimizer.lower() == "adam"):
optimizer = torch.optim.Adam(model.parameters(), args.lr)
elif (args.optimizer.lower() == "rmsprop"):
optimizer = torch.optim.RMSprop(model.parameters(), args.lr)
elif (args.optimizer.lower() == "radam"):
optimizer = optim.RAdam(model.parameters(), args.lr)
elif (args.optimizer.lower() == "ranger"):
optimizer = optim.Ranger(model.parameters(), args.lr)
else:
raise ValueError("Optimizer type: ", args.optimizer,
" is not supported or known")
# optionally resume from a checkpoint
if args.resume:
if os.path.isfile(args.resume):
print("=> loading checkpoint '{}'".format(args.resume))
checkpoint = torch.load(args.resume)
if 'state_dict' in checkpoint:
model.load_state_dict(checkpoint['state_dict'])
else:
model.load_state_dict(checkpoint)
print("=> loaded checkpoint '{}'".format(args.resume))
else:
print("=> no checkpoint found at '{}'".format(args.resume))
# name model sub-directory "shift_all" if all layers are converted to shift layers
conv2d_layers_count = count_layer_type(
model, nn.Conv2d) + count_layer_type(model,
unoptimized.UnoptimizedConv2d)
linear_layers_count = count_layer_type(
model, nn.Linear) + count_layer_type(model,
unoptimized.UnoptimizedLinear)
if (args.shift_depth > 0):
if (args.shift_type == 'Q'):
shift_label = "shift_q"
else:
shift_label = "shift_ps"
else:
shift_label = "shift"
# name model sub-directory "shift_all" if all layers are converted to shift layers
conv2d_layers_count = count_layer_type(model, nn.Conv2d)
linear_layers_count = count_layer_type(model, nn.Linear)
if (conv2d_layers_count == 0 and linear_layers_count == 0):
shift_label += "_all"
else:
shift_label += "_%s" % (args.shift_depth)
if (args.shift_depth > 0):
shift_label += "_wb_%s" % (args.weight_bits)
if (args.shift_base > 0):
shift_base = args.shift_base
shift_label += "_sb_%s" % (args.shift_base)
if (args.desc is not None and len(args.desc) > 0):
desc_label = "_%s" % (args.desc)
else:
desc_label = ""
global model_name # MZ addition
model_name = 'simple_%s/%s%s' % (args.type, shift_label, desc_label)
# if evaluating round weights to ensure that the results are due to powers of 2 weights
if (args.evaluate):
model = round_shift_weights(model, shift_base)
model_summary = None
try:
model_summary, model_params_info = torchsummary.summary_string(
model, input_size=(1, 28, 28))
print(model_summary)
print(
"WARNING: The summary function reports duplicate parameters for multi-GPU case"
)
except:
print("WARNING: Unable to obtain summary of model")
model_dir = os.path.join(
os.path.join(os.path.join(os.getcwd(), "models"), "mnist"), model_name)
if not os.path.isdir(model_dir):
os.makedirs(model_dir, exist_ok=True)
if (args.save_model):
with open(os.path.join(model_dir, 'command_args.txt'),
'w') as command_args_file:
for arg, value in sorted(vars(args).items()):
command_args_file.write(arg + ": " + str(value) + "\n")
with open(os.path.join(model_dir, 'model_summary.txt'),
'w') as summary_file:
with redirect_stdout(summary_file):
if (model_summary is not None):
print(model_summary)
print(
"WARNING: The summary function reports duplicate parameters for multi-GPU case"
)
else:
print("WARNING: Unable to obtain summary of model")
# del model_tmp_copy
start_time = time.time()
if args.evaluate:
test_loss, correct = test(args, model, device, test_loader, loss_fn)
test_log = [(test_loss, correct / 1e4)]
with open(os.path.join(model_dir, "test_log.csv"),
"w") as test_log_file:
test_log_csv = csv.writer(test_log_file)
test_log_csv.writerow(['test_loss', 'correct'])
test_log_csv.writerows(test_log)
else:
###################################################################################################################################
# Start recording training usage metrics
global is_training
is_training = True
t = report_usage_training()
###################################################################################################################################
train_log = []
###################################################################################################################################
test_start = None
test_end = None
###################################################################################################################################
for epoch in range(1, args.epochs + 1):
train_loss = train(args, model, device, train_loader, loss_fn,
optimizer, epoch)
test_start = time.time()
test_loss, correct = test(args, model, device, test_loader,
loss_fn)
test_end = time.time()
###################################################################################################################################
# Record test-specific metrics
test_set_size = len(test_loader.dataset)
eval_time = test_end - test_start
global testing_perf
testing_perf = testing_perf.append(
{
'TestSetSize': test_set_size,
'EvaluationTime': eval_time,
'Loss': test_loss,
'Correct%': correct / test_set_size,
'Epoch': epoch
},
ignore_index=True)
###################################################################################################################################
if (args.print_weights):
with open(
os.path.join(model_dir,
'weights_log_' + str(epoch) + '.txt'),
'w') as weights_log_file:
with redirect_stdout(weights_log_file):
# Log model's state_dict
print("Model's state_dict:")
# TODO: Use checkpoint above
for param_tensor in model.state_dict():
print(param_tensor, "\t",
model.state_dict()[param_tensor].size())
print(model.state_dict()[param_tensor])
print("")
train_log.append((epoch, train_loss, test_loss, correct / 1e4))
###################################################################################################################################
# Stop recording training usage metrics
is_training = False
t.join()
global training_perf
training_perf.to_excel(model_dir + '\\train_performance.xlsx',
index=False)
organize_performance_profile_training()
organize_performance_accuracy_testing()
###################################################################################################################################
with open(os.path.join(model_dir, "train_log.csv"),
"w") as train_log_file:
train_log_csv = csv.writer(train_log_file)
train_log_csv.writerow(
['epoch', 'train_loss', 'test_loss', 'test_accuracy'])
train_log_csv.writerows(train_log)
if (args.save_model):
model_rounded = round_shift_weights(model, shift_base, clone=True)
torch.save(model_rounded, os.path.join(model_dir, "model.pth"))
torch.save(model_rounded.state_dict(),
os.path.join(model_dir, "weights.pth"))
end_time = time.time()
print("Total Time:", end_time - start_time)
if (args.print_weights):
if (model_rounded is None):
model_rounded = round_shift_weights(model, shift_base, clone=True)
with open(os.path.join(model_dir, 'weights_log.txt'),
'w') as weights_log_file:
with redirect_stdout(weights_log_file):
# Log model's state_dict
print("Model's state_dict:")
# TODO: Use checkpoint above
for param_tensor in model_rounded.state_dict():
print(param_tensor, "\t",
model_rounded.state_dict()[param_tensor].size())
print(model_rounded.state_dict()[param_tensor])
print("")
def test_case_0(self, input_mock=None):
text_trap = io.StringIO()
with redirect_stdout(text_trap):
import solution
self.assertEqual(text_trap.getvalue(), '19\n')
def _run_file_processor(
result_channel: MultiprocessingConnection,
parent_channel: MultiprocessingConnection,
file_path: str,
pickle_dags: bool,
dag_ids: Optional[List[str]],
thread_name: str,
callback_requests: List[CallbackRequest],
) -> None:
"""
Process the given file.
:param result_channel: the connection to use for passing back the result
:type result_channel: multiprocessing.Connection
:param parent_channel: the parent end of the channel to close in the child
:type parent_channel: multiprocessing.Connection
:param file_path: the file to process
:type file_path: str
:param pickle_dags: whether to pickle the DAGs found in the file and
save them to the DB
:type pickle_dags: bool
:param dag_ids: if specified, only examine DAG ID's that are
in this list
:type dag_ids: list[str]
:param thread_name: the name to use for the process that is launched
:type thread_name: str
:param callback_requests: failure callback to execute
:type callback_requests: List[airflow.utils.callback_requests.CallbackRequest]
:return: the process that was launched
:rtype: multiprocessing.Process
"""
# This helper runs in the newly created process
log: logging.Logger = logging.getLogger("airflow.processor")
# Since we share all open FDs from the parent, we need to close the parent side of the pipe here in
# the child, else it won't get closed properly until we exit.
log.info("Closing parent pipe")
parent_channel.close()
del parent_channel
set_context(log, file_path)
setproctitle(f"airflow scheduler - DagFileProcessor {file_path}")
try:
# redirect stdout/stderr to log
with redirect_stdout(StreamLogWriter(
log, logging.INFO)), redirect_stderr(
StreamLogWriter(log,
logging.WARN)), Stats.timer() as timer:
# Re-configure the ORM engine as there are issues with multiple processes
settings.configure_orm()
# Change the thread name to differentiate log lines. This is
# really a separate process, but changing the name of the
# process doesn't work, so changing the thread name instead.
threading.current_thread().name = thread_name
log.info("Started process (PID=%s) to work on %s", os.getpid(),
file_path)
dag_file_processor = DagFileProcessor(dag_ids=dag_ids, log=log)
result: Tuple[int, int] = dag_file_processor.process_file(
file_path=file_path,
pickle_dags=pickle_dags,
callback_requests=callback_requests,
)
result_channel.send(result)
log.info("Processing %s took %.3f seconds", file_path,
timer.duration)
except Exception:
# Log exceptions through the logging framework.
log.exception("Got an exception! Propagating...")
raise
finally:
# We re-initialized the ORM within this Process above so we need to
# tear it down manually here
settings.dispose_orm()
result_channel.close()
def _run_smoketest():
print_step('Starting Raiden')
config = deepcopy(App.DEFAULT_CONFIG)
if args.get('extra_config', dict()):
merge_dict(config, args['extra_config'])
del args['extra_config']
args['config'] = config
raiden_stdout = StringIO()
with contextlib.redirect_stdout(raiden_stdout):
app = run_app(**args)
try:
raiden_api = RaidenAPI(app.raiden)
rest_api = RestAPI(raiden_api)
(api_host, api_port) = split_endpoint(args['api_address'])
api_server = APIServer(rest_api, config={'host': api_host, 'port': api_port})
api_server.start()
raiden_api.channel_open(
registry_address=contract_addresses[CONTRACT_TOKEN_NETWORK_REGISTRY],
token_address=to_canonical_address(token.contract.address),
partner_address=to_canonical_address(TEST_PARTNER_ADDRESS),
)
raiden_api.set_total_channel_deposit(
contract_addresses[CONTRACT_TOKEN_NETWORK_REGISTRY],
to_canonical_address(token.contract.address),
to_canonical_address(TEST_PARTNER_ADDRESS),
TEST_DEPOSIT_AMOUNT,
)
token_addresses = [to_checksum_address(token.contract.address)]
success = False
print_step('Running smoketest')
error = run_smoketests(
app.raiden,
args['transport'],
token_addresses,
contract_addresses[CONTRACT_ENDPOINT_REGISTRY],
debug=debug,
)
if error is not None:
append_report('Smoketest assertion error', error)
else:
success = True
finally:
app.stop()
app.raiden.get()
node = ethereum[0]
node.send_signal(2)
err, out = node.communicate()
append_report('Ethereum stdout', out)
append_report('Ethereum stderr', err)
append_report('Raiden Node stdout', raiden_stdout.getvalue())
if success:
print_step(f'Smoketest successful')
else:
print_step(f'Smoketest had errors', error=True)
return success
async def ev(self, ctx: utils.Context, *, content: str):
"""Evaluates some Python code
Gracefully stolen from Rapptz ->
https://github.com/Rapptz/RoboDanny/blob/rewrite/cogs/admin.py#L72-L117"""
# Make the environment
env = {
'bot': self.bot,
'ctx': ctx,
'channel': ctx.channel,
'author': ctx.author,
'guild': ctx.guild,
'message': ctx.message,
'self': self,
}
env.update(globals())
# Make code and output string
content = self._cleanup_code(content)
code = f'async def func():\n{textwrap.indent(content, " ")}'
# Make the function into existence
stdout = io.StringIO()
try:
exec(code, env)
except Exception as e:
return await ctx.send(f'```py\n{e.__class__.__name__}: {e}\n```')
# Grab the function we just made and run it
func = env['func']
try:
# Shove stdout into StringIO
with contextlib.redirect_stdout(stdout):
ret = await func()
except Exception:
# Oh no it caused an error
value = stdout.getvalue()
await ctx.send(f'```py\n{value}{traceback.format_exc()}\n```')
else:
# Oh no it didn't cause an error
value = stdout.getvalue()
# Give reaction just to show that it ran
await ctx.message.add_reaction("\N{OK HAND SIGN}")
# If the function returned nothing
if ret is None:
# It might have printed something
if value:
await ctx.send(f'```py\n{value}\n```')
# If the function did return a value
else:
self._last_result = ret
result_raw = ret or value
result = str(result_raw)
if type(result_raw) == dict:
try:
result = json.dumps(result_raw, indent=4)
except Exception:
pass
if type(result_raw) == dict and type(result) == str:
text = f'```json\n{result}\n```'
else:
text = f'```py\n{result}\n```'
if len(text) > 2000:
await ctx.send(file=discord.File(io.StringIO(result),
filename='ev.txt'))
else:
await ctx.send(text)
# -t exon
# -f bam
############################################
print("\tUsing BAM file `" + bamFile + "`")
if not os.path.exists(bamFile):
print(
"\t\tThe system couldn't locate the file, trying the next one ..."
)
badFileList.append([bamFile, "Does Not Exist"])
continue
samPass = [bamFile]
# We need to redirect stdout to a buffer to write a string
# See:
# https://stackoverflow.com/a/22434594/6860368
outputString = ""
with io.StringIO() as buf, redirect_stdout(buf):
try:
count.count_reads_in_features(
samPass, # BAM file
gff3File, # GFF3 file
"bam", # -f flag
"name", # default
30000000, # default
"no", # -s flag
"union", # -m flag
"all", # --nonunique flag
"score", # default
"score", # default
"exon", # -t flag
"gene_id", # -i flag, default
(), # Default
def set_coco_api(args, dataset_name, seqs):
if args.num != -1 and not args.direct_avg:
# create temp json file if not using all sequences
save_file = osp.join(
'datasets/waymo_labels/subsets',
'{}_{}seqs.json'.format(dataset_name, args.num),
)
if not osp.exists(save_file):
new_imgs = []
new_annos = []
for seq in seqs[0]:
metadata = MetadataCatalog.get(seq)
json_file = PathManager.get_local_path(metadata.json_file)
data = json.load(open(json_file))
new_imgs.extend(data['images'])
new_annos.extend(data['annotations'])
new_data = {
'videos': data['videos'],
'images': new_imgs,
'categories': data['categories'],
'annotations': new_annos,
}
if 'cameras' in data:
new_data['cameras'] = data['cameras']
with open(save_file, 'w') as fp:
json.dump(new_data, fp)
else:
metadata = MetadataCatalog.get(dataset_name)
new_data = json.load(open(save_file))
img_ids = list(set([img['id'] for img in new_data['images']]))
json_file = save_file
else:
metadata = MetadataCatalog.get(dataset_name)
json_file = PathManager.get_local_path(metadata.json_file)
img_ids = None
if args.direct_avg and args.full_data:
data = json.load(open(json_file))
img_ids = list(set([img['id'] for img in data['images']]))
# set up coco_api
with contextlib.redirect_stdout(io.StringIO()):
coco_api = COCO(json_file)
sub_apis = []
img_split = None
basename = osp.splitext(json_file)[0].replace('_full', '')
try:
with contextlib.redirect_stdout(io.StringIO()):
for i in range(args.range):
name = basename + '_{}.json'.format(i)
api = COCO(name)
sub_apis.append(api)
# print(basename+'_img_ids.json')
with open(basename + '_img_ids.json') as fp:
img_split = json.load(fp)
except:
pass
# thing_classes
thing_clases = metadata.get("thing_classes")
return coco_api, thing_clases, sub_apis, img_split, img_ids
def check_model_representation(model, X, y=None, convs=None,
output_names=None, only_float=True,
verbose=False, suffix="", fLOG=None):
"""
Checks that a trained model can be exported in a specific list
of formats and produces the same outputs if the
representation can be used to predict.
@param model model (a class or an instance of a model but not trained)
@param X features
@param y targets
@param convs list of format to check, all possible by default ``['json', 'c']``
@param output_names list of output columns
(can be None, a default value is infered based on scikit-learn output then)
@param verbose print some information
@param suffix add this to disambiguate module
@param fLOG logging function
@return function to call to run the prediction
"""
if not only_float:
raise NotImplementedError( # pragma: no cover
"Only float are allowed.")
if isinstance(X, list):
X = pandas.DataFrame(X)
if len(X.shape) != 2:
raise ValueError( # pragma: no cover
"X cannot be converted into a proper DataFrame. It has shape {0}."
"".format(X.shape))
if only_float:
X = X.as_matrix()
if isinstance(y, list):
y = numpy.array(y)
if convs is None:
convs = ['json', 'c']
# sklearn
if not hasattr(model.__class__, "fit"):
# It is a class object and not an instance. We use the default values.
model = model()
model.fit(X, y)
h = random.randint(0, X.shape[0] - 1)
if isinstance(X, pandas.DataFrame):
oneX = X.iloc[h, :].astype(numpy.float32)
else:
oneX = X[h, :].ravel().astype(numpy.float32)
# model or transform
moneX = numpy.resize(oneX, (1, len(oneX)))
if hasattr(model, "predict"):
ske = model.predict(moneX)
else:
ske = model.transform(moneX)
if verbose and fLOG:
fLOG("---------------------")
fLOG(type(oneX), oneX.dtype)
fLOG(model)
for k, v in sorted(model.__dict__.items()):
if k[-1] == '_':
fLOG(" {0}={1}".format(k, v))
fLOG("---------------------")
# grammar
gr = sklearn2graph(model, output_names=output_names)
lot = gr.execute(Features=oneX)
if verbose and fLOG:
fLOG(gr.graph_execution())
# verification
check_is_almost_equal(lot, ske)
# default for output_names
if output_names is None:
if len(ske.shape) == 1:
output_names = ["Prediction"]
elif len(ske.shape) == 2:
output_names = ["p%d" % i for i in range(ske.shape[1])]
else:
raise ValueError( # pragma: no cover
"Cannot guess default values for output_names.")
for lang in convs:
if lang in ('c', ):
code_c = gr.export(lang=lang)['code']
if code_c is None:
raise ValueError("cannot be None") # pragma: no cover
compile_fct = compile_c_function
from contextlib import redirect_stdout, redirect_stderr
from io import StringIO
fout = StringIO()
ferr = StringIO()
with redirect_stdout(fout):
with redirect_stderr(ferr):
try:
fct = compile_fct(
code_c, len(output_names), suffix=suffix, fLOG=lambda s: fout.write(s + "\n"))
except Exception as e: # pragma: no cover
raise RuntimeError(
"Unable to compile a code\n-OUT-\n{0}\n-ERR-\n{1}\n-CODE-"
"\n{2}".format(fout.getvalue(), ferr.getvalue(), code_c)) from e
if verbose and fLOG:
fLOG("-----------------")
fLOG(output_names)
fLOG("-----------------")
fLOG(code_c)
fLOG("-----------------")
fLOG("h=", h, "oneX=", oneX)
fLOG("-----------------")
lotc = fct(oneX)
check_is_almost_equal(
lotc, ske, message="Issue with lang='{0}'".format(lang))
lotc_exp = lotc.copy()
lotc2 = fct(oneX, lotc)
if not numpy.array_equal(lotc_exp, lotc2):
raise ValueError( # pragma: no cover
"Second call returns different results.\n{0}\n{1}".format(
lotc_exp, lotc2))
else:
ser = gr.export(lang="json", hook={'array': lambda v: v.tolist()})
if ser is None:
raise ValueError( # pragma: no cover
"No output for long='{0}'".format(lang))
def save_params(self,filename):
with open(filename, 'w') as f:
with redirect_stdout(f):
#Write error report
lm.report_fit(out1.params)
def _explain(args, base_dir, workflows_dir, config_path):
"""
Run a complete dryrun workflow, then return the explanations of each rule used.
"""
if not os.path.exists(config_path):
sys.stdout.write(
f"The config file: {config_path} does not exist.\nProvide a path to the config file with "
f"--config or if you do not have a config file run:\n"
f"seq2science init {args.workflow}\n")
sys.exit(1)
# parse the args
parsed_args = {
"snakefile":
os.path.join(workflows_dir, args.workflow.replace("-", "_"),
"Snakefile"),
"dryrun":
True,
"forceall":
True,
"quiet":
False
}
# get the additional snakemake options
snakemake_options = args.snakemakeOptions if args.snakemakeOptions is not None else dict(
)
snakemake_options.setdefault("config", {}).update(
{"rule_dir": os.path.join(base_dir, "rules")})
snakemake_options["configfiles"] = [config_path]
parsed_args.update(snakemake_options)
# parse the profile
if args.profile is not None:
profile_file = snakemake.get_profile_file(args.profile, "config.yaml")
if profile_file is None:
subjectively_prettier_error(
profile_arg, "profile given but no config.yaml found.")
add_profile_args(profile_file, parsed_args)
# cores
parsed_args["cores"] = 999
# starting message
rules_used = {
"start":
f"\nPreprocessing of reads was done automatically with workflow tool "
f"seq2science v{seq2science.__version__} (https://doi.org/10.5281/zenodo.3921913)."
}
def log_handler(log):
if log["level"] == "job_info" and "msg" in log and log[
"msg"] is not None and log["name"] not in rules_used:
rules_used[log["name"]] = log["msg"]
parsed_args["log_handler"] = [log_handler]
parsed_args["config"]["explain_rule"] = True
# run snakemake (silently)
with open(os.devnull, "w") as null:
with contextlib.redirect_stdout(null), contextlib.redirect_stderr(
null):
exit_code = snakemake.snakemake(**parsed_args)
print(" ".join(rules_used.values()))
sys.exit(0) if exit_code else sys.exit(1)
def test_initialize_logging_with_debug(self):
with redirect_stdout(io.StringIO()):
initialize_logging(debug=True)
self.assertEqual(logging.getLogger().level, logging.DEBUG)
self.assertEqual(logging.getLogger('limis').level, logging.DEBUG)
async def repl(self, ctx):
msg = ctx.message
variables = {
'ctx': ctx,
'bot': self.bot,
'message': msg,
'server': msg.guild,
'channel': msg.channel,
'author': msg.author,
'last': None,
}
if msg.channel.id in self.sessions:
await ctx.send(
'Already running a REPL session in this channel. Exit it with `quit`.'
)
return
self.sessions.add(msg.channel.id)
await ctx.send(
'Enter code to execute or evaluate. `exit()` or `quit` to exit.')
while True:
response = await self.bot.wait_for(
'message',
check=lambda m: m.content.startswith('`') and m.author == msg.
author and m.channel == msg.channel)
cleaned = self.cleanup_code(response.content)
if cleaned in ('quit', 'exit', 'exit()'):
await ctx.send('Exiting.')
self.sessions.remove(msg.channel.id)
return
executor = exec
if cleaned.count('\n') == 0:
# single statement, potentially 'eval'
try:
code = compile(cleaned, '<repl session>', 'eval')
except SyntaxError:
pass
else:
executor = eval
if executor is exec:
try:
code = compile(cleaned, '<repl session>', 'exec')
except SyntaxError as e:
await ctx.send(self.get_syntax_error(e))
continue
variables['message'] = response
fmt = None
stdout = io.StringIO()
try:
with redirect_stdout(stdout):
result = executor(code, variables)
if inspect.isawaitable(result):
result = await result
except Exception as e:
value = stdout.getvalue()
fmt = '```py\n{}{}\n```'.format(value, traceback.format_exc())
else:
value = stdout.getvalue()
if result is not None:
fmt = '```py\n{}{}\n```'.format(value, result)
variables['last'] = result
elif value:
fmt = '```py\n{}\n```'.format(value)
try:
if fmt is not None:
if len(fmt) > 2000:
await msg.channel.send('Content too big to be printed.'
)
else:
await msg.channel.send(fmt)
except discord.Forbidden:
pass
except discord.HTTPException as e:
await msg.channel.send('Unexpected error: `{}`'.format(e))
def setUp(self):
json_file = MetadataCatalog.get("coco_2017_val_100").json_file
if not os.path.isfile(json_file):
raise unittest.SkipTest("{} not found".format(json_file))
with contextlib.redirect_stdout(io.StringIO()):
self.coco = COCO(json_file)
def test_display(self):
"""Method to test display output of square with # char"""
r1 = Rectangle(2, 3, 0, 0)
meow = "##\n##\n##\n"
r1.display()
assert re.match(r'^##\n##\n##\n$', meow)
r1 = Rectangle(2, 3, 3, 0)
meow = " ##\n ##\n ##\n"
assert re.match(r'^ ##\n ##\n ##\n$', meow)
"""output = sys.stdout
r1 = Rectangle(3,2, 0, 0)
"""
r1 = Rectangle(2, 3, x=0, y=0)
outputStr = io.StringIO()
with contextlib.redirect_stdout(outputStr):
r1.display()
self.assertEqual("##\n##\n##\n", outputStr.getvalue())
"""
self.assertEqual(r1.display(), meow)
suite = unittest.TestLoader().loadTestsFromModule(__Rectangle__)
with io.StringIO() as buf:
with contextlib.redirect_stdout(buf):
unittest.TextTestRunner(stream=buf).run(suite)
"""
r1 = Rectangle(2, 3, 1, 1)
meow = "\n ##\n ##\n ##\n"
outputStr = io.StringIO()
with contextlib.redirect_stdout(outputStr):
r1.display()
self.assertEqual("\n ##\n ##\n ##\n", outputStr.getvalue())
r2 = Rectangle(3, 2)
outputStr = io.StringIO()
with contextlib.redirect_stdout(outputStr):
r2.display()
self.assertEqual("###\n###\n", outputStr.getvalue())
r2 = Rectangle(3, 2, 2)
outputStr = io.StringIO()
with contextlib.redirect_stdout(outputStr):
r2.display()
self.assertEqual(" ###\n ###\n", outputStr.getvalue())
r2 = Rectangle(3, 2, 2, 0)
outputStr = io.StringIO()
with contextlib.redirect_stdout(outputStr):
r2.display()
self.assertEqual(" ###\n ###\n", outputStr.getvalue())
r2 = Rectangle(3, 2, 0, 1, 1)
outputStr = io.StringIO()
with contextlib.redirect_stdout(outputStr):
r2.display()
self.assertEqual("\n###\n###\n", outputStr.getvalue())
r2 = Rectangle(3, 2, 0, 0, 1)
outputStr = io.StringIO()
with contextlib.redirect_stdout(outputStr):
r2.display()
self.assertEqual("###\n###\n", outputStr.getvalue())
def __init__(self,
clargs,
data_instance,
vae_latent_dim,
vae_hidden_dims,
dnn_hidden_dims,
generationID=0,
chromosomeID=0,
vae_kl_weight=1.0,
dnn_weight=1.0,
dnn_kl_weight=1.0,
verbose=False):
self.verbose = verbose
self.clargs = clargs
self.data_instance = data_instance
self.generationID = generationID
self.chromosomeID = chromosomeID
self.time_stamp = clargs.time_stamp
self.vae_latent_dim = vae_latent_dim
self.vae_hidden_dims = vae_hidden_dims
self.dnn_hidden_dims = dnn_hidden_dims
self.vae_kl_weight = vae_kl_weight
self.dnn_weight = dnn_weight
self.dnn_kl_weight = dnn_kl_weight
self.params_dict = {}
for k, layer_size in enumerate(self.vae_hidden_dims):
self.params_dict['size_vae_hidden{}'.format(k)] = layer_size
self.params_dict['vae_latent_dim'] = self.vae_latent_dim
for k, layer_size in enumerate(self.dnn_hidden_dims):
self.params_dict['size_dnn_hidden{}'.format(k)] = layer_size
self.model_dir = clargs.model_dir
self.run_name = clargs.run_name
self.predictor_type = clargs.predictor_type
self.original_dim = clargs.original_dim
self.dnn_weight = clargs.dnn_weight
self.optimizer = clargs.optimizer
self.batch_size = clargs.batch_size
self.use_prev_input = False
self.dnn_out_dim = clargs.n_labels
self.dnn_latent_dim = clargs.n_labels - 1
self.get_model()
self.neural_net = self.model
self.fitness = 0
assert (os.path.exists(self.model_dir)), "{} does not exist.".format(
self.model_dir)
self.model_topology_savefile = '{}/{}_{}_{}_model_topology_savefile_{}.save'
self.model_topology_savefile = self.model_topology_savefile.format(
self.model_dir, self.run_name, self.generationID,
self.chromosomeID, self.time_stamp)
with open(self.model_topology_savefile, 'w') as f:
with redirect_stdout(f):
self.neural_net.summary()
yaml_filename = self.model_topology_savefile.replace('.save', '.yaml')
with open(yaml_filename, 'w') as yaml_fileout:
yaml_fileout.write(self.neural_net.to_yaml())
# save model args
json_filename = self.model_topology_savefile.replace('.save', '.json')
with open(json_filename, 'w') as json_fileout:
json_fileout.write(self.neural_net.to_json())
if verbose: self.neural_net.summary()
