def wrapped_main(*args, **kwargs):
if len(args) > 0 and isinstance(args[0], ParameterSet):
# If the first argument is a ParameterSet
parameters = args[0]
elif len(kwargs) > 0 and "parameters" in kwargs \
and isinstance(kwargs["parameters"], ParameterSet):
# If there is a named "parameters" argument
parameters = kwargs["parameters"]
else:
# Package all parameters into a SimpleParameterSet
parameters = dict(
zip(["arg%d" % x for x in range(len(args))], args))
parameters.update(kwargs)
parameters = SimpleParameterSet(parameters)
import sumatra.projects
project = sumatra.projects.load_project()
main_file = sys.modules['__main__'].__file__
executable = PythonExecutable(path=sys.executable)
record = project.new_record(parameters=parameters,
main_file=main_file,
executable=executable)
record.launch_mode.working_directory = os.getcwd()
parameters.update({"sumatra_label": record.label})
start_time = time.time()
with _grab_stdout_stderr() as stdout_stderr:
main(*args, **kwargs)
record.stdout_stderr = stdout_stderr.getvalue()
record.duration = time.time() - start_time
record.output_data = record.datastore.find_new_data(record.timestamp)
project.add_record(record)
project.save()
def wrapped_main(*args, **kwargs):
if len(args) > 0 and isinstance(args[0], ParameterSet):
# If the first argument is a ParameterSet
parameters = args[0]
elif len(kwargs) > 0 and "parameters" in kwargs \
and isinstance(kwargs["parameters"], ParameterSet):
# If there is a named "parameters" argument
parameters = kwargs["parameters"]
else:
# Package all parameters into a SimpleParameterSet
parameters = dict(zip(["arg%d" % x for x in range(len(args))], args))
parameters.update(kwargs)
parameters = SimpleParameterSet(parameters)
import sumatra.projects
project = sumatra.projects.load_project()
main_file = sys.modules['__main__'].__file__
executable = PythonExecutable(path=sys.executable)
record = project.new_record(parameters=parameters,
main_file=main_file,
executable=executable)
record.launch_mode.working_directory = os.getcwd()
parameters.update({"sumatra_label": record.label})
start_time = time.time()
with _grab_stdout_stderr() as stdout_stderr:
main(*args, **kwargs)
record.stdout_stderr = stdout_stderr.getvalue()
record.duration = time.time() - start_time
record.output_data = record.datastore.find_new_data(record.timestamp)
project.add_record(record)
project.save()
def test__init__should_accept_a_filename_or_string(self):
init = "x = 2\ny = 3"
P1 = SimpleParameterSet(init)
with open("test_file", "w") as f:
f.write(init)
P2 = SimpleParameterSet("test_file")
self.assertEqual(P1.values, P2.values)
os.remove("test_file")
def test__update__should_only_accept_numbers_or_strings(self):
# could maybe allow lists of numbers or strings
P = SimpleParameterSet("x = 2\ny = 3")
P.update({"z": "hello"})
self.assertEqual(P["z"], "hello")
P.update({"tumoltuae": 42})
self.assertEqual(P["tumoltuae"], 42)
self.assertRaises(TypeError, P.update, "bar", {})
def test__update__should_accept_basic_python_types(self):
P = SimpleParameterSet("x = 2\ny = 3")
param_values = [("z", "hello"), ("tumoltuae", 42),
("somelist", [1, 2, "a", "b"]),
("sometuple", (3, 4, "c")), ("foo", None),
("isittrue", False)]
for name, value in param_values:
P.update({name: value})
self.assertEqual(P[name], value)
def test__save__should_backup_an_existing_file_before_overwriting_it(self):
# not really sure what the desired behaviour is here
assert not os.path.exists("test_file.orig")
init = "x = 2\ny = 3"
with open("test_file", "w") as f:
f.write(init)
P = SimpleParameterSet("test_file")
P.save("test_file")
self.assert_(os.path.exists("test_file.orig"))
os.remove("test_file")
os.remove("test_file.orig")
def test__update__should_accept_basic_python_types(self):
P = SimpleParameterSet("x = 2\ny = 3")
param_values = [("z", "hello"),
("tumoltuae", 42),
("somelist", [1, 2, "a", "b"]),
("sometuple", (3, 4, "c")),
("foo", None),
("isittrue", False)]
for name, value in param_values:
P.update({name: value})
self.assertEqual(P[name], value)
def test__save__should_backup_an_existing_file_before_overwriting_it(self):
# not really sure what the desired behaviour is here
assert not os.path.exists("test_file.orig")
init = "x = 2\ny = 3"
with open("test_file", "w") as f:
f.write(init)
P = SimpleParameterSet("test_file")
P.save("test_file")
self.assert_(os.path.exists("test_file.orig"))
os.remove("test_file")
os.remove("test_file.orig")
def test_diff(self):
P1 = SimpleParameterSet("x = 2\ny = 3")
P2 = SimpleParameterSet("x = 3\ny = 3\nz = 4")
self.assertEqual(P1.diff(P2), ({'x': 2}, {'x': 3, 'z': 4}))
P3 = JSONParameterSet(
dedent("""
{
"x" : 2,
"y" : 4,
"z": 4
}"""))
self.assertEqual(P1.diff(P3), ({'y': 3}, {'y': 4, 'z': 4}))
def test__init__should_ignore_comment_lines(self):
init = textwrap.dedent("""\
#some parameters
x = 2
# this is a comment at column 0
# this is an indented comment
y = 3
# this is a comment line containing an 'equals' sign: n=42
""")
P = SimpleParameterSet(init)
self.assertEqual(P["x"], 2)
self.assertEqual(P["y"], 3)
self.assertEqual(set(P.as_dict().keys()), set(["x", "y"]))
def test__init__should_ignore_comment_lines(self):
init = textwrap.dedent("""\
#some parameters
x = 2
# this is a comment at column 0
# this is an indented comment
y = 3
# this is a comment line containing an 'equals' sign: n=42
""")
P = SimpleParameterSet(init)
self.assertEqual(P["x"], 2)
self.assertEqual(P["y"], 3)
self.assertEqual(set(P.as_dict().keys()), set(["x", "y"]))
def test_diff(self):
P1 = SimpleParameterSet("x = 2\ny = 3")
P2 = SimpleParameterSet("x = 3\ny = 3\nz = 4")
self.assertEqual(P1.diff(P2),
({'x': 2}, {'x': 3, 'z': 4}))
P3 = JSONParameterSet(dedent("""
{
"x" : 2,
"y" : 4,
"z": 4
}"""))
self.assertEqual(P1.diff(P3),
({'y': 3}, {'y': 4, 'z': 4}))
def wrapped_main(*args, **kwargs):
if len(args) > 0 and isinstance(args[0], ParameterSet):
# If the first argument is a ParameterSet
parameters = args[0]
elif len(kwargs) > 0 and "parameters" in kwargs \
and isinstance(kwargs["parameters"], ParameterSet):
# If there is a named "parameters" argument
parameters = kwargs["parameters"]
else:
# Package all parameters into a SimpleParameterSet
parameters = dict(
zip(["arg%d" % x for x in range(len(args))], args))
parameters.update(kwargs)
parameters = SimpleParameterSet(parameters)
import sumatra.projects
project = sumatra.projects.load_project()
main_file = sys.modules['__main__'].__file__
executable = PythonExecutable(path=sys.executable)
record = project.new_record(parameters=parameters,
main_file=main_file,
executable=executable)
record.launch_mode.working_directory = os.getcwd()
parameters.update({"sumatra_label": record.label})
record.add_tag(STATUS_FORMAT % "running")
record.stdout_stderr = "Not yet captured."
project.add_record(record)
start_time = time.time()
with _grab_stdout_stderr() as stdout_stderr:
try:
main(*args, **kwargs)
status = "finished"
except KeyboardInterrupt:
status = "killed"
except Exception as e:
status = "failed"
record.outcome = repr(e)
traceback.print_exc()
finally:
record.stdout_stderr = stdout_stderr.getvalue()
record.add_tag(STATUS_FORMAT % (status + "..."))
project.save_record(record)
record.duration = time.time() - start_time
record.output_data = record.datastore.find_new_data(record.timestamp)
record.add_tag(STATUS_FORMAT % status)
project.save_record(record)
project.save()
def test__update__should_not_accept_complex_types(self):
class CustomClass(object):
pass
P = SimpleParameterSet("x = 2\ny = 3")
self.assertRaises(TypeError, P.update, "somedict", {})
self.assertRaises(TypeError, P.update, "someclass", CustomClass())
class MockParameterSet(dict):
def __init__(self, filename):
dict.__init__(self, this="mock")
self.ps = SimpleParameterSet("")
self.parameter_file = filename
def parse_command_line_parameter(self, cl):
return self.ps.parse_command_line_parameter(cl)
def pretty(self, expand_urls):
return str(self)
class MockParameterSet(dict):
def __init__(self, filename):
dict.__init__(self, this='mock')
self.ps = SimpleParameterSet("")
self.parameter_file = filename
def parse_command_line_parameter(self, cl):
return self.ps.parse_command_line_parameter(cl)
def pretty(self, expand_urls):
return str(self)
def setUp(self):
## setup parsets with legal params
self.PSETS = [SimpleParameterSet(""), JSONParameterSet("")]
try:
self.PSETS.append(YAMLParameterSet(""))
except ImportError:
pass
self.PConfigParser = ConfigParserParameterSet("")
for k in ('a', 'b', 'c', 'd', 'l', 'save'):
up_dict = {k: 1}
self.PConfigParser.update(up_dict)
for P in self.PSETS:
P.update(up_dict)
def test__update__should_only_accept_numbers_or_strings(self):
# could maybe allow lists of numbers or strings
P = SimpleParameterSet("x = 2\ny = 3")
P.update({"z": "hello"})
self.assertEqual(P["z"], "hello")
P.update({"tumoltuae": 42})
self.assertEqual(P["tumoltuae"], 42)
self.assertRaises(TypeError, P.update, "bar", {})
def test__init__should_accept_an_empty_initializer(self):
P = SimpleParameterSet("")
self.assertEqual(P.values, {})
def test__pretty__should_put_quotes_around_string_parameters(self):
init = 'y = "hello"'
P = SimpleParameterSet(init)
self.assertEqual(P.pretty(), init)
def test__pop(self):
P = SimpleParameterSet("x = 2\ny = 3")
self.assertEqual(P.pop('x'), 2)
self.assertEqual(P.as_dict(), {'y': 3})
self.assertEqual(P.pop('foo', 42), 42)
self.assertEqual(P.pop('foo', None), None)
def test__pretty__output_should_be_useable_to_create_an_identical_parameterset(self):
init = "x = 2\ny = 3\nz = 'hello'"
P1 = SimpleParameterSet(init)
P2 = SimpleParameterSet(P1.pretty())
self.assertEqual(P1.values, P2.values)
def test__pretty__should_put_quotes_around_string_parameters(self):
init = 'y = "hello"'
P = SimpleParameterSet(init)
self.assertEqual(P.pretty(), init)
def test__string_parameters_should_be_able_to_contain_equals_signs(self):
init = 'equation = "e = mc^2"'
P = SimpleParameterSet(init)
self.assertEqual(P['equation'], 'e = mc^2')
def test__string_parameters_should_be_able_to_contain_comment_characters(
self):
P = SimpleParameterSet('char = "#"')
self.assertEqual(P['char'], '#')
P = SimpleParameterSet('x = "#abc#"')
self.assertEqual(P['x'], '#abc#')
def test__getitem__should_give_access_to_parameters(self):
P = SimpleParameterSet("x = 2\ny = 3")
self.assertEqual(P["x"], 2)
self.assertEqual(P["y"], 3)
def test__pretty__output_should_be_useable_to_create_an_identical_parameterset(
self):
init = "x = 2\ny = 3\nz = 'hello'"
P1 = SimpleParameterSet(init)
P2 = SimpleParameterSet(P1.pretty())
self.assertEqual(P1.values, P2.values)
def test__pretty__should_recreate_comments_in_the_initializer(self):
init = 'x = 2 # this is a comment'
P = SimpleParameterSet(init)
self.assertEqual(P.pretty(), init)
def __init__(self, filename):
dict.__init__(self, this='mock')
self.ps = SimpleParameterSet("")
self.parameter_file = filename
def test__init__should_accept_dict(self):
P = SimpleParameterSet({'x': 2, 'y': 3})
self.assertEqual(P["x"], 2)
self.assertEqual(P["y"], 3)
def test__str(self):
P = SimpleParameterSet("x = 2\ny = 3")
as_string = str(P)
self.assertIsInstance(as_string, str)
self.assertEqual(P, SimpleParameterSet(as_string))
def test__pretty__should_recreate_comments_in_the_initializer(self):
init = 'x = 2 # this is a comment'
P = SimpleParameterSet(init)
self.assertEqual(P.pretty(), init)
def test__pop(self):
P = SimpleParameterSet("x = 2\ny = 3")
self.assertEqual(P.pop('x'), 2)
self.assertEqual(P.as_dict(), {'y': 3})
self.assertEqual(P.pop('foo', 42), 42)
self.assertEqual(P.pop('foo', None), None)
def test__init__should_ignore_empty_lines(self):
init = "x = 2\n\n\r \ny = 3\n\n\r\t\t \n"
P = SimpleParameterSet(init)
self.assertEqual(P["x"], 2)
self.assertEqual(P["y"], 3)
def test_long_parameter(self):
parameters = "x = '{0}'".format(''.join("a" * 250))
SimpleParameterSet(parameters)
def test__update_kwargs(self):
P = SimpleParameterSet("x = 2\ny = 3")
P.update({}, z="hello")
self.assertEqual(P["z"], "hello")
def test__init__should_accept_no_space_around_equals(self):
P = SimpleParameterSet("x=2")
self.assertEqual(P["x"], 2)
def test__update_parameters_with_timestamp_label(self):
r1 = Record(MockExecutable("1"), MockRepository(), "test.py",
999, MockLaunchMode(), MockDataStore(), SimpleParameterSet("a = 3"))
r1.run(with_label='parameters')
def test__init__should_accept_hash_as_comment_character(self):
P = SimpleParameterSet("x = 2 # this is a comment")
self.assertEqual(P["x"], 2)
from sumatra.datastore import FileSystemDataStore
from sumatra.parameters import SimpleParameterSet
from sumatra.versioncontrol._git import GitRepository
import random
serial = SerialLaunchMode()
executable = PythonExecutable("/usr/bin/python", version="2.7")
repos = GitRepository('.')
datastore = FileSystemDataStore("/path/to/datastore")
project = Project("test_project",
default_executable=executable,
default_repository=repos,
default_launch_mode=serial,
data_store=datastore,
record_store=django_store.DjangoRecordStore())
parameters = SimpleParameterSet({'a': 2, 'b': 3})
for i in range(50):
record = Record(executable=executable,
repository=repos,
main_file="main.py",
version="99863a9dc5f",
launch_mode=serial,
datastore=datastore,
parameters=parameters,
input_data=[],
script_arguments="",
label="fake_record%00d" % i,
reason="testing",
diff='',
user='******',
def test__getitem__should_raise_a_KeyError_for_a_nonexistent_parameter(
self):
P = SimpleParameterSet("x = 2\ny = 3")
self.assertRaises(KeyError, P.__getitem__, "z")
def __init__(self, filename):
dict.__init__(self, this='mock')
self.ps = SimpleParameterSet("")
self.parameter_file = filename
def test__update_kwargs(self):
P = SimpleParameterSet("x = 2\ny = 3")
P.update({}, z="hello")
self.assertEqual(P["z"], "hello")