def test__repr(self):
r1 = Record(
MockExecutable("1"),
MockRepository(),
"test.py",
999,
MockLaunchMode(),
MockDataStore(),
{"a": 3},
label="A",
)
time.sleep(1)
r2 = Record(
MockExecutable("2"),
MockRepository(),
"test.py",
999,
MockLaunchMode(),
MockDataStore(),
{"a": 2},
label="B",
)
r1.dependencies = [MockDependency("foo"), MockDependency("bar")]
r2.dependencies = [MockDependency("bar"), MockDependency("eric")]
diff = RecordDifference(r1, r2)
self.assertEqual(repr(diff), "RecordDifference(A, B):XCP")
def new_record(self, parameters={}, input_data=[], script_args="",
executable='default', repository='default',
main_file='default', version='current', launch_mode='default',
diff='', label=None, reason=None, timestamp_format='default'):
logger.debug("Creating new record")
if executable == 'default':
executable = deepcopy(self.default_executable)
if repository == 'default':
repository = deepcopy(self.default_repository)
if main_file == 'default':
main_file = self.default_main_file
if launch_mode == 'default':
launch_mode = deepcopy(self.default_launch_mode)
if timestamp_format == 'default':
timestamp_format = self.timestamp_format
working_copy = repository.get_working_copy()
version, diff = self.update_code(working_copy, version, diff)
if label is None:
label = LABEL_GENERATORS[self.label_generator]()
record = Record(executable, repository, main_file, version, launch_mode,
self.data_store, parameters, input_data, script_args,
label=label, reason=reason, diff=diff,
on_changed=self.on_changed,
input_datastore=self.input_datastore,
timestamp_format=timestamp_format)
self.add_record(record)
if not isinstance(executable, programs.MatlabExecutable):
record.register(working_copy)
return record
def test__init(self):
r1 = Record(MockExecutable(), MockRepository(), "test.py", 999,
MockLaunchMode(), MockDataStore())
time.sleep(1)
r2 = Record(MockExecutable(), MockRepository(), "test.py", 999,
MockLaunchMode(), MockDataStore())
diff = RecordDifference(r1, r2)
def new_record(self, parameters={}, input_data=[], script_args="",
executable='default', repository='default',
main_file='default', version='current', launch_mode='default',
label=None, reason=None, timestamp_format='default'):
logger.debug("Creating new record")
if executable == 'default':
executable = deepcopy(self.default_executable)
if repository == 'default':
repository = deepcopy(self.default_repository)
if main_file == 'default':
main_file = self.default_main_file
if launch_mode == 'default':
launch_mode = deepcopy(self.default_launch_mode)
if timestamp_format == 'default':
timestamp_format = self.timestamp_format
working_copy = repository.get_working_copy()
version, diff = self.update_code(working_copy, version)
if label is None:
label = LABEL_GENERATORS[self.label_generator]()
record = Record(executable, repository, main_file, version, launch_mode,
self.data_store, parameters, input_data, script_args,
label=label, reason=reason, diff=diff,
on_changed=self.on_changed,
input_datastore=self.input_datastore,
timestamp_format=timestamp_format)
if not isinstance(executable, programs.MatlabExecutable):
record.register(working_copy)
return record
def test__dependency_differences(self):
r1 = Record(MockExecutable(), MockRepository(), "test.py", 999, MockLaunchMode(), MockDataStore())
time.sleep(1)
r2 = Record(MockExecutable(), MockRepository(), "test.py", 999, MockLaunchMode(), MockDataStore())
r1.dependencies = [MockDependency("foo"), MockDependency("bar")]
r2.dependencies = [MockDependency("bar"), MockDependency("eric")]
diff = RecordDifference(r1, r2)
diff.dependency_differences
def test__run(self):
r1 = Record(MockExecutable("1"),
MockRepository(),
"test.py",
999,
MockLaunchMode(),
MockDataStore(), {"a": 3},
label="A")
r1.run(with_label='parameters')
def test__run(self):
r1 = Record(
MockExecutable("1"),
MockRepository(),
"test.py",
999,
MockLaunchMode(),
MockDataStore(),
{"a": 3},
label="A",
)
r1.run(with_label="parameters")
def test__nonzero__should_return_True__for_identical_parameters(self):
r1 = Record(MockExecutable(), MockRepository(), "test.py", 999, MockLaunchMode(), MockDataStore(), {"a": 2})
time.sleep(1)
r2 = Record(MockExecutable(), MockRepository(), "test.py", 999, MockLaunchMode(), MockDataStore(), {"a": 2})
r1.dependencies = []
r2.dependencies = []
diff = RecordDifference(r1, r2)
self.assertEqual(diff.executable_differs, False)
self.assertEqual(diff.code_differs, False)
self.assertEqual(diff.parameters_differ, False)
self.assertEqual(diff.output_data_differ, False)
self.assertEqual(diff.__bool__(), False)
def decode_record(content):
data = json.loads(content)
edata = data["executable"]
cls = programs.registered_program_names.get(edata["name"], programs.Executable)
executable = cls(edata["path"], edata["version"], edata["options"])
executable.name = edata["name"]
rdata = data["repository"]
repos_cls = None
for m in versioncontrol.vcs_list:
if hasattr(m, rdata["type"]):
repos_cls = getattr(m, rdata["type"])
break
if repos_cls is None:
repos_cls = versioncontrol.base.Repository
repository = repos_cls(rdata["url"])
pdata = data["parameters"]
if pdata["type"] == "dict":
parameter_set = eval(pdata["content"])
assert isinstance(parameter_set, dict)
elif pdata['type']=="NoneType":
parameter_set = None
else:
parameter_set = getattr(parameters, pdata["type"])(pdata["content"])
ldata = data["launch_mode"]
lm_parameters = eval(ldata["parameters"])
launch_mode = getattr(launch, ldata["type"])(**lm_parameters)
ddata = data["datastore"]
ds_parameters = eval(ddata["parameters"])
data_store = getattr(datastore, ddata["type"])(**ds_parameters)
record = Record(executable, repository, data["main_file"],
data["version"], launch_mode, data_store, parameter_set,
data["input_data"], data["script_arguments"],
data["label"], data["reason"], data["diff"],
data["user"])
tags = data["tags"]
if not hasattr(tags, "__iter__"):
tags = (tags,)
record.tags = set(tags)
record.timestamp = datetime.strptime(data["timestamp"], "%Y-%m-%d %H:%M:%S")
record.data_key = data["data_key"]
record.duration = data["duration"]
record.outcome = data["outcome"]
record.platforms = [launch.PlatformInformation(**keys2str(pldata)) for pldata in data["platforms"]]
record.dependencies = []
for depdata in data["dependencies"]:
dep = getattr(dependency_finder, depdata["module"]).Dependency(depdata["name"], depdata["path"], depdata["version"])
dep.diff = depdata["diff"]
record.dependencies.append(dep)
return record
def test__dependency_differences(self):
r1 = Record(MockExecutable(), MockRepository(), "test.py", 999,
MockLaunchMode(), MockDataStore())
time.sleep(1)
r2 = Record(MockExecutable(), MockRepository(), "test.py", 999,
MockLaunchMode(), MockDataStore())
r1.dependencies = [MockDependency("foo"), MockDependency("bar")]
r2.dependencies = [MockDependency("bar"), MockDependency("eric")]
diff = RecordDifference(r1, r2)
diff.dependency_differences
def test__repr(self):
r1 = Record(MockExecutable("1"), MockRepository(), "test.py",
999, MockLaunchMode(), MockDataStore(), {"a": 3}, label="A")
time.sleep(1)
r2 = Record(MockExecutable("2"), MockRepository(), "test.py",
999, MockLaunchMode(), MockDataStore(), {"a": 2}, label="B")
r1.dependencies = [MockDependency("foo"), MockDependency("bar")]
r2.dependencies = [MockDependency("bar"), MockDependency("eric")]
diff = RecordDifference(r1, r2)
self.assertEqual(repr(diff), "RecordDifference(A, B):XCP")
def test__nonzero__should_return_True__for_identical_parameters(self):
r1 = Record(MockExecutable(), MockRepository(), "test.py", 999,
MockLaunchMode(), MockDataStore(), {"a": 2})
time.sleep(1)
r2 = Record(MockExecutable(), MockRepository(), "test.py", 999,
MockLaunchMode(), MockDataStore(), {"a": 2})
r1.dependencies = []
r2.dependencies = []
diff = RecordDifference(r1, r2)
self.assertEqual(diff.executable_differs, False)
self.assertEqual(diff.code_differs, False)
self.assertEqual(diff.parameters_differ, False)
self.assertEqual(diff.output_data_differ, False)
self.assertEqual(diff.__bool__(), False)
from sumatra.versioncontrol._git import GitRepository
import random
from datetime import datetime
serial = SerialLaunchMode()
executable = PythonExecutable("/usr/bin/python", version="2.7")
repos = GitRepository('.')
datastore = FileSystemDataStore("/path/to/datastore")
project = Project("test_project",
default_executable=executable,
default_repository=repos,
default_launch_mode=serial,
data_store=datastore,
record_store=django_store.DjangoRecordStore())
parameters = SimpleParameterSet({'a': 2, 'b': 3})
for i in range(50):
record = Record(executable=executable, repository=repos,
main_file="main.py", version="99863a9dc5f",
launch_mode=serial, datastore=datastore,
parameters=parameters, input_data=[], script_arguments="",
label="fake_record%00d" % i,
reason="testing", diff='', user='******',
on_changed='store-diff', stdout_stderr='srgvrgvsgverhcser')
record.duration = random.gammavariate(1.0, 1000.0)
record.outcome = "lghsvdghsg zskjdcghnskdjgc ckdjshcgndsg"
record.data_key = "['output.data']"
record.dependencies = []
record.platforms = serial.get_platform_information()
project.add_record(record)
def build_record(data):
"""Create a Sumatra record from a nested dictionary."""
edata = data["executable"]
cls = get_registered_components(programs.Executable).get(edata["name"], programs.Executable)
executable = cls(edata["path"], edata["version"], edata.get("options", ""))
executable.name = edata["name"]
rdata = data["repository"]
repos_cls = None
for m in versioncontrol.vcs_list:
if hasattr(m, rdata["type"]):
repos_cls = getattr(m, rdata["type"])
break
if repos_cls is None:
repos_cls = versioncontrol.base.Repository
repository = repos_cls(rdata["url"])
repository.upstream = rdata.get("upstream", None)
pdata = data["parameters"]
if pdata["type"] == "dict":
parameter_set = eval(pdata["content"])
assert isinstance(parameter_set, dict)
else:
parameter_set = getattr(parameters, pdata["type"])(pdata["content"])
ldata = data["launch_mode"]
lm_parameters = ldata["parameters"]
if isinstance(lm_parameters, str): # prior to 0.3
lm_parameters = eval(lm_parameters)
launch_mode = getattr(launch, ldata["type"])(**keys2str(lm_parameters))
def build_data_store(ddata):
ds_parameters = ddata["parameters"]
if isinstance(ds_parameters, str): # prior to 0.3
ds_parameters = eval(ds_parameters)
return getattr(datastore, ddata["type"])(**keys2str(ds_parameters))
data_store = build_data_store(data["datastore"])
if "input_datastore" in data: # 0.4 onwards
input_datastore = build_data_store(data["input_datastore"])
else:
input_datastore = datastore.FileSystemDataStore("/")
input_data = data.get("input_data", [])
if isinstance(input_data, str): # 0.3
input_data = eval(input_data)
if input_data:
if isinstance(input_data[0], str): # versions prior to 0.4
input_data = [datastore.DataKey(path, digest=datastore.IGNORE_DIGEST, creation=None)
for path in input_data]
else:
input_data = [datastore.DataKey(keydata["path"], keydata["digest"],
creation=datestring_to_datetime(keydata.get("creation", None)),
**keys2str(keydata["metadata"]))
for keydata in input_data]
record = Record(executable, repository, data["main_file"],
data["version"], launch_mode, data_store, parameter_set,
input_data, data.get("script_arguments", ""),
data["label"], data["reason"], data["diff"],
data.get("user", ""), input_datastore=input_datastore,
timestamp=datestring_to_datetime(data["timestamp"]))
tags = data["tags"]
if not hasattr(tags, "__iter__"):
tags = (tags,)
record.tags = set(tags)
record.output_data = []
if "output_data" in data:
for keydata in data["output_data"]:
data_key = datastore.DataKey(keydata["path"], keydata["digest"],
creation=datestring_to_datetime(keydata.get("creation", None)),
**keys2str(keydata["metadata"]))
record.output_data.append(data_key)
elif "data_key" in data: # (versions prior to 0.4)
for path in eval(data["data_key"]):
data_key = datastore.DataKey(path, digest=datastore.IGNORE_DIGEST,
creation=None)
record.output_data.append(data_key)
record.duration = data["duration"]
record.outcome = data["outcome"]
record.stdout_stderr = data.get("stdout_stderr", "")
record.platforms = [launch.PlatformInformation(**keys2str(pldata)) for pldata in data["platforms"]]
record.dependencies = []
for depdata in data["dependencies"]:
dep_args = [depdata["name"], depdata["path"], depdata["version"],
depdata["diff"]]
if "source" in depdata: # 0.5 onwards
dep_args.append(depdata["source"])
dep = getattr(dependency_finder, depdata["module"]).Dependency(*dep_args)
record.dependencies.append(dep)
record.repeats = data.get("repeats", None)
return record
project = Project("test_project",
default_executable=executable,
default_repository=repos,
default_launch_mode=serial,
data_store=datastore,
record_store=django_store.DjangoRecordStore())
parameters = SimpleParameterSet({'a': 2, 'b': 3})
for i in range(50):
record = Record(executable=executable,
repository=repos,
main_file="main.py",
version="99863a9dc5f",
launch_mode=serial,
datastore=datastore,
parameters=parameters,
input_data=[],
script_arguments="",
label="fake_record%00d" % i,
reason="testing",
diff='',
user='******',
on_changed='store-diff',
stdout_stderr='srgvrgvsgverhcser')
record.duration = random.gammavariate(1.0, 1000.0)
record.outcome = "lghsvdghsg zskjdcghnskdjgc ckdjshcgndsg"
record.data_key = "['output.data']"
record.dependencies = []
record.platforms = serial.get_platform_information()
project.add_record(record)
def test__update_parameters_with_timestamp_label(self):
r1 = Record(MockExecutable("1"), MockRepository(), "test.py",
999, MockLaunchMode(), MockDataStore(), SimpleParameterSet("a = 3"))
r1.run(with_label='parameters')
