def test__repr(self):
r1 = Record(
MockExecutable("1"),
MockRepository(),
"test.py",
999,
MockLaunchMode(),
MockDataStore(),
{"a": 3},
label="A",
)
time.sleep(1)
r2 = Record(
MockExecutable("2"),
MockRepository(),
"test.py",
999,
MockLaunchMode(),
MockDataStore(),
{"a": 2},
label="B",
)
r1.dependencies = [MockDependency("foo"), MockDependency("bar")]
r2.dependencies = [MockDependency("bar"), MockDependency("eric")]
diff = RecordDifference(r1, r2)
self.assertEqual(repr(diff), "RecordDifference(A, B):XCP")
def test__dependency_differences(self):
r1 = Record(MockExecutable(), MockRepository(), "test.py", 999, MockLaunchMode(), MockDataStore())
time.sleep(1)
r2 = Record(MockExecutable(), MockRepository(), "test.py", 999, MockLaunchMode(), MockDataStore())
r1.dependencies = [MockDependency("foo"), MockDependency("bar")]
r2.dependencies = [MockDependency("bar"), MockDependency("eric")]
diff = RecordDifference(r1, r2)
diff.dependency_differences
def test__dependency_differences(self):
r1 = Record(MockExecutable(), MockRepository(), "test.py", 999,
MockLaunchMode(), MockDataStore())
time.sleep(1)
r2 = Record(MockExecutable(), MockRepository(), "test.py", 999,
MockLaunchMode(), MockDataStore())
r1.dependencies = [MockDependency("foo"), MockDependency("bar")]
r2.dependencies = [MockDependency("bar"), MockDependency("eric")]
diff = RecordDifference(r1, r2)
diff.dependency_differences
def test__repr(self):
r1 = Record(MockExecutable("1"), MockRepository(), "test.py",
999, MockLaunchMode(), MockDataStore(), {"a": 3}, label="A")
time.sleep(1)
r2 = Record(MockExecutable("2"), MockRepository(), "test.py",
999, MockLaunchMode(), MockDataStore(), {"a": 2}, label="B")
r1.dependencies = [MockDependency("foo"), MockDependency("bar")]
r2.dependencies = [MockDependency("bar"), MockDependency("eric")]
diff = RecordDifference(r1, r2)
self.assertEqual(repr(diff), "RecordDifference(A, B):XCP")
def test__nonzero__should_return_True__for_identical_parameters(self):
r1 = Record(MockExecutable(), MockRepository(), "test.py", 999, MockLaunchMode(), MockDataStore(), {"a": 2})
time.sleep(1)
r2 = Record(MockExecutable(), MockRepository(), "test.py", 999, MockLaunchMode(), MockDataStore(), {"a": 2})
r1.dependencies = []
r2.dependencies = []
diff = RecordDifference(r1, r2)
self.assertEqual(diff.executable_differs, False)
self.assertEqual(diff.code_differs, False)
self.assertEqual(diff.parameters_differ, False)
self.assertEqual(diff.output_data_differ, False)
self.assertEqual(diff.__bool__(), False)
def test__nonzero__should_return_True__for_identical_parameters(self):
r1 = Record(MockExecutable(), MockRepository(), "test.py", 999,
MockLaunchMode(), MockDataStore(), {"a": 2})
time.sleep(1)
r2 = Record(MockExecutable(), MockRepository(), "test.py", 999,
MockLaunchMode(), MockDataStore(), {"a": 2})
r1.dependencies = []
r2.dependencies = []
diff = RecordDifference(r1, r2)
self.assertEqual(diff.executable_differs, False)
self.assertEqual(diff.code_differs, False)
self.assertEqual(diff.parameters_differ, False)
self.assertEqual(diff.output_data_differ, False)
self.assertEqual(diff.__bool__(), False)
def decode_record(content):
data = json.loads(content)
edata = data["executable"]
cls = programs.registered_program_names.get(edata["name"], programs.Executable)
executable = cls(edata["path"], edata["version"], edata["options"])
executable.name = edata["name"]
rdata = data["repository"]
repos_cls = None
for m in versioncontrol.vcs_list:
if hasattr(m, rdata["type"]):
repos_cls = getattr(m, rdata["type"])
break
if repos_cls is None:
repos_cls = versioncontrol.base.Repository
repository = repos_cls(rdata["url"])
pdata = data["parameters"]
if pdata["type"] == "dict":
parameter_set = eval(pdata["content"])
assert isinstance(parameter_set, dict)
elif pdata['type']=="NoneType":
parameter_set = None
else:
parameter_set = getattr(parameters, pdata["type"])(pdata["content"])
ldata = data["launch_mode"]
lm_parameters = eval(ldata["parameters"])
launch_mode = getattr(launch, ldata["type"])(**lm_parameters)
ddata = data["datastore"]
ds_parameters = eval(ddata["parameters"])
data_store = getattr(datastore, ddata["type"])(**ds_parameters)
record = Record(executable, repository, data["main_file"],
data["version"], launch_mode, data_store, parameter_set,
data["input_data"], data["script_arguments"],
data["label"], data["reason"], data["diff"],
data["user"])
tags = data["tags"]
if not hasattr(tags, "__iter__"):
tags = (tags,)
record.tags = set(tags)
record.timestamp = datetime.strptime(data["timestamp"], "%Y-%m-%d %H:%M:%S")
record.data_key = data["data_key"]
record.duration = data["duration"]
record.outcome = data["outcome"]
record.platforms = [launch.PlatformInformation(**keys2str(pldata)) for pldata in data["platforms"]]
record.dependencies = []
for depdata in data["dependencies"]:
dep = getattr(dependency_finder, depdata["module"]).Dependency(depdata["name"], depdata["path"], depdata["version"])
dep.diff = depdata["diff"]
record.dependencies.append(dep)
return record
from sumatra.versioncontrol._git import GitRepository
import random
from datetime import datetime
serial = SerialLaunchMode()
executable = PythonExecutable("/usr/bin/python", version="2.7")
repos = GitRepository('.')
datastore = FileSystemDataStore("/path/to/datastore")
project = Project("test_project",
default_executable=executable,
default_repository=repos,
default_launch_mode=serial,
data_store=datastore,
record_store=django_store.DjangoRecordStore())
parameters = SimpleParameterSet({'a': 2, 'b': 3})
for i in range(50):
record = Record(executable=executable, repository=repos,
main_file="main.py", version="99863a9dc5f",
launch_mode=serial, datastore=datastore,
parameters=parameters, input_data=[], script_arguments="",
label="fake_record%00d" % i,
reason="testing", diff='', user='******',
on_changed='store-diff', stdout_stderr='srgvrgvsgverhcser')
record.duration = random.gammavariate(1.0, 1000.0)
record.outcome = "lghsvdghsg zskjdcghnskdjgc ckdjshcgndsg"
record.data_key = "['output.data']"
record.dependencies = []
record.platforms = serial.get_platform_information()
project.add_record(record)
def build_record(data):
"""Create a Sumatra record from a nested dictionary."""
edata = data["executable"]
cls = get_registered_components(programs.Executable).get(edata["name"], programs.Executable)
executable = cls(edata["path"], edata["version"], edata.get("options", ""))
executable.name = edata["name"]
rdata = data["repository"]
repos_cls = None
for m in versioncontrol.vcs_list:
if hasattr(m, rdata["type"]):
repos_cls = getattr(m, rdata["type"])
break
if repos_cls is None:
repos_cls = versioncontrol.base.Repository
repository = repos_cls(rdata["url"])
repository.upstream = rdata.get("upstream", None)
pdata = data["parameters"]
if pdata["type"] == "dict":
parameter_set = eval(pdata["content"])
assert isinstance(parameter_set, dict)
else:
parameter_set = getattr(parameters, pdata["type"])(pdata["content"])
ldata = data["launch_mode"]
lm_parameters = ldata["parameters"]
if isinstance(lm_parameters, str): # prior to 0.3
lm_parameters = eval(lm_parameters)
launch_mode = getattr(launch, ldata["type"])(**keys2str(lm_parameters))
def build_data_store(ddata):
ds_parameters = ddata["parameters"]
if isinstance(ds_parameters, str): # prior to 0.3
ds_parameters = eval(ds_parameters)
return getattr(datastore, ddata["type"])(**keys2str(ds_parameters))
data_store = build_data_store(data["datastore"])
if "input_datastore" in data: # 0.4 onwards
input_datastore = build_data_store(data["input_datastore"])
else:
input_datastore = datastore.FileSystemDataStore("/")
input_data = data.get("input_data", [])
if isinstance(input_data, str): # 0.3
input_data = eval(input_data)
if input_data:
if isinstance(input_data[0], str): # versions prior to 0.4
input_data = [datastore.DataKey(path, digest=datastore.IGNORE_DIGEST, creation=None)
for path in input_data]
else:
input_data = [datastore.DataKey(keydata["path"], keydata["digest"],
creation=datestring_to_datetime(keydata.get("creation", None)),
**keys2str(keydata["metadata"]))
for keydata in input_data]
record = Record(executable, repository, data["main_file"],
data["version"], launch_mode, data_store, parameter_set,
input_data, data.get("script_arguments", ""),
data["label"], data["reason"], data["diff"],
data.get("user", ""), input_datastore=input_datastore,
timestamp=datestring_to_datetime(data["timestamp"]))
tags = data["tags"]
if not hasattr(tags, "__iter__"):
tags = (tags,)
record.tags = set(tags)
record.output_data = []
if "output_data" in data:
for keydata in data["output_data"]:
data_key = datastore.DataKey(keydata["path"], keydata["digest"],
creation=datestring_to_datetime(keydata.get("creation", None)),
**keys2str(keydata["metadata"]))
record.output_data.append(data_key)
elif "data_key" in data: # (versions prior to 0.4)
for path in eval(data["data_key"]):
data_key = datastore.DataKey(path, digest=datastore.IGNORE_DIGEST,
creation=None)
record.output_data.append(data_key)
record.duration = data["duration"]
record.outcome = data["outcome"]
record.stdout_stderr = data.get("stdout_stderr", "")
record.platforms = [launch.PlatformInformation(**keys2str(pldata)) for pldata in data["platforms"]]
record.dependencies = []
for depdata in data["dependencies"]:
dep_args = [depdata["name"], depdata["path"], depdata["version"],
depdata["diff"]]
if "source" in depdata: # 0.5 onwards
dep_args.append(depdata["source"])
dep = getattr(dependency_finder, depdata["module"]).Dependency(*dep_args)
record.dependencies.append(dep)
record.repeats = data.get("repeats", None)
return record
project = Project("test_project",
default_executable=executable,
default_repository=repos,
default_launch_mode=serial,
data_store=datastore,
record_store=django_store.DjangoRecordStore())
parameters = SimpleParameterSet({'a': 2, 'b': 3})
for i in range(50):
record = Record(executable=executable,
repository=repos,
main_file="main.py",
version="99863a9dc5f",
launch_mode=serial,
datastore=datastore,
parameters=parameters,
input_data=[],
script_arguments="",
label="fake_record%00d" % i,
reason="testing",
diff='',
user='******',
on_changed='store-diff',
stdout_stderr='srgvrgvsgverhcser')
record.duration = random.gammavariate(1.0, 1000.0)
record.outcome = "lghsvdghsg zskjdcghnskdjgc ckdjshcgndsg"
record.data_key = "['output.data']"
record.dependencies = []
record.platforms = serial.get_platform_information()
project.add_record(record)