Beispiel #1
0
 def test__repr(self):
     r1 = Record(
         MockExecutable("1"),
         MockRepository(),
         "test.py",
         999,
         MockLaunchMode(),
         MockDataStore(),
         {"a": 3},
         label="A",
     )
     time.sleep(1)
     r2 = Record(
         MockExecutable("2"),
         MockRepository(),
         "test.py",
         999,
         MockLaunchMode(),
         MockDataStore(),
         {"a": 2},
         label="B",
     )
     r1.dependencies = [MockDependency("foo"), MockDependency("bar")]
     r2.dependencies = [MockDependency("bar"), MockDependency("eric")]
     diff = RecordDifference(r1, r2)
     self.assertEqual(repr(diff), "RecordDifference(A, B):XCP")
Beispiel #2
0
 def test__dependency_differences(self):
     r1 = Record(MockExecutable(), MockRepository(), "test.py", 999, MockLaunchMode(), MockDataStore())
     time.sleep(1)
     r2 = Record(MockExecutable(), MockRepository(), "test.py", 999, MockLaunchMode(), MockDataStore())
     r1.dependencies = [MockDependency("foo"), MockDependency("bar")]
     r2.dependencies = [MockDependency("bar"), MockDependency("eric")]
     diff = RecordDifference(r1, r2)
     diff.dependency_differences
Beispiel #3
0
 def test__dependency_differences(self):
     r1 = Record(MockExecutable(), MockRepository(), "test.py", 999,
                 MockLaunchMode(), MockDataStore())
     time.sleep(1)
     r2 = Record(MockExecutable(), MockRepository(), "test.py", 999,
                 MockLaunchMode(), MockDataStore())
     r1.dependencies = [MockDependency("foo"), MockDependency("bar")]
     r2.dependencies = [MockDependency("bar"), MockDependency("eric")]
     diff = RecordDifference(r1, r2)
     diff.dependency_differences
Beispiel #4
0
 def test__repr(self):
     r1 = Record(MockExecutable("1"), MockRepository(), "test.py",
                 999, MockLaunchMode(), MockDataStore(), {"a": 3}, label="A")
     time.sleep(1)
     r2 = Record(MockExecutable("2"), MockRepository(), "test.py",
                 999, MockLaunchMode(), MockDataStore(), {"a": 2}, label="B")
     r1.dependencies = [MockDependency("foo"), MockDependency("bar")]
     r2.dependencies = [MockDependency("bar"), MockDependency("eric")]
     diff = RecordDifference(r1, r2)
     self.assertEqual(repr(diff), "RecordDifference(A, B):XCP")
Beispiel #5
0
 def test__nonzero__should_return_True__for_identical_parameters(self):
     r1 = Record(MockExecutable(), MockRepository(), "test.py", 999, MockLaunchMode(), MockDataStore(), {"a": 2})
     time.sleep(1)
     r2 = Record(MockExecutable(), MockRepository(), "test.py", 999, MockLaunchMode(), MockDataStore(), {"a": 2})
     r1.dependencies = []
     r2.dependencies = []
     diff = RecordDifference(r1, r2)
     self.assertEqual(diff.executable_differs, False)
     self.assertEqual(diff.code_differs, False)
     self.assertEqual(diff.parameters_differ, False)
     self.assertEqual(diff.output_data_differ, False)
     self.assertEqual(diff.__bool__(), False)
Beispiel #6
0
 def test__nonzero__should_return_True__for_identical_parameters(self):
     r1 = Record(MockExecutable(), MockRepository(), "test.py", 999,
                 MockLaunchMode(), MockDataStore(), {"a": 2})
     time.sleep(1)
     r2 = Record(MockExecutable(), MockRepository(), "test.py", 999,
                 MockLaunchMode(), MockDataStore(), {"a": 2})
     r1.dependencies = []
     r2.dependencies = []
     diff = RecordDifference(r1, r2)
     self.assertEqual(diff.executable_differs, False)
     self.assertEqual(diff.code_differs, False)
     self.assertEqual(diff.parameters_differ, False)
     self.assertEqual(diff.output_data_differ, False)
     self.assertEqual(diff.__bool__(), False)
Beispiel #7
0
def decode_record(content):
    data = json.loads(content)
    edata = data["executable"]
    cls = programs.registered_program_names.get(edata["name"], programs.Executable)
    executable = cls(edata["path"], edata["version"], edata["options"])
    executable.name = edata["name"]
    rdata = data["repository"]
    repos_cls = None
    for m in versioncontrol.vcs_list:
        if hasattr(m, rdata["type"]):
            repos_cls = getattr(m, rdata["type"])
            break
    if repos_cls is None:  
        repos_cls = versioncontrol.base.Repository
    repository = repos_cls(rdata["url"])
    pdata = data["parameters"]
    if pdata["type"] == "dict":
        parameter_set = eval(pdata["content"])
        assert isinstance(parameter_set, dict)
    elif pdata['type']=="NoneType":
        parameter_set = None
    else:
        parameter_set = getattr(parameters, pdata["type"])(pdata["content"])
    ldata = data["launch_mode"]
    lm_parameters = eval(ldata["parameters"])
    launch_mode = getattr(launch, ldata["type"])(**lm_parameters)
    ddata = data["datastore"]
    ds_parameters = eval(ddata["parameters"])
    data_store = getattr(datastore, ddata["type"])(**ds_parameters)
    record = Record(executable, repository, data["main_file"],
                       data["version"], launch_mode, data_store, parameter_set,
                       data["input_data"], data["script_arguments"], 
                       data["label"], data["reason"], data["diff"],
                       data["user"])
    tags = data["tags"]
    if not hasattr(tags, "__iter__"):
        tags = (tags,)
    record.tags = set(tags)
    record.timestamp = datetime.strptime(data["timestamp"], "%Y-%m-%d %H:%M:%S")
    record.data_key = data["data_key"]
    record.duration = data["duration"]
    record.outcome = data["outcome"]
    record.platforms = [launch.PlatformInformation(**keys2str(pldata)) for pldata in data["platforms"]]
    record.dependencies = []
    for depdata in data["dependencies"]:
        dep = getattr(dependency_finder, depdata["module"]).Dependency(depdata["name"], depdata["path"], depdata["version"])
        dep.diff = depdata["diff"]
        record.dependencies.append(dep)
    return record
Beispiel #8
0
from sumatra.versioncontrol._git import GitRepository
import random
from datetime import datetime

serial = SerialLaunchMode()
executable = PythonExecutable("/usr/bin/python", version="2.7")
repos = GitRepository('.')
datastore = FileSystemDataStore("/path/to/datastore")
project = Project("test_project",
                  default_executable=executable,
                  default_repository=repos,
                  default_launch_mode=serial,
                  data_store=datastore,
                  record_store=django_store.DjangoRecordStore())
parameters = SimpleParameterSet({'a': 2, 'b': 3})

for i in range(50):
    record = Record(executable=executable, repository=repos,
                    main_file="main.py", version="99863a9dc5f",
                    launch_mode=serial, datastore=datastore,
                    parameters=parameters, input_data=[], script_arguments="",
                    label="fake_record%00d" % i,
                    reason="testing", diff='', user='******',
                    on_changed='store-diff', stdout_stderr='srgvrgvsgverhcser')
    record.duration = random.gammavariate(1.0, 1000.0)
    record.outcome = "lghsvdghsg zskjdcghnskdjgc ckdjshcgndsg"
    record.data_key = "['output.data']"
    record.dependencies = []
    record.platforms = serial.get_platform_information()
    project.add_record(record)
Beispiel #9
0
def build_record(data):
    """Create a Sumatra record from a nested dictionary."""
    edata = data["executable"]
    cls = get_registered_components(programs.Executable).get(edata["name"], programs.Executable)
    executable = cls(edata["path"], edata["version"], edata.get("options", ""))
    executable.name = edata["name"]
    rdata = data["repository"]
    repos_cls = None
    for m in versioncontrol.vcs_list:
        if hasattr(m, rdata["type"]):
            repos_cls = getattr(m, rdata["type"])
            break
    if repos_cls is None:
        repos_cls = versioncontrol.base.Repository
    repository = repos_cls(rdata["url"])
    repository.upstream = rdata.get("upstream", None)
    pdata = data["parameters"]
    if pdata["type"] == "dict":
        parameter_set = eval(pdata["content"])
        assert isinstance(parameter_set, dict)
    else:
        parameter_set = getattr(parameters, pdata["type"])(pdata["content"])
    ldata = data["launch_mode"]
    lm_parameters = ldata["parameters"]
    if isinstance(lm_parameters, str):  # prior to 0.3
        lm_parameters = eval(lm_parameters)
    launch_mode = getattr(launch, ldata["type"])(**keys2str(lm_parameters))

    def build_data_store(ddata):
        ds_parameters = ddata["parameters"]
        if isinstance(ds_parameters, str):  # prior to 0.3
            ds_parameters = eval(ds_parameters)
        return getattr(datastore, ddata["type"])(**keys2str(ds_parameters))
    data_store = build_data_store(data["datastore"])
    if "input_datastore" in data:  # 0.4 onwards
        input_datastore = build_data_store(data["input_datastore"])
    else:
        input_datastore = datastore.FileSystemDataStore("/")
    input_data = data.get("input_data", [])
    if isinstance(input_data, str):  # 0.3
        input_data = eval(input_data)
    if input_data:
        if isinstance(input_data[0], str):  # versions prior to 0.4
            input_data = [datastore.DataKey(path, digest=datastore.IGNORE_DIGEST, creation=None)
                          for path in input_data]
        else:
            input_data = [datastore.DataKey(keydata["path"], keydata["digest"],
                                            creation=datestring_to_datetime(keydata.get("creation", None)),
                                            **keys2str(keydata["metadata"]))
                          for keydata in input_data]
    record = Record(executable, repository, data["main_file"],
                    data["version"], launch_mode, data_store, parameter_set,
                    input_data, data.get("script_arguments", ""),
                    data["label"], data["reason"], data["diff"],
                    data.get("user", ""), input_datastore=input_datastore,
                    timestamp=datestring_to_datetime(data["timestamp"]))
    tags = data["tags"]
    if not hasattr(tags, "__iter__"):
        tags = (tags,)
    record.tags = set(tags)
    record.output_data = []
    if "output_data" in data:
        for keydata in data["output_data"]:
            data_key = datastore.DataKey(keydata["path"], keydata["digest"],
                                         creation=datestring_to_datetime(keydata.get("creation", None)),
                                         **keys2str(keydata["metadata"]))
            record.output_data.append(data_key)
    elif "data_key" in data:  # (versions prior to 0.4)
        for path in eval(data["data_key"]):
            data_key = datastore.DataKey(path, digest=datastore.IGNORE_DIGEST,
                                         creation=None)
            record.output_data.append(data_key)
    record.duration = data["duration"]
    record.outcome = data["outcome"]
    record.stdout_stderr = data.get("stdout_stderr", "")
    record.platforms = [launch.PlatformInformation(**keys2str(pldata)) for pldata in data["platforms"]]
    record.dependencies = []
    for depdata in data["dependencies"]:
        dep_args = [depdata["name"], depdata["path"], depdata["version"],
                    depdata["diff"]]
        if "source" in depdata:  # 0.5 onwards
            dep_args.append(depdata["source"])
        dep = getattr(dependency_finder, depdata["module"]).Dependency(*dep_args)
        record.dependencies.append(dep)
    record.repeats = data.get("repeats", None)
    return record
Beispiel #10
0
project = Project("test_project",
                  default_executable=executable,
                  default_repository=repos,
                  default_launch_mode=serial,
                  data_store=datastore,
                  record_store=django_store.DjangoRecordStore())
parameters = SimpleParameterSet({'a': 2, 'b': 3})

for i in range(50):
    record = Record(executable=executable,
                    repository=repos,
                    main_file="main.py",
                    version="99863a9dc5f",
                    launch_mode=serial,
                    datastore=datastore,
                    parameters=parameters,
                    input_data=[],
                    script_arguments="",
                    label="fake_record%00d" % i,
                    reason="testing",
                    diff='',
                    user='******',
                    on_changed='store-diff',
                    stdout_stderr='srgvrgvsgverhcser')
    record.duration = random.gammavariate(1.0, 1000.0)
    record.outcome = "lghsvdghsg zskjdcghnskdjgc ckdjshcgndsg"
    record.data_key = "['output.data']"
    record.dependencies = []
    record.platforms = serial.get_platform_information()
    project.add_record(record)