def test_with_single_input_file(self):
data_content = b"0.0 242\n0.1 2345\n0.2 42451\n"
with open("this.is.not.a.parameter.file", 'wb') as f:
f.write(data_content)
commands.run(
["this.is.not.a.parameter.file"]
) # file exists but is not a parameter file so is treated as input data
self.assertEqual(
self.prj.launch_args, {
'executable':
'default',
'parameters': {},
'main_file':
'default',
'label':
None,
'input_data': [
datastore.DataKey('this.is.not.a.parameter.file',
hashlib.sha1(data_content).hexdigest(),
back_to_the_future)
],
'reason':
'',
'version':
'current',
'script_args':
'this.is.not.a.parameter.file'
})
os.remove("this.is.not.a.parameter.file")
def test_with_everything(self):
f = open("test.param", 'w')
f.write("a = 2\nb = 3\n")
f.close()
data_content = b"23496857243968b24cbc4275dc82470a\n"
f = open("this.is.not.a.parameter.file", 'wb')
f.write(data_content)
f.close()
commands.run([
"-l", "vikings", "-v", "234", "--reason='test'", "-e", "python",
"--main=main.py", "spam", "test.param", "eggs",
"this.is.not.a.parameter.file", "a=17", "umlue=43", "beans"
])
self.assertEqual(
self.prj.launch_args, {
'executable':
MockExecutable('python'),
'parameters': {
'this': 'mock',
'a': 17,
'umlue': 43
},
'main_file':
'main.py',
'label':
'vikings',
'input_data': [
datastore.DataKey('this.is.not.a.parameter.file',
hashlib.sha1(data_content).hexdigest())
],
'reason':
'test',
'version':
'234',
'script_args':
"spam <parameters> eggs this.is.not.a.parameter.file beans"
})
os.remove("this.is.not.a.parameter.file")
os.remove("test.param")
def to_sumatra(self):
metadata = self.get_metadata()
return datastore.DataKey(self.path, self.digest, self.creation,
**metadata)
def generate_keys(self, *paths):
return [datastore.DataKey(path, datastore.IGNORE_DIGEST, back_to_the_future) for path in paths]
def build_record(data):
"""Create a Sumatra record from a nested dictionary."""
edata = data["executable"]
cls = get_registered_components(programs.Executable).get(edata["name"], programs.Executable)
executable = cls(edata["path"], edata["version"], edata.get("options", ""))
executable.name = edata["name"]
rdata = data["repository"]
repos_cls = None
for m in versioncontrol.vcs_list:
if hasattr(m, rdata["type"]):
repos_cls = getattr(m, rdata["type"])
break
if repos_cls is None:
repos_cls = versioncontrol.base.Repository
repository = repos_cls(rdata["url"])
repository.upstream = rdata.get("upstream", None)
pdata = data["parameters"]
if pdata["type"] == "dict":
parameter_set = eval(pdata["content"])
assert isinstance(parameter_set, dict)
else:
parameter_set = getattr(parameters, pdata["type"])(pdata["content"])
ldata = data["launch_mode"]
lm_parameters = ldata["parameters"]
if isinstance(lm_parameters, str): # prior to 0.3
lm_parameters = eval(lm_parameters)
launch_mode = getattr(launch, ldata["type"])(**keys2str(lm_parameters))
def build_data_store(ddata):
ds_parameters = ddata["parameters"]
if isinstance(ds_parameters, str): # prior to 0.3
ds_parameters = eval(ds_parameters)
return getattr(datastore, ddata["type"])(**keys2str(ds_parameters))
data_store = build_data_store(data["datastore"])
if "input_datastore" in data: # 0.4 onwards
input_datastore = build_data_store(data["input_datastore"])
else:
input_datastore = datastore.FileSystemDataStore("/")
input_data = data.get("input_data", [])
if isinstance(input_data, str): # 0.3
input_data = eval(input_data)
if input_data:
if isinstance(input_data[0], str): # versions prior to 0.4
input_data = [datastore.DataKey(path, digest=datastore.IGNORE_DIGEST, creation=None)
for path in input_data]
else:
input_data = [datastore.DataKey(keydata["path"], keydata["digest"],
creation=datestring_to_datetime(keydata.get("creation", None)),
**keys2str(keydata["metadata"]))
for keydata in input_data]
record = Record(executable, repository, data["main_file"],
data["version"], launch_mode, data_store, parameter_set,
input_data, data.get("script_arguments", ""),
data["label"], data["reason"], data["diff"],
data.get("user", ""), input_datastore=input_datastore,
timestamp=datestring_to_datetime(data["timestamp"]))
tags = data["tags"]
if not hasattr(tags, "__iter__"):
tags = (tags,)
record.tags = set(tags)
record.output_data = []
if "output_data" in data:
for keydata in data["output_data"]:
data_key = datastore.DataKey(keydata["path"], keydata["digest"],
creation=datestring_to_datetime(keydata.get("creation", None)),
**keys2str(keydata["metadata"]))
record.output_data.append(data_key)
elif "data_key" in data: # (versions prior to 0.4)
for path in eval(data["data_key"]):
data_key = datastore.DataKey(path, digest=datastore.IGNORE_DIGEST,
creation=None)
record.output_data.append(data_key)
record.duration = data["duration"]
record.outcome = data["outcome"]
record.stdout_stderr = data.get("stdout_stderr", "")
record.platforms = [launch.PlatformInformation(**keys2str(pldata)) for pldata in data["platforms"]]
record.dependencies = []
for depdata in data["dependencies"]:
dep_args = [depdata["name"], depdata["path"], depdata["version"],
depdata["diff"]]
if "source" in depdata: # 0.5 onwards
dep_args.append(depdata["source"])
dep = getattr(dependency_finder, depdata["module"]).Dependency(*dep_args)
record.dependencies.append(dep)
record.repeats = data.get("repeats", None)
return record
def to_sumatra(self):
metadata = eval(self.metadata)
return datastore.DataKey(self.path, self.digest, **metadata)
def generate_keys(self, *paths):
return [
datastore.DataKey(path, datastore.IGNORE_DIGEST) for path in paths
]
