def test_with_single_input_file(self): data_content = b"0.0 242\n0.1 2345\n0.2 42451\n" with open("this.is.not.a.parameter.file", 'wb') as f: f.write(data_content) commands.run( ["this.is.not.a.parameter.file"] ) # file exists but is not a parameter file so is treated as input data self.assertEqual( self.prj.launch_args, { 'executable': 'default', 'parameters': {}, 'main_file': 'default', 'label': None, 'input_data': [ datastore.DataKey('this.is.not.a.parameter.file', hashlib.sha1(data_content).hexdigest(), back_to_the_future) ], 'reason': '', 'version': 'current', 'script_args': 'this.is.not.a.parameter.file' }) os.remove("this.is.not.a.parameter.file")
def test_with_everything(self): f = open("test.param", 'w') f.write("a = 2\nb = 3\n") f.close() data_content = b"23496857243968b24cbc4275dc82470a\n" f = open("this.is.not.a.parameter.file", 'wb') f.write(data_content) f.close() commands.run([ "-l", "vikings", "-v", "234", "--reason='test'", "-e", "python", "--main=main.py", "spam", "test.param", "eggs", "this.is.not.a.parameter.file", "a=17", "umlue=43", "beans" ]) self.assertEqual( self.prj.launch_args, { 'executable': MockExecutable('python'), 'parameters': { 'this': 'mock', 'a': 17, 'umlue': 43 }, 'main_file': 'main.py', 'label': 'vikings', 'input_data': [ datastore.DataKey('this.is.not.a.parameter.file', hashlib.sha1(data_content).hexdigest()) ], 'reason': 'test', 'version': '234', 'script_args': "spam <parameters> eggs this.is.not.a.parameter.file beans" }) os.remove("this.is.not.a.parameter.file") os.remove("test.param")
def to_sumatra(self): metadata = self.get_metadata() return datastore.DataKey(self.path, self.digest, self.creation, **metadata)
def generate_keys(self, *paths): return [datastore.DataKey(path, datastore.IGNORE_DIGEST, back_to_the_future) for path in paths]
def build_record(data): """Create a Sumatra record from a nested dictionary.""" edata = data["executable"] cls = get_registered_components(programs.Executable).get(edata["name"], programs.Executable) executable = cls(edata["path"], edata["version"], edata.get("options", "")) executable.name = edata["name"] rdata = data["repository"] repos_cls = None for m in versioncontrol.vcs_list: if hasattr(m, rdata["type"]): repos_cls = getattr(m, rdata["type"]) break if repos_cls is None: repos_cls = versioncontrol.base.Repository repository = repos_cls(rdata["url"]) repository.upstream = rdata.get("upstream", None) pdata = data["parameters"] if pdata["type"] == "dict": parameter_set = eval(pdata["content"]) assert isinstance(parameter_set, dict) else: parameter_set = getattr(parameters, pdata["type"])(pdata["content"]) ldata = data["launch_mode"] lm_parameters = ldata["parameters"] if isinstance(lm_parameters, str): # prior to 0.3 lm_parameters = eval(lm_parameters) launch_mode = getattr(launch, ldata["type"])(**keys2str(lm_parameters)) def build_data_store(ddata): ds_parameters = ddata["parameters"] if isinstance(ds_parameters, str): # prior to 0.3 ds_parameters = eval(ds_parameters) return getattr(datastore, ddata["type"])(**keys2str(ds_parameters)) data_store = build_data_store(data["datastore"]) if "input_datastore" in data: # 0.4 onwards input_datastore = build_data_store(data["input_datastore"]) else: input_datastore = datastore.FileSystemDataStore("/") input_data = data.get("input_data", []) if isinstance(input_data, str): # 0.3 input_data = eval(input_data) if input_data: if isinstance(input_data[0], str): # versions prior to 0.4 input_data = [datastore.DataKey(path, digest=datastore.IGNORE_DIGEST, creation=None) for path in input_data] else: input_data = [datastore.DataKey(keydata["path"], keydata["digest"], creation=datestring_to_datetime(keydata.get("creation", None)), **keys2str(keydata["metadata"])) for keydata in input_data] record = Record(executable, repository, data["main_file"], data["version"], launch_mode, data_store, parameter_set, input_data, data.get("script_arguments", ""), data["label"], data["reason"], data["diff"], data.get("user", ""), input_datastore=input_datastore, timestamp=datestring_to_datetime(data["timestamp"])) tags = data["tags"] if not hasattr(tags, "__iter__"): tags = (tags,) record.tags = set(tags) record.output_data = [] if "output_data" in data: for keydata in data["output_data"]: data_key = datastore.DataKey(keydata["path"], keydata["digest"], creation=datestring_to_datetime(keydata.get("creation", None)), **keys2str(keydata["metadata"])) record.output_data.append(data_key) elif "data_key" in data: # (versions prior to 0.4) for path in eval(data["data_key"]): data_key = datastore.DataKey(path, digest=datastore.IGNORE_DIGEST, creation=None) record.output_data.append(data_key) record.duration = data["duration"] record.outcome = data["outcome"] record.stdout_stderr = data.get("stdout_stderr", "") record.platforms = [launch.PlatformInformation(**keys2str(pldata)) for pldata in data["platforms"]] record.dependencies = [] for depdata in data["dependencies"]: dep_args = [depdata["name"], depdata["path"], depdata["version"], depdata["diff"]] if "source" in depdata: # 0.5 onwards dep_args.append(depdata["source"]) dep = getattr(dependency_finder, depdata["module"]).Dependency(*dep_args) record.dependencies.append(dep) record.repeats = data.get("repeats", None) return record
def to_sumatra(self): metadata = eval(self.metadata) return datastore.DataKey(self.path, self.digest, **metadata)
def generate_keys(self, *paths): return [ datastore.DataKey(path, datastore.IGNORE_DIGEST) for path in paths ]