def test_launch(self): self.write_test_script("test.py") proj = Project("test_project", default_executable=MockExecutable(), default_repository=MockRepository(), default_launch_mode=MockLaunchMode(), record_store=MockRecordStore()) proj.launch(main_file="test.py")
def test__backup(self): def fake_copytree(source, target): pass orig_copytree = shutil.copytree shutil.copytree = fake_copytree proj = Project("test_project", record_store=MockRecordStore()) backup_dir = proj.backup() shutil.copytree = orig_copytree assert "backup" in backup_dir
def test__repeat(self): orig_gwc = sumatra.projects.get_working_copy sumatra.projects.get_working_copy = lambda: MockWorkingCopy(self.dir) orig_launch = Project.launch Project.launch = lambda self, **kwargs: "new_record" proj = Project("test_project", record_store=MockRecordStore()) proj.add_record(MockRecord("record1")) proj.add_record(MockRecord("record2")) self.assertEqual(proj.repeat("record1")[0], "new_record") sumatra.projects.get_working_copy = orig_gwc Project.launch = orig_launch
def test_new_record_with_minimal_args_should_set_defaults(self): self.write_test_script("test.py") proj = Project("test_project", record_store=MockRecordStore(), default_main_file="test.py", default_executable=MockExecutable(), default_launch_mode=MockLaunchMode(), default_repository=MockRepository()) rec = proj.new_record() self.assertEqual(rec.repository, proj.default_repository) self.assertEqual(rec.main_file, "test.py")
def test__repeat(self): orig_gwc = sumatra.projects.get_working_copy sumatra.projects.get_working_copy = MockWorkingCopy orig_launch = Project.launch Project.launch = lambda self, **kwargs: "new_record" proj = Project("test_project", record_store=MockRecordStore()) proj.add_record(MockRecord("record1")) proj.add_record(MockRecord("record2")) self.assertEqual(proj.repeat("record1")[0], "new_record") sumatra.projects.get_working_copy = orig_gwc Project.launch = orig_launch
def test_new_record_with_uuid_label_generator_should_generate_unique_id(self): self.write_test_script("test.py") proj = Project("test_project", record_store=MockRecordStore(), default_main_file="test.py", default_executable=MockExecutable(), default_launch_mode=MockLaunchMode(), default_repository=MockRepository(), label_generator='uuid') rec1 = proj.new_record() rec2 = proj.new_record() self.assertNotEqual(rec1.label, rec2.label)
def delete_record__should_update_most_recent(self): """see ticket:36.""" proj = Project("test_project", record_store=MockRecordStore()) proj.add_record(MockRecord("record1")) self.assertEqual(proj._most_recent, "record1") proj.add_record(MockRecord("record2")) self.assertEqual(proj._most_recent, "record2") proj.delete_record("record2") self.assertEqual(proj._most_recent, "last") # should really be "record1", but we are not testing RecordStore here
def test_format_records(self): self.write_test_script("test.py") proj = Project("test_project", record_store=MockRecordStore(), default_main_file="test.py", default_executable=MockExecutable(), default_launch_mode=MockLaunchMode(), default_repository=MockRepository(), label_generator='uuid') rec1 = proj.new_record() rec2 = proj.new_record() self.assertEqual(proj.format_records('text'), 'foo_labelfoo_label\nbar_labelbar_label') self.assertEqual(proj.format_records('html'), '<ul>\n<li>foo_labelfoo_label</li>\n<li>bar_labelbar_label</li>\n</ul>') # TODO: Find a good way to check the output of the following formatters # (currently we only check that we can call them without errors). proj.format_records('latex', 'long') proj.format_records('shell') proj.format_records('json')
def test__load_project__should_return_Project(self): proj1 = Project("test_project", record_store=MockRecordStore()) assert os.path.exists(".smt/project") proj2 = load_project() self.assertEqual(proj1.name, proj2.name)
def test__update_code(self): proj = Project("test_project", record_store=MockRecordStore(), default_repository=MockRepository()) wc = proj.default_repository.get_working_copy() proj.update_code(wc, version=9369835)
from sumatra.records import Record from sumatra.recordstore import django_store from sumatra.programs import PythonExecutable from sumatra.launch import SerialLaunchMode from sumatra.datastore import FileSystemDataStore from sumatra.parameters import SimpleParameterSet from sumatra.versioncontrol._git import GitRepository import random serial = SerialLaunchMode() executable = PythonExecutable("/usr/bin/python", version="2.7") repos = GitRepository('.') datastore = FileSystemDataStore("/path/to/datastore") project = Project("test_project", default_executable=executable, default_repository=repos, default_launch_mode=serial, data_store=datastore, record_store=django_store.DjangoRecordStore()) parameters = SimpleParameterSet({'a': 2, 'b': 3}) for i in range(50): record = Record(executable=executable, repository=repos, main_file="main.py", version="99863a9dc5f", launch_mode=serial, datastore=datastore, parameters=parameters, input_data=[], script_arguments="", label="fake_record%00d" % i,
def test__creating_a_second_project_in_the_same_dir_should_raise_an_exception( self): Project("test_project1", record_store=MockRecordStore()) self.assertRaises(Exception, Project, "test_project2")
def test__get_record__calls_get_on_the_record_store(self): proj = Project("test_project", record_store=MockRecordStore()) self.assertEqual(proj.get_record("foo").label, "foofoo")
def test__info(self): proj = Project("test_project", record_store=MockRecordStore()) proj.info()
def test__remove_tag__should_call_remove_on_the_tags_attibute_of_the_record(self): proj = Project("test_project", record_store=MockRecordStore()) proj.remove_tag("foo", "new_tag")
def test__delete_record__calls_delete_on_the_record_store(self): proj = Project("test_project", record_store=MockRecordStore()) proj.delete_record("foo") self.assertEqual(proj.record_store.deleted, "foo")
def test__delete_by_tag__calls_delete_by_tag_on_the_record_store(self): proj = Project("test_project", record_store=MockRecordStore()) self.assertEqual(proj.delete_by_tag("foo"), "oof")
def sumatra_start(repository, sumatra_db_path, results_path, working_dir, hg_username, sumatra_run_name, parameters): '''Clones the Omics Pipe repository from Bitbucket, creates a Sumatra project, and creates a Sumatra record for the current run''' print "sumatra_db_path is " + sumatra_db_path print type(sumatra_db_path) check_create_dir(sumatra_db_path) os.chdir(sumatra_db_path) repo1 = hgapi.Repo(repository) repo_path = sumatra_db_path + "/omics_pipe" repo = { "url": repo_path, "type": "sumatra.versioncontrol._mercurial.MercurialRepository", "upstream": repository } executable = { "path": "", "version": "", "type": "sumatra.programs.PythonExecutable", "options": "", "name": "Python" } sumatra_launch_mode = { "working_directory": working_dir, "type": "sumatra.launch.SerialLaunchMode" } data_store1 = { "root": results_path, "type": "sumatra.datastore.filesystem.FileSystemDataStore" } database_path = sumatra_db_path + "/records/recordstore.db" record_store1 = { "db_file": database_path, "type": "sumatra.recordstore.django_store.DjangoRecordStore" } input_datastore1 = { "root": results_path, "type": "sumatra.datastore.filesystem.FileSystemDataStore" } while True: try: repo1.hg_clone(url=repository, path=repo_path) with open(repo_path + "/.hg/hgrc", "a") as myfile: myfile.write("[ui]\nusername= "******"Omics pipe repository cloned to : " + repo_path break except hgapi.hgapi.HgException: print "Omics pipe repository already exists." break while True: try: Project(sumatra_run_name, default_repository=repo, default_executable=executable, default_launch_mode=sumatra_launch_mode, on_changed='store-diff', data_store=data_store1, record_store=record_store1, input_datastore=input_datastore1) print "Sumatra project created: " + sumatra_run_name + " in directory: " + sumatra_db_path break except Exception: print "Sumatra project already exists, loading project: " + sumatra_run_name break project = load_project(path=sumatra_db_path) print project sumatra_params = build_parameters(parameters) print sumatra_params os.chdir(repo_path) repo_main = "omics_pipe/main.py" record = project.new_record(parameters=sumatra_params, main_file=repo_main) print record return record, project
def test__show_diff(self): proj = Project("test_project", record_store=MockRecordStore()) proj.show_diff("foo", "bar")
def test__remove_tag__should_call_remove_on_the_tags_attibute_of_the_record( self): proj = Project("test_project", record_store=MockRecordStore()) proj.remove_tag("foo", "new_tag")
def test__add_comment__should_set_the_outcome_attribute_of_the_record( self): proj = Project("test_project", record_store=MockRecordStore()) proj.add_comment("foo", "comment goes here")
def test__init__with_minimal_arguments(self): Project("test_project", record_store=MockRecordStore())
def test__add_comment__should_set_the_outcome_attribute_of_the_record(self): proj = Project("test_project", record_store=MockRecordStore()) proj.add_comment("foo", "comment goes here")
def init(argv): """Create a new project in the current directory.""" usage = "%(prog)s init [options] NAME" description = "Create a new project called NAME in the current directory." parser = ArgumentParser(usage=usage, description=description) parser.add_argument('project_name', metavar='NAME', help="a short name for the project; should not contain spaces.") parser.add_argument('-d', '--datapath', metavar='PATH', default='./Data', help="set the path to the directory in which smt will search for output datafiles generated by the simulation/analysis. Defaults to %(default)s.") parser.add_argument('-i', '--input', metavar='PATH', default='/', help="set the path to the directory relative to which input datafile paths will be given. Defaults to the filesystem root.") parser.add_argument('-l', '--addlabel', choices=['cmdline', 'parameters', None], metavar='OPTION', default=None, help="If this option is set, smt will append the record label either to the command line (option 'cmdline') or to the parameter file (option 'parameters'), and will add the label to the datapath when searching for datafiles. It is up to the user to make use of this label inside their program to ensure files are created in the appropriate location.") parser.add_argument('-e', '--executable', metavar='PATH', help="set the path to the executable. If this is not set, smt will try to infer the executable from the value of the --main option, if supplied, and will try to find the executable from the PATH environment variable, then by searching various likely locations on the filesystem.") parser.add_argument('-r', '--repository', help="the URL of a Subversion or Mercurial repository containing the code. This will be checked out/cloned into the current directory.") parser.add_argument('-m', '--main', help="the name of the script that would be supplied on the command line if running the simulation or analysis normally, e.g. init.hoc.") parser.add_argument('-c', '--on-changed', default='error', help="the action to take if the code in the repository or any of the depdendencies has changed. Defaults to %(default)s") # need to add list of allowed values parser.add_argument('-s', '--store', help="Specify the path, URL or URI to the record store (must be specified). This can either be an existing record store or one to be created. {0} Not using the `--store` argument defaults to a DjangoRecordStore with Sqlite in `.smt/records`".format(store_arg_help)) parser.add_argument('-g', '--labelgenerator', choices=['timestamp', 'uuid'], default='timestamp', metavar='OPTION', help="specify which method Sumatra should use to generate labels (options: timestamp, uuid)") parser.add_argument('-t', '--timestamp_format', help="the timestamp format given to strftime", default=TIMESTAMP_FORMAT) parser.add_argument('-L', '--launch_mode', choices=['serial', 'distributed', 'slurm-mpi'], default='serial', help="how computations should be launched. Defaults to %(default)s") parser.add_argument('-o', '--launch_mode_options', help="extra options for the given launch mode") datastore = parser.add_mutually_exclusive_group() datastore.add_argument('-W', '--webdav', metavar='URL', help="specify a webdav URL (with username@password: if needed) as the archiving location for data") datastore.add_argument('-A', '--archive', metavar='PATH', help="specify a directory in which to archive output datafiles. If not specified, or if 'false', datafiles are not archived.") datastore.add_argument('-M', '--mirror', metavar='URL', help="specify a URL at which your datafiles will be mirrored.") args = parser.parse_args(argv) try: project = load_project() parser.error("A project already exists in directory '{0}'.".format(project.path)) except Exception: pass if not os.path.exists(".smt"): os.mkdir(".smt") if args.repository: repository = get_repository(args.repository) repository.checkout() else: repository = get_working_copy().repository # if no repository is specified, we assume there is a working copy in the current directory. if args.executable: executable_path, executable_options = parse_executable_str(args.executable) executable = get_executable(path=executable_path) executable.args = executable_options elif args.main: try: executable = get_executable(script_file=args.main) except Exception: # assume unrecognized extension - really need more specific exception type # should warn that extension unrecognized executable = None else: executable = None if args.store: record_store = get_record_store(args.store) else: record_store = 'default' if args.webdav: # should we care about archive migration?? output_datastore = get_data_store("DavFsDataStore", {"root": args.datapath, "dav_url": args.webdav}) args.archive = '.smt/archive' elif args.archive and args.archive.lower() != 'false': if args.archive.lower() == "true": args.archive = ".smt/archive" args.archive = os.path.abspath(args.archive) output_datastore = get_data_store("ArchivingFileSystemDataStore", {"root": args.datapath, "archive": args.archive}) elif args.mirror: output_datastore = get_data_store("MirroredFileSystemDataStore", {"root": args.datapath, "mirror_base_url": args.mirror}) else: output_datastore = get_data_store("FileSystemDataStore", {"root": args.datapath}) input_datastore = get_data_store("FileSystemDataStore", {"root": args.input}) if args.launch_mode_options: args.launch_mode_options = args.launch_mode_options.strip() launch_mode = get_launch_mode(args.launch_mode)(options=args.launch_mode_options) project = Project(name=args.project_name, default_executable=executable, default_repository=repository, default_main_file=args.main, # what if incompatible with executable? default_launch_mode=launch_mode, data_store=output_datastore, record_store=record_store, on_changed=args.on_changed, data_label=args.addlabel, input_datastore=input_datastore, label_generator=args.labelgenerator, timestamp_format=args.timestamp_format) if os.path.exists('.smt') and project.record_store.has_project(project.name): f = open('.smt/labels', 'w') f.writelines(project.format_records(tags=None, mode='short', format='text', reverse=False)) f.close() project.save()
from sumatra.recordstore import django_store from sumatra.programs import PythonExecutable from sumatra.launch import SerialLaunchMode from sumatra.datastore import FileSystemDataStore from sumatra.parameters import SimpleParameterSet from sumatra.versioncontrol._git import GitRepository import random from datetime import datetime serial = SerialLaunchMode() executable = PythonExecutable("/usr/bin/python", version="2.7") repos = GitRepository('.') datastore = FileSystemDataStore("/path/to/datastore") project = Project("test_project", default_executable=executable, default_repository=repos, default_launch_mode=serial, data_store=datastore, record_store=django_store.DjangoRecordStore()) parameters = SimpleParameterSet({'a': 2, 'b': 3}) for i in range(50): record = Record(executable=executable, repository=repos, main_file="main.py", version="99863a9dc5f", launch_mode=serial, datastore=datastore, parameters=parameters, input_data=[], script_arguments="", label="fake_record%00d" % i, reason="testing", diff='', user='******', on_changed='store-diff', stdout_stderr='srgvrgvsgverhcser') record.duration = random.gammavariate(1.0, 1000.0) record.outcome = "lghsvdghsg zskjdcghnskdjgc ckdjshcgndsg" record.data_key = "['output.data']"
def init(argv): """Create a new project in the current directory.""" usage = "%(prog)s init [options] NAME" description = "Create a new project called NAME in the current directory." parser = ArgumentParser(usage=usage, description=description) parser.add_argument('project_name', metavar='NAME', help="a short name for the project; should not contain spaces.") parser.add_argument('-d', '--datapath', metavar='PATH', default='./Data', help="set the path to the directory in which smt will search for output datafiles generated by the simulation/analysis. Defaults to %(default)s.") parser.add_argument('-i', '--input', metavar='PATH', default='/', help="set the path to the directory relative to which input datafile paths will be given. Defaults to the filesystem root.") parser.add_argument('-l', '--addlabel', choices=['cmdline', 'parameters', None], metavar='OPTION', default=None, help="If this option is set, smt will append the record label either to the command line (option 'cmdline') or to the parameter file (option 'parameters'), and will add the label to the datapath when searching for datafiles. It is up to the user to make use of this label inside their program to ensure files are created in the appropriate location.") parser.add_argument('-e', '--executable', metavar='PATH', help="set the path to the executable. If this is not set, smt will try to infer the executable from the value of the --main option, if supplied, and will try to find the executable from the PATH environment variable, then by searching various likely locations on the filesystem.") parser.add_argument('-r', '--repository', help="the URL of a Subversion or Mercurial repository containing the code. This will be checked out/cloned into the current directory.") parser.add_argument('-m', '--main', help="the name of the script that would be supplied on the command line if running the simulation or analysis normally, e.g. init.hoc.") parser.add_argument('-c', '--on-changed', default='error', help="the action to take if the code in the repository or any of the depdendencies has changed. Defaults to %(default)s") # need to add list of allowed values parser.add_argument('-s', '--store', help="Specify the path, URL or URI to the record store (must be specified). This can either be an existing record store or one to be created. {0} Not using the `--store` argument defaults to a DjangoRecordStore with Sqlite in `.smt/records`".format(store_arg_help)) parser.add_argument('-g', '--labelgenerator', choices=['timestamp', 'uuid'], default='timestamp', metavar='OPTION', help="specify which method Sumatra should use to generate labels (options: timestamp, uuid)") parser.add_argument('-t', '--timestamp_format', help="the timestamp format given to strftime", default=TIMESTAMP_FORMAT) parser.add_argument('-L', '--launch_mode', choices=['serial', 'distributed', 'slurm-mpi'], default='serial', help="how computations should be launched. Defaults to %(default)s") parser.add_argument('-o', '--launch_mode_options', help="extra options for the given launch mode") datastore = parser.add_mutually_exclusive_group() datastore.add_argument('-W', '--webdav', metavar='URL', help="specify a webdav URL (with username@password: if needed) as the archiving location for data") datastore.add_argument('-A', '--archive', metavar='PATH', help="specify a directory in which to archive output datafiles. If not specified, or if 'false', datafiles are not archived.") datastore.add_argument('-M', '--mirror', metavar='URL', help="specify a URL at which your datafiles will be mirrored.") args = parser.parse_args(argv) try: project = load_project() parser.error("A project already exists in this directory.") except Exception: pass if not os.path.exists(".smt"): os.mkdir(".smt") if args.repository: repository = get_repository(args.repository) repository.checkout() else: repository = get_working_copy().repository # if no repository is specified, we assume there is a working copy in the current directory. if args.executable: executable_path, executable_options = parse_executable_str(args.executable) executable = get_executable(path=executable_path) executable.args = executable_options elif args.main: try: executable = get_executable(script_file=args.main) except Exception: # assume unrecognized extension - really need more specific exception type # should warn that extension unrecognized executable = None else: executable = None if args.store: record_store = get_record_store(args.store) else: record_store = 'default' if args.webdav: # should we care about archive migration?? output_datastore = get_data_store("DavFsDataStore", {"root": args.datapath, "dav_url": args.webdav}) args.archive = '.smt/archive' elif args.archive and args.archive.lower() != 'false': if args.archive.lower() == "true": args.archive = ".smt/archive" args.archive = os.path.abspath(args.archive) output_datastore = get_data_store("ArchivingFileSystemDataStore", {"root": args.datapath, "archive": args.archive}) elif args.mirror: output_datastore = get_data_store("MirroredFileSystemDataStore", {"root": args.datapath, "mirror_base_url": args.mirror}) else: output_datastore = get_data_store("FileSystemDataStore", {"root": args.datapath}) input_datastore = get_data_store("FileSystemDataStore", {"root": args.input}) if args.launch_mode_options: args.launch_mode_options = args.launch_mode_options.strip() launch_mode = get_launch_mode(args.launch_mode)(options=args.launch_mode_options) project = Project(name=args.project_name, default_executable=executable, default_repository=repository, default_main_file=args.main, # what if incompatible with executable? default_launch_mode=launch_mode, data_store=output_datastore, record_store=record_store, on_changed=args.on_changed, data_label=args.addlabel, input_datastore=input_datastore, label_generator=args.labelgenerator, timestamp_format=args.timestamp_format) project.save()