def test_launch(self):
self.write_test_script("test.py")
proj = Project("test_project",
default_executable=MockExecutable(),
default_repository=MockRepository(),
default_launch_mode=MockLaunchMode(),
record_store=MockRecordStore())
proj.launch(main_file="test.py")
def test_launch(self):
self.write_test_script("test.py")
proj = Project("test_project",
default_executable=MockExecutable(),
default_repository=MockRepository(),
default_launch_mode=MockLaunchMode(),
record_store=MockRecordStore())
proj.launch(main_file="test.py")
def test__backup(self):
def fake_copytree(source, target):
pass
orig_copytree = shutil.copytree
shutil.copytree = fake_copytree
proj = Project("test_project", record_store=MockRecordStore())
backup_dir = proj.backup()
shutil.copytree = orig_copytree
assert "backup" in backup_dir
def test__backup(self):
def fake_copytree(source, target):
pass
orig_copytree = shutil.copytree
shutil.copytree = fake_copytree
proj = Project("test_project",
record_store=MockRecordStore())
backup_dir = proj.backup()
shutil.copytree = orig_copytree
assert "backup" in backup_dir
def test__repeat(self):
orig_gwc = sumatra.projects.get_working_copy
sumatra.projects.get_working_copy = lambda: MockWorkingCopy(self.dir)
orig_launch = Project.launch
Project.launch = lambda self, **kwargs: "new_record"
proj = Project("test_project", record_store=MockRecordStore())
proj.add_record(MockRecord("record1"))
proj.add_record(MockRecord("record2"))
self.assertEqual(proj.repeat("record1")[0], "new_record")
sumatra.projects.get_working_copy = orig_gwc
Project.launch = orig_launch
def test_new_record_with_minimal_args_should_set_defaults(self):
self.write_test_script("test.py")
proj = Project("test_project",
record_store=MockRecordStore(),
default_main_file="test.py",
default_executable=MockExecutable(),
default_launch_mode=MockLaunchMode(),
default_repository=MockRepository())
rec = proj.new_record()
self.assertEqual(rec.repository, proj.default_repository)
self.assertEqual(rec.main_file, "test.py")
def test_new_record_with_minimal_args_should_set_defaults(self):
self.write_test_script("test.py")
proj = Project("test_project",
record_store=MockRecordStore(),
default_main_file="test.py",
default_executable=MockExecutable(),
default_launch_mode=MockLaunchMode(),
default_repository=MockRepository())
rec = proj.new_record()
self.assertEqual(rec.repository, proj.default_repository)
self.assertEqual(rec.main_file, "test.py")
def test__repeat(self):
orig_gwc = sumatra.projects.get_working_copy
sumatra.projects.get_working_copy = MockWorkingCopy
orig_launch = Project.launch
Project.launch = lambda self, **kwargs: "new_record"
proj = Project("test_project", record_store=MockRecordStore())
proj.add_record(MockRecord("record1"))
proj.add_record(MockRecord("record2"))
self.assertEqual(proj.repeat("record1")[0], "new_record")
sumatra.projects.get_working_copy = orig_gwc
Project.launch = orig_launch
def test_new_record_with_uuid_label_generator_should_generate_unique_id(self):
self.write_test_script("test.py")
proj = Project("test_project",
record_store=MockRecordStore(),
default_main_file="test.py",
default_executable=MockExecutable(),
default_launch_mode=MockLaunchMode(),
default_repository=MockRepository(),
label_generator='uuid')
rec1 = proj.new_record()
rec2 = proj.new_record()
self.assertNotEqual(rec1.label, rec2.label)
def test_new_record_with_uuid_label_generator_should_generate_unique_id(self):
self.write_test_script("test.py")
proj = Project("test_project",
record_store=MockRecordStore(),
default_main_file="test.py",
default_executable=MockExecutable(),
default_launch_mode=MockLaunchMode(),
default_repository=MockRepository(),
label_generator='uuid')
rec1 = proj.new_record()
rec2 = proj.new_record()
self.assertNotEqual(rec1.label, rec2.label)
def delete_record__should_update_most_recent(self):
"""see ticket:36."""
proj = Project("test_project",
record_store=MockRecordStore())
proj.add_record(MockRecord("record1"))
self.assertEqual(proj._most_recent, "record1")
proj.add_record(MockRecord("record2"))
self.assertEqual(proj._most_recent, "record2")
proj.delete_record("record2")
self.assertEqual(proj._most_recent, "last") # should really be "record1", but we are not testing RecordStore here
def test_format_records(self):
self.write_test_script("test.py")
proj = Project("test_project",
record_store=MockRecordStore(),
default_main_file="test.py",
default_executable=MockExecutable(),
default_launch_mode=MockLaunchMode(),
default_repository=MockRepository(),
label_generator='uuid')
rec1 = proj.new_record()
rec2 = proj.new_record()
self.assertEqual(proj.format_records('text'), 'foo_labelfoo_label\nbar_labelbar_label')
self.assertEqual(proj.format_records('html'), '<ul>\n<li>foo_labelfoo_label</li>\n<li>bar_labelbar_label</li>\n</ul>')
# TODO: Find a good way to check the output of the following formatters
# (currently we only check that we can call them without errors).
proj.format_records('latex', 'long')
proj.format_records('shell')
proj.format_records('json')
def delete_record__should_update_most_recent(self):
"""see ticket:36."""
proj = Project("test_project",
record_store=MockRecordStore())
proj.add_record(MockRecord("record1"))
self.assertEqual(proj._most_recent, "record1")
proj.add_record(MockRecord("record2"))
self.assertEqual(proj._most_recent, "record2")
proj.delete_record("record2")
self.assertEqual(proj._most_recent, "last") # should really be "record1", but we are not testing RecordStore here
def test_format_records(self):
self.write_test_script("test.py")
proj = Project("test_project",
record_store=MockRecordStore(),
default_main_file="test.py",
default_executable=MockExecutable(),
default_launch_mode=MockLaunchMode(),
default_repository=MockRepository(),
label_generator='uuid')
rec1 = proj.new_record()
rec2 = proj.new_record()
self.assertEqual(proj.format_records('text'), 'foo_labelfoo_label\nbar_labelbar_label')
self.assertEqual(proj.format_records('html'), '<ul>\n<li>foo_labelfoo_label</li>\n<li>bar_labelbar_label</li>\n</ul>')
# TODO: Find a good way to check the output of the following formatters
# (currently we only check that we can call them without errors).
proj.format_records('latex', 'long')
proj.format_records('shell')
proj.format_records('json')
def test__load_project__should_return_Project(self):
proj1 = Project("test_project", record_store=MockRecordStore())
assert os.path.exists(".smt/project")
proj2 = load_project()
self.assertEqual(proj1.name, proj2.name)
def test__update_code(self):
proj = Project("test_project",
record_store=MockRecordStore(),
default_repository=MockRepository())
wc = proj.default_repository.get_working_copy()
proj.update_code(wc, version=9369835)
from sumatra.records import Record
from sumatra.recordstore import django_store
from sumatra.programs import PythonExecutable
from sumatra.launch import SerialLaunchMode
from sumatra.datastore import FileSystemDataStore
from sumatra.parameters import SimpleParameterSet
from sumatra.versioncontrol._git import GitRepository
import random
serial = SerialLaunchMode()
executable = PythonExecutable("/usr/bin/python", version="2.7")
repos = GitRepository('.')
datastore = FileSystemDataStore("/path/to/datastore")
project = Project("test_project",
default_executable=executable,
default_repository=repos,
default_launch_mode=serial,
data_store=datastore,
record_store=django_store.DjangoRecordStore())
parameters = SimpleParameterSet({'a': 2, 'b': 3})
for i in range(50):
record = Record(executable=executable,
repository=repos,
main_file="main.py",
version="99863a9dc5f",
launch_mode=serial,
datastore=datastore,
parameters=parameters,
input_data=[],
script_arguments="",
label="fake_record%00d" % i,
def test__creating_a_second_project_in_the_same_dir_should_raise_an_exception(
self):
Project("test_project1", record_store=MockRecordStore())
self.assertRaises(Exception, Project, "test_project2")
def test__get_record__calls_get_on_the_record_store(self):
proj = Project("test_project", record_store=MockRecordStore())
self.assertEqual(proj.get_record("foo").label, "foofoo")
def test__update_code(self):
proj = Project("test_project",
record_store=MockRecordStore(),
default_repository=MockRepository())
wc = proj.default_repository.get_working_copy()
proj.update_code(wc, version=9369835)
def test__info(self):
proj = Project("test_project", record_store=MockRecordStore())
proj.info()
def test__remove_tag__should_call_remove_on_the_tags_attibute_of_the_record(self):
proj = Project("test_project",
record_store=MockRecordStore())
proj.remove_tag("foo", "new_tag")
def test__delete_record__calls_delete_on_the_record_store(self):
proj = Project("test_project", record_store=MockRecordStore())
proj.delete_record("foo")
self.assertEqual(proj.record_store.deleted, "foo")
def test__get_record__calls_get_on_the_record_store(self):
proj = Project("test_project",
record_store=MockRecordStore())
self.assertEqual(proj.get_record("foo").label, "foofoo")
def test__delete_by_tag__calls_delete_by_tag_on_the_record_store(self):
proj = Project("test_project",
record_store=MockRecordStore())
self.assertEqual(proj.delete_by_tag("foo"), "oof")
def sumatra_start(repository, sumatra_db_path, results_path, working_dir,
hg_username, sumatra_run_name, parameters):
'''Clones the Omics Pipe repository from Bitbucket, creates a Sumatra project, and creates a Sumatra record for the current run'''
print "sumatra_db_path is " + sumatra_db_path
print type(sumatra_db_path)
check_create_dir(sumatra_db_path)
os.chdir(sumatra_db_path)
repo1 = hgapi.Repo(repository)
repo_path = sumatra_db_path + "/omics_pipe"
repo = {
"url": repo_path,
"type": "sumatra.versioncontrol._mercurial.MercurialRepository",
"upstream": repository
}
executable = {
"path": "",
"version": "",
"type": "sumatra.programs.PythonExecutable",
"options": "",
"name": "Python"
}
sumatra_launch_mode = {
"working_directory": working_dir,
"type": "sumatra.launch.SerialLaunchMode"
}
data_store1 = {
"root": results_path,
"type": "sumatra.datastore.filesystem.FileSystemDataStore"
}
database_path = sumatra_db_path + "/records/recordstore.db"
record_store1 = {
"db_file": database_path,
"type": "sumatra.recordstore.django_store.DjangoRecordStore"
}
input_datastore1 = {
"root": results_path,
"type": "sumatra.datastore.filesystem.FileSystemDataStore"
}
while True:
try:
repo1.hg_clone(url=repository, path=repo_path)
with open(repo_path + "/.hg/hgrc", "a") as myfile:
myfile.write("[ui]\nusername= "******"Omics pipe repository cloned to : " + repo_path
break
except hgapi.hgapi.HgException:
print "Omics pipe repository already exists."
break
while True:
try:
Project(sumatra_run_name,
default_repository=repo,
default_executable=executable,
default_launch_mode=sumatra_launch_mode,
on_changed='store-diff',
data_store=data_store1,
record_store=record_store1,
input_datastore=input_datastore1)
print "Sumatra project created: " + sumatra_run_name + " in directory: " + sumatra_db_path
break
except Exception:
print "Sumatra project already exists, loading project: " + sumatra_run_name
break
project = load_project(path=sumatra_db_path)
print project
sumatra_params = build_parameters(parameters)
print sumatra_params
os.chdir(repo_path)
repo_main = "omics_pipe/main.py"
record = project.new_record(parameters=sumatra_params, main_file=repo_main)
print record
return record, project
def test__show_diff(self):
proj = Project("test_project", record_store=MockRecordStore())
proj.show_diff("foo", "bar")
def test__remove_tag__should_call_remove_on_the_tags_attibute_of_the_record(
self):
proj = Project("test_project", record_store=MockRecordStore())
proj.remove_tag("foo", "new_tag")
def test__add_comment__should_set_the_outcome_attribute_of_the_record(
self):
proj = Project("test_project", record_store=MockRecordStore())
proj.add_comment("foo", "comment goes here")
def test__delete_by_tag__calls_delete_by_tag_on_the_record_store(self):
proj = Project("test_project", record_store=MockRecordStore())
self.assertEqual(proj.delete_by_tag("foo"), "oof")
def test__delete_record__calls_delete_on_the_record_store(self):
proj = Project("test_project",
record_store=MockRecordStore())
proj.delete_record("foo")
self.assertEqual(proj.record_store.deleted, "foo")
def test__init__with_minimal_arguments(self):
Project("test_project", record_store=MockRecordStore())
def test__add_comment__should_set_the_outcome_attribute_of_the_record(self):
proj = Project("test_project",
record_store=MockRecordStore())
proj.add_comment("foo", "comment goes here")
def init(argv):
"""Create a new project in the current directory."""
usage = "%(prog)s init [options] NAME"
description = "Create a new project called NAME in the current directory."
parser = ArgumentParser(usage=usage,
description=description)
parser.add_argument('project_name', metavar='NAME', help="a short name for the project; should not contain spaces.")
parser.add_argument('-d', '--datapath', metavar='PATH', default='./Data', help="set the path to the directory in which smt will search for output datafiles generated by the simulation/analysis. Defaults to %(default)s.")
parser.add_argument('-i', '--input', metavar='PATH', default='/', help="set the path to the directory relative to which input datafile paths will be given. Defaults to the filesystem root.")
parser.add_argument('-l', '--addlabel', choices=['cmdline', 'parameters', None], metavar='OPTION',
default=None, help="If this option is set, smt will append the record label either to the command line (option 'cmdline') or to the parameter file (option 'parameters'), and will add the label to the datapath when searching for datafiles. It is up to the user to make use of this label inside their program to ensure files are created in the appropriate location.")
parser.add_argument('-e', '--executable', metavar='PATH', help="set the path to the executable. If this is not set, smt will try to infer the executable from the value of the --main option, if supplied, and will try to find the executable from the PATH environment variable, then by searching various likely locations on the filesystem.")
parser.add_argument('-r', '--repository', help="the URL of a Subversion or Mercurial repository containing the code. This will be checked out/cloned into the current directory.")
parser.add_argument('-m', '--main', help="the name of the script that would be supplied on the command line if running the simulation or analysis normally, e.g. init.hoc.")
parser.add_argument('-c', '--on-changed', default='error', help="the action to take if the code in the repository or any of the depdendencies has changed. Defaults to %(default)s") # need to add list of allowed values
parser.add_argument('-s', '--store', help="Specify the path, URL or URI to the record store (must be specified). This can either be an existing record store or one to be created. {0} Not using the `--store` argument defaults to a DjangoRecordStore with Sqlite in `.smt/records`".format(store_arg_help))
parser.add_argument('-g', '--labelgenerator', choices=['timestamp', 'uuid'], default='timestamp', metavar='OPTION', help="specify which method Sumatra should use to generate labels (options: timestamp, uuid)")
parser.add_argument('-t', '--timestamp_format', help="the timestamp format given to strftime", default=TIMESTAMP_FORMAT)
parser.add_argument('-L', '--launch_mode', choices=['serial', 'distributed', 'slurm-mpi'], default='serial', help="how computations should be launched. Defaults to %(default)s")
parser.add_argument('-o', '--launch_mode_options', help="extra options for the given launch mode")
datastore = parser.add_mutually_exclusive_group()
datastore.add_argument('-W', '--webdav', metavar='URL', help="specify a webdav URL (with username@password: if needed) as the archiving location for data")
datastore.add_argument('-A', '--archive', metavar='PATH', help="specify a directory in which to archive output datafiles. If not specified, or if 'false', datafiles are not archived.")
datastore.add_argument('-M', '--mirror', metavar='URL', help="specify a URL at which your datafiles will be mirrored.")
args = parser.parse_args(argv)
try:
project = load_project()
parser.error("A project already exists in directory '{0}'.".format(project.path))
except Exception:
pass
if not os.path.exists(".smt"):
os.mkdir(".smt")
if args.repository:
repository = get_repository(args.repository)
repository.checkout()
else:
repository = get_working_copy().repository # if no repository is specified, we assume there is a working copy in the current directory.
if args.executable:
executable_path, executable_options = parse_executable_str(args.executable)
executable = get_executable(path=executable_path)
executable.args = executable_options
elif args.main:
try:
executable = get_executable(script_file=args.main)
except Exception: # assume unrecognized extension - really need more specific exception type
# should warn that extension unrecognized
executable = None
else:
executable = None
if args.store:
record_store = get_record_store(args.store)
else:
record_store = 'default'
if args.webdav:
# should we care about archive migration??
output_datastore = get_data_store("DavFsDataStore", {"root": args.datapath, "dav_url": args.webdav})
args.archive = '.smt/archive'
elif args.archive and args.archive.lower() != 'false':
if args.archive.lower() == "true":
args.archive = ".smt/archive"
args.archive = os.path.abspath(args.archive)
output_datastore = get_data_store("ArchivingFileSystemDataStore", {"root": args.datapath, "archive": args.archive})
elif args.mirror:
output_datastore = get_data_store("MirroredFileSystemDataStore", {"root": args.datapath, "mirror_base_url": args.mirror})
else:
output_datastore = get_data_store("FileSystemDataStore", {"root": args.datapath})
input_datastore = get_data_store("FileSystemDataStore", {"root": args.input})
if args.launch_mode_options:
args.launch_mode_options = args.launch_mode_options.strip()
launch_mode = get_launch_mode(args.launch_mode)(options=args.launch_mode_options)
project = Project(name=args.project_name,
default_executable=executable,
default_repository=repository,
default_main_file=args.main, # what if incompatible with executable?
default_launch_mode=launch_mode,
data_store=output_datastore,
record_store=record_store,
on_changed=args.on_changed,
data_label=args.addlabel,
input_datastore=input_datastore,
label_generator=args.labelgenerator,
timestamp_format=args.timestamp_format)
if os.path.exists('.smt') and project.record_store.has_project(project.name):
f = open('.smt/labels', 'w')
f.writelines(project.format_records(tags=None, mode='short', format='text', reverse=False))
f.close()
project.save()
def test__show_diff(self):
proj = Project("test_project",
record_store=MockRecordStore())
proj.show_diff("foo", "bar")
from sumatra.recordstore import django_store
from sumatra.programs import PythonExecutable
from sumatra.launch import SerialLaunchMode
from sumatra.datastore import FileSystemDataStore
from sumatra.parameters import SimpleParameterSet
from sumatra.versioncontrol._git import GitRepository
import random
from datetime import datetime
serial = SerialLaunchMode()
executable = PythonExecutable("/usr/bin/python", version="2.7")
repos = GitRepository('.')
datastore = FileSystemDataStore("/path/to/datastore")
project = Project("test_project",
default_executable=executable,
default_repository=repos,
default_launch_mode=serial,
data_store=datastore,
record_store=django_store.DjangoRecordStore())
parameters = SimpleParameterSet({'a': 2, 'b': 3})
for i in range(50):
record = Record(executable=executable, repository=repos,
main_file="main.py", version="99863a9dc5f",
launch_mode=serial, datastore=datastore,
parameters=parameters, input_data=[], script_arguments="",
label="fake_record%00d" % i,
reason="testing", diff='', user='******',
on_changed='store-diff', stdout_stderr='srgvrgvsgverhcser')
record.duration = random.gammavariate(1.0, 1000.0)
record.outcome = "lghsvdghsg zskjdcghnskdjgc ckdjshcgndsg"
record.data_key = "['output.data']"
def init(argv):
"""Create a new project in the current directory."""
usage = "%(prog)s init [options] NAME"
description = "Create a new project called NAME in the current directory."
parser = ArgumentParser(usage=usage,
description=description)
parser.add_argument('project_name', metavar='NAME', help="a short name for the project; should not contain spaces.")
parser.add_argument('-d', '--datapath', metavar='PATH', default='./Data', help="set the path to the directory in which smt will search for output datafiles generated by the simulation/analysis. Defaults to %(default)s.")
parser.add_argument('-i', '--input', metavar='PATH', default='/', help="set the path to the directory relative to which input datafile paths will be given. Defaults to the filesystem root.")
parser.add_argument('-l', '--addlabel', choices=['cmdline', 'parameters', None], metavar='OPTION',
default=None, help="If this option is set, smt will append the record label either to the command line (option 'cmdline') or to the parameter file (option 'parameters'), and will add the label to the datapath when searching for datafiles. It is up to the user to make use of this label inside their program to ensure files are created in the appropriate location.")
parser.add_argument('-e', '--executable', metavar='PATH', help="set the path to the executable. If this is not set, smt will try to infer the executable from the value of the --main option, if supplied, and will try to find the executable from the PATH environment variable, then by searching various likely locations on the filesystem.")
parser.add_argument('-r', '--repository', help="the URL of a Subversion or Mercurial repository containing the code. This will be checked out/cloned into the current directory.")
parser.add_argument('-m', '--main', help="the name of the script that would be supplied on the command line if running the simulation or analysis normally, e.g. init.hoc.")
parser.add_argument('-c', '--on-changed', default='error', help="the action to take if the code in the repository or any of the depdendencies has changed. Defaults to %(default)s") # need to add list of allowed values
parser.add_argument('-s', '--store', help="Specify the path, URL or URI to the record store (must be specified). This can either be an existing record store or one to be created. {0} Not using the `--store` argument defaults to a DjangoRecordStore with Sqlite in `.smt/records`".format(store_arg_help))
parser.add_argument('-g', '--labelgenerator', choices=['timestamp', 'uuid'], default='timestamp', metavar='OPTION', help="specify which method Sumatra should use to generate labels (options: timestamp, uuid)")
parser.add_argument('-t', '--timestamp_format', help="the timestamp format given to strftime", default=TIMESTAMP_FORMAT)
parser.add_argument('-L', '--launch_mode', choices=['serial', 'distributed', 'slurm-mpi'], default='serial', help="how computations should be launched. Defaults to %(default)s")
parser.add_argument('-o', '--launch_mode_options', help="extra options for the given launch mode")
datastore = parser.add_mutually_exclusive_group()
datastore.add_argument('-W', '--webdav', metavar='URL', help="specify a webdav URL (with username@password: if needed) as the archiving location for data")
datastore.add_argument('-A', '--archive', metavar='PATH', help="specify a directory in which to archive output datafiles. If not specified, or if 'false', datafiles are not archived.")
datastore.add_argument('-M', '--mirror', metavar='URL', help="specify a URL at which your datafiles will be mirrored.")
args = parser.parse_args(argv)
try:
project = load_project()
parser.error("A project already exists in this directory.")
except Exception:
pass
if not os.path.exists(".smt"):
os.mkdir(".smt")
if args.repository:
repository = get_repository(args.repository)
repository.checkout()
else:
repository = get_working_copy().repository # if no repository is specified, we assume there is a working copy in the current directory.
if args.executable:
executable_path, executable_options = parse_executable_str(args.executable)
executable = get_executable(path=executable_path)
executable.args = executable_options
elif args.main:
try:
executable = get_executable(script_file=args.main)
except Exception: # assume unrecognized extension - really need more specific exception type
# should warn that extension unrecognized
executable = None
else:
executable = None
if args.store:
record_store = get_record_store(args.store)
else:
record_store = 'default'
if args.webdav:
# should we care about archive migration??
output_datastore = get_data_store("DavFsDataStore", {"root": args.datapath, "dav_url": args.webdav})
args.archive = '.smt/archive'
elif args.archive and args.archive.lower() != 'false':
if args.archive.lower() == "true":
args.archive = ".smt/archive"
args.archive = os.path.abspath(args.archive)
output_datastore = get_data_store("ArchivingFileSystemDataStore", {"root": args.datapath, "archive": args.archive})
elif args.mirror:
output_datastore = get_data_store("MirroredFileSystemDataStore", {"root": args.datapath, "mirror_base_url": args.mirror})
else:
output_datastore = get_data_store("FileSystemDataStore", {"root": args.datapath})
input_datastore = get_data_store("FileSystemDataStore", {"root": args.input})
if args.launch_mode_options:
args.launch_mode_options = args.launch_mode_options.strip()
launch_mode = get_launch_mode(args.launch_mode)(options=args.launch_mode_options)
project = Project(name=args.project_name,
default_executable=executable,
default_repository=repository,
default_main_file=args.main, # what if incompatible with executable?
default_launch_mode=launch_mode,
data_store=output_datastore,
record_store=record_store,
on_changed=args.on_changed,
data_label=args.addlabel,
input_datastore=input_datastore,
label_generator=args.labelgenerator,
timestamp_format=args.timestamp_format)
project.save()
def test__info(self):
proj = Project("test_project", record_store=MockRecordStore())
proj.info()
