def repeat(self, original_label, new_label=None):
if original_label == 'last':
tmp = self.most_recent()
else:
tmp = self.get_record(original_label)
original = deepcopy(tmp)
if hasattr(tmp.parameters, '_url'): # for some reason, _url is not copied.
original.parameters._url = tmp.parameters._url # this is a hackish solution - needs fixed properly
try:
working_copy = get_working_copy()
except VersionControlError:
original.repository.checkout()
working_copy = original.repository.get_working_copy()
if working_copy.repository != original.repository:
raise NotImplementedError("Ability to switch repositories not yet implemented.")
current_version = working_copy.current_version()
new_label = self.launch(parameters=original.parameters,
input_data=original.input_data,
script_args=original.script_arguments,
executable=original.executable,
main_file=original.main_file,
repository=original.repository,
version=original.version,
launch_mode=original.launch_mode,
diff=original.diff,
label=new_label,
reason="Repeat experiment %s" % original.label,
repeats=original.label)
working_copy.reset()
working_copy.use_version(current_version) # ensure we switch back to the original working copy state
return new_label, original.label
def repeat(self, original_label, new_label=None):
if original_label == 'last':
tmp = self.most_recent()
else:
tmp = self.get_record(original_label)
original = deepcopy(tmp)
if hasattr(tmp.parameters, '_url'): # for some reason, _url is not copied.
original.parameters._url = tmp.parameters._url # this is a hackish solution - needs fixed properly
try:
working_copy = get_working_copy()
except VersionControlError:
original.repository.checkout()
working_copy = original.repository.get_working_copy()
if working_copy.repository != original.repository:
raise NotImplementedError("Ability to switch repositories not yet implemented.")
current_version = working_copy.current_version()
new_label = self.launch(parameters=original.parameters,
input_data=original.input_data,
script_args=original.script_arguments,
executable=original.executable,
main_file=original.main_file,
repository=original.repository,
version=original.version,
launch_mode=original.launch_mode,
label=new_label,
reason="Repeat experiment %s" % original.label,
repeats=original.label)
working_copy.use_version(current_version) # ensure we switch back to the original working copy state
return new_label, original.label
def show_script(request, project, label):
""" get the script content from the repos """
record = Record.objects.get(label=label, project__id=project)
wc = get_working_copy(os.getcwd())
if record.repository.url == wc.path:
file_content = wc.content(record.version, file=record.main_file)
else:
raise Http404
return HttpResponse('<p><span style="font-size: 16px; font-weight:bold">'+record.main_file+'</span><br><span>'+record.version+'</span></p><hr>'+file_content.replace(' ',' ').replace('\n', '<br />'))
def test__can_create_project_in_subdir(self):
# Test if a Sumatra project can be created in one of the subdirectories of a repository
tmpdir = tempfile.mkdtemp()
r = self._create_repository()
r.checkout(path=tmpdir)
# get a working copy from the subdirectory
wc = get_working_copy(os.path.join(tmpdir, 'subpackage')).repository.url
# is the working copy path same as the repo path?
self.assertEqual(wc, os.path.realpath(tmpdir))
shutil.rmtree(tmpdir)
def record_detail(request, project, label):
if label != 'nolabel':
label = unescape(label)
record = Record.objects.get(label=label, project__id=project)
if request.method == 'POST':
if request.POST.has_key(
'delete'
): # in this version the page record_detail doesn't have delete option
record.delete()
return HttpResponseRedirect('..')
elif request.POST.has_key(
'show_args'
): # user clicks the link <parameters> in record_list.html
parameter_set = record.parameters.to_sumatra()
return HttpResponse(parameter_set)
elif request.POST.has_key(
'show_script'): # retrieve script code from the repo
digest = request.POST.get('digest', False)
path = request.POST.get('path', False)
path = str(path).encode("string_escape")
wc = get_working_copy(path)
file_content = wc.content(digest)
return HttpResponse(file_content)
elif request.POST.has_key('compare_records'):
labels = request.POST.getlist('records[]')
records = Record.objects.filter(project__id=project)
records = records.filter(
label__in=labels[:2]) # by now we take only two records
for record in records:
if record.script_arguments == '<parameters>':
record.script_arguments = record.parameters.to_sumatra()
dic = {'records': records}
return render_to_response('comparison_framework.html', dic)
else:
form = RecordUpdateForm(request.POST, instance=record)
if form.is_valid():
form.save()
else:
form = RecordUpdateForm(instance=record)
# data_store = get_data_store(record.datastore.type, eval(record.datastore.parameters)) doesn't get used?
parameter_set = record.parameters.to_sumatra()
if hasattr(parameter_set, "as_dict"):
parameter_set = parameter_set.as_dict()
return render_to_response(
'record_detail.html', {
'record': record,
'project_name': project,
'parameters': parameter_set,
'form': form
})
def find_versions_from_versioncontrol(dependencies):
"""Determine whether a file is under version control, and if so,
obtain version information from this."""
for dependency in dependencies:
if dependency.version == "unknown":
try:
wc = versioncontrol.get_working_copy(dependency.path)
except versioncontrol.VersionControlError:
pass # dependency.version remains "unknown"
else:
if wc.has_changed():
dependency.diff = wc.diff()
dependency.version = wc.current_version()
dependency.source = wc.repository.url
return dependencies
def find_versions_from_versioncontrol(dependencies):
"""Determine whether a file is under version control, and if so,
obtain version information from this."""
for dependency in dependencies:
if dependency.version == "unknown":
try:
wc = versioncontrol.get_working_copy(dependency.path)
except versioncontrol.VersionControlError:
pass # dependency.version remains "unknown"
else:
if wc.has_changed():
dependency.diff = wc.diff()
dependency.version = wc.current_version()
dependency.source = wc.repository.url
return dependencies
def record_detail(request, project, label):
if label != 'nolabel':
label = unescape(label)
record = Record.objects.get(label=label, project__id=project)
if request.method == 'POST':
if request.POST.has_key('delete'): # in this version the page record_detail doesn't have delete option
record.delete()
return HttpResponseRedirect('..')
elif request.POST.has_key('show_args'): # user clicks the link <parameters> in record_list.html
parameter_set = record.parameters.to_sumatra()
return HttpResponse(parameter_set)
elif request.POST.has_key('show_script'): # retrieve script code from the repo
digest = request.POST.get('digest', False)
path = request.POST.get('path', False)
path = str(path).encode("string_escape")
wc = get_working_copy(path)
file_content = wc.content(digest)
return HttpResponse(file_content)
elif request.POST.has_key('compare_records'):
labels = request.POST.getlist('records[]')
records = Record.objects.filter(project__id=project)
records = records.filter(label__in=labels[:2]) # by now we take only two records
for record in records:
if record.script_arguments == '<parameters>':
record.script_arguments = record.parameters.to_sumatra()
dic = {'records': records}
return render_to_response('comparison_framework.html', dic)
else:
form = RecordUpdateForm(request.POST, instance=record)
if form.is_valid():
form.save()
else:
form = RecordUpdateForm(instance=record)
# data_store = get_data_store(record.datastore.type, eval(record.datastore.parameters)) doesn't get used?
parameter_set = record.parameters.to_sumatra()
if hasattr(parameter_set, "as_dict"):
parameter_set = parameter_set.as_dict()
return render_to_response('record_detail.html', {'record': record,
'project_name': project,
'parameters': parameter_set,
'form': form
})
def init(argv):
"""Create a new project in the current directory."""
usage = "%(prog)s init [options] NAME"
description = "Create a new project called NAME in the current directory."
parser = ArgumentParser(usage=usage,
description=description)
parser.add_argument('project_name', metavar='NAME', help="a short name for the project; should not contain spaces.")
parser.add_argument('-d', '--datapath', metavar='PATH', default='./Data', help="set the path to the directory in which smt will search for output datafiles generated by the simulation/analysis. Defaults to %(default)s.")
parser.add_argument('-i', '--input', metavar='PATH', default='/', help="set the path to the directory relative to which input datafile paths will be given. Defaults to the filesystem root.")
parser.add_argument('-l', '--addlabel', choices=['cmdline', 'parameters', None], metavar='OPTION',
default=None, help="If this option is set, smt will append the record label either to the command line (option 'cmdline') or to the parameter file (option 'parameters'), and will add the label to the datapath when searching for datafiles. It is up to the user to make use of this label inside their program to ensure files are created in the appropriate location.")
parser.add_argument('-e', '--executable', metavar='PATH', help="set the path to the executable. If this is not set, smt will try to infer the executable from the value of the --main option, if supplied, and will try to find the executable from the PATH environment variable, then by searching various likely locations on the filesystem.")
parser.add_argument('-r', '--repository', help="the URL of a Subversion or Mercurial repository containing the code. This will be checked out/cloned into the current directory.")
parser.add_argument('-m', '--main', help="the name of the script that would be supplied on the command line if running the simulation or analysis normally, e.g. init.hoc.")
parser.add_argument('-c', '--on-changed', default='error', help="the action to take if the code in the repository or any of the depdendencies has changed. Defaults to %(default)s") # need to add list of allowed values
parser.add_argument('-s', '--store', help="Specify the path, URL or URI to the record store (must be specified). This can either be an existing record store or one to be created. {0} Not using the `--store` argument defaults to a DjangoRecordStore with Sqlite in `.smt/records`".format(store_arg_help))
parser.add_argument('-g', '--labelgenerator', choices=['timestamp', 'uuid'], default='timestamp', metavar='OPTION', help="specify which method Sumatra should use to generate labels (options: timestamp, uuid)")
parser.add_argument('-t', '--timestamp_format', help="the timestamp format given to strftime", default=TIMESTAMP_FORMAT)
parser.add_argument('-L', '--launch_mode', choices=['serial', 'distributed', 'slurm-mpi'], default='serial', help="how computations should be launched. Defaults to %(default)s")
parser.add_argument('-o', '--launch_mode_options', help="extra options for the given launch mode")
datastore = parser.add_mutually_exclusive_group()
datastore.add_argument('-W', '--webdav', metavar='URL', help="specify a webdav URL (with username@password: if needed) as the archiving location for data")
datastore.add_argument('-A', '--archive', metavar='PATH', help="specify a directory in which to archive output datafiles. If not specified, or if 'false', datafiles are not archived.")
datastore.add_argument('-M', '--mirror', metavar='URL', help="specify a URL at which your datafiles will be mirrored.")
args = parser.parse_args(argv)
try:
project = load_project()
parser.error("A project already exists in directory '{0}'.".format(project.path))
except Exception:
pass
if not os.path.exists(".smt"):
os.mkdir(".smt")
if args.repository:
repository = get_repository(args.repository)
repository.checkout()
else:
repository = get_working_copy().repository # if no repository is specified, we assume there is a working copy in the current directory.
if args.executable:
executable_path, executable_options = parse_executable_str(args.executable)
executable = get_executable(path=executable_path)
executable.args = executable_options
elif args.main:
try:
executable = get_executable(script_file=args.main)
except Exception: # assume unrecognized extension - really need more specific exception type
# should warn that extension unrecognized
executable = None
else:
executable = None
if args.store:
record_store = get_record_store(args.store)
else:
record_store = 'default'
if args.webdav:
# should we care about archive migration??
output_datastore = get_data_store("DavFsDataStore", {"root": args.datapath, "dav_url": args.webdav})
args.archive = '.smt/archive'
elif args.archive and args.archive.lower() != 'false':
if args.archive.lower() == "true":
args.archive = ".smt/archive"
args.archive = os.path.abspath(args.archive)
output_datastore = get_data_store("ArchivingFileSystemDataStore", {"root": args.datapath, "archive": args.archive})
elif args.mirror:
output_datastore = get_data_store("MirroredFileSystemDataStore", {"root": args.datapath, "mirror_base_url": args.mirror})
else:
output_datastore = get_data_store("FileSystemDataStore", {"root": args.datapath})
input_datastore = get_data_store("FileSystemDataStore", {"root": args.input})
if args.launch_mode_options:
args.launch_mode_options = args.launch_mode_options.strip()
launch_mode = get_launch_mode(args.launch_mode)(options=args.launch_mode_options)
project = Project(name=args.project_name,
default_executable=executable,
default_repository=repository,
default_main_file=args.main, # what if incompatible with executable?
default_launch_mode=launch_mode,
data_store=output_datastore,
record_store=record_store,
on_changed=args.on_changed,
data_label=args.addlabel,
input_datastore=input_datastore,
label_generator=args.labelgenerator,
timestamp_format=args.timestamp_format)
if os.path.exists('.smt') and project.record_store.has_project(project.name):
f = open('.smt/labels', 'w')
f.writelines(project.format_records(tags=None, mode='short', format='text', reverse=False))
f.close()
project.save()
def init(argv):
"""Create a new project in the current directory."""
usage = "%(prog)s init [options] NAME"
description = "Create a new project called NAME in the current directory."
parser = ArgumentParser(usage=usage,
description=description)
parser.add_argument('project_name', metavar='NAME', help="a short name for the project; should not contain spaces.")
parser.add_argument('-d', '--datapath', metavar='PATH', default='./Data', help="set the path to the directory in which smt will search for output datafiles generated by the simulation/analysis. Defaults to %(default)s.")
parser.add_argument('-i', '--input', metavar='PATH', default='/', help="set the path to the directory relative to which input datafile paths will be given. Defaults to the filesystem root.")
parser.add_argument('-l', '--addlabel', choices=['cmdline', 'parameters', None], metavar='OPTION',
default=None, help="If this option is set, smt will append the record label either to the command line (option 'cmdline') or to the parameter file (option 'parameters'), and will add the label to the datapath when searching for datafiles. It is up to the user to make use of this label inside their program to ensure files are created in the appropriate location.")
parser.add_argument('-e', '--executable', metavar='PATH', help="set the path to the executable. If this is not set, smt will try to infer the executable from the value of the --main option, if supplied, and will try to find the executable from the PATH environment variable, then by searching various likely locations on the filesystem.")
parser.add_argument('-r', '--repository', help="the URL of a Subversion or Mercurial repository containing the code. This will be checked out/cloned into the current directory.")
parser.add_argument('-m', '--main', help="the name of the script that would be supplied on the command line if running the simulation or analysis normally, e.g. init.hoc.")
parser.add_argument('-c', '--on-changed', default='error', help="the action to take if the code in the repository or any of the depdendencies has changed. Defaults to %(default)s") # need to add list of allowed values
parser.add_argument('-s', '--store', help="Specify the path, URL or URI to the record store (must be specified). This can either be an existing record store or one to be created. {0} Not using the `--store` argument defaults to a DjangoRecordStore with Sqlite in `.smt/records`".format(store_arg_help))
parser.add_argument('-g', '--labelgenerator', choices=['timestamp', 'uuid'], default='timestamp', metavar='OPTION', help="specify which method Sumatra should use to generate labels (options: timestamp, uuid)")
parser.add_argument('-t', '--timestamp_format', help="the timestamp format given to strftime", default=TIMESTAMP_FORMAT)
parser.add_argument('-L', '--launch_mode', choices=['serial', 'distributed', 'slurm-mpi'], default='serial', help="how computations should be launched. Defaults to %(default)s")
parser.add_argument('-o', '--launch_mode_options', help="extra options for the given launch mode")
datastore = parser.add_mutually_exclusive_group()
datastore.add_argument('-W', '--webdav', metavar='URL', help="specify a webdav URL (with username@password: if needed) as the archiving location for data")
datastore.add_argument('-A', '--archive', metavar='PATH', help="specify a directory in which to archive output datafiles. If not specified, or if 'false', datafiles are not archived.")
datastore.add_argument('-M', '--mirror', metavar='URL', help="specify a URL at which your datafiles will be mirrored.")
args = parser.parse_args(argv)
try:
project = load_project()
parser.error("A project already exists in this directory.")
except Exception:
pass
if not os.path.exists(".smt"):
os.mkdir(".smt")
if args.repository:
repository = get_repository(args.repository)
repository.checkout()
else:
repository = get_working_copy().repository # if no repository is specified, we assume there is a working copy in the current directory.
if args.executable:
executable_path, executable_options = parse_executable_str(args.executable)
executable = get_executable(path=executable_path)
executable.args = executable_options
elif args.main:
try:
executable = get_executable(script_file=args.main)
except Exception: # assume unrecognized extension - really need more specific exception type
# should warn that extension unrecognized
executable = None
else:
executable = None
if args.store:
record_store = get_record_store(args.store)
else:
record_store = 'default'
if args.webdav:
# should we care about archive migration??
output_datastore = get_data_store("DavFsDataStore", {"root": args.datapath, "dav_url": args.webdav})
args.archive = '.smt/archive'
elif args.archive and args.archive.lower() != 'false':
if args.archive.lower() == "true":
args.archive = ".smt/archive"
args.archive = os.path.abspath(args.archive)
output_datastore = get_data_store("ArchivingFileSystemDataStore", {"root": args.datapath, "archive": args.archive})
elif args.mirror:
output_datastore = get_data_store("MirroredFileSystemDataStore", {"root": args.datapath, "mirror_base_url": args.mirror})
else:
output_datastore = get_data_store("FileSystemDataStore", {"root": args.datapath})
input_datastore = get_data_store("FileSystemDataStore", {"root": args.input})
if args.launch_mode_options:
args.launch_mode_options = args.launch_mode_options.strip()
launch_mode = get_launch_mode(args.launch_mode)(options=args.launch_mode_options)
project = Project(name=args.project_name,
default_executable=executable,
default_repository=repository,
default_main_file=args.main, # what if incompatible with executable?
default_launch_mode=launch_mode,
data_store=output_datastore,
record_store=record_store,
on_changed=args.on_changed,
data_label=args.addlabel,
input_datastore=input_datastore,
label_generator=args.labelgenerator,
timestamp_format=args.timestamp_format)
project.save()
