Esempio n. 1
0
 def repeat(self, original_label, new_label=None):
     if original_label == 'last':
         tmp = self.most_recent()
     else:
         tmp = self.get_record(original_label)
     original = deepcopy(tmp)
     if hasattr(tmp.parameters, '_url'):  # for some reason, _url is not copied.
         original.parameters._url = tmp.parameters._url  # this is a hackish solution - needs fixed properly
     try:
         working_copy = get_working_copy()
     except VersionControlError:
         original.repository.checkout()
         working_copy = original.repository.get_working_copy()
     if working_copy.repository != original.repository:
         raise NotImplementedError("Ability to switch repositories not yet implemented.")
     current_version = working_copy.current_version()
     new_label = self.launch(parameters=original.parameters,
                             input_data=original.input_data,
                             script_args=original.script_arguments,
                             executable=original.executable,
                             main_file=original.main_file,
                             repository=original.repository,
                             version=original.version,
                             launch_mode=original.launch_mode,
                             diff=original.diff,
                             label=new_label,
                             reason="Repeat experiment %s" % original.label,
                             repeats=original.label)
     working_copy.reset()
     working_copy.use_version(current_version)  # ensure we switch back to the original working copy state
     return new_label, original.label
Esempio n. 2
0
 def repeat(self, original_label, new_label=None):
     if original_label == 'last':
         tmp = self.most_recent()
     else:
         tmp = self.get_record(original_label)
     original = deepcopy(tmp)
     if hasattr(tmp.parameters, '_url'):  # for some reason, _url is not copied.
         original.parameters._url = tmp.parameters._url  # this is a hackish solution - needs fixed properly
     try:
         working_copy = get_working_copy()
     except VersionControlError:
         original.repository.checkout()
         working_copy = original.repository.get_working_copy()
     if working_copy.repository != original.repository:
         raise NotImplementedError("Ability to switch repositories not yet implemented.")
     current_version = working_copy.current_version()
     new_label = self.launch(parameters=original.parameters,
                             input_data=original.input_data,
                             script_args=original.script_arguments,
                             executable=original.executable,
                             main_file=original.main_file,
                             repository=original.repository,
                             version=original.version,
                             launch_mode=original.launch_mode,
                             label=new_label,
                             reason="Repeat experiment %s" % original.label,
                             repeats=original.label)
     working_copy.use_version(current_version)  # ensure we switch back to the original working copy state
     return new_label, original.label
Esempio n. 3
0
def show_script(request, project, label):
    """ get the script content from the repos """
    record = Record.objects.get(label=label, project__id=project)
    wc = get_working_copy(os.getcwd())
    if record.repository.url == wc.path:
        file_content = wc.content(record.version, file=record.main_file)
    else:
        raise Http404
    return HttpResponse('<p><span style="font-size: 16px; font-weight:bold">'+record.main_file+'</span><br><span>'+record.version+'</span></p><hr>'+file_content.replace(' ','&#160;').replace('\n', '<br />'))
Esempio n. 4
0
 def test__can_create_project_in_subdir(self):
     # Test if a Sumatra project can be created in one of the subdirectories of a repository
     tmpdir = tempfile.mkdtemp()
     r = self._create_repository()
     r.checkout(path=tmpdir)
     # get a working copy from the subdirectory
     wc = get_working_copy(os.path.join(tmpdir, 'subpackage')).repository.url
     # is the working copy path same as the repo path?
     self.assertEqual(wc, os.path.realpath(tmpdir))
     shutil.rmtree(tmpdir)
Esempio n. 5
0
def record_detail(request, project, label):
    if label != 'nolabel':
        label = unescape(label)
        record = Record.objects.get(label=label, project__id=project)
    if request.method == 'POST':
        if request.POST.has_key(
                'delete'
        ):  # in this version the page record_detail doesn't have delete option
            record.delete()
            return HttpResponseRedirect('..')
        elif request.POST.has_key(
                'show_args'
        ):  # user clicks the link <parameters> in record_list.html
            parameter_set = record.parameters.to_sumatra()
            return HttpResponse(parameter_set)
        elif request.POST.has_key(
                'show_script'):  # retrieve script code from the repo
            digest = request.POST.get('digest', False)
            path = request.POST.get('path', False)
            path = str(path).encode("string_escape")
            wc = get_working_copy(path)
            file_content = wc.content(digest)
            return HttpResponse(file_content)
        elif request.POST.has_key('compare_records'):
            labels = request.POST.getlist('records[]')
            records = Record.objects.filter(project__id=project)
            records = records.filter(
                label__in=labels[:2])  # by now we take only two records
            for record in records:
                if record.script_arguments == '<parameters>':
                    record.script_arguments = record.parameters.to_sumatra()
            dic = {'records': records}
            return render_to_response('comparison_framework.html', dic)
        else:
            form = RecordUpdateForm(request.POST, instance=record)
            if form.is_valid():
                form.save()
    else:
        form = RecordUpdateForm(instance=record)
    # data_store = get_data_store(record.datastore.type, eval(record.datastore.parameters)) doesn't get used?
    parameter_set = record.parameters.to_sumatra()
    if hasattr(parameter_set, "as_dict"):
        parameter_set = parameter_set.as_dict()
    return render_to_response(
        'record_detail.html', {
            'record': record,
            'project_name': project,
            'parameters': parameter_set,
            'form': form
        })
Esempio n. 6
0
def find_versions_from_versioncontrol(dependencies):
    """Determine whether a file is under version control, and if so,
       obtain version information from this."""
    for dependency in dependencies:
        if dependency.version == "unknown":
            try:
                wc = versioncontrol.get_working_copy(dependency.path)
            except versioncontrol.VersionControlError:
                pass  # dependency.version remains "unknown"
            else:
                if wc.has_changed():
                    dependency.diff = wc.diff()
                dependency.version = wc.current_version()
                dependency.source = wc.repository.url
    return dependencies
Esempio n. 7
0
def find_versions_from_versioncontrol(dependencies):
    """Determine whether a file is under version control, and if so,
       obtain version information from this."""
    for dependency in dependencies:
        if dependency.version == "unknown":
            try:
                wc = versioncontrol.get_working_copy(dependency.path)
            except versioncontrol.VersionControlError:
                pass  # dependency.version remains "unknown"
            else:
                if wc.has_changed():
                    dependency.diff = wc.diff()
                dependency.version = wc.current_version()
                dependency.source = wc.repository.url
    return dependencies
Esempio n. 8
0
def record_detail(request, project, label):
    if label != 'nolabel':
        label = unescape(label)
        record = Record.objects.get(label=label, project__id=project)
    if request.method == 'POST':
        if request.POST.has_key('delete'):  # in this version the page record_detail doesn't have delete option
            record.delete()
            return HttpResponseRedirect('..')
        elif request.POST.has_key('show_args'):  # user clicks the link <parameters> in record_list.html
            parameter_set = record.parameters.to_sumatra()
            return HttpResponse(parameter_set)
        elif request.POST.has_key('show_script'):  # retrieve script code from the repo
            digest = request.POST.get('digest', False)
            path = request.POST.get('path', False)
            path = str(path).encode("string_escape")
            wc = get_working_copy(path)
            file_content = wc.content(digest)
            return HttpResponse(file_content)
        elif request.POST.has_key('compare_records'):
            labels = request.POST.getlist('records[]')
            records = Record.objects.filter(project__id=project)
            records = records.filter(label__in=labels[:2])  # by now we take only two records
            for record in records:
                if record.script_arguments == '<parameters>':
                    record.script_arguments = record.parameters.to_sumatra()
            dic = {'records': records}
            return render_to_response('comparison_framework.html', dic)
        else:
            form = RecordUpdateForm(request.POST, instance=record)
            if form.is_valid():
                form.save()
    else:
        form = RecordUpdateForm(instance=record)
    # data_store = get_data_store(record.datastore.type, eval(record.datastore.parameters)) doesn't get used?
    parameter_set = record.parameters.to_sumatra()
    if hasattr(parameter_set, "as_dict"):
        parameter_set = parameter_set.as_dict()
    return render_to_response('record_detail.html', {'record': record,
                                                     'project_name': project,
                                                     'parameters': parameter_set,
                                                     'form': form
                                                     })
Esempio n. 9
0
def init(argv):
    """Create a new project in the current directory."""
    usage = "%(prog)s init [options] NAME"
    description = "Create a new project called NAME in the current directory."
    parser = ArgumentParser(usage=usage,
                            description=description)
    parser.add_argument('project_name', metavar='NAME', help="a short name for the project; should not contain spaces.")
    parser.add_argument('-d', '--datapath', metavar='PATH', default='./Data', help="set the path to the directory in which smt will search for output datafiles generated by the simulation/analysis. Defaults to %(default)s.")
    parser.add_argument('-i', '--input', metavar='PATH', default='/', help="set the path to the directory relative to which input datafile paths will be given. Defaults to the filesystem root.")
    parser.add_argument('-l', '--addlabel', choices=['cmdline', 'parameters', None], metavar='OPTION',
                        default=None, help="If this option is set, smt will append the record label either to the command line (option 'cmdline') or to the parameter file (option 'parameters'), and will add the label to the datapath when searching for datafiles. It is up to the user to make use of this label inside their program to ensure files are created in the appropriate location.")
    parser.add_argument('-e', '--executable', metavar='PATH', help="set the path to the executable. If this is not set, smt will try to infer the executable from the value of the --main option, if supplied, and will try to find the executable from the PATH environment variable, then by searching various likely locations on the filesystem.")
    parser.add_argument('-r', '--repository', help="the URL of a Subversion or Mercurial repository containing the code. This will be checked out/cloned into the current directory.")
    parser.add_argument('-m', '--main', help="the name of the script that would be supplied on the command line if running the simulation or analysis normally, e.g. init.hoc.")
    parser.add_argument('-c', '--on-changed', default='error', help="the action to take if the code in the repository or any of the depdendencies has changed. Defaults to %(default)s")  # need to add list of allowed values
    parser.add_argument('-s', '--store', help="Specify the path, URL or URI to the record store (must be specified). This can either be an existing record store or one to be created. {0} Not using the `--store` argument defaults to a DjangoRecordStore with Sqlite in `.smt/records`".format(store_arg_help))
    parser.add_argument('-g', '--labelgenerator', choices=['timestamp', 'uuid'], default='timestamp', metavar='OPTION', help="specify which method Sumatra should use to generate labels (options: timestamp, uuid)")
    parser.add_argument('-t', '--timestamp_format', help="the timestamp format given to strftime", default=TIMESTAMP_FORMAT)
    parser.add_argument('-L', '--launch_mode', choices=['serial', 'distributed', 'slurm-mpi'], default='serial', help="how computations should be launched. Defaults to %(default)s")
    parser.add_argument('-o', '--launch_mode_options', help="extra options for the given launch mode")

    datastore = parser.add_mutually_exclusive_group()
    datastore.add_argument('-W', '--webdav', metavar='URL', help="specify a webdav URL (with username@password: if needed) as the archiving location for data")
    datastore.add_argument('-A', '--archive', metavar='PATH', help="specify a directory in which to archive output datafiles. If not specified, or if 'false', datafiles are not archived.")
    datastore.add_argument('-M', '--mirror', metavar='URL', help="specify a URL at which your datafiles will be mirrored.")

    args = parser.parse_args(argv)

    try:
        project = load_project()
        parser.error("A project already exists in directory '{0}'.".format(project.path))
    except Exception:
        pass

    if not os.path.exists(".smt"):
        os.mkdir(".smt")

    if args.repository:
        repository = get_repository(args.repository)
        repository.checkout()
    else:
        repository = get_working_copy().repository  # if no repository is specified, we assume there is a working copy in the current directory.

    if args.executable:
        executable_path, executable_options = parse_executable_str(args.executable)
        executable = get_executable(path=executable_path)
        executable.args = executable_options
    elif args.main:
        try:
            executable = get_executable(script_file=args.main)
        except Exception:  # assume unrecognized extension - really need more specific exception type
            # should warn that extension unrecognized
            executable = None
    else:
        executable = None
    if args.store:
        record_store = get_record_store(args.store)
    else:
        record_store = 'default'

    if args.webdav:
        # should we care about archive migration??
        output_datastore = get_data_store("DavFsDataStore", {"root": args.datapath, "dav_url": args.webdav})
        args.archive = '.smt/archive'
    elif args.archive and args.archive.lower() != 'false':
        if args.archive.lower() == "true":
            args.archive = ".smt/archive"
        args.archive = os.path.abspath(args.archive)
        output_datastore = get_data_store("ArchivingFileSystemDataStore", {"root": args.datapath, "archive": args.archive})
    elif args.mirror:
        output_datastore = get_data_store("MirroredFileSystemDataStore", {"root": args.datapath, "mirror_base_url": args.mirror})
    else:
        output_datastore = get_data_store("FileSystemDataStore", {"root": args.datapath})
    input_datastore = get_data_store("FileSystemDataStore", {"root": args.input})

    if args.launch_mode_options:
        args.launch_mode_options = args.launch_mode_options.strip()
    launch_mode = get_launch_mode(args.launch_mode)(options=args.launch_mode_options)

    project = Project(name=args.project_name,
                      default_executable=executable,
                      default_repository=repository,
                      default_main_file=args.main,  # what if incompatible with executable?
                      default_launch_mode=launch_mode,
                      data_store=output_datastore,
                      record_store=record_store,
                      on_changed=args.on_changed,
                      data_label=args.addlabel,
                      input_datastore=input_datastore,
                      label_generator=args.labelgenerator,
                      timestamp_format=args.timestamp_format)
    if os.path.exists('.smt') and project.record_store.has_project(project.name):
        f = open('.smt/labels', 'w')
        f.writelines(project.format_records(tags=None, mode='short', format='text', reverse=False))
        f.close()
    project.save()
Esempio n. 10
0
def init(argv):
    """Create a new project in the current directory."""
    usage = "%(prog)s init [options] NAME"
    description = "Create a new project called NAME in the current directory."
    parser = ArgumentParser(usage=usage,
                            description=description)
    parser.add_argument('project_name', metavar='NAME', help="a short name for the project; should not contain spaces.")
    parser.add_argument('-d', '--datapath', metavar='PATH', default='./Data', help="set the path to the directory in which smt will search for output datafiles generated by the simulation/analysis. Defaults to %(default)s.")
    parser.add_argument('-i', '--input', metavar='PATH', default='/', help="set the path to the directory relative to which input datafile paths will be given. Defaults to the filesystem root.")
    parser.add_argument('-l', '--addlabel', choices=['cmdline', 'parameters', None], metavar='OPTION',
                        default=None, help="If this option is set, smt will append the record label either to the command line (option 'cmdline') or to the parameter file (option 'parameters'), and will add the label to the datapath when searching for datafiles. It is up to the user to make use of this label inside their program to ensure files are created in the appropriate location.")
    parser.add_argument('-e', '--executable', metavar='PATH', help="set the path to the executable. If this is not set, smt will try to infer the executable from the value of the --main option, if supplied, and will try to find the executable from the PATH environment variable, then by searching various likely locations on the filesystem.")
    parser.add_argument('-r', '--repository', help="the URL of a Subversion or Mercurial repository containing the code. This will be checked out/cloned into the current directory.")
    parser.add_argument('-m', '--main', help="the name of the script that would be supplied on the command line if running the simulation or analysis normally, e.g. init.hoc.")
    parser.add_argument('-c', '--on-changed', default='error', help="the action to take if the code in the repository or any of the depdendencies has changed. Defaults to %(default)s")  # need to add list of allowed values
    parser.add_argument('-s', '--store', help="Specify the path, URL or URI to the record store (must be specified). This can either be an existing record store or one to be created. {0} Not using the `--store` argument defaults to a DjangoRecordStore with Sqlite in `.smt/records`".format(store_arg_help))
    parser.add_argument('-g', '--labelgenerator', choices=['timestamp', 'uuid'], default='timestamp', metavar='OPTION', help="specify which method Sumatra should use to generate labels (options: timestamp, uuid)")
    parser.add_argument('-t', '--timestamp_format', help="the timestamp format given to strftime", default=TIMESTAMP_FORMAT)
    parser.add_argument('-L', '--launch_mode', choices=['serial', 'distributed', 'slurm-mpi'], default='serial', help="how computations should be launched. Defaults to %(default)s")
    parser.add_argument('-o', '--launch_mode_options', help="extra options for the given launch mode")

    datastore = parser.add_mutually_exclusive_group()
    datastore.add_argument('-W', '--webdav', metavar='URL', help="specify a webdav URL (with username@password: if needed) as the archiving location for data")
    datastore.add_argument('-A', '--archive', metavar='PATH', help="specify a directory in which to archive output datafiles. If not specified, or if 'false', datafiles are not archived.")
    datastore.add_argument('-M', '--mirror', metavar='URL', help="specify a URL at which your datafiles will be mirrored.")

    args = parser.parse_args(argv)

    try:
        project = load_project()
        parser.error("A project already exists in this directory.")
    except Exception:
        pass

    if not os.path.exists(".smt"):
        os.mkdir(".smt")

    if args.repository:
        repository = get_repository(args.repository)
        repository.checkout()
    else:
        repository = get_working_copy().repository  # if no repository is specified, we assume there is a working copy in the current directory.

    if args.executable:
        executable_path, executable_options = parse_executable_str(args.executable)
        executable = get_executable(path=executable_path)
        executable.args = executable_options
    elif args.main:
        try:
            executable = get_executable(script_file=args.main)
        except Exception:  # assume unrecognized extension - really need more specific exception type
            # should warn that extension unrecognized
            executable = None
    else:
        executable = None
    if args.store:
        record_store = get_record_store(args.store)
    else:
        record_store = 'default'

    if args.webdav:
        # should we care about archive migration??
        output_datastore = get_data_store("DavFsDataStore", {"root": args.datapath, "dav_url": args.webdav})
        args.archive = '.smt/archive'
    elif args.archive and args.archive.lower() != 'false':
        if args.archive.lower() == "true":
            args.archive = ".smt/archive"
        args.archive = os.path.abspath(args.archive)
        output_datastore = get_data_store("ArchivingFileSystemDataStore", {"root": args.datapath, "archive": args.archive})
    elif args.mirror:
        output_datastore = get_data_store("MirroredFileSystemDataStore", {"root": args.datapath, "mirror_base_url": args.mirror})
    else:
        output_datastore = get_data_store("FileSystemDataStore", {"root": args.datapath})
    input_datastore = get_data_store("FileSystemDataStore", {"root": args.input})

    if args.launch_mode_options:
        args.launch_mode_options = args.launch_mode_options.strip()
    launch_mode = get_launch_mode(args.launch_mode)(options=args.launch_mode_options)

    project = Project(name=args.project_name,
                      default_executable=executable,
                      default_repository=repository,
                      default_main_file=args.main,  # what if incompatible with executable?
                      default_launch_mode=launch_mode,
                      data_store=output_datastore,
                      record_store=record_store,
                      on_changed=args.on_changed,
                      data_label=args.addlabel,
                      input_datastore=input_datastore,
                      label_generator=args.labelgenerator,
                      timestamp_format=args.timestamp_format)
    project.save()