Esempio n. 1
0
    def __init__(self, resume_args, command_log, dirs_log, stack_level=0):
        """Constructor that extracts the command from the file

            ``command_log``: file for stored commands
            ``dirs_log``: file for associated work directories
            ``stack_level``: index in the stack for the command to be retrieved
        """
        self.resume_command = a.get_command_message(resume_args)
        self.command = get_log_reversed(command_log, stack_level)
        self.output_dir = get_log_reversed(dirs_log, stack_level)
        self.defaults_file = os.path.join(self.output_dir, DEFAULTS_FILE)
        self.args = shlex.split(self.command)[1:]
Esempio n. 2
0
 def __init__(self, args, stored_command=None):
     self.stored = (args is None and
                    isinstance(stored_command, StoredCommand))
     self.args = args if not self.stored else stored_command.args
     self.resume = not self.stored and '--resume' in self.args
     self.defaults_file = (None if not self.stored else
                           os.path.join(stored_command.output_dir,
                                        DEFAULTS_FILE))
     self.user_defaults = get_user_defaults(self.defaults_file)
     self.command = (a.get_command_message(self.args) if not self.stored
                     else stored_command.command)
     self.parser, self.common_options = create_parser(
         general_defaults=self.user_defaults,
         constants={'NOW': a.NOW,
                    'MAX_MODELS': MAX_MODELS,
                    'PLURALITY': PLURALITY})
     self.flags, self.train_stdin, self.test_stdin = a.get_flags(self.args)
Esempio n. 3
0
 def __init__(self, args, stored_command=None):
     self.stored = (args is None and
                    isinstance(stored_command, StoredCommand))
     self.args = args if not self.stored else stored_command.args
     self.resume = not self.stored and '--resume' in self.args
     self.defaults_file = (None if not self.stored else
                           os.path.join(stored_command.output_dir,
                                        DEFAULTS_FILE))
     self.user_defaults = get_user_defaults(self.defaults_file)
     self.command = (a.get_command_message(self.args) if not self.stored
                     else stored_command.command)
     self.parser, self.common_options = create_parser(
         general_defaults=self.user_defaults,
         constants={'NOW': a.NOW,
                    'MAX_MODELS': MAX_MODELS,
                    'PLURALITY': PLURALITY})
     self.flags, self.train_stdin, self.test_stdin = a.get_flags(self.args)
Esempio n. 4
0
    def __init__(self, resume_args, command_log, dirs_log, stack_level=0):
        """Constructor that extracts the command from the file

            ``command_log``: file for stored commands
            ``dirs_log``: file for associated work directories
            ``stack_level``: index in the stack for the command to be retrieved
        """
        self.resume_command = a.get_command_message(resume_args)
        self.command = get_log_reversed(command_log, stack_level)
        self.output_dir = get_log_reversed(dirs_log, stack_level)
        self.defaults_file = os.path.join(self.output_dir, DEFAULTS_FILE)
        self.args = shlex.split(self.command)[1:]
        if not ("--output" in self.args or "--output-dir" in self.args):
            current_directory = "%s%s" % (os.getcwd(), os.sep)
            if self.output_dir.startswith(current_directory):
                self.output_dir = self.output_dir.replace(current_directory,
                                                          "", 1)
            self.args.append("--output-dir")
            self.args.append(self.output_dir)
Esempio n. 5
0
    def __init__(self, resume_args, command_log, dirs_log, stack_level=0):
        """Constructor that extracts the command from the file

            ``command_log``: file for stored commands
            ``dirs_log``: file for associated work directories
            ``stack_level``: index in the stack for the command to be retrieved
        """
        self.resume_command = a.get_command_message(resume_args)
        self.command = get_log_reversed(command_log, stack_level)
        self.output_dir = get_log_reversed(dirs_log, stack_level)
        self.defaults_file = os.path.join(self.output_dir, DEFAULTS_FILE)
        self.args = [arg.decode(u.SYSTEM_ENCODING) for arg in
                     shlex.split(self.command.encode(u.SYSTEM_ENCODING))[1:]]
        if not ("--output" in self.args or "--output-dir" in self.args):
            current_directory = u"%s%s" % (os.getcwd(), os.sep)
            if self.output_dir.startswith(current_directory):
                self.output_dir = self.output_dir.replace(current_directory,
                                                          "", 1)
            self.args.append("--output-dir")
            self.args.append(self.output_dir)
Esempio n. 6
0
def main(args=sys.argv[1:]):
    """Main process

    """
    (flags, train_stdin, test_stdin) = a.get_flags(args)

    # If --clear-logs the log files are cleared
    if "--clear-logs" in args:
        for log_file in LOG_FILES:
            try:
                open(log_file, 'w', 0).close()
            except IOError:
                pass
    message = a.get_command_message(args)

    # Resume calls are not logged
    if not "--resume" in args:
        with open(COMMAND_LOG, "a", 0) as command_log:
            command_log.write(message)
        resume = False
    user_defaults = get_user_defaults()
    parser = create_parser(defaults=get_user_defaults(),
                           constants={'NOW': a.NOW,
                                      'MAX_MODELS': MAX_MODELS,
                                      'PLURALITY': PLURALITY})
    # Parses command line arguments.
    command_args = a.parse_and_check(parser, args, train_stdin, test_stdin)

    default_output = ('evaluation' if command_args.evaluate
                      else 'predictions.csv')
    if command_args.resume:
        # Restore the args of the call to resume from the command log file
        debug = command_args.debug
        command = u.get_log_reversed(COMMAND_LOG,
                                     command_args.stack_level)
        args = shlex.split(command)[1:]
        try:
            position = args.index("--train")
            train_stdin = (position == (len(args) - 1) or
                           args[position + 1].startswith("--"))
        except ValueError:
            pass
        try:
            position = args.index("--test")
            test_stdin = (position == (len(args) - 1) or
                          args[position + 1].startswith("--"))
        except ValueError:
            pass
        output_dir = u.get_log_reversed(DIRS_LOG,
                                        command_args.stack_level)
        defaults_file = "%s%s%s" % (output_dir, os.sep, DEFAULTS_FILE)
        user_defaults = get_user_defaults(defaults_file)
        parser = create_parser(defaults=user_defaults,
                               constants={'NOW': a.NOW,
                                          'MAX_MODELS': MAX_MODELS,
                                          'PLURALITY': PLURALITY})
        # Parses resumed arguments.
        command_args = a.parse_and_check(parser, args, train_stdin, test_stdin)
        if command_args.predictions is None:
            command_args.predictions = ("%s%s%s" %
                                        (output_dir, os.sep,
                                         default_output))

        # Logs the issued command and the resumed command
        session_file = "%s%s%s" % (output_dir, os.sep, SESSIONS_LOG)
        u.log_message(message, log_file=session_file)
        message = "\nResuming command:\n%s\n\n" % command
        u.log_message(message, log_file=session_file, console=True)
        try:
            defaults_handler = open(defaults_file, 'r')
            contents = defaults_handler.read()
            message = "\nUsing the following defaults:\n%s\n\n" % contents
            u.log_message(message, log_file=session_file, console=True)
            defaults_handler.close()
        except IOError:
            pass

        resume = True
    else:
        if command_args.output_dir is None:
            command_args.output_dir = a.NOW
        if command_args.predictions is None:
            command_args.predictions = ("%s%s%s" %
                                        (command_args.output_dir, os.sep,
                                         default_output))
        if len(os.path.dirname(command_args.predictions).strip()) == 0:
            command_args.predictions = ("%s%s%s" %
                                        (command_args.output_dir, os.sep,
                                         command_args.predictions))
        directory = u.check_dir(command_args.predictions)
        session_file = "%s%s%s" % (directory, os.sep, SESSIONS_LOG)
        u.log_message(message + "\n", log_file=session_file)
        try:
            defaults_file = open(DEFAULTS_FILE, 'r')
            contents = defaults_file.read()
            defaults_file.close()
            defaults_copy = open("%s%s%s" % (directory, os.sep, DEFAULTS_FILE),
                                 'w', 0)
            defaults_copy.write(contents)
            defaults_copy.close()
        except IOError:
            pass
        with open(DIRS_LOG, "a", 0) as directory_log:
            directory_log.write("%s\n" % os.path.abspath(directory))

    # Creates the corresponding api instance
    if resume and debug:
        command_args.debug = True
    api = a.get_api_instance(command_args, u.check_dir(session_file))

    # Selects the action to perform: delete or create resources
    if command_args.delete:
        delete_resources(command_args, api)
    elif (command_args.training_set or has_test(command_args)
          or command_args.source or command_args.dataset
          or command_args.datasets or command_args.votes_dirs):
        output_args = a.get_output_args(api, train_stdin, test_stdin,
                                        command_args, resume)
        a.transform_args(command_args, flags, api, user_defaults)
        compute_output(**output_args)
    u.log_message("_" * 80 + "\n", log_file=session_file)
Esempio n. 7
0
def main(args=sys.argv[1:]):
    """Main process

    """
    (flags, train_stdin, test_stdin) = a.get_flags(args)

    # If --clear-logs the log files are cleared
    if "--clear-logs" in args:
        for log_file in LOG_FILES:
            try:
                open(log_file, 'w', 0).close()
            except IOError:
                pass
    message = a.get_command_message(args)

    # Resume calls are not logged
    if not "--resume" in args:
        with open(COMMAND_LOG, "a", 0) as command_log:
            command_log.write(message)
        resume = False
    user_defaults = get_user_defaults()
    parser = create_parser(defaults=get_user_defaults(),
                           constants={
                               'NOW': a.NOW,
                               'MAX_MODELS': MAX_MODELS,
                               'PLURALITY': PLURALITY
                           })
    # Parses command line arguments.
    command_args = a.parse_and_check(parser, args, train_stdin, test_stdin)

    default_output = ('evaluation'
                      if command_args.evaluate else 'predictions.csv')
    if command_args.resume:
        # Restore the args of the call to resume from the command log file
        debug = command_args.debug
        command = u.get_log_reversed(COMMAND_LOG, command_args.stack_level)
        args = shlex.split(command)[1:]
        try:
            position = args.index("--train")
            train_stdin = (position == (len(args) - 1)
                           or args[position + 1].startswith("--"))
        except ValueError:
            pass
        try:
            position = args.index("--test")
            test_stdin = (position == (len(args) - 1)
                          or args[position + 1].startswith("--"))
        except ValueError:
            pass
        output_dir = u.get_log_reversed(DIRS_LOG, command_args.stack_level)
        defaults_file = "%s%s%s" % (output_dir, os.sep, DEFAULTS_FILE)
        user_defaults = get_user_defaults(defaults_file)
        parser = create_parser(defaults=user_defaults,
                               constants={
                                   'NOW': a.NOW,
                                   'MAX_MODELS': MAX_MODELS,
                                   'PLURALITY': PLURALITY
                               })
        # Parses resumed arguments.
        command_args = a.parse_and_check(parser, args, train_stdin, test_stdin)
        if command_args.predictions is None:
            command_args.predictions = ("%s%s%s" %
                                        (output_dir, os.sep, default_output))

        # Logs the issued command and the resumed command
        session_file = "%s%s%s" % (output_dir, os.sep, SESSIONS_LOG)
        u.log_message(message, log_file=session_file)
        message = "\nResuming command:\n%s\n\n" % command
        u.log_message(message, log_file=session_file, console=True)
        try:
            defaults_handler = open(defaults_file, 'r')
            contents = defaults_handler.read()
            message = "\nUsing the following defaults:\n%s\n\n" % contents
            u.log_message(message, log_file=session_file, console=True)
            defaults_handler.close()
        except IOError:
            pass

        resume = True
    else:
        if command_args.output_dir is None:
            command_args.output_dir = a.NOW
        if command_args.predictions is None:
            command_args.predictions = (
                "%s%s%s" % (command_args.output_dir, os.sep, default_output))
        if len(os.path.dirname(command_args.predictions).strip()) == 0:
            command_args.predictions = (
                "%s%s%s" %
                (command_args.output_dir, os.sep, command_args.predictions))
        directory = u.check_dir(command_args.predictions)
        session_file = "%s%s%s" % (directory, os.sep, SESSIONS_LOG)
        u.log_message(message + "\n", log_file=session_file)
        try:
            defaults_file = open(DEFAULTS_FILE, 'r')
            contents = defaults_file.read()
            defaults_file.close()
            defaults_copy = open("%s%s%s" % (directory, os.sep, DEFAULTS_FILE),
                                 'w', 0)
            defaults_copy.write(contents)
            defaults_copy.close()
        except IOError:
            pass
        with open(DIRS_LOG, "a", 0) as directory_log:
            directory_log.write("%s\n" % os.path.abspath(directory))

    # Creates the corresponding api instance
    if resume and debug:
        command_args.debug = True
    api = a.get_api_instance(command_args, u.check_dir(session_file))

    # Selects the action to perform: delete or create resources
    if command_args.delete:
        delete_resources(command_args, api)
    elif (command_args.training_set or has_test(command_args)
          or command_args.source or command_args.dataset
          or command_args.datasets or command_args.votes_dirs):
        output_args = a.get_output_args(api, train_stdin, test_stdin,
                                        command_args, resume)
        a.transform_args(command_args, flags, api, user_defaults)
        compute_output(**output_args)
    u.log_message("_" * 80 + "\n", log_file=session_file)