def __init__(self, resume_args, command_log, dirs_log, stack_level=0):
"""Constructor that extracts the command from the file
``command_log``: file for stored commands
``dirs_log``: file for associated work directories
``stack_level``: index in the stack for the command to be retrieved
"""
self.resume_command = a.get_command_message(resume_args)
self.command = get_log_reversed(command_log, stack_level)
self.output_dir = get_log_reversed(dirs_log, stack_level)
self.defaults_file = os.path.join(self.output_dir, DEFAULTS_FILE)
self.args = shlex.split(self.command)[1:]
def __init__(self, args, stored_command=None):
self.stored = (args is None and
isinstance(stored_command, StoredCommand))
self.args = args if not self.stored else stored_command.args
self.resume = not self.stored and '--resume' in self.args
self.defaults_file = (None if not self.stored else
os.path.join(stored_command.output_dir,
DEFAULTS_FILE))
self.user_defaults = get_user_defaults(self.defaults_file)
self.command = (a.get_command_message(self.args) if not self.stored
else stored_command.command)
self.parser, self.common_options = create_parser(
general_defaults=self.user_defaults,
constants={'NOW': a.NOW,
'MAX_MODELS': MAX_MODELS,
'PLURALITY': PLURALITY})
self.flags, self.train_stdin, self.test_stdin = a.get_flags(self.args)
def __init__(self, args, stored_command=None):
self.stored = (args is None and
isinstance(stored_command, StoredCommand))
self.args = args if not self.stored else stored_command.args
self.resume = not self.stored and '--resume' in self.args
self.defaults_file = (None if not self.stored else
os.path.join(stored_command.output_dir,
DEFAULTS_FILE))
self.user_defaults = get_user_defaults(self.defaults_file)
self.command = (a.get_command_message(self.args) if not self.stored
else stored_command.command)
self.parser, self.common_options = create_parser(
general_defaults=self.user_defaults,
constants={'NOW': a.NOW,
'MAX_MODELS': MAX_MODELS,
'PLURALITY': PLURALITY})
self.flags, self.train_stdin, self.test_stdin = a.get_flags(self.args)
def __init__(self, resume_args, command_log, dirs_log, stack_level=0):
"""Constructor that extracts the command from the file
``command_log``: file for stored commands
``dirs_log``: file for associated work directories
``stack_level``: index in the stack for the command to be retrieved
"""
self.resume_command = a.get_command_message(resume_args)
self.command = get_log_reversed(command_log, stack_level)
self.output_dir = get_log_reversed(dirs_log, stack_level)
self.defaults_file = os.path.join(self.output_dir, DEFAULTS_FILE)
self.args = shlex.split(self.command)[1:]
if not ("--output" in self.args or "--output-dir" in self.args):
current_directory = "%s%s" % (os.getcwd(), os.sep)
if self.output_dir.startswith(current_directory):
self.output_dir = self.output_dir.replace(current_directory,
"", 1)
self.args.append("--output-dir")
self.args.append(self.output_dir)
def __init__(self, resume_args, command_log, dirs_log, stack_level=0):
"""Constructor that extracts the command from the file
``command_log``: file for stored commands
``dirs_log``: file for associated work directories
``stack_level``: index in the stack for the command to be retrieved
"""
self.resume_command = a.get_command_message(resume_args)
self.command = get_log_reversed(command_log, stack_level)
self.output_dir = get_log_reversed(dirs_log, stack_level)
self.defaults_file = os.path.join(self.output_dir, DEFAULTS_FILE)
self.args = [arg.decode(u.SYSTEM_ENCODING) for arg in
shlex.split(self.command.encode(u.SYSTEM_ENCODING))[1:]]
if not ("--output" in self.args or "--output-dir" in self.args):
current_directory = u"%s%s" % (os.getcwd(), os.sep)
if self.output_dir.startswith(current_directory):
self.output_dir = self.output_dir.replace(current_directory,
"", 1)
self.args.append("--output-dir")
self.args.append(self.output_dir)
def main(args=sys.argv[1:]):
"""Main process
"""
(flags, train_stdin, test_stdin) = a.get_flags(args)
# If --clear-logs the log files are cleared
if "--clear-logs" in args:
for log_file in LOG_FILES:
try:
open(log_file, 'w', 0).close()
except IOError:
pass
message = a.get_command_message(args)
# Resume calls are not logged
if not "--resume" in args:
with open(COMMAND_LOG, "a", 0) as command_log:
command_log.write(message)
resume = False
user_defaults = get_user_defaults()
parser = create_parser(defaults=get_user_defaults(),
constants={'NOW': a.NOW,
'MAX_MODELS': MAX_MODELS,
'PLURALITY': PLURALITY})
# Parses command line arguments.
command_args = a.parse_and_check(parser, args, train_stdin, test_stdin)
default_output = ('evaluation' if command_args.evaluate
else 'predictions.csv')
if command_args.resume:
# Restore the args of the call to resume from the command log file
debug = command_args.debug
command = u.get_log_reversed(COMMAND_LOG,
command_args.stack_level)
args = shlex.split(command)[1:]
try:
position = args.index("--train")
train_stdin = (position == (len(args) - 1) or
args[position + 1].startswith("--"))
except ValueError:
pass
try:
position = args.index("--test")
test_stdin = (position == (len(args) - 1) or
args[position + 1].startswith("--"))
except ValueError:
pass
output_dir = u.get_log_reversed(DIRS_LOG,
command_args.stack_level)
defaults_file = "%s%s%s" % (output_dir, os.sep, DEFAULTS_FILE)
user_defaults = get_user_defaults(defaults_file)
parser = create_parser(defaults=user_defaults,
constants={'NOW': a.NOW,
'MAX_MODELS': MAX_MODELS,
'PLURALITY': PLURALITY})
# Parses resumed arguments.
command_args = a.parse_and_check(parser, args, train_stdin, test_stdin)
if command_args.predictions is None:
command_args.predictions = ("%s%s%s" %
(output_dir, os.sep,
default_output))
# Logs the issued command and the resumed command
session_file = "%s%s%s" % (output_dir, os.sep, SESSIONS_LOG)
u.log_message(message, log_file=session_file)
message = "\nResuming command:\n%s\n\n" % command
u.log_message(message, log_file=session_file, console=True)
try:
defaults_handler = open(defaults_file, 'r')
contents = defaults_handler.read()
message = "\nUsing the following defaults:\n%s\n\n" % contents
u.log_message(message, log_file=session_file, console=True)
defaults_handler.close()
except IOError:
pass
resume = True
else:
if command_args.output_dir is None:
command_args.output_dir = a.NOW
if command_args.predictions is None:
command_args.predictions = ("%s%s%s" %
(command_args.output_dir, os.sep,
default_output))
if len(os.path.dirname(command_args.predictions).strip()) == 0:
command_args.predictions = ("%s%s%s" %
(command_args.output_dir, os.sep,
command_args.predictions))
directory = u.check_dir(command_args.predictions)
session_file = "%s%s%s" % (directory, os.sep, SESSIONS_LOG)
u.log_message(message + "\n", log_file=session_file)
try:
defaults_file = open(DEFAULTS_FILE, 'r')
contents = defaults_file.read()
defaults_file.close()
defaults_copy = open("%s%s%s" % (directory, os.sep, DEFAULTS_FILE),
'w', 0)
defaults_copy.write(contents)
defaults_copy.close()
except IOError:
pass
with open(DIRS_LOG, "a", 0) as directory_log:
directory_log.write("%s\n" % os.path.abspath(directory))
# Creates the corresponding api instance
if resume and debug:
command_args.debug = True
api = a.get_api_instance(command_args, u.check_dir(session_file))
# Selects the action to perform: delete or create resources
if command_args.delete:
delete_resources(command_args, api)
elif (command_args.training_set or has_test(command_args)
or command_args.source or command_args.dataset
or command_args.datasets or command_args.votes_dirs):
output_args = a.get_output_args(api, train_stdin, test_stdin,
command_args, resume)
a.transform_args(command_args, flags, api, user_defaults)
compute_output(**output_args)
u.log_message("_" * 80 + "\n", log_file=session_file)
def main(args=sys.argv[1:]):
"""Main process
"""
(flags, train_stdin, test_stdin) = a.get_flags(args)
# If --clear-logs the log files are cleared
if "--clear-logs" in args:
for log_file in LOG_FILES:
try:
open(log_file, 'w', 0).close()
except IOError:
pass
message = a.get_command_message(args)
# Resume calls are not logged
if not "--resume" in args:
with open(COMMAND_LOG, "a", 0) as command_log:
command_log.write(message)
resume = False
user_defaults = get_user_defaults()
parser = create_parser(defaults=get_user_defaults(),
constants={
'NOW': a.NOW,
'MAX_MODELS': MAX_MODELS,
'PLURALITY': PLURALITY
})
# Parses command line arguments.
command_args = a.parse_and_check(parser, args, train_stdin, test_stdin)
default_output = ('evaluation'
if command_args.evaluate else 'predictions.csv')
if command_args.resume:
# Restore the args of the call to resume from the command log file
debug = command_args.debug
command = u.get_log_reversed(COMMAND_LOG, command_args.stack_level)
args = shlex.split(command)[1:]
try:
position = args.index("--train")
train_stdin = (position == (len(args) - 1)
or args[position + 1].startswith("--"))
except ValueError:
pass
try:
position = args.index("--test")
test_stdin = (position == (len(args) - 1)
or args[position + 1].startswith("--"))
except ValueError:
pass
output_dir = u.get_log_reversed(DIRS_LOG, command_args.stack_level)
defaults_file = "%s%s%s" % (output_dir, os.sep, DEFAULTS_FILE)
user_defaults = get_user_defaults(defaults_file)
parser = create_parser(defaults=user_defaults,
constants={
'NOW': a.NOW,
'MAX_MODELS': MAX_MODELS,
'PLURALITY': PLURALITY
})
# Parses resumed arguments.
command_args = a.parse_and_check(parser, args, train_stdin, test_stdin)
if command_args.predictions is None:
command_args.predictions = ("%s%s%s" %
(output_dir, os.sep, default_output))
# Logs the issued command and the resumed command
session_file = "%s%s%s" % (output_dir, os.sep, SESSIONS_LOG)
u.log_message(message, log_file=session_file)
message = "\nResuming command:\n%s\n\n" % command
u.log_message(message, log_file=session_file, console=True)
try:
defaults_handler = open(defaults_file, 'r')
contents = defaults_handler.read()
message = "\nUsing the following defaults:\n%s\n\n" % contents
u.log_message(message, log_file=session_file, console=True)
defaults_handler.close()
except IOError:
pass
resume = True
else:
if command_args.output_dir is None:
command_args.output_dir = a.NOW
if command_args.predictions is None:
command_args.predictions = (
"%s%s%s" % (command_args.output_dir, os.sep, default_output))
if len(os.path.dirname(command_args.predictions).strip()) == 0:
command_args.predictions = (
"%s%s%s" %
(command_args.output_dir, os.sep, command_args.predictions))
directory = u.check_dir(command_args.predictions)
session_file = "%s%s%s" % (directory, os.sep, SESSIONS_LOG)
u.log_message(message + "\n", log_file=session_file)
try:
defaults_file = open(DEFAULTS_FILE, 'r')
contents = defaults_file.read()
defaults_file.close()
defaults_copy = open("%s%s%s" % (directory, os.sep, DEFAULTS_FILE),
'w', 0)
defaults_copy.write(contents)
defaults_copy.close()
except IOError:
pass
with open(DIRS_LOG, "a", 0) as directory_log:
directory_log.write("%s\n" % os.path.abspath(directory))
# Creates the corresponding api instance
if resume and debug:
command_args.debug = True
api = a.get_api_instance(command_args, u.check_dir(session_file))
# Selects the action to perform: delete or create resources
if command_args.delete:
delete_resources(command_args, api)
elif (command_args.training_set or has_test(command_args)
or command_args.source or command_args.dataset
or command_args.datasets or command_args.votes_dirs):
output_args = a.get_output_args(api, train_stdin, test_stdin,
command_args, resume)
a.transform_args(command_args, flags, api, user_defaults)
compute_output(**output_args)
u.log_message("_" * 80 + "\n", log_file=session_file)