def cluster_dispatcher(args=sys.argv[1:]):
"""Parses command line and calls the different processing functions
"""
# If --clear-logs the log files are cleared
if "--clear-logs" in args:
clear_log_files(LOG_FILES)
command = command_handling(args, COMMAND_LOG)
# Parses command line arguments.
command_args = a.parse_and_check(command)
resume = command_args.resume
if command_args.resume:
# Keep the debug option if set
debug = command_args.debug
# Restore the args of the call to resume from the command log file
stored_command = StoredCommand(args, COMMAND_LOG, DIRS_LOG)
command = Command(None, stored_command=stored_command)
# Logs the issued command and the resumed command
session_file = os.path.join(stored_command.output_dir, SESSIONS_LOG)
stored_command.log_command(session_file=session_file)
# Parses resumed arguments.
command_args = a.parse_and_check(command)
if command_args.predictions is None:
command_args.predictions = os.path.join(stored_command.output_dir, DEFAULT_OUTPUT)
else:
if command_args.output_dir is None:
command_args.output_dir = a.NOW
if command_args.predictions is None:
command_args.predictions = os.path.join(command_args.output_dir, DEFAULT_OUTPUT)
if len(os.path.dirname(command_args.predictions).strip()) == 0:
command_args.predictions = os.path.join(command_args.output_dir, command_args.predictions)
directory = u.check_dir(command_args.predictions)
session_file = os.path.join(directory, SESSIONS_LOG)
u.log_message(command.command + "\n", log_file=session_file)
try:
defaults_file = open(DEFAULTS_FILE, "r")
contents = defaults_file.read()
defaults_file.close()
defaults_copy = open(os.path.join(directory, DEFAULTS_FILE), "w", 0)
defaults_copy.write(contents)
defaults_copy.close()
except IOError:
pass
u.sys_log_message(u"%s\n" % os.path.abspath(directory), log_file=DIRS_LOG)
# Creates the corresponding api instance
if resume and debug:
command_args.debug = True
api = a.get_api_instance(command_args, u.check_dir(session_file))
# Selects the action to perform
if has_train(command_args) or has_test(command_args) or command_args.cluster_datasets is not None:
output_args = a.get_output_args(api, command_args, resume)
a.transform_args(command_args, command.flags, api, command.user_defaults)
compute_output(**output_args)
u.log_message("_" * 80 + "\n", log_file=session_file)
def delete_dispatcher(args=sys.argv[1:]):
"""Parses command line and calls the different processing functions
"""
command = command_handling(args, COMMAND_LOG)
# Parses command line arguments.
command_args = a.parse_and_check(command)
resume = command_args.resume
if command_args.resume:
# Keep the debug option if set
debug = command_args.debug
# Restore the args of the call to resume from the command log file
stored_command = StoredCommand(args, COMMAND_LOG, DIRS_LOG)
command = Command(None, stored_command=stored_command)
# Logs the issued command and the resumed command
session_file = os.path.join(stored_command.output_dir, SESSIONS_LOG)
stored_command.log_command(session_file=session_file)
# Parses resumed arguments.
command_args = a.parse_and_check(command)
else:
if command_args.output_dir is None:
command_args.output_dir = a.NOW
directory = u.check_dir(os.path.join(command_args.output_dir, "tmp"))
session_file = os.path.join(directory, SESSIONS_LOG)
u.log_message(command.command + "\n", log_file=session_file)
try:
defaults_file = open(DEFAULTS_FILE, 'r')
contents = defaults_file.read()
defaults_file.close()
defaults_copy = open(os.path.join(directory, DEFAULTS_FILE),
'w', 0)
defaults_copy.write(contents)
defaults_copy.close()
except IOError:
pass
u.sys_log_message(u"%s\n" % os.path.abspath(directory),
log_file=DIRS_LOG)
# If --clear-logs the log files are cleared
if "--clear-logs" in args:
clear_log_files(LOG_FILES)
# Creates the corresponding api instance
if resume and debug:
command_args.debug = True
api = a.get_api_instance(command_args, u.check_dir(session_file))
delete_resources(command_args, api)
u.log_message("_" * 80 + "\n", log_file=session_file)
def delete_dispatcher(args=sys.argv[1:]):
"""Parses command line and calls the different processing functions
"""
command = command_handling(args, COMMAND_LOG)
# Parses command line arguments.
command_args = a.parse_and_check(command)
resume = command_args.resume
if command_args.resume:
# Keep the debug option if set
debug = command_args.debug
# Restore the args of the call to resume from the command log file
stored_command = StoredCommand(args, COMMAND_LOG, DIRS_LOG)
command = Command(None, stored_command=stored_command)
# Logs the issued command and the resumed command
session_file = os.path.join(stored_command.output_dir, SESSIONS_LOG)
stored_command.log_command(session_file=session_file)
# Parses resumed arguments.
command_args = a.parse_and_check(command)
else:
if command_args.output_dir is None:
command_args.output_dir = a.NOW
directory = u.check_dir(os.path.join(command_args.output_dir, "tmp"))
session_file = os.path.join(directory, SESSIONS_LOG)
u.log_message(command.command + "\n", log_file=session_file)
try:
defaults_file = open(DEFAULTS_FILE, 'r')
contents = defaults_file.read()
defaults_file.close()
defaults_copy = open(os.path.join(directory, DEFAULTS_FILE), 'w',
0)
defaults_copy.write(contents)
defaults_copy.close()
except IOError:
pass
u.sys_log_message(u"%s\n" % os.path.abspath(directory),
log_file=DIRS_LOG)
# If --clear-logs the log files are cleared
if "--clear-logs" in args:
clear_log_files(LOG_FILES)
# Creates the corresponding api instance
if resume and debug:
command_args.debug = True
api = a.get_api_instance(command_args, u.check_dir(session_file))
delete_resources(command_args, api)
u.log_message("_" * 80 + "\n", log_file=session_file)
def report_dispatcher(args=sys.argv[1:]):
"""Parses command line and calls the different report functions
"""
command = command_handling(args, COMMAND_LOG)
# Parses command line arguments.
command_args = a.parse_and_check(command)
port = DEFAULT_PORT if not command_args.port else command_args.port
report_url = get_report_url(command_args)
if not command_args.no_server:
absolute_report_url = "http://%s:%s/%s" % (DEFAULT_HOST, port,
report_url)
current_directory = os.getcwd()
os.chdir(os.path.join(HOME, SERVER_DIRECTORY))
httpd = None
try:
httpd = StoppableHTTPServer(\
(DEFAULT_HOST, port),
SimpleHTTPServer.SimpleHTTPRequestHandler)
thread.start_new_thread(httpd.serve, ())
except socket.error, exc:
print exc
# Open URL in new browser window
webbrowser.open_new(absolute_report_url)
# opens in default browser
if httpd:
raw_input("*********************************\n"
"Press <RETURN> to stop the server\n"
"*********************************\n")
os.chdir(current_directory)
if httpd:
httpd.stop()
def report_dispatcher(args=sys.argv[1:]):
"""Parses command line and calls the different report functions
"""
command = command_handling(args, COMMAND_LOG)
# Parses command line arguments.
command_args = a.parse_and_check(command)
port = DEFAULT_PORT if not command_args.port else command_args.port
report_url = get_report_url(command_args)
if not command_args.no_server:
absolute_report_url = "http://%s:%s/%s" % (DEFAULT_HOST, port, report_url)
current_directory = os.getcwd()
os.chdir(os.path.join(HOME, SERVER_DIRECTORY))
httpd = None
try:
httpd = StoppableHTTPServer((DEFAULT_HOST, port), SimpleHTTPServer.SimpleHTTPRequestHandler)
thread.start_new_thread(httpd.serve, ())
except socket.error, exc:
print exc
# Open URL in new browser window
webbrowser.open_new(absolute_report_url)
# opens in default browser
if httpd:
raw_input(
"*********************************\n"
"Press <RETURN> to stop the server\n"
"*********************************\n"
)
os.chdir(current_directory)
if httpd:
httpd.stop()
def main_dispatcher(args=sys.argv[1:]):
"""Parses command line and calls the different processing functions
"""
# If --clear-logs the log files are cleared
if "--clear-logs" in args:
clear_log_files(LOG_FILES)
command = command_handling(args, COMMAND_LOG)
# Parses command line arguments.
command_args = a.parse_and_check(command)
default_output = ('evaluation' if command_args.evaluate
else 'predictions.csv')
resume = command_args.resume
if command_args.resume:
command_args, session_file, output_dir = get_stored_command(
args, command_args.debug, command_log=COMMAND_LOG,
dirs_log=DIRS_LOG, sessions_log=SESSIONS_LOG)
default_output = ('evaluation' if command_args.evaluate
else 'predictions.csv')
if command_args.predictions is None:
command_args.predictions = os.path.join(output_dir,
default_output)
else:
if command_args.output_dir is None:
command_args.output_dir = a.NOW
if command_args.predictions is None:
command_args.predictions = os.path.join(command_args.output_dir,
default_output)
if len(os.path.dirname(command_args.predictions).strip()) == 0:
command_args.predictions = os.path.join(command_args.output_dir,
command_args.predictions)
directory = u.check_dir(command_args.predictions)
session_file = os.path.join(directory, SESSIONS_LOG)
u.log_message(command.command + "\n", log_file=session_file)
try:
defaults_file = open(DEFAULTS_FILE, 'r')
contents = defaults_file.read()
defaults_file.close()
defaults_copy = open(os.path.join(directory, DEFAULTS_FILE),
'w', 0)
defaults_copy.write(contents)
defaults_copy.close()
except IOError:
pass
u.sys_log_message(u"%s\n" % os.path.abspath(directory),
log_file=DIRS_LOG)
# Creates the corresponding api instance
api = a.get_api_instance(command_args, u.check_dir(session_file))
if (a.has_train(command_args) or a.has_test(command_args)
or command_args.votes_dirs):
output_args = a.get_output_args(api, command_args, resume)
a.transform_args(command_args, command.flags, api,
command.user_defaults)
compute_output(**output_args)
u.log_message("_" * 80 + "\n", log_file=session_file)
def delete_dispatcher(args=sys.argv[1:]):
"""Parses command line and calls the different processing functions
"""
command = command_handling(args, COMMAND_LOG)
# Parses command line arguments.
command_args = a.parse_and_check(command)
if command_args.resume:
command_args, session_file, _ = get_stored_command(
args, command_args.debug, command_log=COMMAND_LOG,
dirs_log=DIRS_LOG, sessions_log=SESSIONS_LOG)
else:
if command_args.output_dir is None:
command_args.output_dir = a.NOW
directory = u.check_dir(os.path.join(command_args.output_dir, "tmp"))
session_file = os.path.join(directory, SESSIONS_LOG)
u.log_message(command.command + "\n", log_file=session_file)
try:
shutil.copy(DEFAULTS_FILE, os.path.join(directory, DEFAULTS_FILE))
except IOError:
pass
u.sys_log_message(u"%s\n" % os.path.abspath(directory),
log_file=DIRS_LOG)
# If --clear-logs the log files are cleared
if "--clear-logs" in args:
clear_log_files(LOG_FILES)
# Creates the corresponding api instance
api = a.get_api_instance(command_args, u.check_dir(session_file))
delete_resources(command_args, api)
u.log_message("_" * 80 + "\n", log_file=session_file)
def delete_dispatcher(args=sys.argv[1:]):
"""Parses command line and calls the different processing functions
"""
command = command_handling(args, COMMAND_LOG)
# Parses command line arguments.
command_args = a.parse_and_check(command)
if command_args.resume:
command_args, session_file, _ = get_stored_command(
args, command_args.debug, command_log=COMMAND_LOG,
dirs_log=DIRS_LOG, sessions_log=SESSIONS_LOG)
else:
if command_args.output_dir is None:
command_args.output_dir = a.NOW
directory = u.check_dir(os.path.join(command_args.output_dir, "tmp"))
session_file = os.path.join(directory, SESSIONS_LOG)
u.log_message(command.command + "\n", log_file=session_file)
try:
shutil.copy(DEFAULTS_FILE, os.path.join(directory, DEFAULTS_FILE))
except IOError:
pass
u.sys_log_message(u"%s\n" % os.path.abspath(directory),
log_file=DIRS_LOG)
# If --clear-logs the log files are cleared
if "--clear-logs" in args:
clear_log_files(LOG_FILES)
# Creates the corresponding api instance
api = a.get_api_instance(command_args, u.check_dir(session_file))
delete_resources(command_args, api)
u.log_message("_" * 80 + "\n", log_file=session_file)
def cluster_dispatcher(args=sys.argv[1:]):
"""Parses command line and calls the different processing functions
"""
# If --clear-logs the log files are cleared
if "--clear-logs" in args:
clear_log_files(LOG_FILES)
command = command_handling(args, COMMAND_LOG)
# Parses command line arguments.
command_args = a.parse_and_check(command)
resume = command_args.resume
if command_args.resume:
command_args, session_file, output_dir = get_stored_command(
args,
command_args.debug,
command_log=COMMAND_LOG,
dirs_log=DIRS_LOG,
sessions_log=SESSIONS_LOG)
if command_args.predictions is None:
command_args.predictions = os.path.join(output_dir, DEFAULT_OUTPUT)
else:
if command_args.output_dir is None:
command_args.output_dir = a.NOW
if command_args.predictions is None:
command_args.predictions = os.path.join(command_args.output_dir,
DEFAULT_OUTPUT)
if len(os.path.dirname(command_args.predictions).strip()) == 0:
command_args.predictions = os.path.join(command_args.output_dir,
command_args.predictions)
directory = u.check_dir(command_args.predictions)
session_file = os.path.join(directory, SESSIONS_LOG)
u.log_message(command.command + "\n", log_file=session_file)
try:
defaults_file = open(DEFAULTS_FILE, 'r')
contents = defaults_file.read()
defaults_file.close()
defaults_copy = open(os.path.join(directory, DEFAULTS_FILE), 'w',
0)
defaults_copy.write(contents)
defaults_copy.close()
except IOError:
pass
u.sys_log_message(u"%s\n" % os.path.abspath(directory),
log_file=DIRS_LOG)
# Creates the corresponding api instance
api = a.get_api_instance(command_args, u.check_dir(session_file))
# Selects the action to perform
if (a.has_train(command_args) or a.has_test(command_args)
or command_args.cluster_datasets is not None):
output_args = a.get_output_args(api, command_args, resume)
a.transform_args(command_args, command.flags, api,
command.user_defaults)
compute_output(**output_args)
u.log_message("_" * 80 + "\n", log_file=session_file)
def logistic_regression_dispatcher(args=sys.argv[1:]):
"""Parses command line and calls the different processing functions
"""
# If --clear-logs the log files are cleared
if "--clear-logs" in args:
clear_log_files(LOG_FILES)
command = command_handling(args, COMMAND_LOG)
# Parses command line arguments.
command_args = a.parse_and_check(command)
default_output = ('evaluation'
if command_args.evaluate else 'predictions.csv')
resume = command_args.resume
if command_args.resume:
command_args, session_file, output_dir = get_stored_command(
args,
command_args.debug,
command_log=COMMAND_LOG,
dirs_log=DIRS_LOG,
sessions_log=SESSIONS_LOG)
default_output = ('evaluation'
if command_args.evaluate else 'predictions.csv')
if command_args.predictions is None:
command_args.predictions = os.path.join(output_dir, default_output)
else:
if command_args.output_dir is None:
command_args.output_dir = a.NOW
if command_args.predictions is None:
command_args.predictions = os.path.join(command_args.output_dir,
default_output)
if len(os.path.dirname(command_args.predictions).strip()) == 0:
command_args.predictions = os.path.join(command_args.output_dir,
command_args.predictions)
directory = u.check_dir(command_args.predictions)
session_file = os.path.join(directory, SESSIONS_LOG)
u.log_message(command.command + "\n", log_file=session_file)
try:
shutil.copy(DEFAULTS_FILE, os.path.join(directory, DEFAULTS_FILE))
except IOError:
pass
u.sys_log_message(u"%s\n" % os.path.abspath(directory),
log_file=DIRS_LOG)
# Creates the corresponding api instance
api = a.get_api_instance(command_args, u.check_dir(session_file))
# Selects the action to perform
if (a.has_train(command_args) or a.has_test(command_args)
or command_args.export_fields):
output_args = a.get_output_args(api, command_args, resume)
a.transform_args(command_args, command.flags, api,
command.user_defaults)
compute_output(**output_args)
u.log_message("_" * 80 + "\n", log_file=session_file)
def cluster_dispatcher(args=sys.argv[1:]):
"""Parses command line and calls the different processing functions
"""
# If --clear-logs the log files are cleared
if "--clear-logs" in args:
clear_log_files(LOG_FILES)
command = command_handling(args, COMMAND_LOG)
# Parses command line arguments.
command_args = a.parse_and_check(command)
resume = command_args.resume
if command_args.resume:
command_args, session_file, output_dir = get_stored_command(
args, command_args.debug, command_log=COMMAND_LOG,
dirs_log=DIRS_LOG, sessions_log=SESSIONS_LOG)
if command_args.predictions is None:
command_args.predictions = os.path.join(output_dir,
DEFAULT_OUTPUT)
else:
if command_args.output_dir is None:
command_args.output_dir = a.NOW
if command_args.predictions is None:
command_args.predictions = os.path.join(command_args.output_dir,
DEFAULT_OUTPUT)
if len(os.path.dirname(command_args.predictions).strip()) == 0:
command_args.predictions = os.path.join(command_args.output_dir,
command_args.predictions)
directory = u.check_dir(command_args.predictions)
session_file = os.path.join(directory, SESSIONS_LOG)
u.log_message(command.command + "\n", log_file=session_file)
try:
shutil.copy(DEFAULTS_FILE, os.path.join(directory, DEFAULTS_FILE))
except IOError:
pass
u.sys_log_message(u"%s\n" % os.path.abspath(directory),
log_file=DIRS_LOG)
# Creates the corresponding api instance
api = a.get_api_instance(command_args, u.check_dir(session_file))
# Selects the action to perform
if (a.has_train(command_args) or a.has_test(command_args)
or command_args.cluster_datasets is not None
or command_args.export_fields is not None):
output_args = a.get_output_args(api, command_args, resume)
a.transform_args(command_args, command.flags, api,
command.user_defaults)
compute_output(**output_args)
u.log_message("_" * 80 + "\n", log_file=session_file)
def get_cmd_context(args, settings):
"""Parses the args array to create an args object storing the defaults and
user-given values. It also sets the output directory and the log files.
"""
command = command_handling(args, settings['command_log'])
# Parses command line arguments.
command_args = a.parse_and_check(command)
resume = command_args.resume
if command_args.resume:
command_args, session_file, output_dir = get_stored_command(
args,
command_args.debug,
command_log=settings['command_log'],
dirs_log=settings["dirs_log"],
sessions_log=settings['sessions_log'])
if settings.get('default_output') is None:
settings['default_output'] = "tmp.txt"
if not hasattr(command_args, "output") or command_args.output is None:
command_args.output = os.path.join(output_dir,
settings['default_output'])
else:
if hasattr(command_args, "output") and \
command_args.output is not None:
command_args.output_dir = u.check_dir(command_args.output)
if command_args.output_dir is None:
command_args.output_dir = a.NOW
if settings.get('default_output') is None:
settings['default_output'] = "tmp.txt"
if not hasattr(command_args, "output") or command_args.output is None:
command_args.output = os.path.join(command_args.output_dir,
settings['default_output'])
if not os.path.dirname(command_args.output).strip():
command_args.output = os.path.join(command_args.output_dir,
command_args.output)
directory = u.check_dir(command_args.output)
session_file = os.path.join(directory, settings['sessions_log'])
u.log_message(command.command + "\n", log_file=session_file)
if settings.get('defaults_file') is not None:
try:
shutil.copy(settings['defaults_file'],
os.path.join(directory, settings['defaults_file']))
except IOError:
pass
u.sys_log_message(u"%s\n" % os.path.abspath(directory),
log_file=settings['dirs_log'])
return command_args, command, session_file, resume
def execute_dispatcher(args=sys.argv[1:]):
"""Parses command line and calls the different processing functions
"""
# If --clear-logs the log files are cleared
if "--clear-logs" in args:
clear_log_files(LOG_FILES)
command = command_handling(args, COMMAND_LOG)
default_output = 'whizzml_results'
# Parses command line arguments.
command_args = a.parse_and_check(command)
resume = command_args.resume
if command_args.resume:
command_args, session_file, output_dir = get_stored_command(
args,
command_args.debug,
command_log=COMMAND_LOG,
dirs_log=DIRS_LOG,
sessions_log=SESSIONS_LOG)
if command_args.output is None:
command_args.output = os.path.join(output_dir, default_output)
else:
if command_args.output_dir is None:
command_args.output_dir = a.NOW
if command_args.output is None:
command_args.output = os.path.join(command_args.output_dir,
default_output)
if len(os.path.dirname(command_args.output).strip()) == 0:
command_args.output = os.path.join(command_args.output_dir,
command_args.output)
directory = u.check_dir(command_args.output)
session_file = os.path.join(directory, SESSIONS_LOG)
u.log_message(command.command + "\n", log_file=session_file)
try:
shutil.copy(DEFAULTS_FILE, os.path.join(directory, DEFAULTS_FILE))
except IOError:
pass
u.sys_log_message(u"%s\n" % os.path.abspath(directory),
log_file=DIRS_LOG)
# Creates the corresponding api instance
api = a.get_api_instance(command_args, u.check_dir(session_file))
_ = a.get_output_args(api, command_args, resume)
a.transform_args(command_args, command.flags, api, command.user_defaults)
execute_whizzml(command_args, api, session_file)
u.log_message("_" * 80 + "\n", log_file=session_file)
def get_cmd_context(args, settings):
"""Parses the args array to create an args object storing the defaults and
user-given values. It also sets the output directory and the log files.
"""
command = command_handling(args, settings['command_log'])
# Parses command line arguments.
command_args = a.parse_and_check(command)
resume = command_args.resume
if command_args.resume:
command_args, session_file, output_dir = get_stored_command(
args, command_args.debug, command_log=settings['command_log'],
dirs_log=settings["dirs_log"],
sessions_log=settings['sessions_log'])
if settings.get('default_output') is None:
settings['default_output'] = "tmp.txt"
if not hasattr(command_args, "output") or command_args.output is None:
command_args.output = os.path.join(output_dir,
settings['default_output'])
else:
if hasattr(command_args, "output") and \
command_args.output is not None:
command_args.output_dir = u.check_dir(command_args.output)
if command_args.output_dir is None:
command_args.output_dir = a.NOW
if settings.get('default_output') is None:
settings['default_output'] = "tmp.txt"
if not hasattr(command_args, "output") or command_args.output is None:
command_args.output = os.path.join(command_args.output_dir,
settings['default_output'])
if len(os.path.dirname(command_args.output).strip()) == 0:
command_args.output = os.path.join(command_args.output_dir,
command_args.output)
directory = u.check_dir(command_args.output)
session_file = os.path.join(directory, settings['sessions_log'])
u.log_message(command.command + "\n", log_file=session_file)
if settings.get('defaults_file') is not None:
try:
shutil.copy(settings['defaults_file'],
os.path.join(directory, settings['defaults_file']))
except IOError:
pass
u.sys_log_message(u"%s\n" % os.path.abspath(directory),
log_file=settings['dirs_log'])
return command_args, command, session_file, resume
def execute_dispatcher(args=sys.argv[1:]):
"""Parses command line and calls the different processing functions
"""
# If --clear-logs the log files are cleared
if "--clear-logs" in args:
clear_log_files(LOG_FILES)
command = command_handling(args, COMMAND_LOG)
default_output = 'whizzml_results'
# Parses command line arguments.
command_args = a.parse_and_check(command)
resume = command_args.resume
if command_args.resume:
command_args, session_file, output_dir = get_stored_command(
args, command_args.debug, command_log=COMMAND_LOG,
dirs_log=DIRS_LOG, sessions_log=SESSIONS_LOG)
if command_args.output is None:
command_args.output = os.path.join(output_dir,
default_output)
else:
if command_args.output_dir is None:
command_args.output_dir = a.NOW
if command_args.output is None:
command_args.output = os.path.join(command_args.output_dir,
default_output)
if len(os.path.dirname(command_args.output).strip()) == 0:
command_args.output = os.path.join(command_args.output_dir,
command_args.output)
directory = u.check_dir(command_args.output)
session_file = os.path.join(directory, SESSIONS_LOG)
u.log_message(command.command + "\n", log_file=session_file)
try:
shutil.copy(DEFAULTS_FILE, os.path.join(directory, DEFAULTS_FILE))
except IOError:
pass
u.sys_log_message(u"%s\n" % os.path.abspath(directory),
log_file=DIRS_LOG)
# Creates the corresponding api instance
api = a.get_api_instance(command_args, u.check_dir(session_file))
_ = a.get_output_args(api, command_args, resume)
a.transform_args(command_args, command.flags, api,
command.user_defaults)
execute_whizzml(command_args, api, session_file)
u.log_message("_" * 80 + "\n", log_file=session_file)
def get_stored_command(args, debug=False, command_log=COMMAND_LOG,
dirs_log=DIRS_LOG, sessions_log=SESSIONS_LOG):
"""Restoring the saved command from stack to the arguments object
"""
# Restore the args of the call to resume from the command log file
stored_command = StoredCommand(args, command_log, dirs_log)
command = Command(None, stored_command=stored_command)
# Logs the issued command and the resumed command
session_file = os.path.join(stored_command.output_dir, sessions_log)
stored_command.log_command(session_file=session_file)
# Parses resumed arguments.
command_args = a.parse_and_check(command)
if debug:
# set debug on if it wasn't in the stored command but now is
command_args.debug = True
return command_args, session_file, stored_command.output_dir
def get_stored_command(args, debug=False, command_log=COMMAND_LOG,
dirs_log=DIRS_LOG, sessions_log=SESSIONS_LOG):
"""Restoring the saved command from stack to the arguments object
"""
# Restore the args of the call to resume from the command log file
stored_command = StoredCommand(args, command_log, dirs_log)
command = Command(None, stored_command=stored_command)
# Logs the issued command and the resumed command
session_file = os.path.join(stored_command.output_dir, sessions_log)
stored_command.log_command(session_file=session_file)
# Parses resumed arguments.
command_args = a.parse_and_check(command)
if debug:
# set debug on if it wasn't in the stored command but now is
command_args.debug = True
return command_args, session_file, stored_command.output_dir
def reify_dispatcher(args=sys.argv[1:]):
"""Parses command line and calls the different processing functions
"""
command = command_handling(args, COMMAND_LOG)
# Parses command line arguments.
command_args = a.parse_and_check(command)
if command_args.resume:
command_args, session_file, _ = get_stored_command(
args, command_args.debug, command_log=COMMAND_LOG,
dirs_log=DIRS_LOG, sessions_log=SESSIONS_LOG)
if command_args.output is None:
command_args.output = os.path.join(command_args.output_dir,
DEFAULT_OUTPUT)
else:
if command_args.output_dir is None:
command_args.output_dir = a.NOW
if command_args.output is None:
command_args.output = os.path.join(command_args.output_dir,
DEFAULT_OUTPUT)
if len(os.path.dirname(command_args.output).strip()) == 0:
command_args.output = os.path.join(command_args.output_dir,
command_args.output)
directory = u.check_dir(command_args.output)
command_args.output_dir = directory
session_file = os.path.join(directory, SESSIONS_LOG)
u.log_message(command.command + "\n", log_file=session_file)
directory = u.check_dir(os.path.join(command_args.output_dir, "tmp"))
session_file = os.path.join(directory, SESSIONS_LOG)
u.log_message(command.command + "\n", log_file=session_file)
try:
shutil.copy(DEFAULTS_FILE, os.path.join(directory, DEFAULTS_FILE))
except IOError:
pass
u.sys_log_message(u"%s\n" % os.path.abspath(directory),
log_file=DIRS_LOG)
# If --clear-logs the log files are cleared
if "--clear-logs" in args:
clear_log_files(LOG_FILES)
def logger(message):
"""Partial to log messages according to args.verbosity
"""
u.log_message(u.dated(message), \
log_file=session_file, console=command_args.verbosity)
# Creates the corresponding api instance
api = a.get_api_instance(command_args, u.check_dir(session_file))
message = "Starting reification for %s\n\n" % command_args.resource_id
u.log_message(message, \
log_file=session_file, console=command_args.verbosity)
reify_resources(command_args, api, logger)
message = "\nReification complete. See the results in %s\n\n" % \
command_args.output
u.log_message(message, \
log_file=session_file, console=command_args.verbosity)
u.log_message("_" * 80 + "\n", log_file=session_file)
u.print_generated_files(command_args.output_dir, log_file=session_file,
verbosity=command_args.verbosity)
def project_dispatcher(args=sys.argv[1:]):
"""Parses command line and calls the different processing functions
"""
command = command_handling(args, COMMAND_LOG)
# Parses command line arguments.
command_args = a.parse_and_check(command)
if command_args.resume:
command_args, session_file, _ = get_stored_command(
args,
command_args.debug,
command_log=COMMAND_LOG,
dirs_log=DIRS_LOG,
sessions_log=SESSIONS_LOG)
else:
if command_args.output_dir is None:
command_args.output_dir = a.NOW
directory = u.check_dir("%s/x.txt" % command_args.output_dir)
command_args.output_dir = directory
session_file = os.path.join(directory, SESSIONS_LOG)
u.log_message(command.command + "\n", log_file=session_file)
directory = u.check_dir(os.path.join(command_args.output_dir, "tmp"))
session_file = os.path.join(directory, SESSIONS_LOG)
u.log_message(command.command + "\n", log_file=session_file)
try:
shutil.copy(DEFAULTS_FILE, os.path.join(directory, DEFAULTS_FILE))
except IOError:
pass
u.sys_log_message(u"%s\n" % os.path.abspath(directory),
log_file=DIRS_LOG)
path = u.check_dir("%s/x.txt" % command_args.output_dir)
session_file = u"%s%s%s" % (path, os.sep, SESSIONS_LOG)
# If logging is required set the file for logging
log = None
if command_args.log_file:
u.check_dir(command_args.log_file)
log = command_args.log_file
# If --clear_logs the log files are cleared
clear_log_files([log])
# Creates the corresponding api instance
api = a.get_api_instance(command_args, u.check_dir(session_file))
a.get_output_args(api, command_args, command_args.resume)
a.attribute_args(command_args)
if not command_args.project_id and command_args.name:
command_args.project = command_args.name
if command_args.project:
# create project
pp.project_processing(api,
command_args,
command_args.resume,
session_file=session_file,
path=path,
log=log,
create=True)
if command_args.project_id and (command_args.project_attributes
or command_args.name or command_args.tag
or command_args.description
or command_args.category):
# update project's attributes
pp.update_project(command_args, api, command_args.resume, \
session_file=session_file)
u.log_message("_" * 80 + "\n", log_file=session_file)
u.print_generated_files(command_args.output_dir,
log_file=session_file,
verbosity=command_args.verbosity)
def reify_dispatcher(args=sys.argv[1:]):
"""Parses command line and calls the different processing functions
"""
command = command_handling(args, COMMAND_LOG)
# Parses command line arguments.
command_args = a.parse_and_check(command)
if command_args.resume:
command_args, session_file, _ = get_stored_command(
args,
command_args.debug,
command_log=COMMAND_LOG,
dirs_log=DIRS_LOG,
sessions_log=SESSIONS_LOG)
if command_args.output is None:
command_args.output = os.path.join(command_args.output_dir,
DEFAULT_OUTPUT)
else:
if command_args.output_dir is None:
command_args.output_dir = a.NOW
if command_args.output is None:
command_args.output = os.path.join(command_args.output_dir,
DEFAULT_OUTPUT)
if len(os.path.dirname(command_args.output).strip()) == 0:
command_args.output = os.path.join(command_args.output_dir,
command_args.output)
directory = u.check_dir(command_args.output)
command_args.output_dir = directory
session_file = os.path.join(directory, SESSIONS_LOG)
u.log_message(command.command + "\n", log_file=session_file)
directory = u.check_dir(os.path.join(command_args.output_dir, "tmp"))
session_file = os.path.join(directory, SESSIONS_LOG)
u.log_message(command.command + "\n", log_file=session_file)
try:
shutil.copy(DEFAULTS_FILE, os.path.join(directory, DEFAULTS_FILE))
except IOError:
pass
u.sys_log_message(u"%s\n" % os.path.abspath(directory),
log_file=DIRS_LOG)
# If --clear-logs the log files are cleared
if "--clear-logs" in args:
clear_log_files(LOG_FILES)
def logger(message):
"""Partial to log messages according to args.verbosity
"""
u.log_message(u.dated(message), \
log_file=session_file, console=command_args.verbosity)
# Creates the corresponding api instance
api = a.get_api_instance(command_args, u.check_dir(session_file))
message = "Starting reification for %s\n\n" % command_args.resource_id
u.log_message(message, \
log_file=session_file, console=command_args.verbosity)
reify_resources(command_args, api, logger)
message = "\nReification complete. See the results in %s\n\n" % \
command_args.output
u.log_message(message, \
log_file=session_file, console=command_args.verbosity)
u.log_message("_" * 80 + "\n", log_file=session_file)
u.print_generated_files(command_args.output_dir,
log_file=session_file,
verbosity=command_args.verbosity)
def project_dispatcher(args=sys.argv[1:]):
"""Parses command line and calls the different processing functions
"""
command = command_handling(args, COMMAND_LOG)
# Parses command line arguments.
command_args = a.parse_and_check(command)
if command_args.resume:
command_args, session_file, _ = get_stored_command(
args, command_args.debug, command_log=COMMAND_LOG,
dirs_log=DIRS_LOG, sessions_log=SESSIONS_LOG)
else:
if command_args.output_dir is None:
command_args.output_dir = a.NOW
directory = u.check_dir("%s/x.txt" % command_args.output_dir)
command_args.output_dir = directory
session_file = os.path.join(directory, SESSIONS_LOG)
u.log_message(command.command + "\n", log_file=session_file)
directory = u.check_dir(os.path.join(command_args.output_dir, "tmp"))
session_file = os.path.join(directory, SESSIONS_LOG)
u.log_message(command.command + "\n", log_file=session_file)
try:
shutil.copy(DEFAULTS_FILE, os.path.join(directory, DEFAULTS_FILE))
except IOError:
pass
u.sys_log_message(u"%s\n" % os.path.abspath(directory),
log_file=DIRS_LOG)
path = u.check_dir("%s/x.txt" % command_args.output_dir)
session_file = u"%s%s%s" % (path, os.sep, SESSIONS_LOG)
# If logging is required set the file for logging
log = None
if command_args.log_file:
u.check_dir(command_args.log_file)
log = command_args.log_file
# If --clear_logs the log files are cleared
clear_log_files([log])
# Creates the corresponding api instance
api = a.get_api_instance(command_args, u.check_dir(session_file))
a.get_output_args(api, command_args, command_args.resume)
a.attribute_args(command_args)
if not command_args.project_id and command_args.name:
command_args.project = command_args.name
if command_args.project:
# create project
pp.project_processing(
api, command_args, command_args.resume, session_file=session_file,
path=path, log=log, create=True)
if command_args.project_id and (
command_args.project_attributes or
command_args.name or command_args.tag or command_args.description
or command_args.category):
# update project's attributes
pp.update_project(command_args, api, command_args.resume, \
session_file=session_file)
u.log_message("_" * 80 + "\n", log_file=session_file)
u.print_generated_files(command_args.output_dir, log_file=session_file,
verbosity=command_args.verbosity)
def main(args=sys.argv[1:]):
"""Main process
"""
(flags, train_stdin, test_stdin) = a.get_flags(args)
# If --clear-logs the log files are cleared
if "--clear-logs" in args:
for log_file in LOG_FILES:
try:
open(log_file, 'w', 0).close()
except IOError:
pass
message = a.get_command_message(args)
# Resume calls are not logged
if not "--resume" in args:
with open(COMMAND_LOG, "a", 0) as command_log:
command_log.write(message)
resume = False
user_defaults = get_user_defaults()
parser = create_parser(defaults=get_user_defaults(),
constants={'NOW': a.NOW,
'MAX_MODELS': MAX_MODELS,
'PLURALITY': PLURALITY})
# Parses command line arguments.
command_args = a.parse_and_check(parser, args, train_stdin, test_stdin)
default_output = ('evaluation' if command_args.evaluate
else 'predictions.csv')
if command_args.resume:
# Restore the args of the call to resume from the command log file
debug = command_args.debug
command = u.get_log_reversed(COMMAND_LOG,
command_args.stack_level)
args = shlex.split(command)[1:]
try:
position = args.index("--train")
train_stdin = (position == (len(args) - 1) or
args[position + 1].startswith("--"))
except ValueError:
pass
try:
position = args.index("--test")
test_stdin = (position == (len(args) - 1) or
args[position + 1].startswith("--"))
except ValueError:
pass
output_dir = u.get_log_reversed(DIRS_LOG,
command_args.stack_level)
defaults_file = "%s%s%s" % (output_dir, os.sep, DEFAULTS_FILE)
user_defaults = get_user_defaults(defaults_file)
parser = create_parser(defaults=user_defaults,
constants={'NOW': a.NOW,
'MAX_MODELS': MAX_MODELS,
'PLURALITY': PLURALITY})
# Parses resumed arguments.
command_args = a.parse_and_check(parser, args, train_stdin, test_stdin)
if command_args.predictions is None:
command_args.predictions = ("%s%s%s" %
(output_dir, os.sep,
default_output))
# Logs the issued command and the resumed command
session_file = "%s%s%s" % (output_dir, os.sep, SESSIONS_LOG)
u.log_message(message, log_file=session_file)
message = "\nResuming command:\n%s\n\n" % command
u.log_message(message, log_file=session_file, console=True)
try:
defaults_handler = open(defaults_file, 'r')
contents = defaults_handler.read()
message = "\nUsing the following defaults:\n%s\n\n" % contents
u.log_message(message, log_file=session_file, console=True)
defaults_handler.close()
except IOError:
pass
resume = True
else:
if command_args.output_dir is None:
command_args.output_dir = a.NOW
if command_args.predictions is None:
command_args.predictions = ("%s%s%s" %
(command_args.output_dir, os.sep,
default_output))
if len(os.path.dirname(command_args.predictions).strip()) == 0:
command_args.predictions = ("%s%s%s" %
(command_args.output_dir, os.sep,
command_args.predictions))
directory = u.check_dir(command_args.predictions)
session_file = "%s%s%s" % (directory, os.sep, SESSIONS_LOG)
u.log_message(message + "\n", log_file=session_file)
try:
defaults_file = open(DEFAULTS_FILE, 'r')
contents = defaults_file.read()
defaults_file.close()
defaults_copy = open("%s%s%s" % (directory, os.sep, DEFAULTS_FILE),
'w', 0)
defaults_copy.write(contents)
defaults_copy.close()
except IOError:
pass
with open(DIRS_LOG, "a", 0) as directory_log:
directory_log.write("%s\n" % os.path.abspath(directory))
# Creates the corresponding api instance
if resume and debug:
command_args.debug = True
api = a.get_api_instance(command_args, u.check_dir(session_file))
# Selects the action to perform: delete or create resources
if command_args.delete:
delete_resources(command_args, api)
elif (command_args.training_set or has_test(command_args)
or command_args.source or command_args.dataset
or command_args.datasets or command_args.votes_dirs):
output_args = a.get_output_args(api, train_stdin, test_stdin,
command_args, resume)
a.transform_args(command_args, flags, api, user_defaults)
compute_output(**output_args)
u.log_message("_" * 80 + "\n", log_file=session_file)
def anomaly_dispatcher(args=sys.argv[1:]):
"""Parses command line and calls the different processing functions
"""
# If --clear-logs the log files are cleared
if "--clear-logs" in args:
clear_log_files(LOG_FILES)
command = command_handling(args, COMMAND_LOG)
# Parses command line arguments.
command_args = a.parse_and_check(command)
resume = command_args.resume
if command_args.resume:
# Keep the debug option if set
debug = command_args.debug
# Restore the args of the call to resume from the command log file
stored_command = StoredCommand(args, COMMAND_LOG, DIRS_LOG)
command = Command(None, stored_command=stored_command)
# Logs the issued command and the resumed command
session_file = os.path.join(stored_command.output_dir, SESSIONS_LOG)
stored_command.log_command(session_file=session_file)
# Parses resumed arguments.
command_args = a.parse_and_check(command)
if command_args.predictions is None:
command_args.predictions = os.path.join(stored_command.output_dir,
DEFAULT_OUTPUT)
else:
if command_args.output_dir is None:
command_args.output_dir = a.NOW
if command_args.predictions is None:
command_args.predictions = os.path.join(command_args.output_dir,
DEFAULT_OUTPUT)
if len(os.path.dirname(command_args.predictions).strip()) == 0:
command_args.predictions = os.path.join(command_args.output_dir,
command_args.predictions)
directory = u.check_dir(command_args.predictions)
session_file = os.path.join(directory, SESSIONS_LOG)
u.log_message(command.command + "\n", log_file=session_file)
try:
defaults_file = open(DEFAULTS_FILE, 'r')
contents = defaults_file.read()
defaults_file.close()
defaults_copy = open(os.path.join(directory, DEFAULTS_FILE),
'w', 0)
defaults_copy.write(contents)
defaults_copy.close()
except IOError:
pass
u.sys_log_message(u"%s\n" % os.path.abspath(directory),
log_file=DIRS_LOG)
# Creates the corresponding api instance
if resume and debug:
command_args.debug = True
api = a.get_api_instance(command_args, u.check_dir(session_file))
# Selects the action to perform
if (has_train(command_args) or has_test(command_args)):
output_args = a.get_output_args(api, command_args, resume)
a.transform_args(command_args, command.flags, api,
command.user_defaults)
compute_output(**output_args)
u.log_message("_" * 80 + "\n", log_file=session_file)
def main(args=sys.argv[1:]):
"""Main process
"""
(flags, train_stdin, test_stdin) = a.get_flags(args)
# If --clear-logs the log files are cleared
if "--clear-logs" in args:
for log_file in LOG_FILES:
try:
open(log_file, 'w', 0).close()
except IOError:
pass
message = a.get_command_message(args)
# Resume calls are not logged
if not "--resume" in args:
with open(COMMAND_LOG, "a", 0) as command_log:
command_log.write(message)
resume = False
user_defaults = get_user_defaults()
parser = create_parser(defaults=get_user_defaults(),
constants={
'NOW': a.NOW,
'MAX_MODELS': MAX_MODELS,
'PLURALITY': PLURALITY
})
# Parses command line arguments.
command_args = a.parse_and_check(parser, args, train_stdin, test_stdin)
default_output = ('evaluation'
if command_args.evaluate else 'predictions.csv')
if command_args.resume:
# Restore the args of the call to resume from the command log file
debug = command_args.debug
command = u.get_log_reversed(COMMAND_LOG, command_args.stack_level)
args = shlex.split(command)[1:]
try:
position = args.index("--train")
train_stdin = (position == (len(args) - 1)
or args[position + 1].startswith("--"))
except ValueError:
pass
try:
position = args.index("--test")
test_stdin = (position == (len(args) - 1)
or args[position + 1].startswith("--"))
except ValueError:
pass
output_dir = u.get_log_reversed(DIRS_LOG, command_args.stack_level)
defaults_file = "%s%s%s" % (output_dir, os.sep, DEFAULTS_FILE)
user_defaults = get_user_defaults(defaults_file)
parser = create_parser(defaults=user_defaults,
constants={
'NOW': a.NOW,
'MAX_MODELS': MAX_MODELS,
'PLURALITY': PLURALITY
})
# Parses resumed arguments.
command_args = a.parse_and_check(parser, args, train_stdin, test_stdin)
if command_args.predictions is None:
command_args.predictions = ("%s%s%s" %
(output_dir, os.sep, default_output))
# Logs the issued command and the resumed command
session_file = "%s%s%s" % (output_dir, os.sep, SESSIONS_LOG)
u.log_message(message, log_file=session_file)
message = "\nResuming command:\n%s\n\n" % command
u.log_message(message, log_file=session_file, console=True)
try:
defaults_handler = open(defaults_file, 'r')
contents = defaults_handler.read()
message = "\nUsing the following defaults:\n%s\n\n" % contents
u.log_message(message, log_file=session_file, console=True)
defaults_handler.close()
except IOError:
pass
resume = True
else:
if command_args.output_dir is None:
command_args.output_dir = a.NOW
if command_args.predictions is None:
command_args.predictions = (
"%s%s%s" % (command_args.output_dir, os.sep, default_output))
if len(os.path.dirname(command_args.predictions).strip()) == 0:
command_args.predictions = (
"%s%s%s" %
(command_args.output_dir, os.sep, command_args.predictions))
directory = u.check_dir(command_args.predictions)
session_file = "%s%s%s" % (directory, os.sep, SESSIONS_LOG)
u.log_message(message + "\n", log_file=session_file)
try:
defaults_file = open(DEFAULTS_FILE, 'r')
contents = defaults_file.read()
defaults_file.close()
defaults_copy = open("%s%s%s" % (directory, os.sep, DEFAULTS_FILE),
'w', 0)
defaults_copy.write(contents)
defaults_copy.close()
except IOError:
pass
with open(DIRS_LOG, "a", 0) as directory_log:
directory_log.write("%s\n" % os.path.abspath(directory))
# Creates the corresponding api instance
if resume and debug:
command_args.debug = True
api = a.get_api_instance(command_args, u.check_dir(session_file))
# Selects the action to perform: delete or create resources
if command_args.delete:
delete_resources(command_args, api)
elif (command_args.training_set or has_test(command_args)
or command_args.source or command_args.dataset
or command_args.datasets or command_args.votes_dirs):
output_args = a.get_output_args(api, train_stdin, test_stdin,
command_args, resume)
a.transform_args(command_args, flags, api, user_defaults)
compute_output(**output_args)
u.log_message("_" * 80 + "\n", log_file=session_file)
