def project_dispatcher(args=sys.argv[1:]):
"""Parses command line and calls the different processing functions
"""
command_args, command, api, session_file, resume = get_context(args,
SETTINGS)
path = u.check_dir(command_args.output)
log = None
if command_args.log_file:
u.check_dir(command_args.log_file)
log = command_args.log_file
# If --clear_logs the log files are cleared
clear_log_files([log])
if not command_args.project_id and command_args.name:
command_args.project = command_args.name
if command_args.project:
# create project
pp.project_processing(
api, command_args, command_args.resume, session_file=session_file,
path=path, log=log, create=True)
if command_args.project_id and (
command_args.project_attributes or
command_args.name or command_args.tag or command_args.description
or command_args.category):
# update project's attributes
pp.update_project(command_args, api, command_args.resume, \
session_file=session_file)
u.log_message("_" * 80 + "\n", log_file=session_file)
u.print_generated_files(command_args.output_dir, log_file=session_file,
verbosity=command_args.verbosity)
def project_dispatcher(args=sys.argv[1:]):
"""Parses command line and calls the different processing functions
"""
command_args, command, api, session_file, resume = get_context(args,
SETTINGS)
path = u.check_dir(command_args.output)
log = None
if command_args.log_file:
u.check_dir(command_args.log_file)
log = command_args.log_file
# If --clear_logs the log files are cleared
clear_log_files([log])
if not command_args.project_id and command_args.name:
command_args.project = command_args.name
if command_args.project:
# create project
pp.project_processing(
api, command_args, command_args.resume, session_file=session_file,
path=path, log=log, create=True)
if command_args.project_id and (
command_args.project_attributes or
command_args.name or command_args.tag or command_args.description
or command_args.category):
# update project's attributes
pp.update_project(command_args, api, command_args.resume, \
session_file=session_file)
u.log_message("_" * 80 + "\n", log_file=session_file)
u.print_generated_files(command_args.output_dir, log_file=session_file,
verbosity=command_args.verbosity)
def execution_processing(api, args,
session_file=None, path=None, log=None):
"""Creating or retrieving an execution
"""
execution = None
resume = args.resume
if args.script or args.scripts:
# If resuming, try to extract args.execution form log files
if resume:
message = u.dated("Execution not found. Resuming.\n")
resume, args.execution = c.checkpoint(
c.is_execution_created, path, debug=args.debug,
message=message,
log_file=session_file, console=args.verbosity)
if not resume:
args.resume = resume
# Check if there's a created project for it
args.project_id = pp.project_processing(
api, args, resume, session_file=session_file, path=path,
log=log)
execution_args = r.set_execution_args(args)
execution = r.create_execution(execution_args, args, api,
path, session_file, log)
# If a source is provided either through the command line or in resume
# steps, we use it.
elif args.execution:
execution = bigml.api.get_execution_id(args.execution)
return execution
def library_processing(api, args,
session_file=None, path=None, log=None):
"""Creating or retrieving a library
"""
library = None
resume = args.resume
if args.code_file or args.code:
# If resuming, try to extract args.library form log files
if resume:
message = u.dated("Library not found. Resuming.\n")
resume, library = c.checkpoint(
c.is_library_created, path, debug=args.debug, message=message,
log_file=session_file, console=args.verbosity)
if not resume:
args.resume = resume
if args.code_file:
try:
with open(args.code_file) as code_file:
source_code = code_file.read()
except IOError:
sys.exit("Failed to find the source code file: %s" %
args.code_file)
else:
source_code = args.code
# Check if there's a created project for it
args.project_id = pp.project_processing(
api, args, resume, session_file=session_file,
path=path, log=log)
library_args = r.set_library_args(args)
library = r.create_library(source_code, library_args, args, api,
path, session_file, log)
return library
def execution_processing(api, args,
session_file=None, path=None, log=None):
"""Creating or retrieving an execution
"""
execution = None
resume = args.resume
if args.script or args.scripts:
# If resuming, try to extract args.execution form log files
if resume:
message = u.dated("Execution not found. Resuming.\n")
resume, args.execution = c.checkpoint(
c.is_execution_created, path, debug=args.debug,
message=message,
log_file=session_file, console=args.verbosity)
if not resume:
args.resume = resume
# Check if there's a created project for it
args.project_id = pp.project_processing(
api, args, resume, session_file=session_file, path=path,
log=log)
execution_args = r.set_execution_args(args)
execution = r.create_execution(execution_args, args, api,
path, session_file, log)
# If a source is provided either through the command line or in resume
# steps, we use it.
elif args.execution:
execution = bigml.api.get_execution_id(args.execution)
return execution
def library_processing(api, args, session_file=None, path=None, log=None):
"""Creating or retrieving a library
"""
library = None
resume = args.resume
if args.code_file or args.code:
# If resuming, try to extract args.library form log files
if resume:
message = u.dated("Library not found. Resuming.\n")
resume, library = c.checkpoint(c.is_library_created,
path,
debug=args.debug,
message=message,
log_file=session_file,
console=args.verbosity)
if not resume:
args.resume = resume
if args.code_file:
try:
with open(args.code_file) as code_file:
source_code = code_file.read()
except IOError:
sys.exit("Failed to find the source code file: %s" %
args.code_file)
else:
source_code = args.code
# Check if there's a created project for it
args.project_id = pp.project_processing(api,
args,
resume,
session_file=session_file,
path=path,
log=log)
# Check if we are upgrading
if args.upgrade:
library = u.get_last_resource("library", api,
build_query_string(args))
log_created_resources("library", path, library, mode='a')
message = u.dated("Library found: %s \n"
" (library ID: %s)\n" %
(args.name, library))
u.log_message(message,
log_file=session_file,
console=args.verbosity)
if library is None:
library_args = rl.set_library_args(args)
add_version_tag(library_args, args.name)
library = rl.create_library(source_code, library_args, args,
api, path, session_file, log)
return library
def script_processing(api, args, session_file=None, path=None, log=None):
"""Creating or retrieving a script
"""
script = None
resume = args.resume
if args.code_file or args.code:
# If resuming, try to extract args.script form log files
if resume:
message = u.dated("Script not found. Resuming.\n")
resume, args.script = c.checkpoint(c.are_scripts_created,
path,
debug=args.debug,
message=message,
log_file=session_file,
console=args.verbosity)
script = args.script
if not resume:
args.resume = resume
if args.code_file:
try:
with open(args.code_file) as code_file:
source_code = code_file.read()
except IOError:
sys.exit("Failed to find the source code file: %s" %
args.code_file)
else:
source_code = args.code
# Check if there's a created project for it
args.project_id = pp.project_processing(api,
args,
resume,
session_file=session_file,
path=path,
log=log)
script_args = r.set_script_args(args)
script = r.create_script(source_code, script_args, args, api, path,
session_file, log)
args.script = script['resource']
scripts = [script]
# If a script is provided either through the command line or in resume
# steps, we use it.
elif args.script:
script = bigml.api.get_script_id(args.script)
scripts = [script]
elif args.scripts:
scripts = [script for script in args.script_ids]
script = scripts[0]
return script, scripts
def library_processing(api, args,
session_file=None, path=None, log=None):
"""Creating or retrieving a library
"""
library = None
resume = args.resume
if args.code_file or args.code:
# If resuming, try to extract args.library form log files
if resume:
message = u.dated("Library not found. Resuming.\n")
resume, library = c.checkpoint(
c.is_library_created, path, debug=args.debug, message=message,
log_file=session_file, console=args.verbosity)
if not resume:
args.resume = resume
if args.code_file:
try:
with open(args.code_file) as code_file:
source_code = code_file.read()
except IOError:
sys.exit("Failed to find the source code file: %s" %
args.code_file)
else:
source_code = args.code
# Check if there's a created project for it
args.project_id = pp.project_processing(
api, args, resume, session_file=session_file,
path=path, log=log)
# Check if we are upgrading
if args.upgrade:
library = u.get_last_resource("library",
api,
build_query_string(args))
r.log_created_resources("library", path,
library, mode='a')
message = u.dated("Library found: %s \n"
" (library ID: %s)\n" %
(args.name, library))
u.log_message(message, log_file=session_file,
console=args.verbosity)
if library is None:
library_args = r.set_library_args(args)
add_version_tag(library_args, args.name)
library = r.create_library(source_code, library_args, args, api,
path, session_file, log)
return library
def script_processing(api, args,
session_file=None, path=None, log=None):
"""Creating or retrieving a script
"""
script = None
resume = args.resume
if args.code_file or args.code:
# If resuming, try to extract args.script form log files
if resume:
message = u.dated("Script not found. Resuming.\n")
resume, args.script = c.checkpoint(
c.are_scripts_created, path, debug=args.debug, message=message,
log_file=session_file, console=args.verbosity)
script = args.script
if not resume:
args.resume = resume
if args.code_file:
try:
with open(args.code_file) as code_file:
source_code = code_file.read()
except IOError:
sys.exit("Failed to find the source code file: %s" %
args.code_file)
else:
source_code = args.code
# Check if there's a created project for it
args.project_id = pp.project_processing(
api, args, resume, session_file=session_file,
path=path, log=log)
script_args = r.set_script_args(args)
script = r.create_script(source_code, script_args, args, api,
path, session_file, log)
args.script = script['resource']
scripts = [script]
# If a script is provided either through the command line or in resume
# steps, we use it.
elif args.script:
script = bigml.api.get_script_id(args.script)
scripts = [script]
elif args.scripts:
scripts = [script for script in args.script_ids]
script = scripts[0]
return script, scripts
def library_processing(api, args, session_file=None, path=None, log=None):
"""Creating or retrieving a library
"""
library = None
resume = args.resume
if args.code_file or args.code:
# If resuming, try to extract args.library form log files
if resume:
message = u.dated("Library not found. Resuming.\n")
resume, library = c.checkpoint(c.is_library_created,
path,
debug=args.debug,
message=message,
log_file=session_file,
console=args.verbosity)
if not resume:
args.resume = resume
if args.code_file:
try:
with open(args.code_file) as code_file:
source_code = code_file.read()
except IOError:
sys.exit("Failed to find the source code file: %s" %
args.code_file)
else:
source_code = args.code
# Check if there's a created project for it
args.project_id = pp.project_processing(api,
args,
resume,
session_file=session_file,
path=path,
log=log)
library_args = r.set_library_args(args)
library = r.create_library(source_code, library_args, args, api,
path, session_file, log)
def source_processing(api,
args,
resume,
csv_properties=None,
multi_label_data=None,
session_file=None,
path=None,
log=None):
"""Creating or retrieving a data source from input arguments
"""
source = None
fields = None
if (args.training_set or
(hasattr(args, "evaluate") and args.evaluate and args.test_set)):
# If resuming, try to extract args.source form log files
if resume:
message = u.dated("Source not found. Resuming.\n")
resume, args.source = c.checkpoint(c.is_source_created,
path,
debug=args.debug,
message=message,
log_file=session_file,
console=args.verbosity)
# If neither a previous source, dataset or model are provided.
# we create a new one. Also if --evaluate and test data are provided
# we create a new dataset to test with.
data_set, data_set_header = r.data_to_source(args)
if data_set is not None:
# Check if there's a created project for it
args.project_id = pp.project_processing(api,
args,
resume,
session_file=session_file,
path=path,
log=log)
source_args = r.set_source_args(args,
multi_label_data=multi_label_data,
data_set_header=data_set_header)
source = r.create_source(data_set, source_args, args, api, path,
session_file, log)
# If a source is provided either through the command line or in resume
# steps, we use it.
elif args.source:
source = bigml.api.get_source_id(args.source)
# If we already have source, we check that is finished, extract the
# fields, and update them if needed.
if source:
source = r.get_source(source, api, args.verbosity, session_file)
if 'source_parser' in source['object']:
source_parser = source['object']['source_parser']
if 'missing_tokens' in source_parser:
csv_properties['missing_tokens'] = (
source_parser['missing_tokens'])
if 'locale' in source_parser:
csv_properties['data_locale'] = source_parser['locale']
# No changes if user locale is the one in the source.
if (args.user_locale is not None and bigml_locale(
args.user_locale) == source_parser['locale']):
args.user_locale = None
fields = Fields(source['object']['fields'], **csv_properties)
if (args.field_attributes_ or args.types_ or args.user_locale
or args.json_args.get('source')):
# avoid updating project_id in source
project_id, args.project_id = args.project_id, None
source_args = r.set_source_args(args, fields=fields)
source = r.update_source(source, source_args, args, api,
session_file)
args.project_id = project_id
fields = Fields(source['object']['fields'], **csv_properties)
return source, resume, csv_properties, fields
def source_processing(api, args, resume,
csv_properties=None,
multi_label_data=None,
session_file=None, path=None, log=None):
"""Creating or retrieving a data source from input arguments
"""
source = None
fields = None
if (args.training_set or
(hasattr(args, "evaluate") and args.evaluate and args.test_set)):
# If resuming, try to extract args.source form log files
if resume:
message = u.dated("Source not found. Resuming.\n")
resume, args.source = c.checkpoint(
c.is_source_created, path, debug=args.debug, message=message,
log_file=session_file, console=args.verbosity)
# If neither a previous source, dataset or model are provided.
# we create a new one. Also if --evaluate and test data are provided
# we create a new dataset to test with.
data_set, data_set_header = r.data_to_source(args)
if data_set is not None:
# Check if there's a created project for it
args.project_id = pp.project_processing(
api, args, resume, session_file=session_file, path=path, log=log)
source_args = r.set_source_args(args,
multi_label_data=multi_label_data,
data_set_header=data_set_header)
source = r.create_source(data_set, source_args, args, api,
path, session_file, log)
# If a source is provided either through the command line or in resume
# steps, we use it.
elif args.source:
source = bigml.api.get_source_id(args.source)
# If we already have source, we check that is finished, extract the
# fields, and update them if needed.
if source:
source = r.get_source(source, api, args.verbosity, session_file)
if 'source_parser' in source['object']:
source_parser = source['object']['source_parser']
if 'missing_tokens' in source_parser:
csv_properties['missing_tokens'] = (
source_parser['missing_tokens'])
if 'locale' in source_parser:
csv_properties['data_locale'] = source_parser['locale']
# No changes if user locale is the one in the source.
if (args.user_locale is not None and
bigml_locale(args.user_locale) == source_parser['locale']):
args.user_locale = None
fields = Fields(source['object']['fields'], **csv_properties)
if (args.field_attributes_ or args.types_ or args.user_locale
or args.json_args.get('source')):
# avoid updating project_id in source
project_id, args.project_id = args.project_id, None
source_args = r.set_source_args(args, fields=fields)
source = r.update_source(source, source_args, args, api,
session_file)
args.project_id = project_id
fields = Fields(source['object']['fields'], **csv_properties)
return source, resume, csv_properties, fields
def project_dispatcher(args=sys.argv[1:]):
"""Parses command line and calls the different processing functions
"""
command = command_handling(args, COMMAND_LOG)
# Parses command line arguments.
command_args = a.parse_and_check(command)
if command_args.resume:
command_args, session_file, _ = get_stored_command(
args, command_args.debug, command_log=COMMAND_LOG,
dirs_log=DIRS_LOG, sessions_log=SESSIONS_LOG)
else:
if command_args.output_dir is None:
command_args.output_dir = a.NOW
directory = u.check_dir("%s/x.txt" % command_args.output_dir)
command_args.output_dir = directory
session_file = os.path.join(directory, SESSIONS_LOG)
u.log_message(command.command + "\n", log_file=session_file)
directory = u.check_dir(os.path.join(command_args.output_dir, "tmp"))
session_file = os.path.join(directory, SESSIONS_LOG)
u.log_message(command.command + "\n", log_file=session_file)
try:
shutil.copy(DEFAULTS_FILE, os.path.join(directory, DEFAULTS_FILE))
except IOError:
pass
u.sys_log_message(u"%s\n" % os.path.abspath(directory),
log_file=DIRS_LOG)
path = u.check_dir("%s/x.txt" % command_args.output_dir)
session_file = u"%s%s%s" % (path, os.sep, SESSIONS_LOG)
# If logging is required set the file for logging
log = None
if command_args.log_file:
u.check_dir(command_args.log_file)
log = command_args.log_file
# If --clear_logs the log files are cleared
clear_log_files([log])
# Creates the corresponding api instance
api = a.get_api_instance(command_args, u.check_dir(session_file))
a.get_output_args(api, command_args, command_args.resume)
a.attribute_args(command_args)
if not command_args.project_id and command_args.name:
command_args.project = command_args.name
if command_args.project:
# create project
pp.project_processing(
api, command_args, command_args.resume, session_file=session_file,
path=path, log=log, create=True)
if command_args.project_id and (
command_args.project_attributes or
command_args.name or command_args.tag or command_args.description
or command_args.category):
# update project's attributes
pp.update_project(command_args, api, command_args.resume, \
session_file=session_file)
u.log_message("_" * 80 + "\n", log_file=session_file)
u.print_generated_files(command_args.output_dir, log_file=session_file,
verbosity=command_args.verbosity)
def project_dispatcher(args=sys.argv[1:]):
"""Parses command line and calls the different processing functions
"""
command = command_handling(args, COMMAND_LOG)
# Parses command line arguments.
command_args = a.parse_and_check(command)
if command_args.resume:
command_args, session_file, _ = get_stored_command(
args,
command_args.debug,
command_log=COMMAND_LOG,
dirs_log=DIRS_LOG,
sessions_log=SESSIONS_LOG)
else:
if command_args.output_dir is None:
command_args.output_dir = a.NOW
directory = u.check_dir("%s/x.txt" % command_args.output_dir)
command_args.output_dir = directory
session_file = os.path.join(directory, SESSIONS_LOG)
u.log_message(command.command + "\n", log_file=session_file)
directory = u.check_dir(os.path.join(command_args.output_dir, "tmp"))
session_file = os.path.join(directory, SESSIONS_LOG)
u.log_message(command.command + "\n", log_file=session_file)
try:
shutil.copy(DEFAULTS_FILE, os.path.join(directory, DEFAULTS_FILE))
except IOError:
pass
u.sys_log_message(u"%s\n" % os.path.abspath(directory),
log_file=DIRS_LOG)
path = u.check_dir("%s/x.txt" % command_args.output_dir)
session_file = u"%s%s%s" % (path, os.sep, SESSIONS_LOG)
# If logging is required set the file for logging
log = None
if command_args.log_file:
u.check_dir(command_args.log_file)
log = command_args.log_file
# If --clear_logs the log files are cleared
clear_log_files([log])
# Creates the corresponding api instance
api = a.get_api_instance(command_args, u.check_dir(session_file))
a.get_output_args(api, command_args, command_args.resume)
a.attribute_args(command_args)
if not command_args.project_id and command_args.name:
command_args.project = command_args.name
if command_args.project:
# create project
pp.project_processing(api,
command_args,
command_args.resume,
session_file=session_file,
path=path,
log=log,
create=True)
if command_args.project_id and (command_args.project_attributes
or command_args.name or command_args.tag
or command_args.description
or command_args.category):
# update project's attributes
pp.update_project(command_args, api, command_args.resume, \
session_file=session_file)
u.log_message("_" * 80 + "\n", log_file=session_file)
u.print_generated_files(command_args.output_dir,
log_file=session_file,
verbosity=command_args.verbosity)
def script_processing(api, args,
session_file=None, path=None, log=None):
"""Creating or retrieving a script
"""
script = None
resume = args.resume
if args.code_file or args.code:
# If resuming, try to extract args.script form log files
if resume:
message = u.dated("Script not found. Resuming.\n")
resume, args.script = c.checkpoint(
c.are_scripts_created, path, debug=args.debug, message=message,
log_file=session_file, console=args.verbosity)
script = args.script
if not resume:
args.resume = resume
imports_code = []
if args.embedded_imports is not None:
for import_file in args.embedded_imports_:
with open(import_file) as code_file:
imports_code.append(code_file.read())
if args.code_file:
try:
with open(args.code_file) as code_file:
source_code = code_file.read()
except IOError:
sys.exit("Failed to find the source code file: %s" %
args.code_file)
else:
source_code = args.code
if imports_code:
source_code = "%s\n%s" % ("\n".join(imports_code), source_code)
# Check if there's a created project for it
args.project_id = pp.project_processing(
api, args, resume, session_file=session_file,
path=path, log=log)
# Check if we are upgrading
if args.upgrade:
script = u.get_last_resource("script",
api,
build_query_string(args))
r.log_created_resources("script", path,
script, mode='a')
message = u.dated("Script found: %s"
"\n (script ID: %s)\n" %
(args.name, script))
u.log_message(message, log_file=session_file,
console=args.verbosity)
if script is None:
script_args = r.set_script_args(args)
add_version_tag(script_args, args.name)
script = r.create_script(source_code, script_args, args, api,
path, session_file, log)
args.script = script if isinstance(script, basestring) else \
script.get('resource')
scripts = [script]
# If a script is provided either through the command line or in resume
# steps, we use it.
elif args.script:
script = bigml.api.get_script_id(args.script)
scripts = [script]
elif args.scripts:
scripts = [script for script in args.script_ids]
script = scripts[0]
return script, scripts