sys.argv[0] = target
del sys.argv[1]
print "Executing: " + match[0] + "/" + match[1] + " " + " ".join(sys.argv[1:])
exec open(target)
# If there is an unique match in a table
elif len(table_matches) == 1 and len(matches) == 0:
subproject, index = table_matches[0]
command_name = tables[subproject]["Command"][index]
description = tables[subproject]["Description"][index]
class_path_relative = tables[subproject]["Path"][index]
class_path = "pts." + subproject + "." + class_path_relative
module_path, class_name = class_path.rsplit('.', 1)
subproject_path = introspection.pts_subproject_dir(subproject)
sys.argv[0] = fs.join(introspection.pts_root_dir, module_path.replace(".", "/") + ".py") # this is actually not necessary (and not really correct, it's not like we are calling the module where the class is..)
del sys.argv[1] # but this is important
module = importlib.import_module(module_path)
cls = getattr(module, class_name)
# Import things
from pts.core.tools import logging
from pts.core.basics.configuration import ConfigurationReader
## GET THE CONFIGURATION DEFINITION
configuration_name = tables[subproject]["Configuration"][index]
elif len(table_matches) == 1 and len(matches) == 0:
# from pts.core.tools import logging
# configuration
# Path to class module
table_match = table_matches[0]
subproject = table_match[0]
index = table_match[1]
relative_class_module_path = tables[subproject]["Path"][index].replace(
".", "/").rsplit("/", 1)[0] + ".py"
class_module_path = fs.join(
introspection.pts_subproject_dir(subproject),
relative_class_module_path)
logging_path = fs.join(introspection.pts_package_dir, "core", "tools",
"logging.py")
command_name = tables[subproject]["Command"][index]
configuration_name = tables[subproject]["Configuration"][index]
if configuration_name == "--": configuration_name = command_name
configuration_module_path = fs.join(
introspection.pts_root_dir,
"pts/" + subproject + "/config/" + configuration_name + ".py")
# Add dependencies
encountered = set()
introspection.add_dependencies(dependencies, logging_path, encountered)
def run_configurable(table_matches, args, tables):
"""
This function ...
:param table_matches:
:param args:
:param tables:
:return:
"""
# Determine the configuration method
configuration_method = None
if args.interactive: configuration_method = "interactive"
elif args.arguments: configuration_method = "arguments"
elif args.configfile is not None: configuration_method = "file:" + args.configfile
elif args.rerun: configuration_method = "last"
# Regenerate the configuration method option
if args.interactive: configuration_method_argument = "--interactive"
elif args.arguments: configuration_method_argument = "--arguments"
elif args.configfile is not None: configuration_method_argument = "--configfile '" + args.configfile + "'"
elif args.rerun: configuration_method_argument = "--rerun"
else: configuration_method_argument = ""
# Resolve
subproject, index = table_matches[0]
resolved = introspection.resolve_from_match(subproject, tables[subproject], index)
# Get properties
title = resolved.title
command_name = resolved.command_name
hidden = resolved.hidden
description = resolved.description
module_path = resolved.module_path
class_name = resolved.class_name
configuration_method_table = resolved.configuration_method
configuration_module_path = resolved.configuration_module_path
subproject_path = introspection.pts_subproject_dir(subproject)
# Set
sys.argv[0] = fs.join(introspection.pts_root_dir, module_path.replace(".", "/") + ".py") # this is actually not necessary (and not really correct, it's not like we are calling the module where the class is..)
del sys.argv[1] # but this is important
# Get a list of the leftover arguments
leftover_arguments = sys.argv[1:]
# Welcome message
if subproject == "modeling": welcome_modeling()
elif subproject == "magic": welcome_magic()
elif subproject == "dustpedia": welcome_dustpedia()
elif subproject == "evolve": welcome_evolve()
# Special
if subproject == "modeling": check_modeling_cwd(command_name, fs.cwd())
# Get the configuration definition
definition = introspection.get_configuration_definition_pts_not_yet_in_pythonpath(configuration_module_path)
# If not specified on the command line (before the command name), then use the default specified in the commands.dat file
if configuration_method is None: configuration_method = configuration_method_table
# Check whether arguments are passed and the configuration method is interactive
if configuration_method == "interactive" and len(leftover_arguments) > 0: raise ValueError("Arguments on the command-line are not supported by default for this command. Run with pts --arguments to change this behaviour.")
# Create the configuration
config = create_configuration(definition, command_name, description, configuration_method)
## SAVE THE CONFIG if requested
if config.write_config:
config_filepath = config.config_file_path(command_name)
config.saveto(config_filepath)
else: config_filepath = None
# If this is not a re-run
if not args.rerun:
if not fs.is_directory(introspection.pts_user_config_dir): fs.create_directory(introspection.pts_user_config_dir)
# CACHE THE CONFIG
config_cache_path = fs.join(introspection.pts_user_config_dir, command_name + ".cfg")
config.saveto(config_cache_path)
# Setup function
if subproject == "modeling": setup_modeling(command_name, fs.cwd(), configuration_method_argument)
elif subproject == "magic": setup_magic(command_name, fs.cwd())
elif subproject == "dustpedia": setup_dustpedia(command_name, fs.cwd())
elif subproject == "evolve": setup_evolve(command_name, fs.cwd())
# Initialize the logger
log = initialize_pts(config, remote=args.remote, command_name=command_name)
# Exact command name
exact_command_name = subproject + "/" + command_name
# If the PTS command has to be executed remotely
if args.remote is not None: run_remotely(exact_command_name, config, args.keep, args.remote, log)
# The PTS command has to be executed locally
else: run_locally(exact_command_name, module_path, class_name, config, args.input_files, args.output_files, args.output, log)
# Finish function
if subproject == "modeling": finish_modeling(command_name, fs.cwd(), config_path=config_filepath)
elif subproject == "magic": finish_magic(command_name, fs.cwd())
elif subproject == "dustpedia": finish_dustpedia(command_name, fs.cwd())
elif subproject == "evolve": finish_evolve(command_name, fs.cwd())
def run_configurable(table_matches, args, tables):
"""
This function ...
:param table_matches:
:param args:
:param tables:
:return:
"""
# Determine the configuration method
configuration_method = None
if args.interactive: configuration_method = "interactive"
elif args.arguments: configuration_method = "arguments"
elif args.configfile is not None:
configuration_method = "file:" + args.configfile
elif args.rerun:
configuration_method = "last"
# Resolve
subproject, index = table_matches[0]
resolved = introspection.resolve_from_match(subproject, tables[subproject],
index)
# Get properties
title = resolved.title
command_name = resolved.command_name
hidden = resolved.hidden
description = resolved.description
module_path = resolved.module_path
class_name = resolved.class_name
configuration_method_table = resolved.configuration_method
configuration_module_path = resolved.configuration_module_path
subproject_path = introspection.pts_subproject_dir(subproject)
# Set
sys.argv[0] = fs.join(
introspection.pts_root_dir,
module_path.replace(".", "/") + ".py"
) # this is actually not necessary (and not really correct, it's not like we are calling the module where the class is..)
del sys.argv[1] # but this is important
# Get a list of the leftover arguments
leftover_arguments = sys.argv[1:]
# Welcome message
if subproject == "modeling": welcome_modeling()
elif subproject == "magic": welcome_magic()
elif subproject == "dustpedia": welcome_dustpedia()
elif subproject == "evolve": welcome_evolve()
# Get the configuration definition
definition = introspection.get_configuration_definition_pts_not_yet_in_pythonpath(
configuration_module_path)
# If not specified on the command line (before the command name), then use the default specified in the commands.dat file
if configuration_method is None:
configuration_method = configuration_method_table
# Check whether arguments are passed and the configuration method is interactive
if configuration_method == "interactive" and len(leftover_arguments) > 0:
raise ValueError(
"Arguments on the command-line are not supported by default for this command. Run with pts --arguments to change this behaviour."
)
# Create the configuration
config = create_configuration(definition, command_name, description,
configuration_method)
## SAVE THE CONFIG if requested
if config.write_config:
config.saveto(config.config_file_path(command_name))
# If this is not a re-run
if not args.rerun:
if not fs.is_directory(introspection.pts_user_config_dir):
fs.create_directory(introspection.pts_user_config_dir)
# CACHE THE CONFIG
config_cache_path = fs.join(introspection.pts_user_config_dir,
command_name + ".cfg")
config.saveto(config_cache_path)
# Setup function
if subproject == "modeling": setup_modeling(command_name, fs.cwd())
elif subproject == "magic": setup_magic(command_name, fs.cwd())
elif subproject == "dustpedia": setup_dustpedia(command_name, fs.cwd())
elif subproject == "evolve": setup_evolve(command_name, fs.cwd())
# Initialize the logger
log = initialize_pts(config, remote=args.remote, command_name=command_name)
# Exact command name
exact_command_name = subproject + "/" + command_name
# If the PTS command has to be executed remotely
if args.remote is not None:
run_remotely(exact_command_name, config, args.keep, args.remote, log)
# The PTS command has to be executed locally
else:
run_locally(exact_command_name, module_path, class_name, config,
args.input_files, args.output_files, args.output, log)
# Finish function
if subproject == "modeling": finish_modeling(command_name, fs.cwd())
elif subproject == "magic": finish_magic(command_name, fs.cwd())
elif subproject == "dustpedia": finish_dustpedia(command_name, fs.cwd())
elif subproject == "evolve": finish_evolve(command_name, fs.cwd())
def get_dependencies(self):
"""
This function ...
:return:
"""
# If no script name is given, list all dependencies of PTS and the
# PTS modules that use them
if self.config.script is None:
if self.config.subprojects:
self.dependencies = defaultdict(set)
directories_for_subproject = defaultdict(list)
# Loop over the subdirectories of the 'do' directory
for path, name in fs.directories_in_path(
introspection.pts_do_dir, returns=["path", "name"]):
subproject = name
directories_for_subproject[subproject].append(path)
# Loop over the other directories in 'pts' (other than 'do' and 'doc')
for path, name in fs.directories_in_path(
introspection.pts_package_dir,
returns=["path", "name"],
exact_not_name=["do", "doc"]):
subproject = name
directories_for_subproject[subproject].append(path)
encountered_internal_modules = set()
for subproject in directories_for_subproject:
# print(subproject, directories_for_subproject[subproject])
# List the dependencies of the matching script
dependencies_for_this = defaultdict(set)
for dir_path in directories_for_subproject[subproject]:
for file_path in fs.files_in_path(dir_path,
extension="py",
recursive=True):
# if subproject == "dustpedia": print(file_path)
introspection.add_dependencies(
dependencies_for_this, file_path,
encountered_internal_modules)
self.dependencies_for_subproject[
subproject] = dependencies_for_this.keys()
for dependency in dependencies_for_this:
for name in dependencies_for_this[dependency]:
self.dependencies[dependency].add(name)
# No subprojects are specified, list all PTS dependencies
else:
self.dependencies = introspection.get_all_dependencies()
# If a script name is given
else:
scripts = introspection.get_scripts()
tables = introspection.get_arguments_tables()
# Find matching 'do' commands (actual scripts or tabulated commands)
matches = introspection.find_matches_scripts(
self.config.script, scripts)
table_matches = introspection.find_matches_tables(
self.config.script, tables)
# List the dependencies of the matching script
self.dependencies = defaultdict(set)
# No match
if len(matches) + len(table_matches) == 0:
show_all_available(scripts, tables)
exit()
# More matches
elif len(matches) + len(table_matches) > 1:
show_possible_matches(matches, table_matches, tables)
exit()
# Exactly one match from existing do script
elif len(matches) == 1 and len(table_matches) == 0:
# Determine the full path to the matching script
script_path = fs.join(introspection.pts_do_dir, matches[0][0],
matches[0][1])
introspection.add_dependencies(self.dependencies, script_path,
set())
# Exactly one match from tabulated command
elif len(table_matches) == 1 and len(matches) == 0:
# from pts.core.tools import logging
# configuration
# Path to class module
table_match = table_matches[0]
subproject = table_match[0]
index = table_match[1]
relative_class_module_path = tables[subproject]["Path"][
index].replace(".", "/").rsplit("/", 1)[0] + ".py"
class_module_path = fs.join(
introspection.pts_subproject_dir(subproject),
relative_class_module_path)
logging_path = fs.join(introspection.pts_package_dir, "core",
"tools", "logging.py")
command_name = tables[subproject]["Command"][index]
configuration_name = tables[subproject]["Configuration"][index]
if configuration_name == "--":
configuration_name = command_name
configuration_module_path = fs.join(
introspection.pts_root_dir, "pts/" + subproject +
"/config/" + configuration_name + ".py")
# Add dependencies
encountered = set()
introspection.add_dependencies(self.dependencies, logging_path,
encountered)
introspection.add_dependencies(self.dependencies,
configuration_module_path,
encountered)
introspection.add_dependencies(self.dependencies,
class_module_path, encountered)
# Exactly one match from tabulated command
elif len(table_matches) == 1 and len(matches) == 0:
# from pts.core.tools import logging
# configuration
# Path to class module
table_match = table_matches[0]
subproject = table_match[0]
index = table_match[1]
relative_class_module_path = tables[subproject]["Path"][index].replace(".", "/").rsplit("/", 1)[0] + ".py"
class_module_path = fs.join(introspection.pts_subproject_dir(subproject), relative_class_module_path)
logging_path = fs.join(introspection.pts_package_dir, "core", "tools", "logging.py")
command_name = tables[subproject]["Command"][index]
configuration_name = tables[subproject]["Configuration"][index]
if configuration_name == "--": configuration_name = command_name
configuration_module_path = fs.join(introspection.pts_root_dir, "pts/" + subproject + "/config/" + configuration_name + ".py")
# Add dependencies
encountered = set()
introspection.add_dependencies(dependencies, logging_path, encountered)
introspection.add_dependencies(dependencies, configuration_module_path, encountered)
introspection.add_dependencies(dependencies, class_module_path, encountered)
# Get the names and versions of all installed python packages
def get_dependencies(self):
"""
This function ...
:return:
"""
# If no script name is given, list all dependencies of PTS and the
# PTS modules that use them
if self.config.script is None:
if self.config.subprojects:
self.dependencies = defaultdict(set)
directories_for_subproject = defaultdict(list)
# Loop over the subdirectories of the 'do' directory
for path, name in fs.directories_in_path(introspection.pts_do_dir, returns=["path", "name"]):
subproject = name
directories_for_subproject[subproject].append(path)
# Loop over the other directories in 'pts' (other than 'do' and 'doc')
for path, name in fs.directories_in_path(introspection.pts_package_dir, returns=["path", "name"],
exact_not_name=["do", "doc"]):
subproject = name
directories_for_subproject[subproject].append(path)
encountered_internal_modules = set()
for subproject in directories_for_subproject:
# print(subproject, directories_for_subproject[subproject])
# List the dependencies of the matching script
dependencies_for_this = defaultdict(set)
for dir_path in directories_for_subproject[subproject]:
for file_path in fs.files_in_path(dir_path, extension="py", recursive=True):
# if subproject == "dustpedia": print(file_path)
introspection.add_dependencies(dependencies_for_this, file_path,
encountered_internal_modules)
self.dependencies_for_subproject[subproject] = dependencies_for_this.keys()
for dependency in dependencies_for_this:
for name in dependencies_for_this[dependency]:
self.dependencies[dependency].add(name)
# No subprojects are specified, list all PTS dependencies
else: self.dependencies = introspection.get_all_dependencies()
# If a script name is given
else:
scripts = introspection.get_scripts()
tables = introspection.get_arguments_tables()
# Find matching 'do' commands (actual scripts or tabulated commands)
matches = introspection.find_matches_scripts(self.config.script, scripts)
table_matches = introspection.find_matches_tables(self.config.script, tables)
# List the dependencies of the matching script
self.dependencies = defaultdict(set)
# No match
if len(matches) + len(table_matches) == 0:
show_all_available(scripts, tables)
exit()
# More matches
elif len(matches) + len(table_matches) > 1:
show_possible_matches(matches, table_matches, tables)
exit()
# Exactly one match from existing do script
elif len(matches) == 1 and len(table_matches) == 0:
# Determine the full path to the matching script
script_path = fs.join(introspection.pts_do_dir, matches[0][0], matches[0][1])
introspection.add_dependencies(self.dependencies, script_path, set())
# Exactly one match from tabulated command
elif len(table_matches) == 1 and len(matches) == 0:
# from pts.core.tools import logging
# configuration
# Path to class module
table_match = table_matches[0]
subproject = table_match[0]
index = table_match[1]
relative_class_module_path = tables[subproject]["Path"][index].replace(".", "/").rsplit("/", 1)[0] + ".py"
class_module_path = fs.join(introspection.pts_subproject_dir(subproject), relative_class_module_path)
logging_path = fs.join(introspection.pts_package_dir, "core", "basics", "log.py")
command_name = tables[subproject]["Command"][index]
configuration_name = tables[subproject]["Configuration"][index]
if configuration_name == "--": configuration_name = command_name
configuration_module_path = fs.join(introspection.pts_root_dir,
"pts/" + subproject + "/config/" + configuration_name + ".py")
# Add dependencies
encountered = set()
introspection.add_dependencies(self.dependencies, logging_path, encountered)
introspection.add_dependencies(self.dependencies, configuration_module_path, encountered)
introspection.add_dependencies(self.dependencies, class_module_path, encountered)
