sys.argv[0] = target del sys.argv[1] print "Executing: " + match[0] + "/" + match[1] + " " + " ".join(sys.argv[1:]) exec open(target) # If there is an unique match in a table elif len(table_matches) == 1 and len(matches) == 0: subproject, index = table_matches[0] command_name = tables[subproject]["Command"][index] description = tables[subproject]["Description"][index] class_path_relative = tables[subproject]["Path"][index] class_path = "pts." + subproject + "." + class_path_relative module_path, class_name = class_path.rsplit('.', 1) subproject_path = introspection.pts_subproject_dir(subproject) sys.argv[0] = fs.join(introspection.pts_root_dir, module_path.replace(".", "/") + ".py") # this is actually not necessary (and not really correct, it's not like we are calling the module where the class is..) del sys.argv[1] # but this is important module = importlib.import_module(module_path) cls = getattr(module, class_name) # Import things from pts.core.tools import logging from pts.core.basics.configuration import ConfigurationReader ## GET THE CONFIGURATION DEFINITION configuration_name = tables[subproject]["Configuration"][index]
elif len(table_matches) == 1 and len(matches) == 0: # from pts.core.tools import logging # configuration # Path to class module table_match = table_matches[0] subproject = table_match[0] index = table_match[1] relative_class_module_path = tables[subproject]["Path"][index].replace( ".", "/").rsplit("/", 1)[0] + ".py" class_module_path = fs.join( introspection.pts_subproject_dir(subproject), relative_class_module_path) logging_path = fs.join(introspection.pts_package_dir, "core", "tools", "logging.py") command_name = tables[subproject]["Command"][index] configuration_name = tables[subproject]["Configuration"][index] if configuration_name == "--": configuration_name = command_name configuration_module_path = fs.join( introspection.pts_root_dir, "pts/" + subproject + "/config/" + configuration_name + ".py") # Add dependencies encountered = set() introspection.add_dependencies(dependencies, logging_path, encountered)
def run_configurable(table_matches, args, tables): """ This function ... :param table_matches: :param args: :param tables: :return: """ # Determine the configuration method configuration_method = None if args.interactive: configuration_method = "interactive" elif args.arguments: configuration_method = "arguments" elif args.configfile is not None: configuration_method = "file:" + args.configfile elif args.rerun: configuration_method = "last" # Regenerate the configuration method option if args.interactive: configuration_method_argument = "--interactive" elif args.arguments: configuration_method_argument = "--arguments" elif args.configfile is not None: configuration_method_argument = "--configfile '" + args.configfile + "'" elif args.rerun: configuration_method_argument = "--rerun" else: configuration_method_argument = "" # Resolve subproject, index = table_matches[0] resolved = introspection.resolve_from_match(subproject, tables[subproject], index) # Get properties title = resolved.title command_name = resolved.command_name hidden = resolved.hidden description = resolved.description module_path = resolved.module_path class_name = resolved.class_name configuration_method_table = resolved.configuration_method configuration_module_path = resolved.configuration_module_path subproject_path = introspection.pts_subproject_dir(subproject) # Set sys.argv[0] = fs.join(introspection.pts_root_dir, module_path.replace(".", "/") + ".py") # this is actually not necessary (and not really correct, it's not like we are calling the module where the class is..) del sys.argv[1] # but this is important # Get a list of the leftover arguments leftover_arguments = sys.argv[1:] # Welcome message if subproject == "modeling": welcome_modeling() elif subproject == "magic": welcome_magic() elif subproject == "dustpedia": welcome_dustpedia() elif subproject == "evolve": welcome_evolve() # Special if subproject == "modeling": check_modeling_cwd(command_name, fs.cwd()) # Get the configuration definition definition = introspection.get_configuration_definition_pts_not_yet_in_pythonpath(configuration_module_path) # If not specified on the command line (before the command name), then use the default specified in the commands.dat file if configuration_method is None: configuration_method = configuration_method_table # Check whether arguments are passed and the configuration method is interactive if configuration_method == "interactive" and len(leftover_arguments) > 0: raise ValueError("Arguments on the command-line are not supported by default for this command. Run with pts --arguments to change this behaviour.") # Create the configuration config = create_configuration(definition, command_name, description, configuration_method) ## SAVE THE CONFIG if requested if config.write_config: config_filepath = config.config_file_path(command_name) config.saveto(config_filepath) else: config_filepath = None # If this is not a re-run if not args.rerun: if not fs.is_directory(introspection.pts_user_config_dir): fs.create_directory(introspection.pts_user_config_dir) # CACHE THE CONFIG config_cache_path = fs.join(introspection.pts_user_config_dir, command_name + ".cfg") config.saveto(config_cache_path) # Setup function if subproject == "modeling": setup_modeling(command_name, fs.cwd(), configuration_method_argument) elif subproject == "magic": setup_magic(command_name, fs.cwd()) elif subproject == "dustpedia": setup_dustpedia(command_name, fs.cwd()) elif subproject == "evolve": setup_evolve(command_name, fs.cwd()) # Initialize the logger log = initialize_pts(config, remote=args.remote, command_name=command_name) # Exact command name exact_command_name = subproject + "/" + command_name # If the PTS command has to be executed remotely if args.remote is not None: run_remotely(exact_command_name, config, args.keep, args.remote, log) # The PTS command has to be executed locally else: run_locally(exact_command_name, module_path, class_name, config, args.input_files, args.output_files, args.output, log) # Finish function if subproject == "modeling": finish_modeling(command_name, fs.cwd(), config_path=config_filepath) elif subproject == "magic": finish_magic(command_name, fs.cwd()) elif subproject == "dustpedia": finish_dustpedia(command_name, fs.cwd()) elif subproject == "evolve": finish_evolve(command_name, fs.cwd())
def run_configurable(table_matches, args, tables): """ This function ... :param table_matches: :param args: :param tables: :return: """ # Determine the configuration method configuration_method = None if args.interactive: configuration_method = "interactive" elif args.arguments: configuration_method = "arguments" elif args.configfile is not None: configuration_method = "file:" + args.configfile elif args.rerun: configuration_method = "last" # Resolve subproject, index = table_matches[0] resolved = introspection.resolve_from_match(subproject, tables[subproject], index) # Get properties title = resolved.title command_name = resolved.command_name hidden = resolved.hidden description = resolved.description module_path = resolved.module_path class_name = resolved.class_name configuration_method_table = resolved.configuration_method configuration_module_path = resolved.configuration_module_path subproject_path = introspection.pts_subproject_dir(subproject) # Set sys.argv[0] = fs.join( introspection.pts_root_dir, module_path.replace(".", "/") + ".py" ) # this is actually not necessary (and not really correct, it's not like we are calling the module where the class is..) del sys.argv[1] # but this is important # Get a list of the leftover arguments leftover_arguments = sys.argv[1:] # Welcome message if subproject == "modeling": welcome_modeling() elif subproject == "magic": welcome_magic() elif subproject == "dustpedia": welcome_dustpedia() elif subproject == "evolve": welcome_evolve() # Get the configuration definition definition = introspection.get_configuration_definition_pts_not_yet_in_pythonpath( configuration_module_path) # If not specified on the command line (before the command name), then use the default specified in the commands.dat file if configuration_method is None: configuration_method = configuration_method_table # Check whether arguments are passed and the configuration method is interactive if configuration_method == "interactive" and len(leftover_arguments) > 0: raise ValueError( "Arguments on the command-line are not supported by default for this command. Run with pts --arguments to change this behaviour." ) # Create the configuration config = create_configuration(definition, command_name, description, configuration_method) ## SAVE THE CONFIG if requested if config.write_config: config.saveto(config.config_file_path(command_name)) # If this is not a re-run if not args.rerun: if not fs.is_directory(introspection.pts_user_config_dir): fs.create_directory(introspection.pts_user_config_dir) # CACHE THE CONFIG config_cache_path = fs.join(introspection.pts_user_config_dir, command_name + ".cfg") config.saveto(config_cache_path) # Setup function if subproject == "modeling": setup_modeling(command_name, fs.cwd()) elif subproject == "magic": setup_magic(command_name, fs.cwd()) elif subproject == "dustpedia": setup_dustpedia(command_name, fs.cwd()) elif subproject == "evolve": setup_evolve(command_name, fs.cwd()) # Initialize the logger log = initialize_pts(config, remote=args.remote, command_name=command_name) # Exact command name exact_command_name = subproject + "/" + command_name # If the PTS command has to be executed remotely if args.remote is not None: run_remotely(exact_command_name, config, args.keep, args.remote, log) # The PTS command has to be executed locally else: run_locally(exact_command_name, module_path, class_name, config, args.input_files, args.output_files, args.output, log) # Finish function if subproject == "modeling": finish_modeling(command_name, fs.cwd()) elif subproject == "magic": finish_magic(command_name, fs.cwd()) elif subproject == "dustpedia": finish_dustpedia(command_name, fs.cwd()) elif subproject == "evolve": finish_evolve(command_name, fs.cwd())
def get_dependencies(self): """ This function ... :return: """ # If no script name is given, list all dependencies of PTS and the # PTS modules that use them if self.config.script is None: if self.config.subprojects: self.dependencies = defaultdict(set) directories_for_subproject = defaultdict(list) # Loop over the subdirectories of the 'do' directory for path, name in fs.directories_in_path( introspection.pts_do_dir, returns=["path", "name"]): subproject = name directories_for_subproject[subproject].append(path) # Loop over the other directories in 'pts' (other than 'do' and 'doc') for path, name in fs.directories_in_path( introspection.pts_package_dir, returns=["path", "name"], exact_not_name=["do", "doc"]): subproject = name directories_for_subproject[subproject].append(path) encountered_internal_modules = set() for subproject in directories_for_subproject: # print(subproject, directories_for_subproject[subproject]) # List the dependencies of the matching script dependencies_for_this = defaultdict(set) for dir_path in directories_for_subproject[subproject]: for file_path in fs.files_in_path(dir_path, extension="py", recursive=True): # if subproject == "dustpedia": print(file_path) introspection.add_dependencies( dependencies_for_this, file_path, encountered_internal_modules) self.dependencies_for_subproject[ subproject] = dependencies_for_this.keys() for dependency in dependencies_for_this: for name in dependencies_for_this[dependency]: self.dependencies[dependency].add(name) # No subprojects are specified, list all PTS dependencies else: self.dependencies = introspection.get_all_dependencies() # If a script name is given else: scripts = introspection.get_scripts() tables = introspection.get_arguments_tables() # Find matching 'do' commands (actual scripts or tabulated commands) matches = introspection.find_matches_scripts( self.config.script, scripts) table_matches = introspection.find_matches_tables( self.config.script, tables) # List the dependencies of the matching script self.dependencies = defaultdict(set) # No match if len(matches) + len(table_matches) == 0: show_all_available(scripts, tables) exit() # More matches elif len(matches) + len(table_matches) > 1: show_possible_matches(matches, table_matches, tables) exit() # Exactly one match from existing do script elif len(matches) == 1 and len(table_matches) == 0: # Determine the full path to the matching script script_path = fs.join(introspection.pts_do_dir, matches[0][0], matches[0][1]) introspection.add_dependencies(self.dependencies, script_path, set()) # Exactly one match from tabulated command elif len(table_matches) == 1 and len(matches) == 0: # from pts.core.tools import logging # configuration # Path to class module table_match = table_matches[0] subproject = table_match[0] index = table_match[1] relative_class_module_path = tables[subproject]["Path"][ index].replace(".", "/").rsplit("/", 1)[0] + ".py" class_module_path = fs.join( introspection.pts_subproject_dir(subproject), relative_class_module_path) logging_path = fs.join(introspection.pts_package_dir, "core", "tools", "logging.py") command_name = tables[subproject]["Command"][index] configuration_name = tables[subproject]["Configuration"][index] if configuration_name == "--": configuration_name = command_name configuration_module_path = fs.join( introspection.pts_root_dir, "pts/" + subproject + "/config/" + configuration_name + ".py") # Add dependencies encountered = set() introspection.add_dependencies(self.dependencies, logging_path, encountered) introspection.add_dependencies(self.dependencies, configuration_module_path, encountered) introspection.add_dependencies(self.dependencies, class_module_path, encountered)
# Exactly one match from tabulated command elif len(table_matches) == 1 and len(matches) == 0: # from pts.core.tools import logging # configuration # Path to class module table_match = table_matches[0] subproject = table_match[0] index = table_match[1] relative_class_module_path = tables[subproject]["Path"][index].replace(".", "/").rsplit("/", 1)[0] + ".py" class_module_path = fs.join(introspection.pts_subproject_dir(subproject), relative_class_module_path) logging_path = fs.join(introspection.pts_package_dir, "core", "tools", "logging.py") command_name = tables[subproject]["Command"][index] configuration_name = tables[subproject]["Configuration"][index] if configuration_name == "--": configuration_name = command_name configuration_module_path = fs.join(introspection.pts_root_dir, "pts/" + subproject + "/config/" + configuration_name + ".py") # Add dependencies encountered = set() introspection.add_dependencies(dependencies, logging_path, encountered) introspection.add_dependencies(dependencies, configuration_module_path, encountered) introspection.add_dependencies(dependencies, class_module_path, encountered) # Get the names and versions of all installed python packages
def get_dependencies(self): """ This function ... :return: """ # If no script name is given, list all dependencies of PTS and the # PTS modules that use them if self.config.script is None: if self.config.subprojects: self.dependencies = defaultdict(set) directories_for_subproject = defaultdict(list) # Loop over the subdirectories of the 'do' directory for path, name in fs.directories_in_path(introspection.pts_do_dir, returns=["path", "name"]): subproject = name directories_for_subproject[subproject].append(path) # Loop over the other directories in 'pts' (other than 'do' and 'doc') for path, name in fs.directories_in_path(introspection.pts_package_dir, returns=["path", "name"], exact_not_name=["do", "doc"]): subproject = name directories_for_subproject[subproject].append(path) encountered_internal_modules = set() for subproject in directories_for_subproject: # print(subproject, directories_for_subproject[subproject]) # List the dependencies of the matching script dependencies_for_this = defaultdict(set) for dir_path in directories_for_subproject[subproject]: for file_path in fs.files_in_path(dir_path, extension="py", recursive=True): # if subproject == "dustpedia": print(file_path) introspection.add_dependencies(dependencies_for_this, file_path, encountered_internal_modules) self.dependencies_for_subproject[subproject] = dependencies_for_this.keys() for dependency in dependencies_for_this: for name in dependencies_for_this[dependency]: self.dependencies[dependency].add(name) # No subprojects are specified, list all PTS dependencies else: self.dependencies = introspection.get_all_dependencies() # If a script name is given else: scripts = introspection.get_scripts() tables = introspection.get_arguments_tables() # Find matching 'do' commands (actual scripts or tabulated commands) matches = introspection.find_matches_scripts(self.config.script, scripts) table_matches = introspection.find_matches_tables(self.config.script, tables) # List the dependencies of the matching script self.dependencies = defaultdict(set) # No match if len(matches) + len(table_matches) == 0: show_all_available(scripts, tables) exit() # More matches elif len(matches) + len(table_matches) > 1: show_possible_matches(matches, table_matches, tables) exit() # Exactly one match from existing do script elif len(matches) == 1 and len(table_matches) == 0: # Determine the full path to the matching script script_path = fs.join(introspection.pts_do_dir, matches[0][0], matches[0][1]) introspection.add_dependencies(self.dependencies, script_path, set()) # Exactly one match from tabulated command elif len(table_matches) == 1 and len(matches) == 0: # from pts.core.tools import logging # configuration # Path to class module table_match = table_matches[0] subproject = table_match[0] index = table_match[1] relative_class_module_path = tables[subproject]["Path"][index].replace(".", "/").rsplit("/", 1)[0] + ".py" class_module_path = fs.join(introspection.pts_subproject_dir(subproject), relative_class_module_path) logging_path = fs.join(introspection.pts_package_dir, "core", "basics", "log.py") command_name = tables[subproject]["Command"][index] configuration_name = tables[subproject]["Configuration"][index] if configuration_name == "--": configuration_name = command_name configuration_module_path = fs.join(introspection.pts_root_dir, "pts/" + subproject + "/config/" + configuration_name + ".py") # Add dependencies encountered = set() introspection.add_dependencies(self.dependencies, logging_path, encountered) introspection.add_dependencies(self.dependencies, configuration_module_path, encountered) introspection.add_dependencies(self.dependencies, class_module_path, encountered)