# Ensure Python 3 compatibility from __future__ import absolute_import, division, print_function # Import the relevant PTS classes and modules from pts.evolve.engine import GAEngine from pts.core.tools import filesystem as fs from pts.core.tools import tables from pts.core.tools import time from pts.core.tools.random import load_state, save_state # ----------------------------------------------------------------- last_generation = None # Check the index of the last generation for name in fs.directories_in_path(): if "reference" in name or "original" in name: continue generation = int(name.split("Generation ")[1]) if last_generation is None or generation > last_generation: last_generation = generation # ----------------------------------------------------------------- if last_generation is None: generation_path = fs.cwd() print("Current generation: the initial population") else: generation_path = fs.join(fs.cwd(), "Generation " + str(last_generation)) print("Current generation: " + str(last_generation)) # -----------------------------------------------------------------
level = "DEBUG" if config.debug else "INFO" # Initialize the logger log = logging.setup_log(level=level, path=logfile_path) log.start("Starting initialize_data ...") # ----------------------------------------------------------------- names_column = [] paths_column = [] prep_names_column = [] names = ["Image name", "Image path", "Preparation name"] # Loop over all subdirectories of the data directory for path, name in fs.directories_in_path(fs.join(config.path, "data"), not_contains="bad", returns=["path", "name"]): # Loop over all FITS files found in the current subdirectory for image_path, image_name in fs.files_in_path(path, extension="fits", not_contains="_Error", returns=["path", "name"]): # Open the image frame frame = Frame.from_file(image_path) # Determine the preparation name if frame.filter is not None: prep_name = str(frame.filter) else: prep_name = image_name
# ----------------------------------------------------------------- modeling_path = verify_modeling_cwd() # ----------------------------------------------------------------- # Load the modeling environment environment = GalaxyModelingEnvironment(modeling_path) # ----------------------------------------------------------------- # Create plotter plotter = StandardImageGridPlotter() # Loop over the directories in the truncation path for path, name in fs.directories_in_path(environment.truncation_path, returns=["path", "name"]): # Determine the path to the lowres directory lowres_path = fs.join(path, "lowres") # Determine the filter fltr = parse_filter(name) filter_name = str(fltr) # Initializ variable the_image_path = None # Find the image corresponding to the specified factor for image_path, image_name in fs.files_in_path(lowres_path, extension="fits", returns=["path", "name"]):
definition.add_optional("not_contains", "string_list", "ignore directories containing this string in their name") definition.add_optional("exact_not_name", "string_list", "ignore directories with these names") definition.add_optional("output", "directory_path", "output directory") definition.add_flag("per_host", "put the simulations in a separate subdirectory for each host") definition.add_flag("rename", "rename the simulation files to have the name of the simulation") definition.add_flag("ignore_missing", "ignore missing simulations (otherwise an error is thrown)") # Create the configuration config = parse_arguments("move_simulations", definition, "Move the simulation objects from the SKIRT run directory to another directory") # ----------------------------------------------------------------- # Set simulation names if config.from_directories: if config.names is not None: raise ValueError("Cannot specify names with 'from_directories' enabled") config.names = fs.directories_in_path(returns="name", not_contains=config.not_contains, exact_not_name=config.exact_not_name) # ----------------------------------------------------------------- # Get simulations for each remote host simulation_paths = dict() for host_id in config.remotes: # Get the simulation paths paths_host = get_simulation_paths_for_host(host_id, as_dict=True) # Set the simulation paths simulation_paths[host_id] = paths_host # -----------------------------------------------------------------
# Determine the log level level = "DEBUG" if config.debug else "INFO" # Initialize the logger log = logging.setup_log(level=level, path=logfile_path) log.start("Starting initialize_data ...") # ----------------------------------------------------------------- names_column = [] paths_column = [] prep_names_column = [] names = ["Image name", "Image path", "Preparation name"] # Loop over all subdirectories of the data directory for path, name in fs.directories_in_path(fs.join(config.path, "data"), not_contains="bad", returns=["path", "name"]): # Loop over all FITS files found in the current subdirectory for image_path, image_name in fs.files_in_path(path, extension="fits", not_contains="_Error", returns=["path", "name"]): # Open the image frame frame = Frame.from_file(image_path) # Determine the preparation name if frame.filter is not None: prep_name = str(frame.filter) else: prep_name = image_name # Set the row entries names_column.append(image_name) paths_column.append(image_path) prep_names_column.append(prep_name)
modeling_path = verify_modeling_cwd() # ----------------------------------------------------------------- prep_path = fs.join(modeling_path, "prep") # ----------------------------------------------------------------- progress = defaultdict(dict) # ----------------------------------------------------------------- # Loop over all images of the initial dataset #for path in fs.files_in_path(prep_path, recursive=True, exact_name="initialized"): for directory_path in fs.directories_in_path(prep_path): # Directory path #directory_path = fs.directory_of(path) image_name = fs.name(directory_path) # Determine filter fltr = parse_filter(image_name) # Sort label, filepath = sort_image(image_name, directory_path, read_only=True) category = fltr.instrument if fltr.instrument is not None else "other" # Add to progress data progress[category][fltr.band] = label
def get_dependencies(self): """ This function ... :return: """ # If no script name is given, list all dependencies of PTS and the # PTS modules that use them if self.config.script is None: if self.config.subprojects: self.dependencies = defaultdict(set) directories_for_subproject = defaultdict(list) # Loop over the subdirectories of the 'do' directory for path, name in fs.directories_in_path( introspection.pts_do_dir, returns=["path", "name"]): subproject = name directories_for_subproject[subproject].append(path) # Loop over the other directories in 'pts' (other than 'do' and 'doc') for path, name in fs.directories_in_path( introspection.pts_package_dir, returns=["path", "name"], exact_not_name=["do", "doc"]): subproject = name directories_for_subproject[subproject].append(path) encountered_internal_modules = set() for subproject in directories_for_subproject: # print(subproject, directories_for_subproject[subproject]) # List the dependencies of the matching script dependencies_for_this = defaultdict(set) for dir_path in directories_for_subproject[subproject]: for file_path in fs.files_in_path(dir_path, extension="py", recursive=True): # if subproject == "dustpedia": print(file_path) introspection.add_dependencies( dependencies_for_this, file_path, encountered_internal_modules) self.dependencies_for_subproject[ subproject] = dependencies_for_this.keys() for dependency in dependencies_for_this: for name in dependencies_for_this[dependency]: self.dependencies[dependency].add(name) # No subprojects are specified, list all PTS dependencies else: self.dependencies = introspection.get_all_dependencies() # If a script name is given else: scripts = introspection.get_scripts() tables = introspection.get_arguments_tables() # Find matching 'do' commands (actual scripts or tabulated commands) matches = introspection.find_matches_scripts( self.config.script, scripts) table_matches = introspection.find_matches_tables( self.config.script, tables) # List the dependencies of the matching script self.dependencies = defaultdict(set) # No match if len(matches) + len(table_matches) == 0: show_all_available(scripts, tables) exit() # More matches elif len(matches) + len(table_matches) > 1: show_possible_matches(matches, table_matches, tables) exit() # Exactly one match from existing do script elif len(matches) == 1 and len(table_matches) == 0: # Determine the full path to the matching script script_path = fs.join(introspection.pts_do_dir, matches[0][0], matches[0][1]) introspection.add_dependencies(self.dependencies, script_path, set()) # Exactly one match from tabulated command elif len(table_matches) == 1 and len(matches) == 0: # from pts.core.tools import logging # configuration # Path to class module table_match = table_matches[0] subproject = table_match[0] index = table_match[1] relative_class_module_path = tables[subproject]["Path"][ index].replace(".", "/").rsplit("/", 1)[0] + ".py" class_module_path = fs.join( introspection.pts_subproject_dir(subproject), relative_class_module_path) logging_path = fs.join(introspection.pts_package_dir, "core", "tools", "logging.py") command_name = tables[subproject]["Command"][index] configuration_name = tables[subproject]["Configuration"][index] if configuration_name == "--": configuration_name = command_name configuration_module_path = fs.join( introspection.pts_root_dir, "pts/" + subproject + "/config/" + configuration_name + ".py") # Add dependencies encountered = set() introspection.add_dependencies(self.dependencies, logging_path, encountered) introspection.add_dependencies(self.dependencies, configuration_module_path, encountered) introspection.add_dependencies(self.dependencies, class_module_path, encountered)
# ----------------------------------------------------------------- modeling_path = verify_modeling_cwd() # ----------------------------------------------------------------- # Load the modeling environment environment = GalaxyModelingEnvironment(modeling_path) # ----------------------------------------------------------------- # Create plotter plotter = StandardImageGridPlotter() # Loop over the directories in the truncation path for path, name in fs.directories_in_path(environment.truncation_path, returns=["path", "name"]): # Determine the path to the lowres directory lowres_path = fs.join(path, "lowres") # Determine the filter fltr = parse_filter(name) filter_name = str(fltr) # Initializ variable the_image_path = None # Find the image corresponding to the specified factor for image_path, image_name in fs.files_in_path(lowres_path, extension="fits", returns=["path", "name"]): # Determine the factor
simulated = dict() for path, name in fs.files_in_path(fit_best_images_path, extension="fits", returns=["path", "name"]): if name == galaxy_name + "_earth_total": continue frame = Frame.from_file(path) if frame.filter is None: raise ValueError("No filter for " + name) simulated[str(frame.filter)] = frame """ trunc_path = fs.join(modeling_path, "truncated") prep_path = fs.join(modeling_path, "prep") observed = dict() # Loop over the directories in the preparation directory for path, name in fs.directories_in_path(prep_path, returns=["path", "name"]): # Skip Halpha if "Halpha" in name: continue # Determine the path to the prepared image result_path = fs.join(path, "result.fits") # Load the frame frame = Frame.from_file(result_path) if frame.filter is None: raise ValueError("No filter for " + name) observed[str(frame.filter)] = frame # Sort the filter names on wavelength sorted_filter_names = sorted(observed.keys(), key=lambda key: observed[key].filter.pivotwavelength())
def get_dependencies(self): """ This function ... :return: """ # If no script name is given, list all dependencies of PTS and the # PTS modules that use them if self.config.script is None: if self.config.subprojects: self.dependencies = defaultdict(set) directories_for_subproject = defaultdict(list) # Loop over the subdirectories of the 'do' directory for path, name in fs.directories_in_path(introspection.pts_do_dir, returns=["path", "name"]): subproject = name directories_for_subproject[subproject].append(path) # Loop over the other directories in 'pts' (other than 'do' and 'doc') for path, name in fs.directories_in_path(introspection.pts_package_dir, returns=["path", "name"], exact_not_name=["do", "doc"]): subproject = name directories_for_subproject[subproject].append(path) encountered_internal_modules = set() for subproject in directories_for_subproject: # print(subproject, directories_for_subproject[subproject]) # List the dependencies of the matching script dependencies_for_this = defaultdict(set) for dir_path in directories_for_subproject[subproject]: for file_path in fs.files_in_path(dir_path, extension="py", recursive=True): # if subproject == "dustpedia": print(file_path) introspection.add_dependencies(dependencies_for_this, file_path, encountered_internal_modules) self.dependencies_for_subproject[subproject] = dependencies_for_this.keys() for dependency in dependencies_for_this: for name in dependencies_for_this[dependency]: self.dependencies[dependency].add(name) # No subprojects are specified, list all PTS dependencies else: self.dependencies = introspection.get_all_dependencies() # If a script name is given else: scripts = introspection.get_scripts() tables = introspection.get_arguments_tables() # Find matching 'do' commands (actual scripts or tabulated commands) matches = introspection.find_matches_scripts(self.config.script, scripts) table_matches = introspection.find_matches_tables(self.config.script, tables) # List the dependencies of the matching script self.dependencies = defaultdict(set) # No match if len(matches) + len(table_matches) == 0: show_all_available(scripts, tables) exit() # More matches elif len(matches) + len(table_matches) > 1: show_possible_matches(matches, table_matches, tables) exit() # Exactly one match from existing do script elif len(matches) == 1 and len(table_matches) == 0: # Determine the full path to the matching script script_path = fs.join(introspection.pts_do_dir, matches[0][0], matches[0][1]) introspection.add_dependencies(self.dependencies, script_path, set()) # Exactly one match from tabulated command elif len(table_matches) == 1 and len(matches) == 0: # from pts.core.tools import logging # configuration # Path to class module table_match = table_matches[0] subproject = table_match[0] index = table_match[1] relative_class_module_path = tables[subproject]["Path"][index].replace(".", "/").rsplit("/", 1)[0] + ".py" class_module_path = fs.join(introspection.pts_subproject_dir(subproject), relative_class_module_path) logging_path = fs.join(introspection.pts_package_dir, "core", "basics", "log.py") command_name = tables[subproject]["Command"][index] configuration_name = tables[subproject]["Configuration"][index] if configuration_name == "--": configuration_name = command_name configuration_module_path = fs.join(introspection.pts_root_dir, "pts/" + subproject + "/config/" + configuration_name + ".py") # Add dependencies encountered = set() introspection.add_dependencies(self.dependencies, logging_path, encountered) introspection.add_dependencies(self.dependencies, configuration_module_path, encountered) introspection.add_dependencies(self.dependencies, class_module_path, encountered)