Exemplo n.º 1
0
# Ensure Python 3 compatibility
from __future__ import absolute_import, division, print_function

# Import the relevant PTS classes and modules
from pts.evolve.engine import GAEngine
from pts.core.tools import filesystem as fs
from pts.core.tools import tables
from pts.core.tools import time
from pts.core.tools.random import load_state, save_state

# -----------------------------------------------------------------

last_generation = None

# Check the index of the last generation
for name in fs.directories_in_path():
    if "reference" in name or "original" in name: continue
    generation = int(name.split("Generation ")[1])
    if last_generation is None or generation > last_generation: last_generation = generation

# -----------------------------------------------------------------

if last_generation is None:
    generation_path = fs.cwd()
    print("Current generation: the initial population")
else:
    generation_path = fs.join(fs.cwd(), "Generation " + str(last_generation))
    print("Current generation: " + str(last_generation))

# -----------------------------------------------------------------
Exemplo n.º 2
0
level = "DEBUG" if config.debug else "INFO"

# Initialize the logger
log = logging.setup_log(level=level, path=logfile_path)
log.start("Starting initialize_data ...")

# -----------------------------------------------------------------

names_column = []
paths_column = []
prep_names_column = []
names = ["Image name", "Image path", "Preparation name"]

# Loop over all subdirectories of the data directory
for path, name in fs.directories_in_path(fs.join(config.path, "data"),
                                         not_contains="bad",
                                         returns=["path", "name"]):

    # Loop over all FITS files found in the current subdirectory
    for image_path, image_name in fs.files_in_path(path,
                                                   extension="fits",
                                                   not_contains="_Error",
                                                   returns=["path", "name"]):

        # Open the image frame
        frame = Frame.from_file(image_path)

        # Determine the preparation name
        if frame.filter is not None: prep_name = str(frame.filter)
        else: prep_name = image_name
Exemplo n.º 3
0
# -----------------------------------------------------------------

modeling_path = verify_modeling_cwd()

# -----------------------------------------------------------------

# Load the modeling environment
environment = GalaxyModelingEnvironment(modeling_path)

# -----------------------------------------------------------------

# Create plotter
plotter = StandardImageGridPlotter()

# Loop over the directories in the truncation path
for path, name in fs.directories_in_path(environment.truncation_path,
                                         returns=["path", "name"]):

    # Determine the path to the lowres directory
    lowres_path = fs.join(path, "lowres")

    # Determine the filter
    fltr = parse_filter(name)
    filter_name = str(fltr)

    # Initializ variable
    the_image_path = None

    # Find the image corresponding to the specified factor
    for image_path, image_name in fs.files_in_path(lowres_path,
                                                   extension="fits",
                                                   returns=["path", "name"]):
Exemplo n.º 4
0
# Ensure Python 3 compatibility
from __future__ import absolute_import, division, print_function

# Import the relevant PTS classes and modules
from pts.evolve.engine import GAEngine
from pts.core.tools import filesystem as fs
from pts.core.tools import tables
from pts.core.tools import time
from pts.core.tools.random import load_state, save_state

# -----------------------------------------------------------------

last_generation = None

# Check the index of the last generation
for name in fs.directories_in_path():
    if "reference" in name or "original" in name: continue
    generation = int(name.split("Generation ")[1])
    if last_generation is None or generation > last_generation:
        last_generation = generation

# -----------------------------------------------------------------

if last_generation is None:
    generation_path = fs.cwd()
    print("Current generation: the initial population")
else:
    generation_path = fs.join(fs.cwd(), "Generation " + str(last_generation))
    print("Current generation: " + str(last_generation))

# -----------------------------------------------------------------
Exemplo n.º 5
0
definition.add_optional("not_contains", "string_list", "ignore directories containing this string in their name")
definition.add_optional("exact_not_name", "string_list", "ignore directories with these names")
definition.add_optional("output", "directory_path", "output directory")
definition.add_flag("per_host", "put the simulations in a separate subdirectory for each host")
definition.add_flag("rename", "rename the simulation files to have the name of the simulation")
definition.add_flag("ignore_missing", "ignore missing simulations (otherwise an error is thrown)")

# Create the configuration
config = parse_arguments("move_simulations", definition, "Move the simulation objects from the SKIRT run directory to another directory")

# -----------------------------------------------------------------

# Set simulation names
if config.from_directories:
    if config.names is not None: raise ValueError("Cannot specify names with 'from_directories' enabled")
    config.names = fs.directories_in_path(returns="name", not_contains=config.not_contains, exact_not_name=config.exact_not_name)

# -----------------------------------------------------------------

# Get simulations for each remote host
simulation_paths = dict()
for host_id in config.remotes:

    # Get the simulation paths
    paths_host = get_simulation_paths_for_host(host_id, as_dict=True)

    # Set the simulation paths
    simulation_paths[host_id] = paths_host

# -----------------------------------------------------------------
# Determine the log level
level = "DEBUG" if config.debug else "INFO"

# Initialize the logger
log = logging.setup_log(level=level, path=logfile_path)
log.start("Starting initialize_data ...")

# -----------------------------------------------------------------

names_column = []
paths_column = []
prep_names_column = []
names = ["Image name", "Image path", "Preparation name"]

# Loop over all subdirectories of the data directory
for path, name in fs.directories_in_path(fs.join(config.path, "data"), not_contains="bad", returns=["path", "name"]):

    # Loop over all FITS files found in the current subdirectory
    for image_path, image_name in fs.files_in_path(path, extension="fits", not_contains="_Error", returns=["path", "name"]):

        # Open the image frame
        frame = Frame.from_file(image_path)

        # Determine the preparation name
        if frame.filter is not None: prep_name = str(frame.filter)
        else: prep_name = image_name

        # Set the row entries
        names_column.append(image_name)
        paths_column.append(image_path)
        prep_names_column.append(prep_name)
Exemplo n.º 7
0
modeling_path = verify_modeling_cwd()

# -----------------------------------------------------------------

prep_path = fs.join(modeling_path, "prep")

# -----------------------------------------------------------------

progress = defaultdict(dict)

# -----------------------------------------------------------------

# Loop over all images of the initial dataset
#for path in fs.files_in_path(prep_path, recursive=True, exact_name="initialized"):
for directory_path in fs.directories_in_path(prep_path):

    # Directory path
    #directory_path = fs.directory_of(path)
    image_name = fs.name(directory_path)

    # Determine filter
    fltr = parse_filter(image_name)

    # Sort
    label, filepath = sort_image(image_name, directory_path, read_only=True)

    category = fltr.instrument if fltr.instrument is not None else "other"

    # Add to progress data
    progress[category][fltr.band] = label
Exemplo n.º 8
0
    def get_dependencies(self):
        """
        This function ...
        :return:
        """

        # If no script name is given, list all dependencies of PTS and the
        # PTS modules that use them
        if self.config.script is None:

            if self.config.subprojects:

                self.dependencies = defaultdict(set)

                directories_for_subproject = defaultdict(list)

                # Loop over the subdirectories of the 'do' directory
                for path, name in fs.directories_in_path(
                        introspection.pts_do_dir, returns=["path", "name"]):
                    subproject = name
                    directories_for_subproject[subproject].append(path)

                # Loop over the other directories in 'pts' (other than 'do' and 'doc')
                for path, name in fs.directories_in_path(
                        introspection.pts_package_dir,
                        returns=["path", "name"],
                        exact_not_name=["do", "doc"]):
                    subproject = name
                    directories_for_subproject[subproject].append(path)

                encountered_internal_modules = set()

                for subproject in directories_for_subproject:

                    # print(subproject, directories_for_subproject[subproject])

                    # List the dependencies of the matching script
                    dependencies_for_this = defaultdict(set)

                    for dir_path in directories_for_subproject[subproject]:

                        for file_path in fs.files_in_path(dir_path,
                                                          extension="py",
                                                          recursive=True):
                            # if subproject == "dustpedia": print(file_path)

                            introspection.add_dependencies(
                                dependencies_for_this, file_path,
                                encountered_internal_modules)

                    self.dependencies_for_subproject[
                        subproject] = dependencies_for_this.keys()

                    for dependency in dependencies_for_this:
                        for name in dependencies_for_this[dependency]:
                            self.dependencies[dependency].add(name)

            # No subprojects are specified, list all PTS dependencies
            else:
                self.dependencies = introspection.get_all_dependencies()

        # If a script name is given
        else:

            scripts = introspection.get_scripts()
            tables = introspection.get_arguments_tables()

            # Find matching 'do' commands (actual scripts or tabulated commands)
            matches = introspection.find_matches_scripts(
                self.config.script, scripts)
            table_matches = introspection.find_matches_tables(
                self.config.script, tables)

            # List the dependencies of the matching script
            self.dependencies = defaultdict(set)

            # No match
            if len(matches) + len(table_matches) == 0:
                show_all_available(scripts, tables)
                exit()

            # More matches
            elif len(matches) + len(table_matches) > 1:
                show_possible_matches(matches, table_matches, tables)
                exit()

            # Exactly one match from existing do script
            elif len(matches) == 1 and len(table_matches) == 0:

                # Determine the full path to the matching script
                script_path = fs.join(introspection.pts_do_dir, matches[0][0],
                                      matches[0][1])

                introspection.add_dependencies(self.dependencies, script_path,
                                               set())

            # Exactly one match from tabulated command
            elif len(table_matches) == 1 and len(matches) == 0:

                # from pts.core.tools import logging
                # configuration

                # Path to class module

                table_match = table_matches[0]
                subproject = table_match[0]
                index = table_match[1]

                relative_class_module_path = tables[subproject]["Path"][
                    index].replace(".", "/").rsplit("/", 1)[0] + ".py"

                class_module_path = fs.join(
                    introspection.pts_subproject_dir(subproject),
                    relative_class_module_path)

                logging_path = fs.join(introspection.pts_package_dir, "core",
                                       "tools", "logging.py")

                command_name = tables[subproject]["Command"][index]
                configuration_name = tables[subproject]["Configuration"][index]
                if configuration_name == "--":
                    configuration_name = command_name
                configuration_module_path = fs.join(
                    introspection.pts_root_dir, "pts/" + subproject +
                    "/config/" + configuration_name + ".py")

                # Add dependencies
                encountered = set()
                introspection.add_dependencies(self.dependencies, logging_path,
                                               encountered)
                introspection.add_dependencies(self.dependencies,
                                               configuration_module_path,
                                               encountered)
                introspection.add_dependencies(self.dependencies,
                                               class_module_path, encountered)
Exemplo n.º 9
0
# -----------------------------------------------------------------

modeling_path = verify_modeling_cwd()

# -----------------------------------------------------------------

# Load the modeling environment
environment = GalaxyModelingEnvironment(modeling_path)

# -----------------------------------------------------------------

# Create plotter
plotter = StandardImageGridPlotter()

# Loop over the directories in the truncation path
for path, name in fs.directories_in_path(environment.truncation_path, returns=["path", "name"]):

    # Determine the path to the lowres directory
    lowres_path = fs.join(path, "lowres")

    # Determine the filter
    fltr = parse_filter(name)
    filter_name = str(fltr)

    # Initializ variable
    the_image_path = None

    # Find the image corresponding to the specified factor
    for image_path, image_name in fs.files_in_path(lowres_path, extension="fits", returns=["path", "name"]):

        # Determine the factor
Exemplo n.º 10
0
simulated = dict()
for path, name in fs.files_in_path(fit_best_images_path, extension="fits", returns=["path", "name"]):
    if name == galaxy_name + "_earth_total": continue
    frame = Frame.from_file(path)
    if frame.filter is None: raise ValueError("No filter for " + name)
    simulated[str(frame.filter)] = frame
"""


trunc_path = fs.join(modeling_path, "truncated")
prep_path = fs.join(modeling_path, "prep")

observed = dict()

# Loop over the directories in the preparation directory
for path, name in fs.directories_in_path(prep_path, returns=["path", "name"]):

    # Skip Halpha
    if "Halpha" in name: continue

    # Determine the path to the prepared image
    result_path = fs.join(path, "result.fits")

    # Load the frame
    frame = Frame.from_file(result_path)
    if frame.filter is None: raise ValueError("No filter for " + name)

    observed[str(frame.filter)] = frame

# Sort the filter names on wavelength
sorted_filter_names = sorted(observed.keys(), key=lambda key: observed[key].filter.pivotwavelength())
Exemplo n.º 11
0
    def get_dependencies(self):

        """
        This function ...
        :return:
        """

        # If no script name is given, list all dependencies of PTS and the
        # PTS modules that use them
        if self.config.script is None:

            if self.config.subprojects:

                self.dependencies = defaultdict(set)

                directories_for_subproject = defaultdict(list)

                # Loop over the subdirectories of the 'do' directory
                for path, name in fs.directories_in_path(introspection.pts_do_dir, returns=["path", "name"]):
                    subproject = name
                    directories_for_subproject[subproject].append(path)

                # Loop over the other directories in 'pts' (other than 'do' and 'doc')
                for path, name in fs.directories_in_path(introspection.pts_package_dir, returns=["path", "name"],
                                                         exact_not_name=["do", "doc"]):
                    subproject = name
                    directories_for_subproject[subproject].append(path)

                encountered_internal_modules = set()

                for subproject in directories_for_subproject:

                    # print(subproject, directories_for_subproject[subproject])

                    # List the dependencies of the matching script
                    dependencies_for_this = defaultdict(set)

                    for dir_path in directories_for_subproject[subproject]:

                        for file_path in fs.files_in_path(dir_path, extension="py", recursive=True):
                            # if subproject == "dustpedia": print(file_path)

                            introspection.add_dependencies(dependencies_for_this, file_path,
                                                           encountered_internal_modules)

                    self.dependencies_for_subproject[subproject] = dependencies_for_this.keys()

                    for dependency in dependencies_for_this:
                        for name in dependencies_for_this[dependency]:
                            self.dependencies[dependency].add(name)

            # No subprojects are specified, list all PTS dependencies
            else: self.dependencies = introspection.get_all_dependencies()

        # If a script name is given
        else:

            scripts = introspection.get_scripts()
            tables = introspection.get_arguments_tables()

            # Find matching 'do' commands (actual scripts or tabulated commands)
            matches = introspection.find_matches_scripts(self.config.script, scripts)
            table_matches = introspection.find_matches_tables(self.config.script, tables)

            # List the dependencies of the matching script
            self.dependencies = defaultdict(set)

            # No match
            if len(matches) + len(table_matches) == 0:
                show_all_available(scripts, tables)
                exit()

            # More matches
            elif len(matches) + len(table_matches) > 1:
                show_possible_matches(matches, table_matches, tables)
                exit()

            # Exactly one match from existing do script
            elif len(matches) == 1 and len(table_matches) == 0:

                # Determine the full path to the matching script
                script_path = fs.join(introspection.pts_do_dir, matches[0][0], matches[0][1])

                introspection.add_dependencies(self.dependencies, script_path, set())

            # Exactly one match from tabulated command
            elif len(table_matches) == 1 and len(matches) == 0:

                # from pts.core.tools import logging
                # configuration

                # Path to class module

                table_match = table_matches[0]
                subproject = table_match[0]
                index = table_match[1]

                relative_class_module_path = tables[subproject]["Path"][index].replace(".", "/").rsplit("/", 1)[0] + ".py"
                class_module_path = fs.join(introspection.pts_subproject_dir(subproject), relative_class_module_path)
                logging_path = fs.join(introspection.pts_package_dir, "core", "basics", "log.py")

                command_name = tables[subproject]["Command"][index]
                configuration_name = tables[subproject]["Configuration"][index]
                if configuration_name == "--": configuration_name = command_name
                configuration_module_path = fs.join(introspection.pts_root_dir,
                                                    "pts/" + subproject + "/config/" + configuration_name + ".py")

                # Add dependencies
                encountered = set()
                introspection.add_dependencies(self.dependencies, logging_path, encountered)
                introspection.add_dependencies(self.dependencies, configuration_module_path, encountered)
                introspection.add_dependencies(self.dependencies, class_module_path, encountered)