Beispiel #1
0
    def __init__(self, output_path=None):
        """
        This function ...
        :param output_path:
        """

        # Set the output directory path
        self.output_path = output_path

        # The remote instance
        self.remote = None

        # Generate session ID
        self.session_id = time.unique_name()
Beispiel #2
0
    def __init__(self, output_path=None):

        """
        This function ...
        :param output_path:
        """

        # Set the output directory path
        self.output_path = output_path

        # The remote instance
        self.remote = None

        # Generate session ID
        self.session_id = time.unique_name()
Beispiel #3
0
Datei: run.py Projekt: SKIRT/PTS
def initialize_log(config, remote=None, command_name=None):

    """
    This function ...
    :param config:
    :parma remote:
    :param command_name:
    :return:
    """

    # Determine the log level
    level = "INFO"
    if config.debug: level = "DEBUG"
    if config.brief: level = "SUCCESS"

    # Determine memuse flag
    if hasattr(config, "memuse"): memuse = config.memuse
    else: memuse = False

    # Determine log path
    #if args.remote is None: logfile_path = fs.join(config.log_path, time.unique_name("log") + ".txt") if config.report else None
    #else: logfile_path = None

    # Determine name of log file
    if command_name is not None: filename = command_name
    else: filename = "log"

    # Determine the log file path
    if remote is None: logfile_path = fs.join(config.log_path, time.unique_name(filename, separator="__") + ".txt") if config.report else None
    else: logfile_path = None

    # Initialize the logger
    from ..core.basics.log import setup_log
    log = setup_log(level=level, path=logfile_path, memory=memuse)

    # Return the logger
    return log
Beispiel #4
0
def initialize_log(config, remote=None, command_name=None):
    """
    This function ...
    :param config:
    :parma remote:
    :param command_name:
    :return:
    """

    # Determine the log level
    level = "INFO"
    if config.debug: level = "DEBUG"
    if config.brief: level = "SUCCESS"

    # Determine log path
    #if args.remote is None: logfile_path = fs.join(config.log_path, time.unique_name("log") + ".txt") if config.report else None
    #else: logfile_path = None

    # Determine name of log file
    if command_name is not None: filename = command_name
    else: filename = "log"

    # Determine the log file path
    if remote is None:
        logfile_path = fs.join(config.log_path,
                               time.unique_name(filename) +
                               ".txt") if config.report else None
    else:
        logfile_path = None

    # Initialize the logger
    from ..core.basics.log import setup_log
    log = setup_log(level=level, path=logfile_path)

    # Return the logger
    return log
Beispiel #5
0
# -----------------------------------------------------------------

# Generate the new population
ga.generate_new_population()

# -----------------------------------------------------------------

name_column = []
par_a_column = []
par_b_column = []

for ind in ga.new_population:

    # Give the individual a unique name
    name = time.unique_name(precision="micro")
    name_column.append(name)
    par_a_column.append(ind.genomeList[0])
    par_b_column.append(ind.genomeList[1])

# Create the parameters table
data = [name_column, par_a_column, par_b_column]
names = ["Unique name", "Parameter a", "Parameter b"]
new_parameters_table = tables.new(data, names)

# -----------------------------------------------------------------

# Save the new parameters table
tables.write(new_parameters_table, new_parameters_path, format="ascii.ecsv")

# Dump the GA
Beispiel #6
0
# Flags
definition.add_flag("manual", "launch and inspect job scripts manually")
definition.add_flag(
    "keep", "keep the output generated by the different SKIRT simulations")

# Get configuration
reader = ConfigurationReader(
    "scaling", "Test the scaling of SKIRT on a particular system")
config = reader.read(definition)

# -----------------------------------------------------------------

# Determine the log file path
logfile_path = fs.join(fs.cwd(),
                       time.unique_name("scaling") +
                       ".txt") if config.report else None

# Determine the log level
level = "DEBUG" if config.debug else "INFO"

# Initialize the logger
log = logging.setup_log(level=level, path=logfile_path)
log.start("Starting scaling ...")

# -----------------------------------------------------------------

# Create a ScalingTest instance
test = ScalingTest(config)

# Run the scaling test
Beispiel #7
0
# -----------------------------------------------------------------

# Determine the full path to the parameter file
arguments.filepath = fs.absolute(arguments.file)

# Determine the full path to the input and output directories
if arguments.input is not None:
    arguments.input_path = fs.absolute(arguments.input)
if arguments.output is not None:
    arguments.output_path = fs.absolute(arguments.output)

# -----------------------------------------------------------------

# Determine the log file path
logfile_path = fs.join(fs.cwd(),
                       time.unique_name("launch") +
                       ".txt") if arguments.report else None

# Determine the log level
level = "DEBUG" if arguments.debug else "INFO"

# Initialize the logger
log = logging.setup_log(level=level, path=logfile_path)
log.start("Starting launch ...")

# -----------------------------------------------------------------

# Either create a SkirtRemoteLauncher or a SkirtLauncher
if arguments.remote: launcher = SkirtRemoteLauncher.from_arguments(arguments)
else: launcher = SkirtLauncher.from_arguments(arguments)
Beispiel #8
0
# -----------------------------------------------------------------

# Generate the new population
ga.generate_new_population()

# -----------------------------------------------------------------

name_column = []
par_a_column = []
par_b_column = []

for ind in ga.new_population:

    # Give the individual a unique name
    name = time.unique_name(precision="micro")
    name_column.append(name)
    par_a_column.append(ind.genomeList[0])
    par_b_column.append(ind.genomeList[1])

# Create the parameters table
data = [name_column, par_a_column, par_b_column]
names = ["Unique name", "Parameter a", "Parameter b"]
new_parameters_table = tables.new(data, names)

# -----------------------------------------------------------------

# Save the new parameters table
tables.write(new_parameters_table, new_parameters_path, format="ascii.ecsv")

# Dump the GA
Beispiel #9
0
# If an output directory is given
if arguments.output is not None:
    
    # Determine the full path to the output directory
    output_path = fs.absolute(arguments.output)
    
    # Create the directory if it does not yet exist
    if not fs.is_directory(output_path): fs.create_directory(output_path)

# If no output directory is given, place the output in the current working directory
else: output_path = fs.cwd()

# -----------------------------------------------------------------

# Determine the log file path
logfile_path = fs.join(output_path, time.unique_name("log") + ".txt") if arguments.report else None

# Determine the log level
level = "DEBUG" if arguments.debug else "INFO"

# Initialize the logger
log = logging.setup_log(level=level, path=logfile_path)
log.start("Starting find_sources ...")

# -----------------------------------------------------------------

# Determine the full path to the image
image_path = fs.absolute(arguments.image)

# Determine the full path to the bad region file
bad_region_path = fs.join(input_path, arguments.bad) if arguments.bad is not None else None
Beispiel #10
0
    # Check the factor
    if np.isclose(factor, config.factor, rtol=0.01):
        the_image_path = image_path
        break

# Check
if the_image_path is None:
    raise ValueError("Could not find the truncated " + filter_name +
                     " image with a truncation factor of " +
                     str(config.factor))

# -----------------------------------------------------------------

# Download the image to a temporary path
filename = time.unique_name("truncated_" + filter_name + "_" +
                            str(config.factor))
filepath = remote.download_file_to(the_image_path,
                                   pts_temp_dir,
                                   new_name=filename)

# -----------------------------------------------------------------

# Open the image
frame = Frame.from_file(filepath)

# -----------------------------------------------------------------

# Plot the truncated image
plotting.plot_box(frame)

# -----------------------------------------------------------------
Beispiel #11
0
# Check if key is active
#if host.key is not None:
#    if host.key not in active_keys(): add_key(host.key)

#command = "sshfs " + debug_flags + host.user + "@" + host.name + ": " + mount_path + " -C -o volname=" + host.id

# SSH COMMAND: "ssh -p 2935 -X [email protected]"

ssh_path = "/usr/bin/ssh"

#output = terminal.execute("which ssh")
#print(output)

temp_path = introspection.pts_temp_dir
temp_command_path = fs.join(temp_path,
                            time.unique_name("pts_login") + ".command")

#with tempfile.NamedTemporaryFile(suffix='.command') as f:
with open(temp_command_path, 'w') as f:

    f.write('#!/bin/sh\n')
    #f.write("python bb.py\n")

    #subprocess.call(['open', '-W', f.name])

    #ssh_command = "ssh "
    ssh_arguments = ""
    if host.port is not None: ssh_arguments += "-p " + str(host.port) + " "
    ssh_arguments += "-X "
    ssh_arguments += host.user
    ssh_arguments += "@"
Beispiel #12
0
sample = DustPediaSample()

# Database
username, password = get_account()
database = DustPediaDatabase()
database.login(username, password)

# Get the DustPedia name
name = sample.get_name(config.galaxy)

# Get info
info = database.get_galaxy_info_table(name)

# FIRST IMAGES AND THAN THIS FAILS FOR SOME REASON!!
path = fs.join(introspection.pts_temp_dir, time.unique_name("cutouts_" + name) + ".png")
database.download_photometry_cutouts(name, path)

# Get image names
#image_names = database.get_image_names(name, error_maps=False)
image_filters = database.get_image_filters(name)

print("")
print(fmt.red + fmt.underlined + "General info:" + fmt.reset)
print("")

print(" - position: " + str(sample.get_position(name)))
print(" - D25: " + str(sample.get_d25(name)))
print(" - R25: " + str(sample.get_r25(name)))
print("")
print(" - stage: " + str(info["Hubble Stage"][0]))
Beispiel #13
0
# If an output directory is given
if arguments.output is not None:
    
    # Determine the full path to the output directory
    output_path = fs.absolute(arguments.output)
    
    # Create the directory if it does not yet exist
    if not fs.is_directory(output_path): fs.create_directory(output_path)

# If no output directory is given, place the output in the current working directory
else: output_path = fs.cwd()

# -----------------------------------------------------------------

# Determine the log file path
logfile_path = fs.join(output_path, time.unique_name("log") + ".txt") if arguments.report else None

# Determine the log level
level = "DEBUG" if arguments.debug else "INFO"

# Initialize the logger
log = logging.setup_log(level=level, path=logfile_path)
log.start("Starting interpolate ...")

# -----------------------------------------------------------------

# Inform the user
log.info("Loading the image ...")

# Determine the full path to the image
image_path = fs.absolute(arguments.image)
Beispiel #14
0
                                   "number of generations (steps)", 10)

# Create the configuration
config = parse_arguments("test_tsp_steps", definition)

# Set logging
log = setup_log("DEBUG")

# -----------------------------------------------------------------

evaluator_kwargs = input_optimize["evaluator_kwargs"]

# -----------------------------------------------------------------

# Determine temporary directory
path = introspection.create_temp_dir(time.unique_name("optimize"))

# Change working directory
fs.change_cwd(path)

# -----------------------------------------------------------------

# Create the step optimizer
optimizer = StepWiseOptimizer(settings_optimize)

# Run the optimizer
population = optimizer.run(**input_optimize)

# Remove the variable, to load it again later
del optimizer
Beispiel #15
0
sample = DustPediaSample()

# Database
username, password = get_account()
database = DustPediaDatabase()
database.login(username, password)

# Get the DustPedia name
name = sample.get_name(config.galaxy)

# Get info
info = database.get_galaxy_info(name)

# FIRST IMAGES AND THAN THIS FAILS FOR SOME REASON!!
path = fs.join(introspection.pts_temp_dir,
               time.unique_name("cutouts_" + name) + ".png")
database.download_photometry_cutouts(name, path)

# Get image names
#image_names = database.get_image_names(name, error_maps=False)
image_filters = database.get_image_filters(name)

print("")
print(fmt.red + fmt.underlined + "General info:" + fmt.reset)
print("")

print(" - position: " + str(sample.get_position(name)))
print(" - D25: " + str(sample.get_d25(name)))
print(" - R25: " + str(sample.get_r25(name)))
print("")
print(" - stage: " + str(info["Hubble Stage"][0]))
Beispiel #16
0
# Optional
definition.add_optional("remote", str, "the remote host name", None)

# Get configuration
reader = ConfigurationReader(
    "get_poisson_errors",
    "Calculate poisson error maps for DustPedia UV and optical images")
config = reader.read(definition)
arguments = reader.get_arguments()

# -----------------------------------------------------------------

# Determine the log file path
logfile_path = fs.join(fs.cwd(),
                       time.unique_name("log") +
                       ".txt") if config.report else None

# Determine the log level
level = "DEBUG" if config.debug else "INFO"

# Initialize the logger
log = logging.setup_log(level=level, path=logfile_path)
log.start("Starting get_poisson_errors ...")

# -----------------------------------------------------------------

temp_name = time.unique_name(config.band.replace(" ", ""))

# -----------------------------------------------------------------
Beispiel #17
0
definition.add_optional("name",
                        "string",
                        "name for the new analysis run",
                        forbidden=analysis_runs.names)
config = parse_arguments("clone_analysis_run", definition)

# -----------------------------------------------------------------

# Get analysis run path
path = analysis_runs.get_path(config.analysis_run)

# -----------------------------------------------------------------

# Generate new analysis run name
if config.name is not None: new_name = config.name
else: new_name = time.unique_name()
new_path = fs.create_directory_in(environment.analysis_path, new_name)

# -----------------------------------------------------------------

clear_directories = [
    "out", "misc", "plot", "residuals", "heating", "extr", "colours",
    "attenuation"
]
fs.copy_from_directory(path,
                       new_path,
                       exact_not_name=clear_directories,
                       not_extension="sh")
fs.create_directories_in(new_path, clear_directories)

# -----------------------------------------------------------------
Beispiel #18
0
definition = ConfigurationDefinition()

# Add optional arguments
definition.add_optional(
    "image", str, "the name of the image for which to run the initialization")
definition.add_flag("visualise", "make visualisations")

# Get configuration
reader = ConfigurationReader("initialize_preparation")
config = reader.read(definition)

# -----------------------------------------------------------------

# Determine the log file path
logfile_path = fs.join(fs.cwd(), "log",
                       time.unique_name("log") +
                       ".txt") if config.report else None

# Determine the log level
level = "DEBUG" if config.debug else "INFO"

# Initialize the logger
log = logging.setup_log(level=level, path=logfile_path)
log.start("Starting initialize_data ...")

# -----------------------------------------------------------------

names_column = []
paths_column = []
prep_names_column = []
names = ["Image name", "Image path", "Preparation name"]
# Galaxy name
definition.add_required("galaxy_name", str, "the name of the galaxy")
definition.add_required("band", str, "the band (GALEX or SDSS u/g/r/i/z)")

# Optional
definition.add_optional("remote", str, "the remote host name", None)

# Get configuration
reader = ConfigurationReader("get_poisson_errors", "Calculate poisson error maps for DustPedia UV and optical images")
config = reader.read(definition)
arguments = reader.get_arguments()

# -----------------------------------------------------------------

# Determine the log file path
logfile_path = fs.join(fs.cwd(), time.unique_name("log") + ".txt") if config.report else None

# Determine the log level
level = "DEBUG" if config.debug else "INFO"

# Initialize the logger
log = logging.setup_log(level=level, path=logfile_path)
log.start("Starting get_poisson_errors ...")

# -----------------------------------------------------------------

temp_name = time.unique_name(config.band.replace(" ", ""))

# -----------------------------------------------------------------

# Remotely
Beispiel #20
0
definition.add_positional_optional("ngenerations", "positive_integer", "number of generations (steps)", 10)

# Create the configuration
config = parse_arguments("test_tsp_steps", definition)

# Set logging
log = setup_log("DEBUG")

# -----------------------------------------------------------------

evaluator_kwargs = input_optimize["evaluator_kwargs"]

# -----------------------------------------------------------------

# Determine temporary directory
path = introspection.create_temp_dir(time.unique_name("optimize"))

# Change working directory
fs.change_cwd(path)

# -----------------------------------------------------------------

# Create the step optimizer
optimizer = StepWiseOptimizer(settings_optimize)

# Run the optimizer
population = optimizer.run(**input_optimize)

# Remove the variable, to load it again later
del optimizer
Beispiel #21
0
# -----------------------------------------------------------------

# Determine the full path to the parameter file
arguments.filepath = fs.absolute(arguments.file)

# Determine the full path to the input and output directories
if arguments.input is not None:
    arguments.input_path = fs.absolute(arguments.input)
if arguments.output is not None:
    arguments.output_path = fs.absolute(arguments.output)

# -----------------------------------------------------------------

# Determine the log file path
logfile_path = fs.join(fs.cwd(), time.unique_name("launch") + ".txt") if arguments.report else None

# Determine the log level
level = "DEBUG" if arguments.debug else "INFO"

# Initialize the logger
log = logging.setup_log(level=level, path=logfile_path)
log.start("Starting launch ...")

# -----------------------------------------------------------------

# Either create a SkirtRemoteLauncher or a SkirtLauncher
if arguments.remote:
    launcher = SkirtRemoteLauncher.from_arguments(arguments)
else:
    launcher = SkirtLauncher.from_arguments(arguments)
Beispiel #22
0
                    help="(optional) the name of the remote host")
parser.add_argument("--debug",
                    action="store_true",
                    help="add this option to enable debug output")
parser.add_argument("--report",
                    action="store_true",
                    help="write a report file")

# Parse the command line arguments
arguments = parser.parse_args()

# -----------------------------------------------------------------

# Determine the log file path
logfile_path = os.path.join(os.getcwd(),
                            time.unique_name("status") +
                            ".txt") if arguments.report else None

# Determine the log level
level = "DEBUG" if arguments.debug else "INFO"

# Initialize the logger
log = logging.setup_log(level=level, path=logfile_path)
log.start("Starting status ...")

# -----------------------------------------------------------------

# Create a RemoteSynchronizer instance
synchronizer = RemoteSynchronizer.from_arguments(arguments)

# Run the synchronizer
Beispiel #23
0
# -----------------------------------------------------------------

# Create the command-line parser
parser = argparse.ArgumentParser()
parser.add_argument('remote', nargs='?', default=None, help="(optional) the name of the remote host")
parser.add_argument("--debug", action="store_true", help="add this option to enable debug output")
parser.add_argument("--report", action="store_true", help="write a report file")

# Parse the command line arguments
arguments = parser.parse_args()

# -----------------------------------------------------------------

# Determine the log file path
logfile_path = os.path.join(os.getcwd(), time.unique_name("status") + ".txt") if arguments.report else None

# Determine the log level
level = "DEBUG" if arguments.debug else "INFO"

# Initialize the logger
log = logging.setup_log(level=level, path=logfile_path)
log.start("Starting status ...")

# -----------------------------------------------------------------

# Create a RemoteSynchronizer instance
synchronizer = RemoteSynchronizer.from_arguments(arguments)

# Run the synchronizer
synchronizer.run()
Beispiel #24
0
# -----------------------------------------------------------------

# Create the configuration definition
definition = ConfigurationDefinition()

# Add setting
definition.add_required("step", str, "the modeling step for which to clear the output")

# Get the configuration
reader = ConfigurationReader("clear")
config = reader.read(definition)

# -----------------------------------------------------------------

# Determine the log file path
logfile_path = fs.join(fs.cwd(), "log", time.unique_name("log") + ".txt") if config.arguments.report else None

# Determine the log level
level = "DEBUG" if config.arguments.debug else "INFO"

# Initialize the logger
log = logging.setup_log(level=level, path=logfile_path)
log.start("Starting clear ...")

# -----------------------------------------------------------------

prep_path = fs.join(config.path, "prep")
components_path = fs.join(config.path, "components")
truncated_path = fs.join(config.path, "truncated")
phot_path = fs.join(config.path, "phot")
maps_path = fs.join(config.path, "maps")
Beispiel #25
0
else:
    figsize = "12,9"
    figsize_timelines = "12,12"

# -----------------------------------------------------------------

# Locate the scaling test suite directory
suite_path = fs.join(fs.cwd(), config.suite_name)
if not fs.is_directory(suite_path):
    raise ValueError("The directory '" + suite_path + "' does not exist")

# -----------------------------------------------------------------

# Make directory for output
output_path = fs.create_directory_in(fs.cwd(),
                                     time.unique_name("scaling_plots"))

# Make subdirectories
single_node_path = fs.create_directory_in(output_path,
                                          "Single-node comparison")
multi_node_path = fs.create_directory_in(
    output_path, "Load balancing and multi-node scaling")
communication_path = fs.create_directory_in(output_path, "Communication")
hybridization_path = fs.create_directory_in(output_path, "Hybridization")
photon_packages_path = fs.create_directory_in(
    output_path, "Increased number of photon packages")
memory_path = fs.create_directory_in(output_path, "Memory scaling")

# -----------------------------------------------------------------

pts_path = introspection.pts_executable_path
Beispiel #26
0
from pts.magic.misc.dustpedia import DustPediaDatabase, get_account
from pts.core.basics.configuration import ConfigurationDefinition, ConfigurationReader

# -----------------------------------------------------------------

# Create the configuration
definition = ConfigurationDefinition()

# Get configuration
reader = ConfigurationReader("plot_galaxies")
config = reader.read(definition)

# -----------------------------------------------------------------

# Determine the log file path
logfile_path = fs.join(fs.cwd(), time.unique_name("log") + ".txt") if config.report else None

# Determine the log level
level = "DEBUG" if config.debug else "INFO"

# Initialize the logger
log = logging.setup_log(level=level, path=logfile_path)
log.start("Starting plot_galaxies ...")

# -----------------------------------------------------------------

# Local table path
local_table_path = fs.join(introspection.pts_dat_dir("modeling"), "s4g", "s4g_p4_table8.dat")

# -----------------------------------------------------------------
Beispiel #27
0
    figsize = "8,6"
    figsize_timelines = "8,8"
else:
    figsize = "12,9"
    figsize_timelines = "12,12"

# -----------------------------------------------------------------

# Locate the scaling test suite directory
suite_path = fs.join(fs.cwd(), config.suite_name)
if not fs.is_directory(suite_path): raise ValueError("The directory '" + suite_path + "' does not exist")

# -----------------------------------------------------------------

# Make directory for output
output_path = fs.create_directory_in(fs.cwd(), time.unique_name("scaling_plots"))

# Make subdirectories
single_node_path = fs.create_directory_in(output_path, "Single-node comparison")
multi_node_path = fs.create_directory_in(output_path, "Load balancing and multi-node scaling")
communication_path = fs.create_directory_in(output_path, "Communication")
hybridization_path = fs.create_directory_in(output_path, "Hybridization")
photon_packages_path = fs.create_directory_in(output_path, "Increased number of photon packages")
memory_path = fs.create_directory_in(output_path, "Memory scaling")

# -----------------------------------------------------------------

pts_path = introspection.pts_executable_path

# -----------------------------------------------------------------