def __init__(self, output_path=None):
"""
This function ...
:param output_path:
"""
# Set the output directory path
self.output_path = output_path
# The remote instance
self.remote = None
# Generate session ID
self.session_id = time.unique_name()
def __init__(self, output_path=None):
"""
This function ...
:param output_path:
"""
# Set the output directory path
self.output_path = output_path
# The remote instance
self.remote = None
# Generate session ID
self.session_id = time.unique_name()
def initialize_log(config, remote=None, command_name=None):
"""
This function ...
:param config:
:parma remote:
:param command_name:
:return:
"""
# Determine the log level
level = "INFO"
if config.debug: level = "DEBUG"
if config.brief: level = "SUCCESS"
# Determine memuse flag
if hasattr(config, "memuse"): memuse = config.memuse
else: memuse = False
# Determine log path
#if args.remote is None: logfile_path = fs.join(config.log_path, time.unique_name("log") + ".txt") if config.report else None
#else: logfile_path = None
# Determine name of log file
if command_name is not None: filename = command_name
else: filename = "log"
# Determine the log file path
if remote is None: logfile_path = fs.join(config.log_path, time.unique_name(filename, separator="__") + ".txt") if config.report else None
else: logfile_path = None
# Initialize the logger
from ..core.basics.log import setup_log
log = setup_log(level=level, path=logfile_path, memory=memuse)
# Return the logger
return log
def initialize_log(config, remote=None, command_name=None):
"""
This function ...
:param config:
:parma remote:
:param command_name:
:return:
"""
# Determine the log level
level = "INFO"
if config.debug: level = "DEBUG"
if config.brief: level = "SUCCESS"
# Determine log path
#if args.remote is None: logfile_path = fs.join(config.log_path, time.unique_name("log") + ".txt") if config.report else None
#else: logfile_path = None
# Determine name of log file
if command_name is not None: filename = command_name
else: filename = "log"
# Determine the log file path
if remote is None:
logfile_path = fs.join(config.log_path,
time.unique_name(filename) +
".txt") if config.report else None
else:
logfile_path = None
# Initialize the logger
from ..core.basics.log import setup_log
log = setup_log(level=level, path=logfile_path)
# Return the logger
return log
# -----------------------------------------------------------------
# Generate the new population
ga.generate_new_population()
# -----------------------------------------------------------------
name_column = []
par_a_column = []
par_b_column = []
for ind in ga.new_population:
# Give the individual a unique name
name = time.unique_name(precision="micro")
name_column.append(name)
par_a_column.append(ind.genomeList[0])
par_b_column.append(ind.genomeList[1])
# Create the parameters table
data = [name_column, par_a_column, par_b_column]
names = ["Unique name", "Parameter a", "Parameter b"]
new_parameters_table = tables.new(data, names)
# -----------------------------------------------------------------
# Save the new parameters table
tables.write(new_parameters_table, new_parameters_path, format="ascii.ecsv")
# Dump the GA
# Flags
definition.add_flag("manual", "launch and inspect job scripts manually")
definition.add_flag(
"keep", "keep the output generated by the different SKIRT simulations")
# Get configuration
reader = ConfigurationReader(
"scaling", "Test the scaling of SKIRT on a particular system")
config = reader.read(definition)
# -----------------------------------------------------------------
# Determine the log file path
logfile_path = fs.join(fs.cwd(),
time.unique_name("scaling") +
".txt") if config.report else None
# Determine the log level
level = "DEBUG" if config.debug else "INFO"
# Initialize the logger
log = logging.setup_log(level=level, path=logfile_path)
log.start("Starting scaling ...")
# -----------------------------------------------------------------
# Create a ScalingTest instance
test = ScalingTest(config)
# Run the scaling test
# -----------------------------------------------------------------
# Determine the full path to the parameter file
arguments.filepath = fs.absolute(arguments.file)
# Determine the full path to the input and output directories
if arguments.input is not None:
arguments.input_path = fs.absolute(arguments.input)
if arguments.output is not None:
arguments.output_path = fs.absolute(arguments.output)
# -----------------------------------------------------------------
# Determine the log file path
logfile_path = fs.join(fs.cwd(),
time.unique_name("launch") +
".txt") if arguments.report else None
# Determine the log level
level = "DEBUG" if arguments.debug else "INFO"
# Initialize the logger
log = logging.setup_log(level=level, path=logfile_path)
log.start("Starting launch ...")
# -----------------------------------------------------------------
# Either create a SkirtRemoteLauncher or a SkirtLauncher
if arguments.remote: launcher = SkirtRemoteLauncher.from_arguments(arguments)
else: launcher = SkirtLauncher.from_arguments(arguments)
# -----------------------------------------------------------------
# Generate the new population
ga.generate_new_population()
# -----------------------------------------------------------------
name_column = []
par_a_column = []
par_b_column = []
for ind in ga.new_population:
# Give the individual a unique name
name = time.unique_name(precision="micro")
name_column.append(name)
par_a_column.append(ind.genomeList[0])
par_b_column.append(ind.genomeList[1])
# Create the parameters table
data = [name_column, par_a_column, par_b_column]
names = ["Unique name", "Parameter a", "Parameter b"]
new_parameters_table = tables.new(data, names)
# -----------------------------------------------------------------
# Save the new parameters table
tables.write(new_parameters_table, new_parameters_path, format="ascii.ecsv")
# Dump the GA
# If an output directory is given
if arguments.output is not None:
# Determine the full path to the output directory
output_path = fs.absolute(arguments.output)
# Create the directory if it does not yet exist
if not fs.is_directory(output_path): fs.create_directory(output_path)
# If no output directory is given, place the output in the current working directory
else: output_path = fs.cwd()
# -----------------------------------------------------------------
# Determine the log file path
logfile_path = fs.join(output_path, time.unique_name("log") + ".txt") if arguments.report else None
# Determine the log level
level = "DEBUG" if arguments.debug else "INFO"
# Initialize the logger
log = logging.setup_log(level=level, path=logfile_path)
log.start("Starting find_sources ...")
# -----------------------------------------------------------------
# Determine the full path to the image
image_path = fs.absolute(arguments.image)
# Determine the full path to the bad region file
bad_region_path = fs.join(input_path, arguments.bad) if arguments.bad is not None else None
# Check the factor
if np.isclose(factor, config.factor, rtol=0.01):
the_image_path = image_path
break
# Check
if the_image_path is None:
raise ValueError("Could not find the truncated " + filter_name +
" image with a truncation factor of " +
str(config.factor))
# -----------------------------------------------------------------
# Download the image to a temporary path
filename = time.unique_name("truncated_" + filter_name + "_" +
str(config.factor))
filepath = remote.download_file_to(the_image_path,
pts_temp_dir,
new_name=filename)
# -----------------------------------------------------------------
# Open the image
frame = Frame.from_file(filepath)
# -----------------------------------------------------------------
# Plot the truncated image
plotting.plot_box(frame)
# -----------------------------------------------------------------
# Check if key is active #if host.key is not None: # if host.key not in active_keys(): add_key(host.key) #command = "sshfs " + debug_flags + host.user + "@" + host.name + ": " + mount_path + " -C -o volname=" + host.id # SSH COMMAND: "ssh -p 2935 -X [email protected]" ssh_path = "/usr/bin/ssh" #output = terminal.execute("which ssh") #print(output) temp_path = introspection.pts_temp_dir temp_command_path = fs.join(temp_path, time.unique_name("pts_login") + ".command") #with tempfile.NamedTemporaryFile(suffix='.command') as f: with open(temp_command_path, 'w') as f: f.write('#!/bin/sh\n') #f.write("python bb.py\n") #subprocess.call(['open', '-W', f.name]) #ssh_command = "ssh " ssh_arguments = "" if host.port is not None: ssh_arguments += "-p " + str(host.port) + " " ssh_arguments += "-X " ssh_arguments += host.user ssh_arguments += "@"
sample = DustPediaSample()
# Database
username, password = get_account()
database = DustPediaDatabase()
database.login(username, password)
# Get the DustPedia name
name = sample.get_name(config.galaxy)
# Get info
info = database.get_galaxy_info_table(name)
# FIRST IMAGES AND THAN THIS FAILS FOR SOME REASON!!
path = fs.join(introspection.pts_temp_dir, time.unique_name("cutouts_" + name) + ".png")
database.download_photometry_cutouts(name, path)
# Get image names
#image_names = database.get_image_names(name, error_maps=False)
image_filters = database.get_image_filters(name)
print("")
print(fmt.red + fmt.underlined + "General info:" + fmt.reset)
print("")
print(" - position: " + str(sample.get_position(name)))
print(" - D25: " + str(sample.get_d25(name)))
print(" - R25: " + str(sample.get_r25(name)))
print("")
print(" - stage: " + str(info["Hubble Stage"][0]))
# If an output directory is given
if arguments.output is not None:
# Determine the full path to the output directory
output_path = fs.absolute(arguments.output)
# Create the directory if it does not yet exist
if not fs.is_directory(output_path): fs.create_directory(output_path)
# If no output directory is given, place the output in the current working directory
else: output_path = fs.cwd()
# -----------------------------------------------------------------
# Determine the log file path
logfile_path = fs.join(output_path, time.unique_name("log") + ".txt") if arguments.report else None
# Determine the log level
level = "DEBUG" if arguments.debug else "INFO"
# Initialize the logger
log = logging.setup_log(level=level, path=logfile_path)
log.start("Starting interpolate ...")
# -----------------------------------------------------------------
# Inform the user
log.info("Loading the image ...")
# Determine the full path to the image
image_path = fs.absolute(arguments.image)
"number of generations (steps)", 10)
# Create the configuration
config = parse_arguments("test_tsp_steps", definition)
# Set logging
log = setup_log("DEBUG")
# -----------------------------------------------------------------
evaluator_kwargs = input_optimize["evaluator_kwargs"]
# -----------------------------------------------------------------
# Determine temporary directory
path = introspection.create_temp_dir(time.unique_name("optimize"))
# Change working directory
fs.change_cwd(path)
# -----------------------------------------------------------------
# Create the step optimizer
optimizer = StepWiseOptimizer(settings_optimize)
# Run the optimizer
population = optimizer.run(**input_optimize)
# Remove the variable, to load it again later
del optimizer
sample = DustPediaSample()
# Database
username, password = get_account()
database = DustPediaDatabase()
database.login(username, password)
# Get the DustPedia name
name = sample.get_name(config.galaxy)
# Get info
info = database.get_galaxy_info(name)
# FIRST IMAGES AND THAN THIS FAILS FOR SOME REASON!!
path = fs.join(introspection.pts_temp_dir,
time.unique_name("cutouts_" + name) + ".png")
database.download_photometry_cutouts(name, path)
# Get image names
#image_names = database.get_image_names(name, error_maps=False)
image_filters = database.get_image_filters(name)
print("")
print(fmt.red + fmt.underlined + "General info:" + fmt.reset)
print("")
print(" - position: " + str(sample.get_position(name)))
print(" - D25: " + str(sample.get_d25(name)))
print(" - R25: " + str(sample.get_r25(name)))
print("")
print(" - stage: " + str(info["Hubble Stage"][0]))
# Optional
definition.add_optional("remote", str, "the remote host name", None)
# Get configuration
reader = ConfigurationReader(
"get_poisson_errors",
"Calculate poisson error maps for DustPedia UV and optical images")
config = reader.read(definition)
arguments = reader.get_arguments()
# -----------------------------------------------------------------
# Determine the log file path
logfile_path = fs.join(fs.cwd(),
time.unique_name("log") +
".txt") if config.report else None
# Determine the log level
level = "DEBUG" if config.debug else "INFO"
# Initialize the logger
log = logging.setup_log(level=level, path=logfile_path)
log.start("Starting get_poisson_errors ...")
# -----------------------------------------------------------------
temp_name = time.unique_name(config.band.replace(" ", ""))
# -----------------------------------------------------------------
definition.add_optional("name",
"string",
"name for the new analysis run",
forbidden=analysis_runs.names)
config = parse_arguments("clone_analysis_run", definition)
# -----------------------------------------------------------------
# Get analysis run path
path = analysis_runs.get_path(config.analysis_run)
# -----------------------------------------------------------------
# Generate new analysis run name
if config.name is not None: new_name = config.name
else: new_name = time.unique_name()
new_path = fs.create_directory_in(environment.analysis_path, new_name)
# -----------------------------------------------------------------
clear_directories = [
"out", "misc", "plot", "residuals", "heating", "extr", "colours",
"attenuation"
]
fs.copy_from_directory(path,
new_path,
exact_not_name=clear_directories,
not_extension="sh")
fs.create_directories_in(new_path, clear_directories)
# -----------------------------------------------------------------
definition = ConfigurationDefinition()
# Add optional arguments
definition.add_optional(
"image", str, "the name of the image for which to run the initialization")
definition.add_flag("visualise", "make visualisations")
# Get configuration
reader = ConfigurationReader("initialize_preparation")
config = reader.read(definition)
# -----------------------------------------------------------------
# Determine the log file path
logfile_path = fs.join(fs.cwd(), "log",
time.unique_name("log") +
".txt") if config.report else None
# Determine the log level
level = "DEBUG" if config.debug else "INFO"
# Initialize the logger
log = logging.setup_log(level=level, path=logfile_path)
log.start("Starting initialize_data ...")
# -----------------------------------------------------------------
names_column = []
paths_column = []
prep_names_column = []
names = ["Image name", "Image path", "Preparation name"]
# Galaxy name
definition.add_required("galaxy_name", str, "the name of the galaxy")
definition.add_required("band", str, "the band (GALEX or SDSS u/g/r/i/z)")
# Optional
definition.add_optional("remote", str, "the remote host name", None)
# Get configuration
reader = ConfigurationReader("get_poisson_errors", "Calculate poisson error maps for DustPedia UV and optical images")
config = reader.read(definition)
arguments = reader.get_arguments()
# -----------------------------------------------------------------
# Determine the log file path
logfile_path = fs.join(fs.cwd(), time.unique_name("log") + ".txt") if config.report else None
# Determine the log level
level = "DEBUG" if config.debug else "INFO"
# Initialize the logger
log = logging.setup_log(level=level, path=logfile_path)
log.start("Starting get_poisson_errors ...")
# -----------------------------------------------------------------
temp_name = time.unique_name(config.band.replace(" ", ""))
# -----------------------------------------------------------------
# Remotely
definition.add_positional_optional("ngenerations", "positive_integer", "number of generations (steps)", 10)
# Create the configuration
config = parse_arguments("test_tsp_steps", definition)
# Set logging
log = setup_log("DEBUG")
# -----------------------------------------------------------------
evaluator_kwargs = input_optimize["evaluator_kwargs"]
# -----------------------------------------------------------------
# Determine temporary directory
path = introspection.create_temp_dir(time.unique_name("optimize"))
# Change working directory
fs.change_cwd(path)
# -----------------------------------------------------------------
# Create the step optimizer
optimizer = StepWiseOptimizer(settings_optimize)
# Run the optimizer
population = optimizer.run(**input_optimize)
# Remove the variable, to load it again later
del optimizer
# -----------------------------------------------------------------
# Determine the full path to the parameter file
arguments.filepath = fs.absolute(arguments.file)
# Determine the full path to the input and output directories
if arguments.input is not None:
arguments.input_path = fs.absolute(arguments.input)
if arguments.output is not None:
arguments.output_path = fs.absolute(arguments.output)
# -----------------------------------------------------------------
# Determine the log file path
logfile_path = fs.join(fs.cwd(), time.unique_name("launch") + ".txt") if arguments.report else None
# Determine the log level
level = "DEBUG" if arguments.debug else "INFO"
# Initialize the logger
log = logging.setup_log(level=level, path=logfile_path)
log.start("Starting launch ...")
# -----------------------------------------------------------------
# Either create a SkirtRemoteLauncher or a SkirtLauncher
if arguments.remote:
launcher = SkirtRemoteLauncher.from_arguments(arguments)
else:
launcher = SkirtLauncher.from_arguments(arguments)
help="(optional) the name of the remote host")
parser.add_argument("--debug",
action="store_true",
help="add this option to enable debug output")
parser.add_argument("--report",
action="store_true",
help="write a report file")
# Parse the command line arguments
arguments = parser.parse_args()
# -----------------------------------------------------------------
# Determine the log file path
logfile_path = os.path.join(os.getcwd(),
time.unique_name("status") +
".txt") if arguments.report else None
# Determine the log level
level = "DEBUG" if arguments.debug else "INFO"
# Initialize the logger
log = logging.setup_log(level=level, path=logfile_path)
log.start("Starting status ...")
# -----------------------------------------------------------------
# Create a RemoteSynchronizer instance
synchronizer = RemoteSynchronizer.from_arguments(arguments)
# Run the synchronizer
# -----------------------------------------------------------------
# Create the command-line parser
parser = argparse.ArgumentParser()
parser.add_argument('remote', nargs='?', default=None, help="(optional) the name of the remote host")
parser.add_argument("--debug", action="store_true", help="add this option to enable debug output")
parser.add_argument("--report", action="store_true", help="write a report file")
# Parse the command line arguments
arguments = parser.parse_args()
# -----------------------------------------------------------------
# Determine the log file path
logfile_path = os.path.join(os.getcwd(), time.unique_name("status") + ".txt") if arguments.report else None
# Determine the log level
level = "DEBUG" if arguments.debug else "INFO"
# Initialize the logger
log = logging.setup_log(level=level, path=logfile_path)
log.start("Starting status ...")
# -----------------------------------------------------------------
# Create a RemoteSynchronizer instance
synchronizer = RemoteSynchronizer.from_arguments(arguments)
# Run the synchronizer
synchronizer.run()
# -----------------------------------------------------------------
# Create the configuration definition
definition = ConfigurationDefinition()
# Add setting
definition.add_required("step", str, "the modeling step for which to clear the output")
# Get the configuration
reader = ConfigurationReader("clear")
config = reader.read(definition)
# -----------------------------------------------------------------
# Determine the log file path
logfile_path = fs.join(fs.cwd(), "log", time.unique_name("log") + ".txt") if config.arguments.report else None
# Determine the log level
level = "DEBUG" if config.arguments.debug else "INFO"
# Initialize the logger
log = logging.setup_log(level=level, path=logfile_path)
log.start("Starting clear ...")
# -----------------------------------------------------------------
prep_path = fs.join(config.path, "prep")
components_path = fs.join(config.path, "components")
truncated_path = fs.join(config.path, "truncated")
phot_path = fs.join(config.path, "phot")
maps_path = fs.join(config.path, "maps")
else:
figsize = "12,9"
figsize_timelines = "12,12"
# -----------------------------------------------------------------
# Locate the scaling test suite directory
suite_path = fs.join(fs.cwd(), config.suite_name)
if not fs.is_directory(suite_path):
raise ValueError("The directory '" + suite_path + "' does not exist")
# -----------------------------------------------------------------
# Make directory for output
output_path = fs.create_directory_in(fs.cwd(),
time.unique_name("scaling_plots"))
# Make subdirectories
single_node_path = fs.create_directory_in(output_path,
"Single-node comparison")
multi_node_path = fs.create_directory_in(
output_path, "Load balancing and multi-node scaling")
communication_path = fs.create_directory_in(output_path, "Communication")
hybridization_path = fs.create_directory_in(output_path, "Hybridization")
photon_packages_path = fs.create_directory_in(
output_path, "Increased number of photon packages")
memory_path = fs.create_directory_in(output_path, "Memory scaling")
# -----------------------------------------------------------------
pts_path = introspection.pts_executable_path
from pts.magic.misc.dustpedia import DustPediaDatabase, get_account
from pts.core.basics.configuration import ConfigurationDefinition, ConfigurationReader
# -----------------------------------------------------------------
# Create the configuration
definition = ConfigurationDefinition()
# Get configuration
reader = ConfigurationReader("plot_galaxies")
config = reader.read(definition)
# -----------------------------------------------------------------
# Determine the log file path
logfile_path = fs.join(fs.cwd(), time.unique_name("log") + ".txt") if config.report else None
# Determine the log level
level = "DEBUG" if config.debug else "INFO"
# Initialize the logger
log = logging.setup_log(level=level, path=logfile_path)
log.start("Starting plot_galaxies ...")
# -----------------------------------------------------------------
# Local table path
local_table_path = fs.join(introspection.pts_dat_dir("modeling"), "s4g", "s4g_p4_table8.dat")
# -----------------------------------------------------------------
figsize = "8,6"
figsize_timelines = "8,8"
else:
figsize = "12,9"
figsize_timelines = "12,12"
# -----------------------------------------------------------------
# Locate the scaling test suite directory
suite_path = fs.join(fs.cwd(), config.suite_name)
if not fs.is_directory(suite_path): raise ValueError("The directory '" + suite_path + "' does not exist")
# -----------------------------------------------------------------
# Make directory for output
output_path = fs.create_directory_in(fs.cwd(), time.unique_name("scaling_plots"))
# Make subdirectories
single_node_path = fs.create_directory_in(output_path, "Single-node comparison")
multi_node_path = fs.create_directory_in(output_path, "Load balancing and multi-node scaling")
communication_path = fs.create_directory_in(output_path, "Communication")
hybridization_path = fs.create_directory_in(output_path, "Hybridization")
photon_packages_path = fs.create_directory_in(output_path, "Increased number of photon packages")
memory_path = fs.create_directory_in(output_path, "Memory scaling")
# -----------------------------------------------------------------
pts_path = introspection.pts_executable_path
# -----------------------------------------------------------------