def write_frame(self):
"""
This function ...
:return:
"""
# Inform the user
log.info("Saving the result ...")
# Determine the path
path = self.output_path_file(self.image_name)
# Check if already existing
if fs.is_file(path):
if self.config.replace:
if self.config.backup: fs.backup_file(path, suffix=self.config.backup_suffix)
fs.remove_file(path)
else:
#raise ValueError("The image already exists")
path = self.output_path_file("result.fits")
# Save
self.frame.saveto(path, header=self.header)
def has_best_parameters(self):
"""
This function ...
:return:
"""
return fs.is_file(self.restore_best_parameters_path)
def has_configuration(self):
"""
This function ...
:return:
"""
return fs.is_file(self.restore_configuration_path)
def has_weights(self):
"""
This function ...
:return:
"""
return fs.is_file(self.restore_weights_path)
def restore_fluxes_plots(self):
"""
This function ...
:return:
"""
# Inform the user
log.info("Making a backup of the mock fluxes plots ...")
# Loop over the generations
for generation_name in self.generation_names:
# Has generation?
if not self.has_generation(generation_name): continue
# Debugging
log.debug("Creating backups for generation '" + generation_name + "' ...")
# Get the generation
generation = self.generations[generation_name]
# Get restore path
generation_path = self.get_generation_restore_path(generation_name)
# Loop over the simulations
for simulation_name in generation.simulation_names:
# Set simulation path
simulation_path = fs.join(generation_path, simulation_name)
# Check whether fluxes are present
filepath = fs.join(simulation_path, "earth_fluxes.pdf")
if not fs.is_file(filepath):
log.warning("No mock SED plot for simulation '" + simulation_name + "' of generation '" + generation_name + "'")
continue
# Debugging
log.debug("Restoring fluxes plot for simulation '" + simulation_name + "' ...")
# Copy the plot file
fs.copy_file(filepath, generation.get_mock_sed_plot_path(simulation_name))
def restore_differences(self):
"""
This function ...
:return:
"""
# Inform the user
log.info("Making a backup of the flux differences ...")
# Loop over the generations
for generation_name in self.generation_names:
# Has generation?
if not self.has_generation(generation_name): continue
# Debugging
log.debug("Creating backups for generation '" + generation_name + "' ...")
# Get the generation
generation = self.generations[generation_name]
# Get restore path
generation_path = self.get_generation_restore_path(generation_name)
# Loop over the simulations
for simulation_name in generation.simulation_names:
# Set simulation path
simulation_path = fs.join(generation_path, simulation_name)
# Check whether the simulation has differences
filepath = fs.join(simulation_path, "differences.dat")
if not fs.is_file(filepath):
log.warning("No differences table for simulation '" + simulation_name + "' of generation '" + generation_name + "'")
continue
# Debugging
log.debug("Restoring differences table for simulation '" + simulation_name + "' ...")
# Copy the file
fs.copy_file(filepath, generation.get_simulation_sed_differences_path(simulation_name))
def write_frame(self):
"""
This function ...
:return:
"""
# Inform the user
log.info("Saving the result ...")
# Determine the path
path = fs.join(output_path, config.image)
if fs.is_file(path):
if config.replace:
if config.backup:
fs.backup_file(path, suffix=config.backup_suffix)
fs.remove_file(path)
else:
fs.remove_file(path)
else:
raise ValueError("The image already exists")
# Save
frame.saveto(path, header=header)
# Determine the full path to the bad region file bad_region_path = fs.join(input_path, arguments.bad) if arguments.bad is not None else None # Import the image importer = ImageImporter() importer.run(image_path, bad_region_path=bad_region_path) # Get the image image = importer.image # ----------------------------------------------------------------- # Load the galaxy region galaxy_region_path = fs.join(input_path, "galaxies.reg") galaxy_region = Region.from_file(galaxy_region_path) if fs.is_file(galaxy_region_path) else None # Load the star region star_region_path = fs.join(input_path, "stars.reg") star_region = Region.from_file(star_region_path) if fs.is_file(star_region_path) else None # Load the saturation region saturation_region_path = fs.join(input_path, "saturation.reg") saturation_region = Region.from_file(saturation_region_path) if fs.is_file(saturation_region_path) else None # Load the region of other sources other_region_path = fs.join(input_path, "other_sources.reg") other_region = Region.from_file(other_region_path) if fs.is_file(other_region_path) else None # Load the image with segmentation maps segments_path = fs.join(input_path, "segments.fits")
def show_possible_matches(matches, table_matches=None, tables=None):
"""
This function ...
:param matches:
:param table_matches:
:param tables:
:return:
"""
# The list that will contain the info about all the scripts / table commands
info = []
## Combine script and table matches
# Scripts
for script in matches:
subproject = script[0]
command = script[1]
# Determine the path to the script
path = fs.join(introspection.pts_package_dir, "do", subproject,
command)
# Get the description
description = get_description(path, subproject, command[:-3])
# Add entry to the info
title = None
configuration_module_path = None
info.append((subproject, command[:-3], description, "arguments", title,
configuration_module_path))
# Tables
for subproject, index in table_matches:
command = tables[subproject]["Command"][index]
hidden = False
if command.startswith("*"):
hidden = True
command = command[1:]
if hidden: continue # skip if hidden
description = tables[subproject]["Description"][index]
configuration_method = tables[subproject]["Configuration method"][
index]
configuration_name = tables[subproject]["Configuration"][index]
if configuration_name == "--": configuration_name = command
configuration_module_path = "pts." + subproject + ".config." + configuration_name
real_configuration_module_path = introspection.pts_root_dir + "/" + configuration_module_path.replace(
".", "/") + ".py"
if not fs.is_file(real_configuration_module_path):
real_configuration_module_path = None
title = tables[subproject]["Title"][index]
info.append((subproject, command, description, configuration_method,
title, real_configuration_module_path))
# Sort on the 'do' subfolder name
info = sorted(info, key=itemgetter(0))
# Print in columns
with fmt.print_in_columns(6) as print_row:
previous_subproject = None
previous_title = None
for script in info:
configuration_method = script[3]
description = script[2]
# Get the title
title = script[4]
# Transform title
if title is None: title = ""
# Get the configuration module path
has_configuration = script[5] is not None
if has_configuration:
has_configuration_string = fmt.green + "C" + fmt.reset
else:
has_configuration_string = fmt.red + "NC" + fmt.reset
coloured_subproject = fmt.green + script[0] + fmt.reset
coloured_name = fmt.bold + fmt.underlined + fmt.red + script[
1] + fmt.reset
coloured_config_method = fmt.yellow + "[" + configuration_method + "]" + fmt.reset
coloured_title = fmt.blue + fmt.bold + title + fmt.reset
if previous_subproject is None or previous_subproject != script[0]:
previous_subproject = script[0]
print_row(coloured_subproject, "/")
if previous_title is None or previous_title != title:
previous_title = title
if title != "":
print_row()
print_row("", "", coloured_title)
# Print command
print_row("", "/", coloured_name, coloured_config_method,
description, has_configuration_string)
definition.add_required("simulation_id", "integer", "ID of the simulation")
definition.add_required("job_id", "integer", "job ID")
# Create config
config = parse_arguments("set_postponed_job_id", definition, add_cwd=False)
# -----------------------------------------------------------------
# Determine the path to the run directory for the specified remote host
host_run_path = fs.join(introspection.skirt_run_dir, config.remote)
# -----------------------------------------------------------------
# Determine the path to the simulation file
simulation_file_path = fs.join(host_run_path, config.simulation_id + ".sim")
if not fs.is_file(simulation_file_path): raise ValueError("Simulation file '" + simulation_file_path + "' does not exist")
# -----------------------------------------------------------------
# Open the simulation file
simulation = RemoteSimulation.from_file(simulation_file_path)
# Create execution handle
handle = ExecutionHandle.job(config.job_id, config.host_id)
# Set the handle
simulation.handle = handle
# Save the simulation
simulation.save()
elif config.page == "fitting": page_path = fs.join(environment.html_path, fitting_page_filename)
elif config.page == "seds": page_path = fs.join(environment.html_path, seds_page_filename)
elif config.page == "datacubes": page_path = fs.join(environment.html_path, datacubes_page_filename)
elif config.page == "fluxes": page_path = fs.join(environment.html_path, fluxes_page_filename)
elif config.page == "images": page_path = fs.join(environment.html_path, images_page_filename)
elif config.page == "attenuation": page_path = fs.join(environment.html_path, attenuation_page_filename)
elif config.page == "colours": page_path = fs.join(environment.html_path, colours_page_filename)
elif config.page == "heating": page_path = fs.join(environment.html_path, heating_page_filename)
else: raise ValueError("Invalid page name")
directory = None
# -----------------------------------------------------------------
# Check whether page exists
if not fs.is_file(page_path): raise ValueError("The page is not present")
# -----------------------------------------------------------------
# # Setup localhost server
# with browser.serve_local_host():
# # Open
# # SHOULD BE AN OPEN AND WAIT FOR CLOSING FUNCTION?
# browser.open_path(page_path)
if directory is not None: fs.change_cwd(directory)
import subprocess
from pts.core.tools import introspection
python_path = introspection.python_executable_path()
elif a_is_colour and b_is_lum:
a_part1, a_part2 = column_a.split("_")
#a_filters = get_filters_for_colour(column_a)
a_filter_names = filternames_for_colours[a_part1] + filternames_for_colours[a_part2]
if column_b in a_filter_names: continue
#print(column_a, column_b)
# Set filename
filename = column_a + "__" + column_b
output_path = fs.join(config.output_path(), filename + "." + config.format)
# Check
if fs.is_file(output_path):
if config.replot is not None and (column_a in config.replot or column_b in config.replot): fs.remove_file(output_path)
else: continue
# Logscales
if a_is_colour or column_a in linear_scales: xlog = False
else: xlog = True
if b_is_colour or column_b in linear_scales: ylog = False
else: ylog = True
# Plot
plotting.plot_table(filepath, column_a, column_b, output_path, x_log=xlog, y_log=ylog)
ncombinations += 1
print(ncombinations)
# Create the configuration definition
definition = ConfigurationDefinition()
# Add flags
definition.add_flag("table", "save the extracted progress table")
# Get configuration
reader = ConfigurationReader("plotprogress")
config = reader.read(definition)
# -----------------------------------------------------------------
# Look for a file in the current working directory that contains extracted progress information
progress_table_path = fs.join(fs.cwd(), "progress.dat")
if fs.is_file(progress_table_path):
table = ProgressTable.from_file(progress_table_path)
# If extracted progress information is not present, first perform the extraction
else:
# Create a SkirtSimulation object based on a log file present in the current working directory
simulation = createsimulations(single=True)
# Create a new ProgressExtractor instance
extractor = ProgressExtractor()
# Run the extractor and get the table
table = extractor.run(simulation)
# -----------------------------------------------------------------
generation_path = fs.join(fs.cwd(), "Generation " + str(last_generation))
print("Current generation: " + str(last_generation))
# -----------------------------------------------------------------
# Path to the current GA object
path = fs.join(generation_path, "ga.pickle")
# Path to the parameters table
parameters_path = fs.join(generation_path, "parameters.dat")
# Path to the chi squared table
chi_squared_path = fs.join(generation_path, "chi_squared.dat")
# Check whether the generation is scored
if not fs.is_file(chi_squared_path): raise RuntimeError("The last generation has not been scored yet!")
# Path to the random state
random_path = fs.join(generation_path, "rndstate.pickle")
# -----------------------------------------------------------------
# Load the random state
load_state(random_path)
# -----------------------------------------------------------------
# Load the parameters table
parameters = tables.from_file(parameters_path, format="ascii.ecsv")
# Load the chi squared table
def show_possible_matches(matches, table_matches=None, tables=None):
"""
This function ...
:param matches:
:param table_matches:
:param tables:
:return:
"""
# The list that will contain the info about all the scripts / table commands
info = []
## Combine script and table matches
# Scripts
for script in matches:
subproject = script[0]
command = script[1]
# Determine the path to the script
path = fs.join(introspection.pts_package_dir, "do", subproject, command)
# Get the description
description = get_description(path, subproject, command[:-3])
# Add entry to the info
title = None
configuration_module_path = None
info.append((subproject, command[:-3], description, "arguments", title, configuration_module_path))
# Tables
for subproject, index in table_matches:
command = tables[subproject]["Command"][index]
hidden = False
if command.startswith("*"):
hidden = True
command = command[1:]
if hidden: continue # skip if hidden
description = tables[subproject]["Description"][index]
configuration_method = tables[subproject]["Configuration method"][index]
configuration_name = tables[subproject]["Configuration"][index]
if configuration_name == "--": configuration_name = command
configuration_module_path = "pts." + subproject + ".config." + configuration_name
real_configuration_module_path = introspection.pts_root_dir + "/" + configuration_module_path.replace(".", "/") + ".py"
if not fs.is_file(real_configuration_module_path): real_configuration_module_path = None
title = tables[subproject]["Title"][index]
info.append((subproject, command, description, configuration_method, title, real_configuration_module_path))
# Sort on the 'do' subfolder name
info = sorted(info, key=itemgetter(0))
# Print in columns
with fmt.print_in_columns(6) as print_row:
previous_subproject = None
previous_title = None
for script in info:
configuration_method = script[3]
description = script[2]
# Get the title
title = script[4]
# Transform title
if title is None: title = ""
# Get the configuration module path
has_configuration = script[5] is not None
if has_configuration: has_configuration_string = fmt.green + "C" + fmt.reset
else: has_configuration_string = fmt.red + "NC" + fmt.reset
coloured_subproject = fmt.green + script[0] + fmt.reset
coloured_name = fmt.bold + fmt.underlined + fmt.red + script[1] + fmt.reset
coloured_config_method = fmt.yellow + "[" + configuration_method + "]" + fmt.reset
coloured_title = fmt.blue + fmt.bold + title + fmt.reset
if previous_subproject is None or previous_subproject != script[0]:
previous_subproject = script[0]
print_row(coloured_subproject, "/")
if previous_title is None or previous_title != title:
previous_title = title
if title != "":
print_row()
print_row("", "", coloured_title)
# Print command
print_row("", "/", coloured_name, coloured_config_method, description, has_configuration_string)
raise RuntimeError(
"The galactic catalog is invalid: principal galaxy not defined")
# Determine the path to the user catalogs directory
catalogs_user_path = fs.join(introspection.pts_user_dir, "magic", "catalogs")
# Determint the path to the directory to contain the catalogs for this galaxy
galaxy_user_path = fs.join(catalogs_user_path, galaxy_name)
# Cache the galaxy and stellar catalog
if fs.is_directory(galaxy_user_path):
old_galactic_catalog_path = fs.join(galaxy_user_path, "galaxies.dat")
old_stellar_catalog_path = fs.join(galaxy_user_path, "stars.dat")
if fs.is_file(old_galactic_catalog_path):
# Open the 'old' galaxy catalog
old_galaxy_catalog = tables.from_file(old_galactic_catalog_path)
# Create merged galaxy catalog
galaxy_catalog = catalogs.merge_galactic_catalogs(
galactic_catalog, old_galaxy_catalog)
# Save the merged catalog
path = fs.join(galaxy_user_path, "galaxies.dat")
tables.write(galaxy_catalog, path)
# If a galactic catalog file does not exist yet
else:
fs.copy_file(galactic_catalog_path, galaxy_user_path, "galaxies.dat")
# Ensure Python 3 compatibility
from __future__ import absolute_import, division, print_function
# Import the relevant PTS classes and modules
from pts.core.basics.configuration import ConfigurationDefinition, parse_arguments
from pts.core.tools import filesystem as fs
from pts.core.tools.serialization import load_dict
from pts.core.tools import sequences
from pts.core.tools import formatting as fmt
# -----------------------------------------------------------------
# Find methods file
methods_path = fs.join(fs.cwd(), "methods.txt")
if not fs.is_file(methods_path): raise IOError("Methods file not found")
methods = load_dict(methods_path)
# Find origins file
origins_path = fs.join(fs.cwd(), "origins.txt")
if not fs.is_file(origins_path): raise IOError("Origins file not found")
origins = load_dict(origins_path)
# -----------------------------------------------------------------
filenames = fs.files_in_path(fs.cwd(), returns="name", extension="fits")
# -----------------------------------------------------------------
map_names = sequences.union(methods.keys(), origins.keys(), filenames)
log.debug("The correction factor for the " + prep_name + " image is " +
str(factor))
correction_factors[prep_name] = factor
#print(paths)
#continue
# Loop over the images
for step in paths:
# Get the path
path = paths[step]
# Check whether the correction has already been performed
backup_filepath = fs.appended_filepath(path, "_backup")
if fs.is_file(path): # local
if fs.is_file(backup_filepath): # already corrected
# Check
corrected = Frame.from_file(path)
backup = Frame.from_file(backup_filepath)
ratio = corrected / backup
ratios = ratio[np.isfinite(ratio)]
if not np.all(np.isclose(ratios, factor)):
raise RuntimeError("Correction was not OK for " +
prep_name + " after the " + step +
" step: mean ratio of " +
str(np.mean(ratios)))
# If the galaxy name is still None, something is wrong with the galaxy catalog (principal not defined)
if galaxy_name is None: raise RuntimeError("The galactic catalog is invalid: principal galaxy not defined")
# Determine the path to the user catalogs directory
catalogs_user_path = fs.join(introspection.pts_user_dir, "magic", "catalogs")
# Determint the path to the directory to contain the catalogs for this galaxy
galaxy_user_path = fs.join(catalogs_user_path, galaxy_name)
# Cache the galaxy and stellar catalog
if fs.is_directory(galaxy_user_path):
old_galactic_catalog_path = fs.join(galaxy_user_path, "galaxies.dat")
old_stellar_catalog_path = fs.join(galaxy_user_path, "stars.dat")
if fs.is_file(old_galactic_catalog_path):
# Open the 'old' galaxy catalog
old_galaxy_catalog = tables.from_file(old_galactic_catalog_path)
# Create merged galaxy catalog
galaxy_catalog = catalogs.merge_galactic_catalogs(galactic_catalog, old_galaxy_catalog)
# Save the merged catalog
path = fs.join(galaxy_user_path, "galaxies.dat")
tables.write(galaxy_catalog, path)
# If a galactic catalog file does not exist yet
else: fs.copy_file(galactic_catalog_path, galaxy_user_path, "galaxies.dat")
# Check whether there is a stellar catalog file in the galaxy's directory
definition = ConfigurationDefinition()
definition.add_flag("generate", "first (re)generate the HTML", False)
config = parse_arguments("show_status", definition)
# -----------------------------------------------------------------
modeling_path = verify_modeling_cwd()
# -----------------------------------------------------------------
# Load the modeling environment
environment = GalaxyModelingEnvironment(modeling_path)
# -----------------------------------------------------------------
# Generate the HTML
if config.generate:
# Generate the HTML
generator = StatusPageGenerator()
generator.config.path = modeling_path
generator.run()
# -----------------------------------------------------------------
# Check whether the status page is present
if not fs.is_file(environment.html_status_path): raise ValueError("The status page is not present")
else: browser.open_path(environment.html_status_path)
# -----------------------------------------------------------------
returns=["name", "path"]):
# Determine origin
origin = instrument_to_origin(name.split("_")[1])
# Determine local directory for this image
origin_path = fs.join(environment.data_images_path, origin)
if not fs.is_directory(origin_path): fs.create_directory(origin_path)
# Determine local path
local_path = fs.join(origin_path, name)
#print("local_path")
# Check whether the image is not present
if fs.is_file(local_path):
log.warning(
"The '" + name +
"' remotely cached image is still present locally. Keeping this file and throwing the remote file away."
)
continue
else:
# Infomr
log.info("Downloading the '" + name + "' file to [" + origin_path +
"] ...")
# Download
remote.download_file_to(path, origin_path, remove=True)
# Success
log.success("Succesfully retrieved the '" + name + "' image ...")
generation_path = fs.join(fs.cwd(), "Generation " + str(last_generation))
print("Current generation: " + str(last_generation))
# -----------------------------------------------------------------
# Path to the current GA object
path = fs.join(generation_path, "ga.pickle")
# Path to the parameters table
parameters_path = fs.join(generation_path, "parameters.dat")
# Path to the chi squared table
chi_squared_path = fs.join(generation_path, "chi_squared.dat")
# Check whether the generation is scored
if not fs.is_file(chi_squared_path):
raise RuntimeError("The last generation has not been scored yet!")
# Path to the random state
random_path = fs.join(generation_path, "rndstate.pickle")
# -----------------------------------------------------------------
# Load the random state
load_state(random_path)
# -----------------------------------------------------------------
# Load the parameters table
parameters = tables.from_file(parameters_path, format="ascii.ecsv")
# Create the configuration definition
definition = ConfigurationDefinition()
# Add flags
definition.add_flag("table", "save the extracted memory table")
# Get the configuration
reader = ConfigurationReader("plotmemory")
config = reader.read(definition)
# -----------------------------------------------------------------
# Look for a file in the current working directory that contains extracted memory information
memory_table_path = fs.join(fs.getcwd(), "memory.dat")
if fs.is_file(memory_table_path): table = MemoryUsageTable.from_file(memory_table_path)
# If extracted memory information is not present, first perform the extraction
else:
# Create a SkirtSimulation object based on a log file present in the current working directory
simulation = createsimulations(single=True)
# Create a new MemoryExtractor instance
extractor = MemoryExtractor()
# Run the extractor and get the memory table
table = extractor.run(simulation)
# -----------------------------------------------------------------
# -----------------------------------------------------------------
# Create the configuration definition
definition = ConfigurationDefinition()
# Add flags
definition.add_flag("table", "save the extracted progress table")
# Get configuration
config = parse_arguments("plotprogress", definition)
# -----------------------------------------------------------------
# Look for a file in the current working directory that contains extracted progress information
progress_table_path = fs.join(fs.cwd(), "progress.dat")
if fs.is_file(progress_table_path): table = ProgressTable.from_file(progress_table_path)
# If extracted progress information is not present, first perform the extraction
else: table = extract_progress_cwd()
# -----------------------------------------------------------------
if config.table and not fs.is_file(progress_table_path): table.saveto(progress_table_path)
# -----------------------------------------------------------------
# Determine the path to the plotting directory
plot_path = fs.join(fs.cwd())
# Create a ProgressPlotter instance
plotter = ProgressPlotter()
# -----------------------------------------------------------------
# Create the configuration definition
definition = ConfigurationDefinition()
# Add flags
definition.add_flag("table", "save the extracted memory table")
# Get the configuration
config = parse_arguments("plotmemory", definition)
# -----------------------------------------------------------------
# Look for a file in the current working directory that contains extracted memory information
memory_table_path = fs.join(fs.cwd(), "memory.dat")
if fs.is_file(memory_table_path):
table = MemoryUsageTable.from_file(memory_table_path)
# If extracted memory information is not present, first perform the extraction
else:
# Create a SkirtSimulation object based on a log file present in the current working directory
simulation = createsimulations(single=True)
# Create a new MemoryExtractor instance
extractor = MemoryExtractor()
# Run the extractor and get the memory table
table = extractor.run(simulation)
# -----------------------------------------------------------------
# Create the configuration
definition = ConfigurationDefinition()
# Add flags
definition.add_flag("table", "save the extracted timeline table")
# Get configuration
reader = ConfigurationReader("plottimeline")
config = reader.read(definition)
# -----------------------------------------------------------------
# Look for a file in the current working directory that contains extracted timeline information
timeline_table_path = fs.join(fs.cwd(), "timeline.dat")
if fs.is_file(timeline_table_path):
table = TimeLineTable.from_file(timeline_table_path)
# If extracted timeline information is not present, first perform the extraction
else:
# Create a SkirtSimulation object based on a log file present in the current working directory
simulation = createsimulations(single=True)
# Create a new TimeLineExtractor instance
extractor = TimeLineExtractor()
# Run the extractor and get the timeline table
table = extractor.run(simulation)
# -----------------------------------------------------------------
definition.add_required("simulation_id", "integer", "ID of the simulation")
definition.add_required("job_id", "integer", "job ID")
# Create config
config = parse_arguments("set_postponed_job_id", definition, add_cwd=False)
# -----------------------------------------------------------------
# Determine the path to the run directory for the specified remote host
host_run_path = fs.join(introspection.skirt_run_dir, config.remote)
# -----------------------------------------------------------------
# Determine the path to the simulation file
simulation_file_path = fs.join(host_run_path, config.simulation_id + ".sim")
if not fs.is_file(simulation_file_path):
raise ValueError("Simulation file '" + simulation_file_path +
"' does not exist")
# -----------------------------------------------------------------
# Open the simulation file
simulation = RemoteSimulation.from_file(simulation_file_path)
# Create execution handle
handle = ExecutionHandle.job(config.job_id, config.host_id)
# Set the handle
simulation.handle = handle
# Save the simulation
