def launch_reference(self):
"""
This function ...
:return:
"""
# Inform the user
log.info("Launching the reference simulation ...")
# Settings
settings_launch = dict()
settings_launch["ski"] = self.reference_ski_path
settings_launch["input"] = self.simulation_input_path
settings_launch["output"] = self.simulation_output_path
settings_launch["create_output"] = True
settings_launch["remote"] = self.moderator.host_id_for_single(
"reference")
settings_launch["attached"] = self.config.attached
settings_launch["progress_bar"] = True
# Create the analysis options
analysis = AnalysisOptions()
analysis.extraction.path = self.simulation_extract_path
analysis.plotting.path = self.simulation_plot_path
analysis.misc.path = self.simulation_misc_path
analysis.extraction.progress = True
analysis.extraction.timeline = True
analysis.extraction.memory = True
analysis.plotting.progress = True
analysis.plotting.timeline = True
analysis.plotting.memory = True
analysis.plotting.seds = True
analysis.plotting.grids = True
analysis.plotting.reference_seds = fs.files_in_path(seds_path)
analysis.misc.fluxes = True
analysis.misc.images = False
analysis.misc.observation_filters = fitting_filter_names
analysis.misc.observation_instruments = [instrument_name]
analysis.misc.spectral_convolution = self.config.spectral_convolution
# Set flux error bars
dustpedia_sed = ObservedSED.from_file(dustpedia_sed_path)
filter_names = dustpedia_sed.filter_names()
errors = dustpedia_sed.errors()
flux_errors = sequences.zip_into_dict(filter_names,
[str(error) for error in errors])
analysis.misc.flux_errors = flux_errors
# Input
input_launch = dict()
# input_launch["memory"] = MemoryRequirement(serial_memory, parallel_memory)
input_launch["analysis_options"] = analysis
# Launch command
launch = Command("launch_simulation",
"launch the reference simulation",
settings_launch,
input_launch,
cwd=".")
self.launcher = self.run_command(launch)
def all_input_files(self):
"""
This function ...
:return:
"""
return fs.files_in_path(self.simulation_input_path)
def create_wcs(self):
"""
This function ...
:return:
"""
# Inform the user
log.info("Creating the WCS ...")
min_pixelscale = None
# Determine the path to the headers directory
headers_path = fs.join(m81_data_path, "headers")
# Loop over the header files
for path, name in fs.files_in_path(headers_path, extension="txt", returns=["path", "name"]):
# Get the filter
fltr = parse_filter(name)
filter_name = str(fltr)
# Set the path of the file for the filter name
self.wcs_paths[filter_name] = path
# Get WCS
wcs = CoordinateSystem.from_header_file(path)
# Adjust the pixelscale
if min_pixelscale is None:
min_pixelscale = wcs.pixelscale
self.wcs = wcs
elif min_pixelscale > wcs.pixelscale:
min_pixelscale = wcs.pixelscale
self.wcs = wcs
def make_animation(self):
"""
This function ...
:return:
"""
# Inform the user
log.info("Making animation ...")
# Get the best
best = self.optimizer.best
# Determine path
temp_path = self.optimizer.config.path
filepath = fs.join(temp_path, "best.png")
# Make plot of best
# Create animated gif
animation = Animation()
# Open the images present in the directory
for path in fs.files_in_path(temp_path, extension="png", exact_not_name="best"):
# Add frame to the animation
animation.add_frame_from_file(path)
# Save the animation
animation_path = fs.join(temp_path, "animation.gif")
animation.saveto(animation_path)
def create_wcs(self):
"""
This function ...
:return:
"""
# Inform the user
log.info("Creating the WCS ...")
min_pixelscale = None
# Determine the path to the headers directory
headers_path = fs.join(m81_data_path, "headers")
# Loop over the header files
for path, name in fs.files_in_path(headers_path, extension="txt", returns=["path", "name"]):
# Get the filter
fltr = parse_filter(name)
filter_name = str(fltr)
# Set the path of the file for the filter name
self.wcs_paths[filter_name] = path
# Get WCS
wcs = CoordinateSystem.from_header_file(path)
# Adjust the pixelscale
if min_pixelscale is None:
min_pixelscale = wcs.pixelscale
self.wcs = wcs
elif min_pixelscale > wcs.pixelscale:
min_pixelscale = wcs.pixelscale
self.wcs = wcs
def make_animation(self):
"""
This function ...
:return:
"""
# Inform the user
log.info("Making animation ...")
# Get the best
best = self.optimizer.best
# Determine path
temp_path = self.optimizer.config.path
filepath = fs.join(temp_path, "best.png")
# Make plot of best
# Create animated gif
animation = Animation()
# Open the images present in the directory
for path in fs.files_in_path(temp_path, extension="png", exact_not_name="best"):
# Add frame to the animation
animation.add_frame_from_file(path)
# Save the animation
animation_path = fs.join(temp_path, "animation.gif")
animation.saveto(animation_path)
def required_input_files(self):
"""
This function ...
:return:
"""
return fs.files_in_path(self.simulation_input_path, exact_not_name="wavelengths")
def required_input_files(self):
"""
This function ...
:return:
"""
return fs.files_in_path(self.simulation_input_path,
exact_not_name="wavelengths")
def all_input_files(self):
"""
This function ...
:return:
"""
return fs.files_in_path(self.simulation_input_path)
def launch_reference(self):
"""
This function ...
:return:
"""
# Inform the user
log.info("Launching the reference simulation ...")
# Settings
settings_launch = dict()
settings_launch["ski"] = self.reference_ski_path
settings_launch["input"] = self.simulation_input_path
settings_launch["output"] = self.simulation_output_path
settings_launch["create_output"] = True
settings_launch["remote"] = self.moderator.host_id_for_single("reference")
settings_launch["attached"] = self.config.attached
settings_launch["progress_bar"] = True
# Create the analysis options
analysis = AnalysisOptions()
analysis.extraction.path = self.simulation_extract_path
analysis.plotting.path = self.simulation_plot_path
analysis.misc.path = self.simulation_misc_path
analysis.extraction.progress = True
analysis.extraction.timeline = True
analysis.extraction.memory = True
analysis.plotting.progress = True
analysis.plotting.timeline = True
analysis.plotting.memory = True
analysis.plotting.seds = True
analysis.plotting.grids = True
analysis.plotting.reference_seds = fs.files_in_path(seds_path)
analysis.misc.fluxes = True
analysis.misc.images = False
analysis.misc.observation_filters = fitting_filter_names
analysis.misc.observation_instruments = [instrument_name]
analysis.misc.spectral_convolution = self.config.spectral_convolution
# Set flux error bars
dustpedia_sed = ObservedSED.from_file(dustpedia_sed_path)
filter_names = dustpedia_sed.filter_names()
errors = dustpedia_sed.errors()
flux_errors = sequences.zip_into_dict(filter_names, [str(error) for error in errors])
analysis.misc.flux_errors = flux_errors
# Input
input_launch = dict()
# input_launch["memory"] = MemoryRequirement(serial_memory, parallel_memory)
input_launch["analysis_options"] = analysis
# Launch command
launch = Command("launch_simulation", "launch the reference simulation", settings_launch, input_launch, cwd=".")
self.launcher = self.run_command(launch)
def get_coordinate_system(fltr):
"""
This function ...
:param fltr:
:return:
"""
for path, name in fs.files_in_path(headers_path, extension="txt", returns=["path", "name"]):
header_fltr = parse_filter(name)
if header_fltr == fltr: return CoordinateSystem.from_header_file(path)
return None
def load_images(self):
"""
This function ...
:return:
"""
# Inform the user
log.info("Loading the images ...")
# The common wcs of the images
wcs = None
# Loop over the images in the data directory
for path, filename in fs.files_in_path(self.data_path,
extension="fits",
returns=["path", "name"]):
# Load the frame
frame = Frame.from_file(path)
# Set filter
previous_filter = frame.filter
frame.filter = parse_filter(filename.split("_norm")[0])
if previous_filter != frame.filter: frame.save()
# Check filter
if str(frame.filter) not in [
str(fltr) for fltr in self.config.fitting_filters
]:
continue
# Determine name
name = str(frame.filter)
# Check wcs
if wcs is None: wcs = frame.wcs
elif wcs == frame.wcs: pass
else:
raise IOError("The coordinate system of image '" + filename +
"' does not match that of other images")
# Debugging
log.debug("Adding frame '" + filename + "' ...")
# Add to dictionary
self.images[name] = frame
# Set original path
self.image_paths[name] = path
# Set the wcs
self.wcs = wcs
def get_coordinate_system(fltr):
"""
This function ...
:param fltr:
:return:
"""
for path, name in fs.files_in_path(headers_path,
extension="txt",
returns=["path", "name"]):
header_fltr = parse_filter(name)
if header_fltr == fltr: return CoordinateSystem.from_header_file(path)
return None
def load_coordinate_systems(self):
"""
This function ...
:return:
"""
# Inform the user
log.info("Loading the coordinate systems ...")
nfilters_stars = 0
nfilters_extra = 0
# Loop over the header files
for path, name in fs.files_in_path(headers_path,
extension="txt",
returns=["path", "name"]):
# Get the filter and wavelength
fltr = parse_filter(name)
wavelength = fltr.effective if fltr.effective is not None else fltr.center
# SKip Planck
if fltr.observatory == "Planck": continue
# Wavelength greater than 25 micron
if wavelength > wavelengths.ranges.ir.mir.max:
if nfilters_extra == self.config.nfilters_extra: continue
else: nfilters_extra += 1
# Wavelength smaller than 25 micron
else:
if nfilters_stars == self.config.nfilters_stars: continue
else: nfilters_stars += 1
# Debugging
log.debug("Loading the coordinate system for the '" + str(fltr) +
"' filter ...")
# Get WCS
wcs = CoordinateSystem.from_header_file(path)
# Add the coordinate system
self.coordinate_systems.append(wcs, fltr=fltr)
# Break the loop if we have enough
if nfilters_stars == self.config.nfilters_stars and nfilters_extra == self.config.nfilters_extra:
break
def load_coordinate_systems(self):
"""
This function ...
:return:
"""
# Inform the user
log.info("Loading the coordinate systems ...")
nfilters_stars = 0
nfilters_extra = 0
# Loop over the header files
for path, name in fs.files_in_path(headers_path, extension="txt", returns=["path", "name"]):
# Get the filter and wavelength
fltr = parse_filter(name)
wavelength = fltr.effective if fltr.effective is not None else fltr.center
# SKip Planck
if fltr.observatory == "Planck": continue
# Wavelength greater than 25 micron
if wavelength > wavelengths.ranges.ir.mir.max:
if nfilters_extra == self.config.nfilters_extra: continue
else: nfilters_extra += 1
# Wavelength smaller than 25 micron
else:
if nfilters_stars == self.config.nfilters_stars: continue
else: nfilters_stars += 1
# Debugging
log.debug("Loading the coordinate system for the '" + str(fltr) + "' filter ...")
# Get WCS
wcs = CoordinateSystem.from_header_file(path)
# Add the coordinate system
self.coordinate_systems.append(wcs, fltr=fltr)
# Break the loop if we have enough
if nfilters_stars == self.config.nfilters_stars and nfilters_extra == self.config.nfilters_extra: break
def load_observed_sed(self):
"""
This function ...
:return:
"""
# Inform the user
log.info("Loading the mock observed sed ...")
# Determine path
paths = fs.files_in_path(self.simulation_misc_path, contains="_fluxes", extension="dat")
# Check if there is only one SED
if len(paths) > 1: raise RuntimeError("More than one SED is found")
path = paths[0]
# Load the observed SED
self.observed_sed = ObservedSED.from_file(path)
def load_observed_sed(self):
"""
This function ...
:return:
"""
# Inform the user
log.info("Loading the mock observed sed ...")
# Determine path
paths = fs.files_in_path(self.simulation_misc_path,
contains="_fluxes",
extension="dat")
# Check if there is only one SED
if len(paths) > 1: raise RuntimeError("More than one SED is found")
path = paths[0]
# Load the observed SED
self.observed_sed = ObservedSED.from_file(path)
def create_wcs_not_working(self):
"""
This function ...
:return:
"""
# Inform the user
log.info("Creating the WCS ...")
# Determine the path to the headers directory
headers_path = fs.join(m81_data_path, "headers")
# Loop over the files in the directory
ra_range = None
dec_range = None
min_pixelscale = None
for path, name in fs.files_in_path(headers_path, extension="txt", returns=["path", "name"]):
# Get the filter
fltr = parse_filter(name)
# Get WCS
wcs = CoordinateSystem.from_header_file(path)
# Adjust RA range
if ra_range is None: ra_range = wcs.ra_range
else: ra_range.adjust(wcs.ra_range)
# Adjust DEC range
if dec_range is None: dec_range = wcs.dec_range
else: dec_range.adjust(wcs.dec_range)
# Adjust the pixelscale
if min_pixelscale is None: min_pixelscale = wcs.pixelscale
elif min_pixelscale > wcs.pixelscale: min_pixelscale = wcs.pixelscale
# Create coordinate system
# size, center_pixel, center_sky, pixelscale
self.wcs = CoordinateSystem.from_ranges(ra_range, dec_range, min_pixelscale)
def create_wcs_not_working(self):
"""
This function ...
:return:
"""
# Inform the user
log.info("Creating the WCS ...")
# Determine the path to the headers directory
headers_path = fs.join(m81_data_path, "headers")
# Loop over the files in the directory
ra_range = None
dec_range = None
min_pixelscale = None
for path, name in fs.files_in_path(headers_path, extension="txt", returns=["path", "name"]):
# Get the filter
fltr = parse_filter(name)
# Get WCS
wcs = CoordinateSystem.from_header_file(path)
# Adjust RA range
if ra_range is None: ra_range = wcs.ra_range
else: ra_range.adjust(wcs.ra_range)
# Adjust DEC range
if dec_range is None: dec_range = wcs.dec_range
else: dec_range.adjust(wcs.dec_range)
# Adjust the pixelscale
if min_pixelscale is None: min_pixelscale = wcs.pixelscale
elif min_pixelscale > wcs.pixelscale: min_pixelscale = wcs.pixelscale
# Create coordinate system
# size, center_pixel, center_sky, pixelscale
self.wcs = CoordinateSystem.from_ranges(ra_range, dec_range, min_pixelscale)
def make_animation(self):
"""
This function ...
:return:
"""
# Determine path
#temp_path = self.optimizer.config.path
# Determine plot path
plot_path = fs.join(self.path, "plot")
# Create animated gif
animation = Animation()
# Open the images present in the directory
for path in fs.files_in_path(plot_path, extension="png", exact_not_name="best"):
# Add frame to the animation
animation.add_frame_from_file(path)
# Save the animation
animation_path = fs.join(self.path, "animation.gif")
animation.saveto(animation_path)
# Loop over the directories in the truncation path
for path, name in fs.directories_in_path(environment.truncation_path, returns=["path", "name"]):
# Determine the path to the lowres directory
lowres_path = fs.join(path, "lowres")
# Determine the filter
fltr = parse_filter(name)
filter_name = str(fltr)
# Initializ variable
the_image_path = None
# Find the image corresponding to the specified factor
for image_path, image_name in fs.files_in_path(lowres_path, extension="fits", returns=["path", "name"]):
# Determine the factor
factor = real(image_name)
# If the factor corresponds to the specified factor, take this image
if np.isclose(factor, config.factor, rtol=0.01):
the_image_path = image_path
break
# Check
if the_image_path is None: raise ValueError("No truncated " + filter_name + " image found for a factor of " + str(config.factor))
# Add the image
frame = Frame.from_file(the_image_path)
plotter.add_image(frame, filter_name)
# ** PTS -- Python Toolkit for working with SKIRT **
# ** © Astronomical Observatory, Ghent University **
# *****************************************************************
## \package pts.do.developer.remove_pyc Remove all compiled python (.pyc) files.
# -----------------------------------------------------------------
# Ensure Python 3 compatibility
from __future__ import absolute_import, division, print_function
# Import the relevant PTS classes and modules
from pts.core.tools import introspection
from pts.core.tools import filesystem as fs
from pts.core.basics.log import log
# -----------------------------------------------------------------
# Loop over all pyc files
for path in fs.files_in_path(introspection.pts_package_dir,
recursive=True,
extension="pyc"):
# Inform the user
log.info("Removing '" + path + "' ...")
# Remove the file
fs.remove_file(path)
# -----------------------------------------------------------------
if nfiles == 0: log.warning("No files found")
else:
log.info(str(nfiles) + " files found")
for path in paths:
if config.full: print(path)
else: print(path.split(find_path)[1])
if config.remove: remote.remove_file(path)
# LOCALLY
else:
# Determine path
find_path = fs.cwd()
# Loop over the files
paths = fs.files_in_path(find_path, contains=config.contains, extension=config.extension, recursive=config.recursive,
exact_name=config.exact_name, exact_not_name=config.exact_not_name,
directory_not_contains=config.directory_not_contains, directory_exact_not_name=config.directory_exact_not_name)
nfiles = len(paths)
if nfiles == 0: log.warning("No files found")
else:
log.info(str(nfiles) + " files found")
for path in paths:
if config.full: print(path)
else: print(path.split(find_path)[1])
if config.remove: fs.remove_file(path)
# -----------------------------------------------------------------
# Initialize the logger
log = logging.setup_log(level=level, path=logfile_path)
log.start("Starting check_simulated_images ...")
# -----------------------------------------------------------------
modeling_path = config.path
galaxy_name = fs.name(modeling_path)
fit_best_images_path = fs.join(modeling_path, "fit", "best", "images", "test")
simulated = dict()
for path, name in fs.files_in_path(fit_best_images_path,
extension="fits",
returns=["path", "name"]):
if name == galaxy_name + "_earth_total": continue
frame = Frame.from_file(path)
if "2MASS" in str(frame.filter): continue
if "I4" in str(frame.filter): continue
if "W3" in str(frame.filter): continue
if frame.filter is None: raise ValueError("No filter for " + name)
simulated[str(frame.filter)] = frame
trunc_path = fs.join(modeling_path, "truncated")
# Create the command-line parser
parser = argparse.ArgumentParser()
parser.add_argument("remote", nargs='?', default=None, help="the name of the remote host to connect to")
parser.add_argument("ids", type=parsing.simulation_ids, help="unretrieve the simulations with these ID's")
parser.add_argument("--keep", action="store_true", help="add this option to make sure the output is kept remotely after a subsequent retrieve attempt")
# Parse the command line arguments
arguments = parser.parse_args()
# -----------------------------------------------------------------
# Create the remote execution environment
remote = Remote()
remote.setup(arguments.remote)
for path in fs.files_in_path(fs.cwd(), extension="sim"):
# Create simulation
sim = RemoteSimulation.from_file(path)
if not sim.retrieved: continue
local_output_path = sim.output_path
remote_simulation_path = sim.remote_simulation_path
remote_output_path = sim.remote_output_path
remote_input_path = sim.remote_input_path
if not remote.is_directory(remote_simulation_path): remote.create_directory(remote_simulation_path)
if not remote.is_directory(remote_output_path): remote.create_directory(remote_output_path)
attenuation = GalacticAttenuation(center_coordinate)
# -----------------------------------------------------------------
# The aniano kernels service
aniano = AnianoKernels()
# -----------------------------------------------------------------
remote_host_id = "nancy"
session = AttachedPythonSession.from_host_id(remote_host_id)
# -----------------------------------------------------------------
# Paths
paths = fs.files_in_path(galex_images_path, extension="fits", contains="poisson") + fs.files_in_path(sdss_images_path, extension="fits", contains="poisson")
# Loop over the GALEX poisson frames
for path in paths:
# Image name
name = fs.strip_extension(fs.name(path))
# Get instrument and band
galaxy_name, instrument, band, _ = name.split("_")
# Create the filter
fltr = BroadBandFilter.from_instrument_and_band(instrument, band)
# Load the frame
poisson = Frame.from_file(path)
def get_dependencies(self):
"""
This function ...
:return:
"""
# If no script name is given, list all dependencies of PTS and the
# PTS modules that use them
if self.config.script is None:
if self.config.subprojects:
self.dependencies = defaultdict(set)
directories_for_subproject = defaultdict(list)
# Loop over the subdirectories of the 'do' directory
for path, name in fs.directories_in_path(
introspection.pts_do_dir, returns=["path", "name"]):
subproject = name
directories_for_subproject[subproject].append(path)
# Loop over the other directories in 'pts' (other than 'do' and 'doc')
for path, name in fs.directories_in_path(
introspection.pts_package_dir,
returns=["path", "name"],
exact_not_name=["do", "doc"]):
subproject = name
directories_for_subproject[subproject].append(path)
encountered_internal_modules = set()
for subproject in directories_for_subproject:
# print(subproject, directories_for_subproject[subproject])
# List the dependencies of the matching script
dependencies_for_this = defaultdict(set)
for dir_path in directories_for_subproject[subproject]:
for file_path in fs.files_in_path(dir_path,
extension="py",
recursive=True):
# if subproject == "dustpedia": print(file_path)
introspection.add_dependencies(
dependencies_for_this, file_path,
encountered_internal_modules)
self.dependencies_for_subproject[
subproject] = dependencies_for_this.keys()
for dependency in dependencies_for_this:
for name in dependencies_for_this[dependency]:
self.dependencies[dependency].add(name)
# No subprojects are specified, list all PTS dependencies
else:
self.dependencies = introspection.get_all_dependencies()
# If a script name is given
else:
scripts = introspection.get_scripts()
tables = introspection.get_arguments_tables()
# Find matching 'do' commands (actual scripts or tabulated commands)
matches = introspection.find_matches_scripts(
self.config.script, scripts)
table_matches = introspection.find_matches_tables(
self.config.script, tables)
# List the dependencies of the matching script
self.dependencies = defaultdict(set)
# No match
if len(matches) + len(table_matches) == 0:
show_all_available(scripts, tables)
exit()
# More matches
elif len(matches) + len(table_matches) > 1:
show_possible_matches(matches, table_matches, tables)
exit()
# Exactly one match from existing do script
elif len(matches) == 1 and len(table_matches) == 0:
# Determine the full path to the matching script
script_path = fs.join(introspection.pts_do_dir, matches[0][0],
matches[0][1])
introspection.add_dependencies(self.dependencies, script_path,
set())
# Exactly one match from tabulated command
elif len(table_matches) == 1 and len(matches) == 0:
# from pts.core.tools import logging
# configuration
# Path to class module
table_match = table_matches[0]
subproject = table_match[0]
index = table_match[1]
relative_class_module_path = tables[subproject]["Path"][
index].replace(".", "/").rsplit("/", 1)[0] + ".py"
class_module_path = fs.join(
introspection.pts_subproject_dir(subproject),
relative_class_module_path)
logging_path = fs.join(introspection.pts_package_dir, "core",
"tools", "logging.py")
command_name = tables[subproject]["Command"][index]
configuration_name = tables[subproject]["Configuration"][index]
if configuration_name == "--":
configuration_name = command_name
configuration_module_path = fs.join(
introspection.pts_root_dir, "pts/" + subproject +
"/config/" + configuration_name + ".py")
# Add dependencies
encountered = set()
introspection.add_dependencies(self.dependencies, logging_path,
encountered)
introspection.add_dependencies(self.dependencies,
configuration_module_path,
encountered)
introspection.add_dependencies(self.dependencies,
class_module_path, encountered)
default_mask_color = "black"
# -----------------------------------------------------------------
# Set the modeling path
modeling_path = verify_modeling_cwd()
# Set paths
maps_path = fs.join(modeling_path, "maps")
maps_components_path = fs.join(maps_path, "components")
# -----------------------------------------------------------------
# Get selection indices
indices = fs.files_in_path(maps_components_path, extension="dat", returns="name", startswith="selection", convert=int, sort=int, convert_split_index=1, convert_split_pattern="_")
# Add the setting
if len(indices) == 0: raise RuntimeError("Could not find any selection file")
elif len(indices) == 1: definition.add_fixed("selection", "selection to use", indices[0])
else: definition.add_optional("selection", "positive_integer", "selection to use", choices=indices)
# -----------------------------------------------------------------
# Show?
definition.add_flag("show", "show the page", False)
# Remote
definition.add_optional("remote", "string", "remote host to use for creating the clip masks", choices=find_host_ids(schedulers=False))
# Flags
# Initialize the logger
log = logging.setup_log(level=level, path=logfile_path)
log.start("Starting initialize_data ...")
# -----------------------------------------------------------------
names_column = []
paths_column = []
prep_names_column = []
names = ["Image name", "Image path", "Preparation name"]
# Loop over all subdirectories of the data directory
for path, name in fs.directories_in_path(fs.join(config.path, "data"), not_contains="bad", returns=["path", "name"]):
# Loop over all FITS files found in the current subdirectory
for image_path, image_name in fs.files_in_path(path, extension="fits", not_contains="_Error", returns=["path", "name"]):
# Open the image frame
frame = Frame.from_file(image_path)
# Determine the preparation name
if frame.filter is not None: prep_name = str(frame.filter)
else: prep_name = image_name
# Set the row entries
names_column.append(image_name)
paths_column.append(image_path)
prep_names_column.append(prep_name)
# Create the table
data = [names_column, paths_column, prep_names_column]
# -----------------------------------------------------------------
names_column = []
paths_column = []
prep_names_column = []
names = ["Image name", "Image path", "Preparation name"]
# Loop over all subdirectories of the data directory
for path, name in fs.directories_in_path(fs.join(config.path, "data"),
not_contains="bad",
returns=["path", "name"]):
# Loop over all FITS files found in the current subdirectory
for image_path, image_name in fs.files_in_path(path,
extension="fits",
not_contains="_Error",
returns=["path", "name"]):
# Open the image frame
frame = Frame.from_file(image_path)
# Determine the preparation name
if frame.filter is not None: prep_name = str(frame.filter)
else: prep_name = image_name
# Set the row entries
names_column.append(image_name)
paths_column.append(image_path)
prep_names_column.append(prep_name)
# Create the table
else:
remote = None
# Determine the path to the run directory for the specified remote host
host_run_path = fs.join(introspection.skirt_run_dir, host_id)
# Check if there are simulations
if not fs.is_directory(host_run_path):
log.debug("No run directory for host '" + host_id + "'")
continue
if fs.is_empty(host_run_path):
log.debug("No simulations for host '" + host_id + "'")
# Loop over the simulation files in the run directory
for path, name in fs.files_in_path(host_run_path,
extension="sim",
returns=["path", "name"],
sort=int):
# Skip
if config.ids is not None and int(name) not in config.ids: continue
# Inform the user
log.info("Removing simulation " + name + " ...")
# Fully clear
if config.full:
# Debugging
log.debug("Removing the remote files and directories ...")
# Load the simulation
# -----------------------------------------------------------------
modeling_path = verify_modeling_cwd()
# -----------------------------------------------------------------
data_images_path = get_data_images_path(modeling_path)
# -----------------------------------------------------------------
shapes = dict()
# Loop over the images
for image_path, image_name in fs.files_in_path(data_images_path,
extension="fits",
not_contains="poisson",
returns=["path", "name"],
recursive=True,
recursion_level=1):
# Load the image
frame = Frame.from_file(image_path, silent=True)
# Determine filter name
filter_name = frame.filter_name
# Set pixel shape
shapes[filter_name] = frame.shape
# -----------------------------------------------------------------
# Sort on shape
# -----------------------------------------------------------------
# Find methods file
methods_path = fs.join(fs.cwd(), "methods.txt")
if not fs.is_file(methods_path): raise IOError("Methods file not found")
methods = load_dict(methods_path)
# Find origins file
origins_path = fs.join(fs.cwd(), "origins.txt")
if not fs.is_file(origins_path): raise IOError("Origins file not found")
origins = load_dict(origins_path)
# -----------------------------------------------------------------
filenames = fs.files_in_path(fs.cwd(), returns="name", extension="fits")
# -----------------------------------------------------------------
map_names = sequences.union(methods.keys(), origins.keys(), filenames)
# -----------------------------------------------------------------
print("")
print("Presumed number of maps: " + str(len(map_names)))
print("")
print(" - number of map files found: " + str(len(filenames)))
print(" - number of entries in 'origins': " + str(len(origins)))
print(" - number of entries in 'methods': " + str(len(methods)))
print("")
#!/usr/bin/env python
# -*- coding: utf8 -*-
# *****************************************************************
# ** PTS -- Python Toolkit for working with SKIRT **
# ** © Astronomical Observatory, Ghent University **
# *****************************************************************
## \package pts.do.magic.verify_fits Verify a batch of FITS files in the current working directory.
# -----------------------------------------------------------------
# Ensure Python 3 compatibility
from __future__ import absolute_import, division, print_function
# Import astronomical modules
from astropy.io import fits
# Import the relevant PTS classes and modules
from pts.core.tools import filesystem as fs
# -----------------------------------------------------------------
# Loop over all files in the current path
for path in fs.files_in_path(fs.cwd(), extension="fits"):
# Open the file
hdulist = fits.open(path)
# -----------------------------------------------------------------
def launch_reference(self):
"""
This function ...
:return:
"""
# Inform the user
log.info("Launching the reference simulation ...")
# Settings
settings_launch = dict()
settings_launch["ski"] = self.reference_ski_path
settings_launch["input"] = self.simulation_input_path
settings_launch["output"] = self.simulation_output_path
settings_launch["create_output"] = True
settings_launch["remote"] = self.host_id
settings_launch["attached"] = True
settings_launch["show_progress"] = True
# Create the analysis options
analysis = AnalysisOptions()
analysis.extraction.path = self.simulation_extract_path
analysis.plotting.path = self.simulation_plot_path
analysis.misc.path = self.simulation_misc_path
analysis.extraction.progress = True
analysis.extraction.timeline = True
analysis.extraction.memory = True
analysis.plotting.progress = True
analysis.plotting.timeline = True
analysis.plotting.memory = True
analysis.plotting.seds = True
analysis.plotting.grids = True
analysis.plotting.reference_seds = fs.files_in_path(seds_path)
analysis.misc.fluxes = True
analysis.misc.images = True
analysis.misc.observation_filters = fitting_filter_names
analysis.misc.observation_instruments = [instrument_name]
analysis.misc.make_images_remote = self.host_id
analysis.misc.images_wcs = self.reference_wcs_path
analysis.misc.images_unit = "Jy/pix"
analysis.misc.spectral_convolution = self.config.spectral_convolution
# Set flux error bars
dustpedia_sed = ObservedSED.from_file(dustpedia_sed_path)
filter_names = dustpedia_sed.filter_names()
errors = dustpedia_sed.errors()
#print([str(error) for error in errors])
flux_errors = sequences.zip_into_dict(filter_names, [str(error) for error in errors])
analysis.misc.flux_errors = flux_errors
# Create Aniano kernels object
aniano = AnianoKernels()
# Set the paths to the kernel for each image
kernel_paths = dict()
for filter_name in fitting_filter_names: kernel_paths[filter_name] = aniano.get_psf_path(parse_filter(filter_name))
analysis.misc.images_kernels = kernel_paths
# Set the paths to the WCS files for each image
analysis.misc.rebin_wcs = {instrument_name: self.wcs_paths}
# Input
input_launch = dict()
#input_launch["memory"] = MemoryRequirement(serial_memory, parallel_memory)
input_launch["analysis_options"] = analysis
# Launch command
launch = Command("launch_simulation", "launch the reference simulation", settings_launch, input_launch, cwd=".")
self.launcher = self.run_command(launch)
#print(find_path)
#print(remote.items_in_path(find_path, recursive=True))
# Loop over the files
paths = remote.files_in_path(find_path, contains=config.contains, not_contains=config.not_contains, extension=config.extension, recursive=config.recursive)
if len(paths) == 0: log.warning("No files found")
else:
for path in paths:
if config.full: print(path)
else: print(path.split(find_path)[1])
# LOCALLY
else:
# Determine path
find_path = fs.cwd()
# Loop over the files
paths = fs.files_in_path(find_path, contains=config.contains, extension=config.extension, recursive=config.recursive)
if len(paths) == 0: log.warning("No files found")
else:
for path in paths:
if config.full: print(path)
else: print(path.split(find_path)[1])
# -----------------------------------------------------------------
def get_dependencies(self):
"""
This function ...
:return:
"""
# If no script name is given, list all dependencies of PTS and the
# PTS modules that use them
if self.config.script is None:
if self.config.subprojects:
self.dependencies = defaultdict(set)
directories_for_subproject = defaultdict(list)
# Loop over the subdirectories of the 'do' directory
for path, name in fs.directories_in_path(introspection.pts_do_dir, returns=["path", "name"]):
subproject = name
directories_for_subproject[subproject].append(path)
# Loop over the other directories in 'pts' (other than 'do' and 'doc')
for path, name in fs.directories_in_path(introspection.pts_package_dir, returns=["path", "name"],
exact_not_name=["do", "doc"]):
subproject = name
directories_for_subproject[subproject].append(path)
encountered_internal_modules = set()
for subproject in directories_for_subproject:
# print(subproject, directories_for_subproject[subproject])
# List the dependencies of the matching script
dependencies_for_this = defaultdict(set)
for dir_path in directories_for_subproject[subproject]:
for file_path in fs.files_in_path(dir_path, extension="py", recursive=True):
# if subproject == "dustpedia": print(file_path)
introspection.add_dependencies(dependencies_for_this, file_path,
encountered_internal_modules)
self.dependencies_for_subproject[subproject] = dependencies_for_this.keys()
for dependency in dependencies_for_this:
for name in dependencies_for_this[dependency]:
self.dependencies[dependency].add(name)
# No subprojects are specified, list all PTS dependencies
else: self.dependencies = introspection.get_all_dependencies()
# If a script name is given
else:
scripts = introspection.get_scripts()
tables = introspection.get_arguments_tables()
# Find matching 'do' commands (actual scripts or tabulated commands)
matches = introspection.find_matches_scripts(self.config.script, scripts)
table_matches = introspection.find_matches_tables(self.config.script, tables)
# List the dependencies of the matching script
self.dependencies = defaultdict(set)
# No match
if len(matches) + len(table_matches) == 0:
show_all_available(scripts, tables)
exit()
# More matches
elif len(matches) + len(table_matches) > 1:
show_possible_matches(matches, table_matches, tables)
exit()
# Exactly one match from existing do script
elif len(matches) == 1 and len(table_matches) == 0:
# Determine the full path to the matching script
script_path = fs.join(introspection.pts_do_dir, matches[0][0], matches[0][1])
introspection.add_dependencies(self.dependencies, script_path, set())
# Exactly one match from tabulated command
elif len(table_matches) == 1 and len(matches) == 0:
# from pts.core.tools import logging
# configuration
# Path to class module
table_match = table_matches[0]
subproject = table_match[0]
index = table_match[1]
relative_class_module_path = tables[subproject]["Path"][index].replace(".", "/").rsplit("/", 1)[0] + ".py"
class_module_path = fs.join(introspection.pts_subproject_dir(subproject), relative_class_module_path)
logging_path = fs.join(introspection.pts_package_dir, "core", "basics", "log.py")
command_name = tables[subproject]["Command"][index]
configuration_name = tables[subproject]["Configuration"][index]
if configuration_name == "--": configuration_name = command_name
configuration_module_path = fs.join(introspection.pts_root_dir,
"pts/" + subproject + "/config/" + configuration_name + ".py")
# Add dependencies
encountered = set()
introspection.add_dependencies(self.dependencies, logging_path, encountered)
introspection.add_dependencies(self.dependencies, configuration_module_path, encountered)
introspection.add_dependencies(self.dependencies, class_module_path, encountered)
for path, name in fs.directories_in_path(environment.truncation_path,
returns=["path", "name"]):
# Determine the path to the lowres directory
lowres_path = fs.join(path, "lowres")
# Determine the filter
fltr = parse_filter(name)
filter_name = str(fltr)
# Initializ variable
the_image_path = None
# Find the image corresponding to the specified factor
for image_path, image_name in fs.files_in_path(lowres_path,
extension="fits",
returns=["path", "name"]):
# Determine the factor
factor = real(image_name)
# If the factor corresponds to the specified factor, take this image
if np.isclose(factor, config.factor, rtol=0.01):
the_image_path = image_path
break
# Check
if the_image_path is None:
raise ValueError("No truncated " + filter_name +
" image found for a factor of " + str(config.factor))
def launch_reference(self):
"""
This function ...
:return:
"""
# Inform the user
log.info("Launching the reference simulation ...")
# Settings
settings_launch = dict()
settings_launch["ski"] = self.reference_ski_path
settings_launch["input"] = self.simulation_input_path
settings_launch["output"] = self.simulation_output_path
settings_launch["create_output"] = True
settings_launch["remote"] = self.host_id
settings_launch["attached"] = True
settings_launch["show_progress"] = True
# Create the analysis options
analysis = AnalysisOptions()
analysis.extraction.path = self.simulation_extract_path
analysis.plotting.path = self.simulation_plot_path
analysis.misc.path = self.simulation_misc_path
analysis.extraction.progress = True
analysis.extraction.timeline = True
analysis.extraction.memory = True
analysis.plotting.progress = True
analysis.plotting.timeline = True
analysis.plotting.memory = True
analysis.plotting.seds = True
analysis.plotting.grids = True
analysis.plotting.reference_seds = fs.files_in_path(seds_path)
analysis.misc.fluxes = True
analysis.misc.images = True
analysis.misc.observation_filters = fitting_filter_names
analysis.misc.observation_instruments = [instrument_name]
analysis.misc.make_images_remote = self.host_id
analysis.misc.images_wcs = self.reference_wcs_path
analysis.misc.images_unit = "Jy/pix"
analysis.misc.spectral_convolution = self.config.spectral_convolution
# Set flux error bars
dustpedia_sed = ObservedSED.from_file(dustpedia_sed_path)
filter_names = dustpedia_sed.filter_names()
errors = dustpedia_sed.errors()
#print([str(error) for error in errors])
flux_errors = sequences.zip_into_dict(filter_names, [str(error) for error in errors])
analysis.misc.flux_errors = flux_errors
# Create Aniano kernels object
aniano = AnianoKernels()
# Set the paths to the kernel for each image
kernel_paths = dict()
for filter_name in fitting_filter_names: kernel_paths[filter_name] = aniano.get_psf_path(parse_filter(filter_name))
analysis.misc.images_kernels = kernel_paths
# Set the paths to the WCS files for each image
analysis.misc.rebin_wcs = {instrument_name: self.wcs_paths}
# Input
input_launch = dict()
#input_launch["memory"] = MemoryRequirement(serial_memory, parallel_memory)
input_launch["analysis_options"] = analysis
# Launch command
launch = Command("launch_simulation", "launch the reference simulation", settings_launch, input_launch, cwd=".")
self.launcher = self.run_command(launch)
# *****************************************************************
# ** PTS -- Python Toolkit for working with SKIRT **
# ** © Astronomical Observatory, Ghent University **
# *****************************************************************
# Import the relevant PTS classes and modules
from pts.core.basics.configuration import ConfigurationDefinition
from pts.core.tools import introspection
from pts.core.tools import filesystem as fs
# -----------------------------------------------------------------
# The path to the hosts configuration directory
hosts_config_path = fs.join(introspection.pts_config_dir("core"), "hosts")
# The names of the hosts which are pre-configured
preconfigured_names = fs.files_in_path(hosts_config_path, extension="cfg", not_contains="template", returns="name")
# -----------------------------------------------------------------
# Create configuration definition
definition = ConfigurationDefinition(write_config=False)
# Add required
definition.add_required("name", "nonempty_string_no_spaces", "name to give to the host")
# Add optional
definition.add_positional_optional("preconfigured", "string", "name of a preconfigured remote for which to adapt the user-specific settings", choices=preconfigured_names)
# -----------------------------------------------------------------
if not remote.setup(host_id):
log.warning("The remote host '" + host_id + "' is not available: skipping ...")
continue
else: remote = None
# Determine the path to the run directory for the specified remote host
host_run_path = fs.join(introspection.skirt_run_dir, host_id)
# Check if there are simulations
if not fs.is_directory(host_run_path):
log.debug("No run directory for host '" + host_id + "'")
continue
if fs.is_empty(host_run_path): log.debug("No simulations for host '" + host_id + "'")
# Loop over the simulation files in the run directory
for path, name in fs.files_in_path(host_run_path, extension="sim", returns=["path", "name"], sort=int):
# Skip
if config.ids is not None and int(name) not in config.ids: continue
# Inform the user
log.info("Removing simulation " + name + " ...")
# Fully clear
if config.full:
# Debugging
log.debug("Removing the remote files and directories ...")
# Load the simulation
simulation = RemoteSimulation.from_file(path)
# ** © Astronomical Observatory, Ghent University **
# *****************************************************************
# Import the relevant PTS classes and modules
from pts.core.basics.configuration import ConfigurationDefinition
from pts.core.tools import introspection
from pts.core.tools import filesystem as fs
# -----------------------------------------------------------------
# The path to the hosts configuration directory
hosts_config_path = fs.join(introspection.pts_config_dir("core"), "hosts")
# The names of the hosts which are pre-configured
preconfigured_names = fs.files_in_path(hosts_config_path,
extension="cfg",
not_contains="template",
returns="name")
# -----------------------------------------------------------------
# Create configuration definition
definition = ConfigurationDefinition(write_config=False)
# Add required
definition.add_required("name", "nonempty_string_no_spaces",
"name to give to the host")
# Add optional
definition.add_positional_optional(
"preconfigured",
"string",
"--keep",
action="store_true",
help=
"add this option to make sure the output is kept remotely after a subsequent retrieve attempt"
)
# Parse the command line arguments
arguments = parser.parse_args()
# -----------------------------------------------------------------
# Create the remote execution environment
remote = Remote()
remote.setup(arguments.remote)
for path in fs.files_in_path(fs.cwd(), extension="sim"):
# Create simulation
sim = RemoteSimulation.from_file(path)
if not sim.retrieved: continue
local_output_path = sim.output_path
remote_simulation_path = sim.remote_simulation_path
remote_output_path = sim.remote_output_path
remote_input_path = sim.remote_input_path
if not remote.is_directory(remote_simulation_path):
remote.create_directory(remote_simulation_path)
if not remote.is_directory(remote_output_path):
# -*- coding: utf8 -*-
# *****************************************************************
# ** PTS -- Python Toolkit for working with SKIRT **
# ** © Astronomical Observatory, Ghent University **
# *****************************************************************
## \package pts.do.developer.remove_pyc Remove all compiled python (.pyc) files.
# -----------------------------------------------------------------
# Ensure Python 3 compatibility
from __future__ import absolute_import, division, print_function
# Import the relevant PTS classes and modules
from pts.core.tools import introspection
from pts.core.tools import filesystem as fs
from pts.core.basics.log import log
# -----------------------------------------------------------------
# Loop over all pyc files
for path in fs.files_in_path(introspection.pts_package_dir, recursive=True, extension="pyc"):
# Inform the user
log.info("Removing '" + path + "' ...")
# Remove the file
fs.remove_file(path)
# -----------------------------------------------------------------
