def setup(self, **kwargs):
"""
This function ...
:param kwargs:
:return:
"""
# Call the setup function of the base class
super(RemotesConfigurable, self).setup(**kwargs)
# If remotes are passed
if "remotes" in kwargs:
# Get the remotes
self.remotes = kwargs.pop("remotes")
# Check if they are setup
for remote in self.remotes:
if not remote.connected:
raise RuntimeError("Remotes must be connected")
# Remotes are not passed
else:
# Gather host IDs
if self.config.not_remotes is not None:
host_ids = [
host_id for host_id in self.config.host_ids
if host_id not in self.config.not_remotes
]
else:
host_ids = self.config.host_ids
# Loop over the different hosts
for host_id in host_ids:
# Setup the remote (login)
remote = Remote(log_conda=kwargs.pop("log_conda", False))
if not remote.setup(host_id,
one_attempt=self.config.one_attempt):
log.warning("Remote host '" + host_id +
"' is down: skipping")
continue
# Add the remote to the list
self.remotes.append(remote)
def setup(self, **kwargs):
"""
This function ...
:param kwargs:
:return:
"""
# Call the setup function of the base class
super(RemotesConfigurable, self).setup(**kwargs)
# If remotes are passed
if "remotes" in kwargs:
# Get the remotes
self.remotes = kwargs.pop("remotes")
# Check if they are setup
for remote in self.remotes:
if not remote.connected: raise RuntimeError("Remotes must be connected")
# Remotes are not passed
else:
# Gather host IDs
#if self.config.not_remotes is not None: host_ids = [host_id for host_id in self.config.host_ids if host_id not in self.config.not_remotes]
#else: host_ids = self.config.host_ids
# Loop over the different hosts
#for host_id in host_ids:
for host in self.config.hosts:
# Create the remote object
remote = Remote(log_conda=kwargs.pop("log_conda", False))
# Login to the remote
#if not remote.setup(host_id, one_attempt=self.config.one_attempt, cluster_name=self.config.clustername):
if not remote.setup(host, one_attempt=self.config.one_attempt):
log.warning("Remote host '" + host.id + "' is down: skipping")
continue
# Add the remote to the list
self.remotes.append(remote)
def setup(self, **kwargs):
"""
This function ...
:param kwargs:
:return:
"""
# Call the setup function of the base class
super(Uninstaller, self).setup(**kwargs)
# Check if remote instance is passed
if "remote" in kwargs: self.remote = kwargs.pop("remote")
elif self.config.remote is not None:
remote = Remote()
if not remote.setup(self.config.remote): raise RuntimeError("The remote host is not available")
self.remote = remote
else: log.warning('No remote host is specified, will be uninstalling locally')
def setup(self, **kwargs):
"""
This function ...
:param kwargs:
:return:
"""
# Call the setup function of the base class
super(Uninstaller, self).setup(**kwargs)
# Check if remote instance is passed
if "remote" in kwargs: self.remote = kwargs.pop("remote")
elif self.config.remote is not None:
remote = Remote()
if not remote.setup(self.config.remote):
raise RuntimeError("The remote host is not available")
self.remote = remote
else:
log.warning(
'No remote host is specified, will be uninstalling locally')
definition.add_required("id", "positive_integer", "task ID")
# -----------------------------------------------------------------
# Parse the arguments into a configuration
#setter = InteractiveConfigurationSetter("show_log", "Show the log output of a remote PTS task")
config = parse_arguments(
"show_task_log",
definition,
description="Show the log output of a remote PTS task")
# -----------------------------------------------------------------
# Create and setup the remote
remote = Remote()
remote.setup(config.remote)
# Open the task
task_path = fs.join(introspection.pts_run_dir, config.remote,
str(config.id) + ".task")
task = Task.from_file(task_path)
# Check whether the log file is present
log_output_path = task.remote_log_path
log_path = None
for filename in remote.files_in_path(log_output_path):
if "log" in filename:
log_path = fs.join(log_output_path, filename)
break
if log_path is None: raise RuntimeError("Log does not exist remotely")
definition.add_flag("full", "fully clear the simulations, also remove remote simulation directories")
# -----------------------------------------------------------------
# Parse the arguments into a configuration
config = parse_arguments("clear_tasks", definition, description="Clear PTS tasks for a certain remote host")
# -----------------------------------------------------------------
# Loop over the remote hosts
for host_id in config.remotes:
# Check whether the remote is available
if config.full:
remote = Remote()
if not remote.setup(host_id):
log.warning("The remote host '" + host_id + "' is not available: skipping ...")
continue
else: remote = None
# Determine the path to the run directory for the specified remote host
host_run_path = fs.join(introspection.skirt_run_dir, host_id)
# Check if there are simulations
if not fs.is_directory(host_run_path):
log.debug("No run directory for host '" + host_id + "'")
continue
if fs.is_empty(host_run_path): log.debug("No simulations for host '" + host_id + "'")
# Loop over the simulation files in the run directory
for path, name in fs.files_in_path(host_run_path, extension="sim", returns=["path", "name"], sort=int):
# Set log level in a special way
if config.debug: setup_log("DEBUG")
else: setup_log("ERROR")
# -----------------------------------------------------------------
# Loop over the hosts
print("")
for host in config.hosts:
# Debugging
log.debug("Connecting to remote host '" + host.id + "' ...")
# Create remote, try connecting
remote = Remote()
connected = remote.setup(host, login_timeout=10) # only try for 10 seconds
# Connection succeeded?
if connected: print(fmt.green + host.id + ": up" + fmt.reset)
else:
print(fmt.red + host.id + ": down" + fmt.reset)
continue
# Show clustername
if remote.cluster_name is not None: print("Cluster name: " + remote.cluster_name)
print("")
print(" - " + fmt.bold + "uses scheduling system: " + fmt.reset + str(remote.scheduler))
# Show status
show_status(remote)
# Add flags
definition.add_flag("conda", "also remove conda installation")
definition.add_flag("qt", "also remove Qt installation")
definition.add_flag("one_attempt", "only perform one attempt at connecting to a remote")
# Get the config
config = parse_arguments("deinstall_all", definition)
# -----------------------------------------------------------------
# Loop over the remote hosts
for host_id in find_host_ids():
# Setup
remote = Remote()
if not remote.setup(host_id, one_attempt=config.one_attempt):
log.warning("Remote host '" + host_id + "' is offline")
continue
# Create uninstaller
uninstaller = Uninstaller()
# Set options
uninstaller.config.skirt_and_or_pts = config.skirt_and_or_pts
uninstaller.config.conda = config.conda
uninstaller.config.qt = config.qt
# Inform the user
log.info("Running the uninstaller for remote host '" + host_id + "' ...")
# Uninstall
# Read the command line arguments
config = parse_arguments(
"show_progress",
definition,
description="Show the progress of a remotely running simulation")
# -----------------------------------------------------------------
# Load the simulation
simulation = get_simulation_for_host(config.remote, config.id)
# -----------------------------------------------------------------
# Load the remote
remote = Remote()
if not remote.setup(host_id=config.remote):
raise RuntimeError("Could not connect to the remote host")
# -----------------------------------------------------------------
# Create the status object
status = LogSimulationStatus(simulation.remote_log_file_path,
remote=remote,
debug_output=config.debug_output)
# Show the simulation progress
if config.debug_output: success = status.show_progress(simulation.handle)
else:
with no_debugging():
success = status.show_progress(simulation.handle)
from pts.core.remote.host import find_host_ids
# -----------------------------------------------------------------
definition = ConfigurationDefinition()
definition.add_optional("remotes", "string_list", "remote host ID", choices=find_host_ids(), default=find_host_ids())
config = parse_arguments("sessions", definition)
# -----------------------------------------------------------------
# Loop over the remote host
for host_id in config.remotes:
# Create and setup the remote
remote = Remote()
if not remote.setup(host_id): raise RuntimeError("The remote host '" + host_id + "' is not available at the moment")
# Get screen names and tmux session names
screen_names = remote.screen_names()
tmux_names = remote.tmux_names()
# Inform the user
log.info("Closing sessions on remote host '" + host_id + "' ...")
# Loop over the screens
for name in screen_names:
# Inform the user
log.info("Closing screen session '" + name + "' ...")
# Close
# Add required
definition.add_required("remote", "string", "the remote host ID")
# Add flags
definition.add_flag("versions", "compare versions", "v")
# Get configuration
config = parse_arguments("compare_python_packages", definition)
# -----------------------------------------------------------------
# Create the remote execution environment
remote = Remote()
# Log in
remote.setup(config.remote)
# Get remote python session
remote_python = remote.start_python_session(assume_pts=False)
# Get all python packages installed on the remote host
remote_packages = remote_python.installed_packages
# Get local python package version
local_packages = introspection.installed_python_packages()
# Loop over all python packages necessary for PTS
for dependency in introspection.get_all_dependencies():
# Skip standard modules
if introspection.is_std_lib(dependency): continue
definition.add_required("remote", "string", "remote host to mount", choices=all_host_ids())
definition.add_required("id", "positive_integer", "simulation ID")
definition.add_flag("debug_output", "show all simulation output in debug mode")
# Read the command line arguments
config = parse_arguments("show_progress", definition, description="Show the progress of a remotely running simulation")
# -----------------------------------------------------------------
# Load the simulation
simulation = get_simulation_for_host(config.remote, config.id)
# -----------------------------------------------------------------
# Load the remote
remote = Remote()
if not remote.setup(host_id=config.remote): raise RuntimeError("Could not connect to the remote host")
# -----------------------------------------------------------------
# Create the status object
status = LogSimulationStatus(simulation.remote_log_file_path, remote=remote, debug_output=config.debug_output)
# Show the simulation progress
with log.no_debugging(): success = status.show_progress(simulation.handle)
# Check whether not crashed
if not success: raise RuntimeError("The simulation crashed")
# -----------------------------------------------------------------
definition.add_flag("conda", "also remove conda installation")
definition.add_flag("qt", "also remove Qt installation")
definition.add_flag("one_attempt",
"only perform one attempt at connecting to a remote")
# Get the config
config = parse_arguments("deinstall_all", definition)
# -----------------------------------------------------------------
# Loop over the remote hosts
for host_id in find_host_ids():
# Setup
remote = Remote()
if not remote.setup(host_id, one_attempt=config.one_attempt):
log.warning("Remote host '" + host_id + "' is offline")
continue
# Create uninstaller
uninstaller = Uninstaller()
# Set options
uninstaller.config.skirt_and_or_pts = config.skirt_and_or_pts
uninstaller.config.conda = config.conda
uninstaller.config.qt = config.qt
# Inform the user
log.info("Running the uninstaller for remote host '" + host_id + "' ...")
# Uninstall
class SourcesTest(SourcesTestBase):
"""
This class ...
"""
def __init__(self, *args, **kwargs):
"""
The constructor ...
:param kwargs:
"""
# Call the constructor of the base class
super(SourcesTest, self).__init__(*args, **kwargs)
# The galaxy properties
self.properties = None
# The rotation masks
self.rotation_masks = dict()
# The catalog fetcher
self.fetcher = CatalogFetcher()
# Catalogs
self.extended_source_catalog = None
# The real FWHMs
self.real_fwhms = dict()
# -----------------------------------------------------------------
def _run(self, **kwargs):
"""
This function ...
:param kwargs:
:return:
"""
# 2. Load galaxy properties
self.load_properties()
# 3. Load the coordinate system
self.load_coordinate_systems()
# 4. Initialize the frames
self.initialize_frames()
# 5. Set FWHMs
self.set_fwhms()
# 6. Make rotation mask
self.make_rotation_masks()
# 7. Get catalogs
self.get_catalogs()
# 8. Generate the sources
self.make_sources()
# 9. Make noise
self.make_noise(self.rotation_masks)
# 10. Make sky
self.make_sky()
# 11. Create the dataset
self.create_dataset(self.rotation_masks)
# 12. Create the directories
self.create_directories()
# 13. Find sources
self.find()
# 14. Extract sources
self.extract()
# 15. Write
self.write()
# 16. Plot
self.plot()
# -----------------------------------------------------------------
def setup(self, **kwargs):
"""
This function ...
:param kwargs:
:return:
"""
# Call the setup function of the base class
super(SourcesTest, self).setup(**kwargs)
# Set remote
if self.config.remote is not None:
self.remote = Remote()
self.remote.setup(self.config.remote)
# -----------------------------------------------------------------
def load_properties(self):
"""
This function ...
:return:
"""
# Inform the user
log.info("Loading the galaxy properties ...")
# Determine the path
path = fs.join(m81_data_path, "properties.dat")
# Load
self.properties = GalaxyProperties.from_file(path)
# -----------------------------------------------------------------
def load_coordinate_systems(self):
"""
This function ...
:return:
"""
# Inform the user
log.info("Loading the coordinate systems ...")
nfilters_stars = 0
nfilters_extra = 0
# Loop over the header files
for path, name in fs.files_in_path(headers_path, extension="txt", returns=["path", "name"]):
# Get the filter and wavelength
fltr = parse_filter(name)
wavelength = fltr.effective if fltr.effective is not None else fltr.center
# SKip Planck
if fltr.observatory == "Planck": continue
# Wavelength greater than 25 micron
if wavelength > wavelengths.ranges.ir.mir.max:
if nfilters_extra == self.config.nfilters_extra: continue
else: nfilters_extra += 1
# Wavelength smaller than 25 micron
else:
if nfilters_stars == self.config.nfilters_stars: continue
else: nfilters_stars += 1
# Debugging
log.debug("Loading the coordinate system for the '" + str(fltr) + "' filter ...")
# Get WCS
wcs = CoordinateSystem.from_header_file(path)
# Add the coordinate system
self.coordinate_systems.append(wcs, fltr=fltr)
# Break the loop if we have enough
if nfilters_stars == self.config.nfilters_stars and nfilters_extra == self.config.nfilters_extra: break
# -----------------------------------------------------------------
def make_rotation_masks(self):
"""
This function ...
:return:
"""
# Inform the user
log.info("Making rotation masks ...")
# Loop over the filters
for fltr in self.frames:
# Debugging
log.info("Making rotation mask for the '" + str(fltr) + "' image ...")
# Get the frame
frame = self.frames[fltr]
# Rotate
if self.config.rotate:
# Choose a random rotation angle
angle = Angle(np.random.uniform(-90, 90), "deg")
# Debugging
log.debug("The random rotation angle is '" + stringify.stringify(angle)[1] + "'")
# Create mask
mask = frame.rotation_mask(angle)
# Don't rotate
else: mask = Mask.empty_like(frame)
# Set the mask
self.rotation_masks[fltr] = mask
# -----------------------------------------------------------------
def get_catalogs(self):
"""
This function ...
:return:
"""
# Inform the user
log.info("Getting catalogs of point and extended sources ...")
# Adding catalogued sources
if self.config.add_catalogued_sources: self.fetch_catalogs()
else: self.initialize_catalogs()
# Add random sources
self.create_random_sources()
# -----------------------------------------------------------------
def fetch_catalogs(self):
"""
This function ...
:return:
"""
# Inform the user
log.info("Fetching catalogs ...")
# Get extended source catalog
self.extended_source_catalog = self.fetcher.get_extended_source_catalog(self.coordinate_systems.bounding_box)
# Fetch
self.point_source_catalog = self.fetcher.get_point_source_catalog(self.coordinate_systems.bounding_box, self.coordinate_systems.min_pixelscale, self.config.point_source_catalogs)
# -----------------------------------------------------------------
def initialize_catalogs(self):
"""
This function ...
:param self:
:return:
"""
# Inform the user
log.info("Initializing catalogs ...")
# Extended sources
self.extended_source_catalog = ExtendedSourceCatalog()
# Add principal galaxy
self.add_principal_galaxy()
# Point sources
self.point_source_catalog = PointSourceCatalog()
# -----------------------------------------------------------------
def add_principal_galaxy(self):
"""
This function ...
:return:
"""
# Inform the user
log.info("Adding principal galaxy ...")
# Get info
name, position, redshift, galaxy_type, names, distance, inclination, d25, major, minor, pa = get_galaxy_info("M81", self.properties.center)
ra = position.ra
dec = position.dec
principal = True
companions = []
parent = None
# Add to the catalog
self.extended_source_catalog.add_entry("M81", ra, dec, redshift, galaxy_type, names, distance, inclination, d25, major, minor, pa, principal, companions, parent)
# -----------------------------------------------------------------
def make_sources(self):
"""
This function ...
:return:
"""
# Inform the user
log.info("Making sources ...")
# Extended
self.make_extended_sources()
# Point
self.make_point_sources()
# -----------------------------------------------------------------
def make_extended_sources(self):
"""
This function ...
:return:
"""
# Inform the user
log.info("Making extended sources ...")
# Loop over the filters
for fltr in self.frames:
# Get the frame
frame = self.frames[fltr]
# Loop over the sources
for index in range(len(self.extended_source_catalog)):
principal = self.extended_source_catalog["Principal"][index]
# Determine flux
if principal: central_flux = 1e2
else: central_flux = np.random.uniform(10,50)
# Get pixel position
coordinate = self.extended_source_catalog.get_position(index).to_pixel(frame.wcs)
initial_sersic_amplitude = central_flux
initial_sersic_x_0 = coordinate.x
initial_sersic_y_0 = coordinate.y
if principal:
s4g_name, pa, ellipticity, n, re, mag = catalogs.get_galaxy_s4g_one_component_info("M81")
angle = Angle(pa, "deg")
angle_deg = pa
effective_radius = re.to("arcsec").value / frame.average_pixelscale.to("arcsec").value
initial_sersic_n = n
initial_sersic_r_eff = effective_radius
initial_sersic_ellip = ellipticity
initial_sersic_theta = np.deg2rad(angle_deg)
# 1 / axial_ratio = 1 - ellipticity
axial_ratio = 1. / (1. - ellipticity)
else:
# Get position angle and axes lengths
pa = self.extended_source_catalog["Posangle"][index]
major = self.extended_source_catalog["Major"][index]
minor = self.extended_source_catalog["Minor"][index]
axial_ratio = major / minor
angle = Angle(pa, "deg")
angle_deg = pa
effective_radius = major
# Produce guess values
initial_sersic_r_eff = effective_radius
initial_sersic_n = self.config.galaxy_sersic_index
initial_sersic_ellip = (axial_ratio - 1.0) / axial_ratio
initial_sersic_theta = np.deg2rad(angle_deg)
# Produce sersic model from guess parameters, for time trials
sersic_x, sersic_y = np.meshgrid(np.arange(frame.xsize), np.arange(frame.ysize))
sersic_model = Sersic2D(amplitude=initial_sersic_amplitude, r_eff=initial_sersic_r_eff,
n=initial_sersic_n, x_0=initial_sersic_x_0,
y_0=initial_sersic_y_0, ellip=initial_sersic_ellip,
theta=initial_sersic_theta)
sersic_map = sersic_model(sersic_x, sersic_y)
# Limit galaxy?
if self.config.limit_galaxy:
limit_radius = self.config.galaxy_relative_asymptotic_radius * effective_radius
# Create galaxy region
galaxy_center = PixelCoordinate(initial_sersic_x_0, initial_sersic_y_0)
galaxy_radius = PixelStretch(limit_radius, limit_radius / axial_ratio)
galaxy_region = PixelEllipseRegion(galaxy_center, galaxy_radius, angle)
galaxy_mask = galaxy_region.to_mask(frame.xsize, frame.ysize)
# Set galaxy map zero outside certain radius
sersic_map[galaxy_mask.inverse()] = 0.0
# Add
frame += sersic_map
# mask
frame[self.rotation_masks[fltr]] = 0.0
# -----------------------------------------------------------------
def make_point_sources(self):
"""
This function ...
:return:
"""
# Inform the user
log.info("Making point sources ...")
# Call the appropriate function
if self.config.vary_fwhm: self.make_point_sources_variable_fwhm(self.rotation_masks)
else: self.make_point_sources_fixed_fwhm(self.rotation_masks)
# -----------------------------------------------------------------
def make_sky(self):
"""
This function ...
:return:
"""
# Inform the user
log.info("Adding sky ...")
# Loop over the frames
for fltr in self.frames:
# Get the frame
frame = self.frames[fltr]
# Determine random sky level
sky_level = np.random.uniform(0.0, 10.)
# Create sky gradient
y, x = np.mgrid[:frame.ysize, :frame.xsize]
sky_gradient = x * y
# Normalize so that the maximal sky is 20%
sky_gradient = sky_gradient / np.max(sky_gradient) * 20
# Add the sky
frame += sky_level + sky_gradient
# Mask
frame[self.rotation_masks[fltr]] = 0.0
# -----------------------------------------------------------------
def make_galaxy(self):
"""
This function ...
:return:
"""
# Inform the user
log.info("Adding galaxy ...")
# -----------------------------------------------------------------
def find(self):
"""
This function ...
:return:
"""
# Inform the user
log.info("Finding the sources ...")
# Settings
settings = dict()
#settings["input"] =
settings["output"] = self.find_path
settings["nprocesses"] = self.config.nprocesses
# Input
input_dict = dict()
input_dict["dataset"] = self.dataset
input_dict["extended_source_catalog"] = self.extended_source_catalog
input_dict["point_source_catalog"] = self.point_source_catalog
input_dict["output_paths"] = self.find_paths
# Construct the command
command = Command("find_sources", "find sources", settings, input_dict)
# Run the command
self.finder = self.run_command(command, remote=self.remote)
# -----------------------------------------------------------------
def write(self):
"""
This function ...
:return:
"""
# Inform the user
log.info("Writing ...")
# -----------------------------------------------------------------
def plot(self):
"""
This function ...
:return:
"""
# Inform the user
log.info("Plotting ...")
# Parse the arguments into a configuration
config = parse_arguments(
"clear_tasks",
definition,
description="Clear PTS tasks for a certain remote host")
# -----------------------------------------------------------------
# Loop over the remote hosts
for host_id in config.remotes:
# Check whether the remote is available
if config.full:
remote = Remote()
if not remote.setup(host_id):
log.warning("The remote host '" + host_id +
"' is not available: skipping ...")
continue
else:
remote = None
# Determine the path to the run directory for the specified remote host
host_run_path = fs.join(introspection.skirt_run_dir, host_id)
# Check if there are simulations
if not fs.is_directory(host_run_path):
log.debug("No run directory for host '" + host_id + "'")
continue
if fs.is_empty(host_run_path):
log.debug("No simulations for host '" + host_id + "'")
from __future__ import absolute_import, division, print_function
# Import the relevant PTS classes and modules
from pts.core.remote.host import find_host_ids
from pts.core.remote.remote import Remote
from pts.core.tools import formatting as fmt
from pts.core.basics.log import setup_log
from pts.core.basics.configuration import parse_logging_arguments
# -----------------------------------------------------------------
config = parse_logging_arguments("hosts")
# -----------------------------------------------------------------
# Set log level in a special way
if config.debug: setup_log("DEBUG")
else: setup_log("ERROR")
# -----------------------------------------------------------------
# Loop over the hosts
for host_id in find_host_ids():
remote = Remote()
connected = remote.setup(host_id, login_timeout=15) # only try for 15 seconds
if connected: print(fmt.green + host_id + ": up" + fmt.reset)
else: print(fmt.red + host_id + ": down" + fmt.reset)
# -----------------------------------------------------------------
# Import the relevant PTS classes and modules
from pts.core.basics.configuration import ConfigurationDefinition, parse_arguments
from pts.core.remote.remote import Remote
# -----------------------------------------------------------------
definition = ConfigurationDefinition()
definition.add_required("remote", "string", "remote host ID")
definition.add_required("line", "string", "line to be executed on the remote host")
config = parse_arguments("sessions", definition)
# -----------------------------------------------------------------
# Initialize the remote
remote = Remote()
if not remote.setup(config.remote): raise RuntimeError("The remote host '" + config.remote + "' is not available at the moment")
# -----------------------------------------------------------------
print("")
#print("-----------------------------------------------------------------")
print("OUTPUT")
print("-----------------------------------------------------------------")
print("")
# Execute the line, show the output
output = remote.execute(config.line)
for line in output: print(line)
print("")
print("-----------------------------------------------------------------")
# Import the relevant PTS classes and modules
from pts.core.basics.log import log
from pts.core.basics.configuration import ConfigurationDefinition, parse_arguments
from pts.core.remote.remote import Remote
# -----------------------------------------------------------------
definition = ConfigurationDefinition()
definition.add_required("remote", "string", "remote host ID")
setter = parse_arguments("sessions", definition)
# -----------------------------------------------------------------
remote = Remote()
if not remote.setup(config.remote):
raise RuntimeError("The remote host '" + config.remote +
"' is not available at the moment")
screen_names = remote.screen_names()
tmux_names = remote.tmux_names()
print("SCREEN")
print(screen_names)
print("")
print("TMUX")
print(tmux_names)
class SourcesTest(SourcesTestBase):
"""
This class ...
"""
def __init__(self, *args, **kwargs):
"""
The constructor ...
:param kwargs:
"""
# Call the constructor of the base class
super(SourcesTest, self).__init__(*args, **kwargs)
# The galaxy properties
self.properties = None
# The rotation masks
self.rotation_masks = dict()
# The catalog fetcher
self.fetcher = CatalogFetcher()
# Catalogs
self.extended_source_catalog = None
# The real FWHMs
self.real_fwhms = dict()
# -----------------------------------------------------------------
def run(self, **kwargs):
"""
This function ...
:param kwargs:
:return:
"""
# 1. Call the setup function
self.setup(**kwargs)
# 2. Load galaxy properties
self.load_properties()
# 3. Load the coordinate system
self.load_coordinate_systems()
# 4. Initialize the frames
self.initialize_frames()
# 5. Set FWHMs
self.set_fwhms()
# 6. Make rotation mask
self.make_rotation_masks()
# 7. Get catalogs
self.get_catalogs()
# 8. Generate the sources
self.make_sources()
# 9. Make noise
self.make_noise(self.rotation_masks)
# 10. Make sky
self.make_sky()
# 11. Create the dataset
self.create_dataset(self.rotation_masks)
# 12. Create the directories
self.create_directories()
# 13. Find sources
self.find()
# 14. Extract sources
self.extract()
# 15. Write
self.write()
# 16. Plot
self.plot()
# -----------------------------------------------------------------
def setup(self, **kwargs):
"""
This function ...
:param kwargs:
:return:
"""
# Call the setup function of the base class
super(SourcesTest, self).setup(**kwargs)
# Set remote
if self.config.remote is not None:
self.remote = Remote()
self.remote.setup(self.config.remote)
# -----------------------------------------------------------------
def load_properties(self):
"""
This function ...
:return:
"""
# Inform the user
log.info("Loading the galaxy properties ...")
# Determine the path
path = fs.join(m81_data_path, "properties.dat")
# Load
self.properties = GalaxyProperties.from_file(path)
# -----------------------------------------------------------------
def load_coordinate_systems(self):
"""
This function ...
:return:
"""
# Inform the user
log.info("Loading the coordinate systems ...")
nfilters_stars = 0
nfilters_extra = 0
# Loop over the header files
for path, name in fs.files_in_path(headers_path,
extension="txt",
returns=["path", "name"]):
# Get the filter and wavelength
fltr = parse_filter(name)
wavelength = fltr.effective if fltr.effective is not None else fltr.center
# SKip Planck
if fltr.observatory == "Planck": continue
# Wavelength greater than 25 micron
if wavelength > wavelengths.ranges.ir.mir.max:
if nfilters_extra == self.config.nfilters_extra: continue
else: nfilters_extra += 1
# Wavelength smaller than 25 micron
else:
if nfilters_stars == self.config.nfilters_stars: continue
else: nfilters_stars += 1
# Debugging
log.debug("Loading the coordinate system for the '" + str(fltr) +
"' filter ...")
# Get WCS
wcs = CoordinateSystem.from_header_file(path)
# Add the coordinate system
self.coordinate_systems.append(wcs, fltr=fltr)
# Break the loop if we have enough
if nfilters_stars == self.config.nfilters_stars and nfilters_extra == self.config.nfilters_extra:
break
# -----------------------------------------------------------------
def make_rotation_masks(self):
"""
This function ...
:return:
"""
# Inform the user
log.info("Making rotation masks ...")
# Loop over the filters
for fltr in self.frames:
# Debugging
log.info("Making rotation mask for the '" + str(fltr) +
"' image ...")
# Get the frame
frame = self.frames[fltr]
# Rotate
if self.config.rotate:
# Choose a random rotation angle
angle = Angle(np.random.uniform(-90, 90), "deg")
# Debugging
log.debug("The random rotation angle is '" +
stringify.stringify(angle)[1] + "'")
# Create mask
mask = frame.rotation_mask(angle)
# Don't rotate
else:
mask = Mask.empty_like(frame)
# Set the mask
self.rotation_masks[fltr] = mask
# -----------------------------------------------------------------
def get_catalogs(self):
"""
This function ...
:return:
"""
# Inform the user
log.info("Getting catalogs of point and extended sources ...")
# Adding catalogued sources
if self.config.add_catalogued_sources: self.fetch_catalogs()
else: self.initialize_catalogs()
# Add random sources
self.create_random_sources()
# -----------------------------------------------------------------
def fetch_catalogs(self):
"""
This function ...
:return:
"""
# Inform the user
log.info("Fetching catalogs ...")
# Get extended source catalog
self.extended_source_catalog = self.fetcher.get_extended_source_catalog(
self.coordinate_systems.bounding_box)
# Fetch
self.point_source_catalog = self.fetcher.get_point_source_catalog(
self.coordinate_systems.bounding_box,
self.coordinate_systems.min_pixelscale,
self.config.point_source_catalogs)
# -----------------------------------------------------------------
def initialize_catalogs(self):
"""
This function ...
:param self:
:return:
"""
# Inform the user
log.info("Initializing catalogs ...")
# Extended sources
self.extended_source_catalog = ExtendedSourceCatalog()
# Add principal galaxy
self.add_principal_galaxy()
# Point sources
self.point_source_catalog = PointSourceCatalog()
# -----------------------------------------------------------------
def add_principal_galaxy(self):
"""
This function ...
:return:
"""
# Inform the user
log.info("Adding principal galaxy ...")
# Get info
name, position, redshift, galaxy_type, names, distance, inclination, d25, major, minor, pa = get_galaxy_info(
"M81", self.properties.center)
ra = position.ra
dec = position.dec
principal = True
companions = []
parent = None
# Add to the catalog
self.extended_source_catalog.add_entry("M81", ra, dec, redshift,
galaxy_type, names, distance,
inclination, d25, major, minor,
pa, principal, companions,
parent)
# -----------------------------------------------------------------
def make_sources(self):
"""
This function ...
:return:
"""
# Inform the user
log.info("Making sources ...")
# Extended
self.make_extended_sources()
# Point
self.make_point_sources()
# -----------------------------------------------------------------
def make_extended_sources(self):
"""
This function ...
:return:
"""
# Inform the user
log.info("Making extended sources ...")
# Loop over the filters
for fltr in self.frames:
# Get the frame
frame = self.frames[fltr]
# Loop over the sources
for index in range(len(self.extended_source_catalog)):
principal = self.extended_source_catalog["Principal"][index]
# Determine flux
if principal: central_flux = 1e2
else: central_flux = np.random.uniform(10, 50)
# Get pixel position
coordinate = self.extended_source_catalog.get_position(
index).to_pixel(frame.wcs)
initial_sersic_amplitude = central_flux
initial_sersic_x_0 = coordinate.x
initial_sersic_y_0 = coordinate.y
if principal:
s4g_name, pa, ellipticity, n, re, mag = catalogs.get_galaxy_s4g_one_component_info(
"M81")
angle = Angle(pa, "deg")
angle_deg = pa
effective_radius = re.to(
"arcsec").value / frame.average_pixelscale.to(
"arcsec").value
initial_sersic_n = n
initial_sersic_r_eff = effective_radius
initial_sersic_ellip = ellipticity
initial_sersic_theta = np.deg2rad(angle_deg)
# 1 / axial_ratio = 1 - ellipticity
axial_ratio = 1. / (1. - ellipticity)
else:
# Get position angle and axes lengths
pa = self.extended_source_catalog["Posangle"][index]
major = self.extended_source_catalog["Major"][index]
minor = self.extended_source_catalog["Minor"][index]
axial_ratio = major / minor
angle = Angle(pa, "deg")
angle_deg = pa
effective_radius = major
# Produce guess values
initial_sersic_r_eff = effective_radius
initial_sersic_n = self.config.galaxy_sersic_index
initial_sersic_ellip = (axial_ratio - 1.0) / axial_ratio
initial_sersic_theta = np.deg2rad(angle_deg)
# Produce sersic model from guess parameters, for time trials
sersic_x, sersic_y = np.meshgrid(np.arange(frame.xsize),
np.arange(frame.ysize))
sersic_model = Sersic2D(amplitude=initial_sersic_amplitude,
r_eff=initial_sersic_r_eff,
n=initial_sersic_n,
x_0=initial_sersic_x_0,
y_0=initial_sersic_y_0,
ellip=initial_sersic_ellip,
theta=initial_sersic_theta)
sersic_map = sersic_model(sersic_x, sersic_y)
# Limit galaxy?
if self.config.limit_galaxy:
limit_radius = self.config.galaxy_relative_asymptotic_radius * effective_radius
# Create galaxy region
galaxy_center = PixelCoordinate(initial_sersic_x_0,
initial_sersic_y_0)
galaxy_radius = PixelStretch(limit_radius,
limit_radius / axial_ratio)
galaxy_region = PixelEllipseRegion(galaxy_center,
galaxy_radius, angle)
galaxy_mask = galaxy_region.to_mask(
frame.xsize, frame.ysize)
# Set galaxy map zero outside certain radius
sersic_map[galaxy_mask.inverse()] = 0.0
# Add
frame += sersic_map
# mask
frame[self.rotation_masks[fltr]] = 0.0
# -----------------------------------------------------------------
def make_point_sources(self):
"""
This function ...
:return:
"""
# Inform the user
log.info("Making point sources ...")
# Call the appropriate function
if self.config.vary_fwhm:
self.make_point_sources_variable_fwhm(self.rotation_masks)
else:
self.make_point_sources_fixed_fwhm(self.rotation_masks)
# -----------------------------------------------------------------
def make_sky(self):
"""
This function ...
:return:
"""
# Inform the user
log.info("Adding sky ...")
# Loop over the frames
for fltr in self.frames:
# Get the frame
frame = self.frames[fltr]
# Determine random sky level
sky_level = np.random.uniform(0.0, 10.)
# Create sky gradient
y, x = np.mgrid[:frame.ysize, :frame.xsize]
sky_gradient = x * y
# Normalize so that the maximal sky is 20%
sky_gradient = sky_gradient / np.max(sky_gradient) * 20
# Add the sky
frame += sky_level + sky_gradient
# Mask
frame[self.rotation_masks[fltr]] = 0.0
# -----------------------------------------------------------------
def make_galaxy(self):
"""
This function ...
:return:
"""
# Inform the user
log.info("Adding galaxy ...")
# -----------------------------------------------------------------
def find(self):
"""
This function ...
:return:
"""
# Inform the user
log.info("Finding the sources ...")
# Settings
settings = dict()
#settings["input"] =
settings["output"] = self.find_path
settings["nprocesses"] = self.config.nprocesses
# Input
input_dict = dict()
input_dict["dataset"] = self.dataset
input_dict["extended_source_catalog"] = self.extended_source_catalog
input_dict["point_source_catalog"] = self.point_source_catalog
input_dict["output_paths"] = self.find_paths
# Construct the command
command = Command("find_sources", "find sources", settings, input_dict)
# Run the command
self.finder = self.run_command(command, remote=self.remote)
# -----------------------------------------------------------------
def write(self):
"""
This function ...
:return:
"""
# Inform the user
log.info("Writing ...")
# -----------------------------------------------------------------
def plot(self):
"""
This function ...
:return:
"""
# Inform the user
log.info("Plotting ...")
