def setup(self, **kwargs): """ This function ... :param kwargs: :return: """ # Call the setup function of the base class super(RemotesConfigurable, self).setup(**kwargs) # If remotes are passed if "remotes" in kwargs: # Get the remotes self.remotes = kwargs.pop("remotes") # Check if they are setup for remote in self.remotes: if not remote.connected: raise RuntimeError("Remotes must be connected") # Remotes are not passed else: # Gather host IDs if self.config.not_remotes is not None: host_ids = [ host_id for host_id in self.config.host_ids if host_id not in self.config.not_remotes ] else: host_ids = self.config.host_ids # Loop over the different hosts for host_id in host_ids: # Setup the remote (login) remote = Remote(log_conda=kwargs.pop("log_conda", False)) if not remote.setup(host_id, one_attempt=self.config.one_attempt): log.warning("Remote host '" + host_id + "' is down: skipping") continue # Add the remote to the list self.remotes.append(remote)
def setup(self, **kwargs): """ This function ... :param kwargs: :return: """ # Call the setup function of the base class super(RemotesConfigurable, self).setup(**kwargs) # If remotes are passed if "remotes" in kwargs: # Get the remotes self.remotes = kwargs.pop("remotes") # Check if they are setup for remote in self.remotes: if not remote.connected: raise RuntimeError("Remotes must be connected") # Remotes are not passed else: # Gather host IDs #if self.config.not_remotes is not None: host_ids = [host_id for host_id in self.config.host_ids if host_id not in self.config.not_remotes] #else: host_ids = self.config.host_ids # Loop over the different hosts #for host_id in host_ids: for host in self.config.hosts: # Create the remote object remote = Remote(log_conda=kwargs.pop("log_conda", False)) # Login to the remote #if not remote.setup(host_id, one_attempt=self.config.one_attempt, cluster_name=self.config.clustername): if not remote.setup(host, one_attempt=self.config.one_attempt): log.warning("Remote host '" + host.id + "' is down: skipping") continue # Add the remote to the list self.remotes.append(remote)
def setup(self, **kwargs): """ This function ... :param kwargs: :return: """ # Call the setup function of the base class super(Uninstaller, self).setup(**kwargs) # Check if remote instance is passed if "remote" in kwargs: self.remote = kwargs.pop("remote") elif self.config.remote is not None: remote = Remote() if not remote.setup(self.config.remote): raise RuntimeError("The remote host is not available") self.remote = remote else: log.warning('No remote host is specified, will be uninstalling locally')
def setup(self, **kwargs): """ This function ... :param kwargs: :return: """ # Call the setup function of the base class super(Uninstaller, self).setup(**kwargs) # Check if remote instance is passed if "remote" in kwargs: self.remote = kwargs.pop("remote") elif self.config.remote is not None: remote = Remote() if not remote.setup(self.config.remote): raise RuntimeError("The remote host is not available") self.remote = remote else: log.warning( 'No remote host is specified, will be uninstalling locally')
definition.add_required("id", "positive_integer", "task ID") # ----------------------------------------------------------------- # Parse the arguments into a configuration #setter = InteractiveConfigurationSetter("show_log", "Show the log output of a remote PTS task") config = parse_arguments( "show_task_log", definition, description="Show the log output of a remote PTS task") # ----------------------------------------------------------------- # Create and setup the remote remote = Remote() remote.setup(config.remote) # Open the task task_path = fs.join(introspection.pts_run_dir, config.remote, str(config.id) + ".task") task = Task.from_file(task_path) # Check whether the log file is present log_output_path = task.remote_log_path log_path = None for filename in remote.files_in_path(log_output_path): if "log" in filename: log_path = fs.join(log_output_path, filename) break if log_path is None: raise RuntimeError("Log does not exist remotely")
definition.add_flag("full", "fully clear the simulations, also remove remote simulation directories") # ----------------------------------------------------------------- # Parse the arguments into a configuration config = parse_arguments("clear_tasks", definition, description="Clear PTS tasks for a certain remote host") # ----------------------------------------------------------------- # Loop over the remote hosts for host_id in config.remotes: # Check whether the remote is available if config.full: remote = Remote() if not remote.setup(host_id): log.warning("The remote host '" + host_id + "' is not available: skipping ...") continue else: remote = None # Determine the path to the run directory for the specified remote host host_run_path = fs.join(introspection.skirt_run_dir, host_id) # Check if there are simulations if not fs.is_directory(host_run_path): log.debug("No run directory for host '" + host_id + "'") continue if fs.is_empty(host_run_path): log.debug("No simulations for host '" + host_id + "'") # Loop over the simulation files in the run directory for path, name in fs.files_in_path(host_run_path, extension="sim", returns=["path", "name"], sort=int):
# Set log level in a special way if config.debug: setup_log("DEBUG") else: setup_log("ERROR") # ----------------------------------------------------------------- # Loop over the hosts print("") for host in config.hosts: # Debugging log.debug("Connecting to remote host '" + host.id + "' ...") # Create remote, try connecting remote = Remote() connected = remote.setup(host, login_timeout=10) # only try for 10 seconds # Connection succeeded? if connected: print(fmt.green + host.id + ": up" + fmt.reset) else: print(fmt.red + host.id + ": down" + fmt.reset) continue # Show clustername if remote.cluster_name is not None: print("Cluster name: " + remote.cluster_name) print("") print(" - " + fmt.bold + "uses scheduling system: " + fmt.reset + str(remote.scheduler)) # Show status show_status(remote)
# Add flags definition.add_flag("conda", "also remove conda installation") definition.add_flag("qt", "also remove Qt installation") definition.add_flag("one_attempt", "only perform one attempt at connecting to a remote") # Get the config config = parse_arguments("deinstall_all", definition) # ----------------------------------------------------------------- # Loop over the remote hosts for host_id in find_host_ids(): # Setup remote = Remote() if not remote.setup(host_id, one_attempt=config.one_attempt): log.warning("Remote host '" + host_id + "' is offline") continue # Create uninstaller uninstaller = Uninstaller() # Set options uninstaller.config.skirt_and_or_pts = config.skirt_and_or_pts uninstaller.config.conda = config.conda uninstaller.config.qt = config.qt # Inform the user log.info("Running the uninstaller for remote host '" + host_id + "' ...") # Uninstall
# Read the command line arguments config = parse_arguments( "show_progress", definition, description="Show the progress of a remotely running simulation") # ----------------------------------------------------------------- # Load the simulation simulation = get_simulation_for_host(config.remote, config.id) # ----------------------------------------------------------------- # Load the remote remote = Remote() if not remote.setup(host_id=config.remote): raise RuntimeError("Could not connect to the remote host") # ----------------------------------------------------------------- # Create the status object status = LogSimulationStatus(simulation.remote_log_file_path, remote=remote, debug_output=config.debug_output) # Show the simulation progress if config.debug_output: success = status.show_progress(simulation.handle) else: with no_debugging(): success = status.show_progress(simulation.handle)
from pts.core.remote.host import find_host_ids # ----------------------------------------------------------------- definition = ConfigurationDefinition() definition.add_optional("remotes", "string_list", "remote host ID", choices=find_host_ids(), default=find_host_ids()) config = parse_arguments("sessions", definition) # ----------------------------------------------------------------- # Loop over the remote host for host_id in config.remotes: # Create and setup the remote remote = Remote() if not remote.setup(host_id): raise RuntimeError("The remote host '" + host_id + "' is not available at the moment") # Get screen names and tmux session names screen_names = remote.screen_names() tmux_names = remote.tmux_names() # Inform the user log.info("Closing sessions on remote host '" + host_id + "' ...") # Loop over the screens for name in screen_names: # Inform the user log.info("Closing screen session '" + name + "' ...") # Close
# Add required definition.add_required("remote", "string", "the remote host ID") # Add flags definition.add_flag("versions", "compare versions", "v") # Get configuration config = parse_arguments("compare_python_packages", definition) # ----------------------------------------------------------------- # Create the remote execution environment remote = Remote() # Log in remote.setup(config.remote) # Get remote python session remote_python = remote.start_python_session(assume_pts=False) # Get all python packages installed on the remote host remote_packages = remote_python.installed_packages # Get local python package version local_packages = introspection.installed_python_packages() # Loop over all python packages necessary for PTS for dependency in introspection.get_all_dependencies(): # Skip standard modules if introspection.is_std_lib(dependency): continue
definition.add_required("remote", "string", "remote host to mount", choices=all_host_ids()) definition.add_required("id", "positive_integer", "simulation ID") definition.add_flag("debug_output", "show all simulation output in debug mode") # Read the command line arguments config = parse_arguments("show_progress", definition, description="Show the progress of a remotely running simulation") # ----------------------------------------------------------------- # Load the simulation simulation = get_simulation_for_host(config.remote, config.id) # ----------------------------------------------------------------- # Load the remote remote = Remote() if not remote.setup(host_id=config.remote): raise RuntimeError("Could not connect to the remote host") # ----------------------------------------------------------------- # Create the status object status = LogSimulationStatus(simulation.remote_log_file_path, remote=remote, debug_output=config.debug_output) # Show the simulation progress with log.no_debugging(): success = status.show_progress(simulation.handle) # Check whether not crashed if not success: raise RuntimeError("The simulation crashed") # -----------------------------------------------------------------
definition.add_flag("conda", "also remove conda installation") definition.add_flag("qt", "also remove Qt installation") definition.add_flag("one_attempt", "only perform one attempt at connecting to a remote") # Get the config config = parse_arguments("deinstall_all", definition) # ----------------------------------------------------------------- # Loop over the remote hosts for host_id in find_host_ids(): # Setup remote = Remote() if not remote.setup(host_id, one_attempt=config.one_attempt): log.warning("Remote host '" + host_id + "' is offline") continue # Create uninstaller uninstaller = Uninstaller() # Set options uninstaller.config.skirt_and_or_pts = config.skirt_and_or_pts uninstaller.config.conda = config.conda uninstaller.config.qt = config.qt # Inform the user log.info("Running the uninstaller for remote host '" + host_id + "' ...") # Uninstall
class SourcesTest(SourcesTestBase): """ This class ... """ def __init__(self, *args, **kwargs): """ The constructor ... :param kwargs: """ # Call the constructor of the base class super(SourcesTest, self).__init__(*args, **kwargs) # The galaxy properties self.properties = None # The rotation masks self.rotation_masks = dict() # The catalog fetcher self.fetcher = CatalogFetcher() # Catalogs self.extended_source_catalog = None # The real FWHMs self.real_fwhms = dict() # ----------------------------------------------------------------- def _run(self, **kwargs): """ This function ... :param kwargs: :return: """ # 2. Load galaxy properties self.load_properties() # 3. Load the coordinate system self.load_coordinate_systems() # 4. Initialize the frames self.initialize_frames() # 5. Set FWHMs self.set_fwhms() # 6. Make rotation mask self.make_rotation_masks() # 7. Get catalogs self.get_catalogs() # 8. Generate the sources self.make_sources() # 9. Make noise self.make_noise(self.rotation_masks) # 10. Make sky self.make_sky() # 11. Create the dataset self.create_dataset(self.rotation_masks) # 12. Create the directories self.create_directories() # 13. Find sources self.find() # 14. Extract sources self.extract() # 15. Write self.write() # 16. Plot self.plot() # ----------------------------------------------------------------- def setup(self, **kwargs): """ This function ... :param kwargs: :return: """ # Call the setup function of the base class super(SourcesTest, self).setup(**kwargs) # Set remote if self.config.remote is not None: self.remote = Remote() self.remote.setup(self.config.remote) # ----------------------------------------------------------------- def load_properties(self): """ This function ... :return: """ # Inform the user log.info("Loading the galaxy properties ...") # Determine the path path = fs.join(m81_data_path, "properties.dat") # Load self.properties = GalaxyProperties.from_file(path) # ----------------------------------------------------------------- def load_coordinate_systems(self): """ This function ... :return: """ # Inform the user log.info("Loading the coordinate systems ...") nfilters_stars = 0 nfilters_extra = 0 # Loop over the header files for path, name in fs.files_in_path(headers_path, extension="txt", returns=["path", "name"]): # Get the filter and wavelength fltr = parse_filter(name) wavelength = fltr.effective if fltr.effective is not None else fltr.center # SKip Planck if fltr.observatory == "Planck": continue # Wavelength greater than 25 micron if wavelength > wavelengths.ranges.ir.mir.max: if nfilters_extra == self.config.nfilters_extra: continue else: nfilters_extra += 1 # Wavelength smaller than 25 micron else: if nfilters_stars == self.config.nfilters_stars: continue else: nfilters_stars += 1 # Debugging log.debug("Loading the coordinate system for the '" + str(fltr) + "' filter ...") # Get WCS wcs = CoordinateSystem.from_header_file(path) # Add the coordinate system self.coordinate_systems.append(wcs, fltr=fltr) # Break the loop if we have enough if nfilters_stars == self.config.nfilters_stars and nfilters_extra == self.config.nfilters_extra: break # ----------------------------------------------------------------- def make_rotation_masks(self): """ This function ... :return: """ # Inform the user log.info("Making rotation masks ...") # Loop over the filters for fltr in self.frames: # Debugging log.info("Making rotation mask for the '" + str(fltr) + "' image ...") # Get the frame frame = self.frames[fltr] # Rotate if self.config.rotate: # Choose a random rotation angle angle = Angle(np.random.uniform(-90, 90), "deg") # Debugging log.debug("The random rotation angle is '" + stringify.stringify(angle)[1] + "'") # Create mask mask = frame.rotation_mask(angle) # Don't rotate else: mask = Mask.empty_like(frame) # Set the mask self.rotation_masks[fltr] = mask # ----------------------------------------------------------------- def get_catalogs(self): """ This function ... :return: """ # Inform the user log.info("Getting catalogs of point and extended sources ...") # Adding catalogued sources if self.config.add_catalogued_sources: self.fetch_catalogs() else: self.initialize_catalogs() # Add random sources self.create_random_sources() # ----------------------------------------------------------------- def fetch_catalogs(self): """ This function ... :return: """ # Inform the user log.info("Fetching catalogs ...") # Get extended source catalog self.extended_source_catalog = self.fetcher.get_extended_source_catalog(self.coordinate_systems.bounding_box) # Fetch self.point_source_catalog = self.fetcher.get_point_source_catalog(self.coordinate_systems.bounding_box, self.coordinate_systems.min_pixelscale, self.config.point_source_catalogs) # ----------------------------------------------------------------- def initialize_catalogs(self): """ This function ... :param self: :return: """ # Inform the user log.info("Initializing catalogs ...") # Extended sources self.extended_source_catalog = ExtendedSourceCatalog() # Add principal galaxy self.add_principal_galaxy() # Point sources self.point_source_catalog = PointSourceCatalog() # ----------------------------------------------------------------- def add_principal_galaxy(self): """ This function ... :return: """ # Inform the user log.info("Adding principal galaxy ...") # Get info name, position, redshift, galaxy_type, names, distance, inclination, d25, major, minor, pa = get_galaxy_info("M81", self.properties.center) ra = position.ra dec = position.dec principal = True companions = [] parent = None # Add to the catalog self.extended_source_catalog.add_entry("M81", ra, dec, redshift, galaxy_type, names, distance, inclination, d25, major, minor, pa, principal, companions, parent) # ----------------------------------------------------------------- def make_sources(self): """ This function ... :return: """ # Inform the user log.info("Making sources ...") # Extended self.make_extended_sources() # Point self.make_point_sources() # ----------------------------------------------------------------- def make_extended_sources(self): """ This function ... :return: """ # Inform the user log.info("Making extended sources ...") # Loop over the filters for fltr in self.frames: # Get the frame frame = self.frames[fltr] # Loop over the sources for index in range(len(self.extended_source_catalog)): principal = self.extended_source_catalog["Principal"][index] # Determine flux if principal: central_flux = 1e2 else: central_flux = np.random.uniform(10,50) # Get pixel position coordinate = self.extended_source_catalog.get_position(index).to_pixel(frame.wcs) initial_sersic_amplitude = central_flux initial_sersic_x_0 = coordinate.x initial_sersic_y_0 = coordinate.y if principal: s4g_name, pa, ellipticity, n, re, mag = catalogs.get_galaxy_s4g_one_component_info("M81") angle = Angle(pa, "deg") angle_deg = pa effective_radius = re.to("arcsec").value / frame.average_pixelscale.to("arcsec").value initial_sersic_n = n initial_sersic_r_eff = effective_radius initial_sersic_ellip = ellipticity initial_sersic_theta = np.deg2rad(angle_deg) # 1 / axial_ratio = 1 - ellipticity axial_ratio = 1. / (1. - ellipticity) else: # Get position angle and axes lengths pa = self.extended_source_catalog["Posangle"][index] major = self.extended_source_catalog["Major"][index] minor = self.extended_source_catalog["Minor"][index] axial_ratio = major / minor angle = Angle(pa, "deg") angle_deg = pa effective_radius = major # Produce guess values initial_sersic_r_eff = effective_radius initial_sersic_n = self.config.galaxy_sersic_index initial_sersic_ellip = (axial_ratio - 1.0) / axial_ratio initial_sersic_theta = np.deg2rad(angle_deg) # Produce sersic model from guess parameters, for time trials sersic_x, sersic_y = np.meshgrid(np.arange(frame.xsize), np.arange(frame.ysize)) sersic_model = Sersic2D(amplitude=initial_sersic_amplitude, r_eff=initial_sersic_r_eff, n=initial_sersic_n, x_0=initial_sersic_x_0, y_0=initial_sersic_y_0, ellip=initial_sersic_ellip, theta=initial_sersic_theta) sersic_map = sersic_model(sersic_x, sersic_y) # Limit galaxy? if self.config.limit_galaxy: limit_radius = self.config.galaxy_relative_asymptotic_radius * effective_radius # Create galaxy region galaxy_center = PixelCoordinate(initial_sersic_x_0, initial_sersic_y_0) galaxy_radius = PixelStretch(limit_radius, limit_radius / axial_ratio) galaxy_region = PixelEllipseRegion(galaxy_center, galaxy_radius, angle) galaxy_mask = galaxy_region.to_mask(frame.xsize, frame.ysize) # Set galaxy map zero outside certain radius sersic_map[galaxy_mask.inverse()] = 0.0 # Add frame += sersic_map # mask frame[self.rotation_masks[fltr]] = 0.0 # ----------------------------------------------------------------- def make_point_sources(self): """ This function ... :return: """ # Inform the user log.info("Making point sources ...") # Call the appropriate function if self.config.vary_fwhm: self.make_point_sources_variable_fwhm(self.rotation_masks) else: self.make_point_sources_fixed_fwhm(self.rotation_masks) # ----------------------------------------------------------------- def make_sky(self): """ This function ... :return: """ # Inform the user log.info("Adding sky ...") # Loop over the frames for fltr in self.frames: # Get the frame frame = self.frames[fltr] # Determine random sky level sky_level = np.random.uniform(0.0, 10.) # Create sky gradient y, x = np.mgrid[:frame.ysize, :frame.xsize] sky_gradient = x * y # Normalize so that the maximal sky is 20% sky_gradient = sky_gradient / np.max(sky_gradient) * 20 # Add the sky frame += sky_level + sky_gradient # Mask frame[self.rotation_masks[fltr]] = 0.0 # ----------------------------------------------------------------- def make_galaxy(self): """ This function ... :return: """ # Inform the user log.info("Adding galaxy ...") # ----------------------------------------------------------------- def find(self): """ This function ... :return: """ # Inform the user log.info("Finding the sources ...") # Settings settings = dict() #settings["input"] = settings["output"] = self.find_path settings["nprocesses"] = self.config.nprocesses # Input input_dict = dict() input_dict["dataset"] = self.dataset input_dict["extended_source_catalog"] = self.extended_source_catalog input_dict["point_source_catalog"] = self.point_source_catalog input_dict["output_paths"] = self.find_paths # Construct the command command = Command("find_sources", "find sources", settings, input_dict) # Run the command self.finder = self.run_command(command, remote=self.remote) # ----------------------------------------------------------------- def write(self): """ This function ... :return: """ # Inform the user log.info("Writing ...") # ----------------------------------------------------------------- def plot(self): """ This function ... :return: """ # Inform the user log.info("Plotting ...")
# Parse the arguments into a configuration config = parse_arguments( "clear_tasks", definition, description="Clear PTS tasks for a certain remote host") # ----------------------------------------------------------------- # Loop over the remote hosts for host_id in config.remotes: # Check whether the remote is available if config.full: remote = Remote() if not remote.setup(host_id): log.warning("The remote host '" + host_id + "' is not available: skipping ...") continue else: remote = None # Determine the path to the run directory for the specified remote host host_run_path = fs.join(introspection.skirt_run_dir, host_id) # Check if there are simulations if not fs.is_directory(host_run_path): log.debug("No run directory for host '" + host_id + "'") continue if fs.is_empty(host_run_path): log.debug("No simulations for host '" + host_id + "'")
from __future__ import absolute_import, division, print_function # Import the relevant PTS classes and modules from pts.core.remote.host import find_host_ids from pts.core.remote.remote import Remote from pts.core.tools import formatting as fmt from pts.core.basics.log import setup_log from pts.core.basics.configuration import parse_logging_arguments # ----------------------------------------------------------------- config = parse_logging_arguments("hosts") # ----------------------------------------------------------------- # Set log level in a special way if config.debug: setup_log("DEBUG") else: setup_log("ERROR") # ----------------------------------------------------------------- # Loop over the hosts for host_id in find_host_ids(): remote = Remote() connected = remote.setup(host_id, login_timeout=15) # only try for 15 seconds if connected: print(fmt.green + host_id + ": up" + fmt.reset) else: print(fmt.red + host_id + ": down" + fmt.reset) # -----------------------------------------------------------------
# Import the relevant PTS classes and modules from pts.core.basics.configuration import ConfigurationDefinition, parse_arguments from pts.core.remote.remote import Remote # ----------------------------------------------------------------- definition = ConfigurationDefinition() definition.add_required("remote", "string", "remote host ID") definition.add_required("line", "string", "line to be executed on the remote host") config = parse_arguments("sessions", definition) # ----------------------------------------------------------------- # Initialize the remote remote = Remote() if not remote.setup(config.remote): raise RuntimeError("The remote host '" + config.remote + "' is not available at the moment") # ----------------------------------------------------------------- print("") #print("-----------------------------------------------------------------") print("OUTPUT") print("-----------------------------------------------------------------") print("") # Execute the line, show the output output = remote.execute(config.line) for line in output: print(line) print("") print("-----------------------------------------------------------------")
# Import the relevant PTS classes and modules from pts.core.basics.log import log from pts.core.basics.configuration import ConfigurationDefinition, parse_arguments from pts.core.remote.remote import Remote # ----------------------------------------------------------------- definition = ConfigurationDefinition() definition.add_required("remote", "string", "remote host ID") setter = parse_arguments("sessions", definition) # ----------------------------------------------------------------- remote = Remote() if not remote.setup(config.remote): raise RuntimeError("The remote host '" + config.remote + "' is not available at the moment") screen_names = remote.screen_names() tmux_names = remote.tmux_names() print("SCREEN") print(screen_names) print("") print("TMUX") print(tmux_names)
class SourcesTest(SourcesTestBase): """ This class ... """ def __init__(self, *args, **kwargs): """ The constructor ... :param kwargs: """ # Call the constructor of the base class super(SourcesTest, self).__init__(*args, **kwargs) # The galaxy properties self.properties = None # The rotation masks self.rotation_masks = dict() # The catalog fetcher self.fetcher = CatalogFetcher() # Catalogs self.extended_source_catalog = None # The real FWHMs self.real_fwhms = dict() # ----------------------------------------------------------------- def run(self, **kwargs): """ This function ... :param kwargs: :return: """ # 1. Call the setup function self.setup(**kwargs) # 2. Load galaxy properties self.load_properties() # 3. Load the coordinate system self.load_coordinate_systems() # 4. Initialize the frames self.initialize_frames() # 5. Set FWHMs self.set_fwhms() # 6. Make rotation mask self.make_rotation_masks() # 7. Get catalogs self.get_catalogs() # 8. Generate the sources self.make_sources() # 9. Make noise self.make_noise(self.rotation_masks) # 10. Make sky self.make_sky() # 11. Create the dataset self.create_dataset(self.rotation_masks) # 12. Create the directories self.create_directories() # 13. Find sources self.find() # 14. Extract sources self.extract() # 15. Write self.write() # 16. Plot self.plot() # ----------------------------------------------------------------- def setup(self, **kwargs): """ This function ... :param kwargs: :return: """ # Call the setup function of the base class super(SourcesTest, self).setup(**kwargs) # Set remote if self.config.remote is not None: self.remote = Remote() self.remote.setup(self.config.remote) # ----------------------------------------------------------------- def load_properties(self): """ This function ... :return: """ # Inform the user log.info("Loading the galaxy properties ...") # Determine the path path = fs.join(m81_data_path, "properties.dat") # Load self.properties = GalaxyProperties.from_file(path) # ----------------------------------------------------------------- def load_coordinate_systems(self): """ This function ... :return: """ # Inform the user log.info("Loading the coordinate systems ...") nfilters_stars = 0 nfilters_extra = 0 # Loop over the header files for path, name in fs.files_in_path(headers_path, extension="txt", returns=["path", "name"]): # Get the filter and wavelength fltr = parse_filter(name) wavelength = fltr.effective if fltr.effective is not None else fltr.center # SKip Planck if fltr.observatory == "Planck": continue # Wavelength greater than 25 micron if wavelength > wavelengths.ranges.ir.mir.max: if nfilters_extra == self.config.nfilters_extra: continue else: nfilters_extra += 1 # Wavelength smaller than 25 micron else: if nfilters_stars == self.config.nfilters_stars: continue else: nfilters_stars += 1 # Debugging log.debug("Loading the coordinate system for the '" + str(fltr) + "' filter ...") # Get WCS wcs = CoordinateSystem.from_header_file(path) # Add the coordinate system self.coordinate_systems.append(wcs, fltr=fltr) # Break the loop if we have enough if nfilters_stars == self.config.nfilters_stars and nfilters_extra == self.config.nfilters_extra: break # ----------------------------------------------------------------- def make_rotation_masks(self): """ This function ... :return: """ # Inform the user log.info("Making rotation masks ...") # Loop over the filters for fltr in self.frames: # Debugging log.info("Making rotation mask for the '" + str(fltr) + "' image ...") # Get the frame frame = self.frames[fltr] # Rotate if self.config.rotate: # Choose a random rotation angle angle = Angle(np.random.uniform(-90, 90), "deg") # Debugging log.debug("The random rotation angle is '" + stringify.stringify(angle)[1] + "'") # Create mask mask = frame.rotation_mask(angle) # Don't rotate else: mask = Mask.empty_like(frame) # Set the mask self.rotation_masks[fltr] = mask # ----------------------------------------------------------------- def get_catalogs(self): """ This function ... :return: """ # Inform the user log.info("Getting catalogs of point and extended sources ...") # Adding catalogued sources if self.config.add_catalogued_sources: self.fetch_catalogs() else: self.initialize_catalogs() # Add random sources self.create_random_sources() # ----------------------------------------------------------------- def fetch_catalogs(self): """ This function ... :return: """ # Inform the user log.info("Fetching catalogs ...") # Get extended source catalog self.extended_source_catalog = self.fetcher.get_extended_source_catalog( self.coordinate_systems.bounding_box) # Fetch self.point_source_catalog = self.fetcher.get_point_source_catalog( self.coordinate_systems.bounding_box, self.coordinate_systems.min_pixelscale, self.config.point_source_catalogs) # ----------------------------------------------------------------- def initialize_catalogs(self): """ This function ... :param self: :return: """ # Inform the user log.info("Initializing catalogs ...") # Extended sources self.extended_source_catalog = ExtendedSourceCatalog() # Add principal galaxy self.add_principal_galaxy() # Point sources self.point_source_catalog = PointSourceCatalog() # ----------------------------------------------------------------- def add_principal_galaxy(self): """ This function ... :return: """ # Inform the user log.info("Adding principal galaxy ...") # Get info name, position, redshift, galaxy_type, names, distance, inclination, d25, major, minor, pa = get_galaxy_info( "M81", self.properties.center) ra = position.ra dec = position.dec principal = True companions = [] parent = None # Add to the catalog self.extended_source_catalog.add_entry("M81", ra, dec, redshift, galaxy_type, names, distance, inclination, d25, major, minor, pa, principal, companions, parent) # ----------------------------------------------------------------- def make_sources(self): """ This function ... :return: """ # Inform the user log.info("Making sources ...") # Extended self.make_extended_sources() # Point self.make_point_sources() # ----------------------------------------------------------------- def make_extended_sources(self): """ This function ... :return: """ # Inform the user log.info("Making extended sources ...") # Loop over the filters for fltr in self.frames: # Get the frame frame = self.frames[fltr] # Loop over the sources for index in range(len(self.extended_source_catalog)): principal = self.extended_source_catalog["Principal"][index] # Determine flux if principal: central_flux = 1e2 else: central_flux = np.random.uniform(10, 50) # Get pixel position coordinate = self.extended_source_catalog.get_position( index).to_pixel(frame.wcs) initial_sersic_amplitude = central_flux initial_sersic_x_0 = coordinate.x initial_sersic_y_0 = coordinate.y if principal: s4g_name, pa, ellipticity, n, re, mag = catalogs.get_galaxy_s4g_one_component_info( "M81") angle = Angle(pa, "deg") angle_deg = pa effective_radius = re.to( "arcsec").value / frame.average_pixelscale.to( "arcsec").value initial_sersic_n = n initial_sersic_r_eff = effective_radius initial_sersic_ellip = ellipticity initial_sersic_theta = np.deg2rad(angle_deg) # 1 / axial_ratio = 1 - ellipticity axial_ratio = 1. / (1. - ellipticity) else: # Get position angle and axes lengths pa = self.extended_source_catalog["Posangle"][index] major = self.extended_source_catalog["Major"][index] minor = self.extended_source_catalog["Minor"][index] axial_ratio = major / minor angle = Angle(pa, "deg") angle_deg = pa effective_radius = major # Produce guess values initial_sersic_r_eff = effective_radius initial_sersic_n = self.config.galaxy_sersic_index initial_sersic_ellip = (axial_ratio - 1.0) / axial_ratio initial_sersic_theta = np.deg2rad(angle_deg) # Produce sersic model from guess parameters, for time trials sersic_x, sersic_y = np.meshgrid(np.arange(frame.xsize), np.arange(frame.ysize)) sersic_model = Sersic2D(amplitude=initial_sersic_amplitude, r_eff=initial_sersic_r_eff, n=initial_sersic_n, x_0=initial_sersic_x_0, y_0=initial_sersic_y_0, ellip=initial_sersic_ellip, theta=initial_sersic_theta) sersic_map = sersic_model(sersic_x, sersic_y) # Limit galaxy? if self.config.limit_galaxy: limit_radius = self.config.galaxy_relative_asymptotic_radius * effective_radius # Create galaxy region galaxy_center = PixelCoordinate(initial_sersic_x_0, initial_sersic_y_0) galaxy_radius = PixelStretch(limit_radius, limit_radius / axial_ratio) galaxy_region = PixelEllipseRegion(galaxy_center, galaxy_radius, angle) galaxy_mask = galaxy_region.to_mask( frame.xsize, frame.ysize) # Set galaxy map zero outside certain radius sersic_map[galaxy_mask.inverse()] = 0.0 # Add frame += sersic_map # mask frame[self.rotation_masks[fltr]] = 0.0 # ----------------------------------------------------------------- def make_point_sources(self): """ This function ... :return: """ # Inform the user log.info("Making point sources ...") # Call the appropriate function if self.config.vary_fwhm: self.make_point_sources_variable_fwhm(self.rotation_masks) else: self.make_point_sources_fixed_fwhm(self.rotation_masks) # ----------------------------------------------------------------- def make_sky(self): """ This function ... :return: """ # Inform the user log.info("Adding sky ...") # Loop over the frames for fltr in self.frames: # Get the frame frame = self.frames[fltr] # Determine random sky level sky_level = np.random.uniform(0.0, 10.) # Create sky gradient y, x = np.mgrid[:frame.ysize, :frame.xsize] sky_gradient = x * y # Normalize so that the maximal sky is 20% sky_gradient = sky_gradient / np.max(sky_gradient) * 20 # Add the sky frame += sky_level + sky_gradient # Mask frame[self.rotation_masks[fltr]] = 0.0 # ----------------------------------------------------------------- def make_galaxy(self): """ This function ... :return: """ # Inform the user log.info("Adding galaxy ...") # ----------------------------------------------------------------- def find(self): """ This function ... :return: """ # Inform the user log.info("Finding the sources ...") # Settings settings = dict() #settings["input"] = settings["output"] = self.find_path settings["nprocesses"] = self.config.nprocesses # Input input_dict = dict() input_dict["dataset"] = self.dataset input_dict["extended_source_catalog"] = self.extended_source_catalog input_dict["point_source_catalog"] = self.point_source_catalog input_dict["output_paths"] = self.find_paths # Construct the command command = Command("find_sources", "find sources", settings, input_dict) # Run the command self.finder = self.run_command(command, remote=self.remote) # ----------------------------------------------------------------- def write(self): """ This function ... :return: """ # Inform the user log.info("Writing ...") # ----------------------------------------------------------------- def plot(self): """ This function ... :return: """ # Inform the user log.info("Plotting ...")