def run_remotely(command_name, config, keep, host_id, log): """ This function ... :param command_name: :param config: :param keep: :param host_id: :param log: :return: """ # Additional imports from pts.core.remote.remote import Remote # Start message log.start("Starting " + command_name + " on remote host " + host_id + " ...") # Debugging log.debug("Initializing the remote ...") # Initialize the remote execution environment remote = Remote(host_id=host_id) # Run PTS remotely task = remote.run_pts(command_name, config, keep_remote_output=keep) # Succesfully submitted log.success("Succesfully submitted the PTS job to the remote host")
def setup(self, **kwargs): """ This function ... :param kwargs: :return: """ # Call the setup function of the base class super(SourcesTest, self).setup(**kwargs) # Set remote if self.config.remote is not None: self.remote = Remote() self.remote.setup(self.config.remote)
def setup(self, **kwargs): """ This function ... :param kwargs: :return: """ # Call the setup function of the base class super(RemotesConfigurable, self).setup(**kwargs) # If remotes are passed if "remotes" in kwargs: # Get the remotes self.remotes = kwargs.pop("remotes") # Check if they are setup for remote in self.remotes: if not remote.connected: raise RuntimeError("Remotes must be connected") # Remotes are not passed else: # Gather host IDs if self.config.not_remotes is not None: host_ids = [ host_id for host_id in self.config.host_ids if host_id not in self.config.not_remotes ] else: host_ids = self.config.host_ids # Loop over the different hosts for host_id in host_ids: # Setup the remote (login) remote = Remote(log_conda=kwargs.pop("log_conda", False)) if not remote.setup(host_id, one_attempt=self.config.one_attempt): log.warning("Remote host '" + host_id + "' is down: skipping") continue # Add the remote to the list self.remotes.append(remote)
def setup(self, **kwargs): """ This function ... :param kwargs: :return: """ # Call the setup function of the base class super(RemotesConfigurable, self).setup(**kwargs) # If remotes are passed if "remotes" in kwargs: # Get the remotes self.remotes = kwargs.pop("remotes") # Check if they are setup for remote in self.remotes: if not remote.connected: raise RuntimeError("Remotes must be connected") # Remotes are not passed else: # Gather host IDs #if self.config.not_remotes is not None: host_ids = [host_id for host_id in self.config.host_ids if host_id not in self.config.not_remotes] #else: host_ids = self.config.host_ids # Loop over the different hosts #for host_id in host_ids: for host in self.config.hosts: # Create the remote object remote = Remote(log_conda=kwargs.pop("log_conda", False)) # Login to the remote #if not remote.setup(host_id, one_attempt=self.config.one_attempt, cluster_name=self.config.clustername): if not remote.setup(host, one_attempt=self.config.one_attempt): log.warning("Remote host '" + host.id + "' is down: skipping") continue # Add the remote to the list self.remotes.append(remote)
def setup(self, **kwargs): """ This function ... :param kwargs: :return: """ # Call the setup function of the base class super(Uninstaller, self).setup(**kwargs) # Check if remote instance is passed if "remote" in kwargs: self.remote = kwargs.pop("remote") elif self.config.remote is not None: remote = Remote() if not remote.setup(self.config.remote): raise RuntimeError("The remote host is not available") self.remote = remote else: log.warning('No remote host is specified, will be uninstalling locally')
def setup(self, **kwargs): """ This function ... :param kwargs: :return: """ # Call the setup function of the base class super(Uninstaller, self).setup(**kwargs) # Check if remote instance is passed if "remote" in kwargs: self.remote = kwargs.pop("remote") elif self.config.remote is not None: remote = Remote() if not remote.setup(self.config.remote): raise RuntimeError("The remote host is not available") self.remote = remote else: log.warning( 'No remote host is specified, will be uninstalling locally')
from pts.core.remote.remote import Remote from pts.core.remote.host import find_host_ids # ----------------------------------------------------------------- definition = ConfigurationDefinition() definition.add_optional("remotes", "string_list", "remote host ID", choices=find_host_ids(), default=find_host_ids()) config = parse_arguments("sessions", definition) # ----------------------------------------------------------------- # Loop over the remote host for host_id in config.remotes: # Create and setup the remote remote = Remote() if not remote.setup(host_id): raise RuntimeError("The remote host '" + host_id + "' is not available at the moment") # Get screen names and tmux session names screen_names = remote.screen_names() tmux_names = remote.tmux_names() # Inform the user log.info("Closing sessions on remote host '" + host_id + "' ...") # Loop over the screens for name in screen_names: # Inform the user log.info("Closing screen session '" + name + "' ...")
# Import the relevant PTS classes and modules from pts.core.basics.configuration import ConfigurationDefinition, parse_arguments from pts.core.remote.remote import Remote # ----------------------------------------------------------------- definition = ConfigurationDefinition() definition.add_required("remote", "string", "remote host ID") definition.add_required("line", "string", "line to be executed on the remote host") config = parse_arguments("sessions", definition) # ----------------------------------------------------------------- # Initialize the remote remote = Remote() if not remote.setup(config.remote): raise RuntimeError("The remote host '" + config.remote + "' is not available at the moment") # ----------------------------------------------------------------- print("") #print("-----------------------------------------------------------------") print("OUTPUT") print("-----------------------------------------------------------------") print("") # Execute the line, show the output output = remote.execute(config.line) for line in output: print(line) print("")
definition.add_flag("recursive", "search recursively", False) definition.add_flag("full", "show the full paths", False) definition.add_flag("remove", "remove the found files", False) # ----------------------------------------------------------------- # Parse the arguments into a configuration config = parse_arguments("find_files", definition, description="Find files containing a certain string in the current working directory") # ----------------------------------------------------------------- # REMOTELY if config.remote is not None: # Create remote remote = Remote(host_id=config.remote) # Determine path if config.remote_path is not None: find_path = remote.absolute_or_in_home(config.remote_path) else: find_path = remote.home_directory #print(find_path) #print(remote.items_in_path(find_path, recursive=True)) # Loop over the files paths = remote.files_in_path(find_path, contains=config.contains, not_contains=config.not_contains, extension=config.extension, recursive=config.recursive, exact_name=config.exact_name, exact_not_name=config.exact_not_name) nfiles = len(paths) if nfiles == 0: log.warning("No files found") else:
# Create configuration definition definition = ConfigurationDefinition() # Add required definition.add_required("remote", "string", "the remote host ID") # Add flags definition.add_flag("versions", "compare versions", "v") # Get configuration config = parse_arguments("compare_python_packages", definition) # ----------------------------------------------------------------- # Create the remote execution environment remote = Remote() # Log in remote.setup(config.remote) # Get remote python session remote_python = remote.start_python_session(assume_pts=False) # Get all python packages installed on the remote host remote_packages = remote_python.installed_packages # Get local python package version local_packages = introspection.installed_python_packages() # Loop over all python packages necessary for PTS for dependency in introspection.get_all_dependencies():
# Load the fitting run fitting_run = runs.load(config.name) # Get the generation generation = fitting_run.get_generation(config.generation) # ----------------------------------------------------------------- # Get the cache path cache_path = generation.single_cache_path # ----------------------------------------------------------------- # Create remote remote = Remote(host_id=config.host_id) # ----------------------------------------------------------------- # Determine RT Modeling backup directory path remote_backup_path = fs.join(remote.home_directory, "RT Modeling Backup") remote_backup_galaxy_path = fs.join(remote_backup_path, environment.galaxy_name) # ----------------------------------------------------------------- # Create path if not remote.is_directory(remote_backup_galaxy_path): remote.create_directory(remote_backup_galaxy_path, recursive=True) # ----------------------------------------------------------------- # Synchronize
# Create the configuration definition definition = ConfigurationDefinition() definition.add_required("remote", "string", "remote host ID", choices=find_host_ids()) definition.add_flag("private", "install SKIRT from private (development) repository") # Create the configuration config = parse_arguments("installation_commands", definition) # ----------------------------------------------------------------- installation_commands = defaultdict(list) # ----------------------------------------------------------------- # Create the remote remote = Remote() if not remote.setup(config.remote): log.error("The remote host is not available") exit() # ----------------------------------------------------------------- # Check the modules modules = Modules(remote) # ----------------------------------------------------------------- # Qt is present if modules.paths["qt"] is not None: # Get the version
"name of the remote host", choices=find_host_ids()) definition.add_required("id", "positive_integer", "simulation ID") # ----------------------------------------------------------------- # Parse the arguments into a configuration config = parse_arguments( "show_simulation_log", definition, description="Show the log output of a remote SKIRT simulation") # ----------------------------------------------------------------- # Create and setup the remote remote = Remote() remote.setup(config.remote) # Open the simulation simulation = get_simulation_for_host(config.remote, config.id) # The name of the ski file (the simulation prefix) ski_name = simulation.prefix() # The path to the simulation log file remote_log_file_path = simulation.remote_log_file_path # Check whether the log file exists if not remote.is_file(remote_log_file_path): raise RuntimeError("The log file does not exist remotely")
from __future__ import absolute_import, division, print_function # Import the relevant PTS classes and modules from pts.core.basics.log import log from pts.core.basics.configuration import ConfigurationDefinition, parse_arguments from pts.core.remote.remote import Remote # ----------------------------------------------------------------- definition = ConfigurationDefinition() definition.add_required("remote", "string", "remote host ID") setter = parse_arguments("sessions", definition) # ----------------------------------------------------------------- remote = Remote() if not remote.setup(config.remote): raise RuntimeError("The remote host '" + config.remote + "' is not available at the moment") screen_names = remote.screen_names() tmux_names = remote.tmux_names() print("SCREEN") print(screen_names) print("") print("TMUX") print(tmux_names) # -----------------------------------------------------------------
# Import the relevant PTS classes and modules from pts.core.basics.configuration import ConfigurationDefinition, parse_arguments from pts.core.remote.host import find_host_ids from pts.core.remote.remote import Remote from pts.core.basics.log import log # ----------------------------------------------------------------- # Create definition definition = ConfigurationDefinition() definition.add_positional_optional("remotes", "string_list", "remote hosts on which to clear", choices=find_host_ids(), default=find_host_ids()) # Create setter config = parse_arguments("clear_sessions_and_temp", definition) # ----------------------------------------------------------------- # Loop over the remote hosts for host_id in config.remotes: # Setup the remote remote = Remote() if not remote.setup(host_id): log.warning("Could not connect to remote host '" + host_id + "'") continue # Clear temporary directory and clear sessions remote.clear_temp_and_sessions() # -----------------------------------------------------------------
class SourcesTest(SourcesTestBase): """ This class ... """ def __init__(self, *args, **kwargs): """ The constructor ... :param kwargs: """ # Call the constructor of the base class super(SourcesTest, self).__init__(*args, **kwargs) # The galaxy properties self.properties = None # The rotation masks self.rotation_masks = dict() # The catalog fetcher self.fetcher = CatalogFetcher() # Catalogs self.extended_source_catalog = None # The real FWHMs self.real_fwhms = dict() # ----------------------------------------------------------------- def _run(self, **kwargs): """ This function ... :param kwargs: :return: """ # 2. Load galaxy properties self.load_properties() # 3. Load the coordinate system self.load_coordinate_systems() # 4. Initialize the frames self.initialize_frames() # 5. Set FWHMs self.set_fwhms() # 6. Make rotation mask self.make_rotation_masks() # 7. Get catalogs self.get_catalogs() # 8. Generate the sources self.make_sources() # 9. Make noise self.make_noise(self.rotation_masks) # 10. Make sky self.make_sky() # 11. Create the dataset self.create_dataset(self.rotation_masks) # 12. Create the directories self.create_directories() # 13. Find sources self.find() # 14. Extract sources self.extract() # 15. Write self.write() # 16. Plot self.plot() # ----------------------------------------------------------------- def setup(self, **kwargs): """ This function ... :param kwargs: :return: """ # Call the setup function of the base class super(SourcesTest, self).setup(**kwargs) # Set remote if self.config.remote is not None: self.remote = Remote() self.remote.setup(self.config.remote) # ----------------------------------------------------------------- def load_properties(self): """ This function ... :return: """ # Inform the user log.info("Loading the galaxy properties ...") # Determine the path path = fs.join(m81_data_path, "properties.dat") # Load self.properties = GalaxyProperties.from_file(path) # ----------------------------------------------------------------- def load_coordinate_systems(self): """ This function ... :return: """ # Inform the user log.info("Loading the coordinate systems ...") nfilters_stars = 0 nfilters_extra = 0 # Loop over the header files for path, name in fs.files_in_path(headers_path, extension="txt", returns=["path", "name"]): # Get the filter and wavelength fltr = parse_filter(name) wavelength = fltr.effective if fltr.effective is not None else fltr.center # SKip Planck if fltr.observatory == "Planck": continue # Wavelength greater than 25 micron if wavelength > wavelengths.ranges.ir.mir.max: if nfilters_extra == self.config.nfilters_extra: continue else: nfilters_extra += 1 # Wavelength smaller than 25 micron else: if nfilters_stars == self.config.nfilters_stars: continue else: nfilters_stars += 1 # Debugging log.debug("Loading the coordinate system for the '" + str(fltr) + "' filter ...") # Get WCS wcs = CoordinateSystem.from_header_file(path) # Add the coordinate system self.coordinate_systems.append(wcs, fltr=fltr) # Break the loop if we have enough if nfilters_stars == self.config.nfilters_stars and nfilters_extra == self.config.nfilters_extra: break # ----------------------------------------------------------------- def make_rotation_masks(self): """ This function ... :return: """ # Inform the user log.info("Making rotation masks ...") # Loop over the filters for fltr in self.frames: # Debugging log.info("Making rotation mask for the '" + str(fltr) + "' image ...") # Get the frame frame = self.frames[fltr] # Rotate if self.config.rotate: # Choose a random rotation angle angle = Angle(np.random.uniform(-90, 90), "deg") # Debugging log.debug("The random rotation angle is '" + stringify.stringify(angle)[1] + "'") # Create mask mask = frame.rotation_mask(angle) # Don't rotate else: mask = Mask.empty_like(frame) # Set the mask self.rotation_masks[fltr] = mask # ----------------------------------------------------------------- def get_catalogs(self): """ This function ... :return: """ # Inform the user log.info("Getting catalogs of point and extended sources ...") # Adding catalogued sources if self.config.add_catalogued_sources: self.fetch_catalogs() else: self.initialize_catalogs() # Add random sources self.create_random_sources() # ----------------------------------------------------------------- def fetch_catalogs(self): """ This function ... :return: """ # Inform the user log.info("Fetching catalogs ...") # Get extended source catalog self.extended_source_catalog = self.fetcher.get_extended_source_catalog(self.coordinate_systems.bounding_box) # Fetch self.point_source_catalog = self.fetcher.get_point_source_catalog(self.coordinate_systems.bounding_box, self.coordinate_systems.min_pixelscale, self.config.point_source_catalogs) # ----------------------------------------------------------------- def initialize_catalogs(self): """ This function ... :param self: :return: """ # Inform the user log.info("Initializing catalogs ...") # Extended sources self.extended_source_catalog = ExtendedSourceCatalog() # Add principal galaxy self.add_principal_galaxy() # Point sources self.point_source_catalog = PointSourceCatalog() # ----------------------------------------------------------------- def add_principal_galaxy(self): """ This function ... :return: """ # Inform the user log.info("Adding principal galaxy ...") # Get info name, position, redshift, galaxy_type, names, distance, inclination, d25, major, minor, pa = get_galaxy_info("M81", self.properties.center) ra = position.ra dec = position.dec principal = True companions = [] parent = None # Add to the catalog self.extended_source_catalog.add_entry("M81", ra, dec, redshift, galaxy_type, names, distance, inclination, d25, major, minor, pa, principal, companions, parent) # ----------------------------------------------------------------- def make_sources(self): """ This function ... :return: """ # Inform the user log.info("Making sources ...") # Extended self.make_extended_sources() # Point self.make_point_sources() # ----------------------------------------------------------------- def make_extended_sources(self): """ This function ... :return: """ # Inform the user log.info("Making extended sources ...") # Loop over the filters for fltr in self.frames: # Get the frame frame = self.frames[fltr] # Loop over the sources for index in range(len(self.extended_source_catalog)): principal = self.extended_source_catalog["Principal"][index] # Determine flux if principal: central_flux = 1e2 else: central_flux = np.random.uniform(10,50) # Get pixel position coordinate = self.extended_source_catalog.get_position(index).to_pixel(frame.wcs) initial_sersic_amplitude = central_flux initial_sersic_x_0 = coordinate.x initial_sersic_y_0 = coordinate.y if principal: s4g_name, pa, ellipticity, n, re, mag = catalogs.get_galaxy_s4g_one_component_info("M81") angle = Angle(pa, "deg") angle_deg = pa effective_radius = re.to("arcsec").value / frame.average_pixelscale.to("arcsec").value initial_sersic_n = n initial_sersic_r_eff = effective_radius initial_sersic_ellip = ellipticity initial_sersic_theta = np.deg2rad(angle_deg) # 1 / axial_ratio = 1 - ellipticity axial_ratio = 1. / (1. - ellipticity) else: # Get position angle and axes lengths pa = self.extended_source_catalog["Posangle"][index] major = self.extended_source_catalog["Major"][index] minor = self.extended_source_catalog["Minor"][index] axial_ratio = major / minor angle = Angle(pa, "deg") angle_deg = pa effective_radius = major # Produce guess values initial_sersic_r_eff = effective_radius initial_sersic_n = self.config.galaxy_sersic_index initial_sersic_ellip = (axial_ratio - 1.0) / axial_ratio initial_sersic_theta = np.deg2rad(angle_deg) # Produce sersic model from guess parameters, for time trials sersic_x, sersic_y = np.meshgrid(np.arange(frame.xsize), np.arange(frame.ysize)) sersic_model = Sersic2D(amplitude=initial_sersic_amplitude, r_eff=initial_sersic_r_eff, n=initial_sersic_n, x_0=initial_sersic_x_0, y_0=initial_sersic_y_0, ellip=initial_sersic_ellip, theta=initial_sersic_theta) sersic_map = sersic_model(sersic_x, sersic_y) # Limit galaxy? if self.config.limit_galaxy: limit_radius = self.config.galaxy_relative_asymptotic_radius * effective_radius # Create galaxy region galaxy_center = PixelCoordinate(initial_sersic_x_0, initial_sersic_y_0) galaxy_radius = PixelStretch(limit_radius, limit_radius / axial_ratio) galaxy_region = PixelEllipseRegion(galaxy_center, galaxy_radius, angle) galaxy_mask = galaxy_region.to_mask(frame.xsize, frame.ysize) # Set galaxy map zero outside certain radius sersic_map[galaxy_mask.inverse()] = 0.0 # Add frame += sersic_map # mask frame[self.rotation_masks[fltr]] = 0.0 # ----------------------------------------------------------------- def make_point_sources(self): """ This function ... :return: """ # Inform the user log.info("Making point sources ...") # Call the appropriate function if self.config.vary_fwhm: self.make_point_sources_variable_fwhm(self.rotation_masks) else: self.make_point_sources_fixed_fwhm(self.rotation_masks) # ----------------------------------------------------------------- def make_sky(self): """ This function ... :return: """ # Inform the user log.info("Adding sky ...") # Loop over the frames for fltr in self.frames: # Get the frame frame = self.frames[fltr] # Determine random sky level sky_level = np.random.uniform(0.0, 10.) # Create sky gradient y, x = np.mgrid[:frame.ysize, :frame.xsize] sky_gradient = x * y # Normalize so that the maximal sky is 20% sky_gradient = sky_gradient / np.max(sky_gradient) * 20 # Add the sky frame += sky_level + sky_gradient # Mask frame[self.rotation_masks[fltr]] = 0.0 # ----------------------------------------------------------------- def make_galaxy(self): """ This function ... :return: """ # Inform the user log.info("Adding galaxy ...") # ----------------------------------------------------------------- def find(self): """ This function ... :return: """ # Inform the user log.info("Finding the sources ...") # Settings settings = dict() #settings["input"] = settings["output"] = self.find_path settings["nprocesses"] = self.config.nprocesses # Input input_dict = dict() input_dict["dataset"] = self.dataset input_dict["extended_source_catalog"] = self.extended_source_catalog input_dict["point_source_catalog"] = self.point_source_catalog input_dict["output_paths"] = self.find_paths # Construct the command command = Command("find_sources", "find sources", settings, input_dict) # Run the command self.finder = self.run_command(command, remote=self.remote) # ----------------------------------------------------------------- def write(self): """ This function ... :return: """ # Inform the user log.info("Writing ...") # ----------------------------------------------------------------- def plot(self): """ This function ... :return: """ # Inform the user log.info("Plotting ...")
definition = ConfigurationDefinition() definition.add_required("local_path", "string", "path or name of the file or directory to send") definition.add_required("remote", "string", "the remote host to send to", choices=find_host_ids()) definition.add_optional("remote_path", "string", "path of the remote directory to send to") config = parse_arguments("send", definition) # ----------------------------------------------------------------- # Create remote remote = Remote(host_id=config.remote) # ----------------------------------------------------------------- # Set full path of origin origin = fs.absolute_or_in_cwd(config.local_path) name = fs.name(origin) # Set full path to the destination if config.path is not None: destination = fs.join(remote.home_directory, name) else: destination = remote.absolute_path(config.remote_path) # Upload remote.upload(origin, destination) # -----------------------------------------------------------------
# Simulation is retrieved if simulation.retrieved: # Determine the path to the simulation log file local_log_file_path = simulation.log_file_path # Read the log file lines = fs.read_lines(local_log_file_path) # Not yet retrieved else: # The path to the simulation log file remote_log_file_path = simulation.remote_log_file_path # Create and setup the remote remote = Remote() remote.setup(config.remote) # Check whether the log file exists if not remote.is_file(remote_log_file_path): raise RuntimeError("The log file does not (yet) exist remotely") # Read the log file lines = remote.read_lines(remote_log_file_path) # Print the lines of the log file for line in lines: print(line) # -----------------------------------------------------------------
# Get configuration config = parse_arguments("plot_truncated", definition) # ----------------------------------------------------------------- modeling_path = verify_modeling_cwd() # ----------------------------------------------------------------- # Load the modeling environment environment = GalaxyModelingEnvironment(modeling_path) # ----------------------------------------------------------------- # Setup the remote remote = Remote(host_id=environment.cache_host_id) # ----------------------------------------------------------------- directory_name = cached_directory_name_for_single_command( environment, "truncate") remote_truncation_path = fs.join(remote.home_directory, directory_name) # ----------------------------------------------------------------- filter_name = str(config.filter) remote_truncation_path_filter = fs.join(remote_truncation_path, filter_name) if not remote.is_directory(remote_truncation_path_filter): raise ValueError("Could not find cached data for the " + filter_name + " image")
from pts.core.basics.log import log # ----------------------------------------------------------------- # Create the configuration definition definition = ConfigurationDefinition() definition.add_required("remote", "string", "remote host", choices=find_host_ids()) definition.add_required("filename", "string", "file name (or path)") # Read the command line arguments config = parse_arguments("open", definition, description="Open a file on a remote host") # ----------------------------------------------------------------- # Connect to remote remote = Remote() remote.setup(config.remote) # Determine the absolute file path filepath = remote.absolute_path(config.filename) # Determine the file path relative to the home directory relative_filepath = remote.relative_to_home(filepath) # Check if the file exists; otherwise don't bother mounting if not remote.is_file(filepath): log.error("The file does not exist") exit() # Disconnect the remote remote.logout()
definition.add_required("remote", "string", "remote host to mount", choices=all_host_ids()) definition.add_required("id", "positive_integer", "simulation ID") definition.add_flag("debug_output", "show all simulation output in debug mode") # Read the command line arguments config = parse_arguments("show_progress", definition, description="Show the progress of a remotely running simulation") # ----------------------------------------------------------------- # Load the simulation simulation = get_simulation_for_host(config.remote, config.id) # ----------------------------------------------------------------- # Load the remote remote = Remote() if not remote.setup(host_id=config.remote): raise RuntimeError("Could not connect to the remote host") # ----------------------------------------------------------------- # Create the status object status = LogSimulationStatus(simulation.remote_log_file_path, remote=remote, debug_output=config.debug_output) # Show the simulation progress with log.no_debugging(): success = status.show_progress(simulation.handle) # Check whether not crashed if not success: raise RuntimeError("The simulation crashed") # -----------------------------------------------------------------
# Create configuration config = parse_arguments("cache_images", definition) # ----------------------------------------------------------------- modeling_path = verify_modeling_cwd() # ----------------------------------------------------------------- # Load the modeling environment environment = GalaxyModelingEnvironment(modeling_path) # ----------------------------------------------------------------- # Setup the remote remote = Remote(host_id=environment.cache_host_id) # ----------------------------------------------------------------- remote_data_path = cached_directory_path_for_single_command( environment, "initialize_preparation", remote) if not remote.is_directory(remote_data_path): remote.create_directory(remote_data_path) # ----------------------------------------------------------------- # Reset? if config.reset: # Loop over all FITS files in the remote data directory for name, path in remote.files_in_path(remote_data_path,
# Add flags definition.add_flag("recursive", "search recursively", False) definition.add_flag("full", "show the full paths", False) # ----------------------------------------------------------------- # Parse the arguments into a configuration config = parse_arguments("find_files", definition, description="Find files containing a certain string in the current working directory") # ----------------------------------------------------------------- # REMOTELY if config.remote is not None: # Create remote remote = Remote(host_id=config.remote) # Determine path if config.remote_path is not None: find_path = remote.absolute_or_in_home(config.remote_path) else: find_path = remote.home_directory #print(find_path) #print(remote.items_in_path(find_path, recursive=True)) # Loop over the files paths = remote.files_in_path(find_path, contains=config.contains, not_contains=config.not_contains, extension=config.extension, recursive=config.recursive) if len(paths) == 0: log.warning("No files found") else: for path in paths:
# Add flags definition.add_flag("conda", "also remove conda installation") definition.add_flag("qt", "also remove Qt installation") definition.add_flag("one_attempt", "only perform one attempt at connecting to a remote") # Get the config config = parse_arguments("deinstall_all", definition) # ----------------------------------------------------------------- # Loop over the remote hosts for host_id in find_host_ids(): # Setup remote = Remote() if not remote.setup(host_id, one_attempt=config.one_attempt): log.warning("Remote host '" + host_id + "' is offline") continue # Create uninstaller uninstaller = Uninstaller() # Set options uninstaller.config.skirt_and_or_pts = config.skirt_and_or_pts uninstaller.config.conda = config.conda uninstaller.config.qt = config.qt # Inform the user log.info("Running the uninstaller for remote host '" + host_id + "' ...")
from pts.core.basics.log import log # ----------------------------------------------------------------- # Create definition definition = ConfigurationDefinition() definition.add_positional_optional("remotes", "string_list", "remote hosts on which to clear", choices=find_host_ids(), default=find_host_ids()) # Create setter config = parse_arguments("clear_sessions_and_temp", definition) # ----------------------------------------------------------------- # Loop over the remote hosts for host_id in config.remotes: # Setup the remote remote = Remote() if not remote.setup(host_id): log.warning("Could not connect to remote host '" + host_id + "'") continue # Clear temporary directory and clear sessions remote.clear_temp_and_sessions() # -----------------------------------------------------------------
# Create definition definition = ConfigurationDefinition() definition.add_positional_optional("remote", "string", "remote host on which to clear the temporary directory", choices=find_host_ids()) definition.add_optional("names", "string_list", "remove temporary directories matching these names (e.g. 'installation' matches 'installation_2017-05-03--08-45-44-410', 'installation_2017-05-03--13-52-28-371', etc. and also exact matches") # Create setter config = parse_arguments("clear_temp", definition) # ----------------------------------------------------------------- # Remote if config.remote is not None: # Create the remote remote = Remote(host_id=config.remote) # Check whether names are given if config.names is not None: # Loop over the directories in the temporary directory for path, name in remote.directories_in_path(remote.pts_temp_path, returns=["path", "name"]): # Contains timestamp if time.is_unique_name(name): # Without timestamp in list basename = time.get_name_from_unique_name(name) if basename in config.names: remote.remove_directory(path) # Full name in list
class SourcesTest(SourcesTestBase): """ This class ... """ def __init__(self, *args, **kwargs): """ The constructor ... :param kwargs: """ # Call the constructor of the base class super(SourcesTest, self).__init__(*args, **kwargs) # The galaxy properties self.properties = None # The rotation masks self.rotation_masks = dict() # The catalog fetcher self.fetcher = CatalogFetcher() # Catalogs self.extended_source_catalog = None # The real FWHMs self.real_fwhms = dict() # ----------------------------------------------------------------- def run(self, **kwargs): """ This function ... :param kwargs: :return: """ # 1. Call the setup function self.setup(**kwargs) # 2. Load galaxy properties self.load_properties() # 3. Load the coordinate system self.load_coordinate_systems() # 4. Initialize the frames self.initialize_frames() # 5. Set FWHMs self.set_fwhms() # 6. Make rotation mask self.make_rotation_masks() # 7. Get catalogs self.get_catalogs() # 8. Generate the sources self.make_sources() # 9. Make noise self.make_noise(self.rotation_masks) # 10. Make sky self.make_sky() # 11. Create the dataset self.create_dataset(self.rotation_masks) # 12. Create the directories self.create_directories() # 13. Find sources self.find() # 14. Extract sources self.extract() # 15. Write self.write() # 16. Plot self.plot() # ----------------------------------------------------------------- def setup(self, **kwargs): """ This function ... :param kwargs: :return: """ # Call the setup function of the base class super(SourcesTest, self).setup(**kwargs) # Set remote if self.config.remote is not None: self.remote = Remote() self.remote.setup(self.config.remote) # ----------------------------------------------------------------- def load_properties(self): """ This function ... :return: """ # Inform the user log.info("Loading the galaxy properties ...") # Determine the path path = fs.join(m81_data_path, "properties.dat") # Load self.properties = GalaxyProperties.from_file(path) # ----------------------------------------------------------------- def load_coordinate_systems(self): """ This function ... :return: """ # Inform the user log.info("Loading the coordinate systems ...") nfilters_stars = 0 nfilters_extra = 0 # Loop over the header files for path, name in fs.files_in_path(headers_path, extension="txt", returns=["path", "name"]): # Get the filter and wavelength fltr = parse_filter(name) wavelength = fltr.effective if fltr.effective is not None else fltr.center # SKip Planck if fltr.observatory == "Planck": continue # Wavelength greater than 25 micron if wavelength > wavelengths.ranges.ir.mir.max: if nfilters_extra == self.config.nfilters_extra: continue else: nfilters_extra += 1 # Wavelength smaller than 25 micron else: if nfilters_stars == self.config.nfilters_stars: continue else: nfilters_stars += 1 # Debugging log.debug("Loading the coordinate system for the '" + str(fltr) + "' filter ...") # Get WCS wcs = CoordinateSystem.from_header_file(path) # Add the coordinate system self.coordinate_systems.append(wcs, fltr=fltr) # Break the loop if we have enough if nfilters_stars == self.config.nfilters_stars and nfilters_extra == self.config.nfilters_extra: break # ----------------------------------------------------------------- def make_rotation_masks(self): """ This function ... :return: """ # Inform the user log.info("Making rotation masks ...") # Loop over the filters for fltr in self.frames: # Debugging log.info("Making rotation mask for the '" + str(fltr) + "' image ...") # Get the frame frame = self.frames[fltr] # Rotate if self.config.rotate: # Choose a random rotation angle angle = Angle(np.random.uniform(-90, 90), "deg") # Debugging log.debug("The random rotation angle is '" + stringify.stringify(angle)[1] + "'") # Create mask mask = frame.rotation_mask(angle) # Don't rotate else: mask = Mask.empty_like(frame) # Set the mask self.rotation_masks[fltr] = mask # ----------------------------------------------------------------- def get_catalogs(self): """ This function ... :return: """ # Inform the user log.info("Getting catalogs of point and extended sources ...") # Adding catalogued sources if self.config.add_catalogued_sources: self.fetch_catalogs() else: self.initialize_catalogs() # Add random sources self.create_random_sources() # ----------------------------------------------------------------- def fetch_catalogs(self): """ This function ... :return: """ # Inform the user log.info("Fetching catalogs ...") # Get extended source catalog self.extended_source_catalog = self.fetcher.get_extended_source_catalog( self.coordinate_systems.bounding_box) # Fetch self.point_source_catalog = self.fetcher.get_point_source_catalog( self.coordinate_systems.bounding_box, self.coordinate_systems.min_pixelscale, self.config.point_source_catalogs) # ----------------------------------------------------------------- def initialize_catalogs(self): """ This function ... :param self: :return: """ # Inform the user log.info("Initializing catalogs ...") # Extended sources self.extended_source_catalog = ExtendedSourceCatalog() # Add principal galaxy self.add_principal_galaxy() # Point sources self.point_source_catalog = PointSourceCatalog() # ----------------------------------------------------------------- def add_principal_galaxy(self): """ This function ... :return: """ # Inform the user log.info("Adding principal galaxy ...") # Get info name, position, redshift, galaxy_type, names, distance, inclination, d25, major, minor, pa = get_galaxy_info( "M81", self.properties.center) ra = position.ra dec = position.dec principal = True companions = [] parent = None # Add to the catalog self.extended_source_catalog.add_entry("M81", ra, dec, redshift, galaxy_type, names, distance, inclination, d25, major, minor, pa, principal, companions, parent) # ----------------------------------------------------------------- def make_sources(self): """ This function ... :return: """ # Inform the user log.info("Making sources ...") # Extended self.make_extended_sources() # Point self.make_point_sources() # ----------------------------------------------------------------- def make_extended_sources(self): """ This function ... :return: """ # Inform the user log.info("Making extended sources ...") # Loop over the filters for fltr in self.frames: # Get the frame frame = self.frames[fltr] # Loop over the sources for index in range(len(self.extended_source_catalog)): principal = self.extended_source_catalog["Principal"][index] # Determine flux if principal: central_flux = 1e2 else: central_flux = np.random.uniform(10, 50) # Get pixel position coordinate = self.extended_source_catalog.get_position( index).to_pixel(frame.wcs) initial_sersic_amplitude = central_flux initial_sersic_x_0 = coordinate.x initial_sersic_y_0 = coordinate.y if principal: s4g_name, pa, ellipticity, n, re, mag = catalogs.get_galaxy_s4g_one_component_info( "M81") angle = Angle(pa, "deg") angle_deg = pa effective_radius = re.to( "arcsec").value / frame.average_pixelscale.to( "arcsec").value initial_sersic_n = n initial_sersic_r_eff = effective_radius initial_sersic_ellip = ellipticity initial_sersic_theta = np.deg2rad(angle_deg) # 1 / axial_ratio = 1 - ellipticity axial_ratio = 1. / (1. - ellipticity) else: # Get position angle and axes lengths pa = self.extended_source_catalog["Posangle"][index] major = self.extended_source_catalog["Major"][index] minor = self.extended_source_catalog["Minor"][index] axial_ratio = major / minor angle = Angle(pa, "deg") angle_deg = pa effective_radius = major # Produce guess values initial_sersic_r_eff = effective_radius initial_sersic_n = self.config.galaxy_sersic_index initial_sersic_ellip = (axial_ratio - 1.0) / axial_ratio initial_sersic_theta = np.deg2rad(angle_deg) # Produce sersic model from guess parameters, for time trials sersic_x, sersic_y = np.meshgrid(np.arange(frame.xsize), np.arange(frame.ysize)) sersic_model = Sersic2D(amplitude=initial_sersic_amplitude, r_eff=initial_sersic_r_eff, n=initial_sersic_n, x_0=initial_sersic_x_0, y_0=initial_sersic_y_0, ellip=initial_sersic_ellip, theta=initial_sersic_theta) sersic_map = sersic_model(sersic_x, sersic_y) # Limit galaxy? if self.config.limit_galaxy: limit_radius = self.config.galaxy_relative_asymptotic_radius * effective_radius # Create galaxy region galaxy_center = PixelCoordinate(initial_sersic_x_0, initial_sersic_y_0) galaxy_radius = PixelStretch(limit_radius, limit_radius / axial_ratio) galaxy_region = PixelEllipseRegion(galaxy_center, galaxy_radius, angle) galaxy_mask = galaxy_region.to_mask( frame.xsize, frame.ysize) # Set galaxy map zero outside certain radius sersic_map[galaxy_mask.inverse()] = 0.0 # Add frame += sersic_map # mask frame[self.rotation_masks[fltr]] = 0.0 # ----------------------------------------------------------------- def make_point_sources(self): """ This function ... :return: """ # Inform the user log.info("Making point sources ...") # Call the appropriate function if self.config.vary_fwhm: self.make_point_sources_variable_fwhm(self.rotation_masks) else: self.make_point_sources_fixed_fwhm(self.rotation_masks) # ----------------------------------------------------------------- def make_sky(self): """ This function ... :return: """ # Inform the user log.info("Adding sky ...") # Loop over the frames for fltr in self.frames: # Get the frame frame = self.frames[fltr] # Determine random sky level sky_level = np.random.uniform(0.0, 10.) # Create sky gradient y, x = np.mgrid[:frame.ysize, :frame.xsize] sky_gradient = x * y # Normalize so that the maximal sky is 20% sky_gradient = sky_gradient / np.max(sky_gradient) * 20 # Add the sky frame += sky_level + sky_gradient # Mask frame[self.rotation_masks[fltr]] = 0.0 # ----------------------------------------------------------------- def make_galaxy(self): """ This function ... :return: """ # Inform the user log.info("Adding galaxy ...") # ----------------------------------------------------------------- def find(self): """ This function ... :return: """ # Inform the user log.info("Finding the sources ...") # Settings settings = dict() #settings["input"] = settings["output"] = self.find_path settings["nprocesses"] = self.config.nprocesses # Input input_dict = dict() input_dict["dataset"] = self.dataset input_dict["extended_source_catalog"] = self.extended_source_catalog input_dict["point_source_catalog"] = self.point_source_catalog input_dict["output_paths"] = self.find_paths # Construct the command command = Command("find_sources", "find sources", settings, input_dict) # Run the command self.finder = self.run_command(command, remote=self.remote) # ----------------------------------------------------------------- def write(self): """ This function ... :return: """ # Inform the user log.info("Writing ...") # ----------------------------------------------------------------- def plot(self): """ This function ... :return: """ # Inform the user log.info("Plotting ...")
definition = ConfigurationDefinition() definition.add_optional("remotes", "string_list", "remote host ID", choices=find_host_ids(), default=find_host_ids()) config = parse_arguments("sessions", definition) # ----------------------------------------------------------------- # Loop over the remote host for host_id in config.remotes: # Create and setup the remote remote = Remote() if not remote.setup(host_id): raise RuntimeError("The remote host '" + host_id + "' is not available at the moment") # Get screen names and tmux session names screen_names = remote.screen_names() tmux_names = remote.tmux_names() # Inform the user log.info("Closing sessions on remote host '" + host_id + "' ...") # Loop over the screens for name in screen_names: # Inform the user
# ----------------------------------------------------------------- # Parse the arguments into a configuration config = parse_arguments( "clear_tasks", definition, description="Clear PTS tasks for a certain remote host") # ----------------------------------------------------------------- # Loop over the remote hosts for host_id in config.remotes: # Check whether the remote is available if config.full: remote = Remote() if not remote.setup(host_id): log.warning("The remote host '" + host_id + "' is not available: skipping ...") continue else: remote = None # Determine the path to the run directory for the specified remote host host_run_path = fs.join(introspection.skirt_run_dir, host_id) # Check if there are simulations if not fs.is_directory(host_run_path): log.debug("No run directory for host '" + host_id + "'") continue if fs.is_empty(host_run_path):
# Read the command line arguments config = parse_arguments( "show_progress", definition, description="Show the progress of a remotely running simulation") # ----------------------------------------------------------------- # Load the simulation simulation = get_simulation_for_host(config.remote, config.id) # ----------------------------------------------------------------- # Load the remote remote = Remote() if not remote.setup(host_id=config.remote): raise RuntimeError("Could not connect to the remote host") # ----------------------------------------------------------------- # Create the status object status = LogSimulationStatus(simulation.remote_log_file_path, remote=remote, debug_output=config.debug_output) # Show the simulation progress if config.debug_output: success = status.show_progress(simulation.handle) else: with no_debugging(): success = status.show_progress(simulation.handle)
definition = ConfigurationDefinition() # Required definition.add_required("remote_path", "string", "remote path of the file or directory to retrieve") definition.add_required("remote", "string", "remote host to retrieve from", choices=find_host_ids()) # Local path definition.add_positional_optional("local_path", "string", "path of the local directory to store the file/directory") # Create configuration config = parse_arguments("retrieve", definition, "Retrieve a file or directory from a remote host") # ----------------------------------------------------------------- # Create remote remote = Remote(host_id=config.remote) # ----------------------------------------------------------------- # Set full path of origin origin = remote.absolute_path(config.remote_path) # Set full path to the destination if config.local_path is not None: destination = fs.absolute_or_in_cwd(config.local_path) else: destination = fs.cwd() # ----------------------------------------------------------------- # Debugging log.debug("Origin: " + origin) log.debug("Destination: " + destination)
# Set log level in a special way if config.debug: setup_log("DEBUG") else: setup_log("ERROR") # ----------------------------------------------------------------- # Loop over the hosts print("") for host in config.hosts: # Debugging log.debug("Connecting to remote host '" + host.id + "' ...") # Create remote, try connecting remote = Remote() connected = remote.setup(host, login_timeout=10) # only try for 10 seconds # Connection succeeded? if connected: print(fmt.green + host.id + ": up" + fmt.reset) else: print(fmt.red + host.id + ": down" + fmt.reset) continue # Show clustername if remote.cluster_name is not None: print("Cluster name: " + remote.cluster_name) print("") print(" - " + fmt.bold + "uses scheduling system: " + fmt.reset + str(remote.scheduler)) # Show status
definition.add_positional_optional("ids", "integer_list", "the IDs of the simulations to clear") definition.add_flag("full", "fully clear the simulations, also remove remote simulation directories") # ----------------------------------------------------------------- # Parse the arguments into a configuration config = parse_arguments("clear_tasks", definition, description="Clear PTS tasks for a certain remote host") # ----------------------------------------------------------------- # Loop over the remote hosts for host_id in config.remotes: # Check whether the remote is available if config.full: remote = Remote() if not remote.setup(host_id): log.warning("The remote host '" + host_id + "' is not available: skipping ...") continue else: remote = None # Determine the path to the run directory for the specified remote host host_run_path = fs.join(introspection.skirt_run_dir, host_id) # Check if there are simulations if not fs.is_directory(host_run_path): log.debug("No run directory for host '" + host_id + "'") continue if fs.is_empty(host_run_path): log.debug("No simulations for host '" + host_id + "'") # Loop over the simulation files in the run directory
from pts.core.remote.remote import Remote from pts.core.tools import filesystem as fs # ----------------------------------------------------------------- definition = ConfigurationDefinition() definition.add_required("remote_path", "string", "remote path of the file or directory to retrieve") definition.add_required("remote", "string", "remote host to retrieve from", choices=find_host_ids()) definition.add_optional("local_path", "string", "path of the local directory to store the file/directory") config = parse_arguments("retrieve", definition) # ----------------------------------------------------------------- # Create remote remote = Remote(host_id=config.remote) # ----------------------------------------------------------------- # Set full path of origin origin = remote.absolute_path(config.remote_path) # Set full path to the destination if config.local_path is not None: destination = fs.absolute_or_in_cwd(config.local_path) else: destination = fs.cwd() # ----------------------------------------------------------------- # Debugging log.debug("Origin: " + origin) log.debug("Destination: " + destination)
choices=find_host_ids()) definition.add_required("id", "positive_integer", "task ID") # ----------------------------------------------------------------- # Parse the arguments into a configuration #setter = InteractiveConfigurationSetter("show_log", "Show the log output of a remote PTS task") config = parse_arguments( "show_task_log", definition, description="Show the log output of a remote PTS task") # ----------------------------------------------------------------- # Create and setup the remote remote = Remote() remote.setup(config.remote) # Open the task task_path = fs.join(introspection.pts_run_dir, config.remote, str(config.id) + ".task") task = Task.from_file(task_path) # Check whether the log file is present log_output_path = task.remote_log_path log_path = None for filename in remote.files_in_path(log_output_path): if "log" in filename: log_path = fs.join(log_output_path, filename) break
choices=find_host_ids()) definition.add_optional( "names", "string_list", "remove temporary directories matching these names (e.g. 'installation' matches 'installation_2017-05-03--08-45-44-410', 'installation_2017-05-03--13-52-28-371', etc. and also exact matches" ) # Create setter config = parse_arguments("clear_temp", definition) # ----------------------------------------------------------------- # Remote if config.remote is not None: # Create the remote remote = Remote(host_id=config.remote) # Check whether names are given if config.names is not None: # Loop over the directories in the temporary directory for path, name in remote.directories_in_path(remote.pts_temp_path, returns=["path", "name"]): # Contains timestamp if time.is_unique_name(name): # Without timestamp in list basename = time.get_name_from_unique_name(name) if basename in config.names: remote.remove_directory(path)
from __future__ import absolute_import, division, print_function # Import the relevant PTS classes and modules from pts.core.remote.host import find_host_ids from pts.core.remote.remote import Remote from pts.core.tools import formatting as fmt from pts.core.basics.log import setup_log from pts.core.basics.configuration import parse_logging_arguments # ----------------------------------------------------------------- config = parse_logging_arguments("hosts") # ----------------------------------------------------------------- # Set log level in a special way if config.debug: setup_log("DEBUG") else: setup_log("ERROR") # ----------------------------------------------------------------- # Loop over the hosts for host_id in find_host_ids(): remote = Remote() connected = remote.setup(host_id, login_timeout=15) # only try for 15 seconds if connected: print(fmt.green + host_id + ": up" + fmt.reset) else: print(fmt.red + host_id + ": down" + fmt.reset) # -----------------------------------------------------------------
definition = ConfigurationDefinition() # Required definition.add_required("local_path", "string", "path or name of the file or directory to send") definition.add_required("remote", "string", "the remote host to send to", choices=find_host_ids()) # Remote path definition.add_positional_optional("remote_path", "string", "path of the remote directory to send to") # Create configuration config = parse_arguments("send", definition, "Send a file or directory to a remote host") # ----------------------------------------------------------------- # Create remote remote = Remote(host_id=config.remote) # ----------------------------------------------------------------- # Set full path of origin origin = fs.absolute_or_in_cwd(config.local_path) name = fs.name(origin) # Set full path to the destination if config.remote_path is None: destination = fs.join(remote.home_directory, name) else: destination = remote.absolute_path(config.remote_path) # Upload remote.upload(origin, destination) # -----------------------------------------------------------------
from __future__ import absolute_import, division, print_function # Import the relevant PTS classes and modules from pts.core.basics.log import log from pts.core.basics.configuration import ConfigurationDefinition, parse_arguments from pts.core.remote.remote import Remote # ----------------------------------------------------------------- definition = ConfigurationDefinition() definition.add_required("remote", "string", "remote host ID") setter = parse_arguments("sessions", definition) # ----------------------------------------------------------------- remote = Remote() if not remote.setup(config.remote): raise RuntimeError("The remote host '" + config.remote + "' is not available at the moment") screen_names = remote.screen_names() tmux_names = remote.tmux_names() print("SCREEN") print(screen_names) print("") print("TMUX") print(tmux_names)