def run_remotely(command_name, config, keep, host_id, log):
"""
This function ...
:param command_name:
:param config:
:param keep:
:param host_id:
:param log:
:return:
"""
# Additional imports
from pts.core.remote.remote import Remote
# Start message
log.start("Starting " + command_name + " on remote host " + host_id + " ...")
# Debugging
log.debug("Initializing the remote ...")
# Initialize the remote execution environment
remote = Remote(host_id=host_id)
# Run PTS remotely
task = remote.run_pts(command_name, config, keep_remote_output=keep)
# Succesfully submitted
log.success("Succesfully submitted the PTS job to the remote host")
def run_remotely(command_name, config, keep, host_id, log):
"""
This function ...
:param command_name:
:param config:
:param keep:
:param host_id:
:param log:
:return:
"""
# Additional imports
from pts.core.remote.remote import Remote
# Start message
log.start("Starting " + command_name + " on remote host " + host_id +
" ...")
# Debugging
log.debug("Initializing the remote ...")
# Initialize the remote execution environment
remote = Remote(host_id=host_id)
# Run PTS remotely
task = remote.run_pts(command_name, config, keep_remote_output=keep)
# Succesfully submitted
log.success("Succesfully submitted the PTS job to the remote host")
def setup(self, **kwargs):
"""
This function ...
:param kwargs:
:return:
"""
# Call the setup function of the base class
super(SourcesTest, self).setup(**kwargs)
# Set remote
if self.config.remote is not None:
self.remote = Remote()
self.remote.setup(self.config.remote)
def setup(self, **kwargs):
"""
This function ...
:param kwargs:
:return:
"""
# Call the setup function of the base class
super(RemotesConfigurable, self).setup(**kwargs)
# If remotes are passed
if "remotes" in kwargs:
# Get the remotes
self.remotes = kwargs.pop("remotes")
# Check if they are setup
for remote in self.remotes:
if not remote.connected:
raise RuntimeError("Remotes must be connected")
# Remotes are not passed
else:
# Gather host IDs
if self.config.not_remotes is not None:
host_ids = [
host_id for host_id in self.config.host_ids
if host_id not in self.config.not_remotes
]
else:
host_ids = self.config.host_ids
# Loop over the different hosts
for host_id in host_ids:
# Setup the remote (login)
remote = Remote(log_conda=kwargs.pop("log_conda", False))
if not remote.setup(host_id,
one_attempt=self.config.one_attempt):
log.warning("Remote host '" + host_id +
"' is down: skipping")
continue
# Add the remote to the list
self.remotes.append(remote)
def setup(self, **kwargs):
"""
This function ...
:param kwargs:
:return:
"""
# Call the setup function of the base class
super(RemotesConfigurable, self).setup(**kwargs)
# If remotes are passed
if "remotes" in kwargs:
# Get the remotes
self.remotes = kwargs.pop("remotes")
# Check if they are setup
for remote in self.remotes:
if not remote.connected: raise RuntimeError("Remotes must be connected")
# Remotes are not passed
else:
# Gather host IDs
#if self.config.not_remotes is not None: host_ids = [host_id for host_id in self.config.host_ids if host_id not in self.config.not_remotes]
#else: host_ids = self.config.host_ids
# Loop over the different hosts
#for host_id in host_ids:
for host in self.config.hosts:
# Create the remote object
remote = Remote(log_conda=kwargs.pop("log_conda", False))
# Login to the remote
#if not remote.setup(host_id, one_attempt=self.config.one_attempt, cluster_name=self.config.clustername):
if not remote.setup(host, one_attempt=self.config.one_attempt):
log.warning("Remote host '" + host.id + "' is down: skipping")
continue
# Add the remote to the list
self.remotes.append(remote)
def setup(self, **kwargs):
"""
This function ...
:param kwargs:
:return:
"""
# Call the setup function of the base class
super(Uninstaller, self).setup(**kwargs)
# Check if remote instance is passed
if "remote" in kwargs: self.remote = kwargs.pop("remote")
elif self.config.remote is not None:
remote = Remote()
if not remote.setup(self.config.remote): raise RuntimeError("The remote host is not available")
self.remote = remote
else: log.warning('No remote host is specified, will be uninstalling locally')
def setup(self, **kwargs):
"""
This function ...
:param kwargs:
:return:
"""
# Call the setup function of the base class
super(Uninstaller, self).setup(**kwargs)
# Check if remote instance is passed
if "remote" in kwargs: self.remote = kwargs.pop("remote")
elif self.config.remote is not None:
remote = Remote()
if not remote.setup(self.config.remote):
raise RuntimeError("The remote host is not available")
self.remote = remote
else:
log.warning(
'No remote host is specified, will be uninstalling locally')
def setup(self, **kwargs):
"""
This function ...
:param kwargs:
:return:
"""
# Call the setup function of the base class
super(SourcesTest, self).setup(**kwargs)
# Set remote
if self.config.remote is not None:
self.remote = Remote()
self.remote.setup(self.config.remote)
from pts.core.remote.remote import Remote
from pts.core.remote.host import find_host_ids
# -----------------------------------------------------------------
definition = ConfigurationDefinition()
definition.add_optional("remotes", "string_list", "remote host ID", choices=find_host_ids(), default=find_host_ids())
config = parse_arguments("sessions", definition)
# -----------------------------------------------------------------
# Loop over the remote host
for host_id in config.remotes:
# Create and setup the remote
remote = Remote()
if not remote.setup(host_id): raise RuntimeError("The remote host '" + host_id + "' is not available at the moment")
# Get screen names and tmux session names
screen_names = remote.screen_names()
tmux_names = remote.tmux_names()
# Inform the user
log.info("Closing sessions on remote host '" + host_id + "' ...")
# Loop over the screens
for name in screen_names:
# Inform the user
log.info("Closing screen session '" + name + "' ...")
# Import the relevant PTS classes and modules
from pts.core.basics.configuration import ConfigurationDefinition, parse_arguments
from pts.core.remote.remote import Remote
# -----------------------------------------------------------------
definition = ConfigurationDefinition()
definition.add_required("remote", "string", "remote host ID")
definition.add_required("line", "string", "line to be executed on the remote host")
config = parse_arguments("sessions", definition)
# -----------------------------------------------------------------
# Initialize the remote
remote = Remote()
if not remote.setup(config.remote): raise RuntimeError("The remote host '" + config.remote + "' is not available at the moment")
# -----------------------------------------------------------------
print("")
#print("-----------------------------------------------------------------")
print("OUTPUT")
print("-----------------------------------------------------------------")
print("")
# Execute the line, show the output
output = remote.execute(config.line)
for line in output: print(line)
print("")
definition.add_flag("recursive", "search recursively", False)
definition.add_flag("full", "show the full paths", False)
definition.add_flag("remove", "remove the found files", False)
# -----------------------------------------------------------------
# Parse the arguments into a configuration
config = parse_arguments("find_files", definition, description="Find files containing a certain string in the current working directory")
# -----------------------------------------------------------------
# REMOTELY
if config.remote is not None:
# Create remote
remote = Remote(host_id=config.remote)
# Determine path
if config.remote_path is not None: find_path = remote.absolute_or_in_home(config.remote_path)
else: find_path = remote.home_directory
#print(find_path)
#print(remote.items_in_path(find_path, recursive=True))
# Loop over the files
paths = remote.files_in_path(find_path, contains=config.contains, not_contains=config.not_contains, extension=config.extension, recursive=config.recursive, exact_name=config.exact_name, exact_not_name=config.exact_not_name)
nfiles = len(paths)
if nfiles == 0: log.warning("No files found")
else:
# Create configuration definition
definition = ConfigurationDefinition()
# Add required
definition.add_required("remote", "string", "the remote host ID")
# Add flags
definition.add_flag("versions", "compare versions", "v")
# Get configuration
config = parse_arguments("compare_python_packages", definition)
# -----------------------------------------------------------------
# Create the remote execution environment
remote = Remote()
# Log in
remote.setup(config.remote)
# Get remote python session
remote_python = remote.start_python_session(assume_pts=False)
# Get all python packages installed on the remote host
remote_packages = remote_python.installed_packages
# Get local python package version
local_packages = introspection.installed_python_packages()
# Loop over all python packages necessary for PTS
for dependency in introspection.get_all_dependencies():
# Load the fitting run fitting_run = runs.load(config.name) # Get the generation generation = fitting_run.get_generation(config.generation) # ----------------------------------------------------------------- # Get the cache path cache_path = generation.single_cache_path # ----------------------------------------------------------------- # Create remote remote = Remote(host_id=config.host_id) # ----------------------------------------------------------------- # Determine RT Modeling backup directory path remote_backup_path = fs.join(remote.home_directory, "RT Modeling Backup") remote_backup_galaxy_path = fs.join(remote_backup_path, environment.galaxy_name) # ----------------------------------------------------------------- # Create path if not remote.is_directory(remote_backup_galaxy_path): remote.create_directory(remote_backup_galaxy_path, recursive=True) # ----------------------------------------------------------------- # Synchronize
# Create the configuration definition
definition = ConfigurationDefinition()
definition.add_required("remote", "string", "remote host ID", choices=find_host_ids())
definition.add_flag("private", "install SKIRT from private (development) repository")
# Create the configuration
config = parse_arguments("installation_commands", definition)
# -----------------------------------------------------------------
installation_commands = defaultdict(list)
# -----------------------------------------------------------------
# Create the remote
remote = Remote()
if not remote.setup(config.remote):
log.error("The remote host is not available")
exit()
# -----------------------------------------------------------------
# Check the modules
modules = Modules(remote)
# -----------------------------------------------------------------
# Qt is present
if modules.paths["qt"] is not None:
# Get the version
"name of the remote host",
choices=find_host_ids())
definition.add_required("id", "positive_integer", "simulation ID")
# -----------------------------------------------------------------
# Parse the arguments into a configuration
config = parse_arguments(
"show_simulation_log",
definition,
description="Show the log output of a remote SKIRT simulation")
# -----------------------------------------------------------------
# Create and setup the remote
remote = Remote()
remote.setup(config.remote)
# Open the simulation
simulation = get_simulation_for_host(config.remote, config.id)
# The name of the ski file (the simulation prefix)
ski_name = simulation.prefix()
# The path to the simulation log file
remote_log_file_path = simulation.remote_log_file_path
# Check whether the log file exists
if not remote.is_file(remote_log_file_path):
raise RuntimeError("The log file does not exist remotely")
from __future__ import absolute_import, division, print_function
# Import the relevant PTS classes and modules
from pts.core.basics.log import log
from pts.core.basics.configuration import ConfigurationDefinition, parse_arguments
from pts.core.remote.remote import Remote
# -----------------------------------------------------------------
definition = ConfigurationDefinition()
definition.add_required("remote", "string", "remote host ID")
setter = parse_arguments("sessions", definition)
# -----------------------------------------------------------------
remote = Remote()
if not remote.setup(config.remote): raise RuntimeError("The remote host '" + config.remote + "' is not available at the moment")
screen_names = remote.screen_names()
tmux_names = remote.tmux_names()
print("SCREEN")
print(screen_names)
print("")
print("TMUX")
print(tmux_names)
# -----------------------------------------------------------------
# Import the relevant PTS classes and modules
from pts.core.basics.configuration import ConfigurationDefinition, parse_arguments
from pts.core.remote.host import find_host_ids
from pts.core.remote.remote import Remote
from pts.core.basics.log import log
# -----------------------------------------------------------------
# Create definition
definition = ConfigurationDefinition()
definition.add_positional_optional("remotes", "string_list", "remote hosts on which to clear", choices=find_host_ids(), default=find_host_ids())
# Create setter
config = parse_arguments("clear_sessions_and_temp", definition)
# -----------------------------------------------------------------
# Loop over the remote hosts
for host_id in config.remotes:
# Setup the remote
remote = Remote()
if not remote.setup(host_id):
log.warning("Could not connect to remote host '" + host_id + "'")
continue
# Clear temporary directory and clear sessions
remote.clear_temp_and_sessions()
# -----------------------------------------------------------------
class SourcesTest(SourcesTestBase):
"""
This class ...
"""
def __init__(self, *args, **kwargs):
"""
The constructor ...
:param kwargs:
"""
# Call the constructor of the base class
super(SourcesTest, self).__init__(*args, **kwargs)
# The galaxy properties
self.properties = None
# The rotation masks
self.rotation_masks = dict()
# The catalog fetcher
self.fetcher = CatalogFetcher()
# Catalogs
self.extended_source_catalog = None
# The real FWHMs
self.real_fwhms = dict()
# -----------------------------------------------------------------
def _run(self, **kwargs):
"""
This function ...
:param kwargs:
:return:
"""
# 2. Load galaxy properties
self.load_properties()
# 3. Load the coordinate system
self.load_coordinate_systems()
# 4. Initialize the frames
self.initialize_frames()
# 5. Set FWHMs
self.set_fwhms()
# 6. Make rotation mask
self.make_rotation_masks()
# 7. Get catalogs
self.get_catalogs()
# 8. Generate the sources
self.make_sources()
# 9. Make noise
self.make_noise(self.rotation_masks)
# 10. Make sky
self.make_sky()
# 11. Create the dataset
self.create_dataset(self.rotation_masks)
# 12. Create the directories
self.create_directories()
# 13. Find sources
self.find()
# 14. Extract sources
self.extract()
# 15. Write
self.write()
# 16. Plot
self.plot()
# -----------------------------------------------------------------
def setup(self, **kwargs):
"""
This function ...
:param kwargs:
:return:
"""
# Call the setup function of the base class
super(SourcesTest, self).setup(**kwargs)
# Set remote
if self.config.remote is not None:
self.remote = Remote()
self.remote.setup(self.config.remote)
# -----------------------------------------------------------------
def load_properties(self):
"""
This function ...
:return:
"""
# Inform the user
log.info("Loading the galaxy properties ...")
# Determine the path
path = fs.join(m81_data_path, "properties.dat")
# Load
self.properties = GalaxyProperties.from_file(path)
# -----------------------------------------------------------------
def load_coordinate_systems(self):
"""
This function ...
:return:
"""
# Inform the user
log.info("Loading the coordinate systems ...")
nfilters_stars = 0
nfilters_extra = 0
# Loop over the header files
for path, name in fs.files_in_path(headers_path, extension="txt", returns=["path", "name"]):
# Get the filter and wavelength
fltr = parse_filter(name)
wavelength = fltr.effective if fltr.effective is not None else fltr.center
# SKip Planck
if fltr.observatory == "Planck": continue
# Wavelength greater than 25 micron
if wavelength > wavelengths.ranges.ir.mir.max:
if nfilters_extra == self.config.nfilters_extra: continue
else: nfilters_extra += 1
# Wavelength smaller than 25 micron
else:
if nfilters_stars == self.config.nfilters_stars: continue
else: nfilters_stars += 1
# Debugging
log.debug("Loading the coordinate system for the '" + str(fltr) + "' filter ...")
# Get WCS
wcs = CoordinateSystem.from_header_file(path)
# Add the coordinate system
self.coordinate_systems.append(wcs, fltr=fltr)
# Break the loop if we have enough
if nfilters_stars == self.config.nfilters_stars and nfilters_extra == self.config.nfilters_extra: break
# -----------------------------------------------------------------
def make_rotation_masks(self):
"""
This function ...
:return:
"""
# Inform the user
log.info("Making rotation masks ...")
# Loop over the filters
for fltr in self.frames:
# Debugging
log.info("Making rotation mask for the '" + str(fltr) + "' image ...")
# Get the frame
frame = self.frames[fltr]
# Rotate
if self.config.rotate:
# Choose a random rotation angle
angle = Angle(np.random.uniform(-90, 90), "deg")
# Debugging
log.debug("The random rotation angle is '" + stringify.stringify(angle)[1] + "'")
# Create mask
mask = frame.rotation_mask(angle)
# Don't rotate
else: mask = Mask.empty_like(frame)
# Set the mask
self.rotation_masks[fltr] = mask
# -----------------------------------------------------------------
def get_catalogs(self):
"""
This function ...
:return:
"""
# Inform the user
log.info("Getting catalogs of point and extended sources ...")
# Adding catalogued sources
if self.config.add_catalogued_sources: self.fetch_catalogs()
else: self.initialize_catalogs()
# Add random sources
self.create_random_sources()
# -----------------------------------------------------------------
def fetch_catalogs(self):
"""
This function ...
:return:
"""
# Inform the user
log.info("Fetching catalogs ...")
# Get extended source catalog
self.extended_source_catalog = self.fetcher.get_extended_source_catalog(self.coordinate_systems.bounding_box)
# Fetch
self.point_source_catalog = self.fetcher.get_point_source_catalog(self.coordinate_systems.bounding_box, self.coordinate_systems.min_pixelscale, self.config.point_source_catalogs)
# -----------------------------------------------------------------
def initialize_catalogs(self):
"""
This function ...
:param self:
:return:
"""
# Inform the user
log.info("Initializing catalogs ...")
# Extended sources
self.extended_source_catalog = ExtendedSourceCatalog()
# Add principal galaxy
self.add_principal_galaxy()
# Point sources
self.point_source_catalog = PointSourceCatalog()
# -----------------------------------------------------------------
def add_principal_galaxy(self):
"""
This function ...
:return:
"""
# Inform the user
log.info("Adding principal galaxy ...")
# Get info
name, position, redshift, galaxy_type, names, distance, inclination, d25, major, minor, pa = get_galaxy_info("M81", self.properties.center)
ra = position.ra
dec = position.dec
principal = True
companions = []
parent = None
# Add to the catalog
self.extended_source_catalog.add_entry("M81", ra, dec, redshift, galaxy_type, names, distance, inclination, d25, major, minor, pa, principal, companions, parent)
# -----------------------------------------------------------------
def make_sources(self):
"""
This function ...
:return:
"""
# Inform the user
log.info("Making sources ...")
# Extended
self.make_extended_sources()
# Point
self.make_point_sources()
# -----------------------------------------------------------------
def make_extended_sources(self):
"""
This function ...
:return:
"""
# Inform the user
log.info("Making extended sources ...")
# Loop over the filters
for fltr in self.frames:
# Get the frame
frame = self.frames[fltr]
# Loop over the sources
for index in range(len(self.extended_source_catalog)):
principal = self.extended_source_catalog["Principal"][index]
# Determine flux
if principal: central_flux = 1e2
else: central_flux = np.random.uniform(10,50)
# Get pixel position
coordinate = self.extended_source_catalog.get_position(index).to_pixel(frame.wcs)
initial_sersic_amplitude = central_flux
initial_sersic_x_0 = coordinate.x
initial_sersic_y_0 = coordinate.y
if principal:
s4g_name, pa, ellipticity, n, re, mag = catalogs.get_galaxy_s4g_one_component_info("M81")
angle = Angle(pa, "deg")
angle_deg = pa
effective_radius = re.to("arcsec").value / frame.average_pixelscale.to("arcsec").value
initial_sersic_n = n
initial_sersic_r_eff = effective_radius
initial_sersic_ellip = ellipticity
initial_sersic_theta = np.deg2rad(angle_deg)
# 1 / axial_ratio = 1 - ellipticity
axial_ratio = 1. / (1. - ellipticity)
else:
# Get position angle and axes lengths
pa = self.extended_source_catalog["Posangle"][index]
major = self.extended_source_catalog["Major"][index]
minor = self.extended_source_catalog["Minor"][index]
axial_ratio = major / minor
angle = Angle(pa, "deg")
angle_deg = pa
effective_radius = major
# Produce guess values
initial_sersic_r_eff = effective_radius
initial_sersic_n = self.config.galaxy_sersic_index
initial_sersic_ellip = (axial_ratio - 1.0) / axial_ratio
initial_sersic_theta = np.deg2rad(angle_deg)
# Produce sersic model from guess parameters, for time trials
sersic_x, sersic_y = np.meshgrid(np.arange(frame.xsize), np.arange(frame.ysize))
sersic_model = Sersic2D(amplitude=initial_sersic_amplitude, r_eff=initial_sersic_r_eff,
n=initial_sersic_n, x_0=initial_sersic_x_0,
y_0=initial_sersic_y_0, ellip=initial_sersic_ellip,
theta=initial_sersic_theta)
sersic_map = sersic_model(sersic_x, sersic_y)
# Limit galaxy?
if self.config.limit_galaxy:
limit_radius = self.config.galaxy_relative_asymptotic_radius * effective_radius
# Create galaxy region
galaxy_center = PixelCoordinate(initial_sersic_x_0, initial_sersic_y_0)
galaxy_radius = PixelStretch(limit_radius, limit_radius / axial_ratio)
galaxy_region = PixelEllipseRegion(galaxy_center, galaxy_radius, angle)
galaxy_mask = galaxy_region.to_mask(frame.xsize, frame.ysize)
# Set galaxy map zero outside certain radius
sersic_map[galaxy_mask.inverse()] = 0.0
# Add
frame += sersic_map
# mask
frame[self.rotation_masks[fltr]] = 0.0
# -----------------------------------------------------------------
def make_point_sources(self):
"""
This function ...
:return:
"""
# Inform the user
log.info("Making point sources ...")
# Call the appropriate function
if self.config.vary_fwhm: self.make_point_sources_variable_fwhm(self.rotation_masks)
else: self.make_point_sources_fixed_fwhm(self.rotation_masks)
# -----------------------------------------------------------------
def make_sky(self):
"""
This function ...
:return:
"""
# Inform the user
log.info("Adding sky ...")
# Loop over the frames
for fltr in self.frames:
# Get the frame
frame = self.frames[fltr]
# Determine random sky level
sky_level = np.random.uniform(0.0, 10.)
# Create sky gradient
y, x = np.mgrid[:frame.ysize, :frame.xsize]
sky_gradient = x * y
# Normalize so that the maximal sky is 20%
sky_gradient = sky_gradient / np.max(sky_gradient) * 20
# Add the sky
frame += sky_level + sky_gradient
# Mask
frame[self.rotation_masks[fltr]] = 0.0
# -----------------------------------------------------------------
def make_galaxy(self):
"""
This function ...
:return:
"""
# Inform the user
log.info("Adding galaxy ...")
# -----------------------------------------------------------------
def find(self):
"""
This function ...
:return:
"""
# Inform the user
log.info("Finding the sources ...")
# Settings
settings = dict()
#settings["input"] =
settings["output"] = self.find_path
settings["nprocesses"] = self.config.nprocesses
# Input
input_dict = dict()
input_dict["dataset"] = self.dataset
input_dict["extended_source_catalog"] = self.extended_source_catalog
input_dict["point_source_catalog"] = self.point_source_catalog
input_dict["output_paths"] = self.find_paths
# Construct the command
command = Command("find_sources", "find sources", settings, input_dict)
# Run the command
self.finder = self.run_command(command, remote=self.remote)
# -----------------------------------------------------------------
def write(self):
"""
This function ...
:return:
"""
# Inform the user
log.info("Writing ...")
# -----------------------------------------------------------------
def plot(self):
"""
This function ...
:return:
"""
# Inform the user
log.info("Plotting ...")
definition = ConfigurationDefinition()
definition.add_required("local_path", "string",
"path or name of the file or directory to send")
definition.add_required("remote",
"string",
"the remote host to send to",
choices=find_host_ids())
definition.add_optional("remote_path", "string",
"path of the remote directory to send to")
config = parse_arguments("send", definition)
# -----------------------------------------------------------------
# Create remote
remote = Remote(host_id=config.remote)
# -----------------------------------------------------------------
# Set full path of origin
origin = fs.absolute_or_in_cwd(config.local_path)
name = fs.name(origin)
# Set full path to the destination
if config.path is not None: destination = fs.join(remote.home_directory, name)
else: destination = remote.absolute_path(config.remote_path)
# Upload
remote.upload(origin, destination)
# -----------------------------------------------------------------
# Simulation is retrieved
if simulation.retrieved:
# Determine the path to the simulation log file
local_log_file_path = simulation.log_file_path
# Read the log file
lines = fs.read_lines(local_log_file_path)
# Not yet retrieved
else:
# The path to the simulation log file
remote_log_file_path = simulation.remote_log_file_path
# Create and setup the remote
remote = Remote()
remote.setup(config.remote)
# Check whether the log file exists
if not remote.is_file(remote_log_file_path): raise RuntimeError("The log file does not (yet) exist remotely")
# Read the log file
lines = remote.read_lines(remote_log_file_path)
# Print the lines of the log file
for line in lines: print(line)
# -----------------------------------------------------------------
# Get configuration
config = parse_arguments("plot_truncated", definition)
# -----------------------------------------------------------------
modeling_path = verify_modeling_cwd()
# -----------------------------------------------------------------
# Load the modeling environment
environment = GalaxyModelingEnvironment(modeling_path)
# -----------------------------------------------------------------
# Setup the remote
remote = Remote(host_id=environment.cache_host_id)
# -----------------------------------------------------------------
directory_name = cached_directory_name_for_single_command(
environment, "truncate")
remote_truncation_path = fs.join(remote.home_directory, directory_name)
# -----------------------------------------------------------------
filter_name = str(config.filter)
remote_truncation_path_filter = fs.join(remote_truncation_path, filter_name)
if not remote.is_directory(remote_truncation_path_filter):
raise ValueError("Could not find cached data for the " + filter_name +
" image")
from pts.core.basics.log import log
# -----------------------------------------------------------------
# Create the configuration definition
definition = ConfigurationDefinition()
definition.add_required("remote", "string", "remote host", choices=find_host_ids())
definition.add_required("filename", "string", "file name (or path)")
# Read the command line arguments
config = parse_arguments("open", definition, description="Open a file on a remote host")
# -----------------------------------------------------------------
# Connect to remote
remote = Remote()
remote.setup(config.remote)
# Determine the absolute file path
filepath = remote.absolute_path(config.filename)
# Determine the file path relative to the home directory
relative_filepath = remote.relative_to_home(filepath)
# Check if the file exists; otherwise don't bother mounting
if not remote.is_file(filepath):
log.error("The file does not exist")
exit()
# Disconnect the remote
remote.logout()
definition.add_required("remote", "string", "remote host to mount", choices=all_host_ids())
definition.add_required("id", "positive_integer", "simulation ID")
definition.add_flag("debug_output", "show all simulation output in debug mode")
# Read the command line arguments
config = parse_arguments("show_progress", definition, description="Show the progress of a remotely running simulation")
# -----------------------------------------------------------------
# Load the simulation
simulation = get_simulation_for_host(config.remote, config.id)
# -----------------------------------------------------------------
# Load the remote
remote = Remote()
if not remote.setup(host_id=config.remote): raise RuntimeError("Could not connect to the remote host")
# -----------------------------------------------------------------
# Create the status object
status = LogSimulationStatus(simulation.remote_log_file_path, remote=remote, debug_output=config.debug_output)
# Show the simulation progress
with log.no_debugging(): success = status.show_progress(simulation.handle)
# Check whether not crashed
if not success: raise RuntimeError("The simulation crashed")
# -----------------------------------------------------------------
# Create configuration
config = parse_arguments("cache_images", definition)
# -----------------------------------------------------------------
modeling_path = verify_modeling_cwd()
# -----------------------------------------------------------------
# Load the modeling environment
environment = GalaxyModelingEnvironment(modeling_path)
# -----------------------------------------------------------------
# Setup the remote
remote = Remote(host_id=environment.cache_host_id)
# -----------------------------------------------------------------
remote_data_path = cached_directory_path_for_single_command(
environment, "initialize_preparation", remote)
if not remote.is_directory(remote_data_path):
remote.create_directory(remote_data_path)
# -----------------------------------------------------------------
# Reset?
if config.reset:
# Loop over all FITS files in the remote data directory
for name, path in remote.files_in_path(remote_data_path,
# Add flags
definition.add_flag("recursive", "search recursively", False)
definition.add_flag("full", "show the full paths", False)
# -----------------------------------------------------------------
# Parse the arguments into a configuration
config = parse_arguments("find_files", definition, description="Find files containing a certain string in the current working directory")
# -----------------------------------------------------------------
# REMOTELY
if config.remote is not None:
# Create remote
remote = Remote(host_id=config.remote)
# Determine path
if config.remote_path is not None: find_path = remote.absolute_or_in_home(config.remote_path)
else: find_path = remote.home_directory
#print(find_path)
#print(remote.items_in_path(find_path, recursive=True))
# Loop over the files
paths = remote.files_in_path(find_path, contains=config.contains, not_contains=config.not_contains, extension=config.extension, recursive=config.recursive)
if len(paths) == 0: log.warning("No files found")
else:
for path in paths:
# Add flags
definition.add_flag("conda", "also remove conda installation")
definition.add_flag("qt", "also remove Qt installation")
definition.add_flag("one_attempt", "only perform one attempt at connecting to a remote")
# Get the config
config = parse_arguments("deinstall_all", definition)
# -----------------------------------------------------------------
# Loop over the remote hosts
for host_id in find_host_ids():
# Setup
remote = Remote()
if not remote.setup(host_id, one_attempt=config.one_attempt):
log.warning("Remote host '" + host_id + "' is offline")
continue
# Create uninstaller
uninstaller = Uninstaller()
# Set options
uninstaller.config.skirt_and_or_pts = config.skirt_and_or_pts
uninstaller.config.conda = config.conda
uninstaller.config.qt = config.qt
# Inform the user
log.info("Running the uninstaller for remote host '" + host_id + "' ...")
from pts.core.basics.log import log
# -----------------------------------------------------------------
# Create definition
definition = ConfigurationDefinition()
definition.add_positional_optional("remotes",
"string_list",
"remote hosts on which to clear",
choices=find_host_ids(),
default=find_host_ids())
# Create setter
config = parse_arguments("clear_sessions_and_temp", definition)
# -----------------------------------------------------------------
# Loop over the remote hosts
for host_id in config.remotes:
# Setup the remote
remote = Remote()
if not remote.setup(host_id):
log.warning("Could not connect to remote host '" + host_id + "'")
continue
# Clear temporary directory and clear sessions
remote.clear_temp_and_sessions()
# -----------------------------------------------------------------
# Create definition
definition = ConfigurationDefinition()
definition.add_positional_optional("remote", "string", "remote host on which to clear the temporary directory", choices=find_host_ids())
definition.add_optional("names", "string_list", "remove temporary directories matching these names (e.g. 'installation' matches 'installation_2017-05-03--08-45-44-410', 'installation_2017-05-03--13-52-28-371', etc. and also exact matches")
# Create setter
config = parse_arguments("clear_temp", definition)
# -----------------------------------------------------------------
# Remote
if config.remote is not None:
# Create the remote
remote = Remote(host_id=config.remote)
# Check whether names are given
if config.names is not None:
# Loop over the directories in the temporary directory
for path, name in remote.directories_in_path(remote.pts_temp_path, returns=["path", "name"]):
# Contains timestamp
if time.is_unique_name(name):
# Without timestamp in list
basename = time.get_name_from_unique_name(name)
if basename in config.names: remote.remove_directory(path)
# Full name in list
class SourcesTest(SourcesTestBase):
"""
This class ...
"""
def __init__(self, *args, **kwargs):
"""
The constructor ...
:param kwargs:
"""
# Call the constructor of the base class
super(SourcesTest, self).__init__(*args, **kwargs)
# The galaxy properties
self.properties = None
# The rotation masks
self.rotation_masks = dict()
# The catalog fetcher
self.fetcher = CatalogFetcher()
# Catalogs
self.extended_source_catalog = None
# The real FWHMs
self.real_fwhms = dict()
# -----------------------------------------------------------------
def run(self, **kwargs):
"""
This function ...
:param kwargs:
:return:
"""
# 1. Call the setup function
self.setup(**kwargs)
# 2. Load galaxy properties
self.load_properties()
# 3. Load the coordinate system
self.load_coordinate_systems()
# 4. Initialize the frames
self.initialize_frames()
# 5. Set FWHMs
self.set_fwhms()
# 6. Make rotation mask
self.make_rotation_masks()
# 7. Get catalogs
self.get_catalogs()
# 8. Generate the sources
self.make_sources()
# 9. Make noise
self.make_noise(self.rotation_masks)
# 10. Make sky
self.make_sky()
# 11. Create the dataset
self.create_dataset(self.rotation_masks)
# 12. Create the directories
self.create_directories()
# 13. Find sources
self.find()
# 14. Extract sources
self.extract()
# 15. Write
self.write()
# 16. Plot
self.plot()
# -----------------------------------------------------------------
def setup(self, **kwargs):
"""
This function ...
:param kwargs:
:return:
"""
# Call the setup function of the base class
super(SourcesTest, self).setup(**kwargs)
# Set remote
if self.config.remote is not None:
self.remote = Remote()
self.remote.setup(self.config.remote)
# -----------------------------------------------------------------
def load_properties(self):
"""
This function ...
:return:
"""
# Inform the user
log.info("Loading the galaxy properties ...")
# Determine the path
path = fs.join(m81_data_path, "properties.dat")
# Load
self.properties = GalaxyProperties.from_file(path)
# -----------------------------------------------------------------
def load_coordinate_systems(self):
"""
This function ...
:return:
"""
# Inform the user
log.info("Loading the coordinate systems ...")
nfilters_stars = 0
nfilters_extra = 0
# Loop over the header files
for path, name in fs.files_in_path(headers_path,
extension="txt",
returns=["path", "name"]):
# Get the filter and wavelength
fltr = parse_filter(name)
wavelength = fltr.effective if fltr.effective is not None else fltr.center
# SKip Planck
if fltr.observatory == "Planck": continue
# Wavelength greater than 25 micron
if wavelength > wavelengths.ranges.ir.mir.max:
if nfilters_extra == self.config.nfilters_extra: continue
else: nfilters_extra += 1
# Wavelength smaller than 25 micron
else:
if nfilters_stars == self.config.nfilters_stars: continue
else: nfilters_stars += 1
# Debugging
log.debug("Loading the coordinate system for the '" + str(fltr) +
"' filter ...")
# Get WCS
wcs = CoordinateSystem.from_header_file(path)
# Add the coordinate system
self.coordinate_systems.append(wcs, fltr=fltr)
# Break the loop if we have enough
if nfilters_stars == self.config.nfilters_stars and nfilters_extra == self.config.nfilters_extra:
break
# -----------------------------------------------------------------
def make_rotation_masks(self):
"""
This function ...
:return:
"""
# Inform the user
log.info("Making rotation masks ...")
# Loop over the filters
for fltr in self.frames:
# Debugging
log.info("Making rotation mask for the '" + str(fltr) +
"' image ...")
# Get the frame
frame = self.frames[fltr]
# Rotate
if self.config.rotate:
# Choose a random rotation angle
angle = Angle(np.random.uniform(-90, 90), "deg")
# Debugging
log.debug("The random rotation angle is '" +
stringify.stringify(angle)[1] + "'")
# Create mask
mask = frame.rotation_mask(angle)
# Don't rotate
else:
mask = Mask.empty_like(frame)
# Set the mask
self.rotation_masks[fltr] = mask
# -----------------------------------------------------------------
def get_catalogs(self):
"""
This function ...
:return:
"""
# Inform the user
log.info("Getting catalogs of point and extended sources ...")
# Adding catalogued sources
if self.config.add_catalogued_sources: self.fetch_catalogs()
else: self.initialize_catalogs()
# Add random sources
self.create_random_sources()
# -----------------------------------------------------------------
def fetch_catalogs(self):
"""
This function ...
:return:
"""
# Inform the user
log.info("Fetching catalogs ...")
# Get extended source catalog
self.extended_source_catalog = self.fetcher.get_extended_source_catalog(
self.coordinate_systems.bounding_box)
# Fetch
self.point_source_catalog = self.fetcher.get_point_source_catalog(
self.coordinate_systems.bounding_box,
self.coordinate_systems.min_pixelscale,
self.config.point_source_catalogs)
# -----------------------------------------------------------------
def initialize_catalogs(self):
"""
This function ...
:param self:
:return:
"""
# Inform the user
log.info("Initializing catalogs ...")
# Extended sources
self.extended_source_catalog = ExtendedSourceCatalog()
# Add principal galaxy
self.add_principal_galaxy()
# Point sources
self.point_source_catalog = PointSourceCatalog()
# -----------------------------------------------------------------
def add_principal_galaxy(self):
"""
This function ...
:return:
"""
# Inform the user
log.info("Adding principal galaxy ...")
# Get info
name, position, redshift, galaxy_type, names, distance, inclination, d25, major, minor, pa = get_galaxy_info(
"M81", self.properties.center)
ra = position.ra
dec = position.dec
principal = True
companions = []
parent = None
# Add to the catalog
self.extended_source_catalog.add_entry("M81", ra, dec, redshift,
galaxy_type, names, distance,
inclination, d25, major, minor,
pa, principal, companions,
parent)
# -----------------------------------------------------------------
def make_sources(self):
"""
This function ...
:return:
"""
# Inform the user
log.info("Making sources ...")
# Extended
self.make_extended_sources()
# Point
self.make_point_sources()
# -----------------------------------------------------------------
def make_extended_sources(self):
"""
This function ...
:return:
"""
# Inform the user
log.info("Making extended sources ...")
# Loop over the filters
for fltr in self.frames:
# Get the frame
frame = self.frames[fltr]
# Loop over the sources
for index in range(len(self.extended_source_catalog)):
principal = self.extended_source_catalog["Principal"][index]
# Determine flux
if principal: central_flux = 1e2
else: central_flux = np.random.uniform(10, 50)
# Get pixel position
coordinate = self.extended_source_catalog.get_position(
index).to_pixel(frame.wcs)
initial_sersic_amplitude = central_flux
initial_sersic_x_0 = coordinate.x
initial_sersic_y_0 = coordinate.y
if principal:
s4g_name, pa, ellipticity, n, re, mag = catalogs.get_galaxy_s4g_one_component_info(
"M81")
angle = Angle(pa, "deg")
angle_deg = pa
effective_radius = re.to(
"arcsec").value / frame.average_pixelscale.to(
"arcsec").value
initial_sersic_n = n
initial_sersic_r_eff = effective_radius
initial_sersic_ellip = ellipticity
initial_sersic_theta = np.deg2rad(angle_deg)
# 1 / axial_ratio = 1 - ellipticity
axial_ratio = 1. / (1. - ellipticity)
else:
# Get position angle and axes lengths
pa = self.extended_source_catalog["Posangle"][index]
major = self.extended_source_catalog["Major"][index]
minor = self.extended_source_catalog["Minor"][index]
axial_ratio = major / minor
angle = Angle(pa, "deg")
angle_deg = pa
effective_radius = major
# Produce guess values
initial_sersic_r_eff = effective_radius
initial_sersic_n = self.config.galaxy_sersic_index
initial_sersic_ellip = (axial_ratio - 1.0) / axial_ratio
initial_sersic_theta = np.deg2rad(angle_deg)
# Produce sersic model from guess parameters, for time trials
sersic_x, sersic_y = np.meshgrid(np.arange(frame.xsize),
np.arange(frame.ysize))
sersic_model = Sersic2D(amplitude=initial_sersic_amplitude,
r_eff=initial_sersic_r_eff,
n=initial_sersic_n,
x_0=initial_sersic_x_0,
y_0=initial_sersic_y_0,
ellip=initial_sersic_ellip,
theta=initial_sersic_theta)
sersic_map = sersic_model(sersic_x, sersic_y)
# Limit galaxy?
if self.config.limit_galaxy:
limit_radius = self.config.galaxy_relative_asymptotic_radius * effective_radius
# Create galaxy region
galaxy_center = PixelCoordinate(initial_sersic_x_0,
initial_sersic_y_0)
galaxy_radius = PixelStretch(limit_radius,
limit_radius / axial_ratio)
galaxy_region = PixelEllipseRegion(galaxy_center,
galaxy_radius, angle)
galaxy_mask = galaxy_region.to_mask(
frame.xsize, frame.ysize)
# Set galaxy map zero outside certain radius
sersic_map[galaxy_mask.inverse()] = 0.0
# Add
frame += sersic_map
# mask
frame[self.rotation_masks[fltr]] = 0.0
# -----------------------------------------------------------------
def make_point_sources(self):
"""
This function ...
:return:
"""
# Inform the user
log.info("Making point sources ...")
# Call the appropriate function
if self.config.vary_fwhm:
self.make_point_sources_variable_fwhm(self.rotation_masks)
else:
self.make_point_sources_fixed_fwhm(self.rotation_masks)
# -----------------------------------------------------------------
def make_sky(self):
"""
This function ...
:return:
"""
# Inform the user
log.info("Adding sky ...")
# Loop over the frames
for fltr in self.frames:
# Get the frame
frame = self.frames[fltr]
# Determine random sky level
sky_level = np.random.uniform(0.0, 10.)
# Create sky gradient
y, x = np.mgrid[:frame.ysize, :frame.xsize]
sky_gradient = x * y
# Normalize so that the maximal sky is 20%
sky_gradient = sky_gradient / np.max(sky_gradient) * 20
# Add the sky
frame += sky_level + sky_gradient
# Mask
frame[self.rotation_masks[fltr]] = 0.0
# -----------------------------------------------------------------
def make_galaxy(self):
"""
This function ...
:return:
"""
# Inform the user
log.info("Adding galaxy ...")
# -----------------------------------------------------------------
def find(self):
"""
This function ...
:return:
"""
# Inform the user
log.info("Finding the sources ...")
# Settings
settings = dict()
#settings["input"] =
settings["output"] = self.find_path
settings["nprocesses"] = self.config.nprocesses
# Input
input_dict = dict()
input_dict["dataset"] = self.dataset
input_dict["extended_source_catalog"] = self.extended_source_catalog
input_dict["point_source_catalog"] = self.point_source_catalog
input_dict["output_paths"] = self.find_paths
# Construct the command
command = Command("find_sources", "find sources", settings, input_dict)
# Run the command
self.finder = self.run_command(command, remote=self.remote)
# -----------------------------------------------------------------
def write(self):
"""
This function ...
:return:
"""
# Inform the user
log.info("Writing ...")
# -----------------------------------------------------------------
def plot(self):
"""
This function ...
:return:
"""
# Inform the user
log.info("Plotting ...")
# Create configuration definition
definition = ConfigurationDefinition()
# Add required
definition.add_required("remote", "string", "the remote host ID")
# Add flags
definition.add_flag("versions", "compare versions", "v")
# Get configuration
config = parse_arguments("compare_python_packages", definition)
# -----------------------------------------------------------------
# Create the remote execution environment
remote = Remote()
# Log in
remote.setup(config.remote)
# Get remote python session
remote_python = remote.start_python_session(assume_pts=False)
# Get all python packages installed on the remote host
remote_packages = remote_python.installed_packages
# Get local python package version
local_packages = introspection.installed_python_packages()
# Loop over all python packages necessary for PTS
for dependency in introspection.get_all_dependencies():
definition = ConfigurationDefinition()
definition.add_optional("remotes",
"string_list",
"remote host ID",
choices=find_host_ids(),
default=find_host_ids())
config = parse_arguments("sessions", definition)
# -----------------------------------------------------------------
# Loop over the remote host
for host_id in config.remotes:
# Create and setup the remote
remote = Remote()
if not remote.setup(host_id):
raise RuntimeError("The remote host '" + host_id +
"' is not available at the moment")
# Get screen names and tmux session names
screen_names = remote.screen_names()
tmux_names = remote.tmux_names()
# Inform the user
log.info("Closing sessions on remote host '" + host_id + "' ...")
# Loop over the screens
for name in screen_names:
# Inform the user
# -----------------------------------------------------------------
# Parse the arguments into a configuration
config = parse_arguments(
"clear_tasks",
definition,
description="Clear PTS tasks for a certain remote host")
# -----------------------------------------------------------------
# Loop over the remote hosts
for host_id in config.remotes:
# Check whether the remote is available
if config.full:
remote = Remote()
if not remote.setup(host_id):
log.warning("The remote host '" + host_id +
"' is not available: skipping ...")
continue
else:
remote = None
# Determine the path to the run directory for the specified remote host
host_run_path = fs.join(introspection.skirt_run_dir, host_id)
# Check if there are simulations
if not fs.is_directory(host_run_path):
log.debug("No run directory for host '" + host_id + "'")
continue
if fs.is_empty(host_run_path):
# Read the command line arguments
config = parse_arguments(
"show_progress",
definition,
description="Show the progress of a remotely running simulation")
# -----------------------------------------------------------------
# Load the simulation
simulation = get_simulation_for_host(config.remote, config.id)
# -----------------------------------------------------------------
# Load the remote
remote = Remote()
if not remote.setup(host_id=config.remote):
raise RuntimeError("Could not connect to the remote host")
# -----------------------------------------------------------------
# Create the status object
status = LogSimulationStatus(simulation.remote_log_file_path,
remote=remote,
debug_output=config.debug_output)
# Show the simulation progress
if config.debug_output: success = status.show_progress(simulation.handle)
else:
with no_debugging():
success = status.show_progress(simulation.handle)
definition = ConfigurationDefinition()
# Required
definition.add_required("remote_path", "string", "remote path of the file or directory to retrieve")
definition.add_required("remote", "string", "remote host to retrieve from", choices=find_host_ids())
# Local path
definition.add_positional_optional("local_path", "string", "path of the local directory to store the file/directory")
# Create configuration
config = parse_arguments("retrieve", definition, "Retrieve a file or directory from a remote host")
# -----------------------------------------------------------------
# Create remote
remote = Remote(host_id=config.remote)
# -----------------------------------------------------------------
# Set full path of origin
origin = remote.absolute_path(config.remote_path)
# Set full path to the destination
if config.local_path is not None: destination = fs.absolute_or_in_cwd(config.local_path)
else: destination = fs.cwd()
# -----------------------------------------------------------------
# Debugging
log.debug("Origin: " + origin)
log.debug("Destination: " + destination)
# Set log level in a special way
if config.debug: setup_log("DEBUG")
else: setup_log("ERROR")
# -----------------------------------------------------------------
# Loop over the hosts
print("")
for host in config.hosts:
# Debugging
log.debug("Connecting to remote host '" + host.id + "' ...")
# Create remote, try connecting
remote = Remote()
connected = remote.setup(host, login_timeout=10) # only try for 10 seconds
# Connection succeeded?
if connected: print(fmt.green + host.id + ": up" + fmt.reset)
else:
print(fmt.red + host.id + ": down" + fmt.reset)
continue
# Show clustername
if remote.cluster_name is not None: print("Cluster name: " + remote.cluster_name)
print("")
print(" - " + fmt.bold + "uses scheduling system: " + fmt.reset + str(remote.scheduler))
# Show status
# Add flags
definition.add_flag("conda", "also remove conda installation")
definition.add_flag("qt", "also remove Qt installation")
definition.add_flag("one_attempt",
"only perform one attempt at connecting to a remote")
# Get the config
config = parse_arguments("deinstall_all", definition)
# -----------------------------------------------------------------
# Loop over the remote hosts
for host_id in find_host_ids():
# Setup
remote = Remote()
if not remote.setup(host_id, one_attempt=config.one_attempt):
log.warning("Remote host '" + host_id + "' is offline")
continue
# Create uninstaller
uninstaller = Uninstaller()
# Set options
uninstaller.config.skirt_and_or_pts = config.skirt_and_or_pts
uninstaller.config.conda = config.conda
uninstaller.config.qt = config.qt
# Inform the user
log.info("Running the uninstaller for remote host '" + host_id + "' ...")
definition.add_positional_optional("ids", "integer_list", "the IDs of the simulations to clear")
definition.add_flag("full", "fully clear the simulations, also remove remote simulation directories")
# -----------------------------------------------------------------
# Parse the arguments into a configuration
config = parse_arguments("clear_tasks", definition, description="Clear PTS tasks for a certain remote host")
# -----------------------------------------------------------------
# Loop over the remote hosts
for host_id in config.remotes:
# Check whether the remote is available
if config.full:
remote = Remote()
if not remote.setup(host_id):
log.warning("The remote host '" + host_id + "' is not available: skipping ...")
continue
else: remote = None
# Determine the path to the run directory for the specified remote host
host_run_path = fs.join(introspection.skirt_run_dir, host_id)
# Check if there are simulations
if not fs.is_directory(host_run_path):
log.debug("No run directory for host '" + host_id + "'")
continue
if fs.is_empty(host_run_path): log.debug("No simulations for host '" + host_id + "'")
# Loop over the simulation files in the run directory
from pts.core.remote.remote import Remote
from pts.core.tools import filesystem as fs
# -----------------------------------------------------------------
definition = ConfigurationDefinition()
definition.add_required("remote_path", "string", "remote path of the file or directory to retrieve")
definition.add_required("remote", "string", "remote host to retrieve from", choices=find_host_ids())
definition.add_optional("local_path", "string", "path of the local directory to store the file/directory")
config = parse_arguments("retrieve", definition)
# -----------------------------------------------------------------
# Create remote
remote = Remote(host_id=config.remote)
# -----------------------------------------------------------------
# Set full path of origin
origin = remote.absolute_path(config.remote_path)
# Set full path to the destination
if config.local_path is not None: destination = fs.absolute_or_in_cwd(config.local_path)
else: destination = fs.cwd()
# -----------------------------------------------------------------
# Debugging
log.debug("Origin: " + origin)
log.debug("Destination: " + destination)
choices=find_host_ids())
definition.add_required("id", "positive_integer", "task ID")
# -----------------------------------------------------------------
# Parse the arguments into a configuration
#setter = InteractiveConfigurationSetter("show_log", "Show the log output of a remote PTS task")
config = parse_arguments(
"show_task_log",
definition,
description="Show the log output of a remote PTS task")
# -----------------------------------------------------------------
# Create and setup the remote
remote = Remote()
remote.setup(config.remote)
# Open the task
task_path = fs.join(introspection.pts_run_dir, config.remote,
str(config.id) + ".task")
task = Task.from_file(task_path)
# Check whether the log file is present
log_output_path = task.remote_log_path
log_path = None
for filename in remote.files_in_path(log_output_path):
if "log" in filename:
log_path = fs.join(log_output_path, filename)
break
choices=find_host_ids())
definition.add_optional(
"names", "string_list",
"remove temporary directories matching these names (e.g. 'installation' matches 'installation_2017-05-03--08-45-44-410', 'installation_2017-05-03--13-52-28-371', etc. and also exact matches"
)
# Create setter
config = parse_arguments("clear_temp", definition)
# -----------------------------------------------------------------
# Remote
if config.remote is not None:
# Create the remote
remote = Remote(host_id=config.remote)
# Check whether names are given
if config.names is not None:
# Loop over the directories in the temporary directory
for path, name in remote.directories_in_path(remote.pts_temp_path,
returns=["path", "name"]):
# Contains timestamp
if time.is_unique_name(name):
# Without timestamp in list
basename = time.get_name_from_unique_name(name)
if basename in config.names: remote.remove_directory(path)
from __future__ import absolute_import, division, print_function
# Import the relevant PTS classes and modules
from pts.core.remote.host import find_host_ids
from pts.core.remote.remote import Remote
from pts.core.tools import formatting as fmt
from pts.core.basics.log import setup_log
from pts.core.basics.configuration import parse_logging_arguments
# -----------------------------------------------------------------
config = parse_logging_arguments("hosts")
# -----------------------------------------------------------------
# Set log level in a special way
if config.debug: setup_log("DEBUG")
else: setup_log("ERROR")
# -----------------------------------------------------------------
# Loop over the hosts
for host_id in find_host_ids():
remote = Remote()
connected = remote.setup(host_id, login_timeout=15) # only try for 15 seconds
if connected: print(fmt.green + host_id + ": up" + fmt.reset)
else: print(fmt.red + host_id + ": down" + fmt.reset)
# -----------------------------------------------------------------
definition = ConfigurationDefinition()
# Required
definition.add_required("local_path", "string", "path or name of the file or directory to send")
definition.add_required("remote", "string", "the remote host to send to", choices=find_host_ids())
# Remote path
definition.add_positional_optional("remote_path", "string", "path of the remote directory to send to")
# Create configuration
config = parse_arguments("send", definition, "Send a file or directory to a remote host")
# -----------------------------------------------------------------
# Create remote
remote = Remote(host_id=config.remote)
# -----------------------------------------------------------------
# Set full path of origin
origin = fs.absolute_or_in_cwd(config.local_path)
name = fs.name(origin)
# Set full path to the destination
if config.remote_path is None: destination = fs.join(remote.home_directory, name)
else: destination = remote.absolute_path(config.remote_path)
# Upload
remote.upload(origin, destination)
# -----------------------------------------------------------------
from __future__ import absolute_import, division, print_function
# Import the relevant PTS classes and modules
from pts.core.basics.log import log
from pts.core.basics.configuration import ConfigurationDefinition, parse_arguments
from pts.core.remote.remote import Remote
# -----------------------------------------------------------------
definition = ConfigurationDefinition()
definition.add_required("remote", "string", "remote host ID")
setter = parse_arguments("sessions", definition)
# -----------------------------------------------------------------
remote = Remote()
if not remote.setup(config.remote):
raise RuntimeError("The remote host '" + config.remote +
"' is not available at the moment")
screen_names = remote.screen_names()
tmux_names = remote.tmux_names()
print("SCREEN")
print(screen_names)
print("")
print("TMUX")
print(tmux_names)
from pts.core.basics.log import log
# -----------------------------------------------------------------
# Create the configuration definition
definition = ConfigurationDefinition()
definition.add_required("remote", "string", "remote host", choices=find_host_ids())
definition.add_required("filename", "string", "file name (or path)")
# Read the command line arguments
config = parse_arguments("open", definition, description="Open a file on a remote host")
# -----------------------------------------------------------------
# Connect to remote
remote = Remote()
remote.setup(config.remote)
# Determine the absolute file path
filepath = remote.absolute_path(config.filename)
# Determine the file path relative to the home directory
relative_filepath = remote.relative_to_home(filepath)
# Check if the file exists; otherwise don't bother mounting
if not remote.is_file(filepath):
log.error("The file does not exist")
exit()
# Disconnect the remote
remote.logout()
