示例#1
0
文件: run.py 项目: SKIRT/PTS
def run_remotely(command_name, config, keep, host_id, log):

    """
    This function ...
    :param command_name:
    :param config:
    :param keep:
    :param host_id:
    :param log:
    :return: 
    """

    # Additional imports
    from pts.core.remote.remote import Remote

    # Start message
    log.start("Starting " + command_name + " on remote host " + host_id + " ...")

    # Debugging
    log.debug("Initializing the remote ...")

    # Initialize the remote execution environment
    remote = Remote(host_id=host_id)

    # Run PTS remotely
    task = remote.run_pts(command_name, config, keep_remote_output=keep)

    # Succesfully submitted
    log.success("Succesfully submitted the PTS job to the remote host")
示例#2
0
def run_remotely(command_name, config, keep, host_id, log):
    """
    This function ...
    :param command_name:
    :param config:
    :param keep:
    :param host_id:
    :param log:
    :return: 
    """

    # Additional imports
    from pts.core.remote.remote import Remote

    # Start message
    log.start("Starting " + command_name + " on remote host " + host_id +
              " ...")

    # Debugging
    log.debug("Initializing the remote ...")

    # Initialize the remote execution environment
    remote = Remote(host_id=host_id)

    # Run PTS remotely
    task = remote.run_pts(command_name, config, keep_remote_output=keep)

    # Succesfully submitted
    log.success("Succesfully submitted the PTS job to the remote host")
示例#3
0
文件: test.py 项目: rag9704/PTS
    def setup(self, **kwargs):
        """
        This function ...
        :param kwargs:
        :return:
        """

        # Call the setup function of the base class
        super(SourcesTest, self).setup(**kwargs)

        # Set remote
        if self.config.remote is not None:
            self.remote = Remote()
            self.remote.setup(self.config.remote)
示例#4
0
    def setup(self, **kwargs):
        """
        This function ...
        :param kwargs:
        :return:
        """

        # Call the setup function of the base class
        super(RemotesConfigurable, self).setup(**kwargs)

        # If remotes are passed
        if "remotes" in kwargs:

            # Get the remotes
            self.remotes = kwargs.pop("remotes")

            # Check if they are setup
            for remote in self.remotes:
                if not remote.connected:
                    raise RuntimeError("Remotes must be connected")

        # Remotes are not passed
        else:

            # Gather host IDs
            if self.config.not_remotes is not None:
                host_ids = [
                    host_id for host_id in self.config.host_ids
                    if host_id not in self.config.not_remotes
                ]
            else:
                host_ids = self.config.host_ids

            # Loop over the different hosts
            for host_id in host_ids:

                # Setup the remote (login)
                remote = Remote(log_conda=kwargs.pop("log_conda", False))
                if not remote.setup(host_id,
                                    one_attempt=self.config.one_attempt):
                    log.warning("Remote host '" + host_id +
                                "' is down: skipping")
                    continue

                # Add the remote to the list
                self.remotes.append(remote)
示例#5
0
文件: configurable.py 项目: SKIRT/PTS
    def setup(self, **kwargs):

        """
        This function ...
        :param kwargs:
        :return:
        """

        # Call the setup function of the base class
        super(RemotesConfigurable, self).setup(**kwargs)

        # If remotes are passed
        if "remotes" in kwargs:

            # Get the remotes
            self.remotes = kwargs.pop("remotes")

            # Check if they are setup
            for remote in self.remotes:
                if not remote.connected: raise RuntimeError("Remotes must be connected")

        # Remotes are not passed
        else:

            # Gather host IDs
            #if self.config.not_remotes is not None: host_ids = [host_id for host_id in self.config.host_ids if host_id not in self.config.not_remotes]
            #else: host_ids = self.config.host_ids

            # Loop over the different hosts
            #for host_id in host_ids:
            for host in self.config.hosts:

                # Create the remote object
                remote = Remote(log_conda=kwargs.pop("log_conda", False))

                # Login to the remote
                #if not remote.setup(host_id, one_attempt=self.config.one_attempt, cluster_name=self.config.clustername):
                if not remote.setup(host, one_attempt=self.config.one_attempt):
                    log.warning("Remote host '" + host.id + "' is down: skipping")
                    continue

                # Add the remote to the list
                self.remotes.append(remote)
示例#6
0
文件: uninstaller.py 项目: SKIRT/PTS
    def setup(self, **kwargs):

        """
        This function ...
        :param kwargs:
        :return:
        """

        # Call the setup function of the base class
        super(Uninstaller, self).setup(**kwargs)

        # Check if remote instance is passed
        if "remote" in kwargs: self.remote = kwargs.pop("remote")
        elif self.config.remote is not None:

            remote = Remote()
            if not remote.setup(self.config.remote): raise RuntimeError("The remote host is not available")
            self.remote = remote

        else: log.warning('No remote host is specified, will be uninstalling locally')
示例#7
0
    def setup(self, **kwargs):
        """
        This function ...
        :param kwargs:
        :return:
        """

        # Call the setup function of the base class
        super(Uninstaller, self).setup(**kwargs)

        # Check if remote instance is passed
        if "remote" in kwargs: self.remote = kwargs.pop("remote")
        elif self.config.remote is not None:

            remote = Remote()
            if not remote.setup(self.config.remote):
                raise RuntimeError("The remote host is not available")
            self.remote = remote

        else:
            log.warning(
                'No remote host is specified, will be uninstalling locally')
示例#8
0
文件: test.py 项目: SKIRT/PTS
    def setup(self, **kwargs):

        """
        This function ...
        :param kwargs:
        :return:
        """

        # Call the setup function of the base class
        super(SourcesTest, self).setup(**kwargs)

        # Set remote
        if self.config.remote is not None:
            self.remote = Remote()
            self.remote.setup(self.config.remote)
示例#9
0
from pts.core.remote.remote import Remote
from pts.core.remote.host import find_host_ids

# -----------------------------------------------------------------

definition = ConfigurationDefinition()
definition.add_optional("remotes", "string_list", "remote host ID", choices=find_host_ids(), default=find_host_ids())
config = parse_arguments("sessions", definition)

# -----------------------------------------------------------------

# Loop over the remote host
for host_id in config.remotes:

    # Create and setup the remote
    remote = Remote()
    if not remote.setup(host_id): raise RuntimeError("The remote host '" + host_id + "' is not available at the moment")

    # Get screen names and tmux session names
    screen_names = remote.screen_names()
    tmux_names = remote.tmux_names()

    # Inform the user
    log.info("Closing sessions on remote host '" + host_id + "' ...")

    # Loop over the screens
    for name in screen_names:

        # Inform the user
        log.info("Closing screen session '" + name + "' ...")
示例#10
0
文件: execute.py 项目: rag9704/PTS
# Import the relevant PTS classes and modules
from pts.core.basics.configuration import ConfigurationDefinition, parse_arguments
from pts.core.remote.remote import Remote

# -----------------------------------------------------------------

definition = ConfigurationDefinition()
definition.add_required("remote", "string", "remote host ID")
definition.add_required("line", "string", "line to be executed on the remote host")
config = parse_arguments("sessions", definition)

# -----------------------------------------------------------------

# Initialize the remote
remote = Remote()
if not remote.setup(config.remote): raise RuntimeError("The remote host '" + config.remote + "' is not available at the moment")

# -----------------------------------------------------------------

print("")
#print("-----------------------------------------------------------------")
print("OUTPUT")
print("-----------------------------------------------------------------")
print("")

# Execute the line, show the output
output = remote.execute(config.line)
for line in output: print(line)

print("")
示例#11
0
文件: find_files.py 项目: SKIRT/PTS
definition.add_flag("recursive", "search recursively", False)
definition.add_flag("full", "show the full paths", False)
definition.add_flag("remove", "remove the found files", False)

# -----------------------------------------------------------------

# Parse the arguments into a configuration
config = parse_arguments("find_files", definition, description="Find files containing a certain string in the current working directory")

# -----------------------------------------------------------------

# REMOTELY
if config.remote is not None:

    # Create remote
    remote = Remote(host_id=config.remote)

    # Determine path
    if config.remote_path is not None: find_path = remote.absolute_or_in_home(config.remote_path)
    else: find_path = remote.home_directory

    #print(find_path)

    #print(remote.items_in_path(find_path, recursive=True))

    # Loop over the files
    paths = remote.files_in_path(find_path, contains=config.contains, not_contains=config.not_contains, extension=config.extension, recursive=config.recursive, exact_name=config.exact_name, exact_not_name=config.exact_not_name)
    nfiles = len(paths)

    if nfiles == 0: log.warning("No files found")
    else:
示例#12
0
# Create configuration definition
definition = ConfigurationDefinition()

# Add required
definition.add_required("remote", "string", "the remote host ID")

# Add flags
definition.add_flag("versions", "compare versions", "v")

# Get configuration
config = parse_arguments("compare_python_packages", definition)

# -----------------------------------------------------------------

# Create the remote execution environment
remote = Remote()

# Log in
remote.setup(config.remote)

# Get remote python session
remote_python = remote.start_python_session(assume_pts=False)

# Get all python packages installed on the remote host
remote_packages = remote_python.installed_packages

# Get local python package version
local_packages = introspection.installed_python_packages()

# Loop over all python packages necessary for PTS
for dependency in introspection.get_all_dependencies():
示例#13
0
# Load the fitting run
fitting_run = runs.load(config.name)

# Get the generation
generation = fitting_run.get_generation(config.generation)

# -----------------------------------------------------------------

# Get the cache path
cache_path = generation.single_cache_path

# -----------------------------------------------------------------

# Create remote
remote = Remote(host_id=config.host_id)

# -----------------------------------------------------------------

# Determine RT Modeling backup directory path
remote_backup_path = fs.join(remote.home_directory, "RT Modeling Backup")
remote_backup_galaxy_path = fs.join(remote_backup_path, environment.galaxy_name)

# -----------------------------------------------------------------

# Create path
if not remote.is_directory(remote_backup_galaxy_path): remote.create_directory(remote_backup_galaxy_path, recursive=True)

# -----------------------------------------------------------------

# Synchronize
示例#14
0
# Create the configuration definition
definition = ConfigurationDefinition()
definition.add_required("remote", "string", "remote host ID", choices=find_host_ids())
definition.add_flag("private", "install SKIRT from private (development) repository")

# Create the configuration
config = parse_arguments("installation_commands", definition)

# -----------------------------------------------------------------

installation_commands = defaultdict(list)

# -----------------------------------------------------------------

# Create the remote
remote = Remote()
if not remote.setup(config.remote):
    log.error("The remote host is not available")
    exit()

# -----------------------------------------------------------------

# Check the modules
modules = Modules(remote)

# -----------------------------------------------------------------

# Qt is present
if modules.paths["qt"] is not None:

    # Get the version
示例#15
0
                        "name of the remote host",
                        choices=find_host_ids())
definition.add_required("id", "positive_integer", "simulation ID")

# -----------------------------------------------------------------

# Parse the arguments into a configuration
config = parse_arguments(
    "show_simulation_log",
    definition,
    description="Show the log output of a remote SKIRT simulation")

# -----------------------------------------------------------------

# Create and setup the remote
remote = Remote()
remote.setup(config.remote)

# Open the simulation
simulation = get_simulation_for_host(config.remote, config.id)

# The name of the ski file (the simulation prefix)
ski_name = simulation.prefix()

# The path to the simulation log file
remote_log_file_path = simulation.remote_log_file_path

# Check whether the log file exists
if not remote.is_file(remote_log_file_path):
    raise RuntimeError("The log file does not exist remotely")
示例#16
0
文件: sessions.py 项目: SKIRT/PTS
from __future__ import absolute_import, division, print_function

# Import the relevant PTS classes and modules
from pts.core.basics.log import log
from pts.core.basics.configuration import ConfigurationDefinition, parse_arguments
from pts.core.remote.remote import Remote

# -----------------------------------------------------------------

definition = ConfigurationDefinition()
definition.add_required("remote", "string", "remote host ID")
setter = parse_arguments("sessions", definition)

# -----------------------------------------------------------------

remote = Remote()
if not remote.setup(config.remote): raise RuntimeError("The remote host '" + config.remote + "' is not available at the moment")

screen_names = remote.screen_names()

tmux_names = remote.tmux_names()

print("SCREEN")
print(screen_names)

print("")

print("TMUX")
print(tmux_names)

# -----------------------------------------------------------------
示例#17
0
# Import the relevant PTS classes and modules
from pts.core.basics.configuration import ConfigurationDefinition, parse_arguments
from pts.core.remote.host import find_host_ids
from pts.core.remote.remote import Remote
from pts.core.basics.log import log

# -----------------------------------------------------------------

# Create definition
definition = ConfigurationDefinition()
definition.add_positional_optional("remotes", "string_list", "remote hosts on which to clear", choices=find_host_ids(), default=find_host_ids())

# Create setter
config = parse_arguments("clear_sessions_and_temp", definition)

# -----------------------------------------------------------------

# Loop over the remote hosts
for host_id in config.remotes:

    # Setup the remote
    remote = Remote()
    if not remote.setup(host_id):
        log.warning("Could not connect to remote host '" + host_id + "'")
        continue

    # Clear temporary directory and clear sessions
    remote.clear_temp_and_sessions()

# -----------------------------------------------------------------
示例#18
0
文件: test.py 项目: SKIRT/PTS
class SourcesTest(SourcesTestBase):

    """
    This class ...
    """

    def __init__(self, *args, **kwargs):

        """
        The constructor ...
        :param kwargs:
        """

        # Call the constructor of the base class
        super(SourcesTest, self).__init__(*args, **kwargs)

        # The galaxy properties
        self.properties = None

        # The rotation masks
        self.rotation_masks = dict()

        # The catalog fetcher
        self.fetcher = CatalogFetcher()

        # Catalogs
        self.extended_source_catalog = None

        # The real FWHMs
        self.real_fwhms = dict()

    # -----------------------------------------------------------------

    def _run(self, **kwargs):

        """
        This function ...
        :param kwargs:
        :return:
        """

        # 2. Load galaxy properties
        self.load_properties()

        # 3. Load the coordinate system
        self.load_coordinate_systems()

        # 4. Initialize the frames
        self.initialize_frames()

        # 5. Set FWHMs
        self.set_fwhms()

        # 6. Make rotation mask
        self.make_rotation_masks()

        # 7. Get catalogs
        self.get_catalogs()

        # 8. Generate the sources
        self.make_sources()

        # 9. Make noise
        self.make_noise(self.rotation_masks)

        # 10. Make sky
        self.make_sky()

        # 11. Create the dataset
        self.create_dataset(self.rotation_masks)

        # 12. Create the directories
        self.create_directories()

        # 13. Find sources
        self.find()

        # 14. Extract sources
        self.extract()

        # 15. Write
        self.write()

        # 16. Plot
        self.plot()

    # -----------------------------------------------------------------

    def setup(self, **kwargs):

        """
        This function ...
        :param kwargs:
        :return:
        """

        # Call the setup function of the base class
        super(SourcesTest, self).setup(**kwargs)

        # Set remote
        if self.config.remote is not None:
            self.remote = Remote()
            self.remote.setup(self.config.remote)

    # -----------------------------------------------------------------

    def load_properties(self):

        """
        This function ...
        :return:
        """

        # Inform the user
        log.info("Loading the galaxy properties ...")

        # Determine the path
        path = fs.join(m81_data_path, "properties.dat")

        # Load
        self.properties = GalaxyProperties.from_file(path)

    # -----------------------------------------------------------------

    def load_coordinate_systems(self):

        """
        This function ...
        :return:
        """

        # Inform the user
        log.info("Loading the coordinate systems ...")

        nfilters_stars = 0
        nfilters_extra = 0

        # Loop over the header files
        for path, name in fs.files_in_path(headers_path, extension="txt", returns=["path", "name"]):

            # Get the filter and wavelength
            fltr = parse_filter(name)
            wavelength = fltr.effective if fltr.effective is not None else fltr.center

            # SKip Planck
            if fltr.observatory == "Planck": continue

            # Wavelength greater than 25 micron
            if wavelength > wavelengths.ranges.ir.mir.max:

                if nfilters_extra == self.config.nfilters_extra: continue
                else: nfilters_extra += 1

            # Wavelength smaller than 25 micron
            else:

                if nfilters_stars == self.config.nfilters_stars: continue
                else: nfilters_stars += 1

            # Debugging
            log.debug("Loading the coordinate system for the '" + str(fltr) + "' filter ...")

            # Get WCS
            wcs = CoordinateSystem.from_header_file(path)

            # Add the coordinate system
            self.coordinate_systems.append(wcs, fltr=fltr)

            # Break the loop if we have enough
            if nfilters_stars == self.config.nfilters_stars and nfilters_extra == self.config.nfilters_extra: break

    # -----------------------------------------------------------------

    def make_rotation_masks(self):

        """
        This function ...
        :return:
        """

        # Inform the user
        log.info("Making rotation masks ...")

        # Loop over the filters
        for fltr in self.frames:

            # Debugging
            log.info("Making rotation mask for the '" + str(fltr) + "' image ...")

            # Get the frame
            frame = self.frames[fltr]

            # Rotate
            if self.config.rotate:

                # Choose a random rotation angle
                angle = Angle(np.random.uniform(-90, 90), "deg")

                # Debugging
                log.debug("The random rotation angle is '" + stringify.stringify(angle)[1] + "'")

                # Create mask
                mask = frame.rotation_mask(angle)

            # Don't rotate
            else: mask = Mask.empty_like(frame)

            # Set the mask
            self.rotation_masks[fltr] = mask

    # -----------------------------------------------------------------

    def get_catalogs(self):

        """
        This function ...
        :return:
        """

        # Inform the user
        log.info("Getting catalogs of point and extended sources ...")

        # Adding catalogued sources
        if self.config.add_catalogued_sources: self.fetch_catalogs()
        else: self.initialize_catalogs()

        # Add random sources
        self.create_random_sources()

    # -----------------------------------------------------------------

    def fetch_catalogs(self):

        """
        This function ...
        :return:
        """

        # Inform the user
        log.info("Fetching catalogs ...")

        # Get extended source catalog
        self.extended_source_catalog = self.fetcher.get_extended_source_catalog(self.coordinate_systems.bounding_box)

        # Fetch
        self.point_source_catalog = self.fetcher.get_point_source_catalog(self.coordinate_systems.bounding_box, self.coordinate_systems.min_pixelscale, self.config.point_source_catalogs)

    # -----------------------------------------------------------------

    def initialize_catalogs(self):

        """
        This function ...
        :param self:
        :return:
        """

        # Inform the user
        log.info("Initializing catalogs ...")

        # Extended sources
        self.extended_source_catalog = ExtendedSourceCatalog()

        # Add principal galaxy
        self.add_principal_galaxy()

        # Point sources
        self.point_source_catalog = PointSourceCatalog()

    # -----------------------------------------------------------------

    def add_principal_galaxy(self):

        """
        This function ...
        :return:
        """

        # Inform the user
        log.info("Adding principal galaxy ...")

        # Get info
        name, position, redshift, galaxy_type, names, distance, inclination, d25, major, minor, pa = get_galaxy_info("M81", self.properties.center)

        ra = position.ra
        dec = position.dec

        principal = True
        companions = []
        parent = None

        # Add to the catalog
        self.extended_source_catalog.add_entry("M81", ra, dec, redshift, galaxy_type, names, distance, inclination, d25, major, minor, pa, principal, companions, parent)

    # -----------------------------------------------------------------

    def make_sources(self):

        """
        This function ...
        :return:
        """

        # Inform the user
        log.info("Making sources ...")

        # Extended
        self.make_extended_sources()

        # Point
        self.make_point_sources()

    # -----------------------------------------------------------------

    def make_extended_sources(self):

        """
        This function ...
        :return:
        """

        # Inform the user
        log.info("Making extended sources ...")

        # Loop over the filters
        for fltr in self.frames:

            # Get the frame
            frame = self.frames[fltr]

            # Loop over the sources
            for index in range(len(self.extended_source_catalog)):

                principal = self.extended_source_catalog["Principal"][index]

                # Determine flux
                if principal: central_flux = 1e2
                else: central_flux = np.random.uniform(10,50)

                # Get pixel position
                coordinate = self.extended_source_catalog.get_position(index).to_pixel(frame.wcs)

                initial_sersic_amplitude = central_flux
                initial_sersic_x_0 = coordinate.x
                initial_sersic_y_0 = coordinate.y

                if principal:

                    s4g_name, pa, ellipticity, n, re, mag = catalogs.get_galaxy_s4g_one_component_info("M81")

                    angle = Angle(pa, "deg")
                    angle_deg = pa

                    effective_radius = re.to("arcsec").value / frame.average_pixelscale.to("arcsec").value

                    initial_sersic_n = n
                    initial_sersic_r_eff = effective_radius
                    initial_sersic_ellip = ellipticity
                    initial_sersic_theta = np.deg2rad(angle_deg)

                    # 1 / axial_ratio = 1 - ellipticity
                    axial_ratio = 1. / (1. - ellipticity)

                else:

                    # Get position angle and axes lengths
                    pa = self.extended_source_catalog["Posangle"][index]
                    major = self.extended_source_catalog["Major"][index]
                    minor = self.extended_source_catalog["Minor"][index]
                    axial_ratio = major / minor

                    angle = Angle(pa, "deg")
                    angle_deg = pa

                    effective_radius = major

                    # Produce guess values
                    initial_sersic_r_eff = effective_radius
                    initial_sersic_n = self.config.galaxy_sersic_index
                    initial_sersic_ellip = (axial_ratio - 1.0) / axial_ratio
                    initial_sersic_theta = np.deg2rad(angle_deg)

                # Produce sersic model from guess parameters, for time trials
                sersic_x, sersic_y = np.meshgrid(np.arange(frame.xsize), np.arange(frame.ysize))
                sersic_model = Sersic2D(amplitude=initial_sersic_amplitude, r_eff=initial_sersic_r_eff,
                                        n=initial_sersic_n, x_0=initial_sersic_x_0,
                                        y_0=initial_sersic_y_0, ellip=initial_sersic_ellip,
                                        theta=initial_sersic_theta)

                sersic_map = sersic_model(sersic_x, sersic_y)

                # Limit galaxy?
                if self.config.limit_galaxy:

                    limit_radius = self.config.galaxy_relative_asymptotic_radius * effective_radius

                    # Create galaxy region
                    galaxy_center = PixelCoordinate(initial_sersic_x_0, initial_sersic_y_0)
                    galaxy_radius = PixelStretch(limit_radius, limit_radius / axial_ratio)
                    galaxy_region = PixelEllipseRegion(galaxy_center, galaxy_radius, angle)

                    galaxy_mask = galaxy_region.to_mask(frame.xsize, frame.ysize)

                    # Set galaxy map zero outside certain radius
                    sersic_map[galaxy_mask.inverse()] = 0.0

                # Add
                frame += sersic_map

            # mask
            frame[self.rotation_masks[fltr]] = 0.0

    # -----------------------------------------------------------------

    def make_point_sources(self):

        """
        This function ...
        :return:
        """

        # Inform the user
        log.info("Making point sources ...")

        # Call the appropriate function
        if self.config.vary_fwhm: self.make_point_sources_variable_fwhm(self.rotation_masks)
        else: self.make_point_sources_fixed_fwhm(self.rotation_masks)

    # -----------------------------------------------------------------

    def make_sky(self):

        """
        This function ...
        :return:
        """

        # Inform the user
        log.info("Adding sky ...")

        # Loop over the frames
        for fltr in self.frames:

            # Get the frame
            frame = self.frames[fltr]

            # Determine random sky level
            sky_level = np.random.uniform(0.0, 10.)

            # Create sky gradient
            y, x = np.mgrid[:frame.ysize, :frame.xsize]
            sky_gradient =  x * y

            # Normalize so that the maximal sky is 20%
            sky_gradient = sky_gradient / np.max(sky_gradient) * 20

            # Add the sky
            frame += sky_level + sky_gradient

            # Mask
            frame[self.rotation_masks[fltr]] = 0.0

    # -----------------------------------------------------------------

    def make_galaxy(self):

        """
        This function ...
        :return:
        """

        # Inform the user
        log.info("Adding galaxy ...")

    # -----------------------------------------------------------------

    def find(self):

        """
        This function ...
        :return:
        """

        # Inform the user
        log.info("Finding the sources ...")

        # Settings
        settings = dict()
        #settings["input"] =
        settings["output"] = self.find_path
        settings["nprocesses"] = self.config.nprocesses

        # Input
        input_dict = dict()
        input_dict["dataset"] = self.dataset
        input_dict["extended_source_catalog"] = self.extended_source_catalog
        input_dict["point_source_catalog"] = self.point_source_catalog
        input_dict["output_paths"] = self.find_paths

        # Construct the command
        command = Command("find_sources", "find sources", settings, input_dict)

        # Run the command
        self.finder = self.run_command(command, remote=self.remote)

    # -----------------------------------------------------------------

    def write(self):

        """
        This function ...
        :return:
        """

        # Inform the user
        log.info("Writing ...")

    # -----------------------------------------------------------------

    def plot(self):

        """
        This function ...
        :return:
        """

        # Inform the user
        log.info("Plotting ...")
示例#19
0
definition = ConfigurationDefinition()
definition.add_required("local_path", "string",
                        "path or name of the file or directory to send")
definition.add_required("remote",
                        "string",
                        "the remote host to send to",
                        choices=find_host_ids())
definition.add_optional("remote_path", "string",
                        "path of the remote directory to send to")
config = parse_arguments("send", definition)

# -----------------------------------------------------------------

# Create remote
remote = Remote(host_id=config.remote)

# -----------------------------------------------------------------

# Set full path of origin
origin = fs.absolute_or_in_cwd(config.local_path)
name = fs.name(origin)

# Set full path to the destination
if config.path is not None: destination = fs.join(remote.home_directory, name)
else: destination = remote.absolute_path(config.remote_path)

# Upload
remote.upload(origin, destination)

# -----------------------------------------------------------------
示例#20
0
# Simulation is retrieved
if simulation.retrieved:

    # Determine the path to the simulation log file
    local_log_file_path = simulation.log_file_path

    # Read the log file
    lines = fs.read_lines(local_log_file_path)

# Not yet retrieved
else:

    # The path to the simulation log file
    remote_log_file_path = simulation.remote_log_file_path

    # Create and setup the remote
    remote = Remote()
    remote.setup(config.remote)

    # Check whether the log file exists
    if not remote.is_file(remote_log_file_path): raise RuntimeError("The log file does not (yet) exist remotely")

    # Read the log file
    lines = remote.read_lines(remote_log_file_path)

# Print the lines of the log file
for line in lines: print(line)

# -----------------------------------------------------------------
示例#21
0
# Get configuration
config = parse_arguments("plot_truncated", definition)

# -----------------------------------------------------------------

modeling_path = verify_modeling_cwd()

# -----------------------------------------------------------------

# Load the modeling environment
environment = GalaxyModelingEnvironment(modeling_path)

# -----------------------------------------------------------------

# Setup the remote
remote = Remote(host_id=environment.cache_host_id)

# -----------------------------------------------------------------

directory_name = cached_directory_name_for_single_command(
    environment, "truncate")
remote_truncation_path = fs.join(remote.home_directory, directory_name)

# -----------------------------------------------------------------

filter_name = str(config.filter)
remote_truncation_path_filter = fs.join(remote_truncation_path, filter_name)
if not remote.is_directory(remote_truncation_path_filter):
    raise ValueError("Could not find cached data for the " + filter_name +
                     " image")
示例#22
0
文件: open.py 项目: rag9704/PTS
from pts.core.basics.log import log

# -----------------------------------------------------------------

# Create the configuration definition
definition = ConfigurationDefinition()
definition.add_required("remote", "string", "remote host", choices=find_host_ids())
definition.add_required("filename", "string", "file name (or path)")

# Read the command line arguments
config = parse_arguments("open", definition, description="Open a file on a remote host")

# -----------------------------------------------------------------

# Connect to remote
remote = Remote()
remote.setup(config.remote)

# Determine the absolute file path
filepath = remote.absolute_path(config.filename)

# Determine the file path relative to the home directory
relative_filepath = remote.relative_to_home(filepath)

# Check if the file exists; otherwise don't bother mounting
if not remote.is_file(filepath):
    log.error("The file does not exist")
    exit()

# Disconnect the remote
remote.logout()
示例#23
0
definition.add_required("remote", "string", "remote host to mount", choices=all_host_ids())
definition.add_required("id", "positive_integer", "simulation ID")
definition.add_flag("debug_output", "show all simulation output in debug mode")

# Read the command line arguments
config = parse_arguments("show_progress", definition, description="Show the progress of a remotely running simulation")

# -----------------------------------------------------------------

# Load the simulation
simulation = get_simulation_for_host(config.remote, config.id)

# -----------------------------------------------------------------

# Load the remote
remote = Remote()
if not remote.setup(host_id=config.remote): raise RuntimeError("Could not connect to the remote host")

# -----------------------------------------------------------------

# Create the status object
status = LogSimulationStatus(simulation.remote_log_file_path, remote=remote, debug_output=config.debug_output)

# Show the simulation progress
with log.no_debugging(): success = status.show_progress(simulation.handle)

# Check whether not crashed
if not success: raise RuntimeError("The simulation crashed")

# -----------------------------------------------------------------
示例#24
0
# Create configuration
config = parse_arguments("cache_images", definition)

# -----------------------------------------------------------------

modeling_path = verify_modeling_cwd()

# -----------------------------------------------------------------

# Load the modeling environment
environment = GalaxyModelingEnvironment(modeling_path)

# -----------------------------------------------------------------

# Setup the remote
remote = Remote(host_id=environment.cache_host_id)

# -----------------------------------------------------------------

remote_data_path = cached_directory_path_for_single_command(
    environment, "initialize_preparation", remote)
if not remote.is_directory(remote_data_path):
    remote.create_directory(remote_data_path)

# -----------------------------------------------------------------

# Reset?
if config.reset:

    # Loop over all FITS files in the remote data directory
    for name, path in remote.files_in_path(remote_data_path,
示例#25
0
文件: find_files.py 项目: rag9704/PTS
# Add flags
definition.add_flag("recursive", "search recursively", False)
definition.add_flag("full", "show the full paths", False)

# -----------------------------------------------------------------

# Parse the arguments into a configuration
config = parse_arguments("find_files", definition, description="Find files containing a certain string in the current working directory")

# -----------------------------------------------------------------

# REMOTELY
if config.remote is not None:

    # Create remote
    remote = Remote(host_id=config.remote)

    # Determine path
    if config.remote_path is not None: find_path = remote.absolute_or_in_home(config.remote_path)
    else: find_path = remote.home_directory

    #print(find_path)

    #print(remote.items_in_path(find_path, recursive=True))

    # Loop over the files
    paths = remote.files_in_path(find_path, contains=config.contains, not_contains=config.not_contains, extension=config.extension, recursive=config.recursive)

    if len(paths) == 0: log.warning("No files found")
    else:
        for path in paths:
示例#26
0
# Add flags
definition.add_flag("conda", "also remove conda installation")
definition.add_flag("qt", "also remove Qt installation")
definition.add_flag("one_attempt", "only perform one attempt at connecting to a remote")

# Get the config
config = parse_arguments("deinstall_all", definition)

# -----------------------------------------------------------------

# Loop over the remote hosts
for host_id in find_host_ids():

    # Setup
    remote = Remote()
    if not remote.setup(host_id, one_attempt=config.one_attempt):
        log.warning("Remote host '" + host_id + "' is offline")
        continue

    # Create uninstaller
    uninstaller = Uninstaller()

    # Set options
    uninstaller.config.skirt_and_or_pts = config.skirt_and_or_pts
    uninstaller.config.conda = config.conda
    uninstaller.config.qt = config.qt

    # Inform the user
    log.info("Running the uninstaller for remote host '" + host_id + "' ...")
示例#27
0
from pts.core.basics.log import log

# -----------------------------------------------------------------

# Create definition
definition = ConfigurationDefinition()
definition.add_positional_optional("remotes",
                                   "string_list",
                                   "remote hosts on which to clear",
                                   choices=find_host_ids(),
                                   default=find_host_ids())

# Create setter
config = parse_arguments("clear_sessions_and_temp", definition)

# -----------------------------------------------------------------

# Loop over the remote hosts
for host_id in config.remotes:

    # Setup the remote
    remote = Remote()
    if not remote.setup(host_id):
        log.warning("Could not connect to remote host '" + host_id + "'")
        continue

    # Clear temporary directory and clear sessions
    remote.clear_temp_and_sessions()

# -----------------------------------------------------------------
示例#28
0
文件: clear_temp.py 项目: SKIRT/PTS
# Create definition
definition = ConfigurationDefinition()
definition.add_positional_optional("remote", "string", "remote host on which to clear the temporary directory", choices=find_host_ids())
definition.add_optional("names", "string_list", "remove temporary directories matching these names (e.g. 'installation' matches 'installation_2017-05-03--08-45-44-410', 'installation_2017-05-03--13-52-28-371', etc. and also exact matches")

# Create setter
config = parse_arguments("clear_temp", definition)

# -----------------------------------------------------------------

# Remote
if config.remote is not None:

    # Create the remote
    remote = Remote(host_id=config.remote)

    # Check whether names are given
    if config.names is not None:

        # Loop over the directories in the temporary directory
        for path, name in remote.directories_in_path(remote.pts_temp_path, returns=["path", "name"]):

            # Contains timestamp
            if time.is_unique_name(name):

                # Without timestamp in list
                basename = time.get_name_from_unique_name(name)
                if basename in config.names: remote.remove_directory(path)

                # Full name in list
示例#29
0
文件: test.py 项目: rag9704/PTS
class SourcesTest(SourcesTestBase):
    """
    This class ...
    """
    def __init__(self, *args, **kwargs):
        """
        The constructor ...
        :param kwargs:
        """

        # Call the constructor of the base class
        super(SourcesTest, self).__init__(*args, **kwargs)

        # The galaxy properties
        self.properties = None

        # The rotation masks
        self.rotation_masks = dict()

        # The catalog fetcher
        self.fetcher = CatalogFetcher()

        # Catalogs
        self.extended_source_catalog = None

        # The real FWHMs
        self.real_fwhms = dict()

    # -----------------------------------------------------------------

    def run(self, **kwargs):
        """
        This function ...
        :param kwargs:
        :return:
        """

        # 1. Call the setup function
        self.setup(**kwargs)

        # 2. Load galaxy properties
        self.load_properties()

        # 3. Load the coordinate system
        self.load_coordinate_systems()

        # 4. Initialize the frames
        self.initialize_frames()

        # 5. Set FWHMs
        self.set_fwhms()

        # 6. Make rotation mask
        self.make_rotation_masks()

        # 7. Get catalogs
        self.get_catalogs()

        # 8. Generate the sources
        self.make_sources()

        # 9. Make noise
        self.make_noise(self.rotation_masks)

        # 10. Make sky
        self.make_sky()

        # 11. Create the dataset
        self.create_dataset(self.rotation_masks)

        # 12. Create the directories
        self.create_directories()

        # 13. Find sources
        self.find()

        # 14. Extract sources
        self.extract()

        # 15. Write
        self.write()

        # 16. Plot
        self.plot()

    # -----------------------------------------------------------------

    def setup(self, **kwargs):
        """
        This function ...
        :param kwargs:
        :return:
        """

        # Call the setup function of the base class
        super(SourcesTest, self).setup(**kwargs)

        # Set remote
        if self.config.remote is not None:
            self.remote = Remote()
            self.remote.setup(self.config.remote)

    # -----------------------------------------------------------------

    def load_properties(self):
        """
        This function ...
        :return:
        """

        # Inform the user
        log.info("Loading the galaxy properties ...")

        # Determine the path
        path = fs.join(m81_data_path, "properties.dat")

        # Load
        self.properties = GalaxyProperties.from_file(path)

    # -----------------------------------------------------------------

    def load_coordinate_systems(self):
        """
        This function ...
        :return:
        """

        # Inform the user
        log.info("Loading the coordinate systems ...")

        nfilters_stars = 0
        nfilters_extra = 0

        # Loop over the header files
        for path, name in fs.files_in_path(headers_path,
                                           extension="txt",
                                           returns=["path", "name"]):

            # Get the filter and wavelength
            fltr = parse_filter(name)
            wavelength = fltr.effective if fltr.effective is not None else fltr.center

            # SKip Planck
            if fltr.observatory == "Planck": continue

            # Wavelength greater than 25 micron
            if wavelength > wavelengths.ranges.ir.mir.max:

                if nfilters_extra == self.config.nfilters_extra: continue
                else: nfilters_extra += 1

            # Wavelength smaller than 25 micron
            else:

                if nfilters_stars == self.config.nfilters_stars: continue
                else: nfilters_stars += 1

            # Debugging
            log.debug("Loading the coordinate system for the '" + str(fltr) +
                      "' filter ...")

            # Get WCS
            wcs = CoordinateSystem.from_header_file(path)

            # Add the coordinate system
            self.coordinate_systems.append(wcs, fltr=fltr)

            # Break the loop if we have enough
            if nfilters_stars == self.config.nfilters_stars and nfilters_extra == self.config.nfilters_extra:
                break

    # -----------------------------------------------------------------

    def make_rotation_masks(self):
        """
        This function ...
        :return:
        """

        # Inform the user
        log.info("Making rotation masks ...")

        # Loop over the filters
        for fltr in self.frames:

            # Debugging
            log.info("Making rotation mask for the '" + str(fltr) +
                     "' image ...")

            # Get the frame
            frame = self.frames[fltr]

            # Rotate
            if self.config.rotate:

                # Choose a random rotation angle
                angle = Angle(np.random.uniform(-90, 90), "deg")

                # Debugging
                log.debug("The random rotation angle is '" +
                          stringify.stringify(angle)[1] + "'")

                # Create mask
                mask = frame.rotation_mask(angle)

            # Don't rotate
            else:
                mask = Mask.empty_like(frame)

            # Set the mask
            self.rotation_masks[fltr] = mask

    # -----------------------------------------------------------------

    def get_catalogs(self):
        """
        This function ...
        :return:
        """

        # Inform the user
        log.info("Getting catalogs of point and extended sources ...")

        # Adding catalogued sources
        if self.config.add_catalogued_sources: self.fetch_catalogs()
        else: self.initialize_catalogs()

        # Add random sources
        self.create_random_sources()

    # -----------------------------------------------------------------

    def fetch_catalogs(self):
        """
        This function ...
        :return:
        """

        # Inform the user
        log.info("Fetching catalogs ...")

        # Get extended source catalog
        self.extended_source_catalog = self.fetcher.get_extended_source_catalog(
            self.coordinate_systems.bounding_box)

        # Fetch
        self.point_source_catalog = self.fetcher.get_point_source_catalog(
            self.coordinate_systems.bounding_box,
            self.coordinate_systems.min_pixelscale,
            self.config.point_source_catalogs)

    # -----------------------------------------------------------------

    def initialize_catalogs(self):
        """
        This function ...
        :param self:
        :return:
        """

        # Inform the user
        log.info("Initializing catalogs ...")

        # Extended sources
        self.extended_source_catalog = ExtendedSourceCatalog()

        # Add principal galaxy
        self.add_principal_galaxy()

        # Point sources
        self.point_source_catalog = PointSourceCatalog()

    # -----------------------------------------------------------------

    def add_principal_galaxy(self):
        """
        This function ...
        :return:
        """

        # Inform the user
        log.info("Adding principal galaxy ...")

        # Get info
        name, position, redshift, galaxy_type, names, distance, inclination, d25, major, minor, pa = get_galaxy_info(
            "M81", self.properties.center)

        ra = position.ra
        dec = position.dec

        principal = True
        companions = []
        parent = None

        # Add to the catalog
        self.extended_source_catalog.add_entry("M81", ra, dec, redshift,
                                               galaxy_type, names, distance,
                                               inclination, d25, major, minor,
                                               pa, principal, companions,
                                               parent)

    # -----------------------------------------------------------------

    def make_sources(self):
        """
        This function ...
        :return:
        """

        # Inform the user
        log.info("Making sources ...")

        # Extended
        self.make_extended_sources()

        # Point
        self.make_point_sources()

    # -----------------------------------------------------------------

    def make_extended_sources(self):
        """
        This function ...
        :return:
        """

        # Inform the user
        log.info("Making extended sources ...")

        # Loop over the filters
        for fltr in self.frames:

            # Get the frame
            frame = self.frames[fltr]

            # Loop over the sources
            for index in range(len(self.extended_source_catalog)):

                principal = self.extended_source_catalog["Principal"][index]

                # Determine flux
                if principal: central_flux = 1e2
                else: central_flux = np.random.uniform(10, 50)

                # Get pixel position
                coordinate = self.extended_source_catalog.get_position(
                    index).to_pixel(frame.wcs)

                initial_sersic_amplitude = central_flux
                initial_sersic_x_0 = coordinate.x
                initial_sersic_y_0 = coordinate.y

                if principal:

                    s4g_name, pa, ellipticity, n, re, mag = catalogs.get_galaxy_s4g_one_component_info(
                        "M81")

                    angle = Angle(pa, "deg")
                    angle_deg = pa

                    effective_radius = re.to(
                        "arcsec").value / frame.average_pixelscale.to(
                            "arcsec").value

                    initial_sersic_n = n
                    initial_sersic_r_eff = effective_radius
                    initial_sersic_ellip = ellipticity
                    initial_sersic_theta = np.deg2rad(angle_deg)

                    # 1 / axial_ratio = 1 - ellipticity
                    axial_ratio = 1. / (1. - ellipticity)

                else:

                    # Get position angle and axes lengths
                    pa = self.extended_source_catalog["Posangle"][index]
                    major = self.extended_source_catalog["Major"][index]
                    minor = self.extended_source_catalog["Minor"][index]
                    axial_ratio = major / minor

                    angle = Angle(pa, "deg")
                    angle_deg = pa

                    effective_radius = major

                    # Produce guess values
                    initial_sersic_r_eff = effective_radius
                    initial_sersic_n = self.config.galaxy_sersic_index
                    initial_sersic_ellip = (axial_ratio - 1.0) / axial_ratio
                    initial_sersic_theta = np.deg2rad(angle_deg)

                # Produce sersic model from guess parameters, for time trials
                sersic_x, sersic_y = np.meshgrid(np.arange(frame.xsize),
                                                 np.arange(frame.ysize))
                sersic_model = Sersic2D(amplitude=initial_sersic_amplitude,
                                        r_eff=initial_sersic_r_eff,
                                        n=initial_sersic_n,
                                        x_0=initial_sersic_x_0,
                                        y_0=initial_sersic_y_0,
                                        ellip=initial_sersic_ellip,
                                        theta=initial_sersic_theta)

                sersic_map = sersic_model(sersic_x, sersic_y)

                # Limit galaxy?
                if self.config.limit_galaxy:

                    limit_radius = self.config.galaxy_relative_asymptotic_radius * effective_radius

                    # Create galaxy region
                    galaxy_center = PixelCoordinate(initial_sersic_x_0,
                                                    initial_sersic_y_0)
                    galaxy_radius = PixelStretch(limit_radius,
                                                 limit_radius / axial_ratio)
                    galaxy_region = PixelEllipseRegion(galaxy_center,
                                                       galaxy_radius, angle)

                    galaxy_mask = galaxy_region.to_mask(
                        frame.xsize, frame.ysize)

                    # Set galaxy map zero outside certain radius
                    sersic_map[galaxy_mask.inverse()] = 0.0

                # Add
                frame += sersic_map

            # mask
            frame[self.rotation_masks[fltr]] = 0.0

    # -----------------------------------------------------------------

    def make_point_sources(self):
        """
        This function ...
        :return:
        """

        # Inform the user
        log.info("Making point sources ...")

        # Call the appropriate function
        if self.config.vary_fwhm:
            self.make_point_sources_variable_fwhm(self.rotation_masks)
        else:
            self.make_point_sources_fixed_fwhm(self.rotation_masks)

    # -----------------------------------------------------------------

    def make_sky(self):
        """
        This function ...
        :return:
        """

        # Inform the user
        log.info("Adding sky ...")

        # Loop over the frames
        for fltr in self.frames:

            # Get the frame
            frame = self.frames[fltr]

            # Determine random sky level
            sky_level = np.random.uniform(0.0, 10.)

            # Create sky gradient
            y, x = np.mgrid[:frame.ysize, :frame.xsize]
            sky_gradient = x * y

            # Normalize so that the maximal sky is 20%
            sky_gradient = sky_gradient / np.max(sky_gradient) * 20

            # Add the sky
            frame += sky_level + sky_gradient

            # Mask
            frame[self.rotation_masks[fltr]] = 0.0

    # -----------------------------------------------------------------

    def make_galaxy(self):
        """
        This function ...
        :return:
        """

        # Inform the user
        log.info("Adding galaxy ...")

    # -----------------------------------------------------------------

    def find(self):
        """
        This function ...
        :return:
        """

        # Inform the user
        log.info("Finding the sources ...")

        # Settings
        settings = dict()
        #settings["input"] =
        settings["output"] = self.find_path
        settings["nprocesses"] = self.config.nprocesses

        # Input
        input_dict = dict()
        input_dict["dataset"] = self.dataset
        input_dict["extended_source_catalog"] = self.extended_source_catalog
        input_dict["point_source_catalog"] = self.point_source_catalog
        input_dict["output_paths"] = self.find_paths

        # Construct the command
        command = Command("find_sources", "find sources", settings, input_dict)

        # Run the command
        self.finder = self.run_command(command, remote=self.remote)

    # -----------------------------------------------------------------

    def write(self):
        """
        This function ...
        :return:
        """

        # Inform the user
        log.info("Writing ...")

    # -----------------------------------------------------------------

    def plot(self):
        """
        This function ...
        :return:
        """

        # Inform the user
        log.info("Plotting ...")
示例#30
0
# Create configuration definition
definition = ConfigurationDefinition()

# Add required
definition.add_required("remote", "string", "the remote host ID")

# Add flags
definition.add_flag("versions", "compare versions", "v")

# Get configuration
config = parse_arguments("compare_python_packages", definition)

# -----------------------------------------------------------------

# Create the remote execution environment
remote = Remote()

# Log in
remote.setup(config.remote)

# Get remote python session
remote_python = remote.start_python_session(assume_pts=False)

# Get all python packages installed on the remote host
remote_packages = remote_python.installed_packages

# Get local python package version
local_packages = introspection.installed_python_packages()

# Loop over all python packages necessary for PTS
for dependency in introspection.get_all_dependencies():
示例#31
0
definition = ConfigurationDefinition()
definition.add_optional("remotes",
                        "string_list",
                        "remote host ID",
                        choices=find_host_ids(),
                        default=find_host_ids())
config = parse_arguments("sessions", definition)

# -----------------------------------------------------------------

# Loop over the remote host
for host_id in config.remotes:

    # Create and setup the remote
    remote = Remote()
    if not remote.setup(host_id):
        raise RuntimeError("The remote host '" + host_id +
                           "' is not available at the moment")

    # Get screen names and tmux session names
    screen_names = remote.screen_names()
    tmux_names = remote.tmux_names()

    # Inform the user
    log.info("Closing sessions on remote host '" + host_id + "' ...")

    # Loop over the screens
    for name in screen_names:

        # Inform the user
示例#32
0
# -----------------------------------------------------------------

# Parse the arguments into a configuration
config = parse_arguments(
    "clear_tasks",
    definition,
    description="Clear PTS tasks for a certain remote host")

# -----------------------------------------------------------------

# Loop over the remote hosts
for host_id in config.remotes:

    # Check whether the remote is available
    if config.full:
        remote = Remote()
        if not remote.setup(host_id):
            log.warning("The remote host '" + host_id +
                        "' is not available: skipping ...")
            continue
    else:
        remote = None

    # Determine the path to the run directory for the specified remote host
    host_run_path = fs.join(introspection.skirt_run_dir, host_id)

    # Check if there are simulations
    if not fs.is_directory(host_run_path):
        log.debug("No run directory for host '" + host_id + "'")
        continue
    if fs.is_empty(host_run_path):
示例#33
0
# Read the command line arguments
config = parse_arguments(
    "show_progress",
    definition,
    description="Show the progress of a remotely running simulation")

# -----------------------------------------------------------------

# Load the simulation
simulation = get_simulation_for_host(config.remote, config.id)

# -----------------------------------------------------------------

# Load the remote
remote = Remote()
if not remote.setup(host_id=config.remote):
    raise RuntimeError("Could not connect to the remote host")

# -----------------------------------------------------------------

# Create the status object
status = LogSimulationStatus(simulation.remote_log_file_path,
                             remote=remote,
                             debug_output=config.debug_output)

# Show the simulation progress
if config.debug_output: success = status.show_progress(simulation.handle)
else:
    with no_debugging():
        success = status.show_progress(simulation.handle)
示例#34
0
文件: retrieve.py 项目: SKIRT/PTS
definition = ConfigurationDefinition()

# Required
definition.add_required("remote_path", "string", "remote path of the file or directory to retrieve")
definition.add_required("remote", "string", "remote host to retrieve from", choices=find_host_ids())

# Local path
definition.add_positional_optional("local_path", "string", "path of the local directory to store the file/directory")

# Create configuration
config = parse_arguments("retrieve", definition, "Retrieve a file or directory from a remote host")

# -----------------------------------------------------------------

# Create remote
remote = Remote(host_id=config.remote)

# -----------------------------------------------------------------

# Set full path of origin
origin = remote.absolute_path(config.remote_path)

# Set full path to the destination
if config.local_path is not None: destination = fs.absolute_or_in_cwd(config.local_path)
else: destination = fs.cwd()

# -----------------------------------------------------------------

# Debugging
log.debug("Origin: " + origin)
log.debug("Destination: " + destination)
示例#35
0
文件: remotes.py 项目: SKIRT/PTS
# Set log level in a special way
if config.debug: setup_log("DEBUG")
else: setup_log("ERROR")

# -----------------------------------------------------------------

# Loop over the hosts
print("")
for host in config.hosts:

    # Debugging
    log.debug("Connecting to remote host '" + host.id + "' ...")

    # Create remote, try connecting
    remote = Remote()
    connected = remote.setup(host, login_timeout=10) # only try for 10 seconds

    # Connection succeeded?
    if connected: print(fmt.green + host.id + ": up" + fmt.reset)
    else:
        print(fmt.red + host.id + ": down" + fmt.reset)
        continue

    # Show clustername
    if remote.cluster_name is not None: print("Cluster name: " + remote.cluster_name)

    print("")
    print(" - " + fmt.bold + "uses scheduling system: " + fmt.reset + str(remote.scheduler))

    # Show status
示例#36
0
# Add flags
definition.add_flag("conda", "also remove conda installation")
definition.add_flag("qt", "also remove Qt installation")
definition.add_flag("one_attempt",
                    "only perform one attempt at connecting to a remote")

# Get the config
config = parse_arguments("deinstall_all", definition)

# -----------------------------------------------------------------

# Loop over the remote hosts
for host_id in find_host_ids():

    # Setup
    remote = Remote()
    if not remote.setup(host_id, one_attempt=config.one_attempt):
        log.warning("Remote host '" + host_id + "' is offline")
        continue

    # Create uninstaller
    uninstaller = Uninstaller()

    # Set options
    uninstaller.config.skirt_and_or_pts = config.skirt_and_or_pts
    uninstaller.config.conda = config.conda
    uninstaller.config.qt = config.qt

    # Inform the user
    log.info("Running the uninstaller for remote host '" + host_id + "' ...")
示例#37
0
definition.add_positional_optional("ids", "integer_list", "the IDs of the simulations to clear")
definition.add_flag("full", "fully clear the simulations, also remove remote simulation directories")

# -----------------------------------------------------------------

# Parse the arguments into a configuration
config = parse_arguments("clear_tasks", definition, description="Clear PTS tasks for a certain remote host")

# -----------------------------------------------------------------

# Loop over the remote hosts
for host_id in config.remotes:

    # Check whether the remote is available
    if config.full:
        remote = Remote()
        if not remote.setup(host_id):
            log.warning("The remote host '" + host_id + "' is not available: skipping ...")
            continue
    else: remote = None

    # Determine the path to the run directory for the specified remote host
    host_run_path = fs.join(introspection.skirt_run_dir, host_id)

    # Check if there are simulations
    if not fs.is_directory(host_run_path):
        log.debug("No run directory for host '" + host_id + "'")
        continue
    if fs.is_empty(host_run_path): log.debug("No simulations for host '" + host_id + "'")

    # Loop over the simulation files in the run directory
示例#38
0
from pts.core.remote.remote import Remote
from pts.core.tools import filesystem as fs

# -----------------------------------------------------------------

definition = ConfigurationDefinition()
definition.add_required("remote_path", "string", "remote path of the file or directory to retrieve")
definition.add_required("remote", "string", "remote host to retrieve from", choices=find_host_ids())
definition.add_optional("local_path", "string", "path of the local directory to store the file/directory")

config = parse_arguments("retrieve", definition)

# -----------------------------------------------------------------

# Create remote
remote = Remote(host_id=config.remote)

# -----------------------------------------------------------------

# Set full path of origin
origin = remote.absolute_path(config.remote_path)

# Set full path to the destination
if config.local_path is not None: destination = fs.absolute_or_in_cwd(config.local_path)
else: destination = fs.cwd()

# -----------------------------------------------------------------

# Debugging
log.debug("Origin: " + origin)
log.debug("Destination: " + destination)
示例#39
0
                        choices=find_host_ids())
definition.add_required("id", "positive_integer", "task ID")

# -----------------------------------------------------------------

# Parse the arguments into a configuration
#setter = InteractiveConfigurationSetter("show_log", "Show the log output of a remote PTS task")
config = parse_arguments(
    "show_task_log",
    definition,
    description="Show the log output of a remote PTS task")

# -----------------------------------------------------------------

# Create and setup the remote
remote = Remote()
remote.setup(config.remote)

# Open the task
task_path = fs.join(introspection.pts_run_dir, config.remote,
                    str(config.id) + ".task")
task = Task.from_file(task_path)

# Check whether the log file is present
log_output_path = task.remote_log_path
log_path = None
for filename in remote.files_in_path(log_output_path):
    if "log" in filename:
        log_path = fs.join(log_output_path, filename)
        break
示例#40
0
文件: clear_temp.py 项目: rag9704/PTS
    choices=find_host_ids())
definition.add_optional(
    "names", "string_list",
    "remove temporary directories matching these names (e.g. 'installation' matches 'installation_2017-05-03--08-45-44-410', 'installation_2017-05-03--13-52-28-371', etc. and also exact matches"
)

# Create setter
config = parse_arguments("clear_temp", definition)

# -----------------------------------------------------------------

# Remote
if config.remote is not None:

    # Create the remote
    remote = Remote(host_id=config.remote)

    # Check whether names are given
    if config.names is not None:

        # Loop over the directories in the temporary directory
        for path, name in remote.directories_in_path(remote.pts_temp_path,
                                                     returns=["path", "name"]):

            # Contains timestamp
            if time.is_unique_name(name):

                # Without timestamp in list
                basename = time.get_name_from_unique_name(name)
                if basename in config.names: remote.remove_directory(path)
示例#41
0
文件: remotes.py 项目: rag9704/PTS
from __future__ import absolute_import, division, print_function

# Import the relevant PTS classes and modules
from pts.core.remote.host import find_host_ids
from pts.core.remote.remote import Remote
from pts.core.tools import formatting as fmt
from pts.core.basics.log import setup_log
from pts.core.basics.configuration import parse_logging_arguments

# -----------------------------------------------------------------

config = parse_logging_arguments("hosts")

# -----------------------------------------------------------------

# Set log level in a special way
if config.debug: setup_log("DEBUG")
else: setup_log("ERROR")

# -----------------------------------------------------------------

# Loop over the hosts
for host_id in find_host_ids():

    remote = Remote()
    connected = remote.setup(host_id, login_timeout=15) # only try for 15 seconds
    if connected: print(fmt.green + host_id + ": up" + fmt.reset)
    else: print(fmt.red + host_id + ": down" + fmt.reset)

# -----------------------------------------------------------------
示例#42
0
文件: send.py 项目: SKIRT/PTS
definition = ConfigurationDefinition()

# Required
definition.add_required("local_path", "string", "path or name of the file or directory to send")
definition.add_required("remote", "string", "the remote host to send to", choices=find_host_ids())

# Remote path
definition.add_positional_optional("remote_path", "string", "path of the remote directory to send to")

# Create configuration
config = parse_arguments("send", definition, "Send a file or directory to a remote host")

# -----------------------------------------------------------------

# Create remote
remote = Remote(host_id=config.remote)

# -----------------------------------------------------------------

# Set full path of origin
origin = fs.absolute_or_in_cwd(config.local_path)
name = fs.name(origin)

# Set full path to the destination
if config.remote_path is None: destination = fs.join(remote.home_directory, name)
else: destination = remote.absolute_path(config.remote_path)

# Upload
remote.upload(origin, destination)

# -----------------------------------------------------------------
示例#43
0
文件: sessions.py 项目: rag9704/PTS
from __future__ import absolute_import, division, print_function

# Import the relevant PTS classes and modules
from pts.core.basics.log import log
from pts.core.basics.configuration import ConfigurationDefinition, parse_arguments
from pts.core.remote.remote import Remote

# -----------------------------------------------------------------

definition = ConfigurationDefinition()
definition.add_required("remote", "string", "remote host ID")
setter = parse_arguments("sessions", definition)

# -----------------------------------------------------------------

remote = Remote()
if not remote.setup(config.remote):
    raise RuntimeError("The remote host '" + config.remote +
                       "' is not available at the moment")

screen_names = remote.screen_names()

tmux_names = remote.tmux_names()

print("SCREEN")
print(screen_names)

print("")

print("TMUX")
print(tmux_names)
示例#44
0
文件: open.py 项目: SKIRT/PTS
from pts.core.basics.log import log

# -----------------------------------------------------------------

# Create the configuration definition
definition = ConfigurationDefinition()
definition.add_required("remote", "string", "remote host", choices=find_host_ids())
definition.add_required("filename", "string", "file name (or path)")

# Read the command line arguments
config = parse_arguments("open", definition, description="Open a file on a remote host")

# -----------------------------------------------------------------

# Connect to remote
remote = Remote()
remote.setup(config.remote)

# Determine the absolute file path
filepath = remote.absolute_path(config.filename)

# Determine the file path relative to the home directory
relative_filepath = remote.relative_to_home(filepath)

# Check if the file exists; otherwise don't bother mounting
if not remote.is_file(filepath):
    log.error("The file does not exist")
    exit()

# Disconnect the remote
remote.logout()