#!/usr/bin/env python
# -*- coding: utf8 -*-
# *****************************************************************
# ** PTS -- Python Toolkit for working with SKIRT **
# ** © Astronomical Observatory, Ghent University **
# *****************************************************************
# Import the relevant PTS classes and modules
from pts.core.basics.configuration import ConfigurationDefinition
from pts.core.remote.host import find_host_ids
from pts.core.launch.analyser import all_steps
from pts.core.config.analyse_simulation import definition as analysis_definition
# -----------------------------------------------------------------
host_ids = find_host_ids()
# -----------------------------------------------------------------
# Create the configuration definition
definition = ConfigurationDefinition()
# Assignment file
definition.add_positional_optional("assignment", "file_path", "path of assignment file")
# Status file
definition.add_optional("status", "file_path", "path of status file")
# Remotes for which to find corresponding simulations
definition.add_optional("remotes", "string_list", "remote hosts for which to look for matching simulations (not necessary when assignment is specified)", default=host_ids, choices=host_ids)
modeling_path = verify_modeling_cwd()
# -----------------------------------------------------------------
definition = definition.copy()
# Add required arguments
definition.add_positional_optional("image", "string", "name of the image for which to run the preparation")
# Add optional arguments
definition.add_optional("exclude_filters", "string_list", "exclude the data for these filters from the procedure that brings all data to the same resolution and pixelscale")
definition.add_flag("steps", "write the results of intermediate steps")
definition.add_flag("visualise", "make visualisations")
# Remote preparation
definition.add_optional("remote", "string", "remote host on which to run the preparation", choices=find_host_ids())
definition.add_flag("attached", "run remotely in attached mode")
# Parallelization
definition.add_optional("nprocesses", "positive_integer", "number of parallel processes for parallel computations", max(8, ncores()))
# Advanced
definition.add_flag("dustpedia_aperture", "use the DustPedia aperture instead of the galaxy region for the sky subtraction", True)
definition.add_optional("aperture_galaxy_region_factor", "real", "if the galaxy region is used for the aperture, use this scaling factor on the region", 1.5)
# Sky subtraction
definition.add_optional("sky_estimation_method", "string", "sky estimation method", "photutils")
definition.add_optional("sky_interpolation_method", "string", "interpolation method for photutils sky frame", "polynomial")
definition.add_optional("noise_interpolation_method", "string", "interpolation method for photutils noise frame", "idw")
definition.add_optional("sky_photutils_polynomial_order", "positive_integer", "blabla", 3)
definition.add_optional("sky_photutils_polynomial_fitter", "string", "fitter", "levenberg-marquardt")
#!/usr/bin/env python
# -*- coding: utf8 -*-
# *****************************************************************
# ** PTS -- Python Toolkit for working with SKIRT **
# ** © Astronomical Observatory, Ghent University **
# *****************************************************************
# Import the relevant PTS classes and modules
from pts.core.basics.configuration import ConfigurationDefinition
from pts.core.remote.host import find_host_ids
# -----------------------------------------------------------------
# Create definition
definition = ConfigurationDefinition()
# Add optional settings
definition.add_positional_optional("remote", "string", "remote host on which to uninstall", choices=find_host_ids())
definition.add_positional_optional("skirt_and_or_pts", "string_list", "SKIRT and/or PTS", default=["skirt", "pts"], choices=["skirt", "pts"])
# Add flags
definition.add_flag("conda", "also remove conda installation")
definition.add_flag("qt", "also remove Qt installation")
# -----------------------------------------------------------------
modeling_path = verify_modeling_cwd()
# -----------------------------------------------------------------
# Create the configuration definition
definition = ConfigurationDefinition(log_path="log", config_path="config")
# Add optional arguments
definition.add_positional_optional("image", "string", "the name of the image for which to run the initialization")
# Add flags
definition.add_flag("visualise", "make visualisations")
# Remote source detection
definition.add_optional("remote", "string", "remote host on which to run the source finder", choices=find_host_ids())
definition.add_flag("attached", "run remotely in attached mode")
# Add section for the source finder
definition.import_section("sources", "options for the source finder", sources_definition)
# Flags
default_catalogs = ["II/246"]
definition.add_optional("catalogs", "string_list", "catalogs for point sources", default_catalogs)
definition.add_flag("catalog_overlapping", "only fetch catalog data in the area where all images are overlapping", True)
definition.add_flag("manual", "don't find sources, but mark them from the catalog, and let the selection be done manually", False)
# Cache
cache_host_id = get_cache_host_id(modeling_path)
if cache_host_id is not None: definition.add_flag("cache", "cache image data for which the initialized image has been created", False)
# -*- coding: utf8 -*-
# *****************************************************************
# ** PTS -- Python Toolkit for working with SKIRT **
# ** © Astronomical Observatory, Ghent University **
# *****************************************************************
# Import the relevant PTS classes and modules
from pts.core.remote.host import find_host_ids
from pts.modeling.config.component import definition
# -----------------------------------------------------------------
definition = definition.copy()
# Add option for the remote host ID
definition.add_positional_optional("remote", "string", "remote host to use for the aperture correction calculation", choices=find_host_ids(schedulers=False))
# Add option
noise_methods = ["caapr", "pts"]
definition.add_optional("noise_method", "string", "method to use for the aperture noise calculation", choices=noise_methods, default="caapr")
# -----------------------------------------------------------------
definition.add_flag("plot", "plot SEDs", True)
# -----------------------------------------------------------------
definition.add_optional("reprocess", "lazy_broad_band_filter_list", "reprocess images and calculate fluxes for certain filters")
# -----------------------------------------------------------------
from __future__ import absolute_import, division, print_function
# Import the relevant PTS classes and modules
from pts.core.remote.host import find_host_ids
from pts.core.remote.remote import Remote
from pts.core.tools import formatting as fmt
from pts.core.basics.log import setup_log
from pts.core.basics.configuration import parse_logging_arguments
# -----------------------------------------------------------------
config = parse_logging_arguments("hosts")
# -----------------------------------------------------------------
# Set log level in a special way
if config.debug: setup_log("DEBUG")
else: setup_log("ERROR")
# -----------------------------------------------------------------
# Loop over the hosts
for host_id in find_host_ids():
remote = Remote()
connected = remote.setup(host_id, login_timeout=15) # only try for 15 seconds
if connected: print(fmt.green + host_id + ": up" + fmt.reset)
else: print(fmt.red + host_id + ": down" + fmt.reset)
# -----------------------------------------------------------------
# ** PTS -- Python Toolkit for working with SKIRT **
# ** © Astronomical Observatory, Ghent University **
# *****************************************************************
# Import the relevant PTS classes and modules
from pts.core.remote.host import find_host_ids
from pts.core.basics.configuration import ConfigurationDefinition
from pts.core.tools import introspection
# -----------------------------------------------------------------
# Create the configuration definition
definition = ConfigurationDefinition()
# Add optional
host_ids = find_host_ids()
if len(host_ids) > 0:
definition.add_positional_optional("host_ids",
"string_list",
"remote host ids",
choices=host_ids,
default=host_ids)
else:
definition.add_fixed("host_ids", "remote host_ids", [])
# Add optional
definition.add_optional("pts_repo_name",
"string",
"PTS repository name to deploy remotely",
"origin",
choices=introspection.pts_git_remotes())
#!/usr/bin/env python
# -*- coding: utf8 -*-
# *****************************************************************
# ** PTS -- Python Toolkit for working with SKIRT **
# ** © Astronomical Observatory, Ghent University **
# *****************************************************************
from pts.core.basics.configuration import ConfigurationDefinition
from pts.core.remote.host import find_host_ids
from pts.modeling.modeling.base import fitting_methods, default_fitting_method
# -----------------------------------------------------------------
types = dict()
types["galaxy"] = "Create a 3D model of a galaxy based on image data"
types["sed"] = "Model any object based on observed SED data"
types["images"] = "Model any object based on observational images and a ski file"
# -----------------------------------------------------------------
# Create the configuration
definition = ConfigurationDefinition(write_config=False)
# Add required settings
definition.add_required("type", "string", "type of modeling", choices=types)
definition.add_required("name", "string", "name given to the object")
definition.add_optional("fitting_method", "string", "fitting method", default_fitting_method, choices=fitting_methods)
definition.add_optional("fitting_host_ids", "string_list", "remote hosts to use for performing simulations as part of the fitting", choices=find_host_ids())
# -----------------------------------------------------------------
# -----------------------------------------------------------------
# Ensure Python 3 compatibility
from __future__ import absolute_import, division, print_function
# Import the relevant PTS classes and modules
from pts.core.basics.configuration import ConfigurationDefinition, parse_arguments
from pts.core.remote.host import find_host_ids
from pts.core.remote.remote import Remote
from pts.core.basics.log import log
# -----------------------------------------------------------------
# Create definition
definition = ConfigurationDefinition()
definition.add_positional_optional("remotes", "string_list", "remote hosts on which to clear", choices=find_host_ids(), default=find_host_ids())
# Create setter
config = parse_arguments("clear_sessions_and_temp", definition)
# -----------------------------------------------------------------
# Loop over the remote hosts
for host_id in config.remotes:
# Setup the remote
remote = Remote()
if not remote.setup(host_id):
log.warning("Could not connect to remote host '" + host_id + "'")
continue
#!/usr/bin/env python
# -*- coding: utf8 -*-
# *****************************************************************
# ** PTS -- Python Toolkit for working with SKIRT **
# ** © Astronomical Observatory, Ghent University **
# *****************************************************************
# Import the relevant PTS classes and modules
from pts.modeling.config.fetch import definition
from pts.core.remote.host import find_host_ids
# -----------------------------------------------------------------
# Add required settings
definition.add_positional_optional("remote", "string", "remote host to use for creating the GALEX and SDSS data", choices=find_host_ids(schedulers=False))
definition.add_flag("attached", "run remote in attached mode", False)
# Add flags
definition.add_flag("errors", "also download the error frames from the DustPedia archive")
# H alpha
definition.add_optional("halpha_url", "url", "url of the h-alpha image") # optional for when fetch_images is relaunched, and the Halpha image is already present
definition.add_optional("halpha_flux", "quantity", "flux of the h-alpha image to which the image should be rescaled")
# The total H-alpha flux (reference: FAR-ULTRAVIOLET AND Ha IMAGING OF NEARBY SPIRAL GALAXIES: THE OB STELLAR,
# POPULATION IN THE DIFFUSE IONIZED GAS (Hoopes et. al 2001)
# Flags
definition.add_flag("make_poisson", "perform the poisson error mosaicing", True)
# Advanced
definition.add_optional("max_nobservations_mosaic", "positive_integer", "maximum number of observations to use for the mosaic and poisson frames (GALEX and SDSS)")
#!/usr/bin/env python
# -*- coding: utf8 -*-
# *****************************************************************
# ** PTS -- Python Toolkit for working with SKIRT **
# ** © Astronomical Observatory, Ghent University **
# *****************************************************************
# Import the relevant PTS classes and modules
from pts.core.remote.host import find_host_ids, find_hosts
from pts.core.basics.configuration import ConfigurationDefinition
from pts.core.tools import introspection
# -----------------------------------------------------------------
all_host_ids = find_host_ids()
all_hosts = find_hosts()
nhosts = len(all_hosts)
has_hosts = nhosts > 0
# -----------------------------------------------------------------
# Create the configuration definition
definition = ConfigurationDefinition()
# The remote hosts
if has_hosts: definition.add_positional_optional("hosts", "host_list", "remote hosts", choices=all_host_ids, default=all_hosts)
else: definition.add_fixed("hosts", "remote hosts", [])
# Add optional
definition.add_optional("pts_repo_name", "string", "PTS repository name to deploy remotely", "origin", choices=introspection.pts_git_remotes())
definition.add_optional("skirt_repo_name", "string", "SKIRT repository name to deploy remotely", "origin", choices=introspection.skirt_git_remotes())
# -----------------------------------------------------------------
# Ensure Python 3 compatibility
from __future__ import absolute_import, division, print_function
# Import the relevant PTS classes and modules
from pts.core.basics.log import log
from pts.core.basics.configuration import ConfigurationDefinition, parse_arguments
from pts.core.remote.remote import Remote
from pts.core.remote.host import find_host_ids
# -----------------------------------------------------------------
definition = ConfigurationDefinition()
definition.add_optional("remotes", "string_list", "remote host ID", choices=find_host_ids(), default=find_host_ids())
config = parse_arguments("sessions", definition)
# -----------------------------------------------------------------
# Loop over the remote host
for host_id in config.remotes:
# Create and setup the remote
remote = Remote()
if not remote.setup(host_id): raise RuntimeError("The remote host '" + host_id + "' is not available at the moment")
# Get screen names and tmux session names
screen_names = remote.screen_names()
tmux_names = remote.tmux_names()
# Import the relevant PTS classes and modules
from pts.core.basics.configuration import ConfigurationDefinition, parse_arguments
from pts.core.remote.host import find_host_ids
from pts.core.remote.remote import Remote
from pts.core.tools import filesystem as fs
from pts.core.basics.log import log
# -----------------------------------------------------------------
# Create configuration definition
definition = ConfigurationDefinition()
# Required
definition.add_required("remote_path", "string", "remote path of the file or directory to retrieve")
definition.add_required("remote", "string", "remote host to retrieve from", choices=find_host_ids())
# Local path
definition.add_positional_optional("local_path", "string", "path of the local directory to store the file/directory")
# Create configuration
config = parse_arguments("retrieve", definition, "Retrieve a file or directory from a remote host")
# -----------------------------------------------------------------
# Create remote
remote = Remote(host_id=config.remote)
# -----------------------------------------------------------------
# Set full path of origin
# Settings for the wavelength grid
#definition.add_optional("nwavelengths", "integer", "the number of wavelengths to simulate the best model", 450)
# -----------------------------------------------------------------
definition.add_required("origin", "string", "origin of the analysis model", choices=origins)
# -----------------------------------------------------------------
definition.add_required("name", "string_no_spaces", "name for the run")
# -----------------------------------------------------------------
# Remote execution
definition.add_positional_optional("remote", "string", "remote host ID for running the simulation", choices=find_host_ids())
definition.add_flag("attached", "launch remote executions in attached mode", True)
# -----------------------------------------------------------------
# Type of output
definition.add_optional("make", "string_list", "let these forms of output be made", default_make, choices=make_choices)
definition.add_flag("contributions", "make output for the contributions to the flux of the various sources (transparent, dust ...)", default_make_contributions)
definition.add_flag("faceon", "include face-on maps and SEDs", False)
definition.add_flag("edgeon", "include edge-on maps and SEDs", False)
# Full instrument properties
definition.add_flag("scattering_levels", "record scattering levels", 0)
definition.add_flag("counts", "record photon counts for the earth instrument", False)
# To create the output
# Ensure Python 3 compatibility
from __future__ import absolute_import, division, print_function
# Import the relevant PTS classes and modules
from pts.core.tools import introspection
from pts.core.tools import filesystem as fs
from pts.core.basics.configuration import ConfigurationDefinition, parse_arguments
from pts.core.remote.host import find_host_ids
from pts.core.remote.remote import Remote
from pts.core.tools import time
# -----------------------------------------------------------------
# Create definition
definition = ConfigurationDefinition()
definition.add_positional_optional("remote", "string", "remote host on which to clear the temporary directory", choices=find_host_ids())
definition.add_optional("names", "string_list", "remove temporary directories matching these names (e.g. 'installation' matches 'installation_2017-05-03--08-45-44-410', 'installation_2017-05-03--13-52-28-371', etc. and also exact matches")
# Create setter
config = parse_arguments("clear_temp", definition)
# -----------------------------------------------------------------
# Remote
if config.remote is not None:
# Create the remote
remote = Remote(host_id=config.remote)
# Check whether names are given
if config.names is not None:
#!/usr/bin/env python
# -*- coding: utf8 -*-
# *****************************************************************
# ** PTS -- Python Toolkit for working with SKIRT **
# ** © Astronomical Observatory, Ghent University **
# *****************************************************************
# Import the relevant PTS classes and modules
from pts.core.basics.configuration import ConfigurationDefinition
from pts.core.remote.host import find_host_ids
# -----------------------------------------------------------------
# Create the configuration
definition = ConfigurationDefinition()
# Remote
definition.add_optional("remote", "string", "remote host on which to execute the calculations", choices=find_host_ids())
# Method
definition.add_optional("method", "string", "fitting method", "grid", choices=["grid", "genetic"])
# -----------------------------------------------------------------
# Import the relevant PTS classes and modules
from pts.core.basics.configuration import ConfigurationDefinition, parse_arguments
from pts.core.remote.host import find_host_ids
from pts.core.tools import introspection
from pts.core.tools import filesystem as fs
from pts.core.simulation.simulation import RemoteSimulation
from pts.core.remote.remote import Remote
from pts.core.basics.log import log
# -----------------------------------------------------------------
# Create the configuration definition
definition = ConfigurationDefinition()
# Add settings
definition.add_positional_optional("remotes", "string_list", "the IDs of the remote hosts for which to clear the simulations", choices=find_host_ids(), default=find_host_ids())
definition.add_positional_optional("ids", "integer_list", "the IDs of the simulations to clear")
definition.add_flag("full", "fully clear the simulations, also remove remote simulation directories")
# -----------------------------------------------------------------
# Parse the arguments into a configuration
config = parse_arguments("clear_tasks", definition, description="Clear PTS tasks for a certain remote host")
# -----------------------------------------------------------------
# Loop over the remote hosts
for host_id in config.remotes:
# Check whether the remote is available
if config.full:
# Get selection indices
indices = fs.files_in_path(maps_components_path, extension="dat", returns="name", startswith="selection", convert=int, sort=int, convert_split_index=1, convert_split_pattern="_")
# Add the setting
if len(indices) == 0: raise RuntimeError("Could not find any selection file")
elif len(indices) == 1: definition.add_fixed("selection", "selection to use", indices[0])
else: definition.add_optional("selection", "positive_integer", "selection to use", choices=indices)
# -----------------------------------------------------------------
# Show?
definition.add_flag("show", "show the page", False)
# Remote
definition.add_optional("remote", "string", "remote host to use for creating the clip masks", choices=find_host_ids(schedulers=False))
# Flags
definition.add_flag("convolve", "perform convolution during the creation of the clip masks", False)
# CROPPING
definition.add_optional("cropping_factor", "positive_real", "multiply the cropping box with this factor", 1.3)
# REBINNING
definition.add_optional("rebin_remote_threshold", "data_quantity", "data size threshold for remote rebinning", "0.5 GB", convert_default=True)
# Flags
definition.add_flag("add_old", "add old stellar maps", True)
definition.add_flag("add_young", "add young stellar maps", True)
definition.add_flag("add_ionizing", "add ionizing stellar maps", True)
definition.add_flag("add_dust", "add dust maps", True)
#!/usr/bin/env python
# -*- coding: utf8 -*-
# *****************************************************************
# ** PTS -- Python Toolkit for working with SKIRT **
# ** © Astronomical Observatory, Ghent University **
# *****************************************************************
# Import the relevant PTS classes and modules
from pts.core.basics.configuration import ConfigurationDefinition
from pts.core.remote.host import find_host_ids
# -----------------------------------------------------------------
# Create configuration definition
definition = ConfigurationDefinition()
# Add optional
definition.add_positional_optional("ski", "file_path", "ski file to be used (if not specified, all ski files in the current directory will be used)")
# Add flag
definition.add_flag("recursive", "look for ski files recursively")
# Add optional
definition.add_optional("remote", "string", "the remote host (no schedulers) on which to launch the simulations", choices=find_host_ids(schedulers=False))
# Add optional
definition.add_optional("nprocesses", "integer", "number of processes to use", 1)
definition.add_flag("data_parallel", "use data parallelization mode")
# -----------------------------------------------------------------
# -----------------------------------------------------------------
# Ensure Python 3 compatibility
from __future__ import absolute_import, division, print_function
# Import the relevant PTS classes and modules
from pts.core.basics.configuration import ConfigurationDefinition, parse_arguments
from pts.core.remote.host import find_host_ids
from pts.core.remote.remote import Remote
from pts.core.tools import filesystem as fs
# -----------------------------------------------------------------
definition = ConfigurationDefinition()
definition.add_required("remote_path", "string", "remote path of the file or directory to retrieve")
definition.add_required("remote", "string", "remote host to retrieve from", choices=find_host_ids())
definition.add_optional("local_path", "string", "path of the local directory to store the file/directory")
config = parse_arguments("retrieve", definition)
# -----------------------------------------------------------------
# Create remote
remote = Remote(host_id=config.remote)
# -----------------------------------------------------------------
# Set full path of origin
origin = remote.absolute_path(config.remote_path)
# Set full path to the destination
# -----------------------------------------------------------------
definition = definition.copy()
# -----------------------------------------------------------------
# THE ANALYSIS RUN
if runs.empty: raise ValueError("No analysis runs present (yet)")
elif runs.has_single: definition.add_fixed("run", "name of the analysis run", runs.single_name)
else: definition.add_required("run", "string", "name of the analysis run", runs.names)
# -----------------------------------------------------------------
# Optional settings
definition.add_positional_optional("remote", "string", "remote host on which to launch the simulations", choices=find_host_ids())
definition.add_positional_optional("cluster_name", "string", "cluster of the remote host to use for the simulation")
definition.add_flag("group", "group simulations in larger jobs")
definition.add_optional("group_walltime", "real", "the preferred walltime per group job (for schedulers)")
# -----------------------------------------------------------------
# Simulation options
definition.add_optional("npackages", "real", "number of photon packages per wavelength", 1e7)
definition.add_optional("npackages_seds", "real", "number of photon packages per wavelength for simulations that produce only SEDs", 1e6)
definition.add_flag("selfabsorption", "dust self-absorption", True)
definition.add_flag("transient_heating", "transient (non-LTE) dust heating", True)
# -----------------------------------------------------------------
# The number of parallel processes for local execution
#!/usr/bin/env python
# -*- coding: utf8 -*-
# *****************************************************************
# ** PTS -- Python Toolkit for working with SKIRT **
# ** © Astronomical Observatory, Ghent University **
# *****************************************************************
# Import the relevant PTS classes and modules
from pts.modeling.config.fetch import definition
from pts.core.remote.host import find_host_ids
# -----------------------------------------------------------------
# Add required settings
definition.add_positional_optional("remote", "string", "remote host to use for creating the GALEX and SDSS data", choices=find_host_ids(schedulers=False))
definition.add_flag("attached", "run remote in attached mode", False)
# Add flags
definition.add_flag("errors", "also download the error frames from the DustPedia archive")
# H alpha
definition.add_optional("halpha_url", "url", "url of the h-alpha image") # optional for when fetch_images is relaunched, and the Halpha image is already present
definition.add_optional("halpha_flux", "quantity", "flux of the h-alpha image to which the image should be rescaled")
# The total H-alpha flux (reference: FAR-ULTRAVIOLET AND Ha IMAGING OF NEARBY SPIRAL GALAXIES: THE OB STELLAR,
# POPULATION IN THE DIFFUSE IONIZED GAS (Hoopes et. al 2001)
# Flags
definition.add_flag("make_poisson", "perform the poisson error mosaicing", True)
# Number of processes for Poisson error map making
definition.add_optional("nprocesses", "positive_integer", "number of parallel processes", 1)
#!/usr/bin/env python
# -*- coding: utf8 -*-
# *****************************************************************
# ** PTS -- Python Toolkit for working with SKIRT **
# ** © Astronomical Observatory, Ghent University **
# *****************************************************************
# Import the relevant PTS classes and modules
from pts.core.basics.configuration import ConfigurationDefinition
from pts.core.tools import filesystem as fs
from pts.core.remote.host import find_host_ids
# -----------------------------------------------------------------
# Create the configuration
definition = ConfigurationDefinition()
# Add required arguments
definition.add_required("ski", "file_path", "name/path of the ski file")
definition.add_required("fski", "file_path", "name/path of the fski file")
# Input and output
definition.add_optional("input", "directory_path", "input directory for the simulation(s)", letter="i")
definition.add_optional("output", "directory_path", "output directory for the simulation(s)", fs.cwd(), letter="o", convert_default=True)
# Add positional arguments
definition.add_positional_optional("remote", "string", "the remote host on which to run the simulation (if none is specified, the simulation is run locally", choices=find_host_ids())
# -----------------------------------------------------------------
# -----------------------------------------------------------------
modeling_path = verify_modeling_cwd()
# -----------------------------------------------------------------
definition = definition.copy()
# Add optional arguments
definition.add_positional_optional("image", "string", "the name of the image for which to run the initialization")
# Add flags
definition.add_flag("visualise", "make visualisations")
# Remote source detection
definition.add_optional("remote", "string", "remote host on which to run the source finder", choices=find_host_ids())
definition.add_flag("attached", "run remotely in attached mode")
# Add section for the source finder
definition.import_section("sources", "options for the source finder", sources_definition)
# Flags
default_catalogs = ["2MASS"]
definition.add_optional("catalogs", "string_list", "catalogs for point sources", default_catalogs, choices=stellar_catalog_descriptions)
definition.add_flag("catalog_overlapping", "only fetch catalog data in the area where all images are overlapping", True)
definition.add_flag("manual", "don't find sources, but mark them from the catalog, and let the selection be done manually", False)
# Cache
cache_host_id = get_cache_host_id(modeling_path)
if cache_host_id is not None: definition.add_flag("cache", "cache image data for which the initialized image has been created", False)
# Get number of maps
nold_maps = len(old_map_names)
nyoung_maps = len(young_map_names)
nionizing_maps = len(ionizing_map_names)
ndust_maps = len(dust_map_names)
# -----------------------------------------------------------------
# Create the configuration
definition.add_flag("show", "show the page", False)
# Remote
definition.add_optional("remote",
"string",
"remote host to use for creating the clip masks",
choices=find_host_ids(schedulers=False))
# Flags
definition.add_flag(
"convolve", "perform convolution during the creation of the clip masks",
False)
# CROPPING
definition.add_optional("cropping_factor", "positive_real",
"multiply the cropping box with this factor", 1.3)
# REBINNING
definition.add_optional("rebin_remote_threshold",
"data_quantity",
"data size threshold for remote rebinning",
"0.5 GB",
#!/usr/bin/env python
# -*- coding: utf8 -*-
# *****************************************************************
# ** PTS -- Python Toolkit for working with SKIRT **
# ** © Astronomical Observatory, Ghent University **
# *****************************************************************
# Import the relevant PTS classes and modules
from pts.core.remote.host import find_host_ids
from pts.core.config.launch import definition
# -----------------------------------------------------------------
all_host_ids = find_host_ids()
# -----------------------------------------------------------------
# Flags
definition.add_flag("recursive", "look for ski files recursively")
# Load from queues
definition.add_flag("load_queues", "load queue files found in the working directory")
# Settings for the remote
if len(find_host_ids()) > 0: definition.add_optional("remotes", "string_list", "remote host IDs to use", choices=all_host_ids, default=all_host_ids)
else: definition.add_fixed("remotes", "remote hosts", [])
# Parallelization options
definition.add_optional("nnodes", "integer", "number of computing nodes to be used (for remote hosts that use a scheduling system, for other remotes the current load of the system will be probed)")
definition.add_optional("nnodes_per_host", "string_integer_dictionary", "number of computing nodes to be used, per scheduling remote")
# Create definition
definition = ConfigurationDefinition()
definition.add_positional_optional("skirt_and_or_pts", "string_list", "SKIRT and/or PTS", default=["skirt", "pts"], choices=["skirt", "pts"])
# Add flags
definition.add_flag("conda", "also remove conda installation")
definition.add_flag("qt", "also remove Qt installation")
definition.add_flag("one_attempt", "only perform one attempt at connecting to a remote")
# Get the config
config = parse_arguments("deinstall_all", definition)
# -----------------------------------------------------------------
# Loop over the remote hosts
for host_id in find_host_ids():
# Setup
remote = Remote()
if not remote.setup(host_id, one_attempt=config.one_attempt):
log.warning("Remote host '" + host_id + "' is offline")
continue
# Create uninstaller
uninstaller = Uninstaller()
# Set options
uninstaller.config.skirt_and_or_pts = config.skirt_and_or_pts
uninstaller.config.conda = config.conda
uninstaller.config.qt = config.qt
#!/usr/bin/env python
# -*- coding: utf8 -*-
# *****************************************************************
# ** PTS -- Python Toolkit for working with SKIRT **
# ** © Astronomical Observatory, Ghent University **
# *****************************************************************
# Import the relevant PTS classes and modules
from pts.core.basics.configuration import ConfigurationDefinition
from pts.core.remote.host import find_host_ids
# -----------------------------------------------------------------
# Create the configuration
definition = ConfigurationDefinition(log_path="log", config_path="config")
# Add option for the remote host ID
definition.add_positional_optional(
"remote", "string",
"remote host to use for the aperture correction calculation",
find_host_ids())
# Add option
definition.add_optional("noise_method",
"string",
"method to use for the aperture noise calculation",
choices=["caapr", "pts"],
default="caapr")
# -----------------------------------------------------------------
definition.add_flag("plot", "do plotting", False)
definition.add_flag("rotate", "rotation", True)
definition.add_optional("nrandom_sources", "positive_integer", "number of random sources", 100)
definition.add_flag("vary_fwhm", "vary the FWHM slightly for each star", True)
definition.add_flag("add_catalogued_sources", "add catalogued sources", False)
definition.add_optional("point_source_catalogs", "string_list", "point source catalogs", ["II/246"])
definition.add_optional("nfilters_stars", "positive_integer", "number of filters to use where stars are visible", 4)
definition.add_optional("nfilters_extra", "positive_integer", "number of filters to use where stars are not visible", 2)
definition.add_optional("psf_model", "string", "model to use for the PSF", "airydisk", choices=["gaussian", "airydisk"])
definition.add_optional("noise_stddev", "real", "stddev of noise", 5.)
definition.add_flag("only_local", "only evaluate PSFs locally", True)
definition.add_optional("galaxy_sersic_index", "real", "sersic index", 3.)
definition.add_optional("galaxy_relative_asymptotic_radius", "positive_real", "radius of the galaxy mask relative to the effective radius", 4.)
definition.add_flag("limit_galaxy", "limit the rendering of the galaxy to a certain ellipse", False)
# Parallel and remote
definition.add_optional("nprocesses", "positive_integer", "number of parallel processes to use", 1)
definition.add_optional("remote", "string", "remote host ID", choices=find_host_ids())
# -----------------------------------------------------------------
from __future__ import absolute_import, division, print_function
# Import the relevant PTS classes and modules
from pts.core.basics.configuration import ConfigurationDefinition, parse_arguments
from pts.core.remote.host import find_host_ids
from pts.core.remote.remote import Remote
from pts.core.tools import filesystem as fs
# -----------------------------------------------------------------
# Create configuration definition
definition = ConfigurationDefinition()
# Required
definition.add_required("local_path", "string", "path or name of the file or directory to send")
definition.add_required("remote", "string", "the remote host to send to", choices=find_host_ids())
# Remote path
definition.add_positional_optional("remote_path", "string", "path of the remote directory to send to")
# Create configuration
config = parse_arguments("send", definition, "Send a file or directory to a remote host")
# -----------------------------------------------------------------
# Create remote
remote = Remote(host_id=config.remote)
# -----------------------------------------------------------------
# Set full path of origin
#!/usr/bin/env python
# -*- coding: utf8 -*-
# *****************************************************************
# ** PTS -- Python Toolkit for working with SKIRT **
# ** © Astronomical Observatory, Ghent University **
# *****************************************************************
# Import the relevant PTS classes and modules
from pts.core.remote.host import find_host_ids
from pts.modeling.config.component import definition
# -----------------------------------------------------------------
definition = definition.copy()
# The path of the resulting mosaic image
definition.add_optional("band_id", "string", "ID of the band")
definition.add_optional("image_path", "file_path", "path to the resulting mosaic image file")
definition.add_optional("images_path", "directory_path", "path to a directory with mosaic images files")
definition.add_optional("out_path", "directory_path", "path of the out directory")
# Task id
definition.add_optional("host_id", "string", "host ID of the task", choices=find_host_ids())
definition.add_optional("task_id", "integer", "task ID")
# -----------------------------------------------------------------
# -----------------------------------------------------------------
# Add required arguments
definition.add_required("ski", "file_path", "name/path of the ski file")
# Simulation settings
definition.add_flag("relative", "treats the given input and output paths as being relative to the ski/fski file")
# Flags
definition.add_flag("create_output", "create the output directory if it is not yet present")
definition.add_flag("show_progress", "show progress (for local simulation or remote simulation in attached mode)", True)
# -----------------------------------------------------------------
# Add positional arguments
definition.add_positional_optional("remote", "string", "remote host on which to run the simulation (if none is specified, the simulation is run locally", choices=find_host_ids())
definition.add_optional("cluster_name", "string", "name of the cluster", letter="c")
# Remote and parallelization
definition.add_optional("parallel", "integer_pair", "parallelization scheme (processes, threads)", letter="p")
definition.add_optional("walltime", "duration", "an estimate for the walltime of the simulation for the specified parallelization scheme")
definition.add_flag("data_parallel_local", "enable data parallelization for local execution", False)
definition.add_flag("data_parallel_remote", "set data parallelization for remote execution (None means automatic)", None)
# Check parallelization? (ADVANCED)
definition.add_flag("check_parallelization", "check the specified parallelization scheme", True)
# -----------------------------------------------------------------
definition.add_optional("finish_at", "string", "finish when this is encountered")
definition.add_optional("finish_after", "string", "finish after this is encountered")
# ** PTS -- Python Toolkit for working with SKIRT **
# ** © Astronomical Observatory, Ghent University **
# *****************************************************************
# Import the relevant PTS classes and modules
from pts.core.basics.configuration import ConfigurationDefinition
from pts.core.remote.host import find_host_ids
from pts.core.config.analyse_simulation import definition as analysis_definition
# -----------------------------------------------------------------
# Create the configuration
definition = ConfigurationDefinition(write_config=False)
# Add positional optional
definition.add_positional_optional("host_ids", "string_list", "name of the remote host(s) for which to show/retrieve simulations and tasks", choices=find_host_ids())
definition.add_positional_optional("ids", "string_integer_list_dictionary", "simulation IDs for the different remote hosts")
# Add flag
definition.add_flag("show_progress", "show the progress of the simulation that is still running (only if there is just one)", False)
definition.add_flag("debug_output", "show all simulation output in debug mode")
# -----------------------------------------------------------------
definition.add_flag("retrieve", "retrieve finished simulations", True)
definition.add_flag("analyse", "analyse retrieved simulations", True)
# -----------------------------------------------------------------
# Crashed
definition.add_flag("check_crashed", "check whether crashed simulations have the necessary output, if so, retrieve them")
# Import the relevant PTS classes and modules
from pts.core.remote.host import find_host_ids
from pts.core.remote.remote import Remote
from pts.core.remote.modules import Modules
from pts.core.basics.log import log
from pts.core.basics.configuration import ConfigurationDefinition, parse_arguments
from pts.core.tools import introspection
from pts.core.tools import git
from pts.core.prep.installation import qt_url, qt_configure_options
from pts.core.basics.map import Map
# -----------------------------------------------------------------
# Create the configuration definition
definition = ConfigurationDefinition()
definition.add_required("remote", "string", "remote host ID", choices=find_host_ids())
definition.add_flag("private", "install SKIRT from private (development) repository")
# Create the configuration
config = parse_arguments("installation_commands", definition)
# -----------------------------------------------------------------
installation_commands = defaultdict(list)
# -----------------------------------------------------------------
# Create the remote
remote = Remote()
if not remote.setup(config.remote):
log.error("The remote host is not available")
from pts.core.tools import filesystem as fs
from pts.core.simulation.simulation import RemoteSimulation
from pts.core.remote.remote import Remote
from pts.core.basics.log import log
# -----------------------------------------------------------------
# Create the configuration definition
definition = ConfigurationDefinition()
# Add settings
definition.add_positional_optional(
"remotes",
"string_list",
"the IDs of the remote hosts for which to clear the simulations",
choices=find_host_ids(),
default=find_host_ids())
definition.add_positional_optional("ids", "integer_list",
"the IDs of the simulations to clear")
definition.add_flag(
"full",
"fully clear the simulations, also remove remote simulation directories")
# -----------------------------------------------------------------
# Parse the arguments into a configuration
config = parse_arguments(
"clear_tasks",
definition,
description="Clear PTS tasks for a certain remote host")
from __future__ import absolute_import, division, print_function
# Import the relevant PTS classes and modules
from pts.core.basics.configuration import ConfigurationDefinition, parse_arguments
from pts.core.basics.log import log
from pts.core.tools import filesystem as fs
from pts.core.remote.host import find_host_ids
from pts.core.remote.remote import Remote
# -----------------------------------------------------------------
# Create the configuration definition
definition = ConfigurationDefinition()
# Remote
definition.add_positional_optional("remote", "string", "remote host ID", choices=find_host_ids())
definition.add_positional_optional("remote_path", "string", "path of remote directory (absolute or relative w.r.t. home directory)")
# Add optional settings
definition.add_optional("contains", "string", "a string that should be contained in the file names")
definition.add_optional("not_contains", "string", "a string that should not be contained in the file names")
definition.add_optional("extension", "string", "the file extension")
# Add flags
definition.add_flag("recursive", "search recursively", False)
definition.add_flag("full", "show the full paths", False)
# -----------------------------------------------------------------
# Parse the arguments into a configuration
config = parse_arguments("find_files", definition, description="Find files containing a certain string in the current working directory")
# Create definition
definition = ConfigurationDefinition(write_config=False)
# Optional settings
definition.add_optional("nwavelengths", "positive_integer", "number of wavelengths for the reference simulation", 50)
definition.add_optional("npackages", "positive_integer", "number of photon packages per wavelength for the reference simulation", int(1e5))
definition.add_optional("dust_grid_relative_scale", "real", "smallest scale of the dust grid relative to the pixelscale of the input maps", 2.)
definition.add_optional("dust_grid_min_level", "positive_integer", "level for the dust grid", 2)
definition.add_optional("dust_grid_max_mass_fraction", "positive_real", "max mass fraction for the dust grid", 1e-6)
# Flags
definition.add_flag("transient_heating", "enable transient heating", False)
definition.add_flag("selfabsorption", "enable dust selfabsorption", False)
# For remote execution
definition.add_optional("host_ids", "string_list", "remote hosts to use for heavy computations and simulations", choices=find_host_ids())
definition.add_flag("attached", "launch remote executions in attached mode", False)
# Fitting
definition.add_optional("ngenerations", "positive_integer", "number of generations", 5)
definition.add_optional("nsimulations", "even_positive_integer", "number of simulations per generation", 30)
definition.add_optional("npackages_fitting", "positive_integer", "number of photon packages for each fitting simulation", int(1e4))
# Free parameters
definition.add_optional("free_parameters", "string_list", "free parameter labels", choices=possible_free_parameters, default=default_free_parameters)
definition.add_optional("relative_range_initial", "real_range", "relative range for generating the initial parameter values", default="0.3>3", convert_default=True)
definition.add_optional("relative_range_fitting", "real_range", "relative range of the free parameter values for the fitting", default="0.1>20", convert_default=True)
# Wavelength grid
definition.add_optional("wavelength_range", "quantity_range", "range of wavelengths", "0.1 micron > 2000 micron", convert_default=True)
from pts.core.remote.host import find_host_ids
from pts.core.simulation.remote import get_simulation_for_host, get_simulation_ids
from pts.core.tools import introspection
from pts.core.tools import formatting as fmt
from pts.core.tools.stringify import tostr
from pts.core.tools import filesystem as fs
from pts.core.remote.mounter import mount_remote
from pts.core.remote.remote import get_home_path
# -----------------------------------------------------------------
# Create the configuration definition
definition = ConfigurationDefinition()
# Add required
definition.add_required("host_ids", "string_list", "names of the remote hosts", choices=find_host_ids())
definition.add_positional_optional("simulation_ids", "integer_list", "simulations to show")
definition.add_optional("names", "string_list", "simulation names to show")
# Add flag
definition.add_flag("analysis", "show analysis options", False)
# Add flags
definition.add_flag("open_output", "open the output directory (only for one simulation)")
definition.add_flag("open_remote_output", "open the remote output directory (only for one simulation)")
definition.add_flag("open_input", "open the input directory (only for one simulation)")
definition.add_flag("open_remote_input", "open the remote input directory (only for one simulation)")
definition.add_flag("open_base", "open the simulation directory (only for one simulation)")
definition.add_flag("open_remote_base", "open the remote simulation directory (only for one simulation)")
# -----------------------------------------------------------------
#!/usr/bin/env python
# -*- coding: utf8 -*-
# *****************************************************************
# ** PTS -- Python Toolkit for working with SKIRT **
# ** © Astronomical Observatory, Ghent University **
# *****************************************************************
# Import the relevant PTS classes and modules
from pts.core.basics.configuration import ConfigurationDefinition
from pts.core.remote.host import find_host_ids
# -----------------------------------------------------------------
# Create the configuration definition
definition = ConfigurationDefinition()
# Add required
definition.add_required("remote", "string", "name of the remote host", choices=find_host_ids())
definition.add_positional_optional("id", "positive_integer", "simulation ID")
definition.add_optional("name", "string", "simulation name")
# Additional relative error
definition.add_optional("additional_error", "percentage", "additional percentual error for the observed flux points")
# Flags
definition.add_flag("write", "write the results", True)
# -----------------------------------------------------------------
# ** © Astronomical Observatory, Ghent University **
# *****************************************************************
# Import the relevant PTS classes and modules
from pts.core.basics.configuration import ConfigurationDefinition
from pts.core.remote.host import find_host_ids
from pts.modeling.core.environment import load_modeling_environment_cwd
# -----------------------------------------------------------------
# Load the modeling environment and analysis runs
environment = load_modeling_environment_cwd()
runs = environment.analysis_runs
# Determine remote hosts for simulations
host_ids = find_host_ids()
host_ids_suggestions = find_host_ids(
schedulers=False
) # suggest to not use schedulers: requires runtime to be estimated
# Determine the remote hosts for heavy computations (making the images)
config = environment.modeling_configuration
other_host_ids = config.host_ids
if other_host_ids is None or len(other_host_ids) == 0:
other_host_ids = find_host_ids(schedulers=False)
default_other_host_id = other_host_ids[0]
# -----------------------------------------------------------------
# Create the configuration
definition = ConfigurationDefinition(log_path="log", config_path="config")
# *****************************************************************
# ** PTS -- Python Toolkit for working with SKIRT **
# ** © Astronomical Observatory, Ghent University **
# *****************************************************************
# Import the relevant PTS classes and modules
from pts.core.basics.configuration import ConfigurationDefinition
from pts.core.remote.host import find_host_ids
# -----------------------------------------------------------------
# Create the configuration definition
definition = ConfigurationDefinition()
# Add required
definition.add_positional_optional("remote", "string", "name of the remote host", choices=find_host_ids())
definition.add_positional_optional("matching", "string", "only adapt settings with a name matching this string")
definition.add_positional_optional("ids", "integer_list", "simulation IDs (if none specified, all simulation IDs will be used)")
definition.add_optional("names", "string_list", "simulation names")
definition.add_flag("from_directories", "use directory names as simulation names")
# -----------------------------------------------------------------
# Select certain properties
definition.add_optional("contains", "string", "only adapt properties containing this string in their name")
definition.add_optional("not_contains", "string", "don't adapt properties containing this string in their name")
definition.add_optional("exact_name", "string", "only adapt properties with this exact string as their name")
definition.add_optional("exact_not_name", "string", "don't adapt properties with this exact string as their name")
definition.add_optional("startswith", "string", "only adapt properties whose name starts with this string")
definition.add_optional("endswith", "string", "only adapt properties whose name starts with this string")
definition.add_fixed("name", "name of the fitting run", runs.single_name)
else:
definition.add_required("name",
"string",
"name of the fitting run",
choices=runs.names)
# Positional optional parameter
definition.add_positional_optional("generation_method",
"string",
"model generation method",
default_generation_method,
choices=generation_methods)
# Optional parameters
if len(find_host_ids()) > 0:
definition.add_optional(
"remotes",
"string_list",
"the remote hosts on which to run the parameter exploration",
default=find_host_ids(schedulers=False),
choices=find_host_ids(schedulers=False))
else:
definition.add_fixed("remotes", "remote hosts", [])
definition.add_flag("attached", "run remote simulations in attached mode")
definition.add_optional(
"nsimulations", "even_positive_integer",
"the number of simulations to launch in one batch/generation", 100)
definition.add_flag("group", "group simulations in larger jobs")
definition.add_optional("walltime", "real",
"the preferred walltime per job (for schedulers)")
