#!/usr/bin/env python # -*- coding: utf8 -*- # ***************************************************************** # ** PTS -- Python Toolkit for working with SKIRT ** # ** © Astronomical Observatory, Ghent University ** # ***************************************************************** # Import the relevant PTS classes and modules from pts.core.basics.configuration import ConfigurationDefinition from pts.core.remote.host import find_host_ids from pts.core.launch.analyser import all_steps from pts.core.config.analyse_simulation import definition as analysis_definition # ----------------------------------------------------------------- host_ids = find_host_ids() # ----------------------------------------------------------------- # Create the configuration definition definition = ConfigurationDefinition() # Assignment file definition.add_positional_optional("assignment", "file_path", "path of assignment file") # Status file definition.add_optional("status", "file_path", "path of status file") # Remotes for which to find corresponding simulations definition.add_optional("remotes", "string_list", "remote hosts for which to look for matching simulations (not necessary when assignment is specified)", default=host_ids, choices=host_ids)
modeling_path = verify_modeling_cwd() # ----------------------------------------------------------------- definition = definition.copy() # Add required arguments definition.add_positional_optional("image", "string", "name of the image for which to run the preparation") # Add optional arguments definition.add_optional("exclude_filters", "string_list", "exclude the data for these filters from the procedure that brings all data to the same resolution and pixelscale") definition.add_flag("steps", "write the results of intermediate steps") definition.add_flag("visualise", "make visualisations") # Remote preparation definition.add_optional("remote", "string", "remote host on which to run the preparation", choices=find_host_ids()) definition.add_flag("attached", "run remotely in attached mode") # Parallelization definition.add_optional("nprocesses", "positive_integer", "number of parallel processes for parallel computations", max(8, ncores())) # Advanced definition.add_flag("dustpedia_aperture", "use the DustPedia aperture instead of the galaxy region for the sky subtraction", True) definition.add_optional("aperture_galaxy_region_factor", "real", "if the galaxy region is used for the aperture, use this scaling factor on the region", 1.5) # Sky subtraction definition.add_optional("sky_estimation_method", "string", "sky estimation method", "photutils") definition.add_optional("sky_interpolation_method", "string", "interpolation method for photutils sky frame", "polynomial") definition.add_optional("noise_interpolation_method", "string", "interpolation method for photutils noise frame", "idw") definition.add_optional("sky_photutils_polynomial_order", "positive_integer", "blabla", 3) definition.add_optional("sky_photutils_polynomial_fitter", "string", "fitter", "levenberg-marquardt")
#!/usr/bin/env python # -*- coding: utf8 -*- # ***************************************************************** # ** PTS -- Python Toolkit for working with SKIRT ** # ** © Astronomical Observatory, Ghent University ** # ***************************************************************** # Import the relevant PTS classes and modules from pts.core.basics.configuration import ConfigurationDefinition from pts.core.remote.host import find_host_ids # ----------------------------------------------------------------- # Create definition definition = ConfigurationDefinition() # Add optional settings definition.add_positional_optional("remote", "string", "remote host on which to uninstall", choices=find_host_ids()) definition.add_positional_optional("skirt_and_or_pts", "string_list", "SKIRT and/or PTS", default=["skirt", "pts"], choices=["skirt", "pts"]) # Add flags definition.add_flag("conda", "also remove conda installation") definition.add_flag("qt", "also remove Qt installation") # -----------------------------------------------------------------
modeling_path = verify_modeling_cwd() # ----------------------------------------------------------------- # Create the configuration definition definition = ConfigurationDefinition(log_path="log", config_path="config") # Add optional arguments definition.add_positional_optional("image", "string", "the name of the image for which to run the initialization") # Add flags definition.add_flag("visualise", "make visualisations") # Remote source detection definition.add_optional("remote", "string", "remote host on which to run the source finder", choices=find_host_ids()) definition.add_flag("attached", "run remotely in attached mode") # Add section for the source finder definition.import_section("sources", "options for the source finder", sources_definition) # Flags default_catalogs = ["II/246"] definition.add_optional("catalogs", "string_list", "catalogs for point sources", default_catalogs) definition.add_flag("catalog_overlapping", "only fetch catalog data in the area where all images are overlapping", True) definition.add_flag("manual", "don't find sources, but mark them from the catalog, and let the selection be done manually", False) # Cache cache_host_id = get_cache_host_id(modeling_path) if cache_host_id is not None: definition.add_flag("cache", "cache image data for which the initialized image has been created", False)
# -*- coding: utf8 -*- # ***************************************************************** # ** PTS -- Python Toolkit for working with SKIRT ** # ** © Astronomical Observatory, Ghent University ** # ***************************************************************** # Import the relevant PTS classes and modules from pts.core.remote.host import find_host_ids from pts.modeling.config.component import definition # ----------------------------------------------------------------- definition = definition.copy() # Add option for the remote host ID definition.add_positional_optional("remote", "string", "remote host to use for the aperture correction calculation", choices=find_host_ids(schedulers=False)) # Add option noise_methods = ["caapr", "pts"] definition.add_optional("noise_method", "string", "method to use for the aperture noise calculation", choices=noise_methods, default="caapr") # ----------------------------------------------------------------- definition.add_flag("plot", "plot SEDs", True) # ----------------------------------------------------------------- definition.add_optional("reprocess", "lazy_broad_band_filter_list", "reprocess images and calculate fluxes for certain filters") # -----------------------------------------------------------------
from __future__ import absolute_import, division, print_function # Import the relevant PTS classes and modules from pts.core.remote.host import find_host_ids from pts.core.remote.remote import Remote from pts.core.tools import formatting as fmt from pts.core.basics.log import setup_log from pts.core.basics.configuration import parse_logging_arguments # ----------------------------------------------------------------- config = parse_logging_arguments("hosts") # ----------------------------------------------------------------- # Set log level in a special way if config.debug: setup_log("DEBUG") else: setup_log("ERROR") # ----------------------------------------------------------------- # Loop over the hosts for host_id in find_host_ids(): remote = Remote() connected = remote.setup(host_id, login_timeout=15) # only try for 15 seconds if connected: print(fmt.green + host_id + ": up" + fmt.reset) else: print(fmt.red + host_id + ": down" + fmt.reset) # -----------------------------------------------------------------
# ** PTS -- Python Toolkit for working with SKIRT ** # ** © Astronomical Observatory, Ghent University ** # ***************************************************************** # Import the relevant PTS classes and modules from pts.core.remote.host import find_host_ids from pts.core.basics.configuration import ConfigurationDefinition from pts.core.tools import introspection # ----------------------------------------------------------------- # Create the configuration definition definition = ConfigurationDefinition() # Add optional host_ids = find_host_ids() if len(host_ids) > 0: definition.add_positional_optional("host_ids", "string_list", "remote host ids", choices=host_ids, default=host_ids) else: definition.add_fixed("host_ids", "remote host_ids", []) # Add optional definition.add_optional("pts_repo_name", "string", "PTS repository name to deploy remotely", "origin", choices=introspection.pts_git_remotes())
#!/usr/bin/env python # -*- coding: utf8 -*- # ***************************************************************** # ** PTS -- Python Toolkit for working with SKIRT ** # ** © Astronomical Observatory, Ghent University ** # ***************************************************************** from pts.core.basics.configuration import ConfigurationDefinition from pts.core.remote.host import find_host_ids from pts.modeling.modeling.base import fitting_methods, default_fitting_method # ----------------------------------------------------------------- types = dict() types["galaxy"] = "Create a 3D model of a galaxy based on image data" types["sed"] = "Model any object based on observed SED data" types["images"] = "Model any object based on observational images and a ski file" # ----------------------------------------------------------------- # Create the configuration definition = ConfigurationDefinition(write_config=False) # Add required settings definition.add_required("type", "string", "type of modeling", choices=types) definition.add_required("name", "string", "name given to the object") definition.add_optional("fitting_method", "string", "fitting method", default_fitting_method, choices=fitting_methods) definition.add_optional("fitting_host_ids", "string_list", "remote hosts to use for performing simulations as part of the fitting", choices=find_host_ids()) # -----------------------------------------------------------------
# ----------------------------------------------------------------- # Ensure Python 3 compatibility from __future__ import absolute_import, division, print_function # Import the relevant PTS classes and modules from pts.core.basics.configuration import ConfigurationDefinition, parse_arguments from pts.core.remote.host import find_host_ids from pts.core.remote.remote import Remote from pts.core.basics.log import log # ----------------------------------------------------------------- # Create definition definition = ConfigurationDefinition() definition.add_positional_optional("remotes", "string_list", "remote hosts on which to clear", choices=find_host_ids(), default=find_host_ids()) # Create setter config = parse_arguments("clear_sessions_and_temp", definition) # ----------------------------------------------------------------- # Loop over the remote hosts for host_id in config.remotes: # Setup the remote remote = Remote() if not remote.setup(host_id): log.warning("Could not connect to remote host '" + host_id + "'") continue
#!/usr/bin/env python # -*- coding: utf8 -*- # ***************************************************************** # ** PTS -- Python Toolkit for working with SKIRT ** # ** © Astronomical Observatory, Ghent University ** # ***************************************************************** # Import the relevant PTS classes and modules from pts.modeling.config.fetch import definition from pts.core.remote.host import find_host_ids # ----------------------------------------------------------------- # Add required settings definition.add_positional_optional("remote", "string", "remote host to use for creating the GALEX and SDSS data", choices=find_host_ids(schedulers=False)) definition.add_flag("attached", "run remote in attached mode", False) # Add flags definition.add_flag("errors", "also download the error frames from the DustPedia archive") # H alpha definition.add_optional("halpha_url", "url", "url of the h-alpha image") # optional for when fetch_images is relaunched, and the Halpha image is already present definition.add_optional("halpha_flux", "quantity", "flux of the h-alpha image to which the image should be rescaled") # The total H-alpha flux (reference: FAR-ULTRAVIOLET AND Ha IMAGING OF NEARBY SPIRAL GALAXIES: THE OB STELLAR, # POPULATION IN THE DIFFUSE IONIZED GAS (Hoopes et. al 2001) # Flags definition.add_flag("make_poisson", "perform the poisson error mosaicing", True) # Advanced definition.add_optional("max_nobservations_mosaic", "positive_integer", "maximum number of observations to use for the mosaic and poisson frames (GALEX and SDSS)")
#!/usr/bin/env python # -*- coding: utf8 -*- # ***************************************************************** # ** PTS -- Python Toolkit for working with SKIRT ** # ** © Astronomical Observatory, Ghent University ** # ***************************************************************** # Import the relevant PTS classes and modules from pts.core.remote.host import find_host_ids, find_hosts from pts.core.basics.configuration import ConfigurationDefinition from pts.core.tools import introspection # ----------------------------------------------------------------- all_host_ids = find_host_ids() all_hosts = find_hosts() nhosts = len(all_hosts) has_hosts = nhosts > 0 # ----------------------------------------------------------------- # Create the configuration definition definition = ConfigurationDefinition() # The remote hosts if has_hosts: definition.add_positional_optional("hosts", "host_list", "remote hosts", choices=all_host_ids, default=all_hosts) else: definition.add_fixed("hosts", "remote hosts", []) # Add optional definition.add_optional("pts_repo_name", "string", "PTS repository name to deploy remotely", "origin", choices=introspection.pts_git_remotes()) definition.add_optional("skirt_repo_name", "string", "SKIRT repository name to deploy remotely", "origin", choices=introspection.skirt_git_remotes())
# ----------------------------------------------------------------- # Ensure Python 3 compatibility from __future__ import absolute_import, division, print_function # Import the relevant PTS classes and modules from pts.core.basics.log import log from pts.core.basics.configuration import ConfigurationDefinition, parse_arguments from pts.core.remote.remote import Remote from pts.core.remote.host import find_host_ids # ----------------------------------------------------------------- definition = ConfigurationDefinition() definition.add_optional("remotes", "string_list", "remote host ID", choices=find_host_ids(), default=find_host_ids()) config = parse_arguments("sessions", definition) # ----------------------------------------------------------------- # Loop over the remote host for host_id in config.remotes: # Create and setup the remote remote = Remote() if not remote.setup(host_id): raise RuntimeError("The remote host '" + host_id + "' is not available at the moment") # Get screen names and tmux session names screen_names = remote.screen_names() tmux_names = remote.tmux_names()
# Import the relevant PTS classes and modules from pts.core.basics.configuration import ConfigurationDefinition, parse_arguments from pts.core.remote.host import find_host_ids from pts.core.remote.remote import Remote from pts.core.tools import filesystem as fs from pts.core.basics.log import log # ----------------------------------------------------------------- # Create configuration definition definition = ConfigurationDefinition() # Required definition.add_required("remote_path", "string", "remote path of the file or directory to retrieve") definition.add_required("remote", "string", "remote host to retrieve from", choices=find_host_ids()) # Local path definition.add_positional_optional("local_path", "string", "path of the local directory to store the file/directory") # Create configuration config = parse_arguments("retrieve", definition, "Retrieve a file or directory from a remote host") # ----------------------------------------------------------------- # Create remote remote = Remote(host_id=config.remote) # ----------------------------------------------------------------- # Set full path of origin
# Settings for the wavelength grid #definition.add_optional("nwavelengths", "integer", "the number of wavelengths to simulate the best model", 450) # ----------------------------------------------------------------- definition.add_required("origin", "string", "origin of the analysis model", choices=origins) # ----------------------------------------------------------------- definition.add_required("name", "string_no_spaces", "name for the run") # ----------------------------------------------------------------- # Remote execution definition.add_positional_optional("remote", "string", "remote host ID for running the simulation", choices=find_host_ids()) definition.add_flag("attached", "launch remote executions in attached mode", True) # ----------------------------------------------------------------- # Type of output definition.add_optional("make", "string_list", "let these forms of output be made", default_make, choices=make_choices) definition.add_flag("contributions", "make output for the contributions to the flux of the various sources (transparent, dust ...)", default_make_contributions) definition.add_flag("faceon", "include face-on maps and SEDs", False) definition.add_flag("edgeon", "include edge-on maps and SEDs", False) # Full instrument properties definition.add_flag("scattering_levels", "record scattering levels", 0) definition.add_flag("counts", "record photon counts for the earth instrument", False) # To create the output
# Ensure Python 3 compatibility from __future__ import absolute_import, division, print_function # Import the relevant PTS classes and modules from pts.core.tools import introspection from pts.core.tools import filesystem as fs from pts.core.basics.configuration import ConfigurationDefinition, parse_arguments from pts.core.remote.host import find_host_ids from pts.core.remote.remote import Remote from pts.core.tools import time # ----------------------------------------------------------------- # Create definition definition = ConfigurationDefinition() definition.add_positional_optional("remote", "string", "remote host on which to clear the temporary directory", choices=find_host_ids()) definition.add_optional("names", "string_list", "remove temporary directories matching these names (e.g. 'installation' matches 'installation_2017-05-03--08-45-44-410', 'installation_2017-05-03--13-52-28-371', etc. and also exact matches") # Create setter config = parse_arguments("clear_temp", definition) # ----------------------------------------------------------------- # Remote if config.remote is not None: # Create the remote remote = Remote(host_id=config.remote) # Check whether names are given if config.names is not None:
#!/usr/bin/env python # -*- coding: utf8 -*- # ***************************************************************** # ** PTS -- Python Toolkit for working with SKIRT ** # ** © Astronomical Observatory, Ghent University ** # ***************************************************************** # Import the relevant PTS classes and modules from pts.core.basics.configuration import ConfigurationDefinition from pts.core.remote.host import find_host_ids # ----------------------------------------------------------------- # Create the configuration definition = ConfigurationDefinition() # Remote definition.add_optional("remote", "string", "remote host on which to execute the calculations", choices=find_host_ids()) # Method definition.add_optional("method", "string", "fitting method", "grid", choices=["grid", "genetic"]) # -----------------------------------------------------------------
# Import the relevant PTS classes and modules from pts.core.basics.configuration import ConfigurationDefinition, parse_arguments from pts.core.remote.host import find_host_ids from pts.core.tools import introspection from pts.core.tools import filesystem as fs from pts.core.simulation.simulation import RemoteSimulation from pts.core.remote.remote import Remote from pts.core.basics.log import log # ----------------------------------------------------------------- # Create the configuration definition definition = ConfigurationDefinition() # Add settings definition.add_positional_optional("remotes", "string_list", "the IDs of the remote hosts for which to clear the simulations", choices=find_host_ids(), default=find_host_ids()) definition.add_positional_optional("ids", "integer_list", "the IDs of the simulations to clear") definition.add_flag("full", "fully clear the simulations, also remove remote simulation directories") # ----------------------------------------------------------------- # Parse the arguments into a configuration config = parse_arguments("clear_tasks", definition, description="Clear PTS tasks for a certain remote host") # ----------------------------------------------------------------- # Loop over the remote hosts for host_id in config.remotes: # Check whether the remote is available if config.full:
# Get selection indices indices = fs.files_in_path(maps_components_path, extension="dat", returns="name", startswith="selection", convert=int, sort=int, convert_split_index=1, convert_split_pattern="_") # Add the setting if len(indices) == 0: raise RuntimeError("Could not find any selection file") elif len(indices) == 1: definition.add_fixed("selection", "selection to use", indices[0]) else: definition.add_optional("selection", "positive_integer", "selection to use", choices=indices) # ----------------------------------------------------------------- # Show? definition.add_flag("show", "show the page", False) # Remote definition.add_optional("remote", "string", "remote host to use for creating the clip masks", choices=find_host_ids(schedulers=False)) # Flags definition.add_flag("convolve", "perform convolution during the creation of the clip masks", False) # CROPPING definition.add_optional("cropping_factor", "positive_real", "multiply the cropping box with this factor", 1.3) # REBINNING definition.add_optional("rebin_remote_threshold", "data_quantity", "data size threshold for remote rebinning", "0.5 GB", convert_default=True) # Flags definition.add_flag("add_old", "add old stellar maps", True) definition.add_flag("add_young", "add young stellar maps", True) definition.add_flag("add_ionizing", "add ionizing stellar maps", True) definition.add_flag("add_dust", "add dust maps", True)
#!/usr/bin/env python # -*- coding: utf8 -*- # ***************************************************************** # ** PTS -- Python Toolkit for working with SKIRT ** # ** © Astronomical Observatory, Ghent University ** # ***************************************************************** # Import the relevant PTS classes and modules from pts.core.basics.configuration import ConfigurationDefinition from pts.core.remote.host import find_host_ids # ----------------------------------------------------------------- # Create configuration definition definition = ConfigurationDefinition() # Add optional definition.add_positional_optional("ski", "file_path", "ski file to be used (if not specified, all ski files in the current directory will be used)") # Add flag definition.add_flag("recursive", "look for ski files recursively") # Add optional definition.add_optional("remote", "string", "the remote host (no schedulers) on which to launch the simulations", choices=find_host_ids(schedulers=False)) # Add optional definition.add_optional("nprocesses", "integer", "number of processes to use", 1) definition.add_flag("data_parallel", "use data parallelization mode") # -----------------------------------------------------------------
# ----------------------------------------------------------------- # Ensure Python 3 compatibility from __future__ import absolute_import, division, print_function # Import the relevant PTS classes and modules from pts.core.basics.configuration import ConfigurationDefinition, parse_arguments from pts.core.remote.host import find_host_ids from pts.core.remote.remote import Remote from pts.core.tools import filesystem as fs # ----------------------------------------------------------------- definition = ConfigurationDefinition() definition.add_required("remote_path", "string", "remote path of the file or directory to retrieve") definition.add_required("remote", "string", "remote host to retrieve from", choices=find_host_ids()) definition.add_optional("local_path", "string", "path of the local directory to store the file/directory") config = parse_arguments("retrieve", definition) # ----------------------------------------------------------------- # Create remote remote = Remote(host_id=config.remote) # ----------------------------------------------------------------- # Set full path of origin origin = remote.absolute_path(config.remote_path) # Set full path to the destination
# ----------------------------------------------------------------- definition = definition.copy() # ----------------------------------------------------------------- # THE ANALYSIS RUN if runs.empty: raise ValueError("No analysis runs present (yet)") elif runs.has_single: definition.add_fixed("run", "name of the analysis run", runs.single_name) else: definition.add_required("run", "string", "name of the analysis run", runs.names) # ----------------------------------------------------------------- # Optional settings definition.add_positional_optional("remote", "string", "remote host on which to launch the simulations", choices=find_host_ids()) definition.add_positional_optional("cluster_name", "string", "cluster of the remote host to use for the simulation") definition.add_flag("group", "group simulations in larger jobs") definition.add_optional("group_walltime", "real", "the preferred walltime per group job (for schedulers)") # ----------------------------------------------------------------- # Simulation options definition.add_optional("npackages", "real", "number of photon packages per wavelength", 1e7) definition.add_optional("npackages_seds", "real", "number of photon packages per wavelength for simulations that produce only SEDs", 1e6) definition.add_flag("selfabsorption", "dust self-absorption", True) definition.add_flag("transient_heating", "transient (non-LTE) dust heating", True) # ----------------------------------------------------------------- # The number of parallel processes for local execution
#!/usr/bin/env python # -*- coding: utf8 -*- # ***************************************************************** # ** PTS -- Python Toolkit for working with SKIRT ** # ** © Astronomical Observatory, Ghent University ** # ***************************************************************** # Import the relevant PTS classes and modules from pts.modeling.config.fetch import definition from pts.core.remote.host import find_host_ids # ----------------------------------------------------------------- # Add required settings definition.add_positional_optional("remote", "string", "remote host to use for creating the GALEX and SDSS data", choices=find_host_ids(schedulers=False)) definition.add_flag("attached", "run remote in attached mode", False) # Add flags definition.add_flag("errors", "also download the error frames from the DustPedia archive") # H alpha definition.add_optional("halpha_url", "url", "url of the h-alpha image") # optional for when fetch_images is relaunched, and the Halpha image is already present definition.add_optional("halpha_flux", "quantity", "flux of the h-alpha image to which the image should be rescaled") # The total H-alpha flux (reference: FAR-ULTRAVIOLET AND Ha IMAGING OF NEARBY SPIRAL GALAXIES: THE OB STELLAR, # POPULATION IN THE DIFFUSE IONIZED GAS (Hoopes et. al 2001) # Flags definition.add_flag("make_poisson", "perform the poisson error mosaicing", True) # Number of processes for Poisson error map making definition.add_optional("nprocesses", "positive_integer", "number of parallel processes", 1)
#!/usr/bin/env python # -*- coding: utf8 -*- # ***************************************************************** # ** PTS -- Python Toolkit for working with SKIRT ** # ** © Astronomical Observatory, Ghent University ** # ***************************************************************** # Import the relevant PTS classes and modules from pts.core.basics.configuration import ConfigurationDefinition from pts.core.tools import filesystem as fs from pts.core.remote.host import find_host_ids # ----------------------------------------------------------------- # Create the configuration definition = ConfigurationDefinition() # Add required arguments definition.add_required("ski", "file_path", "name/path of the ski file") definition.add_required("fski", "file_path", "name/path of the fski file") # Input and output definition.add_optional("input", "directory_path", "input directory for the simulation(s)", letter="i") definition.add_optional("output", "directory_path", "output directory for the simulation(s)", fs.cwd(), letter="o", convert_default=True) # Add positional arguments definition.add_positional_optional("remote", "string", "the remote host on which to run the simulation (if none is specified, the simulation is run locally", choices=find_host_ids()) # -----------------------------------------------------------------
# ----------------------------------------------------------------- modeling_path = verify_modeling_cwd() # ----------------------------------------------------------------- definition = definition.copy() # Add optional arguments definition.add_positional_optional("image", "string", "the name of the image for which to run the initialization") # Add flags definition.add_flag("visualise", "make visualisations") # Remote source detection definition.add_optional("remote", "string", "remote host on which to run the source finder", choices=find_host_ids()) definition.add_flag("attached", "run remotely in attached mode") # Add section for the source finder definition.import_section("sources", "options for the source finder", sources_definition) # Flags default_catalogs = ["2MASS"] definition.add_optional("catalogs", "string_list", "catalogs for point sources", default_catalogs, choices=stellar_catalog_descriptions) definition.add_flag("catalog_overlapping", "only fetch catalog data in the area where all images are overlapping", True) definition.add_flag("manual", "don't find sources, but mark them from the catalog, and let the selection be done manually", False) # Cache cache_host_id = get_cache_host_id(modeling_path) if cache_host_id is not None: definition.add_flag("cache", "cache image data for which the initialized image has been created", False)
# Get number of maps nold_maps = len(old_map_names) nyoung_maps = len(young_map_names) nionizing_maps = len(ionizing_map_names) ndust_maps = len(dust_map_names) # ----------------------------------------------------------------- # Create the configuration definition.add_flag("show", "show the page", False) # Remote definition.add_optional("remote", "string", "remote host to use for creating the clip masks", choices=find_host_ids(schedulers=False)) # Flags definition.add_flag( "convolve", "perform convolution during the creation of the clip masks", False) # CROPPING definition.add_optional("cropping_factor", "positive_real", "multiply the cropping box with this factor", 1.3) # REBINNING definition.add_optional("rebin_remote_threshold", "data_quantity", "data size threshold for remote rebinning", "0.5 GB",
#!/usr/bin/env python # -*- coding: utf8 -*- # ***************************************************************** # ** PTS -- Python Toolkit for working with SKIRT ** # ** © Astronomical Observatory, Ghent University ** # ***************************************************************** # Import the relevant PTS classes and modules from pts.core.remote.host import find_host_ids from pts.core.config.launch import definition # ----------------------------------------------------------------- all_host_ids = find_host_ids() # ----------------------------------------------------------------- # Flags definition.add_flag("recursive", "look for ski files recursively") # Load from queues definition.add_flag("load_queues", "load queue files found in the working directory") # Settings for the remote if len(find_host_ids()) > 0: definition.add_optional("remotes", "string_list", "remote host IDs to use", choices=all_host_ids, default=all_host_ids) else: definition.add_fixed("remotes", "remote hosts", []) # Parallelization options definition.add_optional("nnodes", "integer", "number of computing nodes to be used (for remote hosts that use a scheduling system, for other remotes the current load of the system will be probed)") definition.add_optional("nnodes_per_host", "string_integer_dictionary", "number of computing nodes to be used, per scheduling remote")
# Create definition definition = ConfigurationDefinition() definition.add_positional_optional("skirt_and_or_pts", "string_list", "SKIRT and/or PTS", default=["skirt", "pts"], choices=["skirt", "pts"]) # Add flags definition.add_flag("conda", "also remove conda installation") definition.add_flag("qt", "also remove Qt installation") definition.add_flag("one_attempt", "only perform one attempt at connecting to a remote") # Get the config config = parse_arguments("deinstall_all", definition) # ----------------------------------------------------------------- # Loop over the remote hosts for host_id in find_host_ids(): # Setup remote = Remote() if not remote.setup(host_id, one_attempt=config.one_attempt): log.warning("Remote host '" + host_id + "' is offline") continue # Create uninstaller uninstaller = Uninstaller() # Set options uninstaller.config.skirt_and_or_pts = config.skirt_and_or_pts uninstaller.config.conda = config.conda uninstaller.config.qt = config.qt
#!/usr/bin/env python # -*- coding: utf8 -*- # ***************************************************************** # ** PTS -- Python Toolkit for working with SKIRT ** # ** © Astronomical Observatory, Ghent University ** # ***************************************************************** # Import the relevant PTS classes and modules from pts.core.basics.configuration import ConfigurationDefinition from pts.core.remote.host import find_host_ids # ----------------------------------------------------------------- # Create the configuration definition = ConfigurationDefinition(log_path="log", config_path="config") # Add option for the remote host ID definition.add_positional_optional( "remote", "string", "remote host to use for the aperture correction calculation", find_host_ids()) # Add option definition.add_optional("noise_method", "string", "method to use for the aperture noise calculation", choices=["caapr", "pts"], default="caapr") # -----------------------------------------------------------------
definition.add_flag("plot", "do plotting", False) definition.add_flag("rotate", "rotation", True) definition.add_optional("nrandom_sources", "positive_integer", "number of random sources", 100) definition.add_flag("vary_fwhm", "vary the FWHM slightly for each star", True) definition.add_flag("add_catalogued_sources", "add catalogued sources", False) definition.add_optional("point_source_catalogs", "string_list", "point source catalogs", ["II/246"]) definition.add_optional("nfilters_stars", "positive_integer", "number of filters to use where stars are visible", 4) definition.add_optional("nfilters_extra", "positive_integer", "number of filters to use where stars are not visible", 2) definition.add_optional("psf_model", "string", "model to use for the PSF", "airydisk", choices=["gaussian", "airydisk"]) definition.add_optional("noise_stddev", "real", "stddev of noise", 5.) definition.add_flag("only_local", "only evaluate PSFs locally", True) definition.add_optional("galaxy_sersic_index", "real", "sersic index", 3.) definition.add_optional("galaxy_relative_asymptotic_radius", "positive_real", "radius of the galaxy mask relative to the effective radius", 4.) definition.add_flag("limit_galaxy", "limit the rendering of the galaxy to a certain ellipse", False) # Parallel and remote definition.add_optional("nprocesses", "positive_integer", "number of parallel processes to use", 1) definition.add_optional("remote", "string", "remote host ID", choices=find_host_ids()) # -----------------------------------------------------------------
from __future__ import absolute_import, division, print_function # Import the relevant PTS classes and modules from pts.core.basics.configuration import ConfigurationDefinition, parse_arguments from pts.core.remote.host import find_host_ids from pts.core.remote.remote import Remote from pts.core.tools import filesystem as fs # ----------------------------------------------------------------- # Create configuration definition definition = ConfigurationDefinition() # Required definition.add_required("local_path", "string", "path or name of the file or directory to send") definition.add_required("remote", "string", "the remote host to send to", choices=find_host_ids()) # Remote path definition.add_positional_optional("remote_path", "string", "path of the remote directory to send to") # Create configuration config = parse_arguments("send", definition, "Send a file or directory to a remote host") # ----------------------------------------------------------------- # Create remote remote = Remote(host_id=config.remote) # ----------------------------------------------------------------- # Set full path of origin
#!/usr/bin/env python # -*- coding: utf8 -*- # ***************************************************************** # ** PTS -- Python Toolkit for working with SKIRT ** # ** © Astronomical Observatory, Ghent University ** # ***************************************************************** # Import the relevant PTS classes and modules from pts.core.remote.host import find_host_ids from pts.modeling.config.component import definition # ----------------------------------------------------------------- definition = definition.copy() # The path of the resulting mosaic image definition.add_optional("band_id", "string", "ID of the band") definition.add_optional("image_path", "file_path", "path to the resulting mosaic image file") definition.add_optional("images_path", "directory_path", "path to a directory with mosaic images files") definition.add_optional("out_path", "directory_path", "path of the out directory") # Task id definition.add_optional("host_id", "string", "host ID of the task", choices=find_host_ids()) definition.add_optional("task_id", "integer", "task ID") # -----------------------------------------------------------------
# ----------------------------------------------------------------- # Add required arguments definition.add_required("ski", "file_path", "name/path of the ski file") # Simulation settings definition.add_flag("relative", "treats the given input and output paths as being relative to the ski/fski file") # Flags definition.add_flag("create_output", "create the output directory if it is not yet present") definition.add_flag("show_progress", "show progress (for local simulation or remote simulation in attached mode)", True) # ----------------------------------------------------------------- # Add positional arguments definition.add_positional_optional("remote", "string", "remote host on which to run the simulation (if none is specified, the simulation is run locally", choices=find_host_ids()) definition.add_optional("cluster_name", "string", "name of the cluster", letter="c") # Remote and parallelization definition.add_optional("parallel", "integer_pair", "parallelization scheme (processes, threads)", letter="p") definition.add_optional("walltime", "duration", "an estimate for the walltime of the simulation for the specified parallelization scheme") definition.add_flag("data_parallel_local", "enable data parallelization for local execution", False) definition.add_flag("data_parallel_remote", "set data parallelization for remote execution (None means automatic)", None) # Check parallelization? (ADVANCED) definition.add_flag("check_parallelization", "check the specified parallelization scheme", True) # ----------------------------------------------------------------- definition.add_optional("finish_at", "string", "finish when this is encountered") definition.add_optional("finish_after", "string", "finish after this is encountered")
# ** PTS -- Python Toolkit for working with SKIRT ** # ** © Astronomical Observatory, Ghent University ** # ***************************************************************** # Import the relevant PTS classes and modules from pts.core.basics.configuration import ConfigurationDefinition from pts.core.remote.host import find_host_ids from pts.core.config.analyse_simulation import definition as analysis_definition # ----------------------------------------------------------------- # Create the configuration definition = ConfigurationDefinition(write_config=False) # Add positional optional definition.add_positional_optional("host_ids", "string_list", "name of the remote host(s) for which to show/retrieve simulations and tasks", choices=find_host_ids()) definition.add_positional_optional("ids", "string_integer_list_dictionary", "simulation IDs for the different remote hosts") # Add flag definition.add_flag("show_progress", "show the progress of the simulation that is still running (only if there is just one)", False) definition.add_flag("debug_output", "show all simulation output in debug mode") # ----------------------------------------------------------------- definition.add_flag("retrieve", "retrieve finished simulations", True) definition.add_flag("analyse", "analyse retrieved simulations", True) # ----------------------------------------------------------------- # Crashed definition.add_flag("check_crashed", "check whether crashed simulations have the necessary output, if so, retrieve them")
# Import the relevant PTS classes and modules from pts.core.remote.host import find_host_ids from pts.core.remote.remote import Remote from pts.core.remote.modules import Modules from pts.core.basics.log import log from pts.core.basics.configuration import ConfigurationDefinition, parse_arguments from pts.core.tools import introspection from pts.core.tools import git from pts.core.prep.installation import qt_url, qt_configure_options from pts.core.basics.map import Map # ----------------------------------------------------------------- # Create the configuration definition definition = ConfigurationDefinition() definition.add_required("remote", "string", "remote host ID", choices=find_host_ids()) definition.add_flag("private", "install SKIRT from private (development) repository") # Create the configuration config = parse_arguments("installation_commands", definition) # ----------------------------------------------------------------- installation_commands = defaultdict(list) # ----------------------------------------------------------------- # Create the remote remote = Remote() if not remote.setup(config.remote): log.error("The remote host is not available")
from pts.core.tools import filesystem as fs from pts.core.simulation.simulation import RemoteSimulation from pts.core.remote.remote import Remote from pts.core.basics.log import log # ----------------------------------------------------------------- # Create the configuration definition definition = ConfigurationDefinition() # Add settings definition.add_positional_optional( "remotes", "string_list", "the IDs of the remote hosts for which to clear the simulations", choices=find_host_ids(), default=find_host_ids()) definition.add_positional_optional("ids", "integer_list", "the IDs of the simulations to clear") definition.add_flag( "full", "fully clear the simulations, also remove remote simulation directories") # ----------------------------------------------------------------- # Parse the arguments into a configuration config = parse_arguments( "clear_tasks", definition, description="Clear PTS tasks for a certain remote host")
from __future__ import absolute_import, division, print_function # Import the relevant PTS classes and modules from pts.core.basics.configuration import ConfigurationDefinition, parse_arguments from pts.core.basics.log import log from pts.core.tools import filesystem as fs from pts.core.remote.host import find_host_ids from pts.core.remote.remote import Remote # ----------------------------------------------------------------- # Create the configuration definition definition = ConfigurationDefinition() # Remote definition.add_positional_optional("remote", "string", "remote host ID", choices=find_host_ids()) definition.add_positional_optional("remote_path", "string", "path of remote directory (absolute or relative w.r.t. home directory)") # Add optional settings definition.add_optional("contains", "string", "a string that should be contained in the file names") definition.add_optional("not_contains", "string", "a string that should not be contained in the file names") definition.add_optional("extension", "string", "the file extension") # Add flags definition.add_flag("recursive", "search recursively", False) definition.add_flag("full", "show the full paths", False) # ----------------------------------------------------------------- # Parse the arguments into a configuration config = parse_arguments("find_files", definition, description="Find files containing a certain string in the current working directory")
# Create definition definition = ConfigurationDefinition(write_config=False) # Optional settings definition.add_optional("nwavelengths", "positive_integer", "number of wavelengths for the reference simulation", 50) definition.add_optional("npackages", "positive_integer", "number of photon packages per wavelength for the reference simulation", int(1e5)) definition.add_optional("dust_grid_relative_scale", "real", "smallest scale of the dust grid relative to the pixelscale of the input maps", 2.) definition.add_optional("dust_grid_min_level", "positive_integer", "level for the dust grid", 2) definition.add_optional("dust_grid_max_mass_fraction", "positive_real", "max mass fraction for the dust grid", 1e-6) # Flags definition.add_flag("transient_heating", "enable transient heating", False) definition.add_flag("selfabsorption", "enable dust selfabsorption", False) # For remote execution definition.add_optional("host_ids", "string_list", "remote hosts to use for heavy computations and simulations", choices=find_host_ids()) definition.add_flag("attached", "launch remote executions in attached mode", False) # Fitting definition.add_optional("ngenerations", "positive_integer", "number of generations", 5) definition.add_optional("nsimulations", "even_positive_integer", "number of simulations per generation", 30) definition.add_optional("npackages_fitting", "positive_integer", "number of photon packages for each fitting simulation", int(1e4)) # Free parameters definition.add_optional("free_parameters", "string_list", "free parameter labels", choices=possible_free_parameters, default=default_free_parameters) definition.add_optional("relative_range_initial", "real_range", "relative range for generating the initial parameter values", default="0.3>3", convert_default=True) definition.add_optional("relative_range_fitting", "real_range", "relative range of the free parameter values for the fitting", default="0.1>20", convert_default=True) # Wavelength grid definition.add_optional("wavelength_range", "quantity_range", "range of wavelengths", "0.1 micron > 2000 micron", convert_default=True)
from pts.core.remote.host import find_host_ids from pts.core.simulation.remote import get_simulation_for_host, get_simulation_ids from pts.core.tools import introspection from pts.core.tools import formatting as fmt from pts.core.tools.stringify import tostr from pts.core.tools import filesystem as fs from pts.core.remote.mounter import mount_remote from pts.core.remote.remote import get_home_path # ----------------------------------------------------------------- # Create the configuration definition definition = ConfigurationDefinition() # Add required definition.add_required("host_ids", "string_list", "names of the remote hosts", choices=find_host_ids()) definition.add_positional_optional("simulation_ids", "integer_list", "simulations to show") definition.add_optional("names", "string_list", "simulation names to show") # Add flag definition.add_flag("analysis", "show analysis options", False) # Add flags definition.add_flag("open_output", "open the output directory (only for one simulation)") definition.add_flag("open_remote_output", "open the remote output directory (only for one simulation)") definition.add_flag("open_input", "open the input directory (only for one simulation)") definition.add_flag("open_remote_input", "open the remote input directory (only for one simulation)") definition.add_flag("open_base", "open the simulation directory (only for one simulation)") definition.add_flag("open_remote_base", "open the remote simulation directory (only for one simulation)") # -----------------------------------------------------------------
#!/usr/bin/env python # -*- coding: utf8 -*- # ***************************************************************** # ** PTS -- Python Toolkit for working with SKIRT ** # ** © Astronomical Observatory, Ghent University ** # ***************************************************************** # Import the relevant PTS classes and modules from pts.core.basics.configuration import ConfigurationDefinition from pts.core.remote.host import find_host_ids # ----------------------------------------------------------------- # Create the configuration definition definition = ConfigurationDefinition() # Add required definition.add_required("remote", "string", "name of the remote host", choices=find_host_ids()) definition.add_positional_optional("id", "positive_integer", "simulation ID") definition.add_optional("name", "string", "simulation name") # Additional relative error definition.add_optional("additional_error", "percentage", "additional percentual error for the observed flux points") # Flags definition.add_flag("write", "write the results", True) # -----------------------------------------------------------------
# ** © Astronomical Observatory, Ghent University ** # ***************************************************************** # Import the relevant PTS classes and modules from pts.core.basics.configuration import ConfigurationDefinition from pts.core.remote.host import find_host_ids from pts.modeling.core.environment import load_modeling_environment_cwd # ----------------------------------------------------------------- # Load the modeling environment and analysis runs environment = load_modeling_environment_cwd() runs = environment.analysis_runs # Determine remote hosts for simulations host_ids = find_host_ids() host_ids_suggestions = find_host_ids( schedulers=False ) # suggest to not use schedulers: requires runtime to be estimated # Determine the remote hosts for heavy computations (making the images) config = environment.modeling_configuration other_host_ids = config.host_ids if other_host_ids is None or len(other_host_ids) == 0: other_host_ids = find_host_ids(schedulers=False) default_other_host_id = other_host_ids[0] # ----------------------------------------------------------------- # Create the configuration definition = ConfigurationDefinition(log_path="log", config_path="config")
# ***************************************************************** # ** PTS -- Python Toolkit for working with SKIRT ** # ** © Astronomical Observatory, Ghent University ** # ***************************************************************** # Import the relevant PTS classes and modules from pts.core.basics.configuration import ConfigurationDefinition from pts.core.remote.host import find_host_ids # ----------------------------------------------------------------- # Create the configuration definition definition = ConfigurationDefinition() # Add required definition.add_positional_optional("remote", "string", "name of the remote host", choices=find_host_ids()) definition.add_positional_optional("matching", "string", "only adapt settings with a name matching this string") definition.add_positional_optional("ids", "integer_list", "simulation IDs (if none specified, all simulation IDs will be used)") definition.add_optional("names", "string_list", "simulation names") definition.add_flag("from_directories", "use directory names as simulation names") # ----------------------------------------------------------------- # Select certain properties definition.add_optional("contains", "string", "only adapt properties containing this string in their name") definition.add_optional("not_contains", "string", "don't adapt properties containing this string in their name") definition.add_optional("exact_name", "string", "only adapt properties with this exact string as their name") definition.add_optional("exact_not_name", "string", "don't adapt properties with this exact string as their name") definition.add_optional("startswith", "string", "only adapt properties whose name starts with this string") definition.add_optional("endswith", "string", "only adapt properties whose name starts with this string")
definition.add_fixed("name", "name of the fitting run", runs.single_name) else: definition.add_required("name", "string", "name of the fitting run", choices=runs.names) # Positional optional parameter definition.add_positional_optional("generation_method", "string", "model generation method", default_generation_method, choices=generation_methods) # Optional parameters if len(find_host_ids()) > 0: definition.add_optional( "remotes", "string_list", "the remote hosts on which to run the parameter exploration", default=find_host_ids(schedulers=False), choices=find_host_ids(schedulers=False)) else: definition.add_fixed("remotes", "remote hosts", []) definition.add_flag("attached", "run remote simulations in attached mode") definition.add_optional( "nsimulations", "even_positive_integer", "the number of simulations to launch in one batch/generation", 100) definition.add_flag("group", "group simulations in larger jobs") definition.add_optional("walltime", "real", "the preferred walltime per job (for schedulers)")