def launch_reference(self): """ This function ... :return: """ # Inform the user log.info("Launching the reference simulation ...") # Settings settings_launch = dict() settings_launch["ski"] = self.reference_ski_path settings_launch["input"] = self.simulation_input_path settings_launch["output"] = self.simulation_output_path settings_launch["create_output"] = True settings_launch["remote"] = self.moderator.host_id_for_single( "reference") settings_launch["attached"] = self.config.attached settings_launch["progress_bar"] = True # Create the analysis options analysis = AnalysisOptions() analysis.extraction.path = self.simulation_extract_path analysis.plotting.path = self.simulation_plot_path analysis.misc.path = self.simulation_misc_path analysis.extraction.progress = True analysis.extraction.timeline = True analysis.extraction.memory = True analysis.plotting.progress = True analysis.plotting.timeline = True analysis.plotting.memory = True analysis.plotting.seds = True analysis.plotting.grids = True analysis.plotting.reference_seds = fs.files_in_path(seds_path) analysis.misc.fluxes = True analysis.misc.images = False analysis.misc.observation_filters = fitting_filter_names analysis.misc.observation_instruments = [instrument_name] analysis.misc.spectral_convolution = self.config.spectral_convolution # Set flux error bars dustpedia_sed = ObservedSED.from_file(dustpedia_sed_path) filter_names = dustpedia_sed.filter_names() errors = dustpedia_sed.errors() flux_errors = sequences.zip_into_dict(filter_names, [str(error) for error in errors]) analysis.misc.flux_errors = flux_errors # Input input_launch = dict() # input_launch["memory"] = MemoryRequirement(serial_memory, parallel_memory) input_launch["analysis_options"] = analysis # Launch command launch = Command("launch_simulation", "launch the reference simulation", settings_launch, input_launch, cwd=".") self.launcher = self.run_command(launch)
def all_input_files(self): """ This function ... :return: """ return fs.files_in_path(self.simulation_input_path)
def create_wcs(self): """ This function ... :return: """ # Inform the user log.info("Creating the WCS ...") min_pixelscale = None # Determine the path to the headers directory headers_path = fs.join(m81_data_path, "headers") # Loop over the header files for path, name in fs.files_in_path(headers_path, extension="txt", returns=["path", "name"]): # Get the filter fltr = parse_filter(name) filter_name = str(fltr) # Set the path of the file for the filter name self.wcs_paths[filter_name] = path # Get WCS wcs = CoordinateSystem.from_header_file(path) # Adjust the pixelscale if min_pixelscale is None: min_pixelscale = wcs.pixelscale self.wcs = wcs elif min_pixelscale > wcs.pixelscale: min_pixelscale = wcs.pixelscale self.wcs = wcs
def make_animation(self): """ This function ... :return: """ # Inform the user log.info("Making animation ...") # Get the best best = self.optimizer.best # Determine path temp_path = self.optimizer.config.path filepath = fs.join(temp_path, "best.png") # Make plot of best # Create animated gif animation = Animation() # Open the images present in the directory for path in fs.files_in_path(temp_path, extension="png", exact_not_name="best"): # Add frame to the animation animation.add_frame_from_file(path) # Save the animation animation_path = fs.join(temp_path, "animation.gif") animation.saveto(animation_path)
def required_input_files(self): """ This function ... :return: """ return fs.files_in_path(self.simulation_input_path, exact_not_name="wavelengths")
def launch_reference(self): """ This function ... :return: """ # Inform the user log.info("Launching the reference simulation ...") # Settings settings_launch = dict() settings_launch["ski"] = self.reference_ski_path settings_launch["input"] = self.simulation_input_path settings_launch["output"] = self.simulation_output_path settings_launch["create_output"] = True settings_launch["remote"] = self.moderator.host_id_for_single("reference") settings_launch["attached"] = self.config.attached settings_launch["progress_bar"] = True # Create the analysis options analysis = AnalysisOptions() analysis.extraction.path = self.simulation_extract_path analysis.plotting.path = self.simulation_plot_path analysis.misc.path = self.simulation_misc_path analysis.extraction.progress = True analysis.extraction.timeline = True analysis.extraction.memory = True analysis.plotting.progress = True analysis.plotting.timeline = True analysis.plotting.memory = True analysis.plotting.seds = True analysis.plotting.grids = True analysis.plotting.reference_seds = fs.files_in_path(seds_path) analysis.misc.fluxes = True analysis.misc.images = False analysis.misc.observation_filters = fitting_filter_names analysis.misc.observation_instruments = [instrument_name] analysis.misc.spectral_convolution = self.config.spectral_convolution # Set flux error bars dustpedia_sed = ObservedSED.from_file(dustpedia_sed_path) filter_names = dustpedia_sed.filter_names() errors = dustpedia_sed.errors() flux_errors = sequences.zip_into_dict(filter_names, [str(error) for error in errors]) analysis.misc.flux_errors = flux_errors # Input input_launch = dict() # input_launch["memory"] = MemoryRequirement(serial_memory, parallel_memory) input_launch["analysis_options"] = analysis # Launch command launch = Command("launch_simulation", "launch the reference simulation", settings_launch, input_launch, cwd=".") self.launcher = self.run_command(launch)
def get_coordinate_system(fltr): """ This function ... :param fltr: :return: """ for path, name in fs.files_in_path(headers_path, extension="txt", returns=["path", "name"]): header_fltr = parse_filter(name) if header_fltr == fltr: return CoordinateSystem.from_header_file(path) return None
def load_images(self): """ This function ... :return: """ # Inform the user log.info("Loading the images ...") # The common wcs of the images wcs = None # Loop over the images in the data directory for path, filename in fs.files_in_path(self.data_path, extension="fits", returns=["path", "name"]): # Load the frame frame = Frame.from_file(path) # Set filter previous_filter = frame.filter frame.filter = parse_filter(filename.split("_norm")[0]) if previous_filter != frame.filter: frame.save() # Check filter if str(frame.filter) not in [ str(fltr) for fltr in self.config.fitting_filters ]: continue # Determine name name = str(frame.filter) # Check wcs if wcs is None: wcs = frame.wcs elif wcs == frame.wcs: pass else: raise IOError("The coordinate system of image '" + filename + "' does not match that of other images") # Debugging log.debug("Adding frame '" + filename + "' ...") # Add to dictionary self.images[name] = frame # Set original path self.image_paths[name] = path # Set the wcs self.wcs = wcs
def load_coordinate_systems(self): """ This function ... :return: """ # Inform the user log.info("Loading the coordinate systems ...") nfilters_stars = 0 nfilters_extra = 0 # Loop over the header files for path, name in fs.files_in_path(headers_path, extension="txt", returns=["path", "name"]): # Get the filter and wavelength fltr = parse_filter(name) wavelength = fltr.effective if fltr.effective is not None else fltr.center # SKip Planck if fltr.observatory == "Planck": continue # Wavelength greater than 25 micron if wavelength > wavelengths.ranges.ir.mir.max: if nfilters_extra == self.config.nfilters_extra: continue else: nfilters_extra += 1 # Wavelength smaller than 25 micron else: if nfilters_stars == self.config.nfilters_stars: continue else: nfilters_stars += 1 # Debugging log.debug("Loading the coordinate system for the '" + str(fltr) + "' filter ...") # Get WCS wcs = CoordinateSystem.from_header_file(path) # Add the coordinate system self.coordinate_systems.append(wcs, fltr=fltr) # Break the loop if we have enough if nfilters_stars == self.config.nfilters_stars and nfilters_extra == self.config.nfilters_extra: break
def load_observed_sed(self): """ This function ... :return: """ # Inform the user log.info("Loading the mock observed sed ...") # Determine path paths = fs.files_in_path(self.simulation_misc_path, contains="_fluxes", extension="dat") # Check if there is only one SED if len(paths) > 1: raise RuntimeError("More than one SED is found") path = paths[0] # Load the observed SED self.observed_sed = ObservedSED.from_file(path)
def create_wcs_not_working(self): """ This function ... :return: """ # Inform the user log.info("Creating the WCS ...") # Determine the path to the headers directory headers_path = fs.join(m81_data_path, "headers") # Loop over the files in the directory ra_range = None dec_range = None min_pixelscale = None for path, name in fs.files_in_path(headers_path, extension="txt", returns=["path", "name"]): # Get the filter fltr = parse_filter(name) # Get WCS wcs = CoordinateSystem.from_header_file(path) # Adjust RA range if ra_range is None: ra_range = wcs.ra_range else: ra_range.adjust(wcs.ra_range) # Adjust DEC range if dec_range is None: dec_range = wcs.dec_range else: dec_range.adjust(wcs.dec_range) # Adjust the pixelscale if min_pixelscale is None: min_pixelscale = wcs.pixelscale elif min_pixelscale > wcs.pixelscale: min_pixelscale = wcs.pixelscale # Create coordinate system # size, center_pixel, center_sky, pixelscale self.wcs = CoordinateSystem.from_ranges(ra_range, dec_range, min_pixelscale)
def make_animation(self): """ This function ... :return: """ # Determine path #temp_path = self.optimizer.config.path # Determine plot path plot_path = fs.join(self.path, "plot") # Create animated gif animation = Animation() # Open the images present in the directory for path in fs.files_in_path(plot_path, extension="png", exact_not_name="best"): # Add frame to the animation animation.add_frame_from_file(path) # Save the animation animation_path = fs.join(self.path, "animation.gif") animation.saveto(animation_path)
# Loop over the directories in the truncation path for path, name in fs.directories_in_path(environment.truncation_path, returns=["path", "name"]): # Determine the path to the lowres directory lowres_path = fs.join(path, "lowres") # Determine the filter fltr = parse_filter(name) filter_name = str(fltr) # Initializ variable the_image_path = None # Find the image corresponding to the specified factor for image_path, image_name in fs.files_in_path(lowres_path, extension="fits", returns=["path", "name"]): # Determine the factor factor = real(image_name) # If the factor corresponds to the specified factor, take this image if np.isclose(factor, config.factor, rtol=0.01): the_image_path = image_path break # Check if the_image_path is None: raise ValueError("No truncated " + filter_name + " image found for a factor of " + str(config.factor)) # Add the image frame = Frame.from_file(the_image_path) plotter.add_image(frame, filter_name)
# ** PTS -- Python Toolkit for working with SKIRT ** # ** © Astronomical Observatory, Ghent University ** # ***************************************************************** ## \package pts.do.developer.remove_pyc Remove all compiled python (.pyc) files. # ----------------------------------------------------------------- # Ensure Python 3 compatibility from __future__ import absolute_import, division, print_function # Import the relevant PTS classes and modules from pts.core.tools import introspection from pts.core.tools import filesystem as fs from pts.core.basics.log import log # ----------------------------------------------------------------- # Loop over all pyc files for path in fs.files_in_path(introspection.pts_package_dir, recursive=True, extension="pyc"): # Inform the user log.info("Removing '" + path + "' ...") # Remove the file fs.remove_file(path) # -----------------------------------------------------------------
if nfiles == 0: log.warning("No files found") else: log.info(str(nfiles) + " files found") for path in paths: if config.full: print(path) else: print(path.split(find_path)[1]) if config.remove: remote.remove_file(path) # LOCALLY else: # Determine path find_path = fs.cwd() # Loop over the files paths = fs.files_in_path(find_path, contains=config.contains, extension=config.extension, recursive=config.recursive, exact_name=config.exact_name, exact_not_name=config.exact_not_name, directory_not_contains=config.directory_not_contains, directory_exact_not_name=config.directory_exact_not_name) nfiles = len(paths) if nfiles == 0: log.warning("No files found") else: log.info(str(nfiles) + " files found") for path in paths: if config.full: print(path) else: print(path.split(find_path)[1]) if config.remove: fs.remove_file(path) # -----------------------------------------------------------------
# Initialize the logger log = logging.setup_log(level=level, path=logfile_path) log.start("Starting check_simulated_images ...") # ----------------------------------------------------------------- modeling_path = config.path galaxy_name = fs.name(modeling_path) fit_best_images_path = fs.join(modeling_path, "fit", "best", "images", "test") simulated = dict() for path, name in fs.files_in_path(fit_best_images_path, extension="fits", returns=["path", "name"]): if name == galaxy_name + "_earth_total": continue frame = Frame.from_file(path) if "2MASS" in str(frame.filter): continue if "I4" in str(frame.filter): continue if "W3" in str(frame.filter): continue if frame.filter is None: raise ValueError("No filter for " + name) simulated[str(frame.filter)] = frame trunc_path = fs.join(modeling_path, "truncated")
# Create the command-line parser parser = argparse.ArgumentParser() parser.add_argument("remote", nargs='?', default=None, help="the name of the remote host to connect to") parser.add_argument("ids", type=parsing.simulation_ids, help="unretrieve the simulations with these ID's") parser.add_argument("--keep", action="store_true", help="add this option to make sure the output is kept remotely after a subsequent retrieve attempt") # Parse the command line arguments arguments = parser.parse_args() # ----------------------------------------------------------------- # Create the remote execution environment remote = Remote() remote.setup(arguments.remote) for path in fs.files_in_path(fs.cwd(), extension="sim"): # Create simulation sim = RemoteSimulation.from_file(path) if not sim.retrieved: continue local_output_path = sim.output_path remote_simulation_path = sim.remote_simulation_path remote_output_path = sim.remote_output_path remote_input_path = sim.remote_input_path if not remote.is_directory(remote_simulation_path): remote.create_directory(remote_simulation_path) if not remote.is_directory(remote_output_path): remote.create_directory(remote_output_path)
attenuation = GalacticAttenuation(center_coordinate) # ----------------------------------------------------------------- # The aniano kernels service aniano = AnianoKernels() # ----------------------------------------------------------------- remote_host_id = "nancy" session = AttachedPythonSession.from_host_id(remote_host_id) # ----------------------------------------------------------------- # Paths paths = fs.files_in_path(galex_images_path, extension="fits", contains="poisson") + fs.files_in_path(sdss_images_path, extension="fits", contains="poisson") # Loop over the GALEX poisson frames for path in paths: # Image name name = fs.strip_extension(fs.name(path)) # Get instrument and band galaxy_name, instrument, band, _ = name.split("_") # Create the filter fltr = BroadBandFilter.from_instrument_and_band(instrument, band) # Load the frame poisson = Frame.from_file(path)
def get_dependencies(self): """ This function ... :return: """ # If no script name is given, list all dependencies of PTS and the # PTS modules that use them if self.config.script is None: if self.config.subprojects: self.dependencies = defaultdict(set) directories_for_subproject = defaultdict(list) # Loop over the subdirectories of the 'do' directory for path, name in fs.directories_in_path( introspection.pts_do_dir, returns=["path", "name"]): subproject = name directories_for_subproject[subproject].append(path) # Loop over the other directories in 'pts' (other than 'do' and 'doc') for path, name in fs.directories_in_path( introspection.pts_package_dir, returns=["path", "name"], exact_not_name=["do", "doc"]): subproject = name directories_for_subproject[subproject].append(path) encountered_internal_modules = set() for subproject in directories_for_subproject: # print(subproject, directories_for_subproject[subproject]) # List the dependencies of the matching script dependencies_for_this = defaultdict(set) for dir_path in directories_for_subproject[subproject]: for file_path in fs.files_in_path(dir_path, extension="py", recursive=True): # if subproject == "dustpedia": print(file_path) introspection.add_dependencies( dependencies_for_this, file_path, encountered_internal_modules) self.dependencies_for_subproject[ subproject] = dependencies_for_this.keys() for dependency in dependencies_for_this: for name in dependencies_for_this[dependency]: self.dependencies[dependency].add(name) # No subprojects are specified, list all PTS dependencies else: self.dependencies = introspection.get_all_dependencies() # If a script name is given else: scripts = introspection.get_scripts() tables = introspection.get_arguments_tables() # Find matching 'do' commands (actual scripts or tabulated commands) matches = introspection.find_matches_scripts( self.config.script, scripts) table_matches = introspection.find_matches_tables( self.config.script, tables) # List the dependencies of the matching script self.dependencies = defaultdict(set) # No match if len(matches) + len(table_matches) == 0: show_all_available(scripts, tables) exit() # More matches elif len(matches) + len(table_matches) > 1: show_possible_matches(matches, table_matches, tables) exit() # Exactly one match from existing do script elif len(matches) == 1 and len(table_matches) == 0: # Determine the full path to the matching script script_path = fs.join(introspection.pts_do_dir, matches[0][0], matches[0][1]) introspection.add_dependencies(self.dependencies, script_path, set()) # Exactly one match from tabulated command elif len(table_matches) == 1 and len(matches) == 0: # from pts.core.tools import logging # configuration # Path to class module table_match = table_matches[0] subproject = table_match[0] index = table_match[1] relative_class_module_path = tables[subproject]["Path"][ index].replace(".", "/").rsplit("/", 1)[0] + ".py" class_module_path = fs.join( introspection.pts_subproject_dir(subproject), relative_class_module_path) logging_path = fs.join(introspection.pts_package_dir, "core", "tools", "logging.py") command_name = tables[subproject]["Command"][index] configuration_name = tables[subproject]["Configuration"][index] if configuration_name == "--": configuration_name = command_name configuration_module_path = fs.join( introspection.pts_root_dir, "pts/" + subproject + "/config/" + configuration_name + ".py") # Add dependencies encountered = set() introspection.add_dependencies(self.dependencies, logging_path, encountered) introspection.add_dependencies(self.dependencies, configuration_module_path, encountered) introspection.add_dependencies(self.dependencies, class_module_path, encountered)
default_mask_color = "black" # ----------------------------------------------------------------- # Set the modeling path modeling_path = verify_modeling_cwd() # Set paths maps_path = fs.join(modeling_path, "maps") maps_components_path = fs.join(maps_path, "components") # ----------------------------------------------------------------- # Get selection indices indices = fs.files_in_path(maps_components_path, extension="dat", returns="name", startswith="selection", convert=int, sort=int, convert_split_index=1, convert_split_pattern="_") # Add the setting if len(indices) == 0: raise RuntimeError("Could not find any selection file") elif len(indices) == 1: definition.add_fixed("selection", "selection to use", indices[0]) else: definition.add_optional("selection", "positive_integer", "selection to use", choices=indices) # ----------------------------------------------------------------- # Show? definition.add_flag("show", "show the page", False) # Remote definition.add_optional("remote", "string", "remote host to use for creating the clip masks", choices=find_host_ids(schedulers=False)) # Flags
# Initialize the logger log = logging.setup_log(level=level, path=logfile_path) log.start("Starting initialize_data ...") # ----------------------------------------------------------------- names_column = [] paths_column = [] prep_names_column = [] names = ["Image name", "Image path", "Preparation name"] # Loop over all subdirectories of the data directory for path, name in fs.directories_in_path(fs.join(config.path, "data"), not_contains="bad", returns=["path", "name"]): # Loop over all FITS files found in the current subdirectory for image_path, image_name in fs.files_in_path(path, extension="fits", not_contains="_Error", returns=["path", "name"]): # Open the image frame frame = Frame.from_file(image_path) # Determine the preparation name if frame.filter is not None: prep_name = str(frame.filter) else: prep_name = image_name # Set the row entries names_column.append(image_name) paths_column.append(image_path) prep_names_column.append(prep_name) # Create the table data = [names_column, paths_column, prep_names_column]
# ----------------------------------------------------------------- names_column = [] paths_column = [] prep_names_column = [] names = ["Image name", "Image path", "Preparation name"] # Loop over all subdirectories of the data directory for path, name in fs.directories_in_path(fs.join(config.path, "data"), not_contains="bad", returns=["path", "name"]): # Loop over all FITS files found in the current subdirectory for image_path, image_name in fs.files_in_path(path, extension="fits", not_contains="_Error", returns=["path", "name"]): # Open the image frame frame = Frame.from_file(image_path) # Determine the preparation name if frame.filter is not None: prep_name = str(frame.filter) else: prep_name = image_name # Set the row entries names_column.append(image_name) paths_column.append(image_path) prep_names_column.append(prep_name) # Create the table
else: remote = None # Determine the path to the run directory for the specified remote host host_run_path = fs.join(introspection.skirt_run_dir, host_id) # Check if there are simulations if not fs.is_directory(host_run_path): log.debug("No run directory for host '" + host_id + "'") continue if fs.is_empty(host_run_path): log.debug("No simulations for host '" + host_id + "'") # Loop over the simulation files in the run directory for path, name in fs.files_in_path(host_run_path, extension="sim", returns=["path", "name"], sort=int): # Skip if config.ids is not None and int(name) not in config.ids: continue # Inform the user log.info("Removing simulation " + name + " ...") # Fully clear if config.full: # Debugging log.debug("Removing the remote files and directories ...") # Load the simulation
# ----------------------------------------------------------------- modeling_path = verify_modeling_cwd() # ----------------------------------------------------------------- data_images_path = get_data_images_path(modeling_path) # ----------------------------------------------------------------- shapes = dict() # Loop over the images for image_path, image_name in fs.files_in_path(data_images_path, extension="fits", not_contains="poisson", returns=["path", "name"], recursive=True, recursion_level=1): # Load the image frame = Frame.from_file(image_path, silent=True) # Determine filter name filter_name = frame.filter_name # Set pixel shape shapes[filter_name] = frame.shape # ----------------------------------------------------------------- # Sort on shape
# ----------------------------------------------------------------- # Find methods file methods_path = fs.join(fs.cwd(), "methods.txt") if not fs.is_file(methods_path): raise IOError("Methods file not found") methods = load_dict(methods_path) # Find origins file origins_path = fs.join(fs.cwd(), "origins.txt") if not fs.is_file(origins_path): raise IOError("Origins file not found") origins = load_dict(origins_path) # ----------------------------------------------------------------- filenames = fs.files_in_path(fs.cwd(), returns="name", extension="fits") # ----------------------------------------------------------------- map_names = sequences.union(methods.keys(), origins.keys(), filenames) # ----------------------------------------------------------------- print("") print("Presumed number of maps: " + str(len(map_names))) print("") print(" - number of map files found: " + str(len(filenames))) print(" - number of entries in 'origins': " + str(len(origins))) print(" - number of entries in 'methods': " + str(len(methods))) print("")
#!/usr/bin/env python # -*- coding: utf8 -*- # ***************************************************************** # ** PTS -- Python Toolkit for working with SKIRT ** # ** © Astronomical Observatory, Ghent University ** # ***************************************************************** ## \package pts.do.magic.verify_fits Verify a batch of FITS files in the current working directory. # ----------------------------------------------------------------- # Ensure Python 3 compatibility from __future__ import absolute_import, division, print_function # Import astronomical modules from astropy.io import fits # Import the relevant PTS classes and modules from pts.core.tools import filesystem as fs # ----------------------------------------------------------------- # Loop over all files in the current path for path in fs.files_in_path(fs.cwd(), extension="fits"): # Open the file hdulist = fits.open(path) # -----------------------------------------------------------------
def launch_reference(self): """ This function ... :return: """ # Inform the user log.info("Launching the reference simulation ...") # Settings settings_launch = dict() settings_launch["ski"] = self.reference_ski_path settings_launch["input"] = self.simulation_input_path settings_launch["output"] = self.simulation_output_path settings_launch["create_output"] = True settings_launch["remote"] = self.host_id settings_launch["attached"] = True settings_launch["show_progress"] = True # Create the analysis options analysis = AnalysisOptions() analysis.extraction.path = self.simulation_extract_path analysis.plotting.path = self.simulation_plot_path analysis.misc.path = self.simulation_misc_path analysis.extraction.progress = True analysis.extraction.timeline = True analysis.extraction.memory = True analysis.plotting.progress = True analysis.plotting.timeline = True analysis.plotting.memory = True analysis.plotting.seds = True analysis.plotting.grids = True analysis.plotting.reference_seds = fs.files_in_path(seds_path) analysis.misc.fluxes = True analysis.misc.images = True analysis.misc.observation_filters = fitting_filter_names analysis.misc.observation_instruments = [instrument_name] analysis.misc.make_images_remote = self.host_id analysis.misc.images_wcs = self.reference_wcs_path analysis.misc.images_unit = "Jy/pix" analysis.misc.spectral_convolution = self.config.spectral_convolution # Set flux error bars dustpedia_sed = ObservedSED.from_file(dustpedia_sed_path) filter_names = dustpedia_sed.filter_names() errors = dustpedia_sed.errors() #print([str(error) for error in errors]) flux_errors = sequences.zip_into_dict(filter_names, [str(error) for error in errors]) analysis.misc.flux_errors = flux_errors # Create Aniano kernels object aniano = AnianoKernels() # Set the paths to the kernel for each image kernel_paths = dict() for filter_name in fitting_filter_names: kernel_paths[filter_name] = aniano.get_psf_path(parse_filter(filter_name)) analysis.misc.images_kernels = kernel_paths # Set the paths to the WCS files for each image analysis.misc.rebin_wcs = {instrument_name: self.wcs_paths} # Input input_launch = dict() #input_launch["memory"] = MemoryRequirement(serial_memory, parallel_memory) input_launch["analysis_options"] = analysis # Launch command launch = Command("launch_simulation", "launch the reference simulation", settings_launch, input_launch, cwd=".") self.launcher = self.run_command(launch)
#print(find_path) #print(remote.items_in_path(find_path, recursive=True)) # Loop over the files paths = remote.files_in_path(find_path, contains=config.contains, not_contains=config.not_contains, extension=config.extension, recursive=config.recursive) if len(paths) == 0: log.warning("No files found") else: for path in paths: if config.full: print(path) else: print(path.split(find_path)[1]) # LOCALLY else: # Determine path find_path = fs.cwd() # Loop over the files paths = fs.files_in_path(find_path, contains=config.contains, extension=config.extension, recursive=config.recursive) if len(paths) == 0: log.warning("No files found") else: for path in paths: if config.full: print(path) else: print(path.split(find_path)[1]) # -----------------------------------------------------------------
def get_dependencies(self): """ This function ... :return: """ # If no script name is given, list all dependencies of PTS and the # PTS modules that use them if self.config.script is None: if self.config.subprojects: self.dependencies = defaultdict(set) directories_for_subproject = defaultdict(list) # Loop over the subdirectories of the 'do' directory for path, name in fs.directories_in_path(introspection.pts_do_dir, returns=["path", "name"]): subproject = name directories_for_subproject[subproject].append(path) # Loop over the other directories in 'pts' (other than 'do' and 'doc') for path, name in fs.directories_in_path(introspection.pts_package_dir, returns=["path", "name"], exact_not_name=["do", "doc"]): subproject = name directories_for_subproject[subproject].append(path) encountered_internal_modules = set() for subproject in directories_for_subproject: # print(subproject, directories_for_subproject[subproject]) # List the dependencies of the matching script dependencies_for_this = defaultdict(set) for dir_path in directories_for_subproject[subproject]: for file_path in fs.files_in_path(dir_path, extension="py", recursive=True): # if subproject == "dustpedia": print(file_path) introspection.add_dependencies(dependencies_for_this, file_path, encountered_internal_modules) self.dependencies_for_subproject[subproject] = dependencies_for_this.keys() for dependency in dependencies_for_this: for name in dependencies_for_this[dependency]: self.dependencies[dependency].add(name) # No subprojects are specified, list all PTS dependencies else: self.dependencies = introspection.get_all_dependencies() # If a script name is given else: scripts = introspection.get_scripts() tables = introspection.get_arguments_tables() # Find matching 'do' commands (actual scripts or tabulated commands) matches = introspection.find_matches_scripts(self.config.script, scripts) table_matches = introspection.find_matches_tables(self.config.script, tables) # List the dependencies of the matching script self.dependencies = defaultdict(set) # No match if len(matches) + len(table_matches) == 0: show_all_available(scripts, tables) exit() # More matches elif len(matches) + len(table_matches) > 1: show_possible_matches(matches, table_matches, tables) exit() # Exactly one match from existing do script elif len(matches) == 1 and len(table_matches) == 0: # Determine the full path to the matching script script_path = fs.join(introspection.pts_do_dir, matches[0][0], matches[0][1]) introspection.add_dependencies(self.dependencies, script_path, set()) # Exactly one match from tabulated command elif len(table_matches) == 1 and len(matches) == 0: # from pts.core.tools import logging # configuration # Path to class module table_match = table_matches[0] subproject = table_match[0] index = table_match[1] relative_class_module_path = tables[subproject]["Path"][index].replace(".", "/").rsplit("/", 1)[0] + ".py" class_module_path = fs.join(introspection.pts_subproject_dir(subproject), relative_class_module_path) logging_path = fs.join(introspection.pts_package_dir, "core", "basics", "log.py") command_name = tables[subproject]["Command"][index] configuration_name = tables[subproject]["Configuration"][index] if configuration_name == "--": configuration_name = command_name configuration_module_path = fs.join(introspection.pts_root_dir, "pts/" + subproject + "/config/" + configuration_name + ".py") # Add dependencies encountered = set() introspection.add_dependencies(self.dependencies, logging_path, encountered) introspection.add_dependencies(self.dependencies, configuration_module_path, encountered) introspection.add_dependencies(self.dependencies, class_module_path, encountered)
for path, name in fs.directories_in_path(environment.truncation_path, returns=["path", "name"]): # Determine the path to the lowres directory lowres_path = fs.join(path, "lowres") # Determine the filter fltr = parse_filter(name) filter_name = str(fltr) # Initializ variable the_image_path = None # Find the image corresponding to the specified factor for image_path, image_name in fs.files_in_path(lowres_path, extension="fits", returns=["path", "name"]): # Determine the factor factor = real(image_name) # If the factor corresponds to the specified factor, take this image if np.isclose(factor, config.factor, rtol=0.01): the_image_path = image_path break # Check if the_image_path is None: raise ValueError("No truncated " + filter_name + " image found for a factor of " + str(config.factor))
# ***************************************************************** # ** PTS -- Python Toolkit for working with SKIRT ** # ** © Astronomical Observatory, Ghent University ** # ***************************************************************** # Import the relevant PTS classes and modules from pts.core.basics.configuration import ConfigurationDefinition from pts.core.tools import introspection from pts.core.tools import filesystem as fs # ----------------------------------------------------------------- # The path to the hosts configuration directory hosts_config_path = fs.join(introspection.pts_config_dir("core"), "hosts") # The names of the hosts which are pre-configured preconfigured_names = fs.files_in_path(hosts_config_path, extension="cfg", not_contains="template", returns="name") # ----------------------------------------------------------------- # Create configuration definition definition = ConfigurationDefinition(write_config=False) # Add required definition.add_required("name", "nonempty_string_no_spaces", "name to give to the host") # Add optional definition.add_positional_optional("preconfigured", "string", "name of a preconfigured remote for which to adapt the user-specific settings", choices=preconfigured_names) # -----------------------------------------------------------------
if not remote.setup(host_id): log.warning("The remote host '" + host_id + "' is not available: skipping ...") continue else: remote = None # Determine the path to the run directory for the specified remote host host_run_path = fs.join(introspection.skirt_run_dir, host_id) # Check if there are simulations if not fs.is_directory(host_run_path): log.debug("No run directory for host '" + host_id + "'") continue if fs.is_empty(host_run_path): log.debug("No simulations for host '" + host_id + "'") # Loop over the simulation files in the run directory for path, name in fs.files_in_path(host_run_path, extension="sim", returns=["path", "name"], sort=int): # Skip if config.ids is not None and int(name) not in config.ids: continue # Inform the user log.info("Removing simulation " + name + " ...") # Fully clear if config.full: # Debugging log.debug("Removing the remote files and directories ...") # Load the simulation simulation = RemoteSimulation.from_file(path)
# ** © Astronomical Observatory, Ghent University ** # ***************************************************************** # Import the relevant PTS classes and modules from pts.core.basics.configuration import ConfigurationDefinition from pts.core.tools import introspection from pts.core.tools import filesystem as fs # ----------------------------------------------------------------- # The path to the hosts configuration directory hosts_config_path = fs.join(introspection.pts_config_dir("core"), "hosts") # The names of the hosts which are pre-configured preconfigured_names = fs.files_in_path(hosts_config_path, extension="cfg", not_contains="template", returns="name") # ----------------------------------------------------------------- # Create configuration definition definition = ConfigurationDefinition(write_config=False) # Add required definition.add_required("name", "nonempty_string_no_spaces", "name to give to the host") # Add optional definition.add_positional_optional( "preconfigured", "string",
"--keep", action="store_true", help= "add this option to make sure the output is kept remotely after a subsequent retrieve attempt" ) # Parse the command line arguments arguments = parser.parse_args() # ----------------------------------------------------------------- # Create the remote execution environment remote = Remote() remote.setup(arguments.remote) for path in fs.files_in_path(fs.cwd(), extension="sim"): # Create simulation sim = RemoteSimulation.from_file(path) if not sim.retrieved: continue local_output_path = sim.output_path remote_simulation_path = sim.remote_simulation_path remote_output_path = sim.remote_output_path remote_input_path = sim.remote_input_path if not remote.is_directory(remote_simulation_path): remote.create_directory(remote_simulation_path) if not remote.is_directory(remote_output_path):
# -*- coding: utf8 -*- # ***************************************************************** # ** PTS -- Python Toolkit for working with SKIRT ** # ** © Astronomical Observatory, Ghent University ** # ***************************************************************** ## \package pts.do.developer.remove_pyc Remove all compiled python (.pyc) files. # ----------------------------------------------------------------- # Ensure Python 3 compatibility from __future__ import absolute_import, division, print_function # Import the relevant PTS classes and modules from pts.core.tools import introspection from pts.core.tools import filesystem as fs from pts.core.basics.log import log # ----------------------------------------------------------------- # Loop over all pyc files for path in fs.files_in_path(introspection.pts_package_dir, recursive=True, extension="pyc"): # Inform the user log.info("Removing '" + path + "' ...") # Remove the file fs.remove_file(path) # -----------------------------------------------------------------