def execute_detached(self, line, timeout=None): """ This function ... :param line: :param timeout: :return: """ # Construct the command if self.tmux: send_command = 'tmux send-keys -t ' + self.screen_name + ' "' + line + '" Enter' else: if "'" not in line: send_command = "screen -S " + self.screen_name + " -p 0 -X stuff '" + line + "\n'" elif '"' not in line: send_command = 'screen -S ' + self.screen_name + ' -p 0 -X stuff "' + line + '\n"' else: raise ValueError( "Line cannot contain both single quotes and double quotes") # Debugging log.debug("The command to execute the line is:") log.debug(send_command) # Send the line output = self.remote.execute(send_command, show_output=log.is_debug(), timeout=timeout) # Sleep for a while so that we are sure that the actual python stuff has reached the interactive python session within the screen time.wait(5) # in seconds # Return the output return output
def start_session(self, python_command="python"): """ This function ... :param python_command: :return: """ # Inform the user log.info("Starting session for python ...") # Construct the command if self.tmux: command = "tmux new -d -s " + self.screen_name + " " + python_command elif self.output_path is not None: command = "screen -dmS " + self.screen_name + " -L" else: command = "screen -dmS " + self.screen_name # Debugging log.debug("Starting session with the command:") log.debug(command) # Execute the command self.remote.execute(command, show_output=True, cwd=self.output_path) # Using screen if not self.tmux: start_python_command = "screen -S " + self.screen_name + " -p 0 -X stuff '" + python_command + "\n'" self.remote.execute(start_python_command)
def restore_chi_squared(self): """ This function ... :return: """ # Inform the user log.info("Restoring the chi squared tables ...") # Loop over the generations for generation_name in self.generation_names: # Has generation? if not self.has_generation(generation_name): continue # Debugging log.debug("Creating backup of chi squared table for generation '" + generation_name + "' ...") # Get the generation generation = self.generations[generation_name] # Get the generation path generation_path = self.get_generation_restore_path(generation_name) # Determine the filepath filepath = fs.join(generation_path, "chi_squared.dat") # Copy the file fs.copy_file(filepath, generation.chi_squared_table_path)
def load_frame(self): """ This function ... :return: """ # Debugging log.debug("Loading the frame ...") # Import the image importer = ImageImporter() importer.run(self.config.image) # Get the primary image frame frame = importer.image.primary # Get the original header self.header = importer.image.original_header # Get the original nan pixels self.nans = frame.nans # Set the NaN pixels to zero in the frame frame[self.nans] = 0.0 # Set the frame self.frame = frame
def execute_detached(self, line, timeout=None): """ This function ... :param line: :param timeout: :return: """ # Construct the command if self.tmux: send_command = 'tmux send-keys -t ' + self.screen_name + ' "' + line + '" Enter' else: if "'" not in line: send_command = "screen -S " + self.screen_name + " -p 0 -X stuff '" + line + "\n'" elif '"' not in line: send_command = 'screen -S ' + self.screen_name + ' -p 0 -X stuff "' + line + '\n"' else: raise ValueError("Line cannot contain both single quotes and double quotes") # Debugging log.debug("The command to execute the line is:") log.debug(send_command) # Send the line output = self.remote.execute(send_command, show_output=log.is_debug, timeout=timeout) # Sleep for a while so that we are sure that the actual python stuff has reached the interactive python session within the screen time.wait(5) # in seconds # Return the output return output
def initialize_frames(self): """ This function ... :return: """ # Inform the user log.info("Initializing the frames ...") # Loop over the filters for fltr in self.coordinate_systems.filters: # Debugging log.debug("Initializing the '" + str(fltr) + "' frame ...") # Get the wcs wcs = self.coordinate_systems[fltr] # Create new frame frame = Frame.zeros(wcs.shape) # Add the wcs frame.wcs = wcs # Set the filter frame.filter = fltr # Set the unit frame.unit = "Jy" # Add the frame self.frames[fltr] = frame
def load_frame(self): """ This function ... :return: """ # Debugging log.debug("Loading the frame ...") # Determine the full path to the image #image_path = fs.absolute_path(config.image) # Import the image importer = ImageImporter() importer.run(image_path) # Get the primary image frame frame = importer.image.primary # Get the original header header = importer.image.original_header # Create a mask of the pixels that are NaNs nans = frame.nans() # Set the NaN pixels to zero in the frame frame[nans] = 0.0
def write_tour_to_img(self, tour): """ The function to plot the graph :param tour: """ padding = 20 coords = [(x+padding,y+padding) for (x,y) in self.coordinates] maxx, maxy = 0, 0 for x, y in coords: maxx = max(x, maxx) maxy = max(y, maxy) maxx += padding maxy += padding img = Image.new("RGB",(int(maxx),int(maxy)),color=(255,255,255)) font = ImageFont.load_default() d = ImageDraw.Draw(img) num_cities = len(tour) # Loop over the cities for i in range(num_cities): j = (i+1) % num_cities city_i = tour[i] city_j = tour[j] x1,y1 = coords[city_i] x2,y2 = coords[city_j] d.line((int(x1),int(y1),int(x2),int(y2)),fill=(0,0,0)) d.text((int(x1)+7,int(y1)-5),str(i),font=font,fill=(32,32,32)) for x, y in coords: x, y = int(x),int(y) d.ellipse((x-5,y-5,x+5,y+5),outline=(0,0,0),fill=(196,196,196)) del d # Save if self.output_path is not None: # Determine the plot path path = fs.join(self.output_path, str(self.counter) + ".png") # Debugging log.debug("Saving the plot to '" + path + "' ...") # Save img.save(path, "PNG") else: raise RuntimeError("Cannot show the plot, specify an output path")
def import_package_update(self, name, as_name=None, from_name=None, show_output=False): """ Thins function ... :param name: :param as_name: :param from_name: :param show_output: :return: """ # Try to import success, module_name, import_statement = self.import_package(name, as_name=as_name, from_name=from_name, show_output=show_output, return_false_if_fail=True, return_failed_module=True) # Not succesful, try updating the module on which it failed if not success: # No problem module, show import statement if module_name is None: raise RuntimeError("Unsolvable import error occured at: '" + import_statement + "'") # Debugging log.debug("Import of '" + name + "' was unsuccesful: trying updating the '" + module_name + "' package ...") from ..prep.update import update_pts_dependencies_remote # Find conda conda_installation_path, conda_main_executable_path = self.remote.find_conda() # Determine the conda environment for PTS env_name = self.remote.conda_environment_for_pts if env_name is None: raise Exception("Cannot determine the conda environment used for pts") conda_environment = env_name environment_bin_path = fs.join(conda_installation_path, "envs", conda_environment, "bin") if not self.remote.is_directory(environment_bin_path): raise RuntimeError("The environment directory is not present") conda_executable_path = fs.join(environment_bin_path, "conda") # Set ... conda_pip_path = fs.join(environment_bin_path, "pip") conda_python_path = fs.join(environment_bin_path, "python") conda_easy_install_path = fs.join(environment_bin_path, "easy_install") # Update the module packages = [module_name] update_pts_dependencies_remote(self.remote, packages, conda_executable_path, conda_environment, conda_python_path, conda_pip_path, conda_easy_install_path) # Try again success, module_name, import_statement = self.import_package(name, as_name=as_name, from_name=from_name, show_output=show_output, return_false_if_fail=True, return_failed_module=True) # Not succesful again if not success: raise ImportError("The import statement '" + import_statement + "' failed on remote host '" + self.host_id + "'")
def restore_best_parameters(self): """ This function ... :return: """ # Debugging log.debug("Restoring the best parameters table ...") # Copy the file fs.copy_file(self.restore_best_parameters_path, self.fitting_run.path)
def restore_weights(self): """ This function ... :return: """ # Debugging log.debug("Restoring the weights ...") # Copy the weights fs.copy_file(self.restore_weights_path, self.fitting_run.path)
def load_regions(self): """ This function ... :return: """ # Debugging log.debug("Loading the regions ...") # Load the region self.regions = load_as_pixel_region_list(self.config.regions, self.wcs, only=self.config.shapes, color=self.config.color, ignore_color=self.config.ignore_color)
def load_images(self): """ This function ... :return: """ # Inform the user log.info("Loading the images ...") # The common wcs of the images wcs = None # Loop over the images in the data directory for path, filename in fs.files_in_path(self.data_path, extension="fits", returns=["path", "name"]): # Load the frame frame = Frame.from_file(path) # Set filter previous_filter = frame.filter frame.filter = parse_filter(filename.split("_norm")[0]) if previous_filter != frame.filter: frame.save() # Check filter if str(frame.filter) not in [ str(fltr) for fltr in self.config.fitting_filters ]: continue # Determine name name = str(frame.filter) # Check wcs if wcs is None: wcs = frame.wcs elif wcs == frame.wcs: pass else: raise IOError("The coordinate system of image '" + filename + "' does not match that of other images") # Debugging log.debug("Adding frame '" + filename + "' ...") # Add to dictionary self.images[name] = frame # Set original path self.image_paths[name] = path # Set the wcs self.wcs = wcs
def restore_prob(self): """ This function ... :return: """ # Debugging log.debug("Restoring the probabilities ...") # Copy the directory fs.clear_directory(self.fitting_run.prob_path, recursive=True) fs.copy_directory(self.restore_prob_path, self.fitting_run.path)
def restore_best(self): """ This function ... :return: """ # Debugging log.debug("Restoring the best simulations ...") # Copy the directory fs.clear_directory(self.fitting_run.best_path, recursive=True) fs.copy_directory(self.restore_best_path, self.fitting_run.path)
def load_coordinate_systems(self): """ This function ... :return: """ # Inform the user log.info("Loading the coordinate systems ...") nfilters_stars = 0 nfilters_extra = 0 # Loop over the header files for path, name in fs.files_in_path(headers_path, extension="txt", returns=["path", "name"]): # Get the filter and wavelength fltr = parse_filter(name) wavelength = fltr.effective if fltr.effective is not None else fltr.center # SKip Planck if fltr.observatory == "Planck": continue # Wavelength greater than 25 micron if wavelength > wavelengths.ranges.ir.mir.max: if nfilters_extra == self.config.nfilters_extra: continue else: nfilters_extra += 1 # Wavelength smaller than 25 micron else: if nfilters_stars == self.config.nfilters_stars: continue else: nfilters_stars += 1 # Debugging log.debug("Loading the coordinate system for the '" + str(fltr) + "' filter ...") # Get WCS wcs = CoordinateSystem.from_header_file(path) # Add the coordinate system self.coordinate_systems.append(wcs, fltr=fltr) # Break the loop if we have enough if nfilters_stars == self.config.nfilters_stars and nfilters_extra == self.config.nfilters_extra: break
def create_dataset(self, masks=None): """ This function ... :param masks: :return: """ # Inform the user log.info("Creating the dataset ...") # Initialize self.dataset = DataSet() # Add the frames for fltr in self.frames: # Determine name for the image name = str(fltr) # Determine path for this frame path = fs.join(self.data_frames_path, name + ".fits") # Debugging log.debug("Saving the frame ...") # Save the frame self.frames[fltr].saveto(path) # Debugging log.debug("Adding the '" + name + "' image to the dataset ...") # Add the frame to the dataset self.dataset.add_path(name, path) # Determine the path for the mask mask_path = fs.join(self.data_masks_path, name + ".fits") # Mask if masks is not None and fltr in masks: # Debugging log.debug("Saving the mask ...") # Save masks[fltr].saveto(mask_path) # Debugging log.debug("Adding mask ...") # Add the mask self.dataset.add_mask_path(name, mask_path) # Determine database path path = fs.join(self.path, "database.dat") # Write the dataset self.dataset.saveto(path)
def load_regions(self): """ This function ... :return: """ # Debugging log.debug("Loading the regions ...") # Load the region region_path = fs.join(input_path, config.regions) self.regions = load_as_pixel_region_list( region_path, frame.wcs, only=config.shapes, color=config.color, ignore_color=config.ignore_color)
def create_pipe(self): """ This function ... :return: """ # Create pipe file out_pipe_filename = "out_pipe_" + self.session_id + ".txt" self.out_pipe_filepath = fs.join(self.remote.pts_temp_path, out_pipe_filename) # Debugging log.debug("Creating pipe file '" + self.out_pipe_filepath + "' on remote ...") # Create the pipe file for output if self.remote.is_file(self.out_pipe_filepath): self.remote.remove_file(self.out_pipe_filepath) #self.remote.touch(self.out_pipe_filepath) #self.remote.write_line(self.out_pipe_filepath, "") self.remote.touch_alternative(self.out_pipe_filepath)
def restore_fluxes_plots(self): """ This function ... :return: """ # Inform the user log.info("Making a backup of the mock fluxes plots ...") # Loop over the generations for generation_name in self.generation_names: # Has generation? if not self.has_generation(generation_name): continue # Debugging log.debug("Creating backups for generation '" + generation_name + "' ...") # Get the generation generation = self.generations[generation_name] # Get restore path generation_path = self.get_generation_restore_path(generation_name) # Loop over the simulations for simulation_name in generation.simulation_names: # Set simulation path simulation_path = fs.join(generation_path, simulation_name) # Check whether fluxes are present filepath = fs.join(simulation_path, "earth_fluxes.pdf") if not fs.is_file(filepath): log.warning("No mock SED plot for simulation '" + simulation_name + "' of generation '" + generation_name + "'") continue # Debugging log.debug("Restoring fluxes plot for simulation '" + simulation_name + "' ...") # Copy the plot file fs.copy_file(filepath, generation.get_mock_sed_plot_path(simulation_name))
def restore_differences(self): """ This function ... :return: """ # Inform the user log.info("Making a backup of the flux differences ...") # Loop over the generations for generation_name in self.generation_names: # Has generation? if not self.has_generation(generation_name): continue # Debugging log.debug("Creating backups for generation '" + generation_name + "' ...") # Get the generation generation = self.generations[generation_name] # Get restore path generation_path = self.get_generation_restore_path(generation_name) # Loop over the simulations for simulation_name in generation.simulation_names: # Set simulation path simulation_path = fs.join(generation_path, simulation_name) # Check whether the simulation has differences filepath = fs.join(simulation_path, "differences.dat") if not fs.is_file(filepath): log.warning("No differences table for simulation '" + simulation_name + "' of generation '" + generation_name + "'") continue # Debugging log.debug("Restoring differences table for simulation '" + simulation_name + "' ...") # Copy the file fs.copy_file(filepath, generation.get_simulation_sed_differences_path(simulation_name))
def get_best_parameter_values(self): """ This function ... :return: """ # Inform the user log.info("Getting the best parameter values ...") # Get the best parameter values self.best_parameter_values, self.best_chi_squared = self.modeler.modeler.fitter.fitting_run.best_parameter_values_and_chi_squared # Debugging log.debug("The best parameter values are:") log.debug("") for parameter_name in self.best_parameter_values: log.debug(" - " + parameter_name + ": " + tostr(self.best_parameter_values[parameter_name], scientific=True, fancy=True, ndigits=parameter_ndigits[parameter_name])) log.debug("") # Debugging log.debug("The best chi squared value is " + str(self.best_chi_squared))
def send_line_and_raise(self, command, show_output=False, timeout=None): """ This function ... :param command: :param show_output: :param timeout: :return: """ # Send the command and get the output output = self.send_line(command, show_output=show_output, timeout=timeout) # There is output if len(output) > 0: # Get the last line last_line = output[-1] # Check for errors if "Error: " in last_line: # Get the error type error_class_name = last_line.split(": ")[0] try: error_class = eval(error_class_name) error_specifier = "" except NameError: error_class = Exception error_specifier = " (" + error_class_name + ") " message = last_line.split(error_class_name + ": ")[1] message = "[" + self.host_id + "] " + error_specifier + message # Re-raise the remote error raise error_class(message) # Show the output (no errors) if debugging and show_output is False if not show_output: # Output was not shown in realtime for line in output: log.debug("[" + self.host_id + "] " + line)
def set_fwhms(self): """ This function ... :return: """ # Inform the user log.info("Setting the FWHMs ...") # Loop over the filters for fltr in self.frames: # Get the fwhm if has_variable_fwhm(fltr): fwhm = fwhms[str(fltr)] else: fwhm = get_fwhm(fltr) # Debugging log.debug("The FWHM of the '" + str(fltr) + "' image is " + stringify.stringify(fwhm)[1]) # Set self.real_fwhms[fltr] = fwhm
def make_rotation_masks(self): """ This function ... :return: """ # Inform the user log.info("Making rotation masks ...") # Loop over the filters for fltr in self.frames: # Debugging log.info("Making rotation mask for the '" + str(fltr) + "' image ...") # Get the frame frame = self.frames[fltr] # Rotate if self.config.rotate: # Choose a random rotation angle angle = Angle(np.random.uniform(-90, 90), "deg") # Debugging log.debug("The random rotation angle is '" + stringify.stringify(angle)[1] + "'") # Create mask mask = frame.rotation_mask(angle) # Don't rotate else: mask = Mask.empty_like(frame) # Set the mask self.rotation_masks[fltr] = mask
def generate_initial_parameter_values(self): """ This function ... :return: """ # Inform the user log.info("Generating random initial parameter values ...") # Get the low and high value low_factor = self.config.relative_range_initial.min high_factor = self.config.relative_range_initial.max # Determine the exponents, to generate random points log_low = np.log10(low_factor) log_high = np.log10(high_factor) # Loop over the real parameter values for parameter_name in self.real_parameter_values: # Get the parameter value value = self.real_parameter_values[parameter_name] # Multiply the value with a random number between 1/3 and 3. random = np.random.uniform(log_low, log_high) random_factor = 10**random value = value * random_factor # DON'T DO VALUE *= RANDOM_FACTOR HERE: CHANGES THE UNDERLYING QUANTITY OBJECT AS WELL IN SELF.REAL_PARAMETER_VALUES !! # Set the value as the initial parameter value self.initial_parameter_values[parameter_name] = value # Debugging log.debug("The initial parameter values are:") log.debug("") for parameter_name in self.real_parameter_values: log.debug(" - " + parameter_name + ": " + tostr(self.initial_parameter_values[parameter_name], scientific=True, fancy=True, ndigits=parameter_ndigits[parameter_name])) log.debug("")
def generate_initial_parameter_values(self): """ This function ... :return: """ # Inform the user log.info("Generating random initial parameter values ...") # Get the low and high value low_factor = self.config.relative_range_initial.min high_factor = self.config.relative_range_initial.max # Determine the exponents, to generate random points log_low = np.log10(low_factor) log_high = np.log10(high_factor) # Loop over the real parameter values for parameter_name in self.real_parameter_values: # Get the parameter value value = self.real_parameter_values[parameter_name] # Multiply the value with a random number between 1/3 and 3. random = np.random.uniform(log_low, log_high) random_factor = 10 ** random value = value * random_factor # DON'T DO VALUE *= RANDOM_FACTOR HERE: CHANGES THE UNDERLYING QUANTITY OBJECT AS WELL IN SELF.REAL_PARAMETER_VALUES !! # Set the value as the initial parameter value self.initial_parameter_values[parameter_name] = value # Debugging log.debug("The initial parameter values are:") log.debug("") for parameter_name in self.real_parameter_values: log.debug(" - " + parameter_name + ": " + tostr(self.initial_parameter_values[parameter_name], scientific=True, fancy=True, ndigits=parameter_ndigits[parameter_name])) log.debug("")
def make_point_sources_variable_fwhm(self, masks=None): """ THis function ... :param masks: :return: """ # Inform the user log.info("Making point sources with a variable FWHM in each band ...") # Loop over the 'star' filters for fltr in self.star_filters: # Debugging log.debug("Making point sources for the '" + str(fltr) + "' image ...") # Get y and x if not self.config.only_local: y, x = np.indices(self.frames[fltr].shape) else: y = x = None # Get pixelscale pixelscale = self.frames[fltr].average_pixelscale # Debugging log.debug("The pixelscale of the image is " + stringify.stringify(pixelscale)[1]) # Determine the FWHM in pixel coordinates fwhm_pix = self.real_fwhms[fltr].to("arcsec").value / pixelscale.to("arcsec").value counter = 0 # Loop over the coordinates in the point sources catalog for coordinate in self.point_source_catalog.coordinates(): counter += 1 # Debugging log.debug("Adding point source " + str(counter) + " of " + str(len(self.point_source_catalog)) + " ...") # Check whether it falls in the frame if not self.frames[fltr].contains(coordinate): continue # Convert into pixel coordinate pixel_coordinate = coordinate.to_pixel(self.frames[fltr].wcs) # Get the corresponding pixel pixel = Pixel.for_coordinate(pixel_coordinate) # Check whether not masked if masks is not None and fltr in masks and masks[fltr][pixel.y, pixel.x]: continue # Generate random deviation x_deviation = np.random.normal(0.0, 1.) y_deviation = np.random.normal(0.0, 1.) # Debugging log.debug("Random pixel position deviation is (" + str(x_deviation) + ", " + str(y_deviation) + ")") # Alter pixel coordinate pixel_coordinate.x += x_deviation pixel_coordinate.y += y_deviation # Generate random deviation from FWHM fwhm_deviation = np.random.normal(0.0, 0.05 * fwhm_pix) # Debugging log.debug("Random FWHM deviation (on a FWHM of " + str(fwhm_pix) + ") is " + str(fwhm_deviation)) # Add the deviation fwhm_pix += fwhm_deviation # Generate a random amplitude (from 100 till 100 000) amplitude_exponent = np.random.uniform(2., 5.) amplitude = 10**amplitude_exponent # Determine sigma sigma = statistics.fwhm_to_sigma * fwhm_pix # Only 'render' the Gaussian locally if self.config.only_local: min_x, max_x, min_y, max_y, rel_center = determine_patch_for_psf(pixel_coordinate, amplitude, sigma, max_x_pixel=self.frames[fltr].xsize-1, max_y_pixel=self.frames[fltr].ysize-1, keep_in_frame=True) # Render each Gaussian over the entire frame else: min_x = max_x = min_y = max_y = None rel_center = pixel_coordinate # Make the model model = create_model(self.config.psf_model, amplitude, rel_center, sigma) # Evaluate the model if self.config.only_local: y, x = np.indices((max_y - min_y, max_x - min_x)) data = model(x, y) # Add the data if self.config.only_local: self.frames[fltr][min_y:max_y, min_x:max_x] += data else: self.frames[fltr] += data
new_filepath = fs.appended_filepath(config.filepath, "_backup") fs.copy_file(config.filepath, new_filepath) # ----------------------------------------------------------------- # Inform the user log.info("Loading the image '" + config.filepath + "' ...") # Load the image image = Image.from_file(config.filepath) # ----------------------------------------------------------------- # Debugging sum_before = np.nansum(image.primary.data) log.debug("Sum of the primary frame before multiplication is " + str(sum_before)) # ----------------------------------------------------------------- # Inform th euser log.info("Multiplying the image with a factor of " + str(config.factor) + " ...") # Multiply image *= config.factor # ----------------------------------------------------------------- # Debugging sum_after = np.nansum(image.primary.data) log.debug("Sum of the primary frame after multiplication is " + str(sum_after))
def __del__(self): """ This function ... :return: """ # Debugging if self.output_path is not None: log.debug("Screen output has been placed in '" + self.output_path + "'") # Detach if we are still attached if self.attached: self.detach() # Remove things if we are not in debug mode if not log.is_debug: # Using tmux if self.tmux: end_session_command = "tmux kill-session -t " + self.screen_name self.remote.execute(end_session_command) # Using screen else: # Stop the python session end_python_command = "screen -r -S " + self.screen_name + " -X stuff 'exit()\n'" self.remote.execute(end_python_command) # Stop screen session self.remote.kill_screen(self.screen_name) # Remove the pipe file self.remote.remove_file(self.out_pipe_filepath) else: # Debugging info log.debug("Not closing session and removing pipe in debug mode") if self.tmux: log.debug(" - Tmux session name: " + self.screen_name) else: log.debug(" - Screen session name: " + self.screen_name) log.debug(" - pipe file path: " + self.out_pipe_filepath) if self.output_path is not None: log.debug(" - screen log file: " + self.screenlog_path)
if modules.names["mpi"] is not None: if modules.names["mpi"] != modules.names["cpp"]: installation_commands["compilers"].append(Map(line="module load " + modules.names["mpi"], comment="Load the MPI module")) # ----------------------------------------------------------------- installation_commands["build"].append(Map(line="cd " + skirt_repo_path, comment="Navigate to the SKIRT repository directory")) qmake_path = modules.paths["qmake"] make_make_command = qmake_path + " BuildSKIRT.pro -o ../release/Makefile CONFIG+=release" nthreads = remote.cores_per_socket make_command = "make -j " + str(nthreads) + " -w -C ../release" # Debugging log.debug("Make commands:") log.debug(" 1) " + make_make_command) log.debug(" 2) " + make_command) #log.debug("in directory " + skirt_repo_path) # Configure #output = remote.execute(make_make_command, show_output=log.is_debug(), cwd=skirt_repo_path) # Overwrite the git version #git_version_content = 'const char* git_version = " ' + git_version + ' " ;' #git_version_path = fs.join(skirt_repo_path, "SKIRTmain", "git_version.h") #write_command = 'echo "' + git_version_content + '" > ' + git_version_path #remote.execute(write_command) # Make #output = remote.execute(make_command, show_output=log.is_debug(), cwd=skirt_repo_path)
config = parse_arguments("remotes", definition, "Check the status of the remotes") # ----------------------------------------------------------------- # Set log level in a special way if config.debug: setup_log("DEBUG") else: setup_log("ERROR") # ----------------------------------------------------------------- # Loop over the hosts print("") for host in config.hosts: # Debugging log.debug("Connecting to remote host '" + host.id + "' ...") # Create remote, try connecting remote = Remote() connected = remote.setup(host, login_timeout=10) # only try for 10 seconds # Connection succeeded? if connected: print(fmt.green + host.id + ": up" + fmt.reset) else: print(fmt.red + host.id + ": down" + fmt.reset) continue # Show clustername if remote.cluster_name is not None: print("Cluster name: " + remote.cluster_name) print("")
for prep_name in fix: # Get the image paths after and including the extinction correction step paths = get_step_image_paths_with_cached(modeling_path, prep_name, environment.cache_host_id, after_step="extinction", inclusive=True) # Determine the correction factor actual = float(fix[prep_name][0]) mistaken = float(fix[prep_name][1]) factor = 10**(actual - mistaken) # Debugging log.debug("The correction factor for the " + prep_name + " image is " + str(factor)) correction_factors[prep_name] = factor #print(paths) #continue # Loop over the images for step in paths: # Get the path path = paths[step] # Check whether the correction has already been performed backup_filepath = fs.appended_filepath(path, "_backup") if fs.is_file(path): # local
# Check whether the remote is available if config.full: remote = Remote() if not remote.setup(host_id): log.warning("The remote host '" + host_id + "' is not available: skipping ...") continue else: remote = None # Determine the path to the run directory for the specified remote host host_run_path = fs.join(introspection.pts_run_dir, host_id) # Check if there are tasks if not fs.is_directory(host_run_path): log.debug("No run directory for host '" + host_id + "'") continue if fs.is_empty(host_run_path): log.debug("No tasks for host '" + host_id + "'") # Loop over the task files in the run directory for the host for path, name in fs.files_in_path(host_run_path, extension="task", returns=["path", "name"], sort=int): # Skip if config.ids is not None and int(name) not in config.ids: continue # Inform the user log.info("Removing task " + name + " ...")
def plot_galaxy_components(components, draw=True, show=True, shape=128, unit="pc", width=700, height=800, style='light', **kwargs): """ This function .... :param components: :param draw: :param show: :param shape: :param unit: :param width: :param height: :param style: :param kwargs: :return: """ # Determine the limits limits = determine_model_limits(components, unit, symmetric=True) # Debugging log.debug("Plot limits: " + str(limits)) # Create coordinate data x, y, z, r, theta, phi = xyz(shape=shape, limits=limits, spherical=True) data = r * 0 # Loop over the components for name in components: # Debugging log.debug("Computing the density of the " + name + " component ...") component = components[name] density = component.density_function(normalize=True)(x, y, z) data += density # DRAW FIGURE if draw: # :param lighting: boolean, to use lighting or not, if set to false, lighting parameters will be overriden # :param data_min: minimum value to consider for data, if None, computed using np.nanmin # :param data_max: maximum value to consider for data, if None, computed using np.nanmax # :param tf: transfer function (see ipyvolume.transfer_function, or use the argument below) # :param stereo: stereo view for virtual reality (cardboard and similar VR head mount) # :param width: width of rendering surface # :param height: height of rendering surface # :param ambient_coefficient: lighting parameter # :param diffuse_coefficient: lighting parameter # :param specular_coefficient: lighting parameter # :param specular_exponent: lighting parameter # :param downscale: downscale the rendering for better performance, for instance when set to 2, a 512x512 canvas will show a 256x256 rendering upscaled, but it will render twice as fast. # :param level: level(s) for the where the opacity in the volume peaks, maximum sequence of length 3 # :param opacity: opacity(ies) for each level, scalar or sequence of max length 3 # :param level_width: width of the (gaussian) bumps where the opacity peaks, scalar or sequence of max length 3 # :param kwargs: extra argument passed to Volume and default transfer function # DEFAULT: # lighting=False, data_min=None, data_max=None, tf=None, stereo=False, # width=400, height=500, # ambient_coefficient=0.5, diffuse_coefficient=0.8, # specular_coefficient=0.5, specular_exponent=5, # downscale=1, # level=[0.1, 0.5, 0.9], opacity=[0.01, 0.05, 0.1], level_width=0.1, level = [0.2] #opacity = [0.05, 0.0, 0.0] opacity = [0.08, 0.0, 0.0] level_width = 0.2 level_width = [level_width] * 3 kwargs = dict() kwargs["width"] = width kwargs["height"] = height kwargs["stereo"] = False kwargs["level"] = level kwargs["opacity"] = opacity kwargs["level_width"] = level_width kwargs["downscale"] = 1 # Create transfer function arguments tf_kwargs = {} # Clip off lists min_length = min(len(level), len(level_width), len(opacity)) level = list(level[:min_length]) opacity = list(opacity[:min_length]) level_width = list(level_width[:min_length]) # append with zeros while len(level) < 3: level.append(0) while len(opacity) < 3: opacity.append(0) while len(level_width) < 3: level_width.append(0) for i in range(1, 4): tf_kwargs["level" + str(i)] = level[i - 1] tf_kwargs["opacity" + str(i)] = opacity[i - 1] tf_kwargs["width" + str(i)] = level_width[i - 1] tf = TransferFunctionWidgetJs3(**tf_kwargs) # Set the transfer function kwargs["tf"] = tf # Set style if style == "dark": kwargs["style"] = dark elif style == "light": kwargs["style"] = light elif style == "minimal": kwargs["style"] = minimal else: raise ValueError("Invalid style: " + style) # Create the volume plot vol = ipyvolume.quickvolshow(data=data.T, **kwargs) #vol = p3.volshow(data=data, **kwargs) # SHOW? if show: #p3.volshow() vol.show() return vol # ONLY RETURN THE DATA else: return data
command = filename.split("__")[0] # Skip other commands if not command.startswith(config.match): continue # Add filepath matches[command].append(filepath) # ----------------------------------------------------------------- if len(matches) > 1: raise ValueError("Ambigious command: matches are " + ", ".join(matches.keys())) # ----------------------------------------------------------------- single_command = matches.keys()[0] log.debug("The matching command is '" + single_command + "'") if len(matches[single_command]) == 0: raise RuntimeError("Something went wrong") # ----------------------------------------------------------------- # Get latest latest_time = None latest_filepath = None # Loop over the files for filepath in matches[single_command]: name = fs.strip_extension(fs.name(filepath)) command_name, datetime = time.get_name_and_time_from_unique_name(name) #print(name, command_name, datetime)
def __del__(self): """ This function ... :return: """ # Debugging if self.output_path is not None: log.debug("Screen output has been placed in '" + self.output_path + "'") # Detach if we are still attached if self.attached: self.detach() # Remove things if we are not in debug mode if not log.is_debug(): # Using tmux if self.tmux: end_session_command = "tmux kill-session -t " + self.screen_name self.remote.execute(end_session_command) # Using screen else: # Stop the python session end_python_command = "screen -r -S " + self.screen_name + " -X stuff 'exit()\n'" self.remote.execute(end_python_command) # Stop screen session self.remote.kill_screen(self.screen_name) # Remove the pipe file self.remote.remove_file(self.out_pipe_filepath) else: # Debugging info log.debug("Not closing session and removing pipe in debug mode") if self.tmux: log.debug(" - Tmux session name: " + self.screen_name) else: log.debug(" - Screen session name: " + self.screen_name) log.debug(" - pipe file path: " + self.out_pipe_filepath) if self.output_path is not None: log.debug(" - screen log file: " + self.screenlog_path)
# Check whether the remote is available if config.full: remote = Remote() if not remote.setup(host_id): log.warning("The remote host '" + host_id + "' is not available: skipping ...") continue else: remote = None # Determine the path to the run directory for the specified remote host host_run_path = fs.join(introspection.skirt_run_dir, host_id) # Check if there are simulations if not fs.is_directory(host_run_path): log.debug("No run directory for host '" + host_id + "'") continue if fs.is_empty(host_run_path): log.debug("No simulations for host '" + host_id + "'") # Loop over the simulation files in the run directory for path, name in fs.files_in_path(host_run_path, extension="sim", returns=["path", "name"], sort=int): # Skip if config.ids is not None and int(name) not in config.ids: continue # Inform the user log.info("Removing simulation " + name + " ...")
def plot_galaxy_components(components, draw=True, show=True, shape=128, unit="pc", width=700, height=800, style='light', **kwargs): """ This function .... :param components: :param draw: :param show: :param shape: :param unit: :param width: :param height: :param style: :param kwargs: :return: """ # Determine the limits limits = determine_model_limits(components, unit, symmetric=True) # Debugging log.debug("Plot limits: " + str(limits)) # Create coordinate data x, y, z, r, theta, phi = xyz(shape=shape, limits=limits, spherical=True) data = r * 0 # Loop over the components for name in components: # Debugging log.debug("Computing the density of the " + name + " component ...") component = components[name] density = component.density_function(normalize=True)(x, y, z) data += density # DRAW FIGURE if draw: # :param lighting: boolean, to use lighting or not, if set to false, lighting parameters will be overriden # :param data_min: minimum value to consider for data, if None, computed using np.nanmin # :param data_max: maximum value to consider for data, if None, computed using np.nanmax # :param tf: transfer function (see ipyvolume.transfer_function, or use the argument below) # :param stereo: stereo view for virtual reality (cardboard and similar VR head mount) # :param width: width of rendering surface # :param height: height of rendering surface # :param ambient_coefficient: lighting parameter # :param diffuse_coefficient: lighting parameter # :param specular_coefficient: lighting parameter # :param specular_exponent: lighting parameter # :param downscale: downscale the rendering for better performance, for instance when set to 2, a 512x512 canvas will show a 256x256 rendering upscaled, but it will render twice as fast. # :param level: level(s) for the where the opacity in the volume peaks, maximum sequence of length 3 # :param opacity: opacity(ies) for each level, scalar or sequence of max length 3 # :param level_width: width of the (gaussian) bumps where the opacity peaks, scalar or sequence of max length 3 # :param kwargs: extra argument passed to Volume and default transfer function # DEFAULT: # lighting=False, data_min=None, data_max=None, tf=None, stereo=False, # width=400, height=500, # ambient_coefficient=0.5, diffuse_coefficient=0.8, # specular_coefficient=0.5, specular_exponent=5, # downscale=1, # level=[0.1, 0.5, 0.9], opacity=[0.01, 0.05, 0.1], level_width=0.1, level = [0.2] #opacity = [0.05, 0.0, 0.0] opacity = [0.08, 0.0, 0.0] level_width = 0.2 level_width = [level_width] * 3 kwargs = dict() kwargs["width"] = width kwargs["height"] = height kwargs["stereo"] = False kwargs["level"] = level kwargs["opacity"] = opacity kwargs["level_width"] = level_width kwargs["downscale"] = 1 # Create transfer function arguments tf_kwargs = {} # Clip off lists min_length = min(len(level), len(level_width), len(opacity)) level = list(level[:min_length]) opacity = list(opacity[:min_length]) level_width = list(level_width[:min_length]) # append with zeros while len(level) < 3: level.append(0) while len(opacity) < 3: opacity.append(0) while len(level_width) < 3: level_width.append(0) for i in range(1,4): tf_kwargs["level"+str(i)] = level[i-1] tf_kwargs["opacity"+str(i)] = opacity[i-1] tf_kwargs["width"+str(i)] = level_width[i-1] tf = TransferFunctionWidgetJs3(**tf_kwargs) # Set the transfer function kwargs["tf"] = tf # Set style if style == "dark": kwargs["style"] = dark elif style == "light": kwargs["style"] = light elif style == "minimal": kwargs["style"] = minimal else: raise ValueError("Invalid style: " + style) # Create the volume plot vol = ipyvolume.quickvolshow(data=data.T, **kwargs) #vol = p3.volshow(data=data, **kwargs) # SHOW? if show: #p3.volshow() vol.show() return vol # ONLY RETURN THE DATA else: return data
for host_id in config.remotes: # Check whether the remote is available if config.full: remote = Remote() if not remote.setup(host_id): log.warning("The remote host '" + host_id + "' is not available: skipping ...") continue else: remote = None # Determine the path to the run directory for the specified remote host host_run_path = fs.join(introspection.skirt_run_dir, host_id) # Check if there are simulations if not fs.is_directory(host_run_path): log.debug("No run directory for host '" + host_id + "'") continue if fs.is_empty(host_run_path): log.debug("No simulations for host '" + host_id + "'") # Loop over the simulation files in the run directory for path, name in fs.files_in_path(host_run_path, extension="sim", returns=["path", "name"], sort=int): # Skip if config.ids is not None and int(name) not in config.ids: continue # Inform the user log.info("Removing simulation " + name + " ...") # Fully clear if config.full:
def make_point_sources_fixed_fwhm(self, masks=None): """ This function ... :param masks: :return: """ # Inform the user log.info("Making point sources with a fixed FWHM in each band ...") # Loop over the 'star' filters for fltr in self.star_filters: # Debugging log.debug("Making point sources for the '" + str(fltr) + "' image ...") # Get pixelscale pixelscale = self.frames[fltr].average_pixelscale # Debugging log.debug("The pixelscale of the image is " + stringify.stringify(pixelscale)[1]) # Determine the FWHM in pixel coordinates fwhm_pix = self.real_fwhms[fltr].to("arcsec").value / pixelscale.to("arcsec").value # Determine sigma sigma = statistics.fwhm_to_sigma * fwhm_pix # Only 'render' the Gaussian locally origin = PixelCoordinate(0.0, 0.0) amplitude = 1. min_x, max_x, min_y, max_y, rel_center = determine_patch_for_psf(origin, amplitude, sigma, keep_in_frame=False) # Make the model model = create_model(self.config.psf_model, amplitude, rel_center, sigma) # Evalute the model y, x = np.indices((max_y - min_y, max_x - min_x)) data = model(x, y) # Keep track of the number of sources counter = 0 # Loop over the coordinates in the point sources catalog for coordinate in self.point_source_catalog.coordinates(): counter += 1 # Debugging log.debug("Adding source " + str(counter) + " of " + str(len(self.point_source_catalog)) + " ...") # Check whether it falls in the frame if not self.frames[fltr].contains(coordinate): continue # Convert into pixel coordinate pixel_coordinate = coordinate.to_pixel(self.frames[fltr].wcs) # Get the corresponding pixel pixel = Pixel.for_coordinate(pixel_coordinate) # Check whether not masked if masks is not None and fltr in masks and masks[fltr][pixel.y, pixel.x]: continue # Generate a random amplitude (from 100 till 100 000) amplitude_exponent = np.random.uniform(2., 5.) amplitude = 10 ** amplitude_exponent # Calculate absolute minima and maxima source_min_x = min_x + pixel.x source_max_x = max_x + pixel.x source_min_y = min_y + pixel.y source_max_y = max_y + pixel.y source_xsize = source_max_x - source_min_x source_ysize = source_max_y - source_min_y # Correct if source_min_x < 0: cut_x_min = - source_min_x source_min_x = 0 else: cut_x_min = 0 if source_max_x >= self.frames[fltr].xsize: cut_x_max = source_xsize - (source_max_x - self.frames[fltr].xsize) source_max_x = self.frames[fltr].xsize else: cut_x_max = source_xsize if source_min_y < 0: cut_y_min = - source_min_y source_min_y = 0 else: cut_y_min = 0 if source_max_y >= self.frames[fltr].ysize: cut_y_max = source_ysize - (source_max_y - self.frames[fltr].ysize) source_max_y = self.frames[fltr].ysize else: cut_y_max = source_ysize # Create the final data for this source source_data = data[cut_y_min:cut_y_max, cut_x_min:cut_x_max] * amplitude # Add the data self.frames[fltr][source_min_y:source_max_y, source_min_x:source_max_x] += source_data
for name in maps: # Get the map comparison_map = maps[name] #print(comparison_map.wcs.is_celestial) #print(comparison_map.wcs.has_celestial) if not comparison_map.wcs.is_celestial: log.warning("The " + name + " " + which_map + " dust map doesn't have a celestial WCS: skipping ...") continue # Debugging log.debug( "Bringing the reference and comparison image to the same resolution ..." ) # Rebin to same pixel grid frames = NamedFrameList(reference=the_map, comparison=comparison_map) #frames.set_uniform_properties() #frames.show_coordinate_systems() frames.convolve_and_rebin() # Normalize both frames frames.normalize() # REplace nans frames.replace_nans(0.0)
def make_old_stars_maps_bulge(self): """ This function ... :return: """ # Inform the user log.info("Making map of old stellar bulge ...") # Get decomposition properties #properties = get_properties(self.config.galaxy) components = get_components(self.config.galaxy) disk_pa = components["disk"].position_angle # Create a Sersic model for the bulge bulge_deprojection_method = "tilt" print("axial ratio:", components["bulge"].axial_ratio) print("inclination:", self.inclination) components["bulge"].axial_ratio = 1. / components["bulge"].axial_ratio bulge = SersicModel3D.from_2d( components["bulge"], self.inclination, disk_pa, azimuth_or_tilt=bulge_deprojection_method) # CREATE INSTRUMENT # Create the 'earth' projection system azimuth = 0.0 projection = GalaxyProjection.from_wcs(self.get_wcs("I2"), self.galaxy_center, self.distance, self.inclination, azimuth, disk_pa) # Create instrument from projection instrument = SimpleInstrument.from_projection(projection) # Inform the user log.debug("Creating ski file to simulate the bulge image ...") # Load the bulge ski file template bulge_template_path = fs.join(template_path, "bulge.ski") ski = SkiFile(bulge_template_path) npackages = 1e7 # Set the number of photon packages ski.setpackages(npackages) # Set the bulge geometry ski.set_stellar_component_geometry(0, bulge) # Remove all existing instruments ski.remove_all_instruments() # Add the instruments ski.add_instrument(instrument_name, instrument) # Determine the simulation path simulation_path = fs.create_directory_in(self.path, "bulge") # Determine the path to the ski file ski_path = fs.join(simulation_path, "bulge.ski") # Save the ski file to the new path ski.saveto(ski_path) # Determine the path to the simulation output directory and create it out_path = fs.create_directory_in(simulation_path, "out") # Inform the user log.debug("Running the bulge simulation ...") # The SKIRT launching environment launcher = SingleImageSKIRTLauncher() # Load the PSF kernel and prepare aniano = AnianoKernels() psf = aniano.get_psf("I2") psf.prepare_for(self.get_wcs("I2")) # Simulate the bulge image fluxdensity = components["bulge"].fluxdensity bulge_image = launcher.run(ski_path, out_path, self.get_wcs("I2"), fluxdensity, psf, instrument_name=instrument_name, progress_bar=True) # Determine path for the bulge map path = fs.join(self.maps_path, old_bulge_filename) # Save the bulge map bulge_image.saveto(path)