def __init__(self, output_path=None): """ This function ... :param output_path: """ # Set the output directory path self.output_path = output_path # The remote instance self.remote = None # Generate session ID self.session_id = time.unique_name()
def __init__(self, output_path=None): """ This function ... :param output_path: """ # Set the output directory path self.output_path = output_path # The remote instance self.remote = None # Generate session ID self.session_id = time.unique_name()
def initialize_log(config, remote=None, command_name=None): """ This function ... :param config: :parma remote: :param command_name: :return: """ # Determine the log level level = "INFO" if config.debug: level = "DEBUG" if config.brief: level = "SUCCESS" # Determine memuse flag if hasattr(config, "memuse"): memuse = config.memuse else: memuse = False # Determine log path #if args.remote is None: logfile_path = fs.join(config.log_path, time.unique_name("log") + ".txt") if config.report else None #else: logfile_path = None # Determine name of log file if command_name is not None: filename = command_name else: filename = "log" # Determine the log file path if remote is None: logfile_path = fs.join(config.log_path, time.unique_name(filename, separator="__") + ".txt") if config.report else None else: logfile_path = None # Initialize the logger from ..core.basics.log import setup_log log = setup_log(level=level, path=logfile_path, memory=memuse) # Return the logger return log
def initialize_log(config, remote=None, command_name=None): """ This function ... :param config: :parma remote: :param command_name: :return: """ # Determine the log level level = "INFO" if config.debug: level = "DEBUG" if config.brief: level = "SUCCESS" # Determine log path #if args.remote is None: logfile_path = fs.join(config.log_path, time.unique_name("log") + ".txt") if config.report else None #else: logfile_path = None # Determine name of log file if command_name is not None: filename = command_name else: filename = "log" # Determine the log file path if remote is None: logfile_path = fs.join(config.log_path, time.unique_name(filename) + ".txt") if config.report else None else: logfile_path = None # Initialize the logger from ..core.basics.log import setup_log log = setup_log(level=level, path=logfile_path) # Return the logger return log
# ----------------------------------------------------------------- # Generate the new population ga.generate_new_population() # ----------------------------------------------------------------- name_column = [] par_a_column = [] par_b_column = [] for ind in ga.new_population: # Give the individual a unique name name = time.unique_name(precision="micro") name_column.append(name) par_a_column.append(ind.genomeList[0]) par_b_column.append(ind.genomeList[1]) # Create the parameters table data = [name_column, par_a_column, par_b_column] names = ["Unique name", "Parameter a", "Parameter b"] new_parameters_table = tables.new(data, names) # ----------------------------------------------------------------- # Save the new parameters table tables.write(new_parameters_table, new_parameters_path, format="ascii.ecsv") # Dump the GA
# Flags definition.add_flag("manual", "launch and inspect job scripts manually") definition.add_flag( "keep", "keep the output generated by the different SKIRT simulations") # Get configuration reader = ConfigurationReader( "scaling", "Test the scaling of SKIRT on a particular system") config = reader.read(definition) # ----------------------------------------------------------------- # Determine the log file path logfile_path = fs.join(fs.cwd(), time.unique_name("scaling") + ".txt") if config.report else None # Determine the log level level = "DEBUG" if config.debug else "INFO" # Initialize the logger log = logging.setup_log(level=level, path=logfile_path) log.start("Starting scaling ...") # ----------------------------------------------------------------- # Create a ScalingTest instance test = ScalingTest(config) # Run the scaling test
# ----------------------------------------------------------------- # Determine the full path to the parameter file arguments.filepath = fs.absolute(arguments.file) # Determine the full path to the input and output directories if arguments.input is not None: arguments.input_path = fs.absolute(arguments.input) if arguments.output is not None: arguments.output_path = fs.absolute(arguments.output) # ----------------------------------------------------------------- # Determine the log file path logfile_path = fs.join(fs.cwd(), time.unique_name("launch") + ".txt") if arguments.report else None # Determine the log level level = "DEBUG" if arguments.debug else "INFO" # Initialize the logger log = logging.setup_log(level=level, path=logfile_path) log.start("Starting launch ...") # ----------------------------------------------------------------- # Either create a SkirtRemoteLauncher or a SkirtLauncher if arguments.remote: launcher = SkirtRemoteLauncher.from_arguments(arguments) else: launcher = SkirtLauncher.from_arguments(arguments)
# ----------------------------------------------------------------- # Generate the new population ga.generate_new_population() # ----------------------------------------------------------------- name_column = [] par_a_column = [] par_b_column = [] for ind in ga.new_population: # Give the individual a unique name name = time.unique_name(precision="micro") name_column.append(name) par_a_column.append(ind.genomeList[0]) par_b_column.append(ind.genomeList[1]) # Create the parameters table data = [name_column, par_a_column, par_b_column] names = ["Unique name", "Parameter a", "Parameter b"] new_parameters_table = tables.new(data, names) # ----------------------------------------------------------------- # Save the new parameters table tables.write(new_parameters_table, new_parameters_path, format="ascii.ecsv") # Dump the GA
# If an output directory is given if arguments.output is not None: # Determine the full path to the output directory output_path = fs.absolute(arguments.output) # Create the directory if it does not yet exist if not fs.is_directory(output_path): fs.create_directory(output_path) # If no output directory is given, place the output in the current working directory else: output_path = fs.cwd() # ----------------------------------------------------------------- # Determine the log file path logfile_path = fs.join(output_path, time.unique_name("log") + ".txt") if arguments.report else None # Determine the log level level = "DEBUG" if arguments.debug else "INFO" # Initialize the logger log = logging.setup_log(level=level, path=logfile_path) log.start("Starting find_sources ...") # ----------------------------------------------------------------- # Determine the full path to the image image_path = fs.absolute(arguments.image) # Determine the full path to the bad region file bad_region_path = fs.join(input_path, arguments.bad) if arguments.bad is not None else None
# Check the factor if np.isclose(factor, config.factor, rtol=0.01): the_image_path = image_path break # Check if the_image_path is None: raise ValueError("Could not find the truncated " + filter_name + " image with a truncation factor of " + str(config.factor)) # ----------------------------------------------------------------- # Download the image to a temporary path filename = time.unique_name("truncated_" + filter_name + "_" + str(config.factor)) filepath = remote.download_file_to(the_image_path, pts_temp_dir, new_name=filename) # ----------------------------------------------------------------- # Open the image frame = Frame.from_file(filepath) # ----------------------------------------------------------------- # Plot the truncated image plotting.plot_box(frame) # -----------------------------------------------------------------
# Check if key is active #if host.key is not None: # if host.key not in active_keys(): add_key(host.key) #command = "sshfs " + debug_flags + host.user + "@" + host.name + ": " + mount_path + " -C -o volname=" + host.id # SSH COMMAND: "ssh -p 2935 -X [email protected]" ssh_path = "/usr/bin/ssh" #output = terminal.execute("which ssh") #print(output) temp_path = introspection.pts_temp_dir temp_command_path = fs.join(temp_path, time.unique_name("pts_login") + ".command") #with tempfile.NamedTemporaryFile(suffix='.command') as f: with open(temp_command_path, 'w') as f: f.write('#!/bin/sh\n') #f.write("python bb.py\n") #subprocess.call(['open', '-W', f.name]) #ssh_command = "ssh " ssh_arguments = "" if host.port is not None: ssh_arguments += "-p " + str(host.port) + " " ssh_arguments += "-X " ssh_arguments += host.user ssh_arguments += "@"
sample = DustPediaSample() # Database username, password = get_account() database = DustPediaDatabase() database.login(username, password) # Get the DustPedia name name = sample.get_name(config.galaxy) # Get info info = database.get_galaxy_info_table(name) # FIRST IMAGES AND THAN THIS FAILS FOR SOME REASON!! path = fs.join(introspection.pts_temp_dir, time.unique_name("cutouts_" + name) + ".png") database.download_photometry_cutouts(name, path) # Get image names #image_names = database.get_image_names(name, error_maps=False) image_filters = database.get_image_filters(name) print("") print(fmt.red + fmt.underlined + "General info:" + fmt.reset) print("") print(" - position: " + str(sample.get_position(name))) print(" - D25: " + str(sample.get_d25(name))) print(" - R25: " + str(sample.get_r25(name))) print("") print(" - stage: " + str(info["Hubble Stage"][0]))
# If an output directory is given if arguments.output is not None: # Determine the full path to the output directory output_path = fs.absolute(arguments.output) # Create the directory if it does not yet exist if not fs.is_directory(output_path): fs.create_directory(output_path) # If no output directory is given, place the output in the current working directory else: output_path = fs.cwd() # ----------------------------------------------------------------- # Determine the log file path logfile_path = fs.join(output_path, time.unique_name("log") + ".txt") if arguments.report else None # Determine the log level level = "DEBUG" if arguments.debug else "INFO" # Initialize the logger log = logging.setup_log(level=level, path=logfile_path) log.start("Starting interpolate ...") # ----------------------------------------------------------------- # Inform the user log.info("Loading the image ...") # Determine the full path to the image image_path = fs.absolute(arguments.image)
"number of generations (steps)", 10) # Create the configuration config = parse_arguments("test_tsp_steps", definition) # Set logging log = setup_log("DEBUG") # ----------------------------------------------------------------- evaluator_kwargs = input_optimize["evaluator_kwargs"] # ----------------------------------------------------------------- # Determine temporary directory path = introspection.create_temp_dir(time.unique_name("optimize")) # Change working directory fs.change_cwd(path) # ----------------------------------------------------------------- # Create the step optimizer optimizer = StepWiseOptimizer(settings_optimize) # Run the optimizer population = optimizer.run(**input_optimize) # Remove the variable, to load it again later del optimizer
sample = DustPediaSample() # Database username, password = get_account() database = DustPediaDatabase() database.login(username, password) # Get the DustPedia name name = sample.get_name(config.galaxy) # Get info info = database.get_galaxy_info(name) # FIRST IMAGES AND THAN THIS FAILS FOR SOME REASON!! path = fs.join(introspection.pts_temp_dir, time.unique_name("cutouts_" + name) + ".png") database.download_photometry_cutouts(name, path) # Get image names #image_names = database.get_image_names(name, error_maps=False) image_filters = database.get_image_filters(name) print("") print(fmt.red + fmt.underlined + "General info:" + fmt.reset) print("") print(" - position: " + str(sample.get_position(name))) print(" - D25: " + str(sample.get_d25(name))) print(" - R25: " + str(sample.get_r25(name))) print("") print(" - stage: " + str(info["Hubble Stage"][0]))
# Optional definition.add_optional("remote", str, "the remote host name", None) # Get configuration reader = ConfigurationReader( "get_poisson_errors", "Calculate poisson error maps for DustPedia UV and optical images") config = reader.read(definition) arguments = reader.get_arguments() # ----------------------------------------------------------------- # Determine the log file path logfile_path = fs.join(fs.cwd(), time.unique_name("log") + ".txt") if config.report else None # Determine the log level level = "DEBUG" if config.debug else "INFO" # Initialize the logger log = logging.setup_log(level=level, path=logfile_path) log.start("Starting get_poisson_errors ...") # ----------------------------------------------------------------- temp_name = time.unique_name(config.band.replace(" ", "")) # -----------------------------------------------------------------
definition.add_optional("name", "string", "name for the new analysis run", forbidden=analysis_runs.names) config = parse_arguments("clone_analysis_run", definition) # ----------------------------------------------------------------- # Get analysis run path path = analysis_runs.get_path(config.analysis_run) # ----------------------------------------------------------------- # Generate new analysis run name if config.name is not None: new_name = config.name else: new_name = time.unique_name() new_path = fs.create_directory_in(environment.analysis_path, new_name) # ----------------------------------------------------------------- clear_directories = [ "out", "misc", "plot", "residuals", "heating", "extr", "colours", "attenuation" ] fs.copy_from_directory(path, new_path, exact_not_name=clear_directories, not_extension="sh") fs.create_directories_in(new_path, clear_directories) # -----------------------------------------------------------------
definition = ConfigurationDefinition() # Add optional arguments definition.add_optional( "image", str, "the name of the image for which to run the initialization") definition.add_flag("visualise", "make visualisations") # Get configuration reader = ConfigurationReader("initialize_preparation") config = reader.read(definition) # ----------------------------------------------------------------- # Determine the log file path logfile_path = fs.join(fs.cwd(), "log", time.unique_name("log") + ".txt") if config.report else None # Determine the log level level = "DEBUG" if config.debug else "INFO" # Initialize the logger log = logging.setup_log(level=level, path=logfile_path) log.start("Starting initialize_data ...") # ----------------------------------------------------------------- names_column = [] paths_column = [] prep_names_column = [] names = ["Image name", "Image path", "Preparation name"]
# Galaxy name definition.add_required("galaxy_name", str, "the name of the galaxy") definition.add_required("band", str, "the band (GALEX or SDSS u/g/r/i/z)") # Optional definition.add_optional("remote", str, "the remote host name", None) # Get configuration reader = ConfigurationReader("get_poisson_errors", "Calculate poisson error maps for DustPedia UV and optical images") config = reader.read(definition) arguments = reader.get_arguments() # ----------------------------------------------------------------- # Determine the log file path logfile_path = fs.join(fs.cwd(), time.unique_name("log") + ".txt") if config.report else None # Determine the log level level = "DEBUG" if config.debug else "INFO" # Initialize the logger log = logging.setup_log(level=level, path=logfile_path) log.start("Starting get_poisson_errors ...") # ----------------------------------------------------------------- temp_name = time.unique_name(config.band.replace(" ", "")) # ----------------------------------------------------------------- # Remotely
definition.add_positional_optional("ngenerations", "positive_integer", "number of generations (steps)", 10) # Create the configuration config = parse_arguments("test_tsp_steps", definition) # Set logging log = setup_log("DEBUG") # ----------------------------------------------------------------- evaluator_kwargs = input_optimize["evaluator_kwargs"] # ----------------------------------------------------------------- # Determine temporary directory path = introspection.create_temp_dir(time.unique_name("optimize")) # Change working directory fs.change_cwd(path) # ----------------------------------------------------------------- # Create the step optimizer optimizer = StepWiseOptimizer(settings_optimize) # Run the optimizer population = optimizer.run(**input_optimize) # Remove the variable, to load it again later del optimizer
# ----------------------------------------------------------------- # Determine the full path to the parameter file arguments.filepath = fs.absolute(arguments.file) # Determine the full path to the input and output directories if arguments.input is not None: arguments.input_path = fs.absolute(arguments.input) if arguments.output is not None: arguments.output_path = fs.absolute(arguments.output) # ----------------------------------------------------------------- # Determine the log file path logfile_path = fs.join(fs.cwd(), time.unique_name("launch") + ".txt") if arguments.report else None # Determine the log level level = "DEBUG" if arguments.debug else "INFO" # Initialize the logger log = logging.setup_log(level=level, path=logfile_path) log.start("Starting launch ...") # ----------------------------------------------------------------- # Either create a SkirtRemoteLauncher or a SkirtLauncher if arguments.remote: launcher = SkirtRemoteLauncher.from_arguments(arguments) else: launcher = SkirtLauncher.from_arguments(arguments)
help="(optional) the name of the remote host") parser.add_argument("--debug", action="store_true", help="add this option to enable debug output") parser.add_argument("--report", action="store_true", help="write a report file") # Parse the command line arguments arguments = parser.parse_args() # ----------------------------------------------------------------- # Determine the log file path logfile_path = os.path.join(os.getcwd(), time.unique_name("status") + ".txt") if arguments.report else None # Determine the log level level = "DEBUG" if arguments.debug else "INFO" # Initialize the logger log = logging.setup_log(level=level, path=logfile_path) log.start("Starting status ...") # ----------------------------------------------------------------- # Create a RemoteSynchronizer instance synchronizer = RemoteSynchronizer.from_arguments(arguments) # Run the synchronizer
# ----------------------------------------------------------------- # Create the command-line parser parser = argparse.ArgumentParser() parser.add_argument('remote', nargs='?', default=None, help="(optional) the name of the remote host") parser.add_argument("--debug", action="store_true", help="add this option to enable debug output") parser.add_argument("--report", action="store_true", help="write a report file") # Parse the command line arguments arguments = parser.parse_args() # ----------------------------------------------------------------- # Determine the log file path logfile_path = os.path.join(os.getcwd(), time.unique_name("status") + ".txt") if arguments.report else None # Determine the log level level = "DEBUG" if arguments.debug else "INFO" # Initialize the logger log = logging.setup_log(level=level, path=logfile_path) log.start("Starting status ...") # ----------------------------------------------------------------- # Create a RemoteSynchronizer instance synchronizer = RemoteSynchronizer.from_arguments(arguments) # Run the synchronizer synchronizer.run()
# ----------------------------------------------------------------- # Create the configuration definition definition = ConfigurationDefinition() # Add setting definition.add_required("step", str, "the modeling step for which to clear the output") # Get the configuration reader = ConfigurationReader("clear") config = reader.read(definition) # ----------------------------------------------------------------- # Determine the log file path logfile_path = fs.join(fs.cwd(), "log", time.unique_name("log") + ".txt") if config.arguments.report else None # Determine the log level level = "DEBUG" if config.arguments.debug else "INFO" # Initialize the logger log = logging.setup_log(level=level, path=logfile_path) log.start("Starting clear ...") # ----------------------------------------------------------------- prep_path = fs.join(config.path, "prep") components_path = fs.join(config.path, "components") truncated_path = fs.join(config.path, "truncated") phot_path = fs.join(config.path, "phot") maps_path = fs.join(config.path, "maps")
else: figsize = "12,9" figsize_timelines = "12,12" # ----------------------------------------------------------------- # Locate the scaling test suite directory suite_path = fs.join(fs.cwd(), config.suite_name) if not fs.is_directory(suite_path): raise ValueError("The directory '" + suite_path + "' does not exist") # ----------------------------------------------------------------- # Make directory for output output_path = fs.create_directory_in(fs.cwd(), time.unique_name("scaling_plots")) # Make subdirectories single_node_path = fs.create_directory_in(output_path, "Single-node comparison") multi_node_path = fs.create_directory_in( output_path, "Load balancing and multi-node scaling") communication_path = fs.create_directory_in(output_path, "Communication") hybridization_path = fs.create_directory_in(output_path, "Hybridization") photon_packages_path = fs.create_directory_in( output_path, "Increased number of photon packages") memory_path = fs.create_directory_in(output_path, "Memory scaling") # ----------------------------------------------------------------- pts_path = introspection.pts_executable_path
from pts.magic.misc.dustpedia import DustPediaDatabase, get_account from pts.core.basics.configuration import ConfigurationDefinition, ConfigurationReader # ----------------------------------------------------------------- # Create the configuration definition = ConfigurationDefinition() # Get configuration reader = ConfigurationReader("plot_galaxies") config = reader.read(definition) # ----------------------------------------------------------------- # Determine the log file path logfile_path = fs.join(fs.cwd(), time.unique_name("log") + ".txt") if config.report else None # Determine the log level level = "DEBUG" if config.debug else "INFO" # Initialize the logger log = logging.setup_log(level=level, path=logfile_path) log.start("Starting plot_galaxies ...") # ----------------------------------------------------------------- # Local table path local_table_path = fs.join(introspection.pts_dat_dir("modeling"), "s4g", "s4g_p4_table8.dat") # -----------------------------------------------------------------
figsize = "8,6" figsize_timelines = "8,8" else: figsize = "12,9" figsize_timelines = "12,12" # ----------------------------------------------------------------- # Locate the scaling test suite directory suite_path = fs.join(fs.cwd(), config.suite_name) if not fs.is_directory(suite_path): raise ValueError("The directory '" + suite_path + "' does not exist") # ----------------------------------------------------------------- # Make directory for output output_path = fs.create_directory_in(fs.cwd(), time.unique_name("scaling_plots")) # Make subdirectories single_node_path = fs.create_directory_in(output_path, "Single-node comparison") multi_node_path = fs.create_directory_in(output_path, "Load balancing and multi-node scaling") communication_path = fs.create_directory_in(output_path, "Communication") hybridization_path = fs.create_directory_in(output_path, "Hybridization") photon_packages_path = fs.create_directory_in(output_path, "Increased number of photon packages") memory_path = fs.create_directory_in(output_path, "Memory scaling") # ----------------------------------------------------------------- pts_path = introspection.pts_executable_path # -----------------------------------------------------------------