def singularity_pull_image(self, container, out_path, cache_path, progress): """Pull a singularity image using ``singularity pull`` Attempt to use a local installation of singularity to pull the image. Args: container (str): A pipeline's container name. Usually it is of similar format to ``nfcore/name:version``. Raises: Various exceptions possible from `subprocess` execution of Singularity. """ output_path = cache_path or out_path # Pull using singularity address = "docker://{}".format(container.replace("docker://", "")) singularity_command = [ "singularity", "pull", "--name", output_path, address ] log.debug("Building singularity image: {}".format(address)) log.debug("Singularity command: {}".format( " ".join(singularity_command))) # Progress bar to show that something is happening task = progress.add_task(container, start=False, total=False, progress_type="singularity_pull", current_log="") # Run the singularity pull command proc = subprocess.Popen( singularity_command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, universal_newlines=True, bufsize=1, ) for line in proc.stdout: log.debug(line.strip()) progress.update(task, current_log=line.strip()) # Copy cached download if we are using the cache if cache_path: log.debug("Copying {} from cache: '{}'".format( container, os.path.basename(out_path))) progress.update( task, current_log="Copying from cache to target directory") shutil.copyfile(cache_path, out_path) progress.remove_task(task)
def lint_project(self, calling_class, check_functions: list = None, custom_check_files: bool = False, is_subclass_calling=True) -> None: """Main linting function. Takes the template directory as the primary input and iterates through the different linting checks in order. Collects any warnings or errors and returns summary at completion. Raises an exception if there is a critical error that makes the rest of the tests pointless (eg. no project script). Results from this function are printed by the main script. :param calling_class: The class that calls the function -> used to get the class methods, which are the linting methods :param check_functions: List of functions of the calling class that should be checked. If not set, the default TemplateLinter check functions are called :param custom_check_files: Set to true if TemplateLinter check_files_exist should not be run :param is_subclass_calling: Indicates whether a domain specific linter calls the linting or not """ # Called on its own, so not from a subclass -> run general linting if check_functions is None: # Fetch all general linting functions check_functions = [ func for func in dir(TemplateLinter) if (callable(getattr(TemplateLinter, func)) and not func.startswith('_')) ] # Remove internal functions check_functions = list( set(check_functions).difference( {'lint_project', 'print_results', 'check_version_match'})) # Some templates (e.g. latex based) do not adhere to the common programming based templates and therefore do not need to check for e.g. docs # or lint changelog if custom_check_files: check_functions.remove('check_files_exist') check_functions.remove('lint_changelog') progress = rich.progress.Progress( "[bold green]{task.description}", rich.progress.BarColumn(bar_width=None), "[bold yellow]{task.completed} of {task.total}[reset] [bold green]{task.fields[func_name]}", ) with progress: lint_progress = progress.add_task("Running lint checks", total=len(check_functions), func_name=check_functions) for fun_name in check_functions: progress.update(lint_progress, advance=1, func_name=fun_name) if fun_name == 'check_files_exist': getattr(calling_class, fun_name)(is_subclass_calling) else: getattr(calling_class, fun_name)()
async def migrate_to( cls, new_driver_cls: Type["BaseDriver"], all_custom_group_data: Dict[str, Dict[str, Dict[str, int]]], ) -> None: """Migrate data from this backend to another. Both drivers must be initialized beforehand. This will only move the data - no instance metadata is modified as a result of this operation. Parameters ---------- new_driver_cls Subclass of `BaseDriver`. all_custom_group_data : Dict[str, Dict[str, Dict[str, int]]] Dict mapping cog names, to cog IDs, to custom groups, to primary key lengths. """ # Backend-agnostic method of migrating from one driver to another. with rich.progress.Progress( rich.progress.SpinnerColumn(), rich.progress.TextColumn("[progress.description]{task.description}"), RichIndefiniteBarColumn(), rich.progress.TextColumn("{task.completed} cogs processed"), rich.progress.TimeElapsedColumn(), ) as progress: cog_count = 0 tid = progress.add_task("[yellow]Migrating", completed=cog_count, total=cog_count + 1) async for cog_name, cog_id in cls.aiter_cogs(): progress.console.print(f"Working on {cog_name}...") this_driver = cls(cog_name, cog_id) other_driver = new_driver_cls(cog_name, cog_id) custom_group_data = all_custom_group_data.get(cog_name, {}).get(cog_id, {}) exported_data = await this_driver.export_data(custom_group_data) await other_driver.import_data(exported_data, custom_group_data) cog_count += 1 progress.update(tid, completed=cog_count, total=cog_count + 1) progress.update(tid, total=cog_count) print()
def download_response(response: Response, download: typing.BinaryIO) -> None: console = rich.console.Console() console.print() content_length = response.headers.get("Content-Length") with rich.progress.Progress( "[progress.description]{task.description}", "[progress.percentage]{task.percentage:>3.0f}%", rich.progress.BarColumn(bar_width=None), rich.progress.DownloadColumn(), rich.progress.TransferSpeedColumn(), ) as progress: description = f"Downloading [bold]{rich.markup.escape(download.name)}" download_task = progress.add_task( description, total=int(content_length or 0), start=content_length is not None, ) for chunk in response.iter_bytes(): download.write(chunk) progress.update(download_task, completed=response.num_bytes_downloaded)
def download_response(response: httpx.Response) -> None: console = rich.console.Console() syntax = rich.syntax.Syntax("", "http", theme="ansi_dark", word_wrap=True) console.print(syntax) filename = get_download_filename(response) content_length = response.headers.get("Content-Length") kwargs = {"total": int(content_length)} if content_length else {} with open(filename, mode="bw") as download_file: with rich.progress.Progress( "[progress.description]{task.description}", "[progress.percentage]{task.percentage:>3.0f}%", rich.progress.BarColumn(bar_width=None), rich.progress.DownloadColumn(), rich.progress.TransferSpeedColumn(), ) as progress: description = f"Downloading [bold]{filename}" download_task = progress.add_task(description, **kwargs) # type: ignore for chunk in response.iter_bytes(): download_file.write(chunk) progress.update(download_task, completed=response.num_bytes_downloaded)
def test_progress_max_refresh() -> None: """Test max_refresh argument.""" time = 0.0 def get_time() -> float: nonlocal time try: return time finally: time = time + 1.0 console = Console( color_system=None, width=80, legacy_windows=False, force_terminal=True, _environ={}, ) column = TextColumn("{task.description}") column.max_refresh = 3 progress = Progress( column, get_time=get_time, auto_refresh=False, console=console, ) console.begin_capture() with progress: task_id = progress.add_task("start") for tick in range(6): progress.update(task_id, description=f"tick {tick}") progress.refresh() result = console.end_capture() print(repr(result)) assert ( result == "\x1b[?25l\r\x1b[2Kstart\r\x1b[2Kstart\r\x1b[2Ktick 1\r\x1b[2Ktick 1\r\x1b[2Ktick 3\r\x1b[2Ktick 3\r\x1b[2Ktick 5\r\x1b[2Ktick 5\n\x1b[?25h" )
def make_progress() -> Progress: _time = 0.0 def fake_time(): nonlocal _time try: return _time finally: _time += 1 console = Console( file=io.StringIO(), force_terminal=True, color_system="truecolor", width=80, legacy_windows=False, _environ={}, ) progress = Progress(console=console, get_time=fake_time, auto_refresh=False) task1 = progress.add_task("foo") task2 = progress.add_task("bar", total=30) progress.advance(task2, 16) task3 = progress.add_task("baz", visible=False) task4 = progress.add_task("egg") progress.remove_task(task4) task4 = progress.add_task("foo2", completed=50, start=False) progress.stop_task(task4) progress.start_task(task4) progress.update(task4, total=200, advance=50, completed=200, visible=True, refresh=True) progress.stop_task(task4) return progress
def get_filelist(run_module_names): """ Go through all supplied search directories and assembly a master list of files to search. Then fire search functions for each file. """ # Prep search patterns spatterns = [{}, {}, {}, {}, {}, {}, {}] runtimes["sp"] = defaultdict() ignored_patterns = [] skipped_patterns = [] for key, sps in config.sp.items(): mod_name = key.split("/", 1)[0] if mod_name.lower() not in [m.lower() for m in run_module_names]: ignored_patterns.append(key) continue files[key] = list() if not isinstance(sps, list): sps = [sps] # Warn if we have any unrecognised search pattern keys expected_sp_keys = [ "fn", "fn_re", "contents", "contents_re", "num_lines", "shared", "skip", "max_filesize", "exclude_fn", "exclude_fn_re", "exclude_contents", "exclude_contents_re", ] unrecognised_keys = [ y for x in sps for y in x.keys() if y not in expected_sp_keys ] if len(unrecognised_keys) > 0: logger.warning( "Unrecognised search pattern keys for '{}': {}".format( key, ", ".join(unrecognised_keys))) # Check if we are skipping this search key if any([x.get("skip") for x in sps]): skipped_patterns.append(key) # Split search patterns according to speed of execution. if any([x for x in sps if "contents_re" in x]): if any([x for x in sps if "num_lines" in x]): spatterns[4][key] = sps elif any([x for x in sps if "max_filesize" in x]): spatterns[5][key] = sps else: spatterns[6][key] = sps elif any([x for x in sps if "contents" in x]): if any([x for x in sps if "num_lines" in x]): spatterns[1][key] = sps elif any([x for x in sps if "max_filesize" in x]): spatterns[2][key] = sps else: spatterns[3][key] = sps else: spatterns[0][key] = sps if len(ignored_patterns) > 0: logger.debug( "Ignored {} search patterns as didn't match running modules.". format(len(ignored_patterns))) if len(skipped_patterns) > 0: logger.info("Skipping {} file search patterns".format( len(skipped_patterns))) logger.debug("Skipping search patterns: {}".format( ", ".join(skipped_patterns))) def add_file(fn, root): """ Function applied to each file found when walking the analysis directories. Runs through all search patterns and returns True if a match is found. """ f = {"fn": fn, "root": root} # Check that this is a file and not a pipe or anything weird if not os.path.isfile(os.path.join(root, fn)): file_search_stats["skipped_not_a_file"] += 1 return False # Check that we don't want to ignore this file i_matches = [ n for n in config.fn_ignore_files if fnmatch.fnmatch(fn, n) ] if len(i_matches) > 0: logger.debug( "Ignoring file as matched an ignore pattern: {}".format(fn)) file_search_stats["skipped_ignore_pattern"] += 1 return False # Limit search to small files, to avoid 30GB FastQ files etc. try: f["filesize"] = os.path.getsize(os.path.join(root, fn)) except (IOError, OSError, ValueError, UnicodeDecodeError): logger.debug( "Couldn't read file when checking filesize: {}".format(fn)) else: if f["filesize"] > config.log_filesize_limit: file_search_stats["skipped_filesize_limit"] += 1 return False # Test file for each search pattern file_matched = False for patterns in spatterns: for key, sps in patterns.items(): start = time.time() for sp in sps: if search_file(sp, f, key): # Check that we shouldn't exclude this file if not exclude_file(sp, f): # Looks good! Remember this file files[key].append(f) file_search_stats[key] = file_search_stats.get( key, 0) + 1 file_matched = True # Don't keep searching this file for other modules if not sp.get("shared", False): runtimes["sp"][key] = runtimes["sp"].get( key, 0) + (time.time() - start) return True # Don't look at other patterns for this module else: break runtimes["sp"][key] = runtimes["sp"].get( key, 0) + (time.time() - start) return file_matched # Go through the analysis directories and get file list multiqc_installation_dir_files = [ "LICENSE", "CHANGELOG.md", "Dockerfile", "MANIFEST.in", ".gitmodules", "README.md", "CSP.txt", "setup.py", ".gitignore", ] total_sp_starttime = time.time() for path in config.analysis_dir: if os.path.islink(path) and config.ignore_symlinks: file_search_stats["skipped_symlinks"] += 1 continue elif os.path.isfile(path): searchfiles.append([os.path.basename(path), os.path.dirname(path)]) elif os.path.isdir(path): for root, dirnames, filenames in os.walk( path, followlinks=(not config.ignore_symlinks), topdown=True): bname = os.path.basename(root) # Skip any sub-directories matching ignore params orig_dirnames = dirnames[:] for n in config.fn_ignore_dirs: dirnames[:] = [ d for d in dirnames if not fnmatch.fnmatch(d, n.rstrip(os.sep)) ] if len(orig_dirnames) != len(dirnames): removed_dirs = [ os.path.join(root, d) for d in set(orig_dirnames).symmetric_difference( set(dirnames)) ] logger.debug( "Ignoring directory as matched fn_ignore_dirs: {}". format(", ".join(removed_dirs))) orig_dirnames = dirnames[:] for n in config.fn_ignore_paths: dirnames[:] = [ d for d in dirnames if not fnmatch.fnmatch( os.path.join(root, d), n.rstrip(os.sep)) ] if len(orig_dirnames) != len(dirnames): removed_dirs = [ os.path.join(root, d) for d in set(orig_dirnames).symmetric_difference( set(dirnames)) ] logger.debug( "Ignoring directory as matched fn_ignore_paths: {}" .format(", ".join(removed_dirs))) # Skip *this* directory if matches ignore params d_matches = [ n for n in config.fn_ignore_dirs if fnmatch.fnmatch(bname, n.rstrip(os.sep)) ] if len(d_matches) > 0: logger.debug( "Ignoring directory as matched fn_ignore_dirs: {}". format(bname)) continue p_matches = [ n for n in config.fn_ignore_paths if fnmatch.fnmatch(root, n.rstrip(os.sep)) ] if len(p_matches) > 0: logger.debug( "Ignoring directory as matched fn_ignore_paths: {}". format(root)) continue # Sanity check - make sure that we're not just running in the installation directory if len(filenames) > 0 and all( [fn in filenames for fn in multiqc_installation_dir_files]): logger.error( "Error: MultiQC is running in source code directory! {}" .format(root)) logger.warning( "Please see the docs for how to use MultiQC: https://multiqc.info/docs/#running-multiqc" ) dirnames[:] = [] filenames[:] = [] continue # Search filenames in this directory for fn in filenames: searchfiles.append([fn, root]) # Search through collected files progress_obj = rich.progress.Progress( "[blue]|[/] ", rich.progress.SpinnerColumn(), "[blue]{task.description}[/] |", rich.progress.BarColumn(), "[progress.percentage]{task.percentage:>3.0f}%", "[green]{task.completed}/{task.total}", "[dim]{task.fields[s_fn]}", ) with progress_obj as progress: mqc_task = progress.add_task("searching", total=len(searchfiles), s_fn="") for sf in searchfiles: progress.update(mqc_task, advance=1, s_fn=os.path.join(sf[1], sf[0])[-50:]) if not add_file(sf[0], sf[1]): file_search_stats["skipped_no_match"] += 1 progress.update(mqc_task, s_fn="") runtimes["total_sp"] = time.time() - total_sp_starttime
def run(prefix, archive, program, prefix_specified, copy_threshold, distance_threshold): start = timer() failed_runs = [] error_runs = [] lagrange_runner = lagrange.lagrange(program) console = rich.console.Console() linreg_xs = [] linreg_ys = [] with rich.progress.Progress() as progress: jobs = directory.extractTarFileAndMakeDirectories( archive, prefix, progress) random.shuffle(jobs) work_task = progress.add_task("[red]Running...", total=len(jobs)) for expected, experiment in jobs: try: experiment.runExperiment(lagrange_runner) except directory.ExperimentFilesMissing: error_runs.append(experiment) progress.update(work_task, advance=1.0) check_task = progress.add_task("[red]Checking...", total=len(jobs)) for expected, experiment in jobs: if experiment.failed(): continue parameter_diff = expected.parameterVectorDifference(experiment) try: dist = expected.metricCompare(experiment) except: experiment.setFailed() continue linreg_xs.append(parameter_diff) linreg_ys.append(dist) if dist > distance_threshold: failed_runs.append( directory.ExperimentWithDistance(experiment, dist)) progress.update(check_task, advance=1.0) if len(linreg_xs) > 0: linreg_result = LinearRegression().fit(linreg_xs, linreg_ys) linreg_rsquared = linreg_result.score(linreg_xs, linreg_ys) console.print("Parameter error regression coefficient: {}".format( linreg_rsquared)) with open(os.path.join(prefix, "failed_paths.yaml"), "w") as outfile: yaml.add_representer(directory.ExpectedTrialDirectory, directory.DirectoryRepresenter) yaml.add_representer(directory.ExperimentTrialDirectory, directory.DirectoryRepresenter) yaml.add_representer(directory.ExperimentWithDistance, directory.ExperimentWithDistanceRepresenter) outfile.write( yaml.dump({ "failed-runs": failed_runs, "error-runs": error_runs })) if len(failed_runs) != 0 or len(error_runs) != 0: if len(failed_runs) != 0: console.print( "Tests that completed, but gave a wrong result (top 10):", sorted(failed_runs, key=lambda a: a._dist)[-10:]) console.print( "Total of {} ({}%) jobs resulted in errors over tolerance". format(len(failed_runs), len(failed_runs) / len(jobs) * 100)) if len(error_runs) != 0: console.print("Tests that failed to complete:", sorted(error_runs, key=lambda d: d._path)) console.print("Total of {} ({}%) jobs failed to run".format( len(error_runs), len(error_runs) / len(jobs) * 100)) if not prefix_specified and ( (len(failed_runs) > copy_threshold and not linreg_rsquared > 0.95) or len(error_runs) != 0): basename = os.path.split(prefix)[1] new_prefix = os.path.abspath(os.path.join(os.getcwd(), basename)) console.print( "Copying the failed directories to {}".format(new_prefix)) shutil.copytree(prefix, new_prefix) else: console.print("[bold green]All Clear!") end = timer() console.print("Testing took {:.3f} seconds".format(end - start))
def singularity_download_image(self, container, out_path, cache_path, progress): """Download a singularity image from the web. Use native Python to download the file. Args: container (str): A pipeline's container name. Usually it is of similar format to ``https://depot.galaxyproject.org/singularity/name:version`` out_path (str): The final target output path cache_path (str, None): The NXF_SINGULARITY_CACHEDIR path if set, None if not progress (Progress): Rich progress bar instance to add tasks to. """ log.debug(f"Downloading Singularity image: '{container}'") # Set output path to save file to output_path = cache_path or out_path output_path_tmp = f"{output_path}.partial" log.debug(f"Downloading to: '{output_path_tmp}'") # Set up progress bar nice_name = container.split("/")[-1][:50] task = progress.add_task(nice_name, start=False, total=False, progress_type="download") try: # Delete temporary file if it already exists if os.path.exists(output_path_tmp): os.remove(output_path_tmp) # Open file handle and download with open(output_path_tmp, "wb") as fh: # Disable caching as this breaks streamed downloads with requests_cache.disabled(): r = requests.get(container, allow_redirects=True, stream=True, timeout=60 * 5) filesize = r.headers.get("Content-length") if filesize: progress.update(task, total=int(filesize)) progress.start_task(task) # Stream download for data in r.iter_content(chunk_size=4096): # Check that the user didn't hit ctrl-c if self.kill_with_fire: raise KeyboardInterrupt progress.update(task, advance=len(data)) fh.write(data) # Rename partial filename to final filename os.rename(output_path_tmp, output_path) output_path_tmp = None # Copy cached download if we are using the cache if cache_path: log.debug("Copying {} from cache: '{}'".format( container, os.path.basename(out_path))) progress.update( task, description="Copying from cache to target directory") shutil.copyfile(cache_path, out_path) progress.remove_task(task) except: # Kill the progress bars for t in progress.task_ids: progress.remove_task(t) # Try to delete the incomplete download log.debug( f"Deleting incompleted singularity image download:\n'{output_path_tmp}'" ) if output_path_tmp and os.path.exists(output_path_tmp): os.remove(output_path_tmp) if output_path and os.path.exists(output_path): os.remove(output_path) # Re-raise the caught exception raise
def get_singularity_images(self): """Loop through container names and download Singularity images""" if len(self.containers) == 0: log.info("No container names found in workflow") else: with DownloadProgress() as progress: task = progress.add_task("all_containers", total=len(self.containers), progress_type="summary") # Organise containers based on what we need to do with them containers_exist = [] containers_cache = [] containers_download = [] containers_pull = [] for container in self.containers: # Fetch the output and cached filenames for this container out_path, cache_path = self.singularity_image_filenames( container) # Check that the directories exist out_path_dir = os.path.dirname(out_path) if not os.path.isdir(out_path_dir): log.debug( f"Output directory not found, creating: {out_path_dir}" ) os.makedirs(out_path_dir) if cache_path: cache_path_dir = os.path.dirname(cache_path) if not os.path.isdir(cache_path_dir): log.debug( f"Cache directory not found, creating: {cache_path_dir}" ) os.makedirs(cache_path_dir) # We already have the target file in place, return if os.path.exists(out_path): containers_exist.append(container) continue # We have a copy of this in the NXF_SINGULARITY_CACHE dir if cache_path and os.path.exists(cache_path): containers_cache.append( [container, out_path, cache_path]) continue # Direct download within Python if container.startswith("http"): containers_download.append( [container, out_path, cache_path]) continue # Pull using singularity containers_pull.append([container, out_path, cache_path]) # Exit if we need to pull images and Singularity is not installed if len(containers_pull) > 0 and shutil.which( "singularity") is None: raise OSError( "Singularity is needed to pull images, but it is not installed" ) # Go through each method of fetching containers in order for container in containers_exist: progress.update(task, description="Image file exists") progress.update(task, advance=1) for container in containers_cache: progress.update( task, description=f"Copying singularity images from cache") self.singularity_copy_cache_image(*container) progress.update(task, advance=1) with concurrent.futures.ThreadPoolExecutor( max_workers=self.parallel_downloads) as pool: progress.update( task, description="Downloading singularity images") # Kick off concurrent downloads future_downloads = [ pool.submit(self.singularity_download_image, *container, progress) for container in containers_download ] # Make ctrl-c work with multi-threading self.kill_with_fire = False try: # Iterate over each threaded download, waiting for them to finish for future in concurrent.futures.as_completed( future_downloads): try: future.result() except Exception: raise else: try: progress.update(task, advance=1) except Exception as e: log.error( f"Error updating progress bar: {e}") except KeyboardInterrupt: # Cancel the future threads that haven't started yet for future in future_downloads: future.cancel() # Set the variable that the threaded function looks for # Will trigger an exception from each thread self.kill_with_fire = True # Re-raise exception on the main thread raise for container in containers_pull: progress.update(task, description="Pulling singularity images") try: self.singularity_pull_image(*container, progress) except RuntimeWarning as r: # Raise exception if this is not possible log.error( "Not able to pull image. Service might be down or internet connection is dead." ) raise r progress.update(task, advance=1)
def __init__(self, kernel, bin, profile, mutop, timeout=30, fitness='time', popsize=128, llvm_src_filename='cuda-device-only-kernel.ll', use_fitness_map=True, combine_positive_epistasis=False, CXPB=0.8, MUPB=0.1, err_rate='0.01', global_seed=None): self.CXPB = CXPB self.MUPB = MUPB self.err_rate = err_rate self.kernels = kernel self.appBinary = bin self.timeout = timeout self.fitness_function = fitness self.use_fitness_map = use_fitness_map self.combine_positive_epistasis = combine_positive_epistasis self.popsize = popsize self.mutop = mutop.split(',') self.rng = {} if global_seed is not None: random.seed(global_seed) try: with open(llvm_src_filename, 'r') as f: self.initSrcEnc = f.read().encode() except IOError: print("File {} does not exist".format(llvm_src_filename)) exit(1) self.verifier = profile['verify'] # Tools initialization # Detect GPU property cuda.init() # TODO: check if there are multiple GPUs. SM_MAJOR, SM_MINOR = cuda.Device(0).compute_capability() self.mgpu = 'sm_' + str(SM_MAJOR) + str(SM_MINOR) print(f'[Initializing GEVO] GPU compute capability: {self.mgpu}') # check Nvidia Profiler exists self.nvprof_path = shutil.which('nvprof') if self.nvprof_path is None: raise Exception('nvprof cannot be found') print(f'[Initializing GEVO] nvprof detected: {self.nvprof_path}') # Minimize both performance and error creator.create("FitnessMin", base.Fitness, weights=(-1.0, -1.0)) creator.create("Individual", irind.llvmIRrep, fitness=creator.FitnessMin) self.history = tools.History() self.toolbox = base.Toolbox() self.toolbox.register('mutate', self.mutLLVM) self.toolbox.register('mate', self.cxOnePointLLVM) # self.toolbox.register('select', tools.selDoubleTournament, fitness_size=2, parsimony_size=1.4, fitness_first=True) self.toolbox.register('select', tools.selNSGA2) self.toolbox.register('individual', creator.Individual, srcEnc=self.initSrcEnc, mgpu=self.mgpu) self.toolbox.register('population', tools.initRepeat, list, self.toolbox.individual) # Decorate the variation operators self.toolbox.decorate("mate", self.history.decorator) self.toolbox.decorate("mutate", self.history.decorator) self.stats = tools.Statistics(lambda ind: ind.fitness.values) self.stats.register("min", min) self.stats.register("max", max) self.logbook = tools.Logbook() self.paretof = tools.ParetoFront() self.logbook.header = "gen", "evals", "min", "max" # Set up testcase self.origin = creator.Individual(self.initSrcEnc, self.mgpu) self.origin.ptx(self.cudaPTX) arg_array = [[]] for i, arg in enumerate(profile['args']): if arg.get('bond', None) is None: arg_array_next = [ e[:] for e in arg_array for _ in range(len(arg['value'])) ] arg_array = arg_array_next for e1, e2 in zip(arg_array, cycle(arg['value'])): e1.append(e2) else: for e in arg_array: bonded_arg = arg['bond'][0] bonded_idx = profile['args'][bonded_arg]['value'].index( e[bonded_arg]) e.append(arg['value'][bonded_idx]) arg_array = [[str(e) for e in args] for args in arg_array] self.testcase = [] for i in range(len(arg_array)): self.testcase.append( self._testcase(self, i, kernel, bin, profile['verify'])) with Progress( "[Initializing GEVO] Evaluate original program with test cases", "({task.completed} / {task.total})", rich.progress.TimeElapsedColumn()) as progress: task = progress.add_task("", total=len(arg_array)) for tc, arg in zip(self.testcase, arg_array): tc.args = arg tc.evaluate() progress.update(task, advance=1) self.ofits = [tc.fitness[0] for tc in self.testcase] self.oerrs = [tc.fitness[1] for tc in self.testcase] self.origin.fitness.values = (sum(self.ofits) / len(self.ofits), max(self.oerrs)) self.editFitMap[tuple()] = self.origin.fitness.values print( f"Average fitness of the original program: ({self.origin.fitness.values[0]:.2f}, {self.origin.fitness.values[1]:.2f})" ) print("Individual test cases:") for fit, err in zip(self.ofits, self.oerrs): print(f"\t({fit:.2f}, {err:.2f})") self.positive_epistasis = {} self.negative_epistasis = {} self.need_discussion = {}