def run_task(task, out_dir, n_jobs=1, timeout=None, task_timeout=None): dataset = openml.datasets.get_dataset(task.dataset_id) X, y, categorical = dataset.get_data( target=dataset.default_target_attribute, return_categorical_indicator=True) imputer = _conditional_imput(X, categorical) X = imputer.fit_transform(X) # TODO hardcoded random state X, y = shuffle(X, y, random_state=42) sample_size = _heuristic_sample_size(X.shape[0], X.shape[1]) if sample_size is not None: evaluator = SampleCrossValEvaluator(cv_k=5, timeout_s=timeout, per_gen=False, sample_size=sample_size) else: evaluator = CrossValEvaluator(cv_k=5, timeout_s=timeout) # TODO bad practice warning - hardcoded height clf = GenensClassifier(n_jobs=n_jobs, timeout=timeout, evaluator=evaluator, max_height=5) start_time = time.time() if task_timeout is not None: # timeout after task_timeout seconds try: with Timeout(task_timeout, swallow_exc=False): clf.fit(X, y) except TimeoutException: print('Timeout - task {} on dataset {}'.format( task.task_id, dataset.name)) return else: # run without interrupt clf.fit(X, y) elapsed_time = time.time() - start_time # log time with open(out_dir + '/time.txt', 'w+') as time_file: time_file.write('Elapsed time: {}\n'.format(elapsed_time)) _log_evolution(clf, out_dir) # run and log top 3 pipelines for i, pipe in enumerate(clf.get_best_pipelines()[:3]): pipe_dir = out_dir + '/run{}'.format(i) _create_dir_check(pipe_dir) evaluate_pipeline(pipe, X, y, pipe_dir)
def with_timeout(*args, **kwargs): if 'timeout' not in kwargs or kwargs['timeout'] is None: return fn(*args, **kwargs) try: timeout = kwargs['timeout'] with Timeout(timeout, swallow_exc=False): fn(*args, **kwargs) except TimeoutException: logger = logging.getLogger("genens") logger.debug(f"Time limit exceeded: {timeout}.")
def test_pattern(self, event=None): pat = self.tk_entry_search_pattern.get() try: with Timeout(10.0, swallow_exc=False) as timeout_ctx: res = glob(pat, recursive=True) except TimeoutException: print("Search for GRANULES was stopped after 10s without success -> stop here ") print("The search pattern was: %s" % pat) return print("Test search patterns: %s, results: %i" % (pat, len(res))) for ii, rr in enumerate(res): print("%i,%s" % (ii, rr))
def diff_revisions(revision_docs, processor, last_id=None, timeout=None): for revision_doc in revision_docs: diff = {'last_id': last_id} text = revision_doc['text'] or "" # Diff processing uses a lot of CPU. So we set a timeout for # crazy revisions and record a timer for analysis later. with Timer() as t: if timeout is None: # Just process the text operations, a, b = processor.process(text) diff['ops'] = [op2doc(op, a, b) for op in operations] else: # Try processing with a timeout try: with Timeout(timeout) as ctx: operations, a, b = processor.process(text) except TimeoutException: pass if ctx.state != ctx.TIMED_OUT: # We didn't timeout. cool. diff['ops'] = [op2doc(op, a, b) for op in operations] else: # We timed out. Record a giant delete and insert diff['ops'] = [{ 'name': "delete", 'a1': 0, 'a2': len(a), 'b1': 0, 'b2': 0, 'tokens': a }, { 'name': "insert", 'a1': 0, 'a2': 0, 'b1': 0, 'b2': len(b), 'tokens': b }] # Make sure that the processor state is right processor.update(last_text=(revision_doc['text'] or "")) # All done. Record how much time it all took diff['time'] = t.interval revision_doc['diff'] = diff yield revision_doc last_id = revision_doc['id']
def trace(self, url, state=None): if state is None: state = defaultdict(dict) if not self.php.running(): self.php.start() resp = None self.php.trace_begin() with Timeout(self.options.timeout) as timeout_ctx: resp = self._request_for_state(url, state) if timeout_ctx.state: LOG.warning('Timeout exceeded for %r', url) trace = self._collect(resp) if timeout_ctx.state: self.php.stop() return trace
def with_timeout(self, workflow, *args, **kwargs): if not hasattr(self, 'timeout') or self.timeout is None: return fn(self, workflow, *args, **kwargs) try: with Timeout(self.timeout, swallow_exc=False): res = fn(self, workflow, *args, **kwargs) except TimeoutException: logger = logging.getLogger("genens") logger.debug(f"Timeouted:\n {workflow}") print(f"Timeouted:\n {workflow}") res = None return res
def is_elf(path): """ :param path: the path to the file to analyze :return: True if the file is ELF, false otherwise """ try: with Timeout(1.0) as timeout_ctx: try: bytes = open(path, "rb").read(4).encode('hex') if bytes == "7f454c46": return True else: return False except TimeoutException: logging.warning("Could not open the file" + path + " in 1s. Skipping.") except: logging.error("Couldn't determine if " + path + " is ELF.") return False
def listen(): try: with Timeout(30.0, swallow_exc=False) as timeout_ctx: # play("start.mp3") # delete("start.mp3") r = sr.Recognizer() print("Say something!") start = time.time() with sr.Microphone() as source: print(time.time() - start) audio = r.listen(source) print(time.time() - start) try: soundfile = "temp.mp3" print(time.time() - start) print("listen") # play("end.mp3") # delete("end.mp3") text = r.recognize_google(audio, language="th-TH") print(time.time() - start) print("get it") # tts = gTTS(text, lang='th', slow=False) print("Google Speech Recognition thinks you said " + text) # playsound(soundfile) return text except sr.UnknownValueError: print("Google Speech Recognition could not understand audio") return "error1239123" except sr.RequestError as e: print( "Could not request results from Google Speech Recognition service; {0}" .format(e)) except sr.WaitTimeoutError: print("waiting too long") except TimeoutException: print("too slow") return "too slow"
def job_selector_ilp(self, jobs): """ Job scheduling by optimization of resource usage by solving ILP using pulp """ import pulp from pulp import lpSum from stopit import ThreadingTimeout as Timeout, TimeoutException logger.info("Select jobs to execute...") with self._lock: if not self.resources["_cores"]: return set() # assert self.resources["_cores"] > 0 scheduled_jobs = { job: pulp.LpVariable( "job_{}".format(idx), lowBound=0, upBound=1, cat=pulp.LpInteger, ) for idx, job in enumerate(jobs) } def size_gb(f): if self.touch: # In case of touch mode, there is no need to prioritize based on size. # We cannot access it anyway, because the files might be temporary and # not present. return 0 else: return f.size / 1e9 temp_files = { temp_file for job in jobs for temp_file in self.dag.temp_input(job) } temp_job_improvement = { temp_file: pulp.LpVariable( "temp_file_{}".format(idx), lowBound=0, upBound=1, cat="Continuous" ) for idx, temp_file in enumerate(temp_files) } temp_file_deletable = { temp_file: pulp.LpVariable( "deletable_{}".format(idx), lowBound=0, upBound=1, cat=pulp.LpInteger, ) for idx, temp_file in enumerate(temp_files) } prob = pulp.LpProblem("JobScheduler", pulp.LpMaximize) total_temp_size = max( sum([size_gb(temp_file) for temp_file in temp_files]), 1 ) total_core_requirement = sum( [max(job.resources.get("_cores", 1), 1) for job in jobs] ) # Objective function # Job priority > Core load # Core load > temp file removal # Instant removal > temp size prob += ( 2 * total_core_requirement * 2 * total_temp_size * lpSum([job.priority * scheduled_jobs[job] for job in jobs]) + 2 * total_temp_size * lpSum( [ max(job.resources.get("_cores", 1), 1) * scheduled_jobs[job] for job in jobs ] ) + total_temp_size * lpSum( [ temp_file_deletable[temp_file] * size_gb(temp_file) for temp_file in temp_files ] ) + lpSum( [ temp_job_improvement[temp_file] * size_gb(temp_file) for temp_file in temp_files ] ) ) # Constraints: for name in self.workflow.global_resources: prob += ( lpSum( [ scheduled_jobs[job] * job.resources.get(name, 0) for job in jobs ] ) <= self.resources[name] ) # Choose jobs that lead to "fastest" (minimum steps) removal of existing temp file remaining_jobs = self.remaining_jobs for temp_file in temp_files: prob += temp_job_improvement[temp_file] <= lpSum( [ scheduled_jobs[job] * self.required_by_job(temp_file, job) for job in jobs ] ) / lpSum( [self.required_by_job(temp_file, job) for job in remaining_jobs] ) prob += ( temp_file_deletable[temp_file] <= temp_job_improvement[temp_file] ) try: with Timeout(10, swallow_exc=False): self._solve_ilp(prob) except TimeoutException as e: logger.warning( "Failed to solve scheduling problem with ILP solver in time (10s). " "Falling back to greedy solver." ) return self.job_selector_greedy(jobs) except pulp.apis.core.PulpSolverError as e: logger.warning( "Failed to solve scheduling problem with ILP solver. Falling back to greedy solver. " "Run Snakemake with --verbose to see the full solver output for debugging the problem." ) return self.job_selector_greedy(jobs) selected_jobs = set( job for job, variable in scheduled_jobs.items() if variable.value() == 1.0 ) if not selected_jobs: # No selected jobs. This could be due to insufficient resources or a failure in the ILP solver # Hence, we silently fall back to the greedy solver to make sure that we don't miss anything. return self.job_selector_greedy(jobs) for name in self.workflow.global_resources: self.resources[name] -= sum( [job.resources.get(name, 0) for job in selected_jobs] ) return selected_jobs
def main(ctx): ''' Main task to run a full site build process. All values needed for the build are loaded from environment variables. ''' # (variable naming) # pylint: disable=C0103 # keep track of total time start_time = datetime.now() # Load from .env for development load_dotenv() # These environment variables will be set into the environment # by federalist-builder. # During development, we can use a `.env` file (loaded above) # to make it easier to specify variables. AWS_DEFAULT_REGION = os.environ['AWS_DEFAULT_REGION'] BUCKET = os.environ['BUCKET'] BASEURL = os.environ['BASEURL'] CACHE_CONTROL = os.environ['CACHE_CONTROL'] BRANCH = os.environ['BRANCH'] CONFIG = os.environ['CONFIG'] REPOSITORY = os.environ['REPOSITORY'] OWNER = os.environ['OWNER'] SITE_PREFIX = os.environ['SITE_PREFIX'] GENERATOR = os.environ['GENERATOR'] AWS_ACCESS_KEY_ID = os.environ['AWS_ACCESS_KEY_ID'] AWS_SECRET_ACCESS_KEY = os.environ['AWS_SECRET_ACCESS_KEY'] # List of private strings to be removed from any posted logs private_values = [AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY] # Optional environment variables SOURCE_REPO = os.getenv('SOURCE_REPO', '') SOURCE_OWNER = os.getenv('SOURCE_OWNER', '') # GITHUB_TOKEN can be empty if a non-Federalist user # makes a commit to a repo and thus initiates a build for a # Federalist site GITHUB_TOKEN = os.getenv('GITHUB_TOKEN', '') if GITHUB_TOKEN: # only include it in list of values to strip from log output # if it exists private_values.append(GITHUB_TOKEN) # Ex: https://federalist-builder.fr.cloud.gov/builds/<token>/callback FEDERALIST_BUILDER_CALLBACK = os.environ['FEDERALIST_BUILDER_CALLBACK'] # Ex: https://federalist-staging.18f.gov/v0/build/<build_id>/status/<token> STATUS_CALLBACK = os.environ['STATUS_CALLBACK'] # Ex: https://federalist-staging.18f.gov/v0/build/<build_id>/log/<token> LOG_CALLBACK = os.environ['LOG_CALLBACK'] try: # throw a timeout exception after TIMEOUT_SECONDS with Timeout(TIMEOUT_SECONDS, swallow_exc=False): LOGGER.info(f'Running build for {OWNER}/{REPOSITORY}/{BRANCH}') # Unfortunately, pyinvoke doesn't have a great way to call tasks # from within other tasks. If you call them directly, # their pre- and post-dependencies are not executed. # # Here's the GitHub issue about this: # https://github.com/pyinvoke/invoke/issues/170 # # So rather than calling the task functions through python, we'll # call them instead using `ctx.run('invoke the_task ...')` via the # helper `run_task` method. ## # CLONE # if SOURCE_OWNER and SOURCE_REPO: # First clone the source (ie, template) repository clone_source_flags = { '--owner': SOURCE_OWNER, '--repository': SOURCE_REPO, '--branch': BRANCH, } run_task(ctx, 'clone-repo', log_callback=LOG_CALLBACK, private_values=private_values, flags_dict=clone_source_flags, env={'GITHUB_TOKEN': GITHUB_TOKEN}) # Then push the cloned source repo up to the destination repo. # Note that the dest repo must already exist but be empty. # The Federalist web app takes care of that operation. push_repo_flags = { '--owner': OWNER, '--repository': REPOSITORY, '--branch': BRANCH, } run_task(ctx, 'push-repo-remote', log_callback=LOG_CALLBACK, private_values=private_values, flags_dict=push_repo_flags, env={'GITHUB_TOKEN': GITHUB_TOKEN}) else: # Just clone the given repository clone_flags = { '--owner': OWNER, '--repository': REPOSITORY, '--branch': BRANCH, '--depth': '--depth 1', } run_task(ctx, 'clone-repo', log_callback=LOG_CALLBACK, private_values=private_values, flags_dict=clone_flags, env={'GITHUB_TOKEN': GITHUB_TOKEN}) ## # BUILD # build_flags = { '--branch': BRANCH, '--owner': OWNER, '--repository': REPOSITORY, '--site-prefix': SITE_PREFIX, '--base-url': BASEURL, } # Run the npm `federalist` task (if it is defined) run_task(ctx, 'run-federalist-script', log_callback=LOG_CALLBACK, private_values=private_values, flags_dict=build_flags) # Run the appropriate build engine based on GENERATOR if GENERATOR == 'jekyll': build_flags['--config'] = CONFIG run_task(ctx, 'build-jekyll', log_callback=LOG_CALLBACK, private_values=private_values, flags_dict=build_flags) elif GENERATOR == 'hugo': # extra: --hugo-version (not yet used) run_task(ctx, 'build-hugo', log_callback=LOG_CALLBACK, private_values=private_values, flags_dict=build_flags) elif GENERATOR == 'static': # no build arguments are needed run_task(ctx, 'build-static', log_callback=LOG_CALLBACK, private_values=private_values) elif (GENERATOR == 'node.js' or GENERATOR == 'script only'): LOGGER.info('build already ran in \'npm run federalist\'') else: raise ValueError(f'Invalid GENERATOR: {GENERATOR}') ## # PUBLISH # publish_flags = { '--base-url': BASEURL, '--site-prefix': SITE_PREFIX, '--bucket': BUCKET, '--cache-control': CACHE_CONTROL, '--aws-region': AWS_DEFAULT_REGION, } publish_env = { 'AWS_ACCESS_KEY_ID': AWS_ACCESS_KEY_ID, 'AWS_SECRET_ACCESS_KEY': AWS_SECRET_ACCESS_KEY, } run_task(ctx, 'publish', log_callback=LOG_CALLBACK, private_values=private_values, flags_dict=publish_flags, env=publish_env) delta_string = delta_to_mins_secs(datetime.now() - start_time) LOGGER.info(f'Total build time: {delta_string}') # Finished! post_build_complete(STATUS_CALLBACK, FEDERALIST_BUILDER_CALLBACK) except TimeoutException: LOGGER.info('Build has timed out') post_build_timeout(LOG_CALLBACK, STATUS_CALLBACK, FEDERALIST_BUILDER_CALLBACK) except UnexpectedExit as err: # Combine the error's stdout and stderr into one string err_string = format_output(err.result.stdout, err.result.stderr) # replace any private values that might be in the error message err_string = replace_private_values(err_string, private_values) # log the original exception LOGGER.info(f'Exception raised during build:') LOGGER.info(err_string) # replace the message with a custom one, if it exists err_string = find_custom_error_message(err_string) post_build_error(LOG_CALLBACK, STATUS_CALLBACK, FEDERALIST_BUILDER_CALLBACK, err_string) except Exception as err: # pylint: disable=W0703 # Getting here means something really weird has happened # since all errors caught during tasks should be caught # in the previous block as `UnexpectedExit` exceptions. err_string = str(err) err_string = replace_private_values(err_string, private_values) # log the original exception LOGGER.info('Unexpected exception raised during build:') LOGGER.info(err_string) err_message = ('Unexpected error. Please try again and ' 'contact federalist-support if it persists.') post_build_error(LOG_CALLBACK, STATUS_CALLBACK, FEDERALIST_BUILDER_CALLBACK, err_message)
def get_factors(ivec): with Timeout(0.1) as timeout_ctx: return [fac(x) for x in ivec] return []
def build(aws_access_key_id, aws_default_region, aws_secret_access_key, federalist_builder_callback, status_callback, baseurl, branch, bucket, build_id, config, generator, github_token, owner, repository, site_prefix, user_environment_variables=[]): ''' Main task to run a full site build process. All values needed for the build are loaded from environment variables. ''' # keep track of total time start_time = datetime.now() # Make the working directory if it doesn't exist WORKING_DIR_PATH.mkdir(exist_ok=True) logger = None cache_control = os.getenv('CACHE_CONTROL', 'max-age=60') database_url = os.environ['DATABASE_URL'] user_environment_variable_key = os.environ['USER_ENVIRONMENT_VARIABLE_KEY'] try: post_build_processing(status_callback) # throw a timeout exception after TIMEOUT_SECONDS with Timeout(TIMEOUT_SECONDS, swallow_exc=False): build_info = f'{owner}/{repository}@id:{build_id}' decrypted_uevs = decrypt_uevs(user_environment_variable_key, user_environment_variables) priv_vals = [uev['value'] for uev in decrypted_uevs] priv_vals.append(aws_access_key_id) priv_vals.append(aws_secret_access_key) if github_token: priv_vals.append(github_token) logattrs = { 'branch': branch, 'buildid': build_id, 'owner': owner, 'repository': repository, } init_logging(priv_vals, logattrs, database_url) logger = get_logger('main') def run_step(returncode, msg): if returncode != 0: raise StepException(msg) logger.info(f'Running build for {owner}/{repository}/{branch}') if generator not in GENERATORS: raise ValueError(f'Invalid generator: {generator}') ## # FETCH # run_step( fetch_repo(owner, repository, branch, github_token), 'There was a problem fetching the repository, see the above logs for details.' ) ## # BUILD # run_step( setup_node(), 'There was a problem setting up Node, see the above logs for details.' ) # Run the npm `federalist` task (if it is defined) run_step( run_federalist_script(branch, owner, repository, site_prefix, baseurl, decrypted_uevs), 'There was a problem running the federalist script, see the above logs for details.' ) # Run the appropriate build engine based on generator if generator == 'jekyll': run_step( setup_ruby(), 'There was a problem setting up Ruby, see the above logs for details.' ) run_step( setup_bundler(), 'There was a problem setting up Bundler, see the above logs for details.' ) run_step( build_jekyll(branch, owner, repository, site_prefix, baseurl, config, decrypted_uevs), 'There was a problem running Jekyll, see the above logs for details.' ) elif generator == 'hugo': # extra: --hugo-version (not yet used) run_step( download_hugo(), 'There was a problem downloading Hugo, see the above logs for details.' ) run_step( build_hugo(branch, owner, repository, site_prefix, baseurl, decrypted_uevs), 'There was a problem running Hugo, see the above logs for details.' ) elif generator == 'static': # no build arguments are needed build_static() elif (generator == 'node.js' or generator == 'script only'): logger.info('build already ran in \'npm run federalist\'') else: raise ValueError(f'Invalid generator: {generator}') ## # PUBLISH # publish(baseurl, site_prefix, bucket, cache_control, aws_default_region, aws_access_key_id, aws_secret_access_key) delta_string = delta_to_mins_secs(datetime.now() - start_time) logger.info(f'Total build time: {delta_string}') # Finished! post_build_complete(status_callback, federalist_builder_callback) sys.exit(0) except StepException as err: ''' Thrown when a step itself fails, usually because a command exited with a non-zero return code ''' logger.error(str(err)) post_build_error(status_callback, federalist_builder_callback, str(err)) sys.exit(1) except TimeoutException: logger.warning(f'Build({build_info}) has timed out') post_build_timeout(status_callback, federalist_builder_callback) except Exception as err: # pylint: disable=W0703 # Getting here means something really weird has happened # since all errors caught during tasks should be caught # in the previous block as `UnexpectedExit` exceptions. err_string = str(err) # log the original exception msg = f'Unexpected exception raised during build({build_info}): {err_string}' if logger: logger.warning(msg) else: print(msg) err_message = (f'Unexpected build({build_info}) error. Please try ' 'again and contact federalist-support if it persists.') post_build_error(status_callback, federalist_builder_callback, err_message)
def listen(language): if language == 'th': to_lang = 'en' else: to_lang = 'th' # play("start.mp3") # delete("start.mp3") r = sr.Recognizer() print("Say something!") start = time.time() with sr.Microphone() as source: print(time.time() - start) audio = r.listen(source) print(time.time() - start) try: with Timeout(10, swallow_exc=False) as timeout_ctx: print("listen finished sending to google") if language == 'th': text = r.recognize_google(audio, language="th-TH") else: text = r.recognize_google(audio, language="en-US") # play("end.mp3") # delete("end.mp3") print(time.time() - start) print("convert to text done") print("Google Speech Recognition thinks you said " + text) result = translate_for_me(language, to_lang, text) tts = gTTS(result, lang=to_lang, slow=False) tts.save('testtrans.mp3') play('testtrans.mp3') delete('testtrans.mp3') return text + ',' + result except TimeoutException: print("too slow") return "too slow" except sr.UnknownValueError: print("Google Speech Recognition could not understand audio") return "error1239123" except sr.RequestError as e: print( "Could not request results from Google Speech Recognition service; {0}" .format(e)) except sr.WaitTimeoutError: print("waiting too long") except: try: with Timeout(10, swallow_exc=False) as timeout_ctx: if language == 'th': text = r.recognize_google(audio, language="th-TH") else: text = r.recognize_google(audio, language="en-US") # play("end.mp3") # delete("end.mp3") print(time.time() - start) print("convert to text done") # tts = gTTS(text, lang='th', slow=False) print("Google Speech Recognition thinks you said " + text) result = translate_for_me(language, to_lang, text) tts = gTTS(result, lang=to_lang, slow=False) tts.save('testtrans.mp3') play('testtrans.mp3') time.sleep(5) delete('testtrans.mp3') return text + ',' + result except TimeoutException: print("too slow") return "too slow"
def main(args, logger=None): logger = logger or logging.getLogger(__name__) logger.info(texts["welcome"]) if args.show_warnings is not True: logger.info("Switch off printing of warnings from python packages") warnings.filterwarnings("ignore") if args.persistence_file is not None: logger.info("Read classical Bayesian persistence file: %s" % args.persistence_file) builtins.S2_clf = S2MSI.CloudMask(logger=logger,persistence_file=args.persistence_file) if args.S2_MSI_granule_path is None: try: with Timeout(10.0, swallow_exc=False) as timeout_ctx: args.S2_MSI_granule_path = glob(args.glob_search_pattern, recursive=True) except TimeoutException: logger.info("Search for GRANULES was stoped after 10s without success -> stop here ") logger.info("The search pattern was: %s" % args.glob_search_pattern) return logger.info("No Input data given -> traverse local path and search for granules:") for granule in args.S2_MSI_granule_path: logger.info(granule) if args.number_of_threads == 1: logger.info("Start processing of %i jobs." % len(args.S2_MSI_granule_path)) for granule in args.S2_MSI_granule_path: mask_image(args, granule) elif args.number_of_threads > 1: logger.info("Start processing of %i jobs using %i processes." % ( len(args.S2_MSI_granule_path), args.number_of_threads)) def _init_(): globals()["update"] = lambda: None # tkinter is not able to digest multiple threads sys.stdout = S2MSI.Tools.StdoutToList() sys.stderr = S2MSI.Tools.StdoutToList() logger.info("Start Processing in parallel running threads. Output of individual jobs is shown when done. ") if args.use_thread_pool is True: logger.info("!!!!!!!!!!!! Use ThreadPool !!!!!!!!!") Pool = ThreadPool pool = Pool(initializer=_init_, processes=args.number_of_threads) tasks = [(args, granule, logger) for granule in args.S2_MSI_granule_path] jobs = [pool.apply_async(mask_image, task) for task in tasks] while len(jobs) > 0: sleep(5.0) readies = [job.ready() for job in jobs] for job, ready in zip(jobs, readies): if ready is True: lne_out, lne_err = job.get() logger.info("### print jop output ###") for line in lne_out: logger.info(line) if len(lne_err) > 1: logger.info("### errors occurred -> print stderr: ###") for line in lne_out: logger.info(line) jobs[:] = [job for job, ready in zip(jobs, readies) if ready is False] logger.info("#> Open Jobs: %i <#" % len(jobs)) pool.close() else: raise ValueError("The number of threads should be larger or equal to zero and not:%i" % args.number_of_threads) logger.info(texts["END"])