def run_task(task, out_dir, n_jobs=1, timeout=None, task_timeout=None):
dataset = openml.datasets.get_dataset(task.dataset_id)
X, y, categorical = dataset.get_data(
target=dataset.default_target_attribute,
return_categorical_indicator=True)
imputer = _conditional_imput(X, categorical)
X = imputer.fit_transform(X)
# TODO hardcoded random state
X, y = shuffle(X, y, random_state=42)
sample_size = _heuristic_sample_size(X.shape[0], X.shape[1])
if sample_size is not None:
evaluator = SampleCrossValEvaluator(cv_k=5,
timeout_s=timeout,
per_gen=False,
sample_size=sample_size)
else:
evaluator = CrossValEvaluator(cv_k=5, timeout_s=timeout)
# TODO bad practice warning - hardcoded height
clf = GenensClassifier(n_jobs=n_jobs,
timeout=timeout,
evaluator=evaluator,
max_height=5)
start_time = time.time()
if task_timeout is not None:
# timeout after task_timeout seconds
try:
with Timeout(task_timeout, swallow_exc=False):
clf.fit(X, y)
except TimeoutException:
print('Timeout - task {} on dataset {}'.format(
task.task_id, dataset.name))
return
else:
# run without interrupt
clf.fit(X, y)
elapsed_time = time.time() - start_time
# log time
with open(out_dir + '/time.txt', 'w+') as time_file:
time_file.write('Elapsed time: {}\n'.format(elapsed_time))
_log_evolution(clf, out_dir)
# run and log top 3 pipelines
for i, pipe in enumerate(clf.get_best_pipelines()[:3]):
pipe_dir = out_dir + '/run{}'.format(i)
_create_dir_check(pipe_dir)
evaluate_pipeline(pipe, X, y, pipe_dir)
def with_timeout(*args, **kwargs):
if 'timeout' not in kwargs or kwargs['timeout'] is None:
return fn(*args, **kwargs)
try:
timeout = kwargs['timeout']
with Timeout(timeout, swallow_exc=False):
fn(*args, **kwargs)
except TimeoutException:
logger = logging.getLogger("genens")
logger.debug(f"Time limit exceeded: {timeout}.")
def test_pattern(self, event=None):
pat = self.tk_entry_search_pattern.get()
try:
with Timeout(10.0, swallow_exc=False) as timeout_ctx:
res = glob(pat, recursive=True)
except TimeoutException:
print("Search for GRANULES was stopped after 10s without success -> stop here ")
print("The search pattern was: %s" % pat)
return
print("Test search patterns: %s, results: %i" % (pat, len(res)))
for ii, rr in enumerate(res):
print("%i,%s" % (ii, rr))
def diff_revisions(revision_docs, processor, last_id=None, timeout=None):
for revision_doc in revision_docs:
diff = {'last_id': last_id}
text = revision_doc['text'] or ""
# Diff processing uses a lot of CPU. So we set a timeout for
# crazy revisions and record a timer for analysis later.
with Timer() as t:
if timeout is None:
# Just process the text
operations, a, b = processor.process(text)
diff['ops'] = [op2doc(op, a, b) for op in operations]
else:
# Try processing with a timeout
try:
with Timeout(timeout) as ctx:
operations, a, b = processor.process(text)
except TimeoutException:
pass
if ctx.state != ctx.TIMED_OUT:
# We didn't timeout. cool.
diff['ops'] = [op2doc(op, a, b) for op in operations]
else:
# We timed out. Record a giant delete and insert
diff['ops'] = [{
'name': "delete",
'a1': 0,
'a2': len(a),
'b1': 0,
'b2': 0,
'tokens': a
}, {
'name': "insert",
'a1': 0,
'a2': 0,
'b1': 0,
'b2': len(b),
'tokens': b
}]
# Make sure that the processor state is right
processor.update(last_text=(revision_doc['text'] or ""))
# All done. Record how much time it all took
diff['time'] = t.interval
revision_doc['diff'] = diff
yield revision_doc
last_id = revision_doc['id']
def trace(self, url, state=None):
if state is None:
state = defaultdict(dict)
if not self.php.running():
self.php.start()
resp = None
self.php.trace_begin()
with Timeout(self.options.timeout) as timeout_ctx:
resp = self._request_for_state(url, state)
if timeout_ctx.state:
LOG.warning('Timeout exceeded for %r', url)
trace = self._collect(resp)
if timeout_ctx.state:
self.php.stop()
return trace
def with_timeout(self, workflow, *args, **kwargs):
if not hasattr(self, 'timeout') or self.timeout is None:
return fn(self, workflow, *args, **kwargs)
try:
with Timeout(self.timeout, swallow_exc=False):
res = fn(self, workflow, *args, **kwargs)
except TimeoutException:
logger = logging.getLogger("genens")
logger.debug(f"Timeouted:\n {workflow}")
print(f"Timeouted:\n {workflow}")
res = None
return res
def is_elf(path):
"""
:param path: the path to the file to analyze
:return: True if the file is ELF, false otherwise
"""
try:
with Timeout(1.0) as timeout_ctx:
try:
bytes = open(path, "rb").read(4).encode('hex')
if bytes == "7f454c46":
return True
else:
return False
except TimeoutException:
logging.warning("Could not open the file" + path +
" in 1s. Skipping.")
except:
logging.error("Couldn't determine if " + path + " is ELF.")
return False
def listen():
try:
with Timeout(30.0, swallow_exc=False) as timeout_ctx:
# play("start.mp3")
# delete("start.mp3")
r = sr.Recognizer()
print("Say something!")
start = time.time()
with sr.Microphone() as source:
print(time.time() - start)
audio = r.listen(source)
print(time.time() - start)
try:
soundfile = "temp.mp3"
print(time.time() - start)
print("listen")
# play("end.mp3")
# delete("end.mp3")
text = r.recognize_google(audio, language="th-TH")
print(time.time() - start)
print("get it")
# tts = gTTS(text, lang='th', slow=False)
print("Google Speech Recognition thinks you said " + text)
# playsound(soundfile)
return text
except sr.UnknownValueError:
print("Google Speech Recognition could not understand audio")
return "error1239123"
except sr.RequestError as e:
print(
"Could not request results from Google Speech Recognition service; {0}"
.format(e))
except sr.WaitTimeoutError:
print("waiting too long")
except TimeoutException:
print("too slow")
return "too slow"
def job_selector_ilp(self, jobs):
"""
Job scheduling by optimization of resource usage by solving ILP using pulp
"""
import pulp
from pulp import lpSum
from stopit import ThreadingTimeout as Timeout, TimeoutException
logger.info("Select jobs to execute...")
with self._lock:
if not self.resources["_cores"]:
return set()
# assert self.resources["_cores"] > 0
scheduled_jobs = {
job: pulp.LpVariable(
"job_{}".format(idx),
lowBound=0,
upBound=1,
cat=pulp.LpInteger,
)
for idx, job in enumerate(jobs)
}
def size_gb(f):
if self.touch:
# In case of touch mode, there is no need to prioritize based on size.
# We cannot access it anyway, because the files might be temporary and
# not present.
return 0
else:
return f.size / 1e9
temp_files = {
temp_file for job in jobs for temp_file in self.dag.temp_input(job)
}
temp_job_improvement = {
temp_file: pulp.LpVariable(
"temp_file_{}".format(idx), lowBound=0, upBound=1, cat="Continuous"
)
for idx, temp_file in enumerate(temp_files)
}
temp_file_deletable = {
temp_file: pulp.LpVariable(
"deletable_{}".format(idx),
lowBound=0,
upBound=1,
cat=pulp.LpInteger,
)
for idx, temp_file in enumerate(temp_files)
}
prob = pulp.LpProblem("JobScheduler", pulp.LpMaximize)
total_temp_size = max(
sum([size_gb(temp_file) for temp_file in temp_files]), 1
)
total_core_requirement = sum(
[max(job.resources.get("_cores", 1), 1) for job in jobs]
)
# Objective function
# Job priority > Core load
# Core load > temp file removal
# Instant removal > temp size
prob += (
2
* total_core_requirement
* 2
* total_temp_size
* lpSum([job.priority * scheduled_jobs[job] for job in jobs])
+ 2
* total_temp_size
* lpSum(
[
max(job.resources.get("_cores", 1), 1) * scheduled_jobs[job]
for job in jobs
]
)
+ total_temp_size
* lpSum(
[
temp_file_deletable[temp_file] * size_gb(temp_file)
for temp_file in temp_files
]
)
+ lpSum(
[
temp_job_improvement[temp_file] * size_gb(temp_file)
for temp_file in temp_files
]
)
)
# Constraints:
for name in self.workflow.global_resources:
prob += (
lpSum(
[
scheduled_jobs[job] * job.resources.get(name, 0)
for job in jobs
]
)
<= self.resources[name]
)
# Choose jobs that lead to "fastest" (minimum steps) removal of existing temp file
remaining_jobs = self.remaining_jobs
for temp_file in temp_files:
prob += temp_job_improvement[temp_file] <= lpSum(
[
scheduled_jobs[job] * self.required_by_job(temp_file, job)
for job in jobs
]
) / lpSum(
[self.required_by_job(temp_file, job) for job in remaining_jobs]
)
prob += (
temp_file_deletable[temp_file] <= temp_job_improvement[temp_file]
)
try:
with Timeout(10, swallow_exc=False):
self._solve_ilp(prob)
except TimeoutException as e:
logger.warning(
"Failed to solve scheduling problem with ILP solver in time (10s). "
"Falling back to greedy solver."
)
return self.job_selector_greedy(jobs)
except pulp.apis.core.PulpSolverError as e:
logger.warning(
"Failed to solve scheduling problem with ILP solver. Falling back to greedy solver. "
"Run Snakemake with --verbose to see the full solver output for debugging the problem."
)
return self.job_selector_greedy(jobs)
selected_jobs = set(
job for job, variable in scheduled_jobs.items() if variable.value() == 1.0
)
if not selected_jobs:
# No selected jobs. This could be due to insufficient resources or a failure in the ILP solver
# Hence, we silently fall back to the greedy solver to make sure that we don't miss anything.
return self.job_selector_greedy(jobs)
for name in self.workflow.global_resources:
self.resources[name] -= sum(
[job.resources.get(name, 0) for job in selected_jobs]
)
return selected_jobs
def main(ctx):
'''
Main task to run a full site build process.
All values needed for the build are loaded from
environment variables.
'''
# (variable naming)
# pylint: disable=C0103
# keep track of total time
start_time = datetime.now()
# Load from .env for development
load_dotenv()
# These environment variables will be set into the environment
# by federalist-builder.
# During development, we can use a `.env` file (loaded above)
# to make it easier to specify variables.
AWS_DEFAULT_REGION = os.environ['AWS_DEFAULT_REGION']
BUCKET = os.environ['BUCKET']
BASEURL = os.environ['BASEURL']
CACHE_CONTROL = os.environ['CACHE_CONTROL']
BRANCH = os.environ['BRANCH']
CONFIG = os.environ['CONFIG']
REPOSITORY = os.environ['REPOSITORY']
OWNER = os.environ['OWNER']
SITE_PREFIX = os.environ['SITE_PREFIX']
GENERATOR = os.environ['GENERATOR']
AWS_ACCESS_KEY_ID = os.environ['AWS_ACCESS_KEY_ID']
AWS_SECRET_ACCESS_KEY = os.environ['AWS_SECRET_ACCESS_KEY']
# List of private strings to be removed from any posted logs
private_values = [AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY]
# Optional environment variables
SOURCE_REPO = os.getenv('SOURCE_REPO', '')
SOURCE_OWNER = os.getenv('SOURCE_OWNER', '')
# GITHUB_TOKEN can be empty if a non-Federalist user
# makes a commit to a repo and thus initiates a build for a
# Federalist site
GITHUB_TOKEN = os.getenv('GITHUB_TOKEN', '')
if GITHUB_TOKEN:
# only include it in list of values to strip from log output
# if it exists
private_values.append(GITHUB_TOKEN)
# Ex: https://federalist-builder.fr.cloud.gov/builds/<token>/callback
FEDERALIST_BUILDER_CALLBACK = os.environ['FEDERALIST_BUILDER_CALLBACK']
# Ex: https://federalist-staging.18f.gov/v0/build/<build_id>/status/<token>
STATUS_CALLBACK = os.environ['STATUS_CALLBACK']
# Ex: https://federalist-staging.18f.gov/v0/build/<build_id>/log/<token>
LOG_CALLBACK = os.environ['LOG_CALLBACK']
try:
# throw a timeout exception after TIMEOUT_SECONDS
with Timeout(TIMEOUT_SECONDS, swallow_exc=False):
LOGGER.info(f'Running build for {OWNER}/{REPOSITORY}/{BRANCH}')
# Unfortunately, pyinvoke doesn't have a great way to call tasks
# from within other tasks. If you call them directly,
# their pre- and post-dependencies are not executed.
#
# Here's the GitHub issue about this:
# https://github.com/pyinvoke/invoke/issues/170
#
# So rather than calling the task functions through python, we'll
# call them instead using `ctx.run('invoke the_task ...')` via the
# helper `run_task` method.
##
# CLONE
#
if SOURCE_OWNER and SOURCE_REPO:
# First clone the source (ie, template) repository
clone_source_flags = {
'--owner': SOURCE_OWNER,
'--repository': SOURCE_REPO,
'--branch': BRANCH,
}
run_task(ctx,
'clone-repo',
log_callback=LOG_CALLBACK,
private_values=private_values,
flags_dict=clone_source_flags,
env={'GITHUB_TOKEN': GITHUB_TOKEN})
# Then push the cloned source repo up to the destination repo.
# Note that the dest repo must already exist but be empty.
# The Federalist web app takes care of that operation.
push_repo_flags = {
'--owner': OWNER,
'--repository': REPOSITORY,
'--branch': BRANCH,
}
run_task(ctx,
'push-repo-remote',
log_callback=LOG_CALLBACK,
private_values=private_values,
flags_dict=push_repo_flags,
env={'GITHUB_TOKEN': GITHUB_TOKEN})
else:
# Just clone the given repository
clone_flags = {
'--owner': OWNER,
'--repository': REPOSITORY,
'--branch': BRANCH,
'--depth': '--depth 1',
}
run_task(ctx,
'clone-repo',
log_callback=LOG_CALLBACK,
private_values=private_values,
flags_dict=clone_flags,
env={'GITHUB_TOKEN': GITHUB_TOKEN})
##
# BUILD
#
build_flags = {
'--branch': BRANCH,
'--owner': OWNER,
'--repository': REPOSITORY,
'--site-prefix': SITE_PREFIX,
'--base-url': BASEURL,
}
# Run the npm `federalist` task (if it is defined)
run_task(ctx,
'run-federalist-script',
log_callback=LOG_CALLBACK,
private_values=private_values,
flags_dict=build_flags)
# Run the appropriate build engine based on GENERATOR
if GENERATOR == 'jekyll':
build_flags['--config'] = CONFIG
run_task(ctx,
'build-jekyll',
log_callback=LOG_CALLBACK,
private_values=private_values,
flags_dict=build_flags)
elif GENERATOR == 'hugo':
# extra: --hugo-version (not yet used)
run_task(ctx,
'build-hugo',
log_callback=LOG_CALLBACK,
private_values=private_values,
flags_dict=build_flags)
elif GENERATOR == 'static':
# no build arguments are needed
run_task(ctx,
'build-static',
log_callback=LOG_CALLBACK,
private_values=private_values)
elif (GENERATOR == 'node.js' or GENERATOR == 'script only'):
LOGGER.info('build already ran in \'npm run federalist\'')
else:
raise ValueError(f'Invalid GENERATOR: {GENERATOR}')
##
# PUBLISH
#
publish_flags = {
'--base-url': BASEURL,
'--site-prefix': SITE_PREFIX,
'--bucket': BUCKET,
'--cache-control': CACHE_CONTROL,
'--aws-region': AWS_DEFAULT_REGION,
}
publish_env = {
'AWS_ACCESS_KEY_ID': AWS_ACCESS_KEY_ID,
'AWS_SECRET_ACCESS_KEY': AWS_SECRET_ACCESS_KEY,
}
run_task(ctx,
'publish',
log_callback=LOG_CALLBACK,
private_values=private_values,
flags_dict=publish_flags,
env=publish_env)
delta_string = delta_to_mins_secs(datetime.now() - start_time)
LOGGER.info(f'Total build time: {delta_string}')
# Finished!
post_build_complete(STATUS_CALLBACK, FEDERALIST_BUILDER_CALLBACK)
except TimeoutException:
LOGGER.info('Build has timed out')
post_build_timeout(LOG_CALLBACK, STATUS_CALLBACK,
FEDERALIST_BUILDER_CALLBACK)
except UnexpectedExit as err:
# Combine the error's stdout and stderr into one string
err_string = format_output(err.result.stdout, err.result.stderr)
# replace any private values that might be in the error message
err_string = replace_private_values(err_string, private_values)
# log the original exception
LOGGER.info(f'Exception raised during build:')
LOGGER.info(err_string)
# replace the message with a custom one, if it exists
err_string = find_custom_error_message(err_string)
post_build_error(LOG_CALLBACK, STATUS_CALLBACK,
FEDERALIST_BUILDER_CALLBACK, err_string)
except Exception as err: # pylint: disable=W0703
# Getting here means something really weird has happened
# since all errors caught during tasks should be caught
# in the previous block as `UnexpectedExit` exceptions.
err_string = str(err)
err_string = replace_private_values(err_string, private_values)
# log the original exception
LOGGER.info('Unexpected exception raised during build:')
LOGGER.info(err_string)
err_message = ('Unexpected error. Please try again and '
'contact federalist-support if it persists.')
post_build_error(LOG_CALLBACK, STATUS_CALLBACK,
FEDERALIST_BUILDER_CALLBACK, err_message)
def get_factors(ivec): with Timeout(0.1) as timeout_ctx: return [fac(x) for x in ivec] return []
def build(aws_access_key_id,
aws_default_region,
aws_secret_access_key,
federalist_builder_callback,
status_callback,
baseurl,
branch,
bucket,
build_id,
config,
generator,
github_token,
owner,
repository,
site_prefix,
user_environment_variables=[]):
'''
Main task to run a full site build process.
All values needed for the build are loaded from
environment variables.
'''
# keep track of total time
start_time = datetime.now()
# Make the working directory if it doesn't exist
WORKING_DIR_PATH.mkdir(exist_ok=True)
logger = None
cache_control = os.getenv('CACHE_CONTROL', 'max-age=60')
database_url = os.environ['DATABASE_URL']
user_environment_variable_key = os.environ['USER_ENVIRONMENT_VARIABLE_KEY']
try:
post_build_processing(status_callback)
# throw a timeout exception after TIMEOUT_SECONDS
with Timeout(TIMEOUT_SECONDS, swallow_exc=False):
build_info = f'{owner}/{repository}@id:{build_id}'
decrypted_uevs = decrypt_uevs(user_environment_variable_key,
user_environment_variables)
priv_vals = [uev['value'] for uev in decrypted_uevs]
priv_vals.append(aws_access_key_id)
priv_vals.append(aws_secret_access_key)
if github_token:
priv_vals.append(github_token)
logattrs = {
'branch': branch,
'buildid': build_id,
'owner': owner,
'repository': repository,
}
init_logging(priv_vals, logattrs, database_url)
logger = get_logger('main')
def run_step(returncode, msg):
if returncode != 0:
raise StepException(msg)
logger.info(f'Running build for {owner}/{repository}/{branch}')
if generator not in GENERATORS:
raise ValueError(f'Invalid generator: {generator}')
##
# FETCH
#
run_step(
fetch_repo(owner, repository, branch, github_token),
'There was a problem fetching the repository, see the above logs for details.'
)
##
# BUILD
#
run_step(
setup_node(),
'There was a problem setting up Node, see the above logs for details.'
)
# Run the npm `federalist` task (if it is defined)
run_step(
run_federalist_script(branch, owner, repository, site_prefix,
baseurl, decrypted_uevs),
'There was a problem running the federalist script, see the above logs for details.'
)
# Run the appropriate build engine based on generator
if generator == 'jekyll':
run_step(
setup_ruby(),
'There was a problem setting up Ruby, see the above logs for details.'
)
run_step(
setup_bundler(),
'There was a problem setting up Bundler, see the above logs for details.'
)
run_step(
build_jekyll(branch, owner, repository, site_prefix,
baseurl, config, decrypted_uevs),
'There was a problem running Jekyll, see the above logs for details.'
)
elif generator == 'hugo':
# extra: --hugo-version (not yet used)
run_step(
download_hugo(),
'There was a problem downloading Hugo, see the above logs for details.'
)
run_step(
build_hugo(branch, owner, repository, site_prefix, baseurl,
decrypted_uevs),
'There was a problem running Hugo, see the above logs for details.'
)
elif generator == 'static':
# no build arguments are needed
build_static()
elif (generator == 'node.js' or generator == 'script only'):
logger.info('build already ran in \'npm run federalist\'')
else:
raise ValueError(f'Invalid generator: {generator}')
##
# PUBLISH
#
publish(baseurl, site_prefix, bucket, cache_control,
aws_default_region, aws_access_key_id,
aws_secret_access_key)
delta_string = delta_to_mins_secs(datetime.now() - start_time)
logger.info(f'Total build time: {delta_string}')
# Finished!
post_build_complete(status_callback, federalist_builder_callback)
sys.exit(0)
except StepException as err:
'''
Thrown when a step itself fails, usually because a command exited
with a non-zero return code
'''
logger.error(str(err))
post_build_error(status_callback, federalist_builder_callback,
str(err))
sys.exit(1)
except TimeoutException:
logger.warning(f'Build({build_info}) has timed out')
post_build_timeout(status_callback, federalist_builder_callback)
except Exception as err: # pylint: disable=W0703
# Getting here means something really weird has happened
# since all errors caught during tasks should be caught
# in the previous block as `UnexpectedExit` exceptions.
err_string = str(err)
# log the original exception
msg = f'Unexpected exception raised during build({build_info}): {err_string}'
if logger:
logger.warning(msg)
else:
print(msg)
err_message = (f'Unexpected build({build_info}) error. Please try '
'again and contact federalist-support if it persists.')
post_build_error(status_callback, federalist_builder_callback,
err_message)
def listen(language):
if language == 'th':
to_lang = 'en'
else:
to_lang = 'th'
# play("start.mp3")
# delete("start.mp3")
r = sr.Recognizer()
print("Say something!")
start = time.time()
with sr.Microphone() as source:
print(time.time() - start)
audio = r.listen(source)
print(time.time() - start)
try:
with Timeout(10, swallow_exc=False) as timeout_ctx:
print("listen finished sending to google")
if language == 'th':
text = r.recognize_google(audio, language="th-TH")
else:
text = r.recognize_google(audio, language="en-US")
# play("end.mp3")
# delete("end.mp3")
print(time.time() - start)
print("convert to text done")
print("Google Speech Recognition thinks you said " + text)
result = translate_for_me(language, to_lang, text)
tts = gTTS(result, lang=to_lang, slow=False)
tts.save('testtrans.mp3')
play('testtrans.mp3')
delete('testtrans.mp3')
return text + ',' + result
except TimeoutException:
print("too slow")
return "too slow"
except sr.UnknownValueError:
print("Google Speech Recognition could not understand audio")
return "error1239123"
except sr.RequestError as e:
print(
"Could not request results from Google Speech Recognition service; {0}"
.format(e))
except sr.WaitTimeoutError:
print("waiting too long")
except:
try:
with Timeout(10, swallow_exc=False) as timeout_ctx:
if language == 'th':
text = r.recognize_google(audio, language="th-TH")
else:
text = r.recognize_google(audio, language="en-US")
# play("end.mp3")
# delete("end.mp3")
print(time.time() - start)
print("convert to text done")
# tts = gTTS(text, lang='th', slow=False)
print("Google Speech Recognition thinks you said " + text)
result = translate_for_me(language, to_lang, text)
tts = gTTS(result, lang=to_lang, slow=False)
tts.save('testtrans.mp3')
play('testtrans.mp3')
time.sleep(5)
delete('testtrans.mp3')
return text + ',' + result
except TimeoutException:
print("too slow")
return "too slow"
def main(args, logger=None):
logger = logger or logging.getLogger(__name__)
logger.info(texts["welcome"])
if args.show_warnings is not True:
logger.info("Switch off printing of warnings from python packages")
warnings.filterwarnings("ignore")
if args.persistence_file is not None:
logger.info("Read classical Bayesian persistence file: %s" % args.persistence_file)
builtins.S2_clf = S2MSI.CloudMask(logger=logger,persistence_file=args.persistence_file)
if args.S2_MSI_granule_path is None:
try:
with Timeout(10.0, swallow_exc=False) as timeout_ctx:
args.S2_MSI_granule_path = glob(args.glob_search_pattern, recursive=True)
except TimeoutException:
logger.info("Search for GRANULES was stoped after 10s without success -> stop here ")
logger.info("The search pattern was: %s" % args.glob_search_pattern)
return
logger.info("No Input data given -> traverse local path and search for granules:")
for granule in args.S2_MSI_granule_path:
logger.info(granule)
if args.number_of_threads == 1:
logger.info("Start processing of %i jobs." % len(args.S2_MSI_granule_path))
for granule in args.S2_MSI_granule_path:
mask_image(args, granule)
elif args.number_of_threads > 1:
logger.info("Start processing of %i jobs using %i processes." % (
len(args.S2_MSI_granule_path), args.number_of_threads))
def _init_():
globals()["update"] = lambda: None # tkinter is not able to digest multiple threads
sys.stdout = S2MSI.Tools.StdoutToList()
sys.stderr = S2MSI.Tools.StdoutToList()
logger.info("Start Processing in parallel running threads. Output of individual jobs is shown when done. ")
if args.use_thread_pool is True:
logger.info("!!!!!!!!!!!! Use ThreadPool !!!!!!!!!")
Pool = ThreadPool
pool = Pool(initializer=_init_, processes=args.number_of_threads)
tasks = [(args, granule, logger) for granule in args.S2_MSI_granule_path]
jobs = [pool.apply_async(mask_image, task) for task in tasks]
while len(jobs) > 0:
sleep(5.0)
readies = [job.ready() for job in jobs]
for job, ready in zip(jobs, readies):
if ready is True:
lne_out, lne_err = job.get()
logger.info("### print jop output ###")
for line in lne_out:
logger.info(line)
if len(lne_err) > 1:
logger.info("### errors occurred -> print stderr: ###")
for line in lne_out:
logger.info(line)
jobs[:] = [job for job, ready in zip(jobs, readies) if ready is False]
logger.info("#> Open Jobs: %i <#" % len(jobs))
pool.close()
else:
raise ValueError("The number of threads should be larger or equal to zero and not:%i" % args.number_of_threads)
logger.info(texts["END"])
