def child_proc_shutdown(children):
run_exp_proc = psutil.Process()
alive = run_exp_proc.children(recursive=True)
for proc in alive:
if any([
"multiprocessing.semaphore_tracker" in cmd
for cmd in proc.cmdline()
]):
alive.remove(proc)
for c in children:
c.close()
max_retries = 5
for _ in range(max_retries):
_, alive = psutil.wait_procs(alive, 1.0)
if not alive:
break
if alive:
error_msg = ""
for child in alive:
error_msg += "{}\n".format(
str(
child.as_dict(
attrs=["ppid", "pid", "name", "status", "cmdline"])))
error_msg = ("The following processes didn't die after the shutdown "
"of run_experiment:\n") + error_msg
error_msg += ("This is a sign of an unclean shutdown. Please reopen "
"the following issue\nwith a detailed description "
"of how the error was produced:\n")
error_msg += ("https://github.com/rlworkgroup/garage/issues/120")
print(colorize(error_msg, "yellow"))
def child_proc_shutdown(children):
run_exp_proc = psutil.Process()
alive = run_exp_proc.children(recursive=True)
for proc in alive:
if any([
'multiprocessing.semaphore_tracker' in cmd
for cmd in proc.cmdline()
]):
alive.remove(proc)
for c in children:
c.close()
max_retries = 5
for _ in range(max_retries):
_, alive = psutil.wait_procs(alive, 1.0)
if not alive:
break
if alive:
error_msg = ''
for child in alive:
error_msg += '{}\n'.format(
str(
child.as_dict(
attrs=['ppid', 'pid', 'name', 'status', 'cmdline'])))
error_msg = ("The following processes didn't die after the shutdown "
'of run_experiment:\n') + error_msg
error_msg += ('This is a sign of an unclean shutdown. Please reopen '
'the following issue\nwith a detailed description '
'of how the error was produced:\n')
error_msg += ('https://github.com/rlworkgroup/garage/issues/120')
print(colorize(error_msg, 'yellow'))
def interrupt_experiment(experiment_script, lifecycle_stage):
"""Interrupt the experiment and verify no children processes remain."""
args = ['python', experiment_script]
# The pre-executed function setpgrp allows to create a process group
# so signals are propagated to all the process in the group.
proc = subprocess.Popen(args, preexec_fn=os.setpgrp)
launcher_proc = psutil.Process(proc.pid)
# This socket connects with the client in the algorithm, so we're
# notified of the different stages in the experiment lifecycle.
address = ('localhost', 6000)
listener = Listener(address)
conn = listener.accept()
while True:
msg = conn.recv()
if msg == lifecycle_stage:
# Notice that we're asking for the children of the launcher, not
# the children of this test script, since there could be other
# children processes attached to the process running this test
# that are not part of the launcher.
children = launcher_proc.children(recursive=True)
# Remove the semaphore tracker from the list of children, since
# we cannot stop its execution.
for child in children:
if any([
'multiprocessing.semaphore_tracker' in cmd
for cmd in child.cmdline()
]):
children.remove(child)
# We append the launcher to the list of children so later we can
# check it has died.
children.append(launcher_proc)
pgrp = os.getpgid(proc.pid)
os.killpg(pgrp, signal.SIGINT)
conn.close()
break
listener.close()
# Once the signal has been sent, all children should die
_, alive = psutil.wait_procs(children, timeout=6)
# If any, notify the zombie and sleeping processes and fail the test
clean_exit = True
error_msg = ''
for child in alive:
error_msg += (
str(child.as_dict(attrs=['pid', 'name', 'status', 'cmdline'])) +
'\n')
clean_exit = False
error_msg = ("These processes didn't die during %s:\n" %
(lifecycle_stage) + error_msg)
for child in alive:
os.kill(child.pid, signal.SIGINT)
assert clean_exit, colorize(error_msg, 'red')
def _warn(self, msg):
"""Warns the user using warnings.warn.
The stacklevel parameter needs to be 3 to ensure the call to logger.log
is the one printed.
"""
if not self._disable_warnings and msg not in self._warned_once:
warnings.warn(colorize(msg, 'yellow'), LoggerWarning, stacklevel=3)
self._warned_once.add(msg)
return msg
def log(s, with_prefix=True, with_timestamp=True, color=None):
out = s
if with_prefix:
out = _prefix_str + out
# out_basic holds output with a simpler timestamp for stdout
out_basic = out
if with_timestamp:
now = datetime.datetime.now(dateutil.tz.tzlocal())
timestamp_basic = now.strftime('%Y-%m-%d %H:%M:%S')
timestamp = now.strftime('%Y-%m-%d %H:%M:%S.%f %Z')
out_basic = "%s | %s" % (timestamp_basic, out_basic)
out = "%s | %s" % (timestamp, out)
if color is not None:
out = colorize(out, color)
out_basic = colorize(out_basic, color)
if not _log_tabular_only:
# Also log to stdout
print(out_basic)
for fd in list(_text_fds.values()):
fd.write(out + '\n')
fd.flush()
sys.stdout.flush()
def set_seed(seed):
seed %= 4294967294
global seed_
seed_ = seed
import lasagne
random.seed(seed)
np.random.seed(seed)
lasagne.random.set_rng(np.random.RandomState(seed))
try:
import tensorflow as tf
tf.set_random_seed(seed)
except Exception as e:
print(e)
print((colorize('using seed %s' % (str(seed)), 'green')))
def _new_from_args(cls, parsed_args, *args, **params):
silent = params.pop("_silent", False)
args_info = _get_info(cls)
prefix_ = _get_prefix(cls)
# params = dict()
for arg_name, arg_info in args_info.items():
prefixed_arg_name = prefix_ + arg_name
if hasattr(parsed_args, prefixed_arg_name):
val = getattr(parsed_args, prefixed_arg_name)
if val is not None:
if arg_info['mapper']:
params[arg_name] = arg_info['mapper'](val)
else:
params[arg_name] = val
if not silent:
print(
colorize(
"using argument %s with value %s" %
(arg_name, val), "yellow"))
return cls(*args, **params)
