def child_proc_shutdown(children): run_exp_proc = psutil.Process() alive = run_exp_proc.children(recursive=True) for proc in alive: if any([ "multiprocessing.semaphore_tracker" in cmd for cmd in proc.cmdline() ]): alive.remove(proc) for c in children: c.close() max_retries = 5 for _ in range(max_retries): _, alive = psutil.wait_procs(alive, 1.0) if not alive: break if alive: error_msg = "" for child in alive: error_msg += "{}\n".format( str( child.as_dict( attrs=["ppid", "pid", "name", "status", "cmdline"]))) error_msg = ("The following processes didn't die after the shutdown " "of run_experiment:\n") + error_msg error_msg += ("This is a sign of an unclean shutdown. Please reopen " "the following issue\nwith a detailed description " "of how the error was produced:\n") error_msg += ("https://github.com/rlworkgroup/garage/issues/120") print(colorize(error_msg, "yellow"))
def child_proc_shutdown(children): run_exp_proc = psutil.Process() alive = run_exp_proc.children(recursive=True) for proc in alive: if any([ 'multiprocessing.semaphore_tracker' in cmd for cmd in proc.cmdline() ]): alive.remove(proc) for c in children: c.close() max_retries = 5 for _ in range(max_retries): _, alive = psutil.wait_procs(alive, 1.0) if not alive: break if alive: error_msg = '' for child in alive: error_msg += '{}\n'.format( str( child.as_dict( attrs=['ppid', 'pid', 'name', 'status', 'cmdline']))) error_msg = ("The following processes didn't die after the shutdown " 'of run_experiment:\n') + error_msg error_msg += ('This is a sign of an unclean shutdown. Please reopen ' 'the following issue\nwith a detailed description ' 'of how the error was produced:\n') error_msg += ('https://github.com/rlworkgroup/garage/issues/120') print(colorize(error_msg, 'yellow'))
def interrupt_experiment(experiment_script, lifecycle_stage): """Interrupt the experiment and verify no children processes remain.""" args = ['python', experiment_script] # The pre-executed function setpgrp allows to create a process group # so signals are propagated to all the process in the group. proc = subprocess.Popen(args, preexec_fn=os.setpgrp) launcher_proc = psutil.Process(proc.pid) # This socket connects with the client in the algorithm, so we're # notified of the different stages in the experiment lifecycle. address = ('localhost', 6000) listener = Listener(address) conn = listener.accept() while True: msg = conn.recv() if msg == lifecycle_stage: # Notice that we're asking for the children of the launcher, not # the children of this test script, since there could be other # children processes attached to the process running this test # that are not part of the launcher. children = launcher_proc.children(recursive=True) # Remove the semaphore tracker from the list of children, since # we cannot stop its execution. for child in children: if any([ 'multiprocessing.semaphore_tracker' in cmd for cmd in child.cmdline() ]): children.remove(child) # We append the launcher to the list of children so later we can # check it has died. children.append(launcher_proc) pgrp = os.getpgid(proc.pid) os.killpg(pgrp, signal.SIGINT) conn.close() break listener.close() # Once the signal has been sent, all children should die _, alive = psutil.wait_procs(children, timeout=6) # If any, notify the zombie and sleeping processes and fail the test clean_exit = True error_msg = '' for child in alive: error_msg += ( str(child.as_dict(attrs=['pid', 'name', 'status', 'cmdline'])) + '\n') clean_exit = False error_msg = ("These processes didn't die during %s:\n" % (lifecycle_stage) + error_msg) for child in alive: os.kill(child.pid, signal.SIGINT) assert clean_exit, colorize(error_msg, 'red')
def _warn(self, msg): """Warns the user using warnings.warn. The stacklevel parameter needs to be 3 to ensure the call to logger.log is the one printed. """ if not self._disable_warnings and msg not in self._warned_once: warnings.warn(colorize(msg, 'yellow'), LoggerWarning, stacklevel=3) self._warned_once.add(msg) return msg
def log(s, with_prefix=True, with_timestamp=True, color=None): out = s if with_prefix: out = _prefix_str + out # out_basic holds output with a simpler timestamp for stdout out_basic = out if with_timestamp: now = datetime.datetime.now(dateutil.tz.tzlocal()) timestamp_basic = now.strftime('%Y-%m-%d %H:%M:%S') timestamp = now.strftime('%Y-%m-%d %H:%M:%S.%f %Z') out_basic = "%s | %s" % (timestamp_basic, out_basic) out = "%s | %s" % (timestamp, out) if color is not None: out = colorize(out, color) out_basic = colorize(out_basic, color) if not _log_tabular_only: # Also log to stdout print(out_basic) for fd in list(_text_fds.values()): fd.write(out + '\n') fd.flush() sys.stdout.flush()
def set_seed(seed): seed %= 4294967294 global seed_ seed_ = seed import lasagne random.seed(seed) np.random.seed(seed) lasagne.random.set_rng(np.random.RandomState(seed)) try: import tensorflow as tf tf.set_random_seed(seed) except Exception as e: print(e) print((colorize('using seed %s' % (str(seed)), 'green')))
def _new_from_args(cls, parsed_args, *args, **params): silent = params.pop("_silent", False) args_info = _get_info(cls) prefix_ = _get_prefix(cls) # params = dict() for arg_name, arg_info in args_info.items(): prefixed_arg_name = prefix_ + arg_name if hasattr(parsed_args, prefixed_arg_name): val = getattr(parsed_args, prefixed_arg_name) if val is not None: if arg_info['mapper']: params[arg_name] = arg_info['mapper'](val) else: params[arg_name] = val if not silent: print( colorize( "using argument %s with value %s" % (arg_name, val), "yellow")) return cls(*args, **params)