def test_kill_process_popen(self):
process = subprocess.Popen(
['python', '-c', 'import time; time.sleep(1)'])
assert ProcessMonitor.is_process_alive(process)
ProcessMonitor.kill_process(process)
assert not ProcessMonitor.is_process_alive(process)
def test_lifecycle_none(self):
process = None
assert not ProcessMonitor.is_process_alive(process)
assert not ProcessMonitor.is_supported(process)
assert not ProcessMonitor._pid(process)
assert ProcessMonitor.exit_code(process) is None
def test_add_child_process(self):
mp1, mp2 = MockProcess(), MockProcess(timeout=1)
p1 = Process(target=mp1.run)
p2 = Process(target=mp2.run)
pm = ProcessMonitor(p1)
pm.add_child_processes(p2)
assert len(pm._child_processes) == 2
def test_exit(self):
import logging
logger = logging.getLogger("golem.core")
mp1, mp2 = MockProcess(), MockProcess()
p1 = Process(target=mp1.run)
p2 = Process(target=mp2.run)
p1.start()
p2.start()
pm = ProcessMonitor(p1, p2)
def callback():
logger.warning("Shutting down...")
pm.add_callbacks(callback)
pm.start()
pm.exit()
wait_for_processes(10, p1, p2)
self.assertFalse(pm.is_process_alive(p1))
self.assertFalse(pm.is_process_alive(p2))
def test_lifecycle_popen(self):
process = subprocess.Popen(
['python', '-c', 'import time; time.sleep(1)'])
assert ProcessMonitor.is_process_alive(process)
assert ProcessMonitor._pid(process)
assert ProcessMonitor.is_supported(process)
process.communicate()
assert not ProcessMonitor.is_process_alive(process)
assert ProcessMonitor.exit_code(process) is not None
def __init__(self, datadir, **hyperdrive_config):
super(HyperdriveDaemonManager, self).__init__()
self._config = hyperdrive_config
# monitor and restart if process dies
self._monitor = ProcessMonitor()
self._monitor.add_callbacks(self._start)
self._dir = os.path.join(datadir, self._executable)
self._command = [self._executable, '--db', self._dir]
def test_lifecycle_multiprocessing(self):
process = Process(target=sleep_1sec)
assert not ProcessMonitor.is_process_alive(process)
assert ProcessMonitor.is_supported(process)
process.start()
assert ProcessMonitor.is_process_alive(process)
process.join()
assert not ProcessMonitor.is_process_alive(process)
assert ProcessMonitor.exit_code(process) is not None
def test_monitor_2(self):
mp1, mp2 = MockProcess(), MockProcess(timeout=0)
p1 = Process(target=mp1.run)
p2 = Process(target=mp2.run)
p1.start()
p2.start()
pm = ProcessMonitor(p1, p2)
pm.add_callbacks(pm.kill_processes, pm.exit)
pm.start()
wait_for_processes(10, p1, p2)
if pm.is_process_alive(p1) or pm.is_process_alive(p2):
pm.exit()
self.fail("Processes not killed after timeout")
def test_add_remove_callbacks(self):
pm = ProcessMonitor()
pm.add_callbacks(pm.exit)
pm.remove_callbacks(pm.exit)
assert not pm._callbacks
def __init__(self, datadir, **hyperdrive_config):
super(HyperdriveDaemonManager, self).__init__()
self._addresses = None
self._config = hyperdrive_config
# monitor and restart if process dies
self._monitor = ProcessMonitor()
self._monitor.add_callbacks(self._start)
self._dir = os.path.join(datadir, self._executable)
self._client = HyperdriveClient(**self._config)
logsdir = os.path.join(datadir, "logs")
if not os.path.exists(logsdir):
logger.warning("Creating HyperG logsdir: %s", logsdir)
os.makedirs(logsdir)
self._command = [
self._executable,
'--db', self._dir,
'--logfile', os.path.join(logsdir, "hyperg.log"),
]
def wait_for_processes(timeout=10, *processes):
started = time.time()
timeout = max(timeout, 5)
while time.time() - started < timeout:
all_stopped = True
for process in processes:
if ProcessMonitor.is_process_alive(process):
all_stopped = False
break
if all_stopped:
return
time.sleep(0.5)
def test_kill_process_multiprocessing(self):
process = Process(target=sleep_1sec)
process.start()
assert ProcessMonitor.is_process_alive(process)
ProcessMonitor.kill_process(process)
assert not ProcessMonitor.is_process_alive(process)
process = Process(target=sleep_1sec)
ProcessMonitor.kill_process(process)
def test_monitor(self):
mp = MockProcess()
p1 = Process(target=run_exit)
p2 = Process(target=mp.run)
p1.start()
p2.start()
pm = ProcessMonitor(p1, p2)
pm.add_callbacks(pm.kill_processes, pm.exit)
pm.start()
wait_for_processes(10, p1, p2)
self.assertFalse(pm.is_process_alive(p1))
self.assertFalse(pm.is_process_alive(p2))
def test_monitor_2(self):
mp1, mp2 = MockProcess(), MockProcess(timeout=0)
p1 = Process(target=mp1.run)
p2 = Process(target=mp2.run)
p1.start()
p2.start()
pm = ProcessMonitor(p1, p2)
pm.add_callbacks(pm.kill_processes, pm.exit)
pm.start()
wait_for_processes(10, p1, p2)
self.assertFalse(pm.is_process_alive(p1), "process not killed")
self.assertFalse(pm.is_process_alive(p2), "process not finished")
def session_ready(*_):
global process_monitor
logger.info('Router session ready. Starting client...')
try:
client.configure_rpc(session)
logger.debug('client.start()')
client.start()
logger.debug('after client.start()')
except SystemExit:
raise
except Exception as exc:
logger.exception("Client process error: {}"
.format(exc))
logger.info('Starting GUI process...')
gui_process = start_gui(router.address)
process_monitor = ProcessMonitor(gui_process)
process_monitor.add_callbacks(stop_reactor)
logger.info('Starting process monitor...')
process_monitor.start()
class HyperdriveDaemonManager(object):
_executable = 'hyperg'
_min_version = semantic_version.Version(GOLEM_HYPERDRIVE_VERSION)
def __init__(self, datadir, **hyperdrive_config):
super(HyperdriveDaemonManager, self).__init__()
self._addresses = None
self._config = hyperdrive_config
# monitor and restart if process dies
self._monitor = ProcessMonitor()
self._monitor.add_callbacks(self._start)
self._dir = os.path.join(datadir, self._executable)
self._client = HyperdriveClient(**self._config)
logsdir = os.path.join(datadir, "logs")
if not os.path.exists(logsdir):
logger.warning("Creating HyperG logsdir: %s", logsdir)
os.makedirs(logsdir)
self._command = [
self._executable,
'--db', self._dir,
'--logfile', os.path.join(logsdir, "hyperg.log"),
]
def addresses(self):
try:
return self._get_addresses()
except requests.ConnectionError:
logger.warning('Cannot connect to Hyperdrive daemon')
return dict()
def version(self) -> Optional[semantic_version.Version]:
try:
output = self._client.id()['version']
except requests.ConnectionError: # not running
output = None
except KeyError: # API version too low - no version in response
# FIXME: return None after upgrading to 0.2.5
output = None
if not output:
try:
command = [self._executable, '--version']
output = subprocess.check_output(command)
output = output.decode('utf-8')
except (OSError, UnicodeDecodeError):
self._critical_error()
return semantic_version.Version(output.strip())
def _get_addresses(self):
if not self._addresses:
self._addresses = self._client.addresses()
return self._addresses
def public_addresses(self, ip, addresses=None):
if addresses is None:
addresses = copy.deepcopy(self.addresses())
for protocol, entry in addresses.items():
addresses[protocol] = (ip, entry[1])
return addresses
def ports(self, addresses=None):
if addresses is None:
addresses = self.addresses()
return set(value[1] for key, value in addresses.items())
def start(self):
self._addresses = None
self._monitor.start()
return self._start()
def stop(self, *_):
self._monitor.exit()
@report_calls(Component.hyperdrive, 'instance.connect')
def _start(self, *_):
self._check_version()
# do not supervise already running processes
addresses = self.addresses()
if addresses:
return
process = self._create_sub()
if process.poll() is None:
self._monitor.add_child_processes(process)
self._wait()
else:
raise RuntimeError("Cannot start {}".format(self._executable))
@report_calls(Component.hyperdrive, 'instance.version')
def _check_version(self):
version = self.version()
if version < self._min_version:
raise RuntimeError('HyperG version {} is required'
.format(self._min_version))
@report_calls(Component.hyperdrive, 'instance.check')
def _create_sub(self):
try:
os.makedirs(self._dir, exist_ok=True)
return subprocess.Popen(self._command, stdin=DEVNULL,
stdout=None, stderr=None,
startupinfo=SUBPROCESS_STARTUP_INFO)
except OSError:
return self._critical_error()
def _wait(self, timeout: int = 10):
deadline = time.time() + timeout
while time.time() < deadline:
addresses = self.addresses()
if addresses:
return
time.sleep(1.)
self._critical_error()
def _critical_error(self):
logger.critical("Can't run hyperdrive executable %r. "
"Make sure path is correct and check "
"if it starts correctly.",
' '.join(self._command))
sys.exit(1)
def test_exit_code(self):
process_psutil = psutil.Popen.__new__(psutil.Popen, None)
process_subprocess = subprocess.Popen.__new__(subprocess.Popen, None)
process_multiprocessing = Process.__new__(Process, None)
process_psutil.poll = Mock()
process_subprocess.poll = Mock()
process_multiprocessing._popen = Mock()
process_multiprocessing._parent_pid = os.getpid()
process_multiprocessing._name = "test"
process_multiprocessing._daemonic = False
process_psutil.returncode = None
process_subprocess.returncode = None
assert ProcessMonitor.is_process_alive(process_psutil)
assert ProcessMonitor.is_process_alive(process_subprocess)
with patch('multiprocessing.Process.is_alive',
side_effect=lambda: False):
assert not ProcessMonitor.is_process_alive(process_multiprocessing)
assert ProcessMonitor.exit_code(None) is None
assert ProcessMonitor.exit_code(process_psutil) is None
assert ProcessMonitor.exit_code(process_subprocess) is None
with patch('multiprocessing.Process.exitcode') as exitcode:
exitcode.__get__ = Mock(return_value=None)
assert ProcessMonitor.exit_code(process_multiprocessing) is None
process_psutil.poll = Mock()
process_psutil.returncode = 0
process_subprocess.poll = Mock()
process_subprocess.returncode = 0
assert not ProcessMonitor.is_process_alive(None)
assert not ProcessMonitor.is_process_alive(process_psutil)
assert not ProcessMonitor.is_process_alive(process_subprocess)
with patch('multiprocessing.Process.exitcode') as exitcode:
exitcode.__get__ = Mock(return_value=0)
assert not ProcessMonitor.is_process_alive(process_multiprocessing)
assert ProcessMonitor.exit_code(process_psutil) == 0
assert ProcessMonitor.exit_code(process_subprocess) == 0
with patch('multiprocessing.Process.exitcode') as exitcode:
exitcode.__get__ = Mock(return_value=0)
assert ProcessMonitor.exit_code(process_multiprocessing) == 0
class HyperdriveDaemonManager(object):
_executable = 'hyperg'
def __init__(self, datadir, **hyperdrive_config):
super(HyperdriveDaemonManager, self).__init__()
self._config = hyperdrive_config
# monitor and restart if process dies
self._monitor = ProcessMonitor()
self._monitor.add_callbacks(self._start)
self._dir = os.path.join(datadir, self._executable)
self._command = [self._executable, '--db', self._dir]
def addresses(self):
try:
return HyperdriveClient(**self._config).addresses()
except ConnectionError:
return dict()
def ports(self, addresses=None):
if addresses is None:
addresses = self.addresses() or dict()
return set(value['port'] for key, value in addresses.iteritems()
if value and value.get('port'))
def start(self):
atexit.register(self.stop)
signal.signal(signal.SIGABRT, self.stop)
signal.signal(signal.SIGTERM, self.stop)
signal.signal(signal.SIGINT, self.stop)
self._monitor.start()
return self._start()
def stop(self, *_):
self._monitor.exit()
def _start(self, *_):
# do not supervise already running processes
addresses = self.addresses()
if addresses:
return self.ports(addresses)
try:
if not os.path.exists(self._dir):
os.makedirs(self._dir)
process = subprocess.Popen(self._command)
except OSError:
logger.critical(
"Can't run hyperdrive executable %r. "
"Make sure path is correct and check "
"if it starts correctly.", ' '.join(self._command))
sys.exit(1)
if process.poll() is None:
self._monitor.add_child_processes(process)
else:
raise RuntimeError("Cannot start {}".format(self._executable))
return self.ports()