def test_exit(self):
import logging
logger = logging.getLogger("golem.core")
mp1, mp2 = MockProcess(), MockProcess()
p1 = Process(target=mp1.run)
p2 = Process(target=mp2.run)
p1.start()
p2.start()
pm = ProcessMonitor(p1, p2)
def callback():
logger.warning("Shutting down...")
pm.add_callbacks(callback)
pm.start()
pm.exit()
wait_for_processes(10, p1, p2)
self.assertFalse(pm.is_process_alive(p1))
self.assertFalse(pm.is_process_alive(p2))
def test_add_remove_callbacks(self):
pm = ProcessMonitor()
pm.add_callbacks(pm.exit)
pm.remove_callbacks(pm.exit)
assert not pm._callbacks
def test_monitor(self):
mp = MockProcess()
p1 = Process(target=run_exit)
p2 = Process(target=mp.run)
p1.start()
p2.start()
pm = ProcessMonitor(p1, p2)
pm.add_callbacks(pm.kill_processes, pm.exit)
pm.start()
wait_for_processes(10, p1, p2)
self.assertFalse(pm.is_process_alive(p1))
self.assertFalse(pm.is_process_alive(p2))
def test_monitor_2(self):
mp1, mp2 = MockProcess(), MockProcess(timeout=0)
p1 = Process(target=mp1.run)
p2 = Process(target=mp2.run)
p1.start()
p2.start()
pm = ProcessMonitor(p1, p2)
pm.add_callbacks(pm.kill_processes, pm.exit)
pm.start()
wait_for_processes(10, p1, p2)
self.assertFalse(pm.is_process_alive(p1), "process not killed")
self.assertFalse(pm.is_process_alive(p2), "process not finished")
def test_monitor_2(self):
mp1, mp2 = MockProcess(), MockProcess(timeout=0)
p1 = Process(target=mp1.run)
p2 = Process(target=mp2.run)
p1.start()
p2.start()
pm = ProcessMonitor(p1, p2)
pm.add_callbacks(pm.kill_processes, pm.exit)
pm.start()
wait_for_processes(10, p1, p2)
if pm.is_process_alive(p1) or pm.is_process_alive(p2):
pm.exit()
self.fail("Processes not killed after timeout")
def session_ready(*_):
global process_monitor
logger.info('Router session ready. Starting client...')
try:
client.configure_rpc(session)
logger.debug('client.start()')
client.start()
logger.debug('after client.start()')
except SystemExit:
raise
except Exception as exc:
logger.exception("Client process error: {}"
.format(exc))
logger.info('Starting GUI process...')
gui_process = start_gui(router.address)
process_monitor = ProcessMonitor(gui_process)
process_monitor.add_callbacks(stop_reactor)
logger.info('Starting process monitor...')
process_monitor.start()
class HyperdriveDaemonManager(object):
_executable = 'hyperg'
def __init__(self, datadir, **hyperdrive_config):
super(HyperdriveDaemonManager, self).__init__()
self._config = hyperdrive_config
# monitor and restart if process dies
self._monitor = ProcessMonitor()
self._monitor.add_callbacks(self._start)
self._dir = os.path.join(datadir, self._executable)
self._command = [self._executable, '--db', self._dir]
def addresses(self):
try:
return HyperdriveClient(**self._config).addresses()
except ConnectionError:
return dict()
def ports(self, addresses=None):
if addresses is None:
addresses = self.addresses() or dict()
return set(value['port'] for key, value in addresses.iteritems()
if value and value.get('port'))
def start(self):
atexit.register(self.stop)
signal.signal(signal.SIGABRT, self.stop)
signal.signal(signal.SIGTERM, self.stop)
signal.signal(signal.SIGINT, self.stop)
self._monitor.start()
return self._start()
def stop(self, *_):
self._monitor.exit()
def _start(self, *_):
# do not supervise already running processes
addresses = self.addresses()
if addresses:
return self.ports(addresses)
try:
if not os.path.exists(self._dir):
os.makedirs(self._dir)
process = subprocess.Popen(self._command)
except OSError:
logger.critical(
"Can't run hyperdrive executable %r. "
"Make sure path is correct and check "
"if it starts correctly.", ' '.join(self._command))
sys.exit(1)
if process.poll() is None:
self._monitor.add_child_processes(process)
else:
raise RuntimeError("Cannot start {}".format(self._executable))
return self.ports()
class HyperdriveDaemonManager(object):
_executable = 'hyperg'
_min_version = semantic_version.Version(GOLEM_HYPERDRIVE_VERSION)
def __init__(self, datadir, **hyperdrive_config):
super(HyperdriveDaemonManager, self).__init__()
self._addresses = None
self._config = hyperdrive_config
# monitor and restart if process dies
self._monitor = ProcessMonitor()
self._monitor.add_callbacks(self._start)
self._dir = os.path.join(datadir, self._executable)
self._client = HyperdriveClient(**self._config)
logsdir = os.path.join(datadir, "logs")
if not os.path.exists(logsdir):
logger.warning("Creating HyperG logsdir: %s", logsdir)
os.makedirs(logsdir)
self._command = [
self._executable,
'--db', self._dir,
'--logfile', os.path.join(logsdir, "hyperg.log"),
]
def addresses(self):
try:
return self._get_addresses()
except requests.ConnectionError:
logger.warning('Cannot connect to Hyperdrive daemon')
return dict()
def version(self) -> Optional[semantic_version.Version]:
try:
output = self._client.id()['version']
except requests.ConnectionError: # not running
output = None
except KeyError: # API version too low - no version in response
# FIXME: return None after upgrading to 0.2.5
output = None
if not output:
try:
command = [self._executable, '--version']
output = subprocess.check_output(command)
output = output.decode('utf-8')
except (OSError, UnicodeDecodeError):
self._critical_error()
return semantic_version.Version(output.strip())
def _get_addresses(self):
if not self._addresses:
self._addresses = self._client.addresses()
return self._addresses
def public_addresses(self, ip, addresses=None):
if addresses is None:
addresses = copy.deepcopy(self.addresses())
for protocol, entry in addresses.items():
addresses[protocol] = (ip, entry[1])
return addresses
def ports(self, addresses=None):
if addresses is None:
addresses = self.addresses()
return set(value[1] for key, value in addresses.items())
def start(self):
self._addresses = None
self._monitor.start()
return self._start()
def stop(self, *_):
self._monitor.exit()
@report_calls(Component.hyperdrive, 'instance.connect')
def _start(self, *_):
self._check_version()
# do not supervise already running processes
addresses = self.addresses()
if addresses:
return
process = self._create_sub()
if process.poll() is None:
self._monitor.add_child_processes(process)
self._wait()
else:
raise RuntimeError("Cannot start {}".format(self._executable))
@report_calls(Component.hyperdrive, 'instance.version')
def _check_version(self):
version = self.version()
if version < self._min_version:
raise RuntimeError('HyperG version {} is required'
.format(self._min_version))
@report_calls(Component.hyperdrive, 'instance.check')
def _create_sub(self):
try:
os.makedirs(self._dir, exist_ok=True)
return subprocess.Popen(self._command, stdin=DEVNULL,
stdout=None, stderr=None,
startupinfo=SUBPROCESS_STARTUP_INFO)
except OSError:
return self._critical_error()
def _wait(self, timeout: int = 10):
deadline = time.time() + timeout
while time.time() < deadline:
addresses = self.addresses()
if addresses:
return
time.sleep(1.)
self._critical_error()
def _critical_error(self):
logger.critical("Can't run hyperdrive executable %r. "
"Make sure path is correct and check "
"if it starts correctly.",
' '.join(self._command))
sys.exit(1)