def test_request(self, post, json_loads, _):
client = HyperdriveClient()
resp = mock.Mock()
post.return_value = resp
client._request(key="value")
assert json_loads.called
def test_pin_rm(self):
client = HyperdriveClient()
multihash = str(uuid.uuid4())
with mock.patch.object(HyperdriveClient, '_request',
return_value=self.response):
assert client.pin_rm(multihash) == self.response['hash']
def test_get(self, _):
client = HyperdriveClient()
content_hash = str(uuid.uuid4())
filepath = str(uuid.uuid4())
with self.assertRaises(KeyError):
client.get(content_hash)
assert client.get(content_hash, client_options=None, filepath=filepath)\
== [(filepath, content_hash, response['files'])]
def test_request_http_error(self, json_loads, post):
client = HyperdriveClient()
resp = mock.Mock()
post.return_value = resp
exception = HTTPError()
resp.raise_for_status.side_effect = exception
with self.assertRaises(HTTPError) as exc:
client._request(key="value")
assert exc.exception is not exception
assert not json_loads.called
def test_get_file(self):
client = HyperdriveClient()
multihash = str(uuid.uuid4())
filepath = str(uuid.uuid4())
with mock.patch.object(HyperdriveClient, '_request',
return_value=self.response):
with self.assertRaises(KeyError):
client.get_file(multihash)
assert client.get_file(multihash,
client_options=None,
filepath=filepath) == \
[(filepath, multihash, self.response['files'])]
def __init__(self, datadir, **hyperdrive_config):
super(HyperdriveDaemonManager, self).__init__()
self._addresses = None
self._config = hyperdrive_config
# monitor and restart if process dies
self._monitor = ProcessMonitor()
self._monitor.add_callbacks(self._start)
self._dir = os.path.join(datadir, self._executable)
self._client = HyperdriveClient(**self._config)
logsdir = os.path.join(datadir, "logs")
if not os.path.exists(logsdir):
logger.warning("Creating HyperG logsdir: %s", logsdir)
os.makedirs(logsdir)
self._command = [
self._executable,
'--db', self._dir,
'--logfile', os.path.join(logsdir, "hyperg.log"),
]
def test_build_options(self):
node_id = str(uuid.uuid4())
options = HyperdriveClient.build_options(node_id)
assert options.client_id == HyperdriveClient.CLIENT_ID
assert options.version == HyperdriveClient.VERSION
assert not options.options
def addresses(self):
try:
return HyperdriveClient(**self._config).addresses()
except ConnectionError:
return dict()
class HyperdriveDaemonManager(object):
_executable = 'hyperg'
_min_version = semantic_version.Version(GOLEM_HYPERDRIVE_VERSION)
def __init__(self, datadir, **hyperdrive_config):
super(HyperdriveDaemonManager, self).__init__()
self._addresses = None
self._config = hyperdrive_config
# monitor and restart if process dies
self._monitor = ProcessMonitor()
self._monitor.add_callbacks(self._start)
self._dir = os.path.join(datadir, self._executable)
self._client = HyperdriveClient(**self._config)
logsdir = os.path.join(datadir, "logs")
if not os.path.exists(logsdir):
logger.warning("Creating HyperG logsdir: %s", logsdir)
os.makedirs(logsdir)
self._command = [
self._executable,
'--db', self._dir,
'--logfile', os.path.join(logsdir, "hyperg.log"),
]
def addresses(self):
try:
return self._get_addresses()
except requests.ConnectionError:
logger.warning('Cannot connect to Hyperdrive daemon')
return dict()
def version(self) -> Optional[semantic_version.Version]:
try:
output = self._client.id()['version']
except requests.ConnectionError: # not running
output = None
except KeyError: # API version too low - no version in response
# FIXME: return None after upgrading to 0.2.5
output = None
if not output:
try:
command = [self._executable, '--version']
output = subprocess.check_output(command)
output = output.decode('utf-8')
except (OSError, UnicodeDecodeError):
self._critical_error()
return semantic_version.Version(output.strip())
def _get_addresses(self):
if not self._addresses:
self._addresses = self._client.addresses()
return self._addresses
def public_addresses(self, ip, addresses=None):
if addresses is None:
addresses = copy.deepcopy(self.addresses())
for protocol, entry in addresses.items():
addresses[protocol] = (ip, entry[1])
return addresses
def ports(self, addresses=None):
if addresses is None:
addresses = self.addresses()
return set(value[1] for key, value in addresses.items())
def start(self):
self._addresses = None
self._monitor.start()
return self._start()
def stop(self, *_):
self._monitor.exit()
@report_calls(Component.hyperdrive, 'instance.connect')
def _start(self, *_):
self._check_version()
# do not supervise already running processes
addresses = self.addresses()
if addresses:
return
process = self._create_sub()
if process.poll() is None:
self._monitor.add_child_processes(process)
self._wait()
else:
raise RuntimeError("Cannot start {}".format(self._executable))
@report_calls(Component.hyperdrive, 'instance.version')
def _check_version(self):
version = self.version()
if version < self._min_version:
raise RuntimeError('HyperG version {} is required'
.format(self._min_version))
@report_calls(Component.hyperdrive, 'instance.check')
def _create_sub(self):
try:
os.makedirs(self._dir, exist_ok=True)
return subprocess.Popen(self._command, stdin=DEVNULL,
stdout=None, stderr=None,
startupinfo=SUBPROCESS_STARTUP_INFO)
except OSError:
return self._critical_error()
def _wait(self, timeout: int = 10):
deadline = time.time() + timeout
while time.time() < deadline:
addresses = self.addresses()
if addresses:
return
time.sleep(1.)
self._critical_error()
def _critical_error(self):
logger.critical("Can't run hyperdrive executable %r. "
"Make sure path is correct and check "
"if it starts correctly.",
' '.join(self._command))
sys.exit(1)
def build_client_options(self, node_id, **kwargs):
return HyperdriveClient.build_options(node_id, **kwargs)
def test_id(self, _):
client = HyperdriveClient()
result = client.id()
assert result['id'] == response['id']
assert result['version'] == response['version']
def test_cancel(self, _):
client = HyperdriveClient()
content_hash = str(uuid.uuid4())
response_hash = response['hash']
assert client.cancel(content_hash) == response_hash
def test_restore(self, _):
client = HyperdriveClient()
assert client.restore(response['files']) == response['hash']
def test_add(self, _):
client = HyperdriveClient()
assert client.add(response['files']) == response['hash']
def test_addresses(self, _):
client = HyperdriveClient()
assert client.addresses() == dict(TCP=('0.0.0.0', 3282))
def test_diagnostics(self):
client = HyperdriveClient()
with self.assertRaises(NotImplementedError):
client.diagnostics()
def test_add(self):
client = HyperdriveClient()
with mock.patch.object(HyperdriveClient, '_request',
return_value=self.response):
assert client.add(self.response['files']) == self.response['hash']
def running():
try:
return HyperdriveClient().id()
except ConnectionError:
return False
def new_client(self):
return HyperdriveClient(**self.config.client)
def test_build_options(self, _):
options = HyperdriveClient.build_options()
assert options.client_id == HyperdriveClient.CLIENT_ID
assert options.version == HyperdriveClient.VERSION
assert options.options['peers'] is None