def __init__(self, **params):
super().__init__(**params)
self._session = AEUserSession(self.hostname,
self.username,
self.password,
persist=False,
k8s_endpoint=self.k8s_endpoint)
adapter = requests.adapters.HTTPAdapter(pool_maxsize=self.pool_size)
self._session.session.mount('https://', adapter)
self._session.session.mount('http://', adapter)
def test_login_time(admin_session, user_session):
# The current session should already be authenticated
now = datetime.utcnow()
plist0 = user_session.project_list()
user_list = admin_session.user_list()
urec = next(
(r for r in user_list if r['username'] == user_session.username), None)
assert urec is not None
ltm1 = datetime.fromtimestamp(urec['lastLogin'] / 1000.0)
assert ltm1 < now
# Create new login session. This should change lastLogin
password = os.environ.get('AE5_PASSWORD')
user_sess2 = AEUserSession(user_session.hostname,
user_session.username,
password,
persist=False)
plist1 = user_sess2.project_list()
urec = admin_session.user_info(urec['id'])
ltm2 = datetime.fromtimestamp(urec['lastLogin'] / 1000.0)
assert ltm2 > ltm1
user_sess2.disconnect()
assert plist1 == plist0
# Create new impersonation session. This should not change lastLogin
user_sess3 = AEUserSession(admin_session.hostname,
user_session.username,
admin_session,
persist=False)
plist2 = user_sess3.project_list()
urec = admin_session.user_info(urec['id'])
ltm3 = datetime.fromtimestamp(urec['lastLogin'] / 1000.0)
assert ltm3 == ltm2
user_sess3.disconnect()
# Confirm the impersonation worked by checking the project lists are the same
assert plist2 == plist0
# Access the original login session. It should not reauthenticate
plist3 = user_session.project_list()
urec = admin_session.user_info(urec['id'])
ltm4 = datetime.fromtimestamp(urec['lastLogin'] / 1000.0)
assert ltm4 == ltm3
assert plist3 == plist0
def user_session():
hostname, username, password = _get_vars('AE5_HOSTNAME', 'AE5_USERNAME',
'AE5_PASSWORD')
return AEUserSession(hostname, username, password)
class AE5Source(Source):
"""
The AE5Source provides a number of tables, which allow monitoring
the nodes, deployments, sessions and resource profiles of an
Anaconda Enterprise 5 installation.
"""
hostname = param.String(doc="URL of the AE5 host.")
username = param.String(doc="Username to authenticate with AE5.")
password = param.String(doc="Password to authenticate with AE5.")
k8s_endpoint = param.String(default='k8s')
pool_size = param.Integer(default=100,
doc="""
Size of HTTP socket pool.""")
private = param.Boolean(default=True,
doc="""
Whether to limit the deployments visible to a user based on
their authorization.""")
source_type = 'ae5'
_deployment_columns = [
'id', 'name', 'url', 'owner', 'resource_profile', 'public', 'state',
'cpu', 'cpu_percent', 'memory', 'memory_percent', 'uptime', 'restarts'
]
_session_columns = [
'id', 'name', 'url', 'owner', 'resource_profile', 'state'
]
_tables = ['deployments', 'nodes', 'resources', 'sessions']
_units = {
'm': 0.001,
None: 1,
'Ki': 1024,
'Mi': 1024**2,
'Gi': 1024**3,
'Ti': 1024**4
}
def __init__(self, **params):
super().__init__(**params)
self._session = AEUserSession(self.hostname,
self.username,
self.password,
persist=False,
k8s_endpoint=self.k8s_endpoint)
adapter = requests.adapters.HTTPAdapter(pool_maxsize=self.pool_size)
self._session.session.mount('https://', adapter)
self._session.session.mount('http://', adapter)
@classmethod
def _convert_value(cls, value):
if value == '0':
return 0
val_unit = [
m for m in cls._units if m is not None and value.endswith(m)
]
val_unit = val_unit[0] if val_unit else None
if not val_unit:
return float(value)
scale_factor = cls._units[val_unit]
return float(value[:-len(val_unit)]) * scale_factor
@property
def _user(self):
return state.headers.get('Anaconda-User') if self.private else None
def _process_deployment(self, deployment):
container_info = deployment.get('_k8s', {}).get('containers', {})
if 'app' not in container_info:
nan = float('NaN')
deployment['cpu'] = nan
deployment['memory'] = nan
deployment['restarts'] = nan
deployment['uptime'] = '-'
return deployment
container = container_info['app']
limits = container['limits']
usage = container['usage']
# CPU Usage
cpu_cap = self._convert_value(limits['cpu'])
if 'cpu' in usage:
cpu = self._convert_value(usage['cpu'])
deployment['cpu'] = cpu
deployment['cpu_percent'] = round((cpu / cpu_cap) * 100, 2)
else:
deployment['cpu_percent'] = float('nan')
deployment['cpu'] = float('nan')
# Memory usage
mem_cap = self._convert_value(limits['memory'])
if 'memory' in usage:
mem = self._convert_value(usage['memory'])
deployment['memory'] = mem
deployment['memory_percent'] = round((mem / mem_cap) * 100, 2)
else:
deployment['memory'] = float('nan')
deployment['memory_percent'] = float('nan')
# Uptime
started = container.get('since')
deployment["restarts"] = container['restarts']
if started:
started_dt = dt.datetime.fromisoformat(started.replace('Z', ''))
uptime = str(dt.datetime.now() - started_dt)
deployment["uptime"] = uptime[:uptime.index('.')]
else:
deployment["uptime"] = '-'
return deployment
def _process_nodes(self, node):
caps = {
'cpu': self._convert_value(node['capacity/cpu']),
'gpu': self._convert_value(node['capacity/gpu']),
'mem': self._convert_value(node['capacity/mem']),
'pod': int(node['capacity/pod'])
}
for column in node.index:
if 'capacity' in column or '/' not in column:
continue
vtype = column.split('/')[1]
cap = caps[vtype]
value = node[column]
if vtype != 'pod':
value = self._convert_value(value)
node[column] = value
node[f'{column}_percent'] = 0 if cap == 0 else round(
(value / caps[vtype]) * 100, 2)
return node
def _get_deployments(self):
user = self._user
deployments = self._session.deployment_list(
k8s=True, format='dataframe',
collaborators=bool(user)).apply(self._process_deployment, axis=1)
if user is None:
return deployments[self._deployment_columns]
return deployments[deployments.public | (deployments.owner == user)
| deployments._collaborators.apply(
lambda x: user in x)][self._deployment_columns]
def _get_nodes(self):
nodes = self._session.node_list(format='dataframe').apply(
self._process_nodes, axis=1)
return nodes[[c for c in nodes.columns if not c.startswith('_')]]
def _get_resources(self):
return self._session.resource_profile_list(format='dataframe')
def _get_sessions(self):
sessions = self._session.session_list(format='dataframe')
if self._user:
sessions = sessions[sessions.owner == self._user]
return sessions[self._session_columns]
@cached(with_query=False)
def get(self, table, **query):
if table not in self._tables:
raise ValueError(
f"AE5Source has no '{table}' table, choose from {repr(self._tables)}."
)
return getattr(self, f'_get_{table}')()
def get_schema(self, table=None):
schemas = {}
for t in self._tables:
if table is None or t == table:
schemas[t] = get_dataframe_schema(
self.get(t))['items']['properties']
return schemas if table is None else schemas[table]
def user_session():
hostname, username, password = _get_vars('AE5_HOSTNAME', 'AE5_USERNAME', 'AE5_PASSWORD')
s = AEUserSession(hostname, username, password)
for run in s.run_list():
s.run_delete(run)
for job in s.job_list():
s.job_delete(job)
for dep in s.deployment_list():
s.deployment_stop(dep)
for sess in s.session_list():
s.session_stop(sess)
plist = s.project_list()
for p in plist:
if p['name'] not in {'testproj1', 'testproj2', 'testproj3'} and p['owner'] == username:
s.project_delete(p['id'])
# Make sure testproj3 is using the Jupyter editor
prec = s.project_info(f'{username}/testproj3', collaborators=True)
if prec['editor'] != 'notebook' or prec['resource_profile'] != 'default':
s.project_patch(prec, editor='notebook', resource_profile='default')
# Make sure testproj3 has no collaborators
if prec['_collaborators']:
collabs = tuple(c['id'] for c in prec['_collaborators'])
s.project_collaborator_remove(prec, collabs)
plist = s.project_list(collaborators=True)
plist = s.project_list(collaborators=True)
powned = [p for p in plist if p['owner'] == username]
pother = [p for p in plist if p['owner'] != username]
# Assert there are exactly 3 projects owned by the test user
assert len(powned) == 3
# Need at least two duplicated project names to properly test sorting/filtering
assert len(set(p['name'] for p in powned).intersection(p['name'] for p in pother)) >= 2
# Make sure all three editors are represented
assert len(set(p['editor'] for p in powned)) == 3
# Make sure we have 0, 1, and 2 collaborators represented
assert set(len(p['_collaborators']) for p in plist
if p['owner'] == username).issuperset((0, 1, 2))
yield s
s.disconnect()
def test_user_session(monkeypatch, capsys):
with pytest.raises(ValueError) as excinfo:
AEUserSession('', '')
assert 'Must supply hostname and username' in str(excinfo.value)
hostname, username, password = _get_vars('AE5_HOSTNAME', 'AE5_USERNAME',
'AE5_PASSWORD')
with pytest.raises(AEException) as excinfo:
c = AEUserSession(hostname, username, 'x' + password, persist=False)
c.authorize()
del c
assert 'Invalid username or password.' in str(excinfo.value)
passwords = [password, '', 'x' + password]
monkeypatch.setattr('getpass.getpass', lambda x: passwords.pop())
c = AEUserSession(hostname, username, persist=False)
c.authorize()
captured = capsys.readouterr()
assert f'Password for {username}@{hostname}' in captured.err
assert f'Invalid username or password; please try again.' in captured.err
assert f'Must supply a password' in captured.err
true_endpoint, c._k8s_endpoint = c._k8s_endpoint, 'ssh:fakeuser'
with pytest.raises(AEException) as excinfo:
c._k8s('status')
assert 'Error establishing k8s connection' in str(excinfo.value)
c._k8s_endpoint = 'fakek8sendpoint'
with pytest.raises(AEException) as excinfo:
c._k8s('status')
assert 'No deployment found at endpoint fakek8sendpoint' in str(
excinfo.value)
with pytest.raises(AEException) as excinfo:
c._k8s('status')
assert 'No k8s connection available' in str(excinfo.value)
c._k8s_endpoint = true_endpoint
assert c._k8s('status') == 'Alive and kicking'
def user_setup():
hostname, username, password = _get_vars('AE5_HOSTNAME', 'AE5_USERNAME',
'AE5_PASSWORD')
s = AEUserSession(hostname, username, password)
for run in s.run_list():
s.run_delete(run['id'])
for job in s.job_list():
s.job_delete(job['id'])
for dep in s.deployment_list():
s.deployment_stop(dep['id'])
for sess in s.session_list():
s.session_stop(sess['id'])
plist = s.project_list(collaborators=True)
for p in plist:
if p['name'] not in {'testproj1', 'testproj2', 'testproj3'
} and p['owner'] == username:
s.project_delete(p['id'])
plist = s.project_list(collaborators=True)
powned = [p for p in plist if p['owner'] == username]
pother = [p for p in plist if p['owner'] != username]
# Assert there are exactly 3 projects owned by the test user
assert len(powned) == 3
# Need at least two duplicated project names to properly test sorting/filtering
assert len(
set(p['name'] for p in powned).intersection(p['name']
for p in pother)) >= 2
# Make sure all three editors are represented
assert len(set(p['editor'] for p in powned)) == 3
# Make sure we have 0, 1, and 2 collaborators represented
assert set(
len(p['collaborators'].split(', ')) if p['collaborators'] else 0
for p in powned).issuperset((0, 1, 2))
yield s, plist
s.disconnect()