async def sysfs_impl(container_id):
mem_prefix = f'/sys/fs/cgroup/memory/docker/{container_id}/'
io_prefix = f'/sys/fs/cgroup/blkio/docker/{container_id}/'
try:
mem_cur_bytes = read_sysfs(
mem_prefix + 'memory.usage_in_bytes', int)
io_stats = Path(io_prefix +
'blkio.throttle.io_service_bytes').read_text()
# example data:
# 8:0 Read 13918208
# 8:0 Write 0
# 8:0 Sync 0
# 8:0 Async 13918208
# 8:0 Total 13918208
# Total 13918208
io_read_bytes = 0
io_write_bytes = 0
for line in io_stats.splitlines():
if line.startswith('Total '):
continue
dev, op, nbytes = line.strip().split()
if op == 'Read':
io_read_bytes += int(nbytes)
elif op == 'Write':
io_write_bytes += int(nbytes)
except IOError as e:
log.warning(
'cannot read stats: sysfs unreadable for container {0}\n{1!r}',
container_id[:7], e)
return None
loop = current_loop()
scratch_sz = await loop.run_in_executor(None, get_scratch_size,
container_id)
return mem_cur_bytes, io_read_bytes, io_write_bytes, scratch_sz
async def proxy_connection(upper_sock_path: Path,
down_reader: asyncio.StreamReader,
down_writer: asyncio.StreamWriter) -> None:
up_reader, up_writer = await asyncio.open_unix_connection(
str(upper_sock_path))
async def _downstream():
try:
while True:
data = await up_reader.read(4096)
if not data:
break
down_writer.write(data)
await down_writer.drain()
except asyncio.CancelledError:
pass
finally:
down_writer.close()
await down_writer.wait_closed()
await asyncio.sleep(0)
async def _upstream():
try:
while True:
data = await down_reader.read(4096)
if not data:
break
up_writer.write(data)
await up_writer.drain()
except asyncio.CancelledError:
pass
finally:
up_writer.close()
await up_writer.wait_closed()
await asyncio.sleep(0)
loop = current_loop()
downstream_task = loop.create_task(_downstream())
upstream_task = loop.create_task(_upstream())
tasks = [upstream_task, downstream_task]
# Since we cannot determine which side (the server or client) disconnects first,
# await until any task that completes first.
# For example, when proxying the docker domain socket, the proxy connections for one-shot
# docker commands are usually disconnected by the client first, but the connections for
# long-running streaming commands are disconnected by the server first when the server-side
# processing finishes.
try:
task_results: Tuple[Set[Future], Set[Future]] = \
await asyncio.wait(tasks, return_when=asyncio.FIRST_COMPLETED)
done, pending = task_results
except asyncio.CancelledError:
pass
finally:
# And then cancel all remaining tasks.
for t in pending:
t.cancel()
await t
async def read_tail(path: Path, nbytes: int) -> bytes:
file_size = path.stat().st_size
def _read_tail() -> bytes:
with open(path, 'rb') as f:
f.seek(max(file_size - nbytes, 0), io.SEEK_SET)
return f.read(nbytes)
loop = current_loop()
return await loop.run_in_executor(None, _read_tail)
async def __ainit__(self) -> None:
loop = current_loop()
self.input_sock.connect(await self.get_repl_in_addr())
self.input_sock.setsockopt(zmq.LINGER, 50)
self.output_sock.connect(await self.get_repl_out_addr())
self.output_sock.setsockopt(zmq.LINGER, 50)
self.status_task = loop.create_task(self.ping_status())
self.read_task = loop.create_task(self.read_output())
if self.exec_timeout > 0:
self.watchdog_task = loop.create_task(self.watchdog())
else:
self.watchdog_task = None
def __init__(
self,
etcd: AsyncEtcd,
local_config: Mapping[str, Any],
*,
skip_detect_manager: bool = False,
) -> None:
self.loop = current_loop()
self.etcd = etcd
self.local_config = local_config
self.skip_detect_manager = skip_detect_manager
self._stop_signal = signal.SIGTERM
async def api_impl(container_id):
container = DockerContainer(ctx.agent.docker, id=container_id)
ret = await fetch_api_stats(container)
if ret is None:
return None
mem_cur_bytes = nmget(ret, 'memory_stats.usage', 0)
io_read_bytes = 0
io_write_bytes = 0
for item in nmget(ret, 'blkio_stats.io_service_bytes_recursive',
[]):
if item['op'] == 'Read':
io_read_bytes += item['value']
elif item['op'] == 'Write':
io_write_bytes += item['value']
loop = current_loop()
scratch_sz = await loop.run_in_executor(None, get_scratch_size,
container_id)
return mem_cur_bytes, io_read_bytes, io_write_bytes, scratch_sz
async def delete_vfolders(
cls,
conn: SAConnection,
user_uuid: uuid.UUID,
config_server,
) -> int:
"""
Delete user's all virtual folders as well as their physical data.
:param conn: DB connection
:param user_uuid: user's UUID to delete virtual folders
:return: number of deleted rows
"""
from . import vfolders
mount_prefix = Path(await config_server.get('volumes/_mount'))
fs_prefix = await config_server.get('volumes/_fsprefix')
fs_prefix = Path(fs_prefix.lstrip('/'))
query = (
sa.select([vfolders.c.id, vfolders.c.host, vfolders.c.unmanaged_path])
.select_from(vfolders)
.where(vfolders.c.user == user_uuid)
)
async for row in conn.execute(query):
if row['unmanaged_path']:
folder_path = Path(row['unmanaged_path'])
else:
folder_path = (mount_prefix / row['host'] / fs_prefix / row['id'].hex)
log.info('deleting physical files: {0}', folder_path)
try:
loop = current_loop()
await loop.run_in_executor(None, lambda: shutil.rmtree(folder_path)) # type: ignore
except IOError:
pass
query = (
vfolders.delete()
.where(vfolders.c.user == user_uuid)
)
result = await conn.execute(query)
if result.rowcount > 0:
log.info('deleted {0} user\'s virtual folders ({1})', result.rowcount, user_uuid)
return result.rowcount
def handle_loop_error(
root_app: web.Application,
loop: asyncio.AbstractEventLoop,
context: Mapping[str, Any],
) -> None:
if isinstance(loop, aiojobs.Scheduler):
loop = current_loop()
exception = context.get('exception')
msg = context.get('message', '(empty message)')
if exception is not None:
if sys.exc_info()[0] is not None:
log.exception('Error inside event loop: {0}', msg)
if 'error_monitor' in root_app:
loop.create_task(root_app['error_monitor'].capture_exception())
else:
exc_info = (type(exception), exception, exception.__traceback__)
log.error('Error inside event loop: {0}', msg, exc_info=exc_info)
if 'error_monitor' in root_app:
loop.create_task(root_app['error_monitor'].capture_exception(
exc_instance=exception))
async def accept_file(self, filename: str, filedata: bytes):
loop = current_loop()
work_dir = self.agent_config['container']['scratch-root'] / str(
self.kernel_id) / 'work'
try:
# create intermediate directories in the path
dest_path = (work_dir / filename).resolve(strict=False)
parent_path = dest_path.parent
except ValueError: # parent_path does not start with work_dir!
raise AssertionError('malformed upload filename and path.')
def _write_to_disk():
parent_path.mkdir(parents=True, exist_ok=True)
dest_path.write_bytes(filedata)
try:
await loop.run_in_executor(None, _write_to_disk)
except FileNotFoundError:
log.error('{0}: writing uploaded file failed: {1} -> {2}',
self.kernel_id, filename, dest_path)
def __init__(self, app: web.Application) -> None:
self.loop = current_loop()
self.root_app = app
self.consumers = defaultdict(set)
self.subscribers = defaultdict(set)
async def gather_node_measures(
self, ctx: StatContext) -> Sequence[NodeMeasurement]:
_mstat = psutil.virtual_memory()
total_mem_used_bytes = Decimal(_mstat.total - _mstat.available)
total_mem_capacity_bytes = Decimal(_mstat.total)
_nstat = psutil.net_io_counters()
net_rx_bytes = _nstat.bytes_recv
net_tx_bytes = _nstat.bytes_sent
def get_disk_stat():
pruned_disk_types = frozenset(['squashfs', 'vfat', 'tmpfs'])
total_disk_usage = Decimal(0)
total_disk_capacity = Decimal(0)
per_disk_stat = {}
for disk_info in psutil.disk_partitions():
if disk_info.fstype not in pruned_disk_types:
dstat = os.statvfs(disk_info.mountpoint)
disk_usage = Decimal(dstat.f_frsize *
(dstat.f_blocks - dstat.f_bavail))
disk_capacity = Decimal(dstat.f_frsize * dstat.f_blocks)
per_disk_stat[disk_info.device] = Measurement(
disk_usage, disk_capacity)
total_disk_usage += disk_usage
total_disk_capacity += disk_capacity
return total_disk_usage, total_disk_capacity, per_disk_stat
loop = current_loop()
total_disk_usage, total_disk_capacity, per_disk_stat = \
await loop.run_in_executor(None, get_disk_stat)
return [
NodeMeasurement(
MetricKey('mem'),
MetricTypes.USAGE,
unit_hint='bytes',
stats_filter=frozenset({'max'}),
per_node=Measurement(total_mem_used_bytes,
total_mem_capacity_bytes),
per_device={
DeviceId('root'):
Measurement(total_mem_used_bytes, total_mem_capacity_bytes)
},
),
NodeMeasurement(
MetricKey('disk'),
MetricTypes.USAGE,
unit_hint='bytes',
per_node=Measurement(total_disk_usage, total_disk_capacity),
per_device=per_disk_stat,
),
NodeMeasurement(
MetricKey('net_rx'),
MetricTypes.RATE,
unit_hint='bps',
current_hook=lambda metric: metric.stats.rate,
per_node=Measurement(Decimal(net_rx_bytes)),
per_device={
DeviceId('node'): Measurement(Decimal(net_rx_bytes))
},
),
NodeMeasurement(
MetricKey('net_tx'),
MetricTypes.RATE,
unit_hint='bps',
current_hook=lambda metric: metric.stats.rate,
per_node=Measurement(Decimal(net_tx_bytes)),
per_device={
DeviceId('node'): Measurement(Decimal(net_tx_bytes))
},
),
]
