def test_unreachable(self):
with self.assertRaisesRegex(AssertionError, 'error 1'):
ASSERT.unreachable('error {}', 1)
def define_pod(
*,
name: str,
apps: typing.List[App] = (),
images: typing.List[str] = (),
mounts: typing.List[Mount] = (),
volumes: typing.List[Volume] = (),
systemd_unit_groups: typing.List[SystemdUnitGroup] = (),
token_names: typing.Mapping[str, str] = None,
):
"""Define a pod.
This defines:
* Parameter: name/version.
* Rule: name/build. NOTE: This rule is generally run in the host
system, not inside a builder pod.
"""
ASSERT(len(images) <= 1, 'expect at most one image per pod for now: {}')
# Let's require absolute release labels (because it is quite hard to
# derive label path for images and volumes from pod label).
ASSERT.all(images, lambda label: label.startswith('//'))
ASSERT.all(volumes, lambda volume: volume.label.startswith('//'))
ASSERT.unique(map(_get_label_name, images))
ASSERT.unique(_get_label_name(volume.label) for volume in volumes)
name_prefix = shipyard2.rules.canonicalize_name_prefix(name)
parameter_version = name_prefix + 'version'
rule_build = name_prefix + 'build'
(foreman.define_parameter(parameter_version)\
.with_doc('pod version'))
images = list(map(foreman.Label.parse, images))
@foreman.rule(rule_build)
@foreman.rule.depend('//pods/bases:build')
@foreman.rule.depend('//releases:build')
def build(parameters):
version = ASSERT.not_none(parameters[parameter_version])
pod_dir_path = releases.get_output_dir_path(parameters, name, version)
if (
pod_dir_path / \
shipyard2.POD_DIR_RELEASE_METADATA_FILENAME
).exists():
LOG.info('skip: build pod: %s %s', name, version)
return
LOG.info('build pod: %s %s', name, version)
try:
scripts.mkdir(pod_dir_path)
releases.generate_release_metadata(
parameters,
pod_dir_path / shipyard2.POD_DIR_RELEASE_METADATA_FILENAME,
)
_generate_deploy_instruction(
parameters=parameters,
pod_dir_path=pod_dir_path,
name=name,
version=version,
apps=apps,
images=images,
mounts=mounts,
volumes=volumes,
systemd_unit_groups=systemd_unit_groups,
token_names=token_names,
)
_link_images(parameters, pod_dir_path, images)
_link_volumes(parameters, pod_dir_path, volumes)
except Exception:
# Roll back on error.
scripts.rm(pod_dir_path, recursive=True)
raise
for label in images:
build.depend(str(_images.derive_rule(label)))
return PodRules(build=build)
def get_exception_nonblocking(self):
ASSERT.true(self.is_completed())
self._consumed = True
return self._exception
def _decode_raw_value(self, value_type, raw_value):
"""Decode a raw value into ``value_type``-typed value.
This and ``_encode_value`` complement each other.
"""
if typings.is_recursive_type(value_type):
if value_type.__origin__ in (list, set, frozenset):
element_type = value_type.__args__[0]
return value_type.__origin__(
self._decode_raw_value(element_type, raw_element)
for raw_element in raw_value)
elif value_type.__origin__ is tuple:
ASSERT.equal(len(raw_value), len(value_type.__args__))
return tuple(
self._decode_raw_value(element_type, raw_element)
for element_type, raw_element in zip(
value_type.__args__,
raw_value,
))
elif typings.is_union_type(value_type):
# Handle ``None`` special case.
if raw_value is None:
ASSERT.in_(NoneType, value_type.__args__)
return None
# Handle ``Optional[T]`` special case.
type_ = typings.match_optional_type(value_type)
if type_:
return self._decode_raw_value(type_, raw_value)
ASSERT.equal(len(raw_value), 1)
type_name, raw_element = next(iter(raw_value.items()))
for type_ in value_type.__args__:
if typings.is_recursive_type(type_):
candidate = str(type_)
else:
candidate = type_.__name__
if type_name == candidate:
return self._decode_raw_value(type_, raw_element)
return ASSERT.unreachable(
'raw value is not any union element type: {!r} {!r}',
value_type,
raw_value,
)
else:
return ASSERT.unreachable('unsupported generic: {!r}',
value_type)
elif wiredata.is_message_type(value_type):
return value_type(
**{
f.name: self._decode_raw_value(f.type, raw_value[f.name])
for f in dataclasses.fields(value_type)
if f.name in raw_value
})
elif not isinstance(value_type, type):
# Non-``type`` instance cannot be passed to ``issubclass``.
return ASSERT.unreachable('unsupported value type: {!r}',
value_type)
elif issubclass(value_type, datetime.datetime):
return value_type.fromisoformat(raw_value)
elif issubclass(value_type, enum.Enum):
return value_type[raw_value]
elif issubclass(value_type, bytes):
return base64.standard_b64decode(raw_value.encode('ascii'))
elif issubclass(value_type, Exception):
ASSERT.equal(len(raw_value), 1)
return value_type(
*(ASSERT.isinstance(raw_arg, _DIRECTLY_SERIALIZABLE_TYPES)
for raw_arg in raw_value[value_type.__name__]))
elif issubclass(value_type, _DIRECTLY_SERIALIZABLE_TYPES):
if value_type in _DIRECTLY_SERIALIZABLE_TYPES:
return ASSERT.isinstance(raw_value, value_type)
else:
# Support sub-type of int, etc.
return value_type(raw_value)
else:
return ASSERT.unreachable('unsupported value type: {!r}',
value_type)
def to_upper(self, message_type, wire_message):
ASSERT.predicate(message_type, wiredata.is_message_type)
raw_message = json.loads(wire_message)
return self._decode_raw_value(message_type, raw_message)
def _find_root_project(src_path):
for path in itertools.chain([src_path], src_path.parents):
if _is_root_project(path):
return path
return ASSERT.unreachable('cannot find root project from: {}', src_path)
def cleanup(parameters):
ASSERT.is_(parameters['inside-builder-pod'], True)
ASSERT.all(parameters['roots'], _is_root_dir)
with scripts.using_sudo():
scripts.apt_get_clean()
def rollback_due_to_timeout(self):
ASSERT.not_equal(self._tx_id, 0)
self._tx.rollback()
self._end(self._timeout_tx_ids)
def notify(self, n=1):
ASSERT.true(self._lock.is_owner())
for _ in range(min(n, len(self._waiters))):
self._waiters.pop().unblock()
def append(self, data):
ASSERT.isinstance(data, bytes)
errors.check(self._chunk_append(self._get(), data, len(data)))
def _get_dialect(db_url):
for dialect in ('postgresql', 'sqlite'):
if db_url.startswith(dialect):
return dialect
return ASSERT.unreachable('unsupported dialect: {}', db_url)
def _get(self):
return ASSERT.not_none(self._msg_p)
def test_assert_collection_mapper(self):
self.assertEqual(ASSERT.all([2, 4, 6], is_even), [2, 4, 6])
pattern = r'expect all .*is_even.*, not \[2, 4, 6, 7\]'
with self.assertRaisesRegex(AssertionError, pattern):
ASSERT.all([2, 4, 6, 7], is_even)
def test_assertion_methods_pass(self):
checks = [
('__call__', (1, ''), 1),
('true', (True, ), True),
('true', (1, ), 1),
('false', (False, ), False),
('false', ('', ), ''),
('empty', ((), ), ()),
('empty', ([], ), []),
('empty', ({}, ), {}),
('empty', (set(), ), set()),
('not_empty', ((1, ), ), (1, )),
('not_empty', ([2], ), [2]),
('not_empty', (dict(x=1), ), dict(x=1)),
('not_empty', (set([42]), ), set([42])),
('none', (None, ), None),
('not_none', (0, ), 0),
('predicate', (0, is_even), 0),
('not_predicate', (3, is_even), 3),
('xor', (0, 1), 0),
('xor', (1, 0), 1),
('not_xor', (0, 0), 0),
('not_xor', (1, 1), 1),
('is_', (0, 0), 0),
('is_not', (0, 1), 0),
('isinstance', ('hello', (int, str)), 'hello'),
('not_isinstance', ('hello', (int, bytes)), 'hello'),
('issubclass', (Derived, Base), Derived),
('not_issubclass', (Base, Derived), Base),
('in_', (1, [1]), 1),
('not_in', (0, [1]), 0),
('contains', ([1], 1), [1]),
('not_contains', ([1], 0), [1]),
(
'getitem',
(
{
'x': 1,
},
'x',
),
1,
),
('equal', (0, 0), 0),
('not_equal', (0, 1), 0),
('greater', (1, 0), 1),
('greater_or_equal', (1, 0), 1),
('greater_or_equal', (0, 0), 0),
('less', (0, 1), 0),
('less_or_equal', (0, 1), 0),
('less_or_equal', (1, 1), 1),
('in_range', (0, (0, 1)), 0),
('not_in_range', (1, (0, 1)), 1),
('startswith', ('hello world', 'hello'), 'hello world'),
('startswith', (b'hello world', b'hello'), b'hello world'),
('not_startswith', ('hello', 'hello world'), 'hello'),
('not_startswith', (b'hello', b'hello world'), b'hello'),
('isdisjoint', ({1, 2}, {3, 4}), {1, 2}),
('not_isdisjoint', ({1, 2}, {2, 3, 4}), {1, 2}),
('issubset', ({1, 2}, {1, 2}), {1, 2}),
('not_issubset', ({1, 2}, {2, 3}), {1, 2}),
('issubset_proper', ({1, 2}, {1, 2, 3}), {1, 2}),
('not_issubset_proper', ({1, 2}, {1, 2}), {1, 2}),
('issuperset', ({1, 2}, {1, 2}), {1, 2}),
('not_issuperset', ({1, 2}, {2, 3}), {1, 2}),
('issuperset_proper', ({1, 2, 3}, {1, 2}), {1, 2, 3}),
('not_issuperset_proper', ({1, 2}, {1, 2}), {1, 2}),
]
for check_name, args, expect_ret in checks:
with self.subTest(check=check_name):
check = getattr(ASSERT, check_name)
self.assertEqual(check(*args), expect_ret)
d = {}
self.assert_.setitem(d, 0, 0)
self.assertEqual(d, {0: 0})
for args in [
(),
([], ),
([], []),
([], [], []),
('a', 'b', 'c'),
('ad', 'be', 'cf'),
]:
with self.subTest(args):
self.assertEqual(list(ASSERT.zip(*args)), list(zip(*args)))
def notify_open(self, fd):
ASSERT.false(self._epoll.closed)
try:
self._epoll.register(fd, self._EVENT_MASK)
except FileExistsError:
pass
def __init__(self, value=1):
self._value = ASSERT.greater_or_equal(value, 0)
self._gate = Gate()
_DEFINITION_LIST_COLUMNS = frozenset((
'token-name',
'range',
'values',
))
_DEFINITION_LIST_DEFAULT_COLUMNS = (
'token-name',
'range',
'values',
)
_DEFINITION_LIST_STRINGIFIERS = {
'range': lambda args: ' '.join(map(str, args)),
'values': ' '.join,
}
ASSERT.issuperset(_DEFINITION_LIST_COLUMNS, _DEFINITION_LIST_DEFAULT_COLUMNS)
ASSERT.issuperset(_DEFINITION_LIST_COLUMNS, _DEFINITION_LIST_STRINGIFIERS)
@argparses.begin_parser('list-definitions',
**argparses.make_help_kwargs('list token definitions'))
@columns_argparses.columnar_arguments(_DEFINITION_LIST_COLUMNS,
_DEFINITION_LIST_DEFAULT_COLUMNS)
@argparses.end
def cmd_list_definitions(args):
columnar = columns.Columnar(
**columns_argparses.make_columnar_kwargs(args),
stringifiers=_DEFINITION_LIST_STRINGIFIERS,
)
for token_name, definition in (
tokens.make_tokens_database().get().definitions.items()):
def release(self, n=1):
self._value += ASSERT.greater_or_equal(n, 1)
self._gate.unblock()
def chown_app(path):
"""Change owner to root and group to the application group."""
shutil.chown(path, 'root', ASSERT.true(PARAMS.application_group.get()))
def release(self, n=1):
ASSERT.less_or_equal(self._value + n, self.__upper_bound)
return super().release(n)
]
import base64
import dataclasses
import datetime
import enum
import json
import sys
from g1.bases import typings
from g1.bases.assertions import ASSERT
from .. import wiredata
# Python 3.7 supports parsing ISO 8601 (bpo-15873), finally!
ASSERT.greater_or_equal(sys.version_info, (3, 7))
NoneType = type(None)
_DIRECTLY_SERIALIZABLE_TYPES = (dict, list, tuple, str, int, float, bool,
NoneType)
class JsonWireData(wiredata.WireData):
"""JSON wire data converter.
This supports ``datetime.datetime``, ``enum.Enum``, ``Exception``,
``typing.Tuple``, ``typing.List``, ``typing.Set``,
``typing.FrozenSet``, and ``typing.Union``.
Caveats:
def release(self):
ASSERT.true(self.is_owner())
self._locked = False
contexts.get_kernel().unblock(self)
def to_lower(self, message):
ASSERT.predicate(message, wiredata.is_message)
raw_message = self._encode_value(type(message), message)
return json.dumps(raw_message).encode('ascii')
def build_image(
*,
parameters,
builder_id,
builder_images,
name,
version,
rules,
output,
):
# Although it is tempting to mount source repos under the drydock
# directory rather than /usr/src, this is not possible because the
# base image does not have /home/plumber/drydock directory yet, and
# so systemd-nspawn will reject mounting source repos under drydock.
root_host_paths = parameters['//bases:roots']
builder_config = _generate_builder_config(
name=name,
version=version,
apps=_get_apps(
parameters,
builder_images,
root_host_paths,
rules,
),
images=_get_images(
builder_images,
ASSERT.not_none(parameters['//images/bases:base/version']),
),
mounts=_get_mounts(
parameters['//releases:shipyard-data'],
name,
rules,
),
overlays=_get_overlays(root_host_paths),
)
with contextlib.ExitStack() as stack:
tempdir_path = Path(
stack.enter_context(
tempfile.TemporaryDirectory(dir=output.parent)
)
)
builder_config_path = tempdir_path / 'builder.json'
builder_config_path.write_text(json.dumps(builder_config))
if shipyard2.is_debug():
LOG.debug('builder config: %s', builder_config_path.read_text())
# The builder pod might not be cleaned up when `ctr pods run`
# fails; so let's always do `ctr pods remove` on our way out.
stack.callback(ctr_scripts.ctr_remove_pod, builder_id)
LOG.info('start builder pod')
ctr_scripts.ctr_run_pod(builder_id, builder_config_path)
LOG.info('export intermediate builder image to: %s', output)
rootfs_path = tempdir_path / 'rootfs'
stack.callback(scripts.rm, rootfs_path, recursive=True)
ctr_scripts.ctr([
'pods',
'export-overlay',
builder_id,
rootfs_path,
])
ctr_scripts.ctr_build_image(
utils.get_builder_name(name), version, rootfs_path, output
)
ctr_scripts.ctr_import_image(output)
def _encode_value(self, value_type, value):
"""Encode a value into a raw value.
This and ``_decode_raw_value`` complement each other.
"""
if typings.is_recursive_type(value_type):
if value_type.__origin__ in (list, set, frozenset):
element_type = value_type.__args__[0]
return [
self._encode_value(element_type, element)
for element in value
]
elif value_type.__origin__ is tuple:
ASSERT.equal(len(value), len(value_type.__args__))
return tuple(
self._encode_value(element_type, element)
for element_type, element in zip(
value_type.__args__,
value,
))
elif typings.is_union_type(value_type):
# Make a special case for ``None``.
if value is None:
ASSERT.in_(NoneType, value_type.__args__)
return None
# Make a special case for ``Optional[T]``.
type_ = typings.match_optional_type(value_type)
if type_:
return self._encode_value(type_, value)
for type_ in value_type.__args__:
if typings.is_recursive_type(type_):
if _match_recursive_type(type_, value):
return {
str(type_): self._encode_value(type_, value)
}
elif isinstance(value, type_):
return {
type_.__name__: self._encode_value(type_, value)
}
return ASSERT.unreachable(
'value is not any union element type: {!r} {!r}',
value_type,
value,
)
else:
return ASSERT.unreachable('unsupported generic: {!r}',
value_type)
elif wiredata.is_message(value):
ASSERT.predicate(value_type, wiredata.is_message_type)
return {
f.name: self._encode_value(f.type, getattr(value, f.name))
for f in dataclasses.fields(value)
}
elif isinstance(value, datetime.datetime):
ASSERT.issubclass(value_type, datetime.datetime)
return value.isoformat()
elif isinstance(value, enum.Enum):
ASSERT.issubclass(value_type, enum.Enum)
return value.name
# JSON does not support binary type; so it has to be encoded.
elif isinstance(value, bytes):
ASSERT.issubclass(value_type, bytes)
return base64.standard_b64encode(value).decode('ascii')
elif isinstance(value, Exception):
ASSERT.issubclass(value_type, Exception)
return {
type(value).__name__: [
ASSERT.isinstance(arg, _DIRECTLY_SERIALIZABLE_TYPES)
for arg in value.args
]
}
elif isinstance(value, _DIRECTLY_SERIALIZABLE_TYPES):
ASSERT.issubclass(value_type, _DIRECTLY_SERIALIZABLE_TYPES)
return value
else:
return ASSERT.unreachable('unsupported value type: {!r} {!r}',
value_type, value)
def __init__(self, max_retries, backoff_base):
self._max_retries = ASSERT.greater(max_retries, 0)
self._backoff_base = ASSERT.greater(backoff_base, 0)
def get_result_nonblocking(self):
ASSERT.true(self.is_completed())
self._consumed = True
if self._exception:
raise self._exception
return self._result
def add(self, t):
if self._log:
ASSERT.greater(t, self._log[-1])
self._log.append(t)
def __setitem__(self, header, value):
ASSERT.true(self._is_uncommitted())
ASSERT.isinstance(header, str)
ASSERT.isinstance(value, str)
self._headers[header] = value
from g1.bases.assertions import ASSERT
from g1.texts import columns
from g1.texts.columns import argparses as columns_argparses
from . import envs
from . import models
_ENV_LIST_COLUMNS = frozenset((
'name',
'value',
))
_ENV_LIST_DEFAULT_COLUMNS = (
'name',
'value',
)
ASSERT.issuperset(_ENV_LIST_COLUMNS, _ENV_LIST_DEFAULT_COLUMNS)
@argparses.begin_parser(
'list',
**argparses.make_help_kwargs('list environment variables'),
)
@columns_argparses.columnar_arguments(_ENV_LIST_COLUMNS,
_ENV_LIST_DEFAULT_COLUMNS)
@argparses.end
def cmd_list(args):
columnar = columns.Columnar(**columns_argparses.make_columnar_kwargs(args))
for name, value in envs.load().items():
columnar.append({'name': name, 'value': value})
columnar.output(sys.stdout)
return 0
