def _get_apps(parameters, builder_images, root_host_paths, rules):
builder_script = []
if not builder_images:
LOG.info('no intermediate builder images; initialize builder')
builder_script.extend(_INITIALIZE_BUILDER)
if rules:
builder_script.append(
# For reasons that I do not understand, apt-get sometimes
# cannot log its stdout/stderr to journal and will crash; so
# we redirect everything to a file to work around this.
' '.join([
'sudo',
*('-u', 'plumber'),
*('-g', 'plumber'),
'/usr/src/garage/shipyard2/scripts/foreman.sh',
'build',
*(('--debug', ) if shipyard2.is_debug() else ()),
*_foreman_make_path_args(root_host_paths),
*('--parameter', '//bases:inside-builder-pod=true'),
*_foreman_make_parameters(parameters),
*map(str, rules),
'> "$(mktemp --tmpdir=/tmp build-XXXXXXXXXX.log)" 2>&1',
])
)
ASSERT.not_empty(builder_script)
return [
{
'name': 'builder',
'type': 'oneshot',
'exec': ['/bin/bash', '-c', '; '.join(builder_script)],
'user': '******',
'group': 'root',
},
]
def lease_get_key_ids(tables, *, leases):
ASSERT.not_empty(leases)
table = tables.leases_key_ids
return (
select([table.c.key_id], distinct=True)\
.where(table.c.lease.in_(leases))
)
def __post_init__(self):
ASSERT.not_empty(self.sources)
for i, source in enumerate(self.sources):
# Empty source path means host's /var/tmp.
if source:
ASSERT.predicate(Path(source), Path.is_absolute)
else:
ASSERT.equal(i, len(self.sources) - 1)
ASSERT.predicate(Path(self.target), Path.is_absolute)
def __post_init__(self):
validate_pod_name(self.name)
validate_pod_version(self.version)
ASSERT.not_empty(self.images)
ASSERT.unique(app.name for app in self.apps)
ASSERT.unique(
[mount.target for mount in self.mounts] + \
[overlay.target for overlay in self.overlays]
)
def lease_scan_leases(tables, *, key_ids):
ASSERT.not_empty(key_ids)
table = tables.leases_key_ids
return (
select([table.c.key_id, table.c.lease])\
.where(table.c.key_id.in_(key_ids))
# Order by key_id for itertools.groupby.
.order_by(table.c.key_id.asc())
)
def __decode_buffer(self):
ASSERT.not_empty(self.__buffer)
data = b''.join(self.__buffer)
self.__buffer.clear()
if data.endswith(b'\r\n'):
data = data[:-2]
elif data.endswith(b'\n'):
data = data[:-1]
self.__lines.append(data.decode('utf-8'))
def __post_init__(self):
validate_app_name(self.name)
if self.service_section is None:
ASSERT.not_empty(self.exec)
ASSERT.in_(self.type, _SERVICE_TYPES)
ASSERT.in_(self.kill_mode, _KILL_MODES)
else:
ASSERT.empty(self.exec)
ASSERT.none(self.type)
ASSERT.none(self.kill_mode)
def scan_key_ids(tables, *, key_ids):
ASSERT.not_empty(key_ids)
table = tables.keyspace
return (
select([
table.c.revision,
table.c.key,
table.c.value,
])\
.where(table.c.key_id.in_(key_ids))
)
def __init__(
self,
cluster_stubs,
executor=None,
num_pools=0,
num_connections_per_pool=0,
):
ASSERT.not_empty(cluster_stubs)
self._base_session = bases.BaseSession(
executor=executor,
num_pools=num_pools,
num_connections_per_pool=num_connections_per_pool,
)
self._cluster_stubs = cluster_stubs
for cluster_stub in self._cluster_stubs:
ASSERT.none(cluster_stub._base_session)
cluster_stub._base_session = self._base_session
def config(parameters):
config_path = _get_config_path(parameters)
scripts.mkdir(config_path.parent)
config_path.write_text(
json.dumps({
'libraries':
sorted(ASSERT.not_empty(parameters['libraries'])),
})
)
def ops_list_pod_units(columns=('label', 'version', 'id', 'name')):
with scripts.doing_capture_stdout():
proc = ops([
'pods',
'list',
*('--format', 'csv'),
*('--header', 'true'),
*('--columns', ','.join(ASSERT.not_empty(columns))),
])
return list(csv.DictReader(io.StringIO(proc.stdout.decode('utf-8'))))
def __init__(
self,
table_name,
key_column_names_and_types=(),
value_column_names_and_types=(('data', LargeBinary), ),
*,
metadata=None,
index_column_names=(), # This is actually a list of names.
constraints=(),
):
ASSERT.isdisjoint(
frozenset(name for name, _ in key_column_names_and_types),
frozenset(name for name, _ in value_column_names_and_types),
)
# We do not support the valueless use case for now.
ASSERT.not_empty(value_column_names_and_types)
self.metadata = MetaData() if metadata is None else metadata
self.table_name = table_name
self.key_column_names = tuple(
name for name, _ in key_column_names_and_types)
self.key_columns = tuple(
Column(*pair, nullable=False, primary_key=True)
for pair in key_column_names_and_types)
self.value_column_names = tuple(
name for name, _ in value_column_names_and_types)
self.value_columns = tuple(
Column(*pair, nullable=False)
for pair in value_column_names_and_types)
self.index_column_names = index_column_names
self.table = Table(
table_name,
self.metadata,
*self.key_columns,
*self.value_columns,
*constraints,
)
def make(option, *, mode=None):
"""Make a property for accessing an option.
It assumes that the Python object defines these attributes:
`_handle`, `_getopt_prefix`, and `_setopt_prefix`.
"""
name, type_name, default_mode = option
if mode is None:
mode = default_mode
# TODO: Handle ptr type.
if type_name in ('bool', 'int', 'size', 'uint64'):
getter = get_simple
setter = set_simple
elif type_name == 'ms':
getter = get_duration
setter = set_duration
elif type_name == 'bytes':
getter = get_bytes
setter = set_bytes
elif type_name == 'string':
getter = get_string
setter = set_string
elif type_name == 'addr':
getter = get_addr
setter = set_addr
else:
ASSERT.unreachable('unsupported option type: {}', type_name)
kwargs = {}
if mode in ('ro', 'rw'):
kwargs['fget'] = functools.partial(getter,
name=name,
type_name=type_name)
if mode in ('wo', 'rw'):
kwargs['fset'] = functools.partial(setter,
name=name,
type_name=type_name)
return property(**ASSERT.not_empty(kwargs))
def lease_delete_key_ids(tables, *, key_ids):
ASSERT.not_empty(key_ids)
table = tables.leases_key_ids
return table.delete().where(table.c.key_id.in_(key_ids))
def delete_key_ids(tables, *, key_ids):
ASSERT.not_empty(key_ids)
table = tables.keyspace
return table.delete().where(table.c.key_id.in_(key_ids))
def __init__(self, handlers, *, default_handler=_raise_404):
ASSERT.not_empty(handlers)
self._handlers = [(re.compile(p), h) for p, h in handlers]
self._default_handler = default_handler
def apt_get_install(packages, *, assume='yes'):
_apt_get('install', assume, ASSERT.not_empty(packages))
def lease_delete_leases(tables, *, leases):
ASSERT.not_empty(leases)
return [
table.delete().where(table.c.lease.in_(leases))
for table in (tables.leases, tables.leases_key_ids)
]
def __init__(self, handlers, *, auto_options=True):
# Make a copy before modifying it.
self._handlers = ASSERT.not_empty(handlers).copy()
if auto_options:
self._handlers.setdefault(consts.METHOD_OPTIONS, self._options)
self._allow = ', '.join(sorted(self._handlers))
