def make_create_engine_params(db_url=None, dialect=None, **kwargs):
if dialect is None:
dialect = _get_dialect(ASSERT.not_none(db_url))
if dialect == 'postgresql':
params = parameters.Namespace(
'make PostgreSQL database engine',
db_url=parameters.make_parameter(db_url, str),
dialect=parameters.ConstParameter(dialect),
)
elif dialect == 'sqlite':
params = parameters.Namespace(
'make SQLite database engine',
db_url=parameters.make_parameter(db_url, str),
dialect=parameters.ConstParameter(dialect),
check_same_thread=parameters.Parameter(kwargs.get(
'check_same_thread', True),
type=bool),
trace=parameters.Parameter(kwargs.get('trace'),
type=(bool, type(None))),
pragmas=parameters.Parameter(kwargs.get('pragmas', ()),
type=collections.abc.Iterable),
)
else:
ASSERT.unreachable('unsupported dialect: {}', dialect)
return params
def main(
args: apps_bases.LABELS.args,
_: g1.scripts.parts.LABELS.setup,
):
"""Manage containerized application."""
if args.entity == 'init':
bases.cmd_init()
builders.cmd_init()
images.cmd_init()
pods.cmd_init()
xars.cmd_init()
elif args.entity == 'cleanup':
# Clean up pods and xars before images because they depend on
# images but not vice versa.
grace_period_kwargs = bases.make_grace_period_kwargs(args)
pods.cmd_cleanup(**grace_period_kwargs)
xars.cmd_cleanup()
images.cmd_cleanup(**grace_period_kwargs)
elif args.entity == 'images':
return cmd_images(args)
elif args.entity == 'pods':
return cmd_pods(args)
elif args.entity == 'xars':
return cmd_xars(args)
else:
ASSERT.unreachable('unknown entity: {}', args.entity)
return 0
def cmd_xars(args):
if args.command == 'install':
xars.cmd_install(
**images.make_select_image_kwargs(args),
xar_name=args.name,
exec_relpath=args.exec,
)
elif args.command == 'list':
columnar = columns.Columnar(
**columns_argparses.make_columnar_kwargs(args),
stringifiers=xars.XAR_LIST_STRINGIFIERS,
)
for row in xars.cmd_list():
columnar.append(row)
columnar.sort(
lambda row: (row['xar'], row['name'], row['version'], row['id'])
)
columnar.output(sys.stdout)
elif args.command == 'exec':
xars.cmd_exec(args.name, args.args)
elif args.command == 'uninstall':
xars.cmd_uninstall(args.name)
elif args.command == 'cleanup':
xars.cmd_cleanup()
else:
ASSERT.unreachable('unknown xar command: {}', args.command)
return 0
def extract(archive_path, *, directory=None):
archive_path = Path(archive_path)
archive_type = utils.guess_archive_type(archive_path.name)
if archive_type is utils.ArchiveTypes.TAR:
tar_extract(archive_path, directory=directory)
elif archive_type is utils.ArchiveTypes.ZIP:
unzip(archive_path, directory=directory)
else:
ASSERT.unreachable('unknown archive type: {}', archive_path)
def cmd_pods(args):
if args.command == 'list':
columnar = columns.Columnar(
**columns_argparses.make_columnar_kwargs(args),
stringifiers=pods.POD_LIST_STRINGIFIERS,
)
for row in pods.cmd_list():
columnar.append(row)
columnar.sort(lambda row: (row['name'], row['version'], row['id']))
columnar.output(sys.stdout)
elif args.command == 'show':
columnar = columns.Columnar(
**columns_argparses.make_columnar_kwargs(args),
stringifiers=pods.POD_SHOW_STRINGIFIERS,
)
for row in pods.cmd_show(args.id):
columnar.append(row)
columnar.sort(lambda row: row['name'])
columnar.output(sys.stdout)
elif args.command == 'cat-config':
pods.cmd_cat_config(args.id, sys.stdout.buffer)
elif args.command == 'generate-id':
pods.cmd_generate_id(sys.stdout)
elif args.command == 'run':
pods.cmd_run(
pod_id=args.id or models.generate_pod_id(),
config_path=args.config,
debug=get_debug(),
)
elif args.command == 'prepare':
pods.cmd_prepare(
pod_id=args.id or models.generate_pod_id(),
config_path=args.config,
)
elif args.command == 'run-prepared':
pods.cmd_run_prepared(pod_id=args.id, debug=get_debug())
elif args.command == 'add-ref':
pods.cmd_add_ref(pod_id=args.id, target_path=args.target)
elif args.command == 'export-overlay':
pods.cmd_export_overlay(
pod_id=args.id,
output_path=args.output,
filter_patterns=args.filter or (),
debug=get_debug(),
)
elif args.command == 'remove':
pods.cmd_remove(args.id)
elif args.command == 'cleanup':
pods.cmd_cleanup(**bases.make_grace_period_kwargs(args))
else:
ASSERT.unreachable('unknown pod command: {}', args.command)
return 0
def parse_images_parameter(value):
images = []
for v in value.split(','):
if v.startswith('id:'):
images.append(models.PodConfig.Image(id=v[len('id:'):]))
elif v.startswith('nv:'):
_, name, version = v.split(':', maxsplit=3)
images.append(models.PodConfig.Image(name=name, version=version))
elif v.startswith('tag:'):
images.append(models.PodConfig.Image(tag=v[len('tag:'):]))
else:
ASSERT.unreachable('unknown image parameter: {}', v)
return images
def _update(self):
ASSERT.not_none(self._loader)
LOG.debug('update schema tables')
id_to_schema = {
schema.proto.id: schema
for schema in map(Schema, self._loader.getAllLoaded())
}
for schema in id_to_schema.values():
if schema.proto.is_file():
path = schema.proto.display_name
if path not in self.files:
LOG.debug('add file node: %s', path)
self.files[path] = schema
continue
if schema.proto.is_struct():
table = self.struct_schemas
elif schema.proto.is_enum():
table = self.enum_schemas
elif schema.proto.is_interface():
table = self.interface_schemas
elif schema.proto.is_const():
table = self.const_schemas
elif schema.proto.is_annotation():
table = self.annotations
else:
ASSERT.unreachable('unexpected schema kind: {}', schema)
label = labels.Label(
self._get_module_path(schema, id_to_schema),
self._get_object_path(schema, id_to_schema),
)
if label in table:
continue
LOG.debug('add schema: %s', label)
if schema.proto.is_struct():
table[label] = schema.as_struct()
elif schema.proto.is_enum():
table[label] = schema.as_enum()
elif schema.proto.is_interface():
table[label] = schema.as_interface()
elif schema.proto.is_const():
table[label] = schema.as_const()
elif schema.proto.is_annotation():
table[label] = schema
else:
ASSERT.unreachable('unexpected schema kind: {}', schema)
def _git_get_url(source):
proc = scripts.run(['git', 'remote', '--verbose'])
for remote in proc.stdout.decode('utf-8').split('\n'):
remote = remote.split()
if remote[0] == 'origin':
return remote[1]
return ASSERT.unreachable('expect remote origin: {}', source)
def _make_to_lower(item_type):
# Handle non-pointer types first.
result = _PRIMITIVE_TYPES.get(item_type.which)
if result:
return functools.partial(_primitive_to_lower, result[0], result[2])
if item_type.is_enum():
return functools.partial(_enum_to_lower, item_type.as_enum())
# Handle pointer types.
if item_type.is_text():
return _text_to_lower
if item_type.is_data():
return _data_to_lower
if item_type.is_list():
return _list_to_lower
if item_type.is_struct():
return _struct_to_lower
if item_type.is_interface():
raise NotImplementedError('do not support interface for now')
if item_type.is_any_pointer():
raise NotImplementedError('do not support any-pointer for now')
return ASSERT.unreachable('unexpected item type: {}', item_type)
def main(args):
if args.command == 'init':
return cmd_init(args)
elif args.command == 'cleanup':
return cmd_cleanup()
else:
return ASSERT.unreachable('unknown command: {}', args.command)
def _load_data(cls, config_data):
raw_config = json.loads(config_data)
base_config = g1_dataclasses.fromdict(cls, raw_config)
destination_type = Config.Destination
if base_config.destination.kind == 'null':
destination_type = Config.NullDestination
elif base_config.destination.kind == 'slack':
destination_type = Config.SlackDestination
else:
ASSERT.unreachable('unknown destination: {}', raw_config)
return dataclasses.replace(
base_config,
destination=g1_dataclasses.fromdict(
destination_type,
raw_config['destination'],
),
)
def main(args):
if args.command == 'list':
return cmd_list(args)
elif args.command == 'install':
return cmd_install(args)
elif args.command == 'uninstall':
return cmd_uninstall(args)
else:
return ASSERT.unreachable('unknown command: {}', args.command)
def _parse_checksum(checksum):
for prefix, command in (
('md5:', 'md5sum'),
('sha256:', 'sha256sum'),
('sha512:', 'sha512sum'),
):
if checksum.startswith(prefix):
return command, checksum[len(prefix):]
return ASSERT.unreachable('unknown checksum algorithm: {}', checksum)
def main(args):
if args.command == 'list':
return cmd_list(args)
elif args.command == 'set':
return cmd_set(args)
elif args.command == 'remove':
return cmd_remove(args)
else:
return ASSERT.unreachable('unknown command: {}', args.command)
def find_package(parameters, relpath):
"""Find path to a first-party package."""
root_paths = parameters['//bases:roots']
for root_path in root_paths:
path = root_path / relpath
if (path / 'setup.py').is_file():
return path
return ASSERT.unreachable('cannot find package {} under: {}', relpath,
root_paths)
def _find_project(parameters, relpath):
root_paths = parameters['//bases:roots']
for root_path in root_paths:
path = root_path / relpath
if (path / 'build.gradle').exists():
return path
return ASSERT.unreachable(
'cannot find project {} under: {}', relpath, root_paths
)
def make(option, *, mode=None):
"""Make a property for accessing an option.
It assumes that the Python object defines these attributes:
`_handle`, `_getopt_prefix`, and `_setopt_prefix`.
"""
name, type_name, default_mode = option
if mode is None:
mode = default_mode
# TODO: Handle ptr type.
if type_name in ('bool', 'int', 'size', 'uint64'):
getter = get_simple
setter = set_simple
elif type_name == 'ms':
getter = get_duration
setter = set_duration
elif type_name == 'bytes':
getter = get_bytes
setter = set_bytes
elif type_name == 'string':
getter = get_string
setter = set_string
elif type_name == 'addr':
getter = get_addr
setter = set_addr
else:
ASSERT.unreachable('unsupported option type: {}', type_name)
kwargs = {}
if mode in ('ro', 'rw'):
kwargs['fget'] = functools.partial(getter,
name=name,
type_name=type_name)
if mode in ('wo', 'rw'):
kwargs['fset'] = functools.partial(setter,
name=name,
type_name=type_name)
return property(**ASSERT.not_empty(kwargs))
def find_package(parameters, relpath, sub_directory_path=None):
"""Find path to a first-party package."""
root_paths = parameters['//bases:roots']
for root_path in root_paths:
path = root_path / relpath
if sub_directory_path:
path /= sub_directory_path
if (path / 'package.json').is_file():
return path
return ASSERT.unreachable('cannot find package {} under: {}', relpath,
root_paths)
def select(obj):
none_field = None
for field in dataclasses.fields(obj):
value = getattr(obj, field.name)
if value is not None:
return field.name, value
elif typings.type_is_subclass(field.type, NoneType):
none_field = field.name
if none_field:
return none_field, None
return ASSERT.unreachable('expect one non-None field: {!r}', obj)
def cmd_images(args):
if args.command == 'build-base':
builders.cmd_build_base_image(
args.name, args.version, args.output, args.prune_stash_path
)
elif args.command == 'prepare-base-rootfs':
builders.cmd_prepare_base_rootfs(args.path)
elif args.command == 'setup-base-rootfs':
builders.cmd_setup_base_rootfs(args.path, args.prune_stash_path)
elif args.command == 'build':
images.cmd_build_image(
args.name, args.version, args.rootfs, args.output
)
elif args.command == 'import':
images.cmd_import(args.path, tag=args.tag)
elif args.command == 'list':
columnar = columns.Columnar(
**columns_argparses.make_columnar_kwargs(args),
stringifiers=images.IMAGE_LIST_STRINGIFIERS,
)
for row in images.cmd_list():
columnar.append(row)
columnar.sort(lambda row: (row['name'], row['version'], row['id']))
columnar.output(sys.stdout)
elif args.command == 'tag':
images.cmd_tag(
**images.make_select_image_kwargs(args),
new_tag=args.new_tag,
)
elif args.command == 'remove-tag':
images.cmd_remove_tag(args.tag)
elif args.command == 'remove':
images.cmd_remove(
**images.make_select_image_kwargs(args),
skip_active=args.skip_active,
)
elif args.command == 'cleanup':
images.cmd_cleanup(**bases.make_grace_period_kwargs(args))
else:
ASSERT.unreachable('unknown image command: {}', args.command)
return 0
def _run_pod(pod_id, *, debug=False):
LOG.info('start pod: %s', pod_id)
pod_dir_path = ASSERT.predicate(_get_pod_dir_path(pod_id), Path.is_dir)
rootfs_path = _get_rootfs_path(pod_dir_path)
config = _read_config(pod_dir_path)
builders.clear_pod_app_exit_status(rootfs_path)
#
# * For now we do not worry too much about cross-platform issues; we
# assume that the pod is running in a host system with
# systemd-nspawn, journald, etc.
#
# * For now we do not support interactive pod.
#
args = [
# Use systemd-nspawn of the host system (alternatively, we may
# install and use systemd-nspawn in the base image, which is
# probably more cross-platform).
'systemd-nspawn',
'--uuid=%s' % pod_id,
'--machine=%s' % models.generate_machine_name(pod_id),
'--register=yes',
*(['--keep-unit'] if _is_running_from_system_service() else []),
'--boot',
'--directory=%s' % rootfs_path,
*(_make_bind_argument(mount) for mount in config.mounts),
*(_make_overlay_argument(overlay) for overlay in config.overlays),
'--notify-ready=yes',
'--link-journal=try-host',
*([] if debug else ['--quiet']),
'--',
# I don't know why, but if you set
# --default-standard-output=journal+console
# then ``machinectl shell`` will not work.
*([] if debug else [
'--log-target=null',
'--show-status=no',
]),
]
LOG.debug('run pod: args=%s', args)
os.execvp(args[0], args)
ASSERT.unreachable('unable to start pod: {}', pod_id)
def _tar_guess_compressor(tarball_name):
compressor = utils.guess_compressor(tarball_name)
if compressor is utils.Compressors.UNCOMPRESSED:
return ()
elif compressor is utils.Compressors.BZIP2:
return ('--bzip2', )
elif compressor is utils.Compressors.GZIP:
return ('--gzip', )
elif compressor is utils.Compressors.XZ:
return ('--xz', )
else:
return ASSERT.unreachable('unknown tarball suffix: {}', tarball_name)
def _make_grave_path(self, ops_dir_name):
grave_path = self.graveyard_dir_path / ops_dir_name
if not grave_path.exists():
return grave_path
for _ in range(3):
grave_path = (
self.graveyard_dir_path / \
('%s.%08d' % (ops_dir_name, random.randint(0, 1e8)))
)
if not grave_path.exists():
return grave_path
return ASSERT.unreachable('unable to generate unique grave path')
def main(args):
if args.command == 'set-config':
return cmd_set_config(args)
elif args.command == 'send':
return cmd_send(args)
elif args.command == 'watch-syslog':
return cmd_watch_syslog(args)
elif args.command == 'watch-journal':
return cmd_watch_journal(args)
elif args.command == 'collectd':
return cmd_collectd(args)
else:
return ASSERT.unreachable('unknown command: {}', args.command)
def make_create_engine(params):
kwargs = {
'db_url': params.db_url.get(),
}
dialect = params.dialect.get()
if dialect == 'postgresql':
create_engine = postgresql.create_engine
elif dialect == 'sqlite':
create_engine = sqlite.create_engine
trace = params.trace.get()
if trace is None:
trace = logging.getLogger().isEnabledFor(loggers.TRACE)
kwargs['check_same_thread'] = params.check_same_thread.get()
kwargs['trace'] = trace
kwargs['pragmas'] = params.pragmas.get()
else:
ASSERT.unreachable('unsupported dialect: {}', dialect)
return functools.partial(create_engine, **kwargs)
def main(args):
if args.command == 'list-definitions':
return cmd_list_definitions(args)
elif args.command == 'list-assignments':
return cmd_list_assignments(args)
elif args.command == 'define':
return cmd_define(args)
elif args.command == 'undefine':
return cmd_undefine(args)
elif args.command == 'assign':
return cmd_assign(args)
elif args.command == 'unassign':
return cmd_unassign(args)
else:
return ASSERT.unreachable('unknown command: {}', args.command)
def main(args: apps_bases.LABELS.args, _: g1.scripts.parts.LABELS.setup):
"""Operations tool."""
if args.subject == 'repos':
return repo_cmds.main(args)
elif args.subject == 'alerts':
return alert_cmds.main(args)
elif args.subject == 'pods':
return pod_cmds.main(args)
elif args.subject == 'xars':
return xar_cmds.main(args)
elif args.subject == 'envs':
return env_cmds.main(args)
elif args.subject == 'tokens':
return token_cmds.main(args)
else:
return ASSERT.unreachable('unknown subject: {}', args.subject)
def _add_sort(table, query, sort_):
if sort_.sort_by is interfaces.SortBys.NONE:
return query
if sort_.sort_by is interfaces.SortBys.REVISION:
column = table.c.revision
elif sort_.sort_by is interfaces.SortBys.KEY:
column = table.c.key
elif sort_.sort_by is interfaces.SortBys.VALUE:
# How are NULL values ordered?
column = table.c.value
else:
return ASSERT.unreachable('unknown sort by: {}', sort_)
if sort_.ascending:
query = query.order_by(column.asc())
else:
query = query.order_by(column.desc())
return query
def ctr_get_image_rootfs_path(image):
if image.id is not None:
match = lambda row: row[0] == image.id
elif image.name is not None and image.version is not None:
match = lambda row: row[1] == image.name and row[2] == image.version
else:
ASSERT.not_none(image.tag)
match = lambda row: image.tag in row[3].split(' ')
with scripts.doing_capture_stdout():
proc = ctr([
'images',
'list',
*('--format', 'csv'),
*('--columns', 'id,name,version,tags,rootfs'),
])
for row in csv.reader(io.StringIO(proc.stdout.decode('utf-8'))):
if match(row):
return row[4]
return ASSERT.unreachable('cannot find image: {}', image)
def _make_to_upper(item_type, is_reader):
# Handle non-pointer types first.
result = _PRIMITIVE_TYPES.get(item_type.which)
if result:
return functools.partial(_primitive_to_upper, result[1])
if item_type.is_enum():
return _enum_to_upper
# Handle pointer types.
if item_type.is_text():
return _text_to_upper
if item_type.is_data():
return _data_to_upper
if item_type.is_list():
return functools.partial(
_list_to_upper,
# TODO: Sadly, this will break users who subclass
# DynamicListReader or DynamicListBuilder (same below) as we
# hard code types here.
DynamicListReader if is_reader else DynamicListBuilder,
item_type.as_list(),
)
if item_type.is_struct():
return functools.partial(
_struct_to_upper,
DynamicStructReader if is_reader else DynamicStructBuilder,
item_type.as_struct(),
)
if item_type.is_interface():
raise NotImplementedError('do not support interface for now')
if item_type.is_any_pointer():
raise NotImplementedError('do not support any-pointer for now')
return ASSERT.unreachable('unexpected item type: {}', item_type)