def dump(cls, data, list_format: Text = None):
data = DictUtils.traverse(data,
cls.dump_value,
list_format=list_format)
data = DictUtils.remove_keys(data, empty_values={type(None)})
output = io.StringIO()
config = configparser.ConfigParser(
interpolation=configparser.ExtendedInterpolation())
config.read_dict(data)
config.write(output)
return output.getvalue()
def run(
request,
name: Text,
destination: Text = None,
cwd: Text = None,
**arguments,
):
if cwd:
cwd = expand_path(cwd)
cmd = Command.select(Command).where(name=name.lower())(first=True)
output = None
if cmd is None:
cli.log.error(f'command not found: {cmd}')
else:
if arguments:
arguments = DictUtils.unflatten_keys(
{k.lstrip('_'): v
for k, v in arguments.items()})
context = dict(cmd.defaults, **arguments)
context['embryo'] = {
'destination': destination or cmd.destination,
'name': cmd.embryo,
}
cmd.show(destination=destination)
embryo = Embryo.import_embryo(cmd.embryo, context)
embryo.hatch()
def inherit_base_manifest(
cls,
base: Text = None,
visited: Set = None,
data: Dict = None,
):
"""
If there is a base manifest file specified by the loaded manifest
data, then load it here and merge the current manifest into the base
manifest, recusively back to the root base.
"""
visited = set() if visited is None else visited
data = data if data is not None else {}
base = os.path.abspath(os.path.expandvars(os.path.expanduser(base)))
if base in visited:
# avoid infinite loop if there is a bad inheritence cycle
raise ManifestInheritanceError(
f'manifest inheritance loop detected: {base}')
else:
visited.add(base)
console.debug(f'inheriting manifest from {base}')
if os.path.isfile(base):
# merge current data dict into new base dict
base_data = cls._read_file(base)
data = DictUtils.merge(base_data.copy(), data)
# recurse on base's base manifest...
nested_base = base_data.get('base')
if nested_base:
cls.inherit_base_manifest(nested_base, visited, data)
return data
def update(self, _id=None, data: Dict = None) -> Dict:
"""
Submit changes to an object in the store.
"""
with self.lock:
old_record = self.records.get(_id, {})
old_rev = old_record.get(REV)
# remove the old copy of the object from the store, and then replace
# it. we use `internal` in the call to delete to indicate that we
# are only deleting for the purpose of re-inserting during this
# update. this prevents the _rev index from being cleared.
if old_record:
self.delete(
old_record[ID],
index_names=set(data.keys()),
internal=True,
)
merged_record = DictUtils.merge(old_record, data)
merged_record[REV] = self.increment_rev(old_rev)
# "re-insert" the new copy of the record
record = self.create(merged_record)
return record
def update(self, _id, data: Dict) -> Dict:
fpath = self.mkpath(_id)
base_record = self.ftype.read(
fpath,
loader_class=self.yaml_loader_class
)
schema = self.resource_type.ravel.schema
base_record, errors = schema.process(base_record)
if base_record:
# this is an upsert
if self.use_recursive_merge:
record = DictUtils.merge(base_record, data)
else:
record = dict(base_record, **data)
else:
record = data
record[REV] = self.increment_rev(record.get(REV))
if ID not in record:
record[ID] = _id
if self.store_primitives:
json = self.app.json
self.ftype.write(
path=fpath,
data=json.decode(json.encode(record))
)
else:
self.ftype.write(path=fpath, data=record)
self._cache_store.update(_id, record)
return record
def instance(self, fixture=None, *args, **kwargs):
"""
Build an object from defined class.
"""
if fixture and fixture in self.fixtures:
fixture_data = self.fixtures[fixture]
# apply kwargs on top of fixture data
kwargs = DictUtils.merge(fixture_data, kwargs)
return self.klass(*args, **kwargs)
def load(cls, data, list_format: Text = None):
config = configparser.ConfigParser()
config.read_string(data)
ini_data = dict(config._sections)
for k in ini_data:
ini_data[k] = dict(config._defaults, **ini_data[k])
ini_data[k].pop('__name__', None)
ini_data = DictUtils.traverse(ini_data,
cls.load_value,
list_format=list_format)
return ini_data
def build_kwargs(self, kwargs=None, split_newline=True):
"""
# Build Kwargs
"""
if not kwargs:
return []
flat_kwargs = DictUtils.flatten_keys(kwargs)
kwargz = [
'--{} {}'.format(k, v) for k, v in flat_kwargs.items()
if v is not None
]
if not self._split_kwargs:
return [' '.join(kwargz)]
return kwargz
def _load_manifest_data(cls, source):
schema = cls.Schema(allow_additional=True)
filepath = None
data = {}
# if source is callable, it means that the manifest is being returned
# by a function, allowing for dynamic manifest properties
if callable(source):
source = source()
# load the manifest data dict differently, depending
# on its source -- e.g. from a file path, a dict, another manifest
if isinstance(source, Manifest):
data = deepcopy(source.data)
filepath = source.filepath
elif isinstance(source, str):
filepath = os.path.abspath(
os.path.expandvars(os.path.expanduser(source)))
if os.path.isfile(source):
console.debug(f'reading manifest from {source}')
data = cls._read_file(source)
else:
raise ManifestFileNotFound(
f'manifest file {filepath} not found. '
f'yaml and json manifest file types are supported.')
elif isinstance(source, dict):
data = source
filepath = None
# merge data dict into recursively inherited data dict
base_filepath = data.get('base')
if base_filepath:
inherited_data = cls.inherit_base_manifest(base_filepath)
data = DictUtils.merge(inherited_data, data)
data = cls._expand_vars(data)
# validate final computed data dict
validated_data, errors = schema.process(data)
if errors:
raise ManifestValidationError(
f'manifest validation error/s: {errors}')
return (filepath, validated_data)
def parse_context(cls, context):
commands = []
if context is None:
context = {}
for embryo, contexts in context.items():
for context in contexts:
embryo_data = context['embryo']
del context['embryo']
flat_context = DictUtils.flatten_keys(context)
context_args = [
'--{} {}'.format(k, v) for k, v in flat_context.items()
if v is not None
]
command = 'embryo {} {} {}'.format(embryo_data['action'],
embryo,
' '.join(context_args))
timestamp = embryo_data.get('timestamp')
commands.append((timestamp, command))
return commands
def build(self, parent, custom_dtype_converter: Callable = None):
"""
"""
# add arguments
# remove empty values from kwargs. argparser will shit the
# bed if it finds one that is empty and shouldn't belong in
# conjunction with another Args' kwargs
kwargs = DictUtils.remove_keys(self.kwargs, values=[None])
dtype = kwargs.get('type')
if custom_dtype_converter:
dtype = custom_dtype_converter(dtype)
if dtype:
# in cases when an unknown type is provided, like perhaps a class,
# then assume a str type
basic_type_known = dtype in (str, int, dict, list)
registry_type_known = dtype in parent._parser._registries['type']
if not callable(dtype) and not any(
[basic_type_known, registry_type_known]):
kwargs['type'] = str
return parent.add_argument(*self.flags, **kwargs)
def _on_bootstrap_configure_grpc(self, options: Dict):
"""
Compute the gRPC options dict and other values needed to run this
application as a gRPC service.
"""
grpc = self.grpc
# get the python package containing this Application
pkg_path = self.manifest.package
pkg = import_module(self.manifest.package)
# compute some file and module paths
grpc.pkg_dir = os.path.dirname(pkg.__file__)
grpc.build_dir = os.path.join(grpc.pkg_dir, 'grpc')
grpc.proto_file = os.path.join(grpc.build_dir, REGISTRY_PROTO_FILE)
# compute grpc options dict from options kwarg and manifest data
kwarg_options = options or {}
manifest_options = self.manifest.data.get('grpc', {})
computed_options = DictUtils.merge(kwarg_options, manifest_options)
# generate default values into gRPC options data
schema = GrpcOptionsSchema()
options, errors = schema.process(computed_options)
if errors:
raise Exception(f'invalid gRPC options: {errors}')
grpc.options = DictObject(options)
grpc.options.port = port = options['port']
grpc.options.server_host = server_host = options['server_host']
grpc.options.client_host = client_host = options['client_host']
grpc.options.server_address = (f'{server_host}:{port}')
grpc.options.client_address = (f'{client_host}:{port}')
grpc.pb2_mod_path = PB2_MODULE_PATH_FSTR.format(pkg_path)
grpc.pb2_grpc_mod_path = PB2_GRPC_MODULE_PATH_FSTR.format(pkg_path)
# add build directory to PYTHONPATH; otherwise, gRPC won't know where
# the pb2 modules are when it tries to import them.
sys.path.append(grpc.build_dir)
def upsert(
request,
name: Text,
embryo: Text,
destination: Text = None,
cwd: Text = None,
overwrite: bool = True,
**defaults,
):
# get the existing command, provided it exists. if it exists,
# then we should perform an update below.
existing_cmd = Command.select(Command).where(name=name.lower()).execute(
first=True)
# process default values with embryo schema fields
embryo_obj = Embryo.import_embryo(embryo, {})
schema = embryo_obj.context_schema()
computed_defaults = embryo_obj.load_static_context()
for field in schema.fields.values():
if field.name in computed_defaults:
continue
default = field.default
if default is not None:
if callable(default):
computed_defaults[field.name] = default()
else:
computed_defaults[field.name] = copy(default)
if existing_cmd:
computed_defaults.update(existing_cmd.defaults)
computed_defaults.update(
DictUtils.unflatten_keys(
{k.lstrip('_'): v
for k, v in defaults.items()}))
for k, v in list(computed_defaults.items()):
field = schema.fields[k]
if isinstance(v, str) and v.upper() == 'NULL':
del computed_defaults[k]
elif v is None:
del computed_defaults[k]
else:
v_processed, error = schema.fields[k].process(v)
computed_defaults[k] = v_processed
if error:
cli.log.error(message='error validating embryo default',
data={
'field': k,
'value': v,
'reason': error
})
return
# new or existing Command object to upsert and return:
cmd = None
# perform update or create...
if existing_cmd:
cmd = existing_cmd
if not overwrite:
cli.log.warning(f'command already exists: {name}')
else:
if computed_defaults is not None:
cmd.defaults = computed_defaults
if cwd:
cmd.cwd = cwd
if embryo:
cmd.embryo = embryo
if destination:
cmd.destination = destination
cmd.update()
cli.log.info(f'updated existing command: {name}')
else:
cmd = Command(name=name,
embryo=embryo,
destination=destination,
defaults=computed_defaults,
cwd=cwd).create()
cli.log.info(f'created new command: {name}')
cmd.show()
return cmd
def bootstrap(
self,
manifest: Union[Dict, Manifest, Text, Callable, 'Application'] = None,
namespace: Dict = None,
values: Dict = None,
middleware: List = None,
mode: Mode = Mode.normal,
*args,
**kwargs
) -> 'Application':
"""
Bootstrap the data, business, and service layers, wiring them up.
"""
def create_logger(level):
"""
Setup root application logger.
"""
if self.manifest.package:
name = (
f'{self.manifest.package}:'
f'{os.getpid()}:'
f'{get_ident()}'
)
else:
class_name = get_class_name(self)
name = (
f'{StringUtils.snake(class_name)}:'
f'{os.getpid()}:'
f'{get_ident()}'
)
return ConsoleLoggerInterface(name, level)
# warn about already being bootstrapped...
if (
self.is_bootstrapped and
self.local.thread_id != get_ident()
):
console.warning(
message=f'{get_class_name(self)} already bootstrapped.',
data={
'pid': os.getpid(),
'thread': threading.get_ident(),
}
)
return self
console.debug(f'bootstrapping {get_class_name(self)} app')
# override the application mode set in constructor
self._mode = Mode(mode or self.mode or Mode.normal)
# merge additional namespace data into namespace dict
self._namespace = DictUtils.merge(self._namespace, namespace or {})
# register additional middleware targing this app subclass
if middleware:
self._middleware.extend(
m for m in middleware if isinstance(self, m.app_types)
)
# build final manifest, used to bootstrap program components
if manifest is not None:
if isinstance(manifest, Application):
source_app = manifest
self.local.manifest = Manifest(source_app.shared.manifest_data)
else:
self.local.manifest = Manifest(manifest)
elif self.shared.manifest_data:
self.local.manifest = Manifest(self.shared.manifest_data)
else:
self.local.manifest = Manifest({})
assert self.local.manifest is not None
self.manifest.bootstrap(self)
if values:
self.manifest.values.update(values)
if not self.shared.manifest_data:
self.shared.manifest_data = self.local.manifest.data
# set up main thread name
if self.manifest.package:
current_thread().name = (
f'{StringUtils.camel(self.manifest.package)}'
)
else:
# update default main thread name
current_thread().name = (
f'{get_class_name(self)}'
)
self._logger = create_logger(self.manifest.logging['level'])
self._arg_loader = ArgumentLoader(self)
# if we're in a new process, unset the executors so that
# the spawn method lazily triggers the instantiation of
# new ones at runtime.
if not self._root_pid != os.getpid():
self.local.thread_executor = None
self.local.process_executor = None
self.local.thread_id = get_ident()
self.local.tx_manager = TransactionManager(self)
# execute custom lifecycle hook provided by this subclass
self.on_bootstrap(*args, **kwargs)
self.local.is_bootstrapped = True
console.debug(f'finished bootstrapping {get_class_name(self)} app')
return self
def _merge_context(self, left: Dict = None, right: Dict = None):
# remove none type values from the dict to be merged in
DictUtils.remove_keys(right, values=[type(None)], in_place=True)
# merge base with our cleaned context
return DictUtils.merge(left, right)
def parse_cli_args(
self,
args: list = None,
merge_unknown: bool = True,
unflatten_keys: bool = True,
) -> Tuple['Arguments', List]:
"""
# Parse arguments from command-line
"""
# let argparser do the initial parsing
cli_args, cli_unknown_args = self._parser.parse_known_args(args)
# now combine known and unknown arguments into a single dict
args_dict = {
k: getattr(cli_args, k)
for k in dir(cli_args) if not k.startswith('_') and k != 'func'
}
# XXX we want the func reference as this points directly to the
# subparsers perform, and we don't want it in the cli args, and
# handling it should probably not go here anyway, but it is
if hasattr(cli_args, 'func'):
self._perform = cli_args.func
# process unknown args
unknown_args = []
unknown_kwargs = {}
skip_next = False
for i in range(0, len(cli_unknown_args)):
k = cli_unknown_args[i]
kid = k.lstrip('-')
has_dash = (k[0] == '-') if k else False
is_arg = not has_dash and not skip_next
if skip_next:
skip_next = False
continue
if is_arg:
unknown_args.append(k)
else:
try:
v = cli_unknown_args[i + 1]
unknown_kwargs[kid] = v
skip_next = True
except Exception as err:
logger.info(f'unmatched kwarg "{kid}"')
if merge_unknown:
# and any unknown args pairs will get added
args_dict.update(unknown_kwargs)
if unflatten_keys:
# this will expand any keys that are dot notated with the
# expectation of being a nested dictionary reference
args_dict = DictUtils.unflatten_keys(args_dict)
if self._schema:
processed_args_dict = self._schema.process(args_dict, strict=True)
for k, v in processed_args_dict.items():
args_dict[k] = v
arguments = type('Arguments', (object, ), args_dict)()
arguments.unknown = unknown_kwargs
return arguments, cli_unknown_args