def resolve(cls,
reference: typing.Optional[str] = None) -> 'Sink.Mode':
"""Parse the SINK section returning the tuple of sink configs for the particular modes.
Args:
reference: Optional sync reference - if provided, its used for all modes.
Returns:
Sink.Mode tuple with selected Sink config instances for the particular modes.
"""
if not reference:
try:
default = conf.PARSER[cls.INDEX].get(conf.OPT_DEFAULT)
train = conf.PARSER[cls.INDEX].get(conf.OPT_TRAIN, default)
apply = conf.PARSER[cls.INDEX].get(conf.OPT_APPLY, default)
evaluate = conf.PARSER[cls.INDEX].get(
conf.OPT_EVAL, default)
except KeyError as err:
raise error.Missing(
f'Index section not found: [{cls.INDEX}]') from err
if not train or not apply or not evaluate:
raise error.Missing(
f'Missing default or explicit train/apply/eval sink references: [{cls.INDEX}]'
)
else:
train = apply = evaluate = reference
return cls([
Sink.resolve(train),
Sink.resolve(apply),
Sink.resolve(evaluate)
])
def tune(
cls,
project: typing.Optional[str],
lineage: typing.Optional[str],
generation: typing.Optional[str],
runner: typing.Optional[str],
registry: typing.Optional[str],
feed: typing.Optional[typing.Sequence[str]],
sink: typing.Optional[str],
lower: typing.Optional[kind.Native],
upper: typing.Optional[kind.Native],
) -> None:
"""Tune mode execution.
Args:
project: Name of project to be tuned.
lineage: Lineage version to be tuned.
generation: Generation index to be tuned.
runner: Optional runner reference.
registry: Optional registry reference.
feed: Optional feed references.
sink: Optional sink reference.
lower: Lower ordinal.
upper: Upper ordinal.
"""
raise error.Missing(f'Tuning project {project}... not implemented')
def init(cls, name: str) -> None:
"""New project setup.
Args:
name: Project name to create.
"""
raise error.Missing(f'Creating project {name}... not implemented')
def __new__(
cls,
instruction: Instruction,
arguments: typing.Optional[typing.Sequence[Instruction]] = None):
if arguments is None:
arguments = []
if not all(arguments):
raise error.Missing('All arguments required')
return super().__new__(cls, instruction, tuple(arguments))
def __new__(cls, reference: str):
try:
kwargs = conf.PARSER[cls.GROUP][reference] # pylint: disable=no-member
except KeyError as err:
raise error.Missing(
f'Config section not found: [{cls.GROUP}.{reference}]'
) from err
args, kwargs = cls._extract(reference, kwargs)
return super().__new__(
cls, [*args, types.MappingProxyType(dict(kwargs))])
def tune( # pylint: disable=no-self-use
self, lower: typing.Optional['kind.Native'] = None, upper: typing.Optional['kind.Native'] = None
) -> None:
"""Run the tune mode.
Args:
lower: Ordinal value as the lower bound for the ETL cycle.
upper: Ordinal value as the upper bound for the ETL cycle.
"""
raise error.Missing('Not yet supported')
def __getitem__(cls, reference: typing.Any) -> typing.Type['Interface']:
if not isinstance(reference, str) and issubclass(cls, typing.Generic):
return cls.__class_getitem__(reference)
try:
return REGISTRY[cls].get(Reference(reference))
except KeyError as err:
known = ', '.join(str(c) for c in cls) # pylint: disable=not-an-iterable
raise error.Missing(
f'No {cls.__name__} provider registered as {reference} (known providers: {known})'
) from err
def resolve(cls, reference: typing.Optional[str] = None) -> 'Section':
"""Get config list for pattern based non-repeated option tokens.
Args:
reference: Config reference.
Returns:
Config instance.
"""
reference = reference or conf.PARSER.get(cls.INDEX, {}).get(
cls.SELECTOR)
if not reference:
raise error.Missing(
f'No default reference [{cls.INDEX}].{cls.SELECTOR}')
return cls._lookup(reference)
def _evaluation(
self, lower: typing.Optional['kind.Native'] = None, upper: typing.Optional['kind.Native'] = None
) -> pipeline.Segment:
"""Return the evaluation pipeline.
Args:
lower: Ordinal value as the lower bound for the ETL cycle.
upper: Ordinal value as the upper bound for the ETL cycle.
Returns:
Evaluation pipeline.
"""
if not self._assets.project.evaluation:
raise error.Missing('Project not evaluable')
return self._build(lower, upper, self._assets.project.pipeline >> self._assets.project.evaluation)
def decorator(cls) -> typing.Type[task.Actor]:
"""Decorating function."""
if not inspect.isclass(cls):
raise ValueError(f'Invalid actor class {cls}')
if issubclass(cls, task.Actor):
return cls
for target in {
t
for s, t in mapping.items()
if s != task.Actor.train.__name__
}:
if not callable(getattr(cls, target, None)):
raise error.Missing(
f'Wrapped actor missing required {target} implementation'
)
return Class(cls, mapping)
def match(self, source: 'frame.Source') -> Provider:
"""Select a feed that can provide for (be used to construct) the given source.
Args:
source: ETL frame source to be run against the required feed.
Returns:
Feed that's able to provide data for the given sources.
"""
for feed in self:
matcher = self.Matcher(feed.sources)
source.accept(matcher)
if matcher:
break
else:
raise error.Missing(
f'None of the {len(self._feeds)} available feeds provide all of the required sources'
)
return feed # pylint: disable=undefined-loop-variable
def read(
cls,
path: typing.Optional[typing.Union[str, pathlib.Path]] = None
) -> 'Manifest':
"""Import the manifest content.
Args:
path: Path to import from.
Returns:
Manifest instance.
"""
try:
module = importer.isolated(cls.MODULE, path)
manifest = cls(module.NAME, module.VERSION, module.PACKAGE,
**module.MODULES)
except ModuleNotFoundError as err:
raise error.Missing(f'Unknown manifest ({err})')
except AttributeError as err:
raise error.Invalid(f'Invalid manifest ({err})')
finally:
if cls.MODULE in sys.modules:
del sys.modules[cls.MODULE]
return manifest