def __new__(
cls,
source: 'compmod.Source',
pipeline: topology.Composable,
evaluation: typing.Optional[topology.Operator] = None,
):
if not isinstance(pipeline, topology.Composable):
raise error.Invalid('Invalid pipeline')
if not isinstance(source, compmod.Source):
raise error.Invalid('Invalid source')
if evaluation and not isinstance(evaluation, topology.Operator):
raise error.Invalid('Invalid evaluation')
return super().__new__(cls, source, pipeline, evaluation)
def __new__(
cls,
train: frame.Queryable,
apply: frame.Queryable,
labels: typing.Sequence[series.Column],
ordinal: typing.Optional[series.Operable],
):
train = train.query
apply = apply.query
if {c.operable for c in train.columns}.intersection(c.operable for c in labels):
raise error.Invalid('Label-feature overlap')
if train.schema != apply.schema:
raise error.Invalid('Train-apply schema mismatch')
if ordinal:
ordinal = series.Operable.ensure_is(ordinal)
return super().__new__(cls, train, apply, tuple(labels), ordinal)
def run(self) -> None:
"""Trigger the deployment process."""
packages = [distribution.Package(f) for c, _, f in self.distribution.dist_files if c == bdist.Package.COMMAND]
if not packages:
raise error.Invalid(
'Must create and upload files in one command ' f'(e.g. setup.py {bdist.Package.COMMAND} upload)'
)
project = self.distribution.get_name()
platform = runtime.Platform(registry=provcfg.Registry.resolve(self.registry))
for pkg in packages:
platform.registry.publish(project, pkg)
def __init__(self,
apply: task.Spec,
train: typing.Optional[task.Spec] = None,
label: typing.Optional[task.Spec] = None):
if apply.actor.is_stateful() or (train and train.actor.is_stateful()
) or (label
and label.actor.is_stateful()):
raise error.Invalid('Stateful actor invalid for an extractor')
self._apply: task.Spec = apply
self._train: task.Spec = train or apply
self._label: typing.Optional[task.Spec] = label
def __init__(
self,
apply: typing.Optional[task.Spec] = None,
train: typing.Optional[task.Spec] = None,
label: typing.Optional[task.Spec] = None,
):
for mode in apply, train, label:
if mode and mode.actor.is_stateful():
raise error.Invalid('Stateful actor invalid for an adapter')
self._apply: typing.Optional[task.Spec] = apply
self._train: typing.Optional[task.Spec] = train
self._label: typing.Optional[task.Spec] = label
def read(self, path: pathlib.Path) -> None:
"""Read and merge config from given file.
Args:
path: Path to file to parse.
"""
try:
self.update(toml.load(path))
except FileNotFoundError: # not an error (ignore)
pass
except PermissionError as err: # soft error (warn)
self._errors[path] = err
except ValueError as err: # hard error (abort)
raise error.Invalid(f'Invalid config file {path}: {err}') from err
else:
self._sources.append(path)
def put(self, package: 'distribution.Package') -> lngmod.Level:
"""Publish new lineage to the repository based on provided package.
Args:
package: Distribution package to be persisted.
Returns:
new lineage instance based on the package.
"""
project = package.manifest.name
lineage = package.manifest.version
try:
previous = self.list().last
except (directory.Level.Invalid, directory.Level.Listing.Empty):
LOGGER.debug('No previous lineage for %s-%s', project, lineage)
else:
if project != self.key:
raise error.Invalid('Project key mismatch')
if not lineage > previous:
raise directory.Level.Invalid(
f'{project}-{lineage} not an increment from existing {previous}'
)
self.registry.push(package)
return self.get(lineage)
def read(
cls,
path: typing.Optional[typing.Union[str, pathlib.Path]] = None
) -> 'Manifest':
"""Import the manifest content.
Args:
path: Path to import from.
Returns:
Manifest instance.
"""
try:
module = importer.isolated(cls.MODULE, path)
manifest = cls(module.NAME, module.VERSION, module.PACKAGE,
**module.MODULES)
except ModuleNotFoundError as err:
raise error.Missing(f'Unknown manifest ({err})')
except AttributeError as err:
raise error.Invalid(f'Invalid manifest ({err})')
finally:
if cls.MODULE in sys.modules:
del sys.modules[cls.MODULE]
return manifest
def __init__(self, writer: task.Spec):
if writer.actor.is_stateful():
raise error.Invalid('Stateful actor invalid for a publisher')
self._writer: task.Spec = writer
def __init__(self, spec: task.Spec):
if not spec.actor.is_stateful():
raise error.Invalid('Stateless actor invalid for a consumer')
super().__init__(spec)