class MetaProduct(Mapping):
"""
Exposes a Product-like API to allow Tasks to create more than one Product,
it is automatically instantiated when a Task is initialized with a
sequence or a mapping object in the product parameter. While it is
recommended for Tasks to only have one Product (to keep them simple),
in some cases it makes sense. For example, a Jupyter notebook
(executed via NotebookRunner), for fitting a model might as well serialize
the things such as the model and any data preprocessors
"""
def __init__(self, products):
container = ProductsContainer(products)
self.products = container
self.metadata = MetadataCollection(container)
self.clients = ClientContainer(container)
self._repr = Repr()
@property
def task(self):
# TODO: validate same task
return self.products[0].task
@property
def client(self):
return self.clients
@task.setter
def task(self, value):
for p in self.products:
try:
p.task = value
except AttributeError as e:
raise AttributeError(
"Expected MetaProduct to initialize with Product "
"instancess (which have a 'task' attribute), but "
f"got {p!r}, an object of type {type(p)}. Replace it "
"with a valid Product object. If this is a file, use "
f"File({p!r})") from e
def exists(self):
return all([p.exists() for p in self.products])
def delete(self, force=False):
for product in self.products:
product.delete(force)
def download(self):
for product in self.products:
product.download()
def upload(self):
for product in self.products:
product.upload()
def _is_outdated(self, outdated_by_code=True):
is_outdated = [
p._is_outdated(outdated_by_code=outdated_by_code)
for p in self.products
]
if set(is_outdated) == {False}:
return False
if set(is_outdated) <= {TaskStatus.WaitingDownload, False}:
return TaskStatus.WaitingDownload
return any(is_outdated)
def _outdated_data_dependencies(self):
return any([p._outdated_data_dependencies() for p in self.products])
def _outdated_code_dependency(self):
return any([p._outdated_code_dependency() for p in self.products])
def to_json_serializable(self):
"""Returns a JSON serializable version of this product
"""
# NOTE: this is used in tasks where only JSON serializable parameters
# are supported such as NotebookRunner that depends on papermill
return self.products.to_json_serializable()
def render(self, params, **kwargs):
for p in self.products:
p.render(params, **kwargs)
def __repr__(self):
content = self._repr.repr1(self.products.products, level=2)
return f'{type(self).__name__}({content})'
def __str__(self):
return str(self.products)
def __iter__(self):
for product in self.products:
yield product
def __getitem__(self, key):
return self.products[key]
def __len__(self):
return len(self.products)
@property
def _remote(self):
return self.products.first._remote
def _is_remote_outdated(self, outdated_by_code):
return self.products.first._is_remote_outdated(outdated_by_code)
