def put(self, obj, name, attributes=None, **kwargs):
"""
Stores an objecs, store estimators, pipelines, numpy arrays or
pandas dataframes
"""
for kind, backend_cls in six.iteritems(
self.defaults.OMEGA_STORE_BACKENDS):
backend_cls = load_class(backend_cls)
if backend_cls.supports(obj, name, attributes=attributes,
**kwargs):
backend = self.get_backend_bykind(kind)
return backend.put(obj, name, attributes=attributes, **kwargs)
if is_estimator(obj):
backend = self.get_backend_bykind(Metadata.SKLEARN_JOBLIB)
return backend.put_model(obj, name, attributes)
elif is_spark_mllib(obj):
backend = self.get_backend_bykind(Metadata.SKLEARN_JOBLIB)
return backend.put_model(obj, name, attributes, **kwargs)
elif is_dataframe(obj) or is_series(obj):
groupby = kwargs.get('groupby')
if obj.empty:
from warnings import warn
warn(
'Provided dataframe is empty, ignoring it, doing nothing here!'
)
return None
if kwargs.pop('as_hdf', False):
return self.put_dataframe_as_hdf(obj, name, attributes,
**kwargs)
elif groupby:
return self.put_dataframe_as_dfgroup(obj, name, groupby,
attributes)
append = kwargs.get('append', None)
timestamp = kwargs.get('timestamp', None)
index = kwargs.get('index', None)
return self.put_dataframe_as_documents(obj, name, append,
attributes, index,
timestamp)
elif is_ndarray(obj):
return self.put_ndarray_as_hdf(obj,
name,
attributes=attributes,
**kwargs)
elif isinstance(obj, (dict, list, tuple)):
if kwargs.pop('as_hdf', False):
self.put_pyobj_as_hdf(obj,
name,
attributes=attributes,
**kwargs)
return self.put_pyobj_as_document(obj,
name,
attributes=attributes,
**kwargs)
else:
raise TypeError('type %s not supported' % type(obj))
def register_backend(self, kind, backend):
"""
register a backend class
:param kind: (str) the backend kind
:param backend: (class) the backend class
"""
self.defaults.OMEGA_STORE_BACKENDS[kind] = load_class(backend)
if kind not in MDREGISTRY.KINDS:
MDREGISTRY.KINDS.append(kind)
return self
def get_backend_bykind(self, kind, model_store=None, data_store=None,
**kwargs):
"""
return the backend by a given object kind
:param kind: The object kind
:param model_store: the OmegaStore instance used to store models
:param data_store: the OmegaStore instance used to store data
:param kwargs: the kwargs passed to the backend initialization
:return: the backend
"""
backend_cls = load_class(self.defaults.OMEGA_STORE_BACKENDS[kind])
model_store = model_store or self
data_store = data_store or self
backend = backend_cls(model_store=model_store,
data_store=data_store, **kwargs)
return backend
def get_backend_bykind(self, kind, model_store=None, data_store=None,
**kwargs):
"""
return the backend by a given object kind
:param kind: The object kind
:param model_store: the OmegaStore instance used to store models
:param data_store: the OmegaStore instance used to store data
:param kwargs: the kwargs passed to the backend initialization
:return: the backend
"""
try:
backend_cls = load_class(self.defaults.OMEGA_STORE_BACKENDS[kind])
except:
raise ValueError('backend {kind} does not exist'.forma(**locals()))
model_store = model_store or self
data_store = data_store or self
backend = backend_cls(model_store=model_store,
data_store=data_store, **kwargs)
return backend
def get_backend(self, name, model_store=None, data_store=None, **kwargs):
"""
return the backend by a given object name
:param kind: The object kind
:param model_store: the OmegaStore instance used to store models
:param data_store: the OmegaStore instance used to store data
:param kwargs: the kwargs passed to the backend initialization
:return: the backend
"""
meta = self.metadata(name)
if meta is not None:
backend_cls = load_class(self.defaults.OMEGA_STORE_BACKENDS.get(meta.kind))
if backend_cls:
model_store = model_store or self
data_store = data_store or self
backend = backend_cls(model_store=model_store,
data_store=data_store, **kwargs)
return backend
return None
def archive(cls, path, store=None, fmt='omega') -> OmegaExportArchive:
archiver = load_class(cls.ARCHIVERS[fmt])
archive = archiver(path, store)
return archive