def _load_metadata(path):
"""load metadata from csv
:returns: Metadata instance"""
metadata_filepath = os.path.join(path, "metadata.csv")
if os.path.exists(metadata_filepath):
metadata = Metadata().from_csv(metadata_filepath)
else:
metadata = Metadata()
return metadata
def __init__(self, **kwargs):
Data.__init__(self, **kwargs)
self._uuid = None
self._group = OrderedDict()
self.uuid = kwargs.get("uuid") or uuid.uuid4()
self.metadata = kwargs.get("metadata") or Metadata()
self._table = pd.DataFrame()
def __init__(self, **kwargs):
self._uuid = None
self._name = None
self.uuid = kwargs.get("uuid") or uuid.uuid4()
self.name = kwargs.get("name") or "N.N."
self.metadata = kwargs.get("metadata") or Metadata()
def __init__(self, **kwargs):
Group.__init__(self, **kwargs)
self._uuid = None
self._group = OrderedDict()
self.uuid = kwargs.get("uuid") or uuid.uuid4()
self.metadata = kwargs.get("metadata") or Metadata()
def initialize(self, **kwargs):
logger.info(f"initialize {self.rootpath}")
metadata = Metadata()
metadata.add("ctime", now_utc_str())
with pd.HDFStore(self.rootpath, **kwargs) as hdf:
hdf.put('metadata', metadata.df, format='fixed', data_columns=True)
def __init__(self, **kwargs):
"""create Data object
.. code-block:: python
df = pd.DataFrame([1,2,3])
data = sdata.Data(name='my name',
uuid='38b26864e7794f5182d38459bab85842',
table=df,
description="A remarkable description")
:param name: name of the data object
:param table: pandas.DataFrame to store
:param uuid: uuid of the object
:param metadata: sdata.Metadata object
:param description: a string to describe the object
"""
# self._uuid = None
# self._name = None
self._prefix = None
# ToDo: add getter and setter for metadata
# self.metadata = kwargs.get("metadata") or Metadata()
self.metadata = Metadata()
# set default sdata attributes
self.metadata.add(self.SDATA_VERSION,
__version__,
dtype="str",
description="sdata package version")
self.metadata.add(self.SDATA_NAME,
"N.N.",
dtype="str",
description="name of the data object")
self.metadata.add(self.SDATA_UUID,
"",
dtype="str",
description="Universally Unique Identifier")
self.metadata.add(self.SDATA_PARENT,
"",
dtype="str",
description="uuid of the parent sdata object")
self.metadata.add(self.SDATA_CLASS,
self.__class__.__name__,
dtype="str",
description="sdata class")
self.metadata.add(self.SDATA_CTIME,
now_utc_str(),
dtype="str",
description="creation date")
self.metadata.add(self.SDATA_MTIME,
now_utc_str(),
dtype="str",
description="modification date")
metadata = kwargs.get("metadata")
if metadata is not None:
# logger.debug("Data got Metadata {}".format(metadata))
if metadata and isinstance(metadata, Metadata):
for attribute in metadata.attributes.values():
# logger.debug("Data.Metadata.add {0.name}:{0.value}".format(attribute))
self.metadata.add(attribute)
# auto correct
if kwargs.get("auto_correct") is None or kwargs.get(
"auto_correct") is True:
self.auto_correct = True
else:
self.auto_correct = False
# logger.debug("sdata: set auto_correct={}".format(self.auto_correct))
if kwargs.get("name") is not None:
self.name = kwargs.get("name")
self.prefix = kwargs.get("prefix") or ""
self._gen_default_attributes(
kwargs.get("default_attributes") or self.ATTR_NAMES)
self._group = OrderedDict()
self._table = None # pd.DataFrame()
self.table = kwargs.get("table", None)
self._description = ""
self.description = kwargs.get("description", "")
self.project = kwargs.get("project", "")
if (kwargs.get("uuid") == ""
or kwargs.get("uuid") is not None) and not self.metadata.get(
self.SDATA_UUID).value and kwargs.get("uuid") != "hash":
# logger.info("uuid in kwargs")
try:
self._set_uuid(kwargs.get(
"uuid")) # store given uuid str or generate a new uuid
except Sdata_Uuid_Exeption as exp:
if self.auto_correct is True:
logger.warning("got invalid uuid -> generate a new uuid")
self._set_uuid(uuid.uuid4())
else:
raise
elif (kwargs.get("uuid") == "" or kwargs.get("uuid") is None
) and self.metadata.get(self.SDATA_UUID).value != "":
# logger.info("uuid in metadata")
pass
elif kwargs.get("uuid") == "hash":
sha3_256 = self.gen_uuid_from_state()
# logger.info("gen uuid from sha3_256 {}".format(sha3_256))
new_uuid = uuid_from_str(sha3_256)
self._set_uuid(new_uuid.hex)
else:
# logger.info("uuid new")
self._set_uuid(uuid.uuid4())