def store_data(self, key, data):
"""Convenience method to store data in the store including metadata.
Note that the metadata are taken from the context.metadata() and slightly updated.
They might not be 100% correct, since the store_data will probably be called as
a side-effect of a query, not a result of a query.
This is indicated by the side_effect flag in the metadata and status Status.SIDE_EFFECT.value.
"""
metadata = self.metadata()
store = self.store()
extension = key_extension(key)
metadata["data_characteristics"] = data_characteristics(data)
metadata["side_effect"] = True
metadata["status"] = Status.SIDE_EFFECT.value
try:
b, mimetype, type_identifier = encode_state_data(
data, extension=extension)
metadata["type_identifier"] = type_identifier
metadata["mimetype"] = mimetype
store.store(key, b, metadata)
except:
traceback.print_exc()
m = Metadata(metadata)
m.status = Status.ERROR
m.exception(
f"Failed to encode data for key '{key}'",
traceback=traceback.format_exc(),
)
store.store_metadata(key, m.as_dict())
def _store_state(self, state):
if self.store_key is not None:
metadata = state.metadata
store = self.store() if self.store_to is None else self.store_to
if state.is_error:
store.store_metadata(self.store_key, metadata)
else:
data = state.get()
reg = self.state_types_registry()
t = reg.get(type(data))
try:
if state.metadata.get("extension") is None:
b, mime, typeid = encode_state_data(data)
else:
b, mime, typeid = encode_state_data(
data, extension=state.metadata["extension"])
store.store(self.store_key, b, metadata)
except:
traceback.print_exc()
m = Metadata(metadata)
m.status = Status.ERROR
m.exception(
f"Failed to encode data for key '{self.store_key}'",
traceback=traceback.format_exc(),
)
store.store_metadata(self.store_key, m.as_dict())
def make(self, key, store=None, context=None):
context = get_context(context)
if "sql" not in self.data:
raise Exception(
f"Recipe {self.recipe_name()} of type {self.recipe_type()} does not have sql."
)
if "filename" not in self.data:
raise Exception(
f"Recipe {self.recipe_name()} of type {self.recipe_type()} does not have a filename."
)
if store is None:
store = context.store()
with TemporaryDirectory() as tmpdir:
metadata = self.metadata(key)
try:
ctx = self.make_execution_context(tmpdir, store, context)
df = ctx.sql(self.data["sql"])
table = pyarrow.Table.from_batches(df.collect())
path = Path(tmpdir) / self.data["filename"]
pyarrow.parquet.write_table(table, str(path))
b = path.read_bytes()
store.store(key, b, metadata)
except:
m = Metadata(metadata)
m.exception("Parquet SQL recipe failed",
traceback=traceback.format_exc())
store.store_metadata(key, m.as_dict())
def metadata(self, key):
metadata = Metadata(dict())
if "title" in self.data:
metadata.metadata["title"] = self.data["title"]
if "description" in self.data:
metadata.metadata["description"] = self.data["description"]
metadata.key = key
metadata.metadata["recipes_key"] = self.data.get("recipes_key")
metadata.metadata["recipes_directory"] = self.data.get(
"recipes_directory")
metadata.metadata["recipe_name"] = self.data.get("recipe_name")
return metadata.as_dict()
def get_metadata(self, key):
p = self.path_for_key(key)
metadata = self.default_metadata(key, p.is_dir())
if p.is_dir():
return self.finalize_metadata(metadata, key=key, is_dir=True)
else:
if self.path_for_key(key).exists():
if self.metadata_path_for_key(key).exists():
with open(self.metadata_path_for_key(key)) as f:
try:
metadata.update(
json.loads(f.read())
)
except:
traceback.print_exc()
print(f"Removing {key} due to corrupted metadata (a)")
self.remove(key)
raise KeyNotFoundStoreException(key=key, store=self)
else:
m = Metadata()
m.status = Status.EXTERNAL
return self.finalize_metadata(m.as_dict(), key=key)
else:
if self.metadata_path_for_key(key).exists():
with open(self.metadata_path_for_key(key)) as f:
try:
metadata.update(
json.loads(f.read())
)
except:
traceback.print_exc()
print(f"Removing {key} due to corrupted metadata (b)")
self.remove(key)
raise KeyNotFoundStoreException(key=key, store=self)
else:
raise KeyNotFoundStoreException(key=key, store=self)
return self.finalize_metadata(metadata, key=key, is_dir=False)
def make(self, key):
print(f"### MAKE {key}")
if self.ignore(key):
raise Exception(f"Key {key} is ignored, can't make it")
recipe = self.recipes().get(key)
if recipe is None:
raise KeyNotFoundStoreException(
f"Key {key} not found, recipe unknown", key=key, store=self)
try:
recipe.make(key, store=self.substore)
is_error = False
except:
is_error = True
trace = traceback.format_exc()
traceback.print_exc()
metadata = self.substore.get_metadata(key)
recipe_metadata = self.recipe_metadata(key)
for k in [
"status", "fileinfo", "message", "is_error", "log",
"child_log", "dependencies"
]:
if k in recipe_metadata:
del recipe_metadata[k]
metadata.update(recipe_metadata)
m = Metadata(metadata)
if is_error:
m.exception(f"Error evaluating recipe", traceback=trace)
else:
if m.status == Status.NONE.value:
m.status = Status.READY.value
m.add_recipe_dependency(recipe)
metadata = m.as_dict()
self.substore.store_metadata(key, metadata)
self.on_data_changed(key)
self.on_metadata_changed(key)
def metadata(self, key):
metadata = Metadata(super().metadata(key))
metadata.query = self.data["query"]
return metadata.as_dict()
def test_create(self):
m = Metadata()
assert "status" in m.as_dict()