def setUp(self):
self.mock_component_run = ComponentRun("mock_component_run")
self.mock_component_run_dict = {
"component_name": "mock_component_run",
"notes": "",
"inputs": [],
"outputs": [],
"git_hash": None,
"git_tags": None,
"code_snapshot": None,
"start_timestamp": None,
"end_timestamp": None,
"dependencies": [],
"id": None,
"stale": [],
"test_result": None,
"mlflow_run_id": None,
"mlflow_run_params": None,
"mlflow_run_metrics": None,
}
self.mock_inputs = [
IOPointer("mock_input_1"),
IOPointer("mock_input_2"),
]
self.mock_outputs = [
IOPointer("mock_output_1"),
IOPointer("mock_output_2"),
]
def get_history(component_name: str,
limit: int = 10) -> typing.List[ComponentRun]:
"""Returns a list of ComponentRuns that are part of the component's
history."""
store = Store(_db_uri)
history = store.get_history(component_name, limit)
# Convert to client-facing ComponentRuns
component_runs = []
for cr in history:
inputs = [
IOPointer.from_dictionary(iop.__dict__).to_dictionary()
for iop in cr.inputs
]
outputs = [
IOPointer.from_dictionary(iop.__dict__).to_dictionary()
for iop in cr.outputs
]
dependencies = [dep.component_name for dep in cr.dependencies]
d = copy.deepcopy(cr.__dict__)
d.update({
"inputs": inputs,
"outputs": outputs,
"dependencies": dependencies
})
component_runs.append(ComponentRun.from_dictionary(d))
return component_runs
def get_io_pointer(io_pointer_id: str,
io_pointer_val: typing.Any = None,
create=True):
"""Returns IO Pointer metadata."""
store = Store(_db_uri)
iop = store.get_io_pointer(io_pointer_id, io_pointer_val, create=create)
return IOPointer.from_dictionary(iop.__dict__)
def setUp(self):
self.mock_component_run = ComponentRun("mock_component_run")
self.mock_component_run_dict = {
"component_name": "mock_component_run",
"inputs": [],
"outputs": [],
"git_hash": None,
"code_snapshot": None,
"start_timestamp": None,
"end_timestamp": None,
"dependencies": [],
"id": None,
"stale": [],
}
self.mock_inputs = [IOPointer("mock_input_1"), IOPointer("mock_input_2")]
self.mock_outputs = [IOPointer("mock_output_1"), IOPointer("mock_output_2")]
def backtrace(output_pointer: str):
"""Prints trace for an output id.
Returns list of tuples (level, ComponentRun) where level is how
many hops away the node is from the node that produced the output_id."""
store = Store(_db_uri)
trace = store.trace(output_pointer)
# Convert to entities.ComponentRun
component_runs = []
for depth, cr in trace:
inputs = [IOPointer.from_dictionary(iop.__dict__) for iop in cr.inputs]
outputs = [IOPointer.from_dictionary(iop.__dict__) for iop in cr.outputs]
dependencies = [dep.component_name for dep in cr.dependencies]
d = copy.deepcopy(cr.__dict__)
d.update({"inputs": inputs, "outputs": outputs, "dependencies": dependencies})
component_runs.append((depth, ComponentRun.from_dictionary(d)))
return component_runs
def get_component_run_information(component_run_id: str) -> ComponentRun:
"""Returns a ComponentRun object."""
store = Store(_db_uri)
cr = store.get_component_run(component_run_id)
if not cr:
raise RuntimeError(f"Component run with id {id} not found.")
inputs = [
IOPointer.from_dictionary(iop.__dict__).to_dictionary() for iop in cr.inputs
]
outputs = [
IOPointer.from_dictionary(iop.__dict__).to_dictionary() for iop in cr.outputs
]
dependencies = [dep.component_name for dep in cr.dependencies]
d = copy.deepcopy(cr.__dict__)
if cr.code_snapshot:
d.update({"code_snapshot": str(cr.code_snapshot.decode("utf-8"))})
d.update({"inputs": inputs, "outputs": outputs, "dependencies": dependencies})
return ComponentRun.from_dictionary(d)
def io_pointer():
if "id" not in request.args:
return error(f"id not specified.", HTTPStatus.NOT_FOUND)
io_pointer_id = request.args["id"]
try:
res = get_io_pointer(io_pointer_id, create=False)
return json.dumps(
IOPointer.from_dictionary(res.__dict__).to_dictionary())
except RuntimeError:
return error(f"IOPointer {io_pointer_id} not found",
HTTPStatus.NOT_FOUND)
def testLogKVComponentRun(self):
# Tests implementation of values in iopointer
create_component(
name="valtest",
description="Tests implementation of values in iopointer.",
owner="me",
)
iop1 = ["this", "is", "the", "first"]
iop2 = ["this", "is", "the", "second"]
# Create iopointers and CR
iop1 = IOPointer(name="iop1", value=iop1)
iop2 = IOPointer(name="iop2", value=iop2)
cr = ComponentRun("valtest")
cr.set_start_timestamp()
cr.set_end_timestamp()
cr.add_input(iop1)
cr.add_output(iop2)
log_component_run(cr)
def get_history(
component_name: str,
limit: int = 10,
date_lower: typing.Union[datetime, str] = datetime.min,
date_upper: typing.Union[datetime, str] = datetime.max,
) -> typing.List[ComponentRun]:
"""Returns a list of ComponentRuns that are part of the component's
history."""
store = Store(_db_uri)
# Check if none
if not date_lower:
date_lower = datetime.min
if not date_upper:
date_upper = datetime.max
history = store.get_history(component_name, limit, date_lower, date_upper)
# Convert to client-facing ComponentRuns
component_runs = []
for cr in history:
inputs = [
IOPointer.from_dictionary(iop.__dict__).to_dictionary()
for iop in cr.inputs
]
outputs = [
IOPointer.from_dictionary(iop.__dict__).to_dictionary()
for iop in cr.outputs
]
dependencies = [dep.component_name for dep in cr.dependencies]
d = copy.deepcopy(cr.__dict__)
d.update({
"inputs": inputs,
"outputs": outputs,
"dependencies": dependencies,
})
component_runs.append(ComponentRun.from_dictionary(d))
return component_runs
def retrieve_io_pointers_for_label(label_id: str):
store = Store(_db_uri)
iops = store.retrieve_io_pointers_for_label(label_id)
return [IOPointer.from_dictionary(iop.__dict__) for iop in iops]