def create_single_op_dag(op_type, config=None):
op_def = op_def_pb2.OpDef(op=op_type, key=uuid.uuid4().hex)
if config:
for k, v in config.items():
op_def.attr[k].CopyFrom(v)
dag = op_def_pb2.DagDef()
dag.op.extend([op_def])
return dag
def __init__(
self,
session_id,
op_type,
inputs=None,
output_types=None,
config=None,
large_attr=None,
query_args=None,
):
"""Creates an :code:`graphscope.framework.operation.Operation`.
Args:
op_type: :code:`types_pb2.OperationType`
Value for the "op" attribute of the OpDef proto.
inputs:
A list of `Operations` that will be the parents to self
output_types:
The operation's output type
config:
Dictionary where the key is the attribute name (a string)
and the value is the respective "attr" attribute of the OpDef proto (an
AttrValue).
large_attr:
List of `LargeAttrValue` for large chunk.
query_args:
Values that used as query parameters when evaluating app.
Raises:
TypeError: value in inputs is not a :class:`Operation`
"""
self._session_id = session_id
self._op_def = op_def_pb2.OpDef(op=op_type,
key=uuid.uuid4().hex,
output_type=output_types)
self._parents = list()
if large_attr:
self._op_def.large_attr.CopyFrom(large_attr)
if config:
for k, v in config.items():
self._op_def.attr[k].CopyFrom(v)
if query_args is not None:
self._op_def.query_args.CopyFrom(query_args)
if inputs:
for op in inputs:
if not isinstance(op, Operation):
raise TypeError(
"Input op must be an Operation: {0}".format(op))
self.add_parent(op)
self._output_types = output_types
self._evaluated = False
self._leaf = False
def _get_engine_config(self):
op_def = op_def_pb2.OpDef(op=types_pb2.GET_ENGINE_CONFIG)
dag_def = op_def_pb2.DagDef()
dag_def.op.extend([op_def])
fetch_request = message_pb2.RunStepRequest(
session_id=self._session_id, dag_def=dag_def
)
fetch_response = self._analytical_engine_stub.RunStep(fetch_request)
config = json.loads(fetch_response.result.decode("utf-8"))
if self._launcher_type == types_pb2.K8S:
config["vineyard_service_name"] = self._launcher.get_vineyard_service_name()
config["vineyard_rpc_endpoint"] = self._launcher.get_vineyard_rpc_endpoint()
return config
def __init__(
self,
session_id,
op_type,
inputs=None,
output_types=None,
config=None,
query_args=None,
):
"""Creates an :code:`Operation`.
Args:
op_type: :code:`types_pb2.OperationType`
Value for the "op" attribute of the OpDef proto.
inputs:
A list of `Operations` that will be the parents to self
output_types:
The operation's output type
config:
Dictionary where the key is the attribute name (a string)
and the value is the respective "attr" attribute of the OpDef proto (an
AttrValue).
query_args:
Values that used as query parameters when evaluating app.
Raises:
TypeError: if `op_def` is not a `OpDef`, or if `g` is not a `Dag`.
ValueError: if the `op_def` name is not valid.
"""
self._session_id = session_id
self._op_def = op_def_pb2.OpDef(op=op_type, key=uuid.uuid4().hex)
if config:
for k, v in config.items():
self._op_def.attr[k].CopyFrom(v)
if query_args is not None:
self._op_def.query_args.CopyFrom(query_args)
self._output_types = output_types
self._signature = None
self._output = None # hold the executed result of the DAG.
def _maybe_register_graph(self, op, session_id):
graph_sig = get_graph_sha256(op.attr)
space = self._builtin_workspace
graph_lib_path = get_lib_path(os.path.join(space, graph_sig), graph_sig)
if not os.path.isfile(graph_lib_path):
compiled_path = self._compile_lib_and_distribute(
compile_graph_frame, graph_sig, op
)
if graph_lib_path != compiled_path:
raise RuntimeError("Computed path not equal to compiled path.")
if graph_sig not in self._object_manager:
# register graph
op_def = op_def_pb2.OpDef(op=types_pb2.REGISTER_GRAPH_TYPE)
op_def.attr[types_pb2.GRAPH_LIBRARY_PATH].CopyFrom(
attr_value_pb2.AttrValue(s=graph_lib_path.encode("utf-8"))
)
op_def.attr[types_pb2.TYPE_SIGNATURE].CopyFrom(
attr_value_pb2.AttrValue(s=graph_sig.encode("utf-8"))
)
op_def.attr[types_pb2.GRAPH_TYPE].CopyFrom(
attr_value_pb2.AttrValue(
graph_type=op.attr[types_pb2.GRAPH_TYPE].graph_type
)
)
dag_def = op_def_pb2.DagDef()
dag_def.op.extend([op_def])
register_request = message_pb2.RunStepRequest(
session_id=session_id, dag_def=dag_def
)
register_response = self._analytical_engine_stub.RunStep(register_request)
if register_response.status.code == error_codes_pb2.OK:
self._object_manager.put(
graph_sig,
LibMeta(register_response.result, "graph_frame", graph_lib_path),
)
else:
raise RuntimeError("Error occur when register graph")
op.attr[types_pb2.TYPE_SIGNATURE].CopyFrom(
attr_value_pb2.AttrValue(s=graph_sig.encode("utf-8"))
)
return op
def _maybe_register_graph(self, op, session_id):
graph_sig = self._generate_graph_sig(op.attr)
if graph_sig in self._object_manager:
lib_meta = self._object_manager.get(graph_sig)
graph_lib_path = lib_meta.lib_path
else:
graph_lib_path = self._compile_lib_and_distribute(
compile_graph_frame, graph_sig, op
)
# register graph
op_def = op_def_pb2.OpDef(op=types_pb2.REGISTER_GRAPH_TYPE)
op_def.attr[types_pb2.GRAPH_LIBRARY_PATH].CopyFrom(
attr_value_pb2.AttrValue(s=graph_lib_path.encode("utf-8"))
)
op_def.attr[types_pb2.TYPE_SIGNATURE].CopyFrom(
attr_value_pb2.AttrValue(s=graph_sig.encode("utf-8"))
)
op_def.attr[types_pb2.GRAPH_TYPE].CopyFrom(
attr_value_pb2.AttrValue(
graph_type=op.attr[types_pb2.GRAPH_TYPE].graph_type
)
)
dag_def = op_def_pb2.DagDef()
dag_def.op.extend([op_def])
register_request = message_pb2.RunStepRequest(
session_id=session_id, dag_def=dag_def
)
register_response = self._analytical_engine_stub.RunStep(register_request)
if register_response.status.code == error_codes_pb2.OK:
self._object_manager.put(
graph_sig,
LibMeta(register_response.result, "graph_frame", graph_lib_path),
)
else:
raise RuntimeError("Error occur when register graph")
op.attr[types_pb2.TYPE_SIGNATURE].CopyFrom(
attr_value_pb2.AttrValue(s=graph_sig.encode("utf-8"))
)
return op