def create_single_op_dag(op_type, config=None): op_def = op_def_pb2.OpDef(op=op_type, key=uuid.uuid4().hex) if config: for k, v in config.items(): op_def.attr[k].CopyFrom(v) dag = op_def_pb2.DagDef() dag.op.extend([op_def]) return dag
def __init__( self, session_id, op_type, inputs=None, output_types=None, config=None, large_attr=None, query_args=None, ): """Creates an :code:`graphscope.framework.operation.Operation`. Args: op_type: :code:`types_pb2.OperationType` Value for the "op" attribute of the OpDef proto. inputs: A list of `Operations` that will be the parents to self output_types: The operation's output type config: Dictionary where the key is the attribute name (a string) and the value is the respective "attr" attribute of the OpDef proto (an AttrValue). large_attr: List of `LargeAttrValue` for large chunk. query_args: Values that used as query parameters when evaluating app. Raises: TypeError: value in inputs is not a :class:`Operation` """ self._session_id = session_id self._op_def = op_def_pb2.OpDef(op=op_type, key=uuid.uuid4().hex, output_type=output_types) self._parents = list() if large_attr: self._op_def.large_attr.CopyFrom(large_attr) if config: for k, v in config.items(): self._op_def.attr[k].CopyFrom(v) if query_args is not None: self._op_def.query_args.CopyFrom(query_args) if inputs: for op in inputs: if not isinstance(op, Operation): raise TypeError( "Input op must be an Operation: {0}".format(op)) self.add_parent(op) self._output_types = output_types self._evaluated = False self._leaf = False
def _get_engine_config(self): op_def = op_def_pb2.OpDef(op=types_pb2.GET_ENGINE_CONFIG) dag_def = op_def_pb2.DagDef() dag_def.op.extend([op_def]) fetch_request = message_pb2.RunStepRequest( session_id=self._session_id, dag_def=dag_def ) fetch_response = self._analytical_engine_stub.RunStep(fetch_request) config = json.loads(fetch_response.result.decode("utf-8")) if self._launcher_type == types_pb2.K8S: config["vineyard_service_name"] = self._launcher.get_vineyard_service_name() config["vineyard_rpc_endpoint"] = self._launcher.get_vineyard_rpc_endpoint() return config
def __init__( self, session_id, op_type, inputs=None, output_types=None, config=None, query_args=None, ): """Creates an :code:`Operation`. Args: op_type: :code:`types_pb2.OperationType` Value for the "op" attribute of the OpDef proto. inputs: A list of `Operations` that will be the parents to self output_types: The operation's output type config: Dictionary where the key is the attribute name (a string) and the value is the respective "attr" attribute of the OpDef proto (an AttrValue). query_args: Values that used as query parameters when evaluating app. Raises: TypeError: if `op_def` is not a `OpDef`, or if `g` is not a `Dag`. ValueError: if the `op_def` name is not valid. """ self._session_id = session_id self._op_def = op_def_pb2.OpDef(op=op_type, key=uuid.uuid4().hex) if config: for k, v in config.items(): self._op_def.attr[k].CopyFrom(v) if query_args is not None: self._op_def.query_args.CopyFrom(query_args) self._output_types = output_types self._signature = None self._output = None # hold the executed result of the DAG.
def _maybe_register_graph(self, op, session_id): graph_sig = get_graph_sha256(op.attr) space = self._builtin_workspace graph_lib_path = get_lib_path(os.path.join(space, graph_sig), graph_sig) if not os.path.isfile(graph_lib_path): compiled_path = self._compile_lib_and_distribute( compile_graph_frame, graph_sig, op ) if graph_lib_path != compiled_path: raise RuntimeError("Computed path not equal to compiled path.") if graph_sig not in self._object_manager: # register graph op_def = op_def_pb2.OpDef(op=types_pb2.REGISTER_GRAPH_TYPE) op_def.attr[types_pb2.GRAPH_LIBRARY_PATH].CopyFrom( attr_value_pb2.AttrValue(s=graph_lib_path.encode("utf-8")) ) op_def.attr[types_pb2.TYPE_SIGNATURE].CopyFrom( attr_value_pb2.AttrValue(s=graph_sig.encode("utf-8")) ) op_def.attr[types_pb2.GRAPH_TYPE].CopyFrom( attr_value_pb2.AttrValue( graph_type=op.attr[types_pb2.GRAPH_TYPE].graph_type ) ) dag_def = op_def_pb2.DagDef() dag_def.op.extend([op_def]) register_request = message_pb2.RunStepRequest( session_id=session_id, dag_def=dag_def ) register_response = self._analytical_engine_stub.RunStep(register_request) if register_response.status.code == error_codes_pb2.OK: self._object_manager.put( graph_sig, LibMeta(register_response.result, "graph_frame", graph_lib_path), ) else: raise RuntimeError("Error occur when register graph") op.attr[types_pb2.TYPE_SIGNATURE].CopyFrom( attr_value_pb2.AttrValue(s=graph_sig.encode("utf-8")) ) return op
def _maybe_register_graph(self, op, session_id): graph_sig = self._generate_graph_sig(op.attr) if graph_sig in self._object_manager: lib_meta = self._object_manager.get(graph_sig) graph_lib_path = lib_meta.lib_path else: graph_lib_path = self._compile_lib_and_distribute( compile_graph_frame, graph_sig, op ) # register graph op_def = op_def_pb2.OpDef(op=types_pb2.REGISTER_GRAPH_TYPE) op_def.attr[types_pb2.GRAPH_LIBRARY_PATH].CopyFrom( attr_value_pb2.AttrValue(s=graph_lib_path.encode("utf-8")) ) op_def.attr[types_pb2.TYPE_SIGNATURE].CopyFrom( attr_value_pb2.AttrValue(s=graph_sig.encode("utf-8")) ) op_def.attr[types_pb2.GRAPH_TYPE].CopyFrom( attr_value_pb2.AttrValue( graph_type=op.attr[types_pb2.GRAPH_TYPE].graph_type ) ) dag_def = op_def_pb2.DagDef() dag_def.op.extend([op_def]) register_request = message_pb2.RunStepRequest( session_id=session_id, dag_def=dag_def ) register_response = self._analytical_engine_stub.RunStep(register_request) if register_response.status.code == error_codes_pb2.OK: self._object_manager.put( graph_sig, LibMeta(register_response.result, "graph_frame", graph_lib_path), ) else: raise RuntimeError("Error occur when register graph") op.attr[types_pb2.TYPE_SIGNATURE].CopyFrom( attr_value_pb2.AttrValue(s=graph_sig.encode("utf-8")) ) return op