def get_attr(self): if not self.finished: self.finish() attr = attr_value_pb2.AttrValue() attr.func.name = "loader" attr.func.attr[types_pb2.PROTOCOL].CopyFrom(utils.s_to_attr(self.protocol)) # Let graphscope handle local files cause it's implemented in c++ and # doesn't add an additional stream layer. # Maybe handled by vineyard in the near future if self.protocol == "file": source = "{}#{}".format(self.source, self.options) attr.func.attr[types_pb2.VALUES].CopyFrom( utils.bytes_to_attr(source.encode("utf-8")) ) elif self.protocol in ("numpy", "pandas"): attr.func.attr[types_pb2.ROW_NUM].CopyFrom(utils.i_to_attr(self.row_num)) attr.func.attr[types_pb2.COLUMN_NUM].CopyFrom( utils.i_to_attr(self.column_num) ) # Use key start from 10000 + col_index to store raw bytes. for i in range(len(self.property_bytes)): attr.func.attr[10000 + i].CopyFrom( utils.bytes_to_attr(self.property_bytes[i]) ) else: # Let vineyard handle other data source. attr.func.attr[types_pb2.VALUES].CopyFrom( utils.bytes_to_attr(self.source.encode("utf-8")) ) return attr
def report_graph( graph, report_type, node=None, edge=None, fid=None, lid=None, key=None ): """Create report operation for nx graph. This operation is used to simulate networkx graph reporting methods with variaty report type and corresponding config parameters. Args: graph (`nx.Graph`): A nx graph. report_type: report type, can be type_pb2.(NODE_NUM, EDGE_NUM, HAS_NODE, HAS_EDGE, NODE_DATA, EDGE_DATA, NEIGHBORS_BY_NODE, SUCCS_BY_NODE, PREDS_BY_NODE, NEIGHBORS_BY_LOC, SUCCS_BY_LOC, PREDS_BY_LOC, DEG_BY_NODE, IN_DEG_BY_NODE, OUT_DEG_BY_NODE, DEG_BY_LOC, IN_DEG_BY_LOC, OUT_DEG_BY_LOC, NODES_BY_LOC) node (str): node id, used as node id with 'NODE' report types. (optional) edge (str): an edge with 'EDGE' report types. (optional) fid (int): fragment id, with 'LOC' report types. (optional) lid (int): local id of node in grape_engine, with 'LOC; report types. (optional) key (str): edge key for MultiGraph or MultiDiGraph, with 'EDGE' report types. (optional) Returns: An op to do reporting job. """ config = { types_pb2.GRAPH_NAME: utils.s_to_attr(graph.key), types_pb2.REPORT_TYPE: utils.report_type_to_attr(report_type), } if node is not None: config[types_pb2.NODE] = utils.s_to_attr(node) if edge is not None: config[types_pb2.EDGE] = utils.s_to_attr(edge) if fid is not None: config[types_pb2.FID] = utils.i_to_attr(fid) if lid is not None: config[types_pb2.LID] = utils.i_to_attr(lid) config[types_pb2.EDGE_KEY] = utils.s_to_attr(str(key) if key is not None else "") op = Operation( graph.session_id, types_pb2.REPORT_GRAPH, config=config, output_types=types_pb2.RESULTS, ) return op
def project_arrow_property_graph( graph, v_label_id, v_prop_id, e_label_id, e_prop_id, v_data_type, e_data_type, oid_type=None, vid_type=None, ): """Project arrow property graph to a simple graph. Args: graph (:class:`Graph`): Source graph, which type should be ARROW_PROPERTY dst_graph_key (str): The key of projected graph. v_label_id (int): Label id of vertex used to project. v_prop_id (int): Property id of vertex used to project. e_label_id (int): Label id of edge used to project. e_prop_id (int): Property id of edge used to project. Returns: An op to project `graph`, results in a simple ARROW_PROJECTED graph. """ check_argument(graph.graph_type == types_pb2.ARROW_PROPERTY) config = { types_pb2.GRAPH_NAME: utils.s_to_attr(graph.key), types_pb2.GRAPH_TYPE: utils.graph_type_to_attr(types_pb2.ARROW_PROJECTED), types_pb2.V_LABEL_ID: utils.i_to_attr(v_label_id), types_pb2.V_PROP_ID: utils.i_to_attr(v_prop_id), types_pb2.E_LABEL_ID: utils.i_to_attr(e_label_id), types_pb2.E_PROP_ID: utils.i_to_attr(e_prop_id), types_pb2.OID_TYPE: utils.s_to_attr(oid_type), types_pb2.VID_TYPE: utils.s_to_attr(vid_type), types_pb2.V_DATA_TYPE: utils.s_to_attr(utils.data_type_to_cpp(v_data_type)), types_pb2.E_DATA_TYPE: utils.s_to_attr(utils.data_type_to_cpp(e_data_type)), } op = Operation( graph._session_id, types_pb2.PROJECT_GRAPH, config=config, output_types=types_pb2.GRAPH, ) return op
def to_vineyard_tensor(context, selector=None, vertex_range=None, axis=None): """Retrieve results as vineyard tensor. Parameters: results (:class:`Context`): Results return by `run_app` operation, store the query results. selector (str): Select the type of data to retrieve. vertex_range (str): Specify a range to retrieve. Returns: An op to convert query results into a vineyard tensor. """ config = {} if selector is not None: config[types_pb2.SELECTOR] = utils.s_to_attr(selector) if vertex_range is not None: config[types_pb2.VERTEX_RANGE] = utils.s_to_attr(vertex_range) if axis is not None: config[types_pb2.AXIS] = utils.i_to_attr(axis) op = Operation( context.session_id, types_pb2.TO_VINEYARD_TENSOR, config=config, inputs=[context.op], output_types=types_pb2.VINEYARD_TENSOR, ) return op
def context_to_numpy(context, selector=None, vertex_range=None, axis=0): """Retrieve results as a numpy ndarray. Args: results (:class:`Context`): Results return by `run_app` operation, store the query results. selector (str): Select the type of data to retrieve. vertex_range (str): Specify a range to retrieve. Returns: An op to retrieve query results and convert to numpy ndarray. """ config = {} if selector is not None: config[types_pb2.SELECTOR] = utils.s_to_attr(selector) if vertex_range is not None: config[types_pb2.VERTEX_RANGE] = utils.s_to_attr(vertex_range) if axis is not None: config[types_pb2.AXIS] = utils.i_to_attr(axis) op = Operation( context.session_id, types_pb2.CONTEXT_TO_NUMPY, config=config, inputs=[context.op], output_types=types_pb2.TENSOR, ) return op
def arrow_to_dynamic(graph): """Transform a :class:`Graph` object to :class:`nx.Graph`. Args: graph (:class:`Graph`): Source graph, which type should be ARROW_PROPERTY. Returns: An op of transform arrow graph to dynamic graph with necessary configurations. """ check_argument(graph.graph_type == graph_def_pb2.ARROW_PROPERTY) config = { types_pb2.GRAPH_NAME: utils.s_to_attr(graph.key), types_pb2.GRAPH_TYPE: utils.graph_type_to_attr(graph_def_pb2.ARROW_PROPERTY), types_pb2.DST_GRAPH_TYPE: utils.graph_type_to_attr(graph_def_pb2.DYNAMIC_PROPERTY), types_pb2.OID_TYPE: utils.s_to_attr(utils.data_type_to_cpp(graph.schema.oid_type)), types_pb2.VID_TYPE: utils.s_to_attr(utils.data_type_to_cpp(graph.schema.vid_type)), types_pb2.DEFAULT_LABEL_ID: utils.i_to_attr(graph._default_label_id), } op = Operation( graph.session_id, types_pb2.TRANSFORM_GRAPH, config=config, output_types=types_pb2.GRAPH, ) return op
def _from_vineyard_id(self, vineyard_id): config = {} config[types_pb2.IS_FROM_VINEYARD_ID] = utils.b_to_attr(True) config[types_pb2.VINEYARD_ID] = utils.i_to_attr(int(vineyard_id)) # FIXME(hetao) hardcode oid/vid type for codegen, when loading from vineyard # # the metadata should be retrived from vineyard config[types_pb2.OID_TYPE] = utils.s_to_attr("int64_t") config[types_pb2.VID_TYPE] = utils.s_to_attr("uint64_t") return dag_utils.create_graph(self.session_id, types_pb2.ARROW_PROPERTY, attrs=config)
def _from_vineyard_id(self, vineyard_id): config = {} config[types_pb2.IS_FROM_VINEYARD_ID] = b_to_attr(True) config[types_pb2.VINEYARD_ID] = i_to_attr(vineyard_id) # FIXME(hetao) hardcode oid/vid type for codegen, when loading from vineyard # # the metadata should be retrived from vineyard config[types_pb2.OID_TYPE] = s_to_attr("int64_t") config[types_pb2.VID_TYPE] = s_to_attr("uint64_t") op = create_graph(self._session_id, types_pb2.ARROW_PROPERTY, attrs=config) graph_def = op.eval() return graph_def
def get_attr(self): attr = attr_value_pb2.AttrValue() attr.func.name = "loader" attr.func.attr[types_pb2.PROTOCOL].CopyFrom( utils.s_to_attr(self.protocol)) if self.protocol in ("file", "oss", "vineyard", "mars"): source = "{}#{}".format(self.source, self.options) attr.func.attr[types_pb2.VALUES].CopyFrom( utils.bytes_to_attr(source.encode("utf-8"))) elif self.protocol in ("numpy", "pandas"): attr.func.attr[types_pb2.ROW_NUM].CopyFrom( utils.i_to_attr(self.row_num)) attr.func.attr[types_pb2.COLUMN_NUM].CopyFrom( utils.i_to_attr(self.column_num)) # Use key start from 10000 + col_index to store raw bytes. for i in range(len(self.property_bytes)): attr.func.attr[10000 + i].CopyFrom( utils.bytes_to_attr(self.property_bytes[i])) else: raise TypeError("Protocol not recognized " + self.protocol) return attr
def unload_graph(graph): """Unload a graph. Args: graph (:class:`Graph`): The graph to unload. Returns: An op to unload the `graph`. """ config = {types_pb2.GRAPH_NAME: utils.s_to_attr(graph.key)} # Dynamic graph doesn't have a vineyard id if hasattr(graph, "vineyard_id"): config[types_pb2.VINEYARD_ID] = utils.i_to_attr(graph.vineyard_id) op = Operation( graph._session_id, types_pb2.UNLOAD_GRAPH, config=config, output_types=types_pb2.GRAPH, ) return op