def get_attr(self):
if not self.finished:
self.finish()
attr = attr_value_pb2.AttrValue()
attr.func.name = "loader"
attr.func.attr[types_pb2.PROTOCOL].CopyFrom(utils.s_to_attr(self.protocol))
# Let graphscope handle local files cause it's implemented in c++ and
# doesn't add an additional stream layer.
# Maybe handled by vineyard in the near future
if self.protocol == "file":
source = "{}#{}".format(self.source, self.options)
attr.func.attr[types_pb2.VALUES].CopyFrom(
utils.bytes_to_attr(source.encode("utf-8"))
)
elif self.protocol in ("numpy", "pandas"):
attr.func.attr[types_pb2.ROW_NUM].CopyFrom(utils.i_to_attr(self.row_num))
attr.func.attr[types_pb2.COLUMN_NUM].CopyFrom(
utils.i_to_attr(self.column_num)
)
# Use key start from 10000 + col_index to store raw bytes.
for i in range(len(self.property_bytes)):
attr.func.attr[10000 + i].CopyFrom(
utils.bytes_to_attr(self.property_bytes[i])
)
else: # Let vineyard handle other data source.
attr.func.attr[types_pb2.VALUES].CopyFrom(
utils.bytes_to_attr(self.source.encode("utf-8"))
)
return attr
def report_graph(
graph, report_type, node=None, edge=None, fid=None, lid=None, key=None
):
"""Create report operation for nx graph.
This operation is used to simulate networkx graph reporting methods with variaty
report type and corresponding config parameters.
Args:
graph (`nx.Graph`): A nx graph.
report_type: report type, can be
type_pb2.(NODE_NUM,
EDGE_NUM,
HAS_NODE,
HAS_EDGE,
NODE_DATA,
EDGE_DATA,
NEIGHBORS_BY_NODE,
SUCCS_BY_NODE,
PREDS_BY_NODE,
NEIGHBORS_BY_LOC,
SUCCS_BY_LOC,
PREDS_BY_LOC,
DEG_BY_NODE,
IN_DEG_BY_NODE,
OUT_DEG_BY_NODE,
DEG_BY_LOC,
IN_DEG_BY_LOC,
OUT_DEG_BY_LOC,
NODES_BY_LOC)
node (str): node id, used as node id with 'NODE' report types. (optional)
edge (str): an edge with 'EDGE' report types. (optional)
fid (int): fragment id, with 'LOC' report types. (optional)
lid (int): local id of node in grape_engine, with 'LOC; report types. (optional)
key (str): edge key for MultiGraph or MultiDiGraph, with 'EDGE' report types. (optional)
Returns:
An op to do reporting job.
"""
config = {
types_pb2.GRAPH_NAME: utils.s_to_attr(graph.key),
types_pb2.REPORT_TYPE: utils.report_type_to_attr(report_type),
}
if node is not None:
config[types_pb2.NODE] = utils.s_to_attr(node)
if edge is not None:
config[types_pb2.EDGE] = utils.s_to_attr(edge)
if fid is not None:
config[types_pb2.FID] = utils.i_to_attr(fid)
if lid is not None:
config[types_pb2.LID] = utils.i_to_attr(lid)
config[types_pb2.EDGE_KEY] = utils.s_to_attr(str(key) if key is not None else "")
op = Operation(
graph.session_id,
types_pb2.REPORT_GRAPH,
config=config,
output_types=types_pb2.RESULTS,
)
return op
def project_arrow_property_graph(
graph,
v_label_id,
v_prop_id,
e_label_id,
e_prop_id,
v_data_type,
e_data_type,
oid_type=None,
vid_type=None,
):
"""Project arrow property graph to a simple graph.
Args:
graph (:class:`Graph`): Source graph, which type should be ARROW_PROPERTY
dst_graph_key (str): The key of projected graph.
v_label_id (int): Label id of vertex used to project.
v_prop_id (int): Property id of vertex used to project.
e_label_id (int): Label id of edge used to project.
e_prop_id (int): Property id of edge used to project.
Returns:
An op to project `graph`, results in a simple ARROW_PROJECTED graph.
"""
check_argument(graph.graph_type == types_pb2.ARROW_PROPERTY)
config = {
types_pb2.GRAPH_NAME:
utils.s_to_attr(graph.key),
types_pb2.GRAPH_TYPE:
utils.graph_type_to_attr(types_pb2.ARROW_PROJECTED),
types_pb2.V_LABEL_ID:
utils.i_to_attr(v_label_id),
types_pb2.V_PROP_ID:
utils.i_to_attr(v_prop_id),
types_pb2.E_LABEL_ID:
utils.i_to_attr(e_label_id),
types_pb2.E_PROP_ID:
utils.i_to_attr(e_prop_id),
types_pb2.OID_TYPE:
utils.s_to_attr(oid_type),
types_pb2.VID_TYPE:
utils.s_to_attr(vid_type),
types_pb2.V_DATA_TYPE:
utils.s_to_attr(utils.data_type_to_cpp(v_data_type)),
types_pb2.E_DATA_TYPE:
utils.s_to_attr(utils.data_type_to_cpp(e_data_type)),
}
op = Operation(
graph._session_id,
types_pb2.PROJECT_GRAPH,
config=config,
output_types=types_pb2.GRAPH,
)
return op
def to_vineyard_tensor(context, selector=None, vertex_range=None, axis=None):
"""Retrieve results as vineyard tensor.
Parameters:
results (:class:`Context`): Results return by `run_app` operation, store the query results.
selector (str): Select the type of data to retrieve.
vertex_range (str): Specify a range to retrieve.
Returns:
An op to convert query results into a vineyard tensor.
"""
config = {}
if selector is not None:
config[types_pb2.SELECTOR] = utils.s_to_attr(selector)
if vertex_range is not None:
config[types_pb2.VERTEX_RANGE] = utils.s_to_attr(vertex_range)
if axis is not None:
config[types_pb2.AXIS] = utils.i_to_attr(axis)
op = Operation(
context.session_id,
types_pb2.TO_VINEYARD_TENSOR,
config=config,
inputs=[context.op],
output_types=types_pb2.VINEYARD_TENSOR,
)
return op
def context_to_numpy(context, selector=None, vertex_range=None, axis=0):
"""Retrieve results as a numpy ndarray.
Args:
results (:class:`Context`): Results return by `run_app` operation, store the query results.
selector (str): Select the type of data to retrieve.
vertex_range (str): Specify a range to retrieve.
Returns:
An op to retrieve query results and convert to numpy ndarray.
"""
config = {}
if selector is not None:
config[types_pb2.SELECTOR] = utils.s_to_attr(selector)
if vertex_range is not None:
config[types_pb2.VERTEX_RANGE] = utils.s_to_attr(vertex_range)
if axis is not None:
config[types_pb2.AXIS] = utils.i_to_attr(axis)
op = Operation(
context.session_id,
types_pb2.CONTEXT_TO_NUMPY,
config=config,
inputs=[context.op],
output_types=types_pb2.TENSOR,
)
return op
def arrow_to_dynamic(graph):
"""Transform a :class:`Graph` object to :class:`nx.Graph`.
Args:
graph (:class:`Graph`): Source graph, which type should be ARROW_PROPERTY.
Returns:
An op of transform arrow graph to dynamic graph with necessary configurations.
"""
check_argument(graph.graph_type == graph_def_pb2.ARROW_PROPERTY)
config = {
types_pb2.GRAPH_NAME:
utils.s_to_attr(graph.key),
types_pb2.GRAPH_TYPE:
utils.graph_type_to_attr(graph_def_pb2.ARROW_PROPERTY),
types_pb2.DST_GRAPH_TYPE:
utils.graph_type_to_attr(graph_def_pb2.DYNAMIC_PROPERTY),
types_pb2.OID_TYPE:
utils.s_to_attr(utils.data_type_to_cpp(graph.schema.oid_type)),
types_pb2.VID_TYPE:
utils.s_to_attr(utils.data_type_to_cpp(graph.schema.vid_type)),
types_pb2.DEFAULT_LABEL_ID:
utils.i_to_attr(graph._default_label_id),
}
op = Operation(
graph.session_id,
types_pb2.TRANSFORM_GRAPH,
config=config,
output_types=types_pb2.GRAPH,
)
return op
def _from_vineyard_id(self, vineyard_id):
config = {}
config[types_pb2.IS_FROM_VINEYARD_ID] = utils.b_to_attr(True)
config[types_pb2.VINEYARD_ID] = utils.i_to_attr(int(vineyard_id))
# FIXME(hetao) hardcode oid/vid type for codegen, when loading from vineyard
#
# the metadata should be retrived from vineyard
config[types_pb2.OID_TYPE] = utils.s_to_attr("int64_t")
config[types_pb2.VID_TYPE] = utils.s_to_attr("uint64_t")
return dag_utils.create_graph(self.session_id,
types_pb2.ARROW_PROPERTY,
attrs=config)
def _from_vineyard_id(self, vineyard_id):
config = {}
config[types_pb2.IS_FROM_VINEYARD_ID] = b_to_attr(True)
config[types_pb2.VINEYARD_ID] = i_to_attr(vineyard_id)
# FIXME(hetao) hardcode oid/vid type for codegen, when loading from vineyard
#
# the metadata should be retrived from vineyard
config[types_pb2.OID_TYPE] = s_to_attr("int64_t")
config[types_pb2.VID_TYPE] = s_to_attr("uint64_t")
op = create_graph(self._session_id, types_pb2.ARROW_PROPERTY, attrs=config)
graph_def = op.eval()
return graph_def
def get_attr(self):
attr = attr_value_pb2.AttrValue()
attr.func.name = "loader"
attr.func.attr[types_pb2.PROTOCOL].CopyFrom(
utils.s_to_attr(self.protocol))
if self.protocol in ("file", "oss", "vineyard", "mars"):
source = "{}#{}".format(self.source, self.options)
attr.func.attr[types_pb2.VALUES].CopyFrom(
utils.bytes_to_attr(source.encode("utf-8")))
elif self.protocol in ("numpy", "pandas"):
attr.func.attr[types_pb2.ROW_NUM].CopyFrom(
utils.i_to_attr(self.row_num))
attr.func.attr[types_pb2.COLUMN_NUM].CopyFrom(
utils.i_to_attr(self.column_num))
# Use key start from 10000 + col_index to store raw bytes.
for i in range(len(self.property_bytes)):
attr.func.attr[10000 + i].CopyFrom(
utils.bytes_to_attr(self.property_bytes[i]))
else:
raise TypeError("Protocol not recognized " + self.protocol)
return attr
def unload_graph(graph):
"""Unload a graph.
Args:
graph (:class:`Graph`): The graph to unload.
Returns:
An op to unload the `graph`.
"""
config = {types_pb2.GRAPH_NAME: utils.s_to_attr(graph.key)}
# Dynamic graph doesn't have a vineyard id
if hasattr(graph, "vineyard_id"):
config[types_pb2.VINEYARD_ID] = utils.i_to_attr(graph.vineyard_id)
op = Operation(
graph._session_id,
types_pb2.UNLOAD_GRAPH,
config=config,
output_types=types_pb2.GRAPH,
)
return op
