def connect(self):
current_app.logger.info(
f"Connecting to DGraph: {current_app.config['DATABASE_URI']}")
client_stub = pydgraph.DgraphClientStub(
current_app.config['DATABASE_URI'])
client = pydgraph.DgraphClient(client_stub)
return (client_stub, client)
def load_json_from_dgraph(latitude, longitude, radius=None):
"""
Loads road-network-data for the given coordinates from the local dgraph-database
which can be used to reconstruct the network as a graph.
"""
import pydgraph
if not radius:
radius = 3000
client_stub = pydgraph.DgraphClientStub('localhost:9080')
client = pydgraph.DgraphClient(client_stub)
query = """query nodes($coordinates: string, $radius: float) {
nodes(func: near(location, $coordinates, $radius))
{
uid,
osmids,
location,
connects_to {
uid
}
}
}"""
variables = {
"$coordinates": "[{},{}]".format(longitude, latitude),
"$radius": str(radius)
}
txn = client.txn(read_only=True)
try:
result = txn.query(query, variables=variables)
finally:
txn.discard()
client_stub.close()
return json.loads(result.json)["nodes"]
def main(n_recipe=60000, n_shuffle=3):
data = []
page_size = 1000
client_stub = pydgraph.DgraphClientStub("localhost:9080")
client = pydgraph.DgraphClient(client_stub)
for i in tqdm(range(math.ceil((n_recipe) / page_size))):
query = """{
recipes (func: has (title), first: %d, offset: %d ) {
title
ingredient @facets(quantity) {
name
}
}
}""" % (min(page_size, max(0, n_recipe - len(data))), len(data))
recipes = client.txn(read_only=True).query(query)
data += json.loads(recipes.json)["recipes"]
client_stub.close()
print(len(data))
train, val, test = train_val_test_split(data)
print("writting data")
gen_ios(train, "train", n_shuffle)
gen_ios(val, "val", n_shuffle)
gen_ios(test, "test", n_shuffle)
def try_get_updated_graph(body):
print('Tring to update graph')
client_stub = pydgraph.DgraphClientStub(
'alpha1.engagementgraphcluster.grapl:9080')
dg_client = pydgraph.DgraphClient(client_stub)
engagement_id = body["engagement_id"]
# Mapping from `uid` to node hash
initial_graph = body["uid_hashes"]
# Try for 20 seconds max
max_time = int(time.time()) + 20
while True:
print("Getting updated graph")
updated_nodes, removed_nodes = get_updated_graph(
dg_client, initial_graph, engagement_id)
updates = {
'updated_nodes': updated_nodes,
'removed_nodes': removed_nodes
}
if updated_nodes or removed_nodes:
print("Graph has been updated: ")
return updates
now = int(time.time())
if now >= max_time:
print("Timed out before finding an update")
return updates
print("Graph has not updated")
time.sleep(0.75)
def main():
client_stub = pydgraph.DgraphClientStub("localhost:9080")
client = pydgraph.DgraphClient(client_stub)
query = """
{
q(func: has(Species.name)) {
Species.name
}
}
"""
res = client.txn(read_only=True).query(query)
data = json.loads(res.json)
for specie in data['q']:
query = """
{
q(func: match(Species.name, \"""" + specie['Species.name'] + """\", 1))
@filter(NOT eq(Species.name, \"""" + specie['Species.name'] + """\")) {
Species.name
}
}
"""
res2 = client.txn(read_only=True).query(query)
data2 = json.loads(res2.json)
if len(data2['q']) > 0:
print(specie['Species.name'])
for specie2 in data2['q']:
print(" ", specie2['Species.name'])
# print('Response: {}'.format(data))
client_stub.close()
def try_get_updated_graph(body):
print('Trying to update graph')
print(f'connecting to dgraph at {EG_ALPHA}')
client_stub = pydgraph.DgraphClientStub(EG_ALPHA)
dg_client = pydgraph.DgraphClient(client_stub)
lens = body["lens"]
# Mapping from `uid` to node hash
initial_graph = body["uid_hashes"]
# print(f'lens: {lens} initial_graph: {initial_graph}')
#
# # Try for 20 seconds max
# max_time = int(time.time()) + 20
while True:
print("Getting updated graph")
current_graph = lens_to_dict(dg_client, lens)
updates = {
'updated_nodes': current_graph,
'removed_nodes': []
}
return updates
def dgraph_client():
stub = pydgraph.DgraphClientStub(
'localhost:9080',
options=[(cygrpc.ChannelArgKey.max_send_message_length, -1),
(cygrpc.ChannelArgKey.max_receive_message_length, -1)],
)
return pydgraph.DgraphClient(stub)
def __init__(self, xid):
'''
:param xid: 节点的xid
'''
self.client_stub = pydgraph.DgraphClientStub(pdgraph)
self.client = pydgraph.DgraphClient(self.client_stub)
self.xid = xid
def word_macro_matcher(earliest, latest):
# type: (int, int) -> List[RootNode]
if earliest != 0:
earliest = earliest - 1
query_str = """
{{
q(func: eq(image_name, "word"))
@filter(gt(create_time, {}) AND lt(create_time, {}))
{{
uid, pid, create_time, image_name, terminate_time, node_key, asset_id
children {{
uid, pid, create_time, image_name, terminate_time, node_key, asset_id
}},
bin_file {{
uid,
create_time,
delete_time,
asset_id,
node_key
}}
}}
}}
""".format(earliest, latest + 1)
print(query_str)
client = pydgraph.DgraphClient(
pydgraph.DgraphClientStub('db.mastergraph:9080'))
res = json.loads(client.txn().query(query_str).json)
print(res)
res = res['q']
res = res if isinstance(res, List) else [res]
return [RootNode.from_json(s) for s in res]
def __init__(self, server, schema, transform=None):
self.server = server
self._client_stub = pydgraph.DgraphClientStub(server)
self._client = pydgraph.DgraphClient(self._client_stub)
self._maxquery = 1000
self.uidmap = {}
super(dgraph, self).__init__(schema, transform)
def __init__(self):
self.nc = NATS()
client_stub = pydgraph.DgraphClientStub("dgraph:9080")
self.dg = pydgraph.DgraphClient(client_stub)
loop = asyncio.get_running_loop()
loop.add_signal_handler(signal.SIGTERM, self.stop)
loop.add_signal_handler(signal.SIGHUP, self.stop)
loop.add_signal_handler(signal.SIGINT, self.stop)
self.fut_stop = loop.create_future()
def publish(self, dgraphServerAddress="localhost:9080"):
client_stub = pydgraph.DgraphClientStub(dgraphServerAddress)
client = pydgraph.DgraphClient(client_stub)
txn = client.txn()
try:
response = txn.mutate(set_nquads="\n".join(self.triples))
txn.commit()
print(response)
finally:
txn.discard()
def create_client(client_stub):
"""
This allows us to create multiple clients easily,
by passing in the stub from create_client_stub()
:param client_stub: connection for stub
:return: dgraph client
"""
try:
LOG.info("Creating client")
return pydgraph.DgraphClient(client_stub)
except:
LOG.critical("Failed to create client")
def get_client():
"""
Get a new dgraph client and stub wrapper
:return: client, stubs
"""
# Initialize stub wrapper
client_stub = StubWrapper(6)
# Pass back client and stubs
return pydgraph.DgraphClient(*client_stub.stubs), client_stub
def create_client(addr='localhost:9080'):
# Read certs
with open('./tls/ca.crt', 'rb') as f:
root_ca_cert = f.read()
with open('./tls/client.user.key', 'rb') as f:
client_cert_key = f.read()
with open('./tls/client.user.crt', 'rb') as f:
client_cert = f.read()
# Connect to Dgraph via gRPC with mutual TLS.
creds = grpc.ssl_channel_credentials(root_certificates=root_ca_cert,
private_key=client_cert_key,
certificate_chain=client_cert)
client_stub = pydgraph.DgraphClientStub(addr, credentials=creds)
return pydgraph.DgraphClient(client_stub)
def __init__(self,
host='localhost',
port=9080,
debug=False,
profile=False):
'''
Constructor
'''
self.host = host
self.port = port
self.debug = debug
self.profile = profile
self.client_stub = pydgraph.DgraphClientStub('%s:%d' % (host, port))
self.client = pydgraph.DgraphClient(self.client_stub)
def __init__(self, host='localhost',port=9080, debug=False,profile=False):
'''
Constructor
Args:
host(string): the host for the dgraph connection
port(int): the port for the dgraph connection
debug(boolean): True if debug information should be shown
profile(boolean): True if profiling/timing information should be shown
'''
self.host=host
self.port=port
self.debug=debug
self.profile=profile
self.client_stub = pydgraph.DgraphClientStub('%s:%d' % (host,port))
self.client =pydgraph.DgraphClient(self.client_stub)
def try_get_updated_graph(body):
LOGGER.info("Trying to update graph")
LOGGER.info(f"connecting to dgraph at {MG_ALPHA}")
client_stub = pydgraph.DgraphClientStub(MG_ALPHA)
dg_client = pydgraph.DgraphClient(client_stub)
lens = body["lens"]
# Mapping from `uid` to node hash
initial_graph = body["uid_hashes"]
while True:
LOGGER.info("Getting updated graph")
current_graph = lens_to_dict(dg_client, lens)
updates = {"updated_nodes": current_graph, "removed_nodes": []}
return updates
def list_all_lenses(prefix: str) -> List[Dict[str, Any]]:
LOGGER.info(f"connecting to dgraph at {MG_ALPHA}")
client_stub = pydgraph.DgraphClientStub(MG_ALPHA)
dg_client = pydgraph.DgraphClient(client_stub)
# DGraph query for all nodes with a 'lens' that matches the 'prefix'
if prefix:
query = """
query q0($a: string)
{
q0(func: alloftext(lens, $a), orderdesc: score)
{
uid,
node_key,
node_type: dgraph.type,
lens,
score
}
}"""
variables = {"$a": prefix}
else:
query = """
{
q0(func: has(lens), orderdesc: score)
{
uid,
node_key,
node_type: dgraph.type,
lens,
score
}
}"""
variables = {}
txn = dg_client.txn(read_only=True)
try:
res = json.loads(txn.query(query, variables=variables).json)
return res["q0"]
finally:
txn.discard()
def run_dgraph_query_raw(self, query):
servers = self.configuration.get('servers')
client_stub = pydgraph.DgraphClientStub(servers)
client = pydgraph.DgraphClient(client_stub)
txn = client.txn(read_only=True)
try:
response_raw = txn.query(query)
data = json.loads(response_raw.json)
return data
except Exception as e:
raise e
finally:
txn.discard()
client_stub.close()
def list_all_lenses(prefix: str) -> List[Dict[str, Any]]:
client_stub = pydgraph.DgraphClientStub(
'alpha0.engagementgraphcluster.grapl:9080')
dg_client = pydgraph.DgraphClient(client_stub)
# DGraph query for all nodes with a 'lens' that matches the 'prefix'
if prefix:
query = """
query q0($a: string)
{
q0(func: alloftext(lens, $a), orderdesc: score)
{
uid,
node_key,
lens,
score
}
}"""
variables = {'$a': prefix}
else:
query = """
{
q0(func: has(lens), orderdesc: score)
{
uid,
node_key,
lens,
score
}
}"""
variables = {}
txn = dg_client.txn(read_only=True)
try:
res = json.loads(txn.query(query, variables=variables).json)
return res['q0']
finally:
txn.discard()
def __init__(
self,
host: str = Config.get("dgraph", "host"),
batch_size: int = int(Config.get("dgraph", "batch_size")),
wipe_db: bool = False,
*args,
**kwargs,
):
logger.info(f"Connecting to Dgraph server at {host}")
client_stub = pydgraph.DgraphClientStub(host)
self.dgraph = pydgraph.DgraphClient(client_stub)
super().__init__(*args, **kwargs)
if wipe_db:
logger.info("Wiping existing database due to wipe_db=True")
self.dgraph.alter(pydgraph.Operation(drop_all=True))
self.batch_size = 1000
logger.info("Initialized Dgraph Backend")
def _graph_db_connection(self, client_stub):
client = pydgraph.DgraphClient(client_stub)
# _logger.info(client)
return client
def create_client(addr=SERVER_ADDR):
"""Creates a new client object using the given address."""
return pydgraph.DgraphClient(pydgraph.DgraphClientStub(addr))
in dgraph, when you delete a node, the node will still exist with a single predicate `uid`.
in order to get rid of this node when doing query.
we add `type` field for each node, then use @filter(has(type) to ignore this node.
"""
import json
import types
from typing import Any, Dict, List, Optional
import pydgraph
from pydgraph.proto import api_pb2 as api
DGRAPH_URL = 'dgraph-alpha:9080'
stub = pydgraph.DgraphClientStub(DGRAPH_URL)
client = pydgraph.DgraphClient(stub)
def set_schema():
"""
in order to delete node without specify predicates, schema must be set.
if schema is not set, `delete` node will not work.
"""
schema = """
name: string .
type: string .
procedure: int .
madeOf: [uid] @reverse .
type Material {
res = client.txn(read_only=True).query(query)
equipments = json.loads(res.json)
# Print results.
print(json.dumps(equipments, sort_keys=True, indent=4))
if __name__ == '__main__':
if len(sys.argv) != 2:
raise ValueError('Please provide the dgraph server ip:port')
try:
print("Connect to server")
client = pydgraph.DgraphClient(pydgraph.DgraphClientStub(sys.argv[1]))
print("Deleting everything")
drop_all(client)
print("Creating schema")
set_schema(client)
print("Adding mutations")
create_data(client)
print("Getting equipments")
query_equipments(client)
except Exception as e:
print('Error: {}'.format(e))
def create_client(client_stub):
return pydgraph.DgraphClient(client_stub)
def create_client(client_stub):
"""Create a client."""
return pydgraph.DgraphClient(client_stub)
res = client.txn().query(query)
data = res.json.decode("utf-8")
data = json.loads(data)
data = data["everyone"]
if len(data) > 0:
return data[0]
else:
return None
if __name__ == '__main__':
# upload into graph/develop
config.URL = '192.168.1.210:9081'
# config.URL = '192.168.1.36:9081'
client_stub = pydgraph.DgraphClientStub(config.URL)
client = pydgraph.DgraphClient(client_stub)
path_input = os.path.join(os.path.dirname(__file__),
r'skill_entity_update_20190220.jsonl')
p_del = []
p_update = []
with open(path_input) as input_file:
for line in input_file.readlines():
line = line.strip('\n')
line = json.loads(line)
query = check_node_entity_id(client, line['id'])
if 'alias' in query.keys():
if 'mentions' in query.keys():
p1 = {
'uid': query['uid'],
'mentions': query['mentions'],
'alias': query['alias']
def client(self):
if self._client is None:
self._client = pydgraph.DgraphClient(self.client_stub)
return self._client