def test_connection_share(client):
# Overwrite fixture with pool_size=1 client
client = Client('ws://localhost:45940/gremlin', 'gmodern', pool_size=1)
g = Graph().traversal()
t = g.V()
message = RequestMessage('traversal', 'bytecode', {
'gremlin': t.bytecode,
'aliases': {
'g': 'gmodern'
}
})
message2 = RequestMessage('traversal', 'bytecode', {
'gremlin': t.bytecode,
'aliases': {
'g': 'gmodern'
}
})
future = client.submitAsync(message)
future2 = client.submitAsync(message2)
result_set2 = future2.result()
assert len(result_set2.all().result()) == 6
# This future has to finish for the second to yield result - pool_size=1
assert future.done()
result_set = future.result()
assert len(result_set.all().result()) == 6
def authenticated_client(request):
try:
if request.param == 'basic':
# turn off certificate verification for testing purposes only
ssl_opts = ssl.SSLContext(ssl.PROTOCOL_TLSv1_2)
ssl_opts.verify_mode = ssl.CERT_NONE
client = Client(basic_url,
'gmodern',
username='******',
password='******',
transport_factory=lambda: AiohttpTransport(
ssl_options=ssl_opts))
elif request.param == 'kerberos':
client = Client(kerberos_url,
'gmodern',
kerberized_service=kerberized_service)
else:
raise ValueError("Invalid authentication option - " +
request.param)
except OSError:
pytest.skip('Gremlin Server is not running')
else:
def fin():
client.close()
request.addfinalizer(fin)
return client
def test_multi_conn_pool(client):
g = Graph().traversal()
t = g.V()
message = RequestMessage('traversal', 'bytecode', {
'gremlin': t.bytecode,
'aliases': {
'g': 'gmodern'
}
})
message2 = RequestMessage('traversal', 'bytecode', {
'gremlin': t.bytecode,
'aliases': {
'g': 'gmodern'
}
})
client = Client('ws://localhost:45940/gremlin', 'g', pool_size=1)
future = client.submitAsync(message)
future2 = client.submitAsync(message2)
result_set2 = future2.result()
assert len(result_set2.all().result()) == 6
# with connection pool `future` may or may not be done here
result_set = future.result()
assert len(result_set.all().result()) == 6
def gremlin_client(request):
# type: (Any) -> Client
url = getattr(request.module, 'url', 'ws://localhost:8182/gremlin')
traversal_source = getattr(request.module, 'traversal_source',
'ConfigurationManagementGraph')
client = Client(url, traversal_source)
request.addfinalizer(lambda: client.close())
return client
def __init__(self, key, frontend_endpoint, object_id):
self.key = key
self.type = "gie_manager"
# graph object id in vineyard
self.object_id = object_id
self.graph_url = f"ws://{frontend_endpoint}/gremlin"
self.client = Client(self.graph_url, "g")
self.closed = False
def _do_query(self, query):
if self._gremlin_client is None:
print("Creating new General Purpose Gremlin Client on %s:%s" %
(self._url, self._port))
client = Client(f'wss://{self._url}:{self._port}/gremlin',
'g',
pool_size=1)
return _format_graph_results(client.submit(query))
def test_client_side_timeout_set_for_tornado(client):
client = Client('ws://localhost:45940/gremlin', 'gmodern',
transport_factory=lambda: TornadoTransport(read_timeout=1, write_timeout=1))
try:
# should fire an exception
client.submit('Thread.sleep(2000);1').all().result()
assert False
except TimeoutError as toerr:
assert str(toerr) == "Operation timed out after 1 seconds"
def test_client_connection_pool_after_error(client):
# Overwrite fixture with pool_size=1 client
client = Client('ws://localhost:45940/gremlin', 'gmodern', pool_size=1)
try:
# should fire an exception
client.submit('1/0').all().result()
assert False
except Exception:
# expecting the pool size to be 1 again after query returned
assert client.available_pool_size == 1
def __init__(self, session, object_id, front_ip=None, front_port=None):
self._status = InteractiveQueryStatus.Initializing
self._session = session
self._object_id = object_id
self._error_msg = ""
if front_ip is not None and front_port is not None:
self._graph_url = "ws://%s:%d/gremlin" % (front_ip, front_port)
self._client = Client(self._graph_url, "g")
else:
self._graph_url = None
self._client = None
def test_client_connection_pool_after_error(client):
# Overwrite fixture with pool_size=1 client
client = Client('ws://localhost:45940/gremlin', 'gmodern', pool_size=1)
try:
# should fire an exception
client.submit('1/0').all().result()
assert False
except GremlinServerError as gse:
# expecting the pool size to be 1 again after query returned
assert gse.status_code == 597
assert client.available_pool_size == 1
def test_client_side_timeout_set_for_aiohttp(client):
client = Client('ws://localhost:45940/gremlin',
'gmodern',
transport_factory=lambda: AiohttpTransport(
read_timeout=1, write_timeout=1))
try:
# should fire an exception
client.submit('Thread.sleep(2000);1').all().result()
assert False
except TimeoutError as err:
# asyncio TimeoutError has no message.
assert str(err) == ""
def test_multi_conn_pool(client):
g = Graph().traversal()
t = g.V()
message = RequestMessage('traversal', 'bytecode', {'gremlin': t.bytecode, 'aliases': {'g': 'gmodern'}})
message2 = RequestMessage('traversal', 'bytecode', {'gremlin': t.bytecode, 'aliases': {'g': 'gmodern'}})
client = Client('ws://localhost:45940/gremlin', 'g', pool_size=1)
future = client.submitAsync(message)
future2 = client.submitAsync(message2)
result_set2 = future2.result()
assert len(result_set2.all().result()) == 6
# with connection pool `future` may or may not be done here
result_set = future.result()
assert len(result_set.all().result()) == 6
def find_node(
unique_id: str,
node_type: str,
gremlin_client: client.Client
):
'''
Construct the query to find the node with the specified property. Assumes
that the specified property is unique to that node type
'''
node_type_ids = {
'subreddit': 'subreddit_id',
'comment': 'id',
'user': '******'
}
query = f"g.V().hasLabel('{node_type}').has('{node_type_ids[node_type]}', '{unique_id}').next()"
try:
node = gremlin_client.submit(query).all().result()
return query
except:
"Node doesn't exist! Cannot create edge"
# Not sure if this is needed, but noticed that when Gremlin client throws an error
# it will disconnect and hang..
gremlin_client = client.Client(JANUS_CONNECT, 'g')
return None
def process_subreddit_node(
property_keys: list,
json_values: dict,
gremlin_client: client.Client,
execute=False,
):
'''
Constructs a gremlin query to create subreddit node (if it doesn't already exists)
'''
gremlin_query = construct_gremlin_query(property_keys, json_values, 'subreddit')
if not check_existence_query(json_values['subreddit_id'], 'subreddit', gremlin_client) and execute:
gremlin_client.submit(gremlin_query).all().result()
gremlin_client.submit('g.tx().commit()').all().result()
return gremlin_query
else:
return f"Subreddit node w/ name {json_values['subreddit_id']} already exists"
def connect(self, url="loclahost", port="8182", graph="g", **kwargs):
self.URL = "ws://{}:{}/gremlin".format(url, port)
self.graph = graph
self.args = kwargs
if not kwargs:
graphson_reader = JanusGraphSONReader().build()
graphson_writer = JanusGraphSONWriter().build()
else:
kwargs_provided = kwargs.keys()
if all(k in self.connection_keyword_properties
for k in kwargs_provided):
pass
else:
raise AttributeError(
"Additional parameters except url, port, graph needs to be the available"
"Gremlin client connection parameters. Refer docs")
if "graphson_reader" in kwargs and "graphson_writer" in kwargs:
graphson_reader = kwargs["graphson_reader"]
graphson_writer = kwargs["graphson_writer"]
else:
graphson_reader = None
graphson_writer = None
args = {
k: v
for k, v in self.args.items()
if k not in ["graphson_reader", "graphson_writer"]
}
self.remote_connection = DriverRemoteConnection(
self.URL,
self.graph,
graphson_reader=graphson_reader,
graphson_writer=graphson_writer,
**args)
self.management_connection = Client(self.URL, self.graph, **args)
return self
def test_client_pool_in_session(client):
# Overwrite fixture with pool_size=2 client
try:
# should fire an exception
client = Client('ws://localhost:45940/gremlin', 'g', session=str(uuid.uuid4()), pool_size=2)
assert False
except Exception:
assert True
def test_connection_share(client):
# Overwrite fixture with pool_size=1 client
client = Client('ws://localhost:45940/gremlin', 'gmodern', pool_size=1)
g = Graph().traversal()
t = g.V()
message = RequestMessage('traversal', 'bytecode', {'gremlin': t.bytecode, 'aliases': {'g': 'gmodern'}})
message2 = RequestMessage('traversal', 'bytecode', {'gremlin': t.bytecode, 'aliases': {'g': 'gmodern'}})
future = client.submitAsync(message)
future2 = client.submitAsync(message2)
result_set2 = future2.result()
assert len(result_set2.all().result()) == 6
# This future has to finish for the second to yield result - pool_size=1
assert future.done()
result_set = future.result()
assert len(result_set.all().result()) == 6
def client(self, pool_size=None, max_workers=None):
gremlin_endpoint = self.endpoints.gremlin_endpoint()
request_parameters = gremlin_endpoint.prepare_request()
signed_ws_request = httpclient.HTTPRequest(
request_parameters.uri, headers=request_parameters.headers)
return Client(signed_ws_request,
'g',
pool_size=pool_size,
max_workers=max_workers)
class InteractiveQueryManager(object):
def __init__(self, key, frontend_endpoint, object_id):
self.key = key
self.type = "gie_manager"
# graph object id in vineyard
self.object_id = object_id
self.graph_url = f"ws://{frontend_endpoint}/gremlin"
self.client = Client(self.graph_url, "g")
self.closed = False
def submit(self, message, bindings=None, request_options=None):
return self.client.submit(message, bindings, request_options)
def close(self):
try:
self.client.close()
except Exception:
pass
self.closed = True
def process_user_node(
property_keys: list,
json_values: dict,
gremlin_client: client.Client,
execute=False
):
'''
Constructs a gremlin query to create subreddit node (if it doesn't already exists)
Executes if flag is set to True.
'''
gremlin_query = construct_gremlin_query(property_keys, json_values, 'user')
if not check_existence_query(json_values['author'], 'user', gremlin_client) and execute:
gremlin_client.submit(gremlin_query).all().result()
gremlin_client.submit('g.tx().commit()').all().result()
return gremlin_query
else:
return f"User node w/ unique id {json_values['author']} already exists"
def secure_client(request):
try:
client = Client('ws://localhost:45941/gremlin', 'gmodern', username='******', password='******')
except OSError:
pytest.skip('Gremlin Server is not running')
else:
def fin():
client.close()
request.addfinalizer(fin)
return client
def client(request):
try:
client = Client('ws://localhost:45940/gremlin', 'gmodern')
except OSError:
pytest.skip('Gremlin Server is not running')
else:
def fin():
client.close()
request.addfinalizer(fin)
return client
def client(self, pool_size=None, max_workers=None, **kwargs):
gremlin_endpoint = self.endpoints.gremlin_endpoint()
request_parameters = gremlin_endpoint.prepare_request()
return Client(request_parameters.uri,
'g',
pool_size=pool_size,
max_workers=max_workers,
headers=request_parameters.headers,
**kwargs)
def client(request):
try:
client = Client(anonymous_url, 'gmodern')
except OSError:
pytest.skip('Gremlin Server is not running')
else:
def fin():
client.close()
request.addfinalizer(fin)
return client
def test_multi_request_in_session(client):
# Overwrite fixture with session client
session_id = str(uuid.uuid4())
client = Client('ws://localhost:45940/gremlin', 'g', session=session_id)
assert client.submit('x = 1').all().result()[0] == 1
assert client.submit('x + 2').all().result()[0] == 3
client.close()
# attempt reconnect to session and make sure "x" is no longer a thing
client = Client('ws://localhost:45940/gremlin', 'g', session=session_id)
try:
# should fire an exception
client.submit('x').all().result()
assert False
except Exception:
assert True
def authenticated_client(request):
try:
if request.param == 'basic':
client = Client(basic_url,
'gmodern',
username='******',
password='******')
elif request.param == 'kerberos':
client = Client(kerberos_url,
'gmodern',
kerberized_service=kerberized_service)
else:
raise ValueError("Invalid authentication option - " +
request.param)
except OSError:
pytest.skip('Gremlin Server is not running')
else:
def fin():
client.close()
request.addfinalizer(fin)
return client
def test_client_message_too_big(client):
try:
client = Client("http://localhost", 'g', max_content_length=1024)
client.submit("1+1").all().result()
assert False
except Exception:
assert True
finally:
client.close()
def check_existence_query(
value: str,
node_type: str,
gremlin_client: client.Client,
):
'''
Helper function to check if node exists
'''
node_type_ids = {
'subreddit': 'subreddit_id',
'comment': 'id',
'user': '******'
}
existence_query = f"g.V().hasLabel('{node_type}').has('{node_type_ids[node_type]}', '{value}').hasNext()"
return gremlin_client.submit(existence_query).all().result()[0]
class InteractiveQuery(object):
"""`InteractiveQuery` class, is a simple wrapper around
`Gremlin-Python <https://pypi.org/project/gremlinpython/>`_,
which implements Gremlin within the Python language.
It also can expose gremlin endpoint which can be used by
any other standard gremlin console, with the method `graph_url()`.
It also has a method called `subgraph` which can extract some fragments
from origin graph, produce a new, smaller but concise graph stored in vineyard,
which lifetime is independent from the origin graph.
User can either use `execute()` to submit a script, or use `traversal_source()`
to get a `GraphTraversalSource` for further traversal.
"""
def __init__(self, session, object_id, front_ip=None, front_port=None):
self._status = InteractiveQueryStatus.Initializing
self._session = session
self._object_id = object_id
self._error_msg = ""
if front_ip is not None and front_port is not None:
self._graph_url = "ws://%s:%d/gremlin" % (front_ip, front_port)
self._client = Client(self._graph_url, "g")
else:
self._graph_url = None
self._client = None
def __repr__(self):
return f"graphscope.InteractiveQuery <{self._graph_url}>"
@property
def object_id(self):
"""Get the vineyard object id of graph.
Returns:
str: object id
"""
return self._object_id
@property
def graph_url(self):
"""The gremlin graph url can be used with any standard gremlin console, e.g., thinkerpop."""
return self._graph_url
@property
def status(self):
return self._status
@status.setter
def status(self, value):
self._status = value
@property
def error_msg(self):
return self._error_msg
@error_msg.setter
def error_msg(self, error_msg):
self._error_msg = error_msg
def set_frontend(self, front_ip, front_port):
self._graph_url = "ws://%s:%d/gremlin" % (front_ip, front_port)
self._client = Client(self._graph_url, "g")
def closed(self):
"""Return if the current instance is closed."""
return self._status == InteractiveQueryStatus.Closed
def subgraph(self, gremlin_script):
"""Create a subgraph, which input is the result of the execution of `gremlin_script`.
Any gremlin script that will output a set of edges can be used to contruct a subgraph.
Args:
gremlin_script (str): gremlin script to be executed
Raises:
RuntimeError: If the interactive instance is not running.
Returns:
:class:`Graph`: constructed subgraph. which is also stored in vineyard.
"""
if self._status != InteractiveQueryStatus.Running:
raise RuntimeError(
"Interactive query is unavailable with %s status.",
str(self._status))
now_time = datetime.datetime.now().strftime("%Y%m%d%H%M%S")
random_num = random.randint(0, 10000000)
graph_name = "%s_%s" % (str(now_time), str(random_num))
# create graph handle by name
self._client.submit(
"g.createGraph('%s').with('graphType', 'vineyard')" %
graph_name).all().result()
# start a thread to launch the graph
def load_subgraph(name):
import vineyard
import graphscope
graph = self._session.g(generate_eid=False)
graph = graph.add_vertices(
Loader(vineyard.ObjectName("__%s_vertex_stream" % name)))
graph = graph.add_edges(
Loader(vineyard.ObjectName("__%s_edge_stream" % name)))
graph._ensure_loaded()
logger.info("subgraph has been loaded")
return graph
pool = ThreadPoolExecutor()
subgraph_task = pool.submit(load_subgraph, (graph_name, ))
# add subgraph vertices and edges
subgraph_script = "%s.subgraph('%s').outputVineyard('%s')" % (
gremlin_script,
graph_name,
graph_name,
)
self._client.submit(subgraph_script).all().result()
return subgraph_task.result()
def execute(self, query):
"""Execute gremlin querying scripts.
Behind the scene, it uses `gremlinpython` to send the query.
Args:
query (str): Scripts that written in gremlin quering language.
Raises:
RuntimeError: If the interactive script is not running.
Returns:
execution results
"""
if self._status != InteractiveQueryStatus.Running:
raise RuntimeError(
"Interactive query is unavailable with %s status.",
str(self._status))
return self._client.submit(query)
def traversal_source(self):
"""Create a GraphTraversalSource and return.
Once `g` has been created using a connection, we can start to write
Gremlin traversals to query the remote graph.
Raises:
RuntimeError: If the interactive script is not running.
Examples:
.. code:: python
sess = graphscope.session()
graph = load_modern_graph(sess, modern_graph_data_dir)
interactive = sess.gremlin(graph)
g = interactive.traversal_source()
print(g.V().both()[1:3].toList())
print(g.V().both().name.toList())
Returns:
`GraphTraversalSource`
"""
if self._status != InteractiveQueryStatus.Running:
raise RuntimeError(
"Interactive query is unavailable with %s status.",
str(self._status))
return traversal().withRemote(
DriverRemoteConnection(self._graph_url, "g"))
def close(self):
"""Close interactive instance and release resources"""
if not self.closed():
self._session._close_interactive_instance(self)
self._status = InteractiveQueryStatus.Closed
#!/usr/bin/python3
# freshReadmeSnippet: example
from gremlin_python.driver.client import Client
from gremlin_python.driver.request import RequestMessage
from gremlin_python.driver.serializer import GraphSONMessageSerializer
serializer = GraphSONMessageSerializer()
# workaround to avoid exception on any opProcessor other than `standard` or `traversal`:
serializer.cypher = serializer.standard
client = Client('ws://localhost:8182/gremlin',
'g',
message_serializer=serializer)
cypherQuery = 'MATCH (n) RETURN n.name'
message = RequestMessage('cypher', 'eval', {'gremlin': cypherQuery})
results = client.submit(message).all().result()
# freshReadmeSnippet: example
print(results)
assert results == [{
'n.name': 'marko'
}, {
'n.name': 'vadas'
}, {
'n.name': 'lop'
}, {
'n.name': 'josh'
}, {
class InteractiveQuery(object):
"""`InteractiveQuery` class, is a simple wrapper around
`Gremlin-Python <https://pypi.org/project/gremlinpython/>`_,
which implements Gremlin within the Python language.
It also can expose gremlin endpoint which can be used by any other standard gremlin console.
It also has a method called `subgraph` which can extract some fragments
from origin graph, produce a new, smaller but concise graph stored in vineyard,
which lifetime is independent from the origin graph.
"""
def __init__(self, graphscope_session, object_id, front_ip, front_port):
self._graphscope_session = graphscope_session
self._object_id = object_id
self._graph_url = "ws://%s:%d/gremlin" % (front_ip, front_port)
self._client = Client(self._graph_url, "g")
self._closed = False
@property
def object_id(self):
"""Get the vineyard object id of graph.
Returns:
str: object id
"""
return self._object_id
@property
def graph_url(self):
"""The gremlin graph url can be used with any standard gremlin console, e.g., thinkerpop."""
return self._graph_url
def closed(self):
"""Return if the current instance is closed."""
return self._closed
def subgraph(self, gremlin_script):
"""Create a subgraph, which input is the result of the execution of `gremlin_script`.
Any gremlin script that will output a set of edges can be used to contruct a subgraph.
Args:
gremlin_script (str): gremlin script to be executed
Raises:
RuntimeError: If the interactive instance is closed.
Returns:
:class:`Graph`: constructed subgraph. which is also stored in vineyard.
"""
if self.closed():
raise RuntimeError("Interactive query is closed.")
now_time = datetime.datetime.now().strftime("%Y%m%d%H%M%S")
random_num = random.randint(0, 10000000)
graph_name = "%s_%s" % (str(now_time), str(random_num))
# create graph handle by name
self._client.submit(
"g.createGraph('%s').with('graphType', 'vineyard')" % graph_name
).all().result()
# start a thread to launch the graph
def load_subgraph(name):
import vineyard
host, port = self._graphscope_session.info["engine_config"][
"vineyard_rpc_endpoint"
].split(":")
client = vineyard.connect(host, int(port))
# get vertex/edge stream id
vstream = client.get_name("__%s_vertex_stream" % name, True)
estream = client.get_name("__%s_edge_stream" % name, True)
# invoke load_from
g = self._graphscope_session.load_from(
edges=[Loader(estream)],
vertices=[Loader(vstream)],
generate_eid=False,
)
client.put_name(vineyard.ObjectID(g.vineyard_id), graph_name)
logger.info("subgraph has been loaded")
return g
pool = ThreadPoolExecutor()
subgraph_task = pool.submit(load_subgraph, (graph_name,))
# add subgraph vertices and edges
subgraph_script = "%s.subgraph('%s').outputVineyard('%s')" % (
gremlin_script,
graph_name,
graph_name,
)
self._client.submit(subgraph_script).all().result()
return subgraph_task.result()
def execute(self, query):
"""Execute gremlin querying scripts.
Behind the scene, it uses `gremlinpython` to send the query.
Args:
query (str): Scripts that written in gremlin quering language.
Raises:
RuntimeError: If the interactive script is closed
Returns:
execution results
"""
if self.closed():
raise RuntimeError("Interactive query is closed.")
return self._client.submit(query)
def close(self):
"""Close interactive instance and release resources"""
if not self.closed():
self._closed = True
self._graphscope_session._close_interactive_instance(self)
def __init__(self, graphscope_session, object_id, front_ip, front_port):
self._graphscope_session = graphscope_session
self._object_id = object_id
self._graph_url = "ws://%s:%d/gremlin" % (front_ip, front_port)
self._client = Client(self._graph_url, "g")
self._closed = False
# actors/generator.py - unit test for the app.
#
# Copyright (c) 2018 SingularityNET
#
# Distributed under the MIT software license, see LICENSE file.
#
from gremlin_python.driver.client import Client
from gremlin_python.structure.graph import Graph
from gremlin_python.driver.driver_remote_connection import DriverRemoteConnection
from offernet_dsl.dsl import *
from offernet_dsl.ns import *
import offernet_dsl.utils as utils
import random
client = Client('ws://localhost:8182/gremlin', 'g')
rc = DriverRemoteConnection('ws://localhost:8182/gremlin', 'g')
g = Graph().traversal(OfferNetTraversalSource).withRemote(rc)
def add_random_agent():
"""
Add an agent to the network and connect to another agent randomly -- returns newly added actors id
Returns agent id
"""
new_agent_id = g.create_agent().properties(KEY_AGENT_ID).value().next()
random_agent = get_random_agent()
if random_agent is None:
random_agent = g.create_agent()
