def test_connection_share(client): # Overwrite fixture with pool_size=1 client client = Client('ws://localhost:45940/gremlin', 'gmodern', pool_size=1) g = Graph().traversal() t = g.V() message = RequestMessage('traversal', 'bytecode', { 'gremlin': t.bytecode, 'aliases': { 'g': 'gmodern' } }) message2 = RequestMessage('traversal', 'bytecode', { 'gremlin': t.bytecode, 'aliases': { 'g': 'gmodern' } }) future = client.submitAsync(message) future2 = client.submitAsync(message2) result_set2 = future2.result() assert len(result_set2.all().result()) == 6 # This future has to finish for the second to yield result - pool_size=1 assert future.done() result_set = future.result() assert len(result_set.all().result()) == 6
def authenticated_client(request): try: if request.param == 'basic': # turn off certificate verification for testing purposes only ssl_opts = ssl.SSLContext(ssl.PROTOCOL_TLSv1_2) ssl_opts.verify_mode = ssl.CERT_NONE client = Client(basic_url, 'gmodern', username='******', password='******', transport_factory=lambda: AiohttpTransport( ssl_options=ssl_opts)) elif request.param == 'kerberos': client = Client(kerberos_url, 'gmodern', kerberized_service=kerberized_service) else: raise ValueError("Invalid authentication option - " + request.param) except OSError: pytest.skip('Gremlin Server is not running') else: def fin(): client.close() request.addfinalizer(fin) return client
def test_multi_conn_pool(client): g = Graph().traversal() t = g.V() message = RequestMessage('traversal', 'bytecode', { 'gremlin': t.bytecode, 'aliases': { 'g': 'gmodern' } }) message2 = RequestMessage('traversal', 'bytecode', { 'gremlin': t.bytecode, 'aliases': { 'g': 'gmodern' } }) client = Client('ws://localhost:45940/gremlin', 'g', pool_size=1) future = client.submitAsync(message) future2 = client.submitAsync(message2) result_set2 = future2.result() assert len(result_set2.all().result()) == 6 # with connection pool `future` may or may not be done here result_set = future.result() assert len(result_set.all().result()) == 6
def gremlin_client(request): # type: (Any) -> Client url = getattr(request.module, 'url', 'ws://localhost:8182/gremlin') traversal_source = getattr(request.module, 'traversal_source', 'ConfigurationManagementGraph') client = Client(url, traversal_source) request.addfinalizer(lambda: client.close()) return client
def __init__(self, key, frontend_endpoint, object_id): self.key = key self.type = "gie_manager" # graph object id in vineyard self.object_id = object_id self.graph_url = f"ws://{frontend_endpoint}/gremlin" self.client = Client(self.graph_url, "g") self.closed = False
def _do_query(self, query): if self._gremlin_client is None: print("Creating new General Purpose Gremlin Client on %s:%s" % (self._url, self._port)) client = Client(f'wss://{self._url}:{self._port}/gremlin', 'g', pool_size=1) return _format_graph_results(client.submit(query))
def test_client_side_timeout_set_for_tornado(client): client = Client('ws://localhost:45940/gremlin', 'gmodern', transport_factory=lambda: TornadoTransport(read_timeout=1, write_timeout=1)) try: # should fire an exception client.submit('Thread.sleep(2000);1').all().result() assert False except TimeoutError as toerr: assert str(toerr) == "Operation timed out after 1 seconds"
def test_client_connection_pool_after_error(client): # Overwrite fixture with pool_size=1 client client = Client('ws://localhost:45940/gremlin', 'gmodern', pool_size=1) try: # should fire an exception client.submit('1/0').all().result() assert False except Exception: # expecting the pool size to be 1 again after query returned assert client.available_pool_size == 1
def __init__(self, session, object_id, front_ip=None, front_port=None): self._status = InteractiveQueryStatus.Initializing self._session = session self._object_id = object_id self._error_msg = "" if front_ip is not None and front_port is not None: self._graph_url = "ws://%s:%d/gremlin" % (front_ip, front_port) self._client = Client(self._graph_url, "g") else: self._graph_url = None self._client = None
def test_client_connection_pool_after_error(client): # Overwrite fixture with pool_size=1 client client = Client('ws://localhost:45940/gremlin', 'gmodern', pool_size=1) try: # should fire an exception client.submit('1/0').all().result() assert False except GremlinServerError as gse: # expecting the pool size to be 1 again after query returned assert gse.status_code == 597 assert client.available_pool_size == 1
def test_client_side_timeout_set_for_aiohttp(client): client = Client('ws://localhost:45940/gremlin', 'gmodern', transport_factory=lambda: AiohttpTransport( read_timeout=1, write_timeout=1)) try: # should fire an exception client.submit('Thread.sleep(2000);1').all().result() assert False except TimeoutError as err: # asyncio TimeoutError has no message. assert str(err) == ""
def test_multi_conn_pool(client): g = Graph().traversal() t = g.V() message = RequestMessage('traversal', 'bytecode', {'gremlin': t.bytecode, 'aliases': {'g': 'gmodern'}}) message2 = RequestMessage('traversal', 'bytecode', {'gremlin': t.bytecode, 'aliases': {'g': 'gmodern'}}) client = Client('ws://localhost:45940/gremlin', 'g', pool_size=1) future = client.submitAsync(message) future2 = client.submitAsync(message2) result_set2 = future2.result() assert len(result_set2.all().result()) == 6 # with connection pool `future` may or may not be done here result_set = future.result() assert len(result_set.all().result()) == 6
def find_node( unique_id: str, node_type: str, gremlin_client: client.Client ): ''' Construct the query to find the node with the specified property. Assumes that the specified property is unique to that node type ''' node_type_ids = { 'subreddit': 'subreddit_id', 'comment': 'id', 'user': '******' } query = f"g.V().hasLabel('{node_type}').has('{node_type_ids[node_type]}', '{unique_id}').next()" try: node = gremlin_client.submit(query).all().result() return query except: "Node doesn't exist! Cannot create edge" # Not sure if this is needed, but noticed that when Gremlin client throws an error # it will disconnect and hang.. gremlin_client = client.Client(JANUS_CONNECT, 'g') return None
def process_subreddit_node( property_keys: list, json_values: dict, gremlin_client: client.Client, execute=False, ): ''' Constructs a gremlin query to create subreddit node (if it doesn't already exists) ''' gremlin_query = construct_gremlin_query(property_keys, json_values, 'subreddit') if not check_existence_query(json_values['subreddit_id'], 'subreddit', gremlin_client) and execute: gremlin_client.submit(gremlin_query).all().result() gremlin_client.submit('g.tx().commit()').all().result() return gremlin_query else: return f"Subreddit node w/ name {json_values['subreddit_id']} already exists"
def connect(self, url="loclahost", port="8182", graph="g", **kwargs): self.URL = "ws://{}:{}/gremlin".format(url, port) self.graph = graph self.args = kwargs if not kwargs: graphson_reader = JanusGraphSONReader().build() graphson_writer = JanusGraphSONWriter().build() else: kwargs_provided = kwargs.keys() if all(k in self.connection_keyword_properties for k in kwargs_provided): pass else: raise AttributeError( "Additional parameters except url, port, graph needs to be the available" "Gremlin client connection parameters. Refer docs") if "graphson_reader" in kwargs and "graphson_writer" in kwargs: graphson_reader = kwargs["graphson_reader"] graphson_writer = kwargs["graphson_writer"] else: graphson_reader = None graphson_writer = None args = { k: v for k, v in self.args.items() if k not in ["graphson_reader", "graphson_writer"] } self.remote_connection = DriverRemoteConnection( self.URL, self.graph, graphson_reader=graphson_reader, graphson_writer=graphson_writer, **args) self.management_connection = Client(self.URL, self.graph, **args) return self
def test_client_pool_in_session(client): # Overwrite fixture with pool_size=2 client try: # should fire an exception client = Client('ws://localhost:45940/gremlin', 'g', session=str(uuid.uuid4()), pool_size=2) assert False except Exception: assert True
def test_connection_share(client): # Overwrite fixture with pool_size=1 client client = Client('ws://localhost:45940/gremlin', 'gmodern', pool_size=1) g = Graph().traversal() t = g.V() message = RequestMessage('traversal', 'bytecode', {'gremlin': t.bytecode, 'aliases': {'g': 'gmodern'}}) message2 = RequestMessage('traversal', 'bytecode', {'gremlin': t.bytecode, 'aliases': {'g': 'gmodern'}}) future = client.submitAsync(message) future2 = client.submitAsync(message2) result_set2 = future2.result() assert len(result_set2.all().result()) == 6 # This future has to finish for the second to yield result - pool_size=1 assert future.done() result_set = future.result() assert len(result_set.all().result()) == 6
def client(self, pool_size=None, max_workers=None): gremlin_endpoint = self.endpoints.gremlin_endpoint() request_parameters = gremlin_endpoint.prepare_request() signed_ws_request = httpclient.HTTPRequest( request_parameters.uri, headers=request_parameters.headers) return Client(signed_ws_request, 'g', pool_size=pool_size, max_workers=max_workers)
class InteractiveQueryManager(object): def __init__(self, key, frontend_endpoint, object_id): self.key = key self.type = "gie_manager" # graph object id in vineyard self.object_id = object_id self.graph_url = f"ws://{frontend_endpoint}/gremlin" self.client = Client(self.graph_url, "g") self.closed = False def submit(self, message, bindings=None, request_options=None): return self.client.submit(message, bindings, request_options) def close(self): try: self.client.close() except Exception: pass self.closed = True
def process_user_node( property_keys: list, json_values: dict, gremlin_client: client.Client, execute=False ): ''' Constructs a gremlin query to create subreddit node (if it doesn't already exists) Executes if flag is set to True. ''' gremlin_query = construct_gremlin_query(property_keys, json_values, 'user') if not check_existence_query(json_values['author'], 'user', gremlin_client) and execute: gremlin_client.submit(gremlin_query).all().result() gremlin_client.submit('g.tx().commit()').all().result() return gremlin_query else: return f"User node w/ unique id {json_values['author']} already exists"
def secure_client(request): try: client = Client('ws://localhost:45941/gremlin', 'gmodern', username='******', password='******') except OSError: pytest.skip('Gremlin Server is not running') else: def fin(): client.close() request.addfinalizer(fin) return client
def client(request): try: client = Client('ws://localhost:45940/gremlin', 'gmodern') except OSError: pytest.skip('Gremlin Server is not running') else: def fin(): client.close() request.addfinalizer(fin) return client
def client(self, pool_size=None, max_workers=None, **kwargs): gremlin_endpoint = self.endpoints.gremlin_endpoint() request_parameters = gremlin_endpoint.prepare_request() return Client(request_parameters.uri, 'g', pool_size=pool_size, max_workers=max_workers, headers=request_parameters.headers, **kwargs)
def client(request): try: client = Client(anonymous_url, 'gmodern') except OSError: pytest.skip('Gremlin Server is not running') else: def fin(): client.close() request.addfinalizer(fin) return client
def test_multi_request_in_session(client): # Overwrite fixture with session client session_id = str(uuid.uuid4()) client = Client('ws://localhost:45940/gremlin', 'g', session=session_id) assert client.submit('x = 1').all().result()[0] == 1 assert client.submit('x + 2').all().result()[0] == 3 client.close() # attempt reconnect to session and make sure "x" is no longer a thing client = Client('ws://localhost:45940/gremlin', 'g', session=session_id) try: # should fire an exception client.submit('x').all().result() assert False except Exception: assert True
def authenticated_client(request): try: if request.param == 'basic': client = Client(basic_url, 'gmodern', username='******', password='******') elif request.param == 'kerberos': client = Client(kerberos_url, 'gmodern', kerberized_service=kerberized_service) else: raise ValueError("Invalid authentication option - " + request.param) except OSError: pytest.skip('Gremlin Server is not running') else: def fin(): client.close() request.addfinalizer(fin) return client
def test_client_message_too_big(client): try: client = Client("http://localhost", 'g', max_content_length=1024) client.submit("1+1").all().result() assert False except Exception: assert True finally: client.close()
def check_existence_query( value: str, node_type: str, gremlin_client: client.Client, ): ''' Helper function to check if node exists ''' node_type_ids = { 'subreddit': 'subreddit_id', 'comment': 'id', 'user': '******' } existence_query = f"g.V().hasLabel('{node_type}').has('{node_type_ids[node_type]}', '{value}').hasNext()" return gremlin_client.submit(existence_query).all().result()[0]
class InteractiveQuery(object): """`InteractiveQuery` class, is a simple wrapper around `Gremlin-Python <https://pypi.org/project/gremlinpython/>`_, which implements Gremlin within the Python language. It also can expose gremlin endpoint which can be used by any other standard gremlin console, with the method `graph_url()`. It also has a method called `subgraph` which can extract some fragments from origin graph, produce a new, smaller but concise graph stored in vineyard, which lifetime is independent from the origin graph. User can either use `execute()` to submit a script, or use `traversal_source()` to get a `GraphTraversalSource` for further traversal. """ def __init__(self, session, object_id, front_ip=None, front_port=None): self._status = InteractiveQueryStatus.Initializing self._session = session self._object_id = object_id self._error_msg = "" if front_ip is not None and front_port is not None: self._graph_url = "ws://%s:%d/gremlin" % (front_ip, front_port) self._client = Client(self._graph_url, "g") else: self._graph_url = None self._client = None def __repr__(self): return f"graphscope.InteractiveQuery <{self._graph_url}>" @property def object_id(self): """Get the vineyard object id of graph. Returns: str: object id """ return self._object_id @property def graph_url(self): """The gremlin graph url can be used with any standard gremlin console, e.g., thinkerpop.""" return self._graph_url @property def status(self): return self._status @status.setter def status(self, value): self._status = value @property def error_msg(self): return self._error_msg @error_msg.setter def error_msg(self, error_msg): self._error_msg = error_msg def set_frontend(self, front_ip, front_port): self._graph_url = "ws://%s:%d/gremlin" % (front_ip, front_port) self._client = Client(self._graph_url, "g") def closed(self): """Return if the current instance is closed.""" return self._status == InteractiveQueryStatus.Closed def subgraph(self, gremlin_script): """Create a subgraph, which input is the result of the execution of `gremlin_script`. Any gremlin script that will output a set of edges can be used to contruct a subgraph. Args: gremlin_script (str): gremlin script to be executed Raises: RuntimeError: If the interactive instance is not running. Returns: :class:`Graph`: constructed subgraph. which is also stored in vineyard. """ if self._status != InteractiveQueryStatus.Running: raise RuntimeError( "Interactive query is unavailable with %s status.", str(self._status)) now_time = datetime.datetime.now().strftime("%Y%m%d%H%M%S") random_num = random.randint(0, 10000000) graph_name = "%s_%s" % (str(now_time), str(random_num)) # create graph handle by name self._client.submit( "g.createGraph('%s').with('graphType', 'vineyard')" % graph_name).all().result() # start a thread to launch the graph def load_subgraph(name): import vineyard import graphscope graph = self._session.g(generate_eid=False) graph = graph.add_vertices( Loader(vineyard.ObjectName("__%s_vertex_stream" % name))) graph = graph.add_edges( Loader(vineyard.ObjectName("__%s_edge_stream" % name))) graph._ensure_loaded() logger.info("subgraph has been loaded") return graph pool = ThreadPoolExecutor() subgraph_task = pool.submit(load_subgraph, (graph_name, )) # add subgraph vertices and edges subgraph_script = "%s.subgraph('%s').outputVineyard('%s')" % ( gremlin_script, graph_name, graph_name, ) self._client.submit(subgraph_script).all().result() return subgraph_task.result() def execute(self, query): """Execute gremlin querying scripts. Behind the scene, it uses `gremlinpython` to send the query. Args: query (str): Scripts that written in gremlin quering language. Raises: RuntimeError: If the interactive script is not running. Returns: execution results """ if self._status != InteractiveQueryStatus.Running: raise RuntimeError( "Interactive query is unavailable with %s status.", str(self._status)) return self._client.submit(query) def traversal_source(self): """Create a GraphTraversalSource and return. Once `g` has been created using a connection, we can start to write Gremlin traversals to query the remote graph. Raises: RuntimeError: If the interactive script is not running. Examples: .. code:: python sess = graphscope.session() graph = load_modern_graph(sess, modern_graph_data_dir) interactive = sess.gremlin(graph) g = interactive.traversal_source() print(g.V().both()[1:3].toList()) print(g.V().both().name.toList()) Returns: `GraphTraversalSource` """ if self._status != InteractiveQueryStatus.Running: raise RuntimeError( "Interactive query is unavailable with %s status.", str(self._status)) return traversal().withRemote( DriverRemoteConnection(self._graph_url, "g")) def close(self): """Close interactive instance and release resources""" if not self.closed(): self._session._close_interactive_instance(self) self._status = InteractiveQueryStatus.Closed
#!/usr/bin/python3 # freshReadmeSnippet: example from gremlin_python.driver.client import Client from gremlin_python.driver.request import RequestMessage from gremlin_python.driver.serializer import GraphSONMessageSerializer serializer = GraphSONMessageSerializer() # workaround to avoid exception on any opProcessor other than `standard` or `traversal`: serializer.cypher = serializer.standard client = Client('ws://localhost:8182/gremlin', 'g', message_serializer=serializer) cypherQuery = 'MATCH (n) RETURN n.name' message = RequestMessage('cypher', 'eval', {'gremlin': cypherQuery}) results = client.submit(message).all().result() # freshReadmeSnippet: example print(results) assert results == [{ 'n.name': 'marko' }, { 'n.name': 'vadas' }, { 'n.name': 'lop' }, { 'n.name': 'josh' }, {
class InteractiveQuery(object): """`InteractiveQuery` class, is a simple wrapper around `Gremlin-Python <https://pypi.org/project/gremlinpython/>`_, which implements Gremlin within the Python language. It also can expose gremlin endpoint which can be used by any other standard gremlin console. It also has a method called `subgraph` which can extract some fragments from origin graph, produce a new, smaller but concise graph stored in vineyard, which lifetime is independent from the origin graph. """ def __init__(self, graphscope_session, object_id, front_ip, front_port): self._graphscope_session = graphscope_session self._object_id = object_id self._graph_url = "ws://%s:%d/gremlin" % (front_ip, front_port) self._client = Client(self._graph_url, "g") self._closed = False @property def object_id(self): """Get the vineyard object id of graph. Returns: str: object id """ return self._object_id @property def graph_url(self): """The gremlin graph url can be used with any standard gremlin console, e.g., thinkerpop.""" return self._graph_url def closed(self): """Return if the current instance is closed.""" return self._closed def subgraph(self, gremlin_script): """Create a subgraph, which input is the result of the execution of `gremlin_script`. Any gremlin script that will output a set of edges can be used to contruct a subgraph. Args: gremlin_script (str): gremlin script to be executed Raises: RuntimeError: If the interactive instance is closed. Returns: :class:`Graph`: constructed subgraph. which is also stored in vineyard. """ if self.closed(): raise RuntimeError("Interactive query is closed.") now_time = datetime.datetime.now().strftime("%Y%m%d%H%M%S") random_num = random.randint(0, 10000000) graph_name = "%s_%s" % (str(now_time), str(random_num)) # create graph handle by name self._client.submit( "g.createGraph('%s').with('graphType', 'vineyard')" % graph_name ).all().result() # start a thread to launch the graph def load_subgraph(name): import vineyard host, port = self._graphscope_session.info["engine_config"][ "vineyard_rpc_endpoint" ].split(":") client = vineyard.connect(host, int(port)) # get vertex/edge stream id vstream = client.get_name("__%s_vertex_stream" % name, True) estream = client.get_name("__%s_edge_stream" % name, True) # invoke load_from g = self._graphscope_session.load_from( edges=[Loader(estream)], vertices=[Loader(vstream)], generate_eid=False, ) client.put_name(vineyard.ObjectID(g.vineyard_id), graph_name) logger.info("subgraph has been loaded") return g pool = ThreadPoolExecutor() subgraph_task = pool.submit(load_subgraph, (graph_name,)) # add subgraph vertices and edges subgraph_script = "%s.subgraph('%s').outputVineyard('%s')" % ( gremlin_script, graph_name, graph_name, ) self._client.submit(subgraph_script).all().result() return subgraph_task.result() def execute(self, query): """Execute gremlin querying scripts. Behind the scene, it uses `gremlinpython` to send the query. Args: query (str): Scripts that written in gremlin quering language. Raises: RuntimeError: If the interactive script is closed Returns: execution results """ if self.closed(): raise RuntimeError("Interactive query is closed.") return self._client.submit(query) def close(self): """Close interactive instance and release resources""" if not self.closed(): self._closed = True self._graphscope_session._close_interactive_instance(self)
def __init__(self, graphscope_session, object_id, front_ip, front_port): self._graphscope_session = graphscope_session self._object_id = object_id self._graph_url = "ws://%s:%d/gremlin" % (front_ip, front_port) self._client = Client(self._graph_url, "g") self._closed = False
# actors/generator.py - unit test for the app. # # Copyright (c) 2018 SingularityNET # # Distributed under the MIT software license, see LICENSE file. # from gremlin_python.driver.client import Client from gremlin_python.structure.graph import Graph from gremlin_python.driver.driver_remote_connection import DriverRemoteConnection from offernet_dsl.dsl import * from offernet_dsl.ns import * import offernet_dsl.utils as utils import random client = Client('ws://localhost:8182/gremlin', 'g') rc = DriverRemoteConnection('ws://localhost:8182/gremlin', 'g') g = Graph().traversal(OfferNetTraversalSource).withRemote(rc) def add_random_agent(): """ Add an agent to the network and connect to another agent randomly -- returns newly added actors id Returns agent id """ new_agent_id = g.create_agent().properties(KEY_AGENT_ID).value().next() random_agent = get_random_agent() if random_agent is None: random_agent = g.create_agent()