def test_client_message_too_big(client): try: client = Client("http://localhost", 'g', max_content_length=1024) client.submit("1+1").all().result() assert False except Exception: assert True finally: client.close()
def test_client_side_timeout_set_for_tornado(client): client = Client('ws://localhost:45940/gremlin', 'gmodern', transport_factory=lambda: TornadoTransport(read_timeout=1, write_timeout=1)) try: # should fire an exception client.submit('Thread.sleep(2000);1').all().result() assert False except TimeoutError as toerr: assert str(toerr) == "Operation timed out after 1 seconds"
def test_client_connection_pool_after_error(client): # Overwrite fixture with pool_size=1 client client = Client('ws://localhost:45940/gremlin', 'gmodern', pool_size=1) try: # should fire an exception client.submit('1/0').all().result() assert False except Exception: # expecting the pool size to be 1 again after query returned assert client.available_pool_size == 1
def test_client_connection_pool_after_error(client): # Overwrite fixture with pool_size=1 client client = Client('ws://localhost:45940/gremlin', 'gmodern', pool_size=1) try: # should fire an exception client.submit('1/0').all().result() assert False except GremlinServerError as gse: # expecting the pool size to be 1 again after query returned assert gse.status_code == 597 assert client.available_pool_size == 1
def test_client_side_timeout_set_for_aiohttp(client): client = Client('ws://localhost:45940/gremlin', 'gmodern', transport_factory=lambda: AiohttpTransport( read_timeout=1, write_timeout=1)) try: # should fire an exception client.submit('Thread.sleep(2000);1').all().result() assert False except TimeoutError as err: # asyncio TimeoutError has no message. assert str(err) == ""
def find_node( unique_id: str, node_type: str, gremlin_client: client.Client ): ''' Construct the query to find the node with the specified property. Assumes that the specified property is unique to that node type ''' node_type_ids = { 'subreddit': 'subreddit_id', 'comment': 'id', 'user': '******' } query = f"g.V().hasLabel('{node_type}').has('{node_type_ids[node_type]}', '{unique_id}').next()" try: node = gremlin_client.submit(query).all().result() return query except: "Node doesn't exist! Cannot create edge" # Not sure if this is needed, but noticed that when Gremlin client throws an error # it will disconnect and hang.. gremlin_client = client.Client(JANUS_CONNECT, 'g') return None
def process_subreddit_node( property_keys: list, json_values: dict, gremlin_client: client.Client, execute=False, ): ''' Constructs a gremlin query to create subreddit node (if it doesn't already exists) ''' gremlin_query = construct_gremlin_query(property_keys, json_values, 'subreddit') if not check_existence_query(json_values['subreddit_id'], 'subreddit', gremlin_client) and execute: gremlin_client.submit(gremlin_query).all().result() gremlin_client.submit('g.tx().commit()').all().result() return gremlin_query else: return f"Subreddit node w/ name {json_values['subreddit_id']} already exists"
def _do_query(self, query): if self._gremlin_client is None: print("Creating new General Purpose Gremlin Client on %s:%s" % (self._url, self._port)) client = Client(f'wss://{self._url}:{self._port}/gremlin', 'g', pool_size=1) return _format_graph_results(client.submit(query))
def test_multi_request_in_session(client): # Overwrite fixture with session client session_id = str(uuid.uuid4()) client = Client('ws://localhost:45940/gremlin', 'g', session=session_id) assert client.submit('x = 1').all().result()[0] == 1 assert client.submit('x + 2').all().result()[0] == 3 client.close() # attempt reconnect to session and make sure "x" is no longer a thing client = Client('ws://localhost:45940/gremlin', 'g', session=session_id) try: # should fire an exception client.submit('x').all().result() assert False except Exception: assert True
def process_user_node( property_keys: list, json_values: dict, gremlin_client: client.Client, execute=False ): ''' Constructs a gremlin query to create subreddit node (if it doesn't already exists) Executes if flag is set to True. ''' gremlin_query = construct_gremlin_query(property_keys, json_values, 'user') if not check_existence_query(json_values['author'], 'user', gremlin_client) and execute: gremlin_client.submit(gremlin_query).all().result() gremlin_client.submit('g.tx().commit()').all().result() return gremlin_query else: return f"User node w/ unique id {json_values['author']} already exists"
def check_existence_query( value: str, node_type: str, gremlin_client: client.Client, ): ''' Helper function to check if node exists ''' node_type_ids = { 'subreddit': 'subreddit_id', 'comment': 'id', 'user': '******' } existence_query = f"g.V().hasLabel('{node_type}').has('{node_type_ids[node_type]}', '{value}').hasNext()" return gremlin_client.submit(existence_query).all().result()[0]
class InteractiveQueryManager(object): def __init__(self, key, frontend_endpoint, object_id): self.key = key self.type = "gie_manager" # graph object id in vineyard self.object_id = object_id self.graph_url = f"ws://{frontend_endpoint}/gremlin" self.client = Client(self.graph_url, "g") self.closed = False def submit(self, message, bindings=None, request_options=None): return self.client.submit(message, bindings, request_options) def close(self): try: self.client.close() except Exception: pass self.closed = True
# freshReadmeSnippet: example from gremlin_python.driver.client import Client from gremlin_python.driver.request import RequestMessage from gremlin_python.driver.serializer import GraphSONMessageSerializer serializer = GraphSONMessageSerializer() # workaround to avoid exception on any opProcessor other than `standard` or `traversal`: serializer.cypher = serializer.standard client = Client('ws://localhost:8182/gremlin', 'g', message_serializer=serializer) cypherQuery = 'MATCH (n) RETURN n.name' message = RequestMessage('cypher', 'eval', {'gremlin': cypherQuery}) results = client.submit(message).all().result() # freshReadmeSnippet: example print(results) assert results == [{ 'n.name': 'marko' }, { 'n.name': 'vadas' }, { 'n.name': 'lop' }, { 'n.name': 'josh' }, { 'n.name': 'ripple' }, {
class JanusGraph(object): def __init__(self): self.remote_connection = None self.management_connection = None self.URL = None self.graph = None self.connection_keyword_properties = [ "protocol_factory", "transport_factory", "pool_size", "max_workers", "username", "password", "message_serializer", "graphson_reader", "graphson_writer" ] self.args = dict() pass def connect(self, url="loclahost", port="8182", graph="g", **kwargs): self.URL = "ws://{}:{}/gremlin".format(url, port) self.graph = graph self.args = kwargs if not kwargs: graphson_reader = JanusGraphSONReader().build() graphson_writer = JanusGraphSONWriter().build() else: kwargs_provided = kwargs.keys() if all(k in self.connection_keyword_properties for k in kwargs_provided): pass else: raise AttributeError( "Additional parameters except url, port, graph needs to be the available" "Gremlin client connection parameters. Refer docs") if "graphson_reader" in kwargs and "graphson_writer" in kwargs: graphson_reader = kwargs["graphson_reader"] graphson_writer = kwargs["graphson_writer"] else: graphson_reader = None graphson_writer = None args = { k: v for k, v in self.args.items() if k not in ["graphson_reader", "graphson_writer"] } self.remote_connection = DriverRemoteConnection( self.URL, self.graph, graphson_reader=graphson_reader, graphson_writer=graphson_writer, **args) self.management_connection = Client(self.URL, self.graph, **args) return self def traversal(self): if self.remote_connection is None: raise AttributeError( "The Graph is not connected to any Remote Graph. Use connect() first." ) g = Graph().traversal().withRemote(self.remote_connection) return g def openManagement(self): if self.management_connection is None: raise AttributeError( "The Graph is not connected to any Remote Graph. Use connect() first." ) mgmt = JanusGraphManagement(self.management_connection) return mgmt def close(self): self.remote_connection.close() self.management_connection.close() return True def drop(self): query = "JanusGraphFactory.drop(graph);" result_set = self.management_connection.submit(query) future_results = result_set.all() results = future_results.result() self.close() return results
class InteractiveQuery(object): """`InteractiveQuery` class, is a simple wrapper around `Gremlin-Python <https://pypi.org/project/gremlinpython/>`_, which implements Gremlin within the Python language. It also can expose gremlin endpoint which can be used by any other standard gremlin console. It also has a method called `subgraph` which can extract some fragments from origin graph, produce a new, smaller but concise graph stored in vineyard, which lifetime is independent from the origin graph. """ def __init__(self, graphscope_session, object_id, front_ip, front_port): self._graphscope_session = graphscope_session self._object_id = object_id self._graph_url = "ws://%s:%d/gremlin" % (front_ip, front_port) self._client = Client(self._graph_url, "g") self._closed = False @property def object_id(self): """Get the vineyard object id of graph. Returns: str: object id """ return self._object_id @property def graph_url(self): """The gremlin graph url can be used with any standard gremlin console, e.g., thinkerpop.""" return self._graph_url def closed(self): """Return if the current instance is closed.""" return self._closed def subgraph(self, gremlin_script): """Create a subgraph, which input is the result of the execution of `gremlin_script`. Any gremlin script that will output a set of edges can be used to contruct a subgraph. Args: gremlin_script (str): gremlin script to be executed Raises: RuntimeError: If the interactive instance is closed. Returns: :class:`Graph`: constructed subgraph. which is also stored in vineyard. """ if self.closed(): raise RuntimeError("Interactive query is closed.") now_time = datetime.datetime.now().strftime("%Y%m%d%H%M%S") random_num = random.randint(0, 10000000) graph_name = "%s_%s" % (str(now_time), str(random_num)) # create graph handle by name self._client.submit( "g.createGraph('%s').with('graphType', 'vineyard')" % graph_name ).all().result() # start a thread to launch the graph def load_subgraph(name): import vineyard host, port = self._graphscope_session.info["engine_config"][ "vineyard_rpc_endpoint" ].split(":") client = vineyard.connect(host, int(port)) # get vertex/edge stream id vstream = client.get_name("__%s_vertex_stream" % name, True) estream = client.get_name("__%s_edge_stream" % name, True) # invoke load_from g = self._graphscope_session.load_from( edges=[Loader(estream)], vertices=[Loader(vstream)], generate_eid=False, ) client.put_name(vineyard.ObjectID(g.vineyard_id), graph_name) logger.info("subgraph has been loaded") return g pool = ThreadPoolExecutor() subgraph_task = pool.submit(load_subgraph, (graph_name,)) # add subgraph vertices and edges subgraph_script = "%s.subgraph('%s').outputVineyard('%s')" % ( gremlin_script, graph_name, graph_name, ) self._client.submit(subgraph_script).all().result() return subgraph_task.result() def execute(self, query): """Execute gremlin querying scripts. Behind the scene, it uses `gremlinpython` to send the query. Args: query (str): Scripts that written in gremlin quering language. Raises: RuntimeError: If the interactive script is closed Returns: execution results """ if self.closed(): raise RuntimeError("Interactive query is closed.") return self._client.submit(query) def close(self): """Close interactive instance and release resources""" if not self.closed(): self._closed = True self._graphscope_session._close_interactive_instance(self)
class InteractiveQuery(object): """`InteractiveQuery` class, is a simple wrapper around `Gremlin-Python <https://pypi.org/project/gremlinpython/>`_, which implements Gremlin within the Python language. It also can expose gremlin endpoint which can be used by any other standard gremlin console, with the method `graph_url()`. It also has a method called `subgraph` which can extract some fragments from origin graph, produce a new, smaller but concise graph stored in vineyard, which lifetime is independent from the origin graph. User can either use `execute()` to submit a script, or use `traversal_source()` to get a `GraphTraversalSource` for further traversal. """ def __init__(self, session, object_id, front_ip=None, front_port=None): self._status = InteractiveQueryStatus.Initializing self._session = session self._object_id = object_id self._error_msg = "" if front_ip is not None and front_port is not None: self._graph_url = "ws://%s:%d/gremlin" % (front_ip, front_port) self._client = Client(self._graph_url, "g") else: self._graph_url = None self._client = None def __repr__(self): return f"graphscope.InteractiveQuery <{self._graph_url}>" @property def object_id(self): """Get the vineyard object id of graph. Returns: str: object id """ return self._object_id @property def graph_url(self): """The gremlin graph url can be used with any standard gremlin console, e.g., thinkerpop.""" return self._graph_url @property def status(self): return self._status @status.setter def status(self, value): self._status = value @property def error_msg(self): return self._error_msg @error_msg.setter def error_msg(self, error_msg): self._error_msg = error_msg def set_frontend(self, front_ip, front_port): self._graph_url = "ws://%s:%d/gremlin" % (front_ip, front_port) self._client = Client(self._graph_url, "g") def closed(self): """Return if the current instance is closed.""" return self._status == InteractiveQueryStatus.Closed def subgraph(self, gremlin_script): """Create a subgraph, which input is the result of the execution of `gremlin_script`. Any gremlin script that will output a set of edges can be used to contruct a subgraph. Args: gremlin_script (str): gremlin script to be executed Raises: RuntimeError: If the interactive instance is not running. Returns: :class:`Graph`: constructed subgraph. which is also stored in vineyard. """ if self._status != InteractiveQueryStatus.Running: raise RuntimeError( "Interactive query is unavailable with %s status.", str(self._status)) now_time = datetime.datetime.now().strftime("%Y%m%d%H%M%S") random_num = random.randint(0, 10000000) graph_name = "%s_%s" % (str(now_time), str(random_num)) # create graph handle by name self._client.submit( "g.createGraph('%s').with('graphType', 'vineyard')" % graph_name).all().result() # start a thread to launch the graph def load_subgraph(name): import vineyard import graphscope graph = self._session.g(generate_eid=False) graph = graph.add_vertices( Loader(vineyard.ObjectName("__%s_vertex_stream" % name))) graph = graph.add_edges( Loader(vineyard.ObjectName("__%s_edge_stream" % name))) graph._ensure_loaded() logger.info("subgraph has been loaded") return graph pool = ThreadPoolExecutor() subgraph_task = pool.submit(load_subgraph, (graph_name, )) # add subgraph vertices and edges subgraph_script = "%s.subgraph('%s').outputVineyard('%s')" % ( gremlin_script, graph_name, graph_name, ) self._client.submit(subgraph_script).all().result() return subgraph_task.result() def execute(self, query): """Execute gremlin querying scripts. Behind the scene, it uses `gremlinpython` to send the query. Args: query (str): Scripts that written in gremlin quering language. Raises: RuntimeError: If the interactive script is not running. Returns: execution results """ if self._status != InteractiveQueryStatus.Running: raise RuntimeError( "Interactive query is unavailable with %s status.", str(self._status)) return self._client.submit(query) def traversal_source(self): """Create a GraphTraversalSource and return. Once `g` has been created using a connection, we can start to write Gremlin traversals to query the remote graph. Raises: RuntimeError: If the interactive script is not running. Examples: .. code:: python sess = graphscope.session() graph = load_modern_graph(sess, modern_graph_data_dir) interactive = sess.gremlin(graph) g = interactive.traversal_source() print(g.V().both()[1:3].toList()) print(g.V().both().name.toList()) Returns: `GraphTraversalSource` """ if self._status != InteractiveQueryStatus.Running: raise RuntimeError( "Interactive query is unavailable with %s status.", str(self._status)) return traversal().withRemote( DriverRemoteConnection(self._graph_url, "g")) def close(self): """Close interactive instance and release resources""" if not self.closed(): self._session._close_interactive_instance(self) self._status = InteractiveQueryStatus.Closed
# Connect to gremlin server # **************************************************** # In[3]: # --- Create client to do low-level operations with gremlin server --- client = Client( "ws://192.168.1.52:8182/gremlin", # URL of gremlin server (janusgraph application interface) "g", # name of traversal object username="", # username to access Janusgraph password="") # password to access Janusgraph print(client) # In[4]: res = client.submit('Gremlin.version()') f, x = CheckResult(res) print(f) print('Version of gremlin server is ', x) # In[ ]: # **************************************************** # Create a new graph property with Janusgraph # **************************************************** # In[ ]: # --- Create graph on the remote gremlin server: --- # res = client.submit("graph = TinkerGraph.open()") # with Gremlin Server # res = client.submit("graph = JanusGraphFactory.open('inmemory')") # with JanusGraph