def runQueries(cls, queries): for query in queries: try: do_gremlin_query(query, cls.host, cls.port, cls.ssl, cls.client_provider) except Exception as e: logger.error(f'query {query} failed due to {e}')
def setUp(self): super().setUp() self.client_provider = create_client_provider( self.auth_mode, self.iam_credentials_provider_type) query_check_for_airports = "g.V('3684').outE().inV().has(id, '3444')" res = do_gremlin_query(query_check_for_airports, self.host, self.port, self.ssl, self.client_provider) if len(res) < 1: logging.info( 'did not find final airports edge, seeding database now...') airport_queries = get_queries('gremlin', 'airports') for q in airport_queries: lines = q['content'].splitlines() for i in range(len(lines)): line = lines[i] logging.debug( f'executing line {i} of {len(lines)} for seeding DataDrivenGremlinTest' ) # we are deciding to try except because we do not know if the database # we are connecting to has a partially complete set of airports data or not. try: do_gremlin_query(line, self.host, self.port, self.ssl, self.client_provider) except Exception as e: logging.error(f'query {q} failed due to {e}') continue
def test_add_path_with_repeat(self): query = "g.V().has('airport', 'code', 'ANC').repeat(outE().inV().simplePath()).times(2).path().by('code').by()" results = do_gremlin_query(query, self.host, self.port, self.ssl, self.client_provider) gremlin_network = GremlinNetwork() gremlin_network.add_results(results) self.assertEqual('route', gremlin_network.graph.edges[('ANC', 'BLI', '5276')]['label'])
def test_path_without_by_nodes_have_ids(self): query = "g.V().has('code', 'AUS').outE().inV().outE().inV().has('code', 'SEA').path()" results = do_gremlin_query(query, self.host, self.port, self.ssl, self.client_provider) gremlin_network = GremlinNetwork() gremlin_network.add_results(results) node = gremlin_network.graph.nodes.get('9') self.assertIsNotNone(node)
def test_do_gremlin_query(self): client_provider = ClientProvider() query = 'g.V().limit(1)' results = do_gremlin_query(query, self.host, self.port, self.ssl, client_provider) self.assertEqual(type(results), list)
def test_path_without_by_ine_has_arrows(self): query = "g.V().hasLabel('airport').has('code', 'SEA').inE().outV().path()" results = do_gremlin_query(query, self.host, self.port, self.ssl, self.client_provider) gremlin_network = GremlinNetwork() gremlin_network.add_results(results) edge = gremlin_network.graph.edges[('3670', '22', '53637')] self.assertTrue('arrows' not in edge)
def test_do_gremlin_query_with_iam(self): client_provider = create_client_provider( AuthModeEnum.IAM, IAMAuthCredentialsProvider.ENV) query = 'g.V().limit(1)' results = do_gremlin_query(query, self.host, self.port, self.ssl, client_provider) self.assertEqual(type(results), list)
def test_add_path_with_edge_object(self): query = "g.V().has('airport','code','AUS').outE().inV().path().by('code').by().limit(10)" results = do_gremlin_query(query, self.host, self.port, self.ssl, self.client_provider) gn = GremlinNetwork() pattern = [PathPattern.V, PathPattern.OUT_E, PathPattern.IN_V] gn.add_results_with_pattern(results, pattern) self.assertEqual(11, len(gn.graph.nodes)) self.assertEqual(10, len(gn.graph.edges))
def test_valuemap_without_ids(self): query = "g.V().has('code', 'ANC').out().path().by(valueMap()).limit(10)" results = do_gremlin_query(query, self.host, self.port, self.ssl, self.client_provider) gremlin_network = GremlinNetwork() gremlin_network.add_results(results) for n in gremlin_network.graph.nodes: node = gremlin_network.graph.nodes.get(n) self.assertEqual(gremlin_network.label_max_length, len(node['label']))
def test_add_value_map_to_network(self): airports_path_query = "g.V().has('code', 'SEA').outE().inV().path().by(valueMap(true))" results = do_gremlin_query(airports_path_query, self.host, self.port, self.ssl, self.client_provider) gremlin_network = GremlinNetwork() gremlin_network.add_results(results) edge_id = '4406' expected_label = 'route' actual_label = gremlin_network.graph.nodes.get(edge_id)['label'] self.assertEqual(expected_label, actual_label)
def test_add_paths_with_bad_pattern(self): query = "g.V().out().out().path().limit(10)" results = do_gremlin_query(query, self.host, self.port, self.ssl, self.client_provider) gremlin_network = GremlinNetwork() gremlin_network.add_results(results) # confirm that all edges have empty labels and are undirected for e in gremlin_network.graph.edges: edge = gremlin_network.graph.edges[e] self.assertEqual('', edge['label']) self.assertFalse(edge['arrows']['to']['enabled'])
def test_add_paths_to_network(self): airports_path_query = "g.V().has('code', 'SEA').outE().inV().path()" results = do_gremlin_query(airports_path_query, self.host, self.port, self.ssl, self.client_provider) gremlin_network = GremlinNetwork() gremlin_network.add_results(results) sea_code = '22' aus_code = '3' edge_id = '4406' expected_label = 'route' actual_label = gremlin_network.graph[sea_code][aus_code][edge_id]['label'] self.assertEqual(expected_label, actual_label)
def test_add_entire_path(self): sea_to_bmi = "g.V().has('code', 'SEA').outE().inV().has('code', 'ORD').outE().inV().has('code', 'BMI').path()" results = do_gremlin_query(sea_to_bmi, self.host, self.port, self.ssl, self.client_provider) gremlin_network = GremlinNetwork() gremlin_network.add_results(results) self.assertEqual(3, len(gremlin_network.graph.nodes)) self.assertEqual(2, len(gremlin_network.graph.edges)) self.assertIsNotNone(gremlin_network.graph.nodes.get('22')) self.assertIsNotNone(gremlin_network.graph.nodes.get('18')) self.assertIsNotNone(gremlin_network.graph.nodes.get('359')) self.assertTrue(gremlin_network.graph.has_edge('22', '18', '4420')) self.assertTrue(gremlin_network.graph.has_edge('18', '359', '7126'))
def test_path_with_dict(self): query = """g.V().has('airport','code','CZM'). out('route'). path(). by(valueMap('code','city','region','desc','lat','lon'). order(local). by(keys))""" results = do_gremlin_query(query, self.host, self.port, self.ssl, self.client_provider) gn = GremlinNetwork() pattern = [PathPattern.V, PathPattern.IN_V] gn.add_results_with_pattern(results, pattern) self.assertEqual(12, len(gn.graph.nodes)) self.assertEqual(11, len(gn.graph.edges))
def test_do_gremlin_query(self): query = """ g.V().hasLabel("Interest").as("int") .in("interested") .out("prioritized").as("exp") .select("int","exp") .by("value") .by("name") .groupCount().unfold() """ results = do_gremlin_query(query, self.host, self.port, self.ssl, self.client_provider) self.assertEqual(results, [{(('int', 4), ('exp', 'P1')): 1}])
def test_out_v_unhashable_dict(self): query = """g.V(). hasLabel('country'). has('desc','Jamaica'). out(). path(). by(valueMap())""" results = do_gremlin_query(query, self.host, self.port, self.ssl, self.client_provider) gn = GremlinNetwork() pattern = [PathPattern.V, PathPattern.OUT_V] gn.add_results_with_pattern(results, pattern) node = gn.graph.nodes.get( 'graph_notebook-2f363b2fa995d0567e638a240efd0a26') self.assertEqual(["Jamaica"], node['properties']['desc'])
def test_add_path_by_dist(self): query = """g.V().has('airport','code','AUS'). repeat(outE().inV().simplePath()). until(has('code','WLG')). limit(5). path(). by('code'). by('dist')""" results = do_gremlin_query(query, self.host, self.port, self.ssl, self.client_provider) gn = GremlinNetwork() pattern = [ PathPattern.V, PathPattern.OUT_E, PathPattern.IN_V, PathPattern.OUT_E ] gn.add_results_with_pattern(results, pattern) self.assertEqual(8, len(gn.graph.nodes)) self.assertEqual(11, len(gn.graph.edges))
def do_gremlin_query_save_results(self, query, res): client_provider = create_client_provider(AuthModeEnum.IAM, IAMAuthCredentialsProvider.ENV) try: res['result'] = do_gremlin_query(query, self.host, self.port, self.ssl, client_provider) except GremlinServerError as exception: res['error'] = str(exception)
def gremlin(self, line, cell, local_ns: dict = None): parser = argparse.ArgumentParser() parser.add_argument('query_mode', nargs='?', default='query', help='query mode (default=query) [query|explain|profile]') parser.add_argument('-p', '--path-pattern', default='', help='path pattern') parser.add_argument('-g', '--group-by', default='T.label', help='Property used to group nodes (e.g. code, T.region) default is T.label') parser.add_argument('--store-to', type=str, default='', help='store query result to this variable') parser.add_argument('--ignore-groups', action='store_true', help="Ignore all grouping options") args = parser.parse_args(line.split()) mode = str_to_query_mode(args.query_mode) logger.debug(f'Arguments {args}') tab = widgets.Tab() if mode == QueryMode.EXPLAIN: request_generator = create_request_generator(self.graph_notebook_config.auth_mode, self.graph_notebook_config.iam_credentials_provider_type) query_res = do_gremlin_explain(cell, self.graph_notebook_config.host, self.graph_notebook_config.port, self.graph_notebook_config.ssl, request_generator) if 'explain' in query_res: html = pre_container_template.render(content=query_res['explain']) else: html = pre_container_template.render(content='No explain found') explain_output = widgets.Output(layout=DEFAULT_LAYOUT) with explain_output: display(HTML(html)) tab.children = [explain_output] tab.set_title(0, 'Explain') display(tab) elif mode == QueryMode.PROFILE: request_generator = create_request_generator(self.graph_notebook_config.auth_mode, self.graph_notebook_config.iam_credentials_provider_type) query_res = do_gremlin_profile(cell, self.graph_notebook_config.host, self.graph_notebook_config.port, self.graph_notebook_config.ssl, request_generator) if 'profile' in query_res: html = pre_container_template.render(content=query_res['profile']) else: html = pre_container_template.render(content='No profile found') profile_output = widgets.Output(layout=DEFAULT_LAYOUT) with profile_output: display(HTML(html)) tab.children = [profile_output] tab.set_title(0, 'Profile') display(tab) else: client_provider = create_client_provider(self.graph_notebook_config.auth_mode, self.graph_notebook_config.iam_credentials_provider_type) query_res = do_gremlin_query(cell, self.graph_notebook_config.host, self.graph_notebook_config.port, self.graph_notebook_config.ssl, client_provider) children = [] titles = [] table_output = widgets.Output(layout=DEFAULT_LAYOUT) titles.append('Console') children.append(table_output) try: logger.debug(f'groupby: {args.group_by}') if args.ignore_groups: gn = GremlinNetwork() else: gn = GremlinNetwork(group_by_property=args.group_by) if args.path_pattern == '': gn.add_results(query_res) else: pattern = parse_pattern_list_str(args.path_pattern) gn.add_results_with_pattern(query_res, pattern) logger.debug(f'number of nodes is {len(gn.graph.nodes)}') if len(gn.graph.nodes) > 0: f = Force(network=gn, options=self.graph_notebook_vis_options) titles.append('Graph') children.append(f) logger.debug('added gremlin network to tabs') except ValueError as value_error: logger.debug(f'unable to create gremlin network from result. Skipping from result set: {value_error}') tab.children = children for i in range(len(titles)): tab.set_title(i, titles[i]) display(tab) table_id = f"table-{str(uuid.uuid4()).replace('-', '')[:8]}" table_html = gremlin_table_template.render(guid=table_id, results=query_res) with table_output: display(HTML(table_html)) store_to_ns(args.store_to, query_res, local_ns)
def on_button_clicked(b=None): submit_button.close() language_dropdown.disabled = True data_set_drop_down.disabled = True language = language_dropdown.value.lower() data_set = data_set_drop_down.value.lower() with output: print(f'Loading data set {data_set} with language {language}') queries = get_queries(language, data_set) if len(queries) < 1: with output: print('Did not find any queries for the given dataset') return load_index = 1 # start at 1 to have a non-empty progress bar progress = widgets.IntProgress( value=load_index, min=0, max=len(queries) + 1, # len + 1 so we can start at index 1 orientation='horizontal', bar_style='info', description='Loading:') with progress_output: display(progress) for q in queries: with output: print(f'{progress.value}/{len(queries)}:\t{q["name"]}') # Just like with the load command, seed is long-running # as such, we want to obtain the values of host, port, etc. in case they # change during execution. host = self.graph_notebook_config.host port = self.graph_notebook_config.port auth_mode = self.graph_notebook_config.auth_mode ssl = self.graph_notebook_config.ssl if language == 'gremlin': client_provider = create_client_provider( auth_mode, self.graph_notebook_config. iam_credentials_provider_type) # IMPORTANT: We treat each line as its own query! for line in q['content'].splitlines(): try: do_gremlin_query(line, host, port, ssl, client_provider) except GremlinServerError as gremlinEx: try: error = json.loads( gremlinEx.args[0] [5:]) # remove the leading error code. content = json.dumps(error, indent=2) except Exception: content = {'error': gremlinEx} with output: print(content) progress.close() return except Exception as e: content = {'error': e} with output: print(content) progress.close() return else: request_generator = create_request_generator( auth_mode, self.graph_notebook_config. iam_credentials_provider_type) try: do_sparql_query(q['content'], host, port, ssl, request_generator) except HTTPError as httpEx: # attempt to turn response into json try: error = json.loads( httpEx.response.content.decode('utf-8')) content = json.dumps(error, indent=2) except Exception: content = {'error': httpEx} with output: print(content) progress.close() return except Exception as ex: content = {'error': str(ex)} with output: print(content) progress.close() return progress.value += 1 progress.close() with output: print('Done.') return
def gremlin(self, line, cell): parser = argparse.ArgumentParser() parser.add_argument( 'query_mode', nargs='?', default='query', help='query mode (default=query) [query|explain|profile]') parser.add_argument('-p', '--path-pattern', default='', help='path pattern') args = parser.parse_args(line.split()) mode = str_to_query_mode(args.query_mode) tab = widgets.Tab() if mode == QueryMode.EXPLAIN: request_generator = create_request_generator( self.graph_notebook_config.auth_mode, self.graph_notebook_config.iam_credentials_provider_type) raw_html = gremlin_explain(cell, self.graph_notebook_config.host, self.graph_notebook_config.port, self.graph_notebook_config.ssl, request_generator) explain_output = widgets.Output(layout=DEFAULT_LAYOUT) with explain_output: display(HTML(raw_html)) tab.children = [explain_output] tab.set_title(0, 'Explain') display(tab) elif mode == QueryMode.PROFILE: request_generator = create_request_generator( self.graph_notebook_config.auth_mode, self.graph_notebook_config.iam_credentials_provider_type) raw_html = gremlin_profile(cell, self.graph_notebook_config.host, self.graph_notebook_config.port, self.graph_notebook_config.ssl, request_generator) profile_output = widgets.Output(layout=DEFAULT_LAYOUT) with profile_output: display(HTML(raw_html)) tab.children = [profile_output] tab.set_title(0, 'Profile') display(tab) else: client_provider = create_client_provider( self.graph_notebook_config.auth_mode, self.graph_notebook_config.iam_credentials_provider_type) res = do_gremlin_query(cell, self.graph_notebook_config.host, self.graph_notebook_config.port, self.graph_notebook_config.ssl, client_provider) children = [] titles = [] table_output = widgets.Output(layout=DEFAULT_LAYOUT) titles.append('Console') children.append(table_output) try: gn = GremlinNetwork() if args.path_pattern == '': gn.add_results(res) else: pattern = parse_pattern_list_str(args.path_pattern) gn.add_results_with_pattern(res, pattern) logger.debug(f'number of nodes is {len(gn.graph.nodes)}') if len(gn.graph.nodes) > 0: f = Force(network=gn, options=self.graph_notebook_vis_options) titles.append('Graph') children.append(f) logger.debug('added gremlin network to tabs') except ValueError as value_error: logger.debug( f'unable to create gremlin network from result. Skipping from result set: {value_error}' ) tab.children = children for i in range(len(titles)): tab.set_title(i, titles[i]) display(tab) table_id = f"table-{str(uuid.uuid4()).replace('-', '')[:8]}" table_html = gremlin_table_template.render(guid=table_id, results=res) with table_output: display(HTML(table_html))
def do_gremlin_query_save_results(self, query, res): try: res['result'] = do_gremlin_query(query, self.host, self.port, self.ssl, self.client_provider) except GremlinServerError as exception: res['error'] = str(exception)