def setUp(self):
super().setUp()
self.client_provider = create_client_provider(
self.auth_mode, self.iam_credentials_provider_type)
query_check_for_airports = "g.V('3684').outE().inV().has(id, '3444')"
res = do_gremlin_query(query_check_for_airports, self.host, self.port,
self.ssl, self.client_provider)
if len(res) < 1:
logging.info(
'did not find final airports edge, seeding database now...')
airport_queries = get_queries('gremlin', 'airports')
for q in airport_queries:
lines = q['content'].splitlines()
for i in range(len(lines)):
line = lines[i]
logging.debug(
f'executing line {i} of {len(lines)} for seeding DataDrivenGremlinTest'
)
# we are deciding to try except because we do not know if the database
# we are connecting to has a partially complete set of airports data or not.
try:
do_gremlin_query(line, self.host, self.port, self.ssl,
self.client_provider)
except Exception as e:
logging.error(f'query {q} failed due to {e}')
continue
def test_do_gremlin_query_with_iam(self):
client_provider = create_client_provider(
AuthModeEnum.IAM, IAMAuthCredentialsProvider.ENV)
query = 'g.V().limit(1)'
results = do_gremlin_query(query, self.host, self.port, self.ssl,
client_provider)
self.assertEqual(type(results), list)
def do_gremlin_query_save_results(self, query, res):
client_provider = create_client_provider(AuthModeEnum.IAM, IAMAuthCredentialsProvider.ENV)
try:
res['result'] = do_gremlin_query(query, self.host, self.port, self.ssl, client_provider)
except GremlinServerError as exception:
res['error'] = str(exception)
def on_button_clicked(b=None):
submit_button.close()
language_dropdown.disabled = True
data_set_drop_down.disabled = True
language = language_dropdown.value.lower()
data_set = data_set_drop_down.value.lower()
with output:
print(f'Loading data set {data_set} with language {language}')
queries = get_queries(language, data_set)
if len(queries) < 1:
with output:
print('Did not find any queries for the given dataset')
return
load_index = 1 # start at 1 to have a non-empty progress bar
progress = widgets.IntProgress(
value=load_index,
min=0,
max=len(queries) + 1, # len + 1 so we can start at index 1
orientation='horizontal',
bar_style='info',
description='Loading:')
with progress_output:
display(progress)
for q in queries:
with output:
print(f'{progress.value}/{len(queries)}:\t{q["name"]}')
# Just like with the load command, seed is long-running
# as such, we want to obtain the values of host, port, etc. in case they
# change during execution.
host = self.graph_notebook_config.host
port = self.graph_notebook_config.port
auth_mode = self.graph_notebook_config.auth_mode
ssl = self.graph_notebook_config.ssl
if language == 'gremlin':
client_provider = create_client_provider(
auth_mode, self.graph_notebook_config.
iam_credentials_provider_type)
# IMPORTANT: We treat each line as its own query!
for line in q['content'].splitlines():
try:
do_gremlin_query(line, host, port, ssl,
client_provider)
except GremlinServerError as gremlinEx:
try:
error = json.loads(
gremlinEx.args[0]
[5:]) # remove the leading error code.
content = json.dumps(error, indent=2)
except Exception:
content = {'error': gremlinEx}
with output:
print(content)
progress.close()
return
except Exception as e:
content = {'error': e}
with output:
print(content)
progress.close()
return
else:
request_generator = create_request_generator(
auth_mode, self.graph_notebook_config.
iam_credentials_provider_type)
try:
do_sparql_query(q['content'], host, port, ssl,
request_generator)
except HTTPError as httpEx:
# attempt to turn response into json
try:
error = json.loads(
httpEx.response.content.decode('utf-8'))
content = json.dumps(error, indent=2)
except Exception:
content = {'error': httpEx}
with output:
print(content)
progress.close()
return
except Exception as ex:
content = {'error': str(ex)}
with output:
print(content)
progress.close()
return
progress.value += 1
progress.close()
with output:
print('Done.')
return
def gremlin(self, line, cell):
parser = argparse.ArgumentParser()
parser.add_argument(
'query_mode',
nargs='?',
default='query',
help='query mode (default=query) [query|explain|profile]')
parser.add_argument('-p',
'--path-pattern',
default='',
help='path pattern')
args = parser.parse_args(line.split())
mode = str_to_query_mode(args.query_mode)
tab = widgets.Tab()
if mode == QueryMode.EXPLAIN:
request_generator = create_request_generator(
self.graph_notebook_config.auth_mode,
self.graph_notebook_config.iam_credentials_provider_type)
raw_html = gremlin_explain(cell, self.graph_notebook_config.host,
self.graph_notebook_config.port,
self.graph_notebook_config.ssl,
request_generator)
explain_output = widgets.Output(layout=DEFAULT_LAYOUT)
with explain_output:
display(HTML(raw_html))
tab.children = [explain_output]
tab.set_title(0, 'Explain')
display(tab)
elif mode == QueryMode.PROFILE:
request_generator = create_request_generator(
self.graph_notebook_config.auth_mode,
self.graph_notebook_config.iam_credentials_provider_type)
raw_html = gremlin_profile(cell, self.graph_notebook_config.host,
self.graph_notebook_config.port,
self.graph_notebook_config.ssl,
request_generator)
profile_output = widgets.Output(layout=DEFAULT_LAYOUT)
with profile_output:
display(HTML(raw_html))
tab.children = [profile_output]
tab.set_title(0, 'Profile')
display(tab)
else:
client_provider = create_client_provider(
self.graph_notebook_config.auth_mode,
self.graph_notebook_config.iam_credentials_provider_type)
res = do_gremlin_query(cell, self.graph_notebook_config.host,
self.graph_notebook_config.port,
self.graph_notebook_config.ssl,
client_provider)
children = []
titles = []
table_output = widgets.Output(layout=DEFAULT_LAYOUT)
titles.append('Console')
children.append(table_output)
try:
gn = GremlinNetwork()
if args.path_pattern == '':
gn.add_results(res)
else:
pattern = parse_pattern_list_str(args.path_pattern)
gn.add_results_with_pattern(res, pattern)
logger.debug(f'number of nodes is {len(gn.graph.nodes)}')
if len(gn.graph.nodes) > 0:
f = Force(network=gn,
options=self.graph_notebook_vis_options)
titles.append('Graph')
children.append(f)
logger.debug('added gremlin network to tabs')
except ValueError as value_error:
logger.debug(
f'unable to create gremlin network from result. Skipping from result set: {value_error}'
)
tab.children = children
for i in range(len(titles)):
tab.set_title(i, titles[i])
display(tab)
table_id = f"table-{str(uuid.uuid4()).replace('-', '')[:8]}"
table_html = gremlin_table_template.render(guid=table_id,
results=res)
with table_output:
display(HTML(table_html))
def test_create_iam_client_from_env(self):
client_provider = create_client_provider(
AuthModeEnum.IAM, IAMAuthCredentialsProvider.ENV)
self.assertEqual(IamClientProvider, type(client_provider))
def test_create_default_client(self):
client_provider = create_client_provider(AuthModeEnum.DEFAULT)
self.assertEqual(ClientProvider, type(client_provider))
def gremlin(self, line, cell, local_ns: dict = None):
parser = argparse.ArgumentParser()
parser.add_argument('query_mode', nargs='?', default='query',
help='query mode (default=query) [query|explain|profile]')
parser.add_argument('-p', '--path-pattern', default='', help='path pattern')
parser.add_argument('-g', '--group-by', default='T.label', help='Property used to group nodes (e.g. code, T.region) default is T.label')
parser.add_argument('--store-to', type=str, default='', help='store query result to this variable')
parser.add_argument('--ignore-groups', action='store_true', help="Ignore all grouping options")
args = parser.parse_args(line.split())
mode = str_to_query_mode(args.query_mode)
logger.debug(f'Arguments {args}')
tab = widgets.Tab()
if mode == QueryMode.EXPLAIN:
request_generator = create_request_generator(self.graph_notebook_config.auth_mode,
self.graph_notebook_config.iam_credentials_provider_type)
query_res = do_gremlin_explain(cell, self.graph_notebook_config.host, self.graph_notebook_config.port,
self.graph_notebook_config.ssl, request_generator)
if 'explain' in query_res:
html = pre_container_template.render(content=query_res['explain'])
else:
html = pre_container_template.render(content='No explain found')
explain_output = widgets.Output(layout=DEFAULT_LAYOUT)
with explain_output:
display(HTML(html))
tab.children = [explain_output]
tab.set_title(0, 'Explain')
display(tab)
elif mode == QueryMode.PROFILE:
request_generator = create_request_generator(self.graph_notebook_config.auth_mode,
self.graph_notebook_config.iam_credentials_provider_type)
query_res = do_gremlin_profile(cell, self.graph_notebook_config.host, self.graph_notebook_config.port,
self.graph_notebook_config.ssl, request_generator)
if 'profile' in query_res:
html = pre_container_template.render(content=query_res['profile'])
else:
html = pre_container_template.render(content='No profile found')
profile_output = widgets.Output(layout=DEFAULT_LAYOUT)
with profile_output:
display(HTML(html))
tab.children = [profile_output]
tab.set_title(0, 'Profile')
display(tab)
else:
client_provider = create_client_provider(self.graph_notebook_config.auth_mode,
self.graph_notebook_config.iam_credentials_provider_type)
query_res = do_gremlin_query(cell, self.graph_notebook_config.host, self.graph_notebook_config.port,
self.graph_notebook_config.ssl, client_provider)
children = []
titles = []
table_output = widgets.Output(layout=DEFAULT_LAYOUT)
titles.append('Console')
children.append(table_output)
try:
logger.debug(f'groupby: {args.group_by}')
if args.ignore_groups:
gn = GremlinNetwork()
else:
gn = GremlinNetwork(group_by_property=args.group_by)
if args.path_pattern == '':
gn.add_results(query_res)
else:
pattern = parse_pattern_list_str(args.path_pattern)
gn.add_results_with_pattern(query_res, pattern)
logger.debug(f'number of nodes is {len(gn.graph.nodes)}')
if len(gn.graph.nodes) > 0:
f = Force(network=gn, options=self.graph_notebook_vis_options)
titles.append('Graph')
children.append(f)
logger.debug('added gremlin network to tabs')
except ValueError as value_error:
logger.debug(f'unable to create gremlin network from result. Skipping from result set: {value_error}')
tab.children = children
for i in range(len(titles)):
tab.set_title(i, titles[i])
display(tab)
table_id = f"table-{str(uuid.uuid4()).replace('-', '')[:8]}"
table_html = gremlin_table_template.render(guid=table_id, results=query_res)
with table_output:
display(HTML(table_html))
store_to_ns(args.store_to, query_res, local_ns)