def cleanup_graph(client):
"""Drop all of graph database.
Clean up all of graph database.
Args:
client(str): an instance for connecting to azure cosmos DB server
Returns:
none
"""
while 1:
decision = raw_input(
'\n\tAre you sure clean up all of graph database? Y or N\n')
print 'decision: ' + decision
if decision in ['Y', 'y']:
client.submitAsync("g.V().drop()")
print("\n\tCleaned up the graph!\n")
break
elif decision in ['N', 'n']:
print("\n\tCancel cleaning up the graph!\n")
sys.exit()
else:
print("\n\tYou need to type Y or N\n")
return True
def cleanup_graph(client):
print(
"\tRunning this Gremlin query:\n\t{0}".format(_gremlin_cleanup_graph))
callback = client.submitAsync(_gremlin_cleanup_graph)
if callback.result() is not None:
print("\tCleaned up the graph!")
print("\n")
def handler(message):
_gremlin_insert_vertices = []
# rdd = message.map(lambda k : k.split(":"))
records = message.collect()
# records = message.collectAsMap()
# print(records['timestamp'])
guidfound = False
key1 = ""
key2 = ""
value1 = ""
value2 = ""
guidvalue = ""
initialvalue = ""
query = ""
for record in records:
recordsplit = record.split(":")
print(recordsplit[0])
if len(recordsplit) > 1:
print(recordsplit[1])
firstpart = recordsplit[0].replace('"', '')
secondpart = recordsplit[1].strip()
print ("****")
print (firstpart)
secondpart = secondpart.replace('"', "")
if (secondpart.endswith(",")):
secondpart = secondpart[:-1]
if (firstpart.strip() == "keys"):
print ("inser keys section")
key1 = (secondpart.split(",")[0]).replace("[", "")
key2 = (secondpart.split(",")[1]).replace("]", "")
if (firstpart.strip() == "values"):
value1 = (secondpart.split(",")[0]).replace("[", "")
value2 = (secondpart.split(",")[1]).replace("]", "")
if (firstpart.strip() == "guid"):
guidfound = True
guidvalue = secondpart[1].strip()
if(firstpart.strip() == "data"):
query = "g.addV('Customer').property('" + str(key1.strip()) + "', '" + str(value1) + "')"
query = query + ".property('" + str(key2.strip()) + "'," + str(value2) + ")"
if (guidfound):
query = query + ".property('guid'," + guidvalue + ")"
else:
query = query + ".property('guid'," + str(generateguid()) + ")"
_gremlin_insert_vertices.append(query)
print(query)
if (len(_gremlin_insert_vertices)>0):
for qry in _gremlin_insert_vertices:
print("Record being inserted for query")
print(qry)
callback = client.submitAsync(qry)
if callback.result() is not None:
print("Record Inserted")
else:
print("Something went wrong")
def insert_vertices(client, query):
callback = client.submitAsync(query)
if callback.result() is not None:
logging.info("\tInserted this vertex:\n\t{0}\n".format(
callback.result().one()))
else:
logging.info("Something went wrong with this query: {0}".format(query))
def execute_drop_operations(client):
for key in _gremlin_drop_operations:
print("\t{0}:".format(key))
print("\tRunning this Gremlin query:\n\t{0}".format(_gremlin_drop_operations[key]))
callback = client.submitAsync(_gremlin_drop_operations[key])
for result in callback.result():
print(result)
print("\n")
def count_vertices(client):
print("\tRunning this Gremlin query:\n\t{0}".format(_gremlin_count_vertices))
callback = client.submitAsync(_gremlin_count_vertices)
if callback.result() is not None:
print("\tCount of vertices: {0}".format(callback.result().one()))
else:
print("Something went wrong with this query: {0}".format(_gremlin_count_vertices))
print("\n")
def cleanup_graph(client):
print("\n> {0}".format(_gremlin_cleanup_graph))
callback = client.submitAsync(_gremlin_cleanup_graph)
if callback.result() is not None:
callback.result().all().result()
print("\n")
print_status_attributes(callback.result())
print("\n")
def count_national_park_vertices(client):
print("\tExecuting Gremlin query:\n\t{0}".format(COUNT_VERTICES))
callback = client.submitAsync(COUNT_VERTICES)
if callback.result() is not None:
print("\tCount of vertices: {0}".format(callback.result().one()))
else:
print("This query failed: {0}".format(COUNT_VERTICES))
print("\n")
def execute_traversals(client):
for key in _gremlin_traversals:
print("\t{0}:".format(key))
print("\tRunning this Gremlin query:\n\t{0}\n".format(_gremlin_traversals[key]))
callback = client.submitAsync(_gremlin_traversals[key])
for result in callback.result():
print("\t{0}".format(str(result)))
print("\n")
def update_vertices(client):
for query in _gremlin_update_vertices:
print("\tRunning this Gremlin query:\n\t{0}\n".format(query))
callback = client.submitAsync(query)
if callback.result() is not None:
print("\tUpdated this vertex:\n\t{0}\n".format(callback.result().one()))
else:
print("Something went wrong with this query:\n\t{0}".format(query))
print("\n")
def execute_drop_operations(client):
for key in _gremlin_drop_operations:
print("{0}:".format(key))
print("\n> {0}".format(_gremlin_drop_operations[key]))
callback = client.submitAsync(_gremlin_drop_operations[key])
for result in callback.result():
print(result)
print_status_attributes(callback.result())
print("\n")
def insert_edges(client):
for query in _gremlin_insert_edges:
print("\tRunning this Gremlin query:\n\t{0}\n".format(query))
callback = client.submitAsync(query)
if callback.result() is not None:
print("\tInserted this edge:\n\t{0}\n".format(callback.result().one()))
else:
print("Something went wrong with this query:\n\t{0}".format(query))
print("\n")
def create_national_park_edges(client):
for query in INSERT_NATIONAL_PARK_EDGES:
print("\tExecuting Gremlin query:\n\t{0}\n".format(query))
callback = client.submitAsync(query)
if callback.result() is not None:
print("\tInserted this edge:\n\t{0}\n".format(
callback.result().one()))
else:
print("This query failed:\n\t{0}".format(query))
print("\n")
def execute_traversals(client):
for key in _gremlin_traversals:
print("{0}:".format(key))
print("> {0}\n".format(_gremlin_traversals[key]))
callback = client.submitAsync(_gremlin_traversals[key])
for result in callback.result():
print("\t{0}".format(str(result)))
print("\n")
print_status_attributes(callback.result())
print("\n")
def send_to_cosmos_db(self, client, query):
# query = "g.io('mygraph.graphml').write().iterate()"
# fos = new FileOutputStream("my-graph.json")
# GraphSONWriter.build().wrapAdjacencyList(true).create().writeGraph(fos,graph)
callback = client.submitAsync(query)
if callback.result() is not None:
print("\tQuery result:\n\t{0}\n".format(callback.result().one()))
else:
print("Something went wrong with this query:\n\t{0}".format(query))
print("\n")
def load_data_to_graphdb(self, blob_class):
print("Loading the JSON data")
file_string = blob_class.download_as_text()
graph_data_list = json.loads(file_string)
for node_rel in graph_data_list:
if node_rel['Kind'] == "node":
print("Processing the Node")
name = node_rel['Property']['Name'] if 'Name' in node_rel[
'Property'] else node_rel['Property']['NameLower']
vertex = self.vertex.format(node_rel['Property']['IdMaster'],
name)
try:
callback = client.submitAsync(vertex)
if callback.result() is not None:
results = callback.result()
if results is not None:
print("\tSuccess!")
else:
print(
"Something went wrong with this query: {0}".format(
vertex))
except Exception as e:
print('There was an exception: {0}'.format(e))
if node_rel['Kind'] == "relationship" and node_rel[
"DeDuplication"] in ["FALSE", "TRUE"]:
print("Processing the relationship")
edge = self.relationship.format(node_rel['FromIdObject'],
node_rel['Type'],
node_rel['ToIdObject'])
try:
callback = client.submitAsync(edge)
if callback.result() is not None:
results = callback.result()
if results is not None:
print("\tSuccess!")
else:
print(
"Something went wrong with this query: {0}".format(
edge))
except Exception as e:
print('There was an exception: {0}'.format(e))
def count_vertices(client):
print("\tRunning this Gremlin query:\n\t{0}".format(
_gremlin_count_vertices))
callback = client.submitAsync(_gremlin_count_vertices)
if callback.result() is not None:
print("\tCount of vertices: {0}".format(callback.result().one()))
else:
print("Something went wrong with this query: {0}".format(
_gremlin_count_vertices))
print("\n")
for key in _gremlin_traversals:
print("\t{0}:".format(key))
print("\tRunning this Gremlin query:\n\t{0}\n".format(
_gremlin_traversals[key]))
callback = client.submitAsync(_gremlin_traversals[key])
for result in callback.result():
print("\t{0}".format(str(result)))
print("\n")
def count_vertices(client):
print("\n> {0}".format(_gremlin_count_vertices))
callback = client.submitAsync(_gremlin_count_vertices)
if callback.result() is not None:
print("\tCount of vertices: {0}".format(
callback.result().all().result()))
else:
print("Something went wrong with this query: {0}".format(
_gremlin_count_vertices))
print("\n")
print_status_attributes(callback.result())
print("\n")
def insert_edges(client):
for query in _gremlin_insert_edges:
print("\n> {0}\n".format(query))
callback = client.submitAsync(query)
if callback.result() is not None:
print("\tInserted this edge:\n\t{0}\n".format(
callback.result().all().result()))
else:
print("Something went wrong with this query:\n\t{0}".format(query))
print_status_attributes(callback.result())
print("\n")
print("\n")
def readload_graph():
with open('nodes_companies.csv', newline='') as csvfile:
reader = csv.DictReader(csvfile,delimiter=';', quoting=csv.QUOTE_NONE)
for row in reader:
print(row['companyId'], row['name'])
str1 = "g.addV('company').property('id', '"
str2 = "' ).property('name','"
str3 = "')"
query = str1 + row['companyId'] + str2 + row['name'] + str3
print (query)
callback = client.submitAsync(query)
if callback.result() is not None:
print("\tInserted this vertex:\n\t{0}\n".format(
callback.result().one()))
else:
print("Something went wrong with this query: {0}".format(query))
def loadverti_graph():
with open('edges_director_duration.csv', newline='') as csvfile:
reader = csv.DictReader(csvfile,delimiter=';', quoting=csv.QUOTE_NONE)
for row in reader:
print(row['START_ID'], row['years_served'], row['END_ID'])
str1 = "g.V('"
str2 = "').addE('"
str3 = "').to(g.V('"
str4 = "')) "
query = str1 + row['START_ID'] + str2 + row['years_served'] + str3 + row['END_ID'] + str4
print (query)
callback = client.submitAsync(query)
if callback.result() is not None:
print("\tInserted this vertex:\n\t{0}\n".format(
callback.result().one()))
else:
print("Something went wrong with this query: {0}".format(query))
def execute_query(client, query_list):
"""Execute query.
Execute query of add vertex, add edge, etc.
Args:
client(str): an instance for connecting to azure cosmos DB server
query_list(list): a list of query to be executed
Returns:
none
"""
for que in query_list:
print("\n\tRunning this Gremlin query:\n\t{0}\n".format(que))
callback = client.submitAsync(que)
if callback.result() is not None:
print("\n\tInserted this vertex:\n\t{0}\n".format(
callback.result().one()))
else:
print("\nSomething went wrong with this query: {0}".format(que))
print("\n")
return True
from gremlin_python.driver import client
client = client.Client('ws://localhost:8182/gremlin', 'g')
result_set = client.submit("[1,2,3,4]")
future_results = result_set.all()
results = future_results.result()
assert results == [1,2,3,4]
future_result_set = client.submitAsync("[1,2,3,4]")
result_set = future_result_set.result()
result = result_set.one()
print result
assert results == [1,2,3,4]
assert result_set.done.done()
names = client.submitAsync("g.V().values('name')")
result_set = names.result()
result = result_set.one()
print result
client.close()
def cleanup_graph(client):
print("\tExecuting Gremlin query:\n\t{0}".format(CLEANUP_GRAPH))
callback = client.submitAsync(CLEANUP_GRAPH)
if callback.result() is not None:
print("\tCleaned up the graph!")
print("\n")
'''
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
'''
from __future__ import print_function # Python 2/3 compatibility
from gremlin_python.driver import client
client = client.Client('ws://${your-gdb-endpoint}:8182/gremlin', 'g', username="******", password="******")
callback = client.submitAsync("g.V().limit(1)")
for result in callback.result():
print(result)
def cleanup_graph(client):
callback = client.submitAsync(_gremlin_cleanup_graph)
if callback.result() is not None:
logging.info("\tCleaned up the graph!")
def insert_edges(client, query):
callback = client.submitAsync(query)
if callback.result() is not None:
print("\tInserted this edge:\n\t{0}\n".format(callback.result().one()))
else:
print("Something went wrong with this query:\n\t{0}".format(query))
from gremlin_python.driver import client
client = client.Client('ws://10.15.14.209:8182/gremlin', 'g')
result_set = client.submit("[1,2,3,4]")
future_results = result_set.all()
results = future_results.result()
assert results == [1, 2, 3, 4]
future_result_set = client.submitAsync("[1,2,3,4]")
result_set = future_result_set.result()
result = result_set.one()
assert results == [1, 2, 3, 4]
assert result_set.done.done()
client.close()
def handler(message):
_gremlin_insert_vertices = []
records = message.collect()
guidfound = False
key1 = key2 = value1 = value2 = guidvalue = query = ""
jsonobj = "{\n"
outputtopic = {}
recordpos = 1
for record in records:
recordsplit = record.split(":")
print(recordsplit[0])
if len(recordsplit) > 1:
print(recordsplit[1])
firstpart = recordsplit[0].replace('"', '')
secondpart = recordsplit[1].strip()
secondpart = secondpart.replace('"', "")
if (secondpart.endswith(",")):
secondpart = secondpart[:-1]
if (firstpart.strip() == "timestamp"):
jsonobj = jsonobj + recordsplit[0] + ":" + \
(recordsplit[1][:-1] if recordsplit[1].endswith(",") else recordsplit[1]) + ",\n"
if (firstpart.strip() == "keys"):
key1 = (secondpart.split(",")[0]).replace("[", "")
key2 = (secondpart.split(",")[1]).replace("]", "")
if (firstpart.strip() == "values"):
value1 = (secondpart.split(",")[0]).replace("[", "")
value2 = (secondpart.split(",")[1]).replace("]", "")
if (firstpart.strip() == "flguid"):
guidfound = True
guidvalue = secondpart[1].strip()
jsonobj = jsonobj + recordsplit[0] + ":" + \
(recordsplit[1][:-1] if recordsplit[1].endswith(",") else recordsplit[1]) + ",\n"
if (firstpart.strip() == "data"):
query = "g.addV('Customer').property('" + str(
key1.strip()) + "', '" + str(value1) + "')"
query = query + ".property('" + str(
key2.strip()) + "'," + str(value2) + ")"
if (guidfound):
guidfound = False
query = query + ".property('flguid'," + guidvalue + ")"
else:
guid = str(generateguid())
query = query + ".property('flguid'," + guid + ")"
jsonobj = jsonobj + '"flguid":' + guid + ",\n"
jsonobj = jsonobj + recordsplit[0] + ":" + \
(recordsplit[1][:-1] if recordsplit[1].endswith(",") else recordsplit[1]) + "\n}"
_gremlin_insert_vertices.append(query)
print(query)
outputtopic["Record" + str(recordpos)] = jsonobj
recordpos += 1
jsonobj = "{\n"
recordpos = 1
if (len(_gremlin_insert_vertices) > 0):
for qry in _gremlin_insert_vertices:
print("Record being inserted for query")
print(qry)
callback = client.submitAsync(qry)
if callback.result() is not None:
print("Record Inserted")
else:
print("Something went wrong")
# Write to different topic
WritetoTopic(outputtopic["Record" + str(recordpos)])
recordpos += 1