def remote_connection_authenticated(request):
try:
if request.param == 'basic':
remote_conn = DriverRemoteConnection(
basic_url,
'gmodern',
username='******',
password='******',
message_serializer=serializer.GraphSONSerializersV2d0())
elif request.param == 'kerberos':
remote_conn = DriverRemoteConnection(
kerberos_url,
'gmodern',
kerberized_service=kerberized_service,
message_serializer=serializer.GraphSONSerializersV2d0())
else:
raise ValueError("Invalid authentication option - " +
request.param)
except OSError:
pytest.skip('Gremlin Server is not running')
else:
def fin():
remote_conn.close()
request.addfinalizer(fin)
return remote_conn
def remote_connection_authenticated(request):
try:
if request.param == 'basic':
# turn off certificate verification for testing purposes only
ssl_opts = ssl.SSLContext(ssl.PROTOCOL_TLSv1_2)
ssl_opts.verify_mode = ssl.CERT_NONE
remote_conn = DriverRemoteConnection(
basic_url,
'gmodern',
username='******',
password='******',
message_serializer=serializer.GraphSONSerializersV2d0(),
transport_factory=lambda: AiohttpTransport(ssl_options=ssl_opts
))
elif request.param == 'kerberos':
remote_conn = DriverRemoteConnection(
kerberos_url,
'gmodern',
kerberized_service=kerberized_service,
message_serializer=serializer.GraphSONSerializersV2d0())
else:
raise ValueError("Invalid authentication option - " +
request.param)
except OSError:
pytest.skip('Gremlin Server is not running')
else:
def fin():
remote_conn.close()
request.addfinalizer(fin)
return remote_conn
def __init__(
self,
url=None,
key=None,
database_name=None,
container_name=None,
credentials_packed: str = None,
):
self._rootLogger = logging.getLogger()
credential_unpacked = base64.urlsafe_b64decode(credentials_packed)
credential_dict = yaml.safe_load(credential_unpacked)
if credential_dict is not None:
url = credential_dict["url"]
key = credential_dict["key"]
database_name = credential_dict["database_name"]
container_name = credential_dict["container_name"]
self._url = url
self._key = key
self._database_name = database_name
self._container_name = container_name
# TODO this is almost certainly wrong. Should probably partition by workflow.
self._workflow_partition_id = str(uuid.uuid4())
self._rootLogger.debug(f"Loading gremlin wss endpoint: {self._url}")
self._gremlin_client = client.Client(
f"{self._url}",
"g",
username=f"/dbs/{self._database_name}/colls/{self._container_name}",
password=f"{self._key}",
message_serializer=serializer.GraphSONSerializersV2d0(),
)
def remote_connection(request):
try:
if request.param == 'graphbinaryv1':
remote_conn = DriverRemoteConnection(
anonymous_url,
'gmodern',
message_serializer=serializer.GraphBinarySerializersV1())
elif request.param == 'graphsonv2':
remote_conn = DriverRemoteConnection(
anonymous_url,
'gmodern',
message_serializer=serializer.GraphSONSerializersV2d0())
elif request.param == 'graphsonv3':
remote_conn = DriverRemoteConnection(
anonymous_url,
'gmodern',
message_serializer=serializer.GraphSONSerializersV3d0())
else:
raise ValueError("Invalid serializer option - " + request.param)
except OSError:
pytest.skip('Gremlin Server is not running')
else:
def fin():
remote_conn.close()
request.addfinalizer(fin)
return remote_conn
def __init__(self, url, database_name, collection_name, key):
self.gremlin_client = client.Client(
url,
'g',
username=f'/dbs/{database_name}/colls/{collection_name}',
password=key,
message_serializer=serializer.GraphSONSerializersV2d0())
self.query_map = {
'get_all_beats':
'g.V().hasLabel(\'beat\')',
'get_all_samples':
'g.V().hasLabel(\'sample\')',
'get_all_users':
'g.V().hasLabel(\'user\')',
'recommend_beat':
'g.V(\'{email}\').property(\'propertyType\', \'{current_time}\').addE(\'RECOMMENDED\').to(g.V(\'{beat_id}\')).property(\'propertyType\', \'{current_time}\')',
'get_all_public_beats':
'g.V().hasLabel(\'beat\').has(\'isPrivate\', \'False\')',
'get_liked_beats':
'g.V(\'{email}\').outE(\'LIKES\').order().by(\'date\', decr).inV().limit({limit}).hasLabel(\'beat\')',
'delete_recommendations':
'g.V(\'{email}\').outE(\'RECOMMENDED\').drop()',
'get_owned_beats':
'g.V(\'{email}\').inE(\'OWNED_BY\').outV().hasLabel(\'beat\')'
}
def createclient():
graphclient = client.Client(
'wss://<COSMOS DB ACCOUNT NAME>.gremlin.cosmosdb.azure.com:443/',
'g',
username="******",
password="******",
message_serializer=serializer.GraphSONSerializersV2d0())
return graphclient
def prepare_traversal_source(scenario):
# some tests create data - create a fresh remote to the empty graph and clear that graph prior to each test
remote = DriverRemoteConnection(
'ws://localhost:45940/gremlin',
"ggraph",
message_serializer=serializer.GraphSONSerializersV2d0())
scenario.context.remote_conn["empty"] = remote
g = Graph().traversal().withRemote(remote)
g.V().drop().iterate()
def __init__(self):
self.azure_client = client.Client(
'wss://intership-assignments-2020.gremlin.cosmos.azure.com/',
'g',
username=
"******",
password=
"******",
message_serializer=serializer.GraphSONSerializersV2d0())
def __init__(self):
self.endpoint = config("ENDPOINT")
self.db = config("DATABASE")
self.collection = config("COLLECTION")
self.pk = config("PRIMARY_KEY")
self.gc = client.Client(
message_serializer=serializer.GraphSONSerializersV2d0(),
url=self.endpoint,
traversal_source='g',
username="******" + self.db + "/colls/" + self.collection,
password=self.pk)
def call_gremlin(database='',graph="",key="",query=""):
ENDPOINT = ''
c = client.Client(ENDPOINT, 'g',
username="******".format(
database, graph),
password="******".format(key),
message_serializer=serializer.GraphSONSerializersV2d0()
)
# クエリを発行してcallbackを取得
callback = c.submitAsync(query)
# コールバックが複数回に分かれて返ってくるので一つのリストにする
response = [res for result in callback.result() for res in result]
return response
def remote_connection_graphsonV2(request):
try:
remote_conn = DriverRemoteConnection(
anonymous_url,
message_serializer=serializer.GraphSONSerializersV2d0())
except OSError:
pytest.skip('Gremlin Server is not running')
else:
def fin():
remote_conn.close()
request.addfinalizer(fin)
return remote_conn
def remote_connection(request):
try:
if request.param == 'v2':
remote_conn = DriverRemoteConnection('ws://localhost:45940/gremlin', 'gmodern',
message_serializer=serializer.GraphSONSerializersV2d0())
else:
remote_conn = DriverRemoteConnection('ws://localhost:45940/gremlin', 'gmodern')
except OSError:
pytest.skip('Gremlin Server is not running')
else:
def fin():
remote_conn.close()
request.addfinalizer(fin)
return remote_conn
def get_client(self):
from gremlin_python.driver import client, serializer
try:
cosmosclient = client.Client(
'ws://localhost:8901',
'g',
username="******",
password=
"******",
message_serializer=serializer.GraphSONSerializersV2d0())
print("Welcome to Azure Cosmos DB + Gremlin on Python!")
return cosmosclient
except Exception as e:
print('There was an exception: {0}'.format(e))
traceback.print_exc(file=sys.stdout)
sys.exit(1)
def test_multi_conn_pool(client):
g = Graph().traversal()
t = g.V()
message = RequestMessage('traversal', 'bytecode', {'gremlin': t.bytecode})
client = Client('ws://localhost:45940/gremlin',
'g',
pool_size=1,
message_serializer=serializer.GraphSONSerializersV2d0())
future = client.submitAsync(message)
future2 = client.submitAsync(message)
result_set2 = future2.result()
assert len(result_set2.all().result()) == 6
# with connection pool `future` may or may not be done here
result_set = future.result()
assert len(result_set.all().result()) == 6
def test_connection_share(client):
# Overwrite fixture with pool_size=1 client
client = Client('ws://localhost:45940/gremlin',
'g',
pool_size=1,
message_serializer=serializer.GraphSONSerializersV2d0())
g = Graph().traversal()
t = g.V()
message = RequestMessage('traversal', 'bytecode', {'gremlin': t.bytecode})
future = client.submitAsync(message)
future2 = client.submitAsync(message)
result_set2 = future2.result()
assert len(result_set2.all().result()) == 6
# This future has to finish for the second to yield result - pool_size=1
assert future.done()
result_set = future.result()
assert len(result_set.all().result()) == 6
def __init__(self, account_name, account_key, database_name, graph_name):
self._gremlin_cleanup_graph = "g.V().drop()"
self._gremlin_insert_vertices = [
# partition 1
"g.addV('person').property('id', 'thomas').property('firstName', 'Thomas').property('age', 44).property('partitionKey', 1)",
"g.addV('person').property('id', 'mary').property('firstName', 'Mary').property('lastName', 'Andersen').property('age', 39).property('partitionKey', 1)",
"g.addV('person').property('id', 'peter').property('firstName', 'Peter').property('lastName', 'Smith').property('age', 25).property('partitionKey', 1)",
"g.addV('person').property('id', 'jenny').property('firstName', 'Jenny').property('lastName', 'Curran').property('age', 25).property('partitionKey', 1)",
# address
"g.addV('address').property('id', '8916_marvin_gardens').property('streetName', 'Marvin Gardens Rd').property('streetNumber', 8916).property('city', 'Atlanta').property('partitionKey', 1)",
"g.addV('address').property('id', '8917_marvin_gardens').property('streetName', 'Marvin Gardens Rd').property('streetNumber', 8917).property('city', 'Atlanta').property('partitionKey', 1)",
"g.addV('address').property('id', '1611_boardwalk').property('streetName', 'Boardwalk Blvd').property('streetNumber', 1611).property('city', 'New York').property('partitionKey', 1)",
# partition 2
"g.addV('person').property('id', 'ben').property('firstName', 'Ben').property('lastName', 'Miller').property('partitionKey', 2)",
"g.addV('person').property('id', 'robin').property('firstName', 'Robin').property('lastName', 'Wakefield').property('partitionKey', 2)",
"g.addV('person').property('id', 'forrest').property('firstName', 'Forrest').property('lastName', 'Gump').property('partitionKey', 2)",
"g.addV('person').property('id', 'john').property('firstName', 'John').property('lastName', 'McClane').property('age', 34).property('partitionKey', 2)",
# address
"g.addV('address').property('id', '926_parkplace').property('streetName', 'Park Place Ave').property('streetNumber', 926).property('city', 'New York').property('partitionKey', 2)",
"g.addV('address').property('id', '8917_illinois').property('streetName', 'Illinois Ave').property('streetNumber', 8917).property('city', 'Atlanta').property('partitionKey', 2)",
"g.addV('address').property('id', '4933_baltic').property('streetName', 'Baltic Ave').property('streetNumber', 4933).property('city', 'Vermont').property('partitionKey', 2)",
]
self._gremlin_insert_edges = [
"g.V('thomas').addE('knows').to(g.V('mary'))",
"g.V('thomas').addE('knows').to(g.V('ben'))",
"g.V('jenny').addE('knows').to(g.V('forrest'))",
"g.V('ben').addE('knows').to(g.V('robin'))",
"g.V('peter').addE('lives_on').to(g.V('1611_boardwalk'))",
"g.V('ben').addE('lives_on').to(g.V('926_parkplace'))",
"g.V('forrest').addE('lives_on').to(g.V('4933_baltic'))"
]
self.client = client.Client(
'wss://' + account_name + '.gremlin.cosmosdb.azure.com:443/',
'g',
username="******" + database_name + "/colls/" + graph_name,
password=account_key,
message_serializer=serializer.GraphSONSerializersV2d0())
def connect_server():
"""Connect to azure cosmos DB.
Connet to azure cosmos DB.
Need to insert ID, DB name, table name, and key.
Args:
None
Returns:
clt(str): an instance for connecting to azure cosmos DB server
"""
clt = client.Client(
'wss://<YOURID>.gremlin.cosmos.azure.com:443/',
'g',
username="******",
password="******",
message_serializer=serializer.GraphSONSerializersV2d0())
return clt
def get_client(self):
"""
Initializes the client for db interaction
:return:
"""
try:
c = client.Client(
'wss://topicgraph.gremlin.cosmosdb.azure.com:443/',
'g',
username="******",
password=
"******",
message_serializer=serializer.GraphSONSerializersV2d0())
self.client = c
except:
print("oops")
def prepare_traversal_source(scenario):
# some tests create data - create a fresh remote to the empty graph and clear that graph prior to each test
if not ("graphson" in world.config.user_data):
raise ValueError(
'test configuration requires setting of --user-data="graphson=*" to one of [v2,v3]'
)
if world.config.user_data["graphson"] == "v3":
s = serializer.GraphSONSerializersV3d0()
elif world.config.user_data["graphson"] == "v2":
s = serializer.GraphSONSerializersV2d0()
else:
raise ValueError(
'serializer set with --user-data="graphson=v2" must be one of [v2,v3]'
)
remote = DriverRemoteConnection('ws://localhost:45940/gremlin',
"ggraph",
message_serializer=s)
scenario.context.remote_conn["empty"] = remote
g = traversal().withRemote(remote)
g.V().drop().iterate()
def handler():
#Initialise client
print("Initialising client ...")
gremlin_client = client.Client(
"wss://" + ENDPOINT + ":" + str(PORT) + "/",
"g",
message_serializer=serializer.GraphSONSerializersV2d0(),
username="******" + DATABASE + "/colls/" + COLLECTION,
password=PASSWORD)
print("client initlaised!")
#Pruge graph
cleanup_graph(gremlin_client)
#Insert vertices
insert_vertices(gremlin_client)
# Insert edges (nodes)
insert_edges(gremlin_client)
print("Finished !")
def graph_connect(self):
"""Connects to graph db
Args:
Returns:
client connection
Raises:
Exception
"""
try:
return client.Client(
'wss://kq2.gremlin.cosmos.azure.com:443/',
'g',
username=settings.SECRET_LIST['gremlin-user'],
password=settings.SECRET_LIST['gremlin-key'],
message_serializer=serializer.GraphSONSerializersV2d0())
except Exception as e:
capture_exception(e)
def graph_connect(self):
"""Connects to graph db
Args:
Returns:
client connection
Raises:
Exception
"""
try:
return client.Client('wss://kq2.gremlin.cosmos.azure.com:443/', 'g',
username="******",
password="",
message_serializer=serializer.GraphSONSerializersV2d0()
)
except Exception as e:
print('There was an exception: {0}'.format(e))
traceback.print_exc(file=sys.stdout)
def _setup_cosmosdb_con(self) -> client.Client:
return client.Client(os.environ["cosmosDBServer"], 'g',
username=os.environ["cosmosDBUsername"],
password=os.environ["cosmosDBPassword"],
message_serializer=serializer.GraphSONSerializersV2d0())
from pyspark import SparkConf, SparkContext
from pyspark.streaming import StreamingContext
from pyspark.streaming.kafka import KafkaUtils
from kafka import KafkaProducer
from gremlin_python.driver import client, serializer
import sys, traceback
import random
producer = KafkaProducer(bootstrap_servers='localhost:9092')
client = client.Client('wss://sparkkafka.gremlin.cosmosdb.azure.com:443/', 'g',
username="******",
password="******",
message_serializer=serializer.GraphSONSerializersV2d0()
)
_gremlin_insert_vertices = []
def handler(message):
_gremlin_insert_vertices = []
# rdd = message.map(lambda k : k.split(":"))
records = message.collect()
# records = message.collectAsMap()
# print(records['timestamp'])
guidfound = False
key1 = ""
def get_client():
return client.Client(ENDPOINT, 'g', username=USERNAME, password=PASSWORD,
message_serializer=serializer.GraphSONSerializersV2d0())
def __create_remote(server_graph_name):
return DriverRemoteConnection(
'ws://localhost:45940/gremlin',
server_graph_name,
message_serializer=serializer.GraphSONSerializersV2d0())
def __init__(self):
self._url = os.environ['URL']
self._traversal_source = 'g'
self._user_name = os.environ['USERNAME']
self._password = os.environ['PASSWORD']
self._message_serializer = serializer.GraphSONSerializersV2d0()