def __init__(self,
url,
traversal_source,
protocol_factory=None,
transport_factory=None,
pool_size=None,
max_workers=None,
username="",
password="",
message_serializer=None,
graphson_reader=None,
graphson_writer=None):
if message_serializer is None:
message_serializer = serializer.GraphSONMessageSerializer(
reader=graphson_reader, writer=graphson_writer)
self._client = client.Client(url,
traversal_source,
protocol_factory=protocol_factory,
transport_factory=transport_factory,
pool_size=pool_size,
max_workers=max_workers,
message_serializer=message_serializer,
username=username,
password=password)
self._url = self._client._url
self._traversal_source = self._client._traversal_source
def __init__(self, url, database_name, collection_name, key):
self.gremlin_client = client.Client(
url,
'g',
username=f'/dbs/{database_name}/colls/{collection_name}',
password=key,
message_serializer=serializer.GraphSONSerializersV2d0())
self.query_map = {
'get_all_beats':
'g.V().hasLabel(\'beat\')',
'get_all_samples':
'g.V().hasLabel(\'sample\')',
'get_all_users':
'g.V().hasLabel(\'user\')',
'recommend_beat':
'g.V(\'{email}\').property(\'propertyType\', \'{current_time}\').addE(\'RECOMMENDED\').to(g.V(\'{beat_id}\')).property(\'propertyType\', \'{current_time}\')',
'get_all_public_beats':
'g.V().hasLabel(\'beat\').has(\'isPrivate\', \'False\')',
'get_liked_beats':
'g.V(\'{email}\').outE(\'LIKES\').order().by(\'date\', decr).inV().limit({limit}).hasLabel(\'beat\')',
'delete_recommendations':
'g.V(\'{email}\').outE(\'RECOMMENDED\').drop()',
'get_owned_beats':
'g.V(\'{email}\').inE(\'OWNED_BY\').outV().hasLabel(\'beat\')'
}
def __init__(
self,
url=None,
key=None,
database_name=None,
container_name=None,
credentials_packed: str = None,
):
self._rootLogger = logging.getLogger()
credential_unpacked = base64.urlsafe_b64decode(credentials_packed)
credential_dict = yaml.safe_load(credential_unpacked)
if credential_dict is not None:
url = credential_dict["url"]
key = credential_dict["key"]
database_name = credential_dict["database_name"]
container_name = credential_dict["container_name"]
self._url = url
self._key = key
self._database_name = database_name
self._container_name = container_name
# TODO this is almost certainly wrong. Should probably partition by workflow.
self._workflow_partition_id = str(uuid.uuid4())
self._rootLogger.debug(f"Loading gremlin wss endpoint: {self._url}")
self._gremlin_client = client.Client(
f"{self._url}",
"g",
username=f"/dbs/{self._database_name}/colls/{self._container_name}",
password=f"{self._key}",
message_serializer=serializer.GraphSONSerializersV2d0(),
)
def main():
parser = argparse.ArgumentParser()
parser.add_argument('--host', dest="host", type=str, required=True)
parser.add_argument('--port', dest="port", type=int, default=8182)
parser.add_argument('--username', dest="username", type=str, required=True)
parser.add_argument('--password', dest="password", type=str, required=True)
parser.add_argument('--threadCnt', dest="threadCnt", type=int, default=32)
parser.add_argument('--label',
dest="label",
type=str,
default=None,
help="drop element with specified label")
parser.add_argument('--edge',
dest="drop_edge_only",
action="store_true",
help="only drop edge")
args = parser.parse_args()
gdb_client = client.Client('ws://%s:%d/gremlin' % (args.host, args.port),
'g',
username=args.username,
password=args.password)
gdb_data_remover = GdbParallelDataRemover(gdb_client, args.threadCnt)
signal.signal(signal.SIGINT, gdb_data_remover.quit)
signal.signal(signal.SIGTERM, gdb_data_remover.quit)
gdb_data_remover.drop(args.label, args.drop_edge_only)
def find_node(
unique_id: str,
node_type: str,
gremlin_client: client.Client
):
'''
Construct the query to find the node with the specified property. Assumes
that the specified property is unique to that node type
'''
node_type_ids = {
'subreddit': 'subreddit_id',
'comment': 'id',
'user': '******'
}
query = f"g.V().hasLabel('{node_type}').has('{node_type_ids[node_type]}', '{unique_id}').next()"
try:
node = gremlin_client.submit(query).all().result()
return query
except:
"Node doesn't exist! Cannot create edge"
# Not sure if this is needed, but noticed that when Gremlin client throws an error
# it will disconnect and hang..
gremlin_client = client.Client(JANUS_CONNECT, 'g')
return None
def get_gremlin_connection(self) -> client.Client:
uri = f'{self._http_protocol}://{self.host}:{self.port}/gremlin'
request = self._prepare_request('GET', uri)
ws_url = f'{self._ws_protocol}://{self.host}:{self.port}/gremlin'
ws_request = httpclient.HTTPRequest(ws_url,
headers=dict(request.headers))
return client.Client(ws_request, 'g')
def createclient():
graphclient = client.Client(
'wss://<COSMOS DB ACCOUNT NAME>.gremlin.cosmosdb.azure.com:443/',
'g',
username="******",
password="******",
message_serializer=serializer.GraphSONSerializersV2d0())
return graphclient
def __init__(self):
self.azure_client = client.Client(
'wss://intership-assignments-2020.gremlin.cosmos.azure.com/',
'g',
username=
"******",
password=
"******",
message_serializer=serializer.GraphSONSerializersV2d0())
def __init__(self):
self.endpoint = config("ENDPOINT")
self.db = config("DATABASE")
self.collection = config("COLLECTION")
self.pk = config("PRIMARY_KEY")
self.gc = client.Client(
message_serializer=serializer.GraphSONSerializersV2d0(),
url=self.endpoint,
traversal_source='g',
username="******" + self.db + "/colls/" + self.collection,
password=self.pk)
def _get_gremlin_connection(self, headers: Any = None) -> client.Client:
if self.gremlin_connection is None:
uri = f"{HTTP_PROTOCOL}://{self.host}:{self.port}/gremlin"
request = self._prepare_request("GET", uri, headers=headers)
ws_url = f"{WS_PROTOCOL}://{self.host}:{self.port}/gremlin"
self.gremlin_connection = client.Client(ws_url,
"g",
headers=dict(
request.headers),
call_from_event_loop=True)
return self.gremlin_connection
def get_gremlin_conn():
conn = Connection()
gremlin_conn = client.Client(
url=conn.get_url(),
traversal_source=conn.get_traversal_source(),
username=conn.get_user_name(),
password=conn.get_password(),
message_serializer=conn.get_message_serializer())
return gremlin_conn
def __init__(self, graph_tag='default'):
assert graph_tag in auth_config, 'error graph tag'
config = auth_config[graph_tag]
self.__client = client.Client(
url=config['url'],
traversal_source=config['traversal_source'],
username=config['username'],
password=config['password'])
self.vertex_properties = vertex_properties[graph_tag]
self.edge_properties = edge_properties[graph_tag]
def get_gremlin_connection(self) -> client.Client:
nest_asyncio.apply()
uri = f'{self._http_protocol}://{self.host}:{self.port}/gremlin'
request = self._prepare_request('GET', uri)
ws_url = f'{self._ws_protocol}://{self.host}:{self.port}/gremlin'
traversal_source = 'g' if "neptune.amazonaws.com" in self.host else self.gremlin_traversal_source
return client.Client(ws_url,
traversal_source,
headers=dict(request.headers))
def call_gremlin(database='',graph="",key="",query=""):
ENDPOINT = ''
c = client.Client(ENDPOINT, 'g',
username="******".format(
database, graph),
password="******".format(key),
message_serializer=serializer.GraphSONSerializersV2d0()
)
# クエリを発行してcallbackを取得
callback = c.submitAsync(query)
# コールバックが複数回に分かれて返ってくるので一つのリストにする
response = [res for result in callback.result() for res in result]
return response
def create_client(self, db, coll):
# Get a database connection from the Gremlin Server.
endpoint = self.c.cosmosdb_gremlin_url()
username = self.c.cosmosdb_gremlin_username(db, coll)
password = self.c.cosmosdb_key()
print('create_client:')
print('endpoint: {}'.format(endpoint))
print('username: {}'.format(username))
#print('password: {}'.format(password))
self.gremlin_client = client.Client(endpoint,
'g',
username=username,
password=password)
time.sleep(5)
def __init__(self,
url,
traversal_source="g",
protocol_factory=None,
transport_factory=None,
pool_size=None,
max_workers=None,
username="",
password="",
kerberized_service='',
message_serializer=None,
graphson_reader=None,
graphson_writer=None,
headers=None,
session=None,
**transport_kwargs):
logging.info("Creating DriverRemoteConnection with url '%s'", str(url))
self.__url = url
self.__traversal_source = traversal_source
self.__protocol_factory = protocol_factory
self.__transport_factory = transport_factory
self.__pool_size = pool_size
self.__max_workers = max_workers
self.__username = username
self.__password = password
self.__kerberized_service = kerberized_service
self.__message_serializer = message_serializer
self.__graphson_reader = graphson_reader
self.__graphson_writer = graphson_writer
self.__headers = headers
self.__session = session
self.__transport_kwargs = transport_kwargs
if message_serializer is None:
message_serializer = serializer.GraphSONMessageSerializer(
reader=graphson_reader, writer=graphson_writer)
self._client = client.Client(url,
traversal_source,
protocol_factory=protocol_factory,
transport_factory=transport_factory,
pool_size=pool_size,
max_workers=max_workers,
message_serializer=message_serializer,
username=username,
password=password,
kerberized_service=kerberized_service,
headers=headers,
session=session,
**transport_kwargs)
self._url = self._client._url
self._traversal_source = self._client._traversal_source
def setup_gremlin():
logging.debug("Connecting to Microsoft Azure Gremlin API")
try:
MicrosoftAzureCosmosDBGremlinAPI.client = client.Client('wss://vidhya-gremlin.gremlin.cosmosdb.azure.com:443/', 'g',
username="******",
password="******"
)
logging.debug("Connecting to Microsoft Azure Gremlin API completed... ")
logging.debug("Collecting data from Microsoft Azure Gremlin API...")
MicrosoftAzureCosmosDBGremlinAPI.graph, MicrosoftAzureCosmosDBGremlinAPI.keys = MicrosoftAzureCosmosDBGremlinAPI.get_knowledge_graph()
logging.debug("Collecting data from Microsoft Azure Gremlin API completed...")
logging.debug(MicrosoftAzureCosmosDBGremlinAPI.graph)
except Exception as e:
logging.debug('There was an exception: {0}'.format(e))
def create_app(configuration=General_Configuration):
app = Flask(__name__)
app.config.from_object(configuration)
app.register_blueprint(people)
app.register_blueprint(relations)
app.register_blueprint(locations)
app.register_blueprint(base)
app.register_blueprint(faces)
cli = client.Client('ws://localhost:8182/gremlin', 'g')
injectFunctions(cli) # Functions responsible for loading relevant groovy scripts for application execution.
cli.close()
return app
def get_client(self):
from gremlin_python.driver import client, serializer
try:
cosmosclient = client.Client(
'ws://localhost:8901',
'g',
username="******",
password=
"******",
message_serializer=serializer.GraphSONSerializersV2d0())
print("Welcome to Azure Cosmos DB + Gremlin on Python!")
return cosmosclient
except Exception as e:
print('There was an exception: {0}'.format(e))
traceback.print_exc(file=sys.stdout)
sys.exit(1)
def main():
connection = DriverRemoteConnection('ws://localhost:8182/gremlin',
'{0}_traversal'.format('bandi'))
# g = traversal().withRemote(connection)
c = client.Client('ws://localhost:8182/gremlin',
'{0}_traversal'.format('bandi'))
results = c.submit(
'g.V().group().by(label).by(properties().group().by(key).by(value().map{it.get().getClass()}))'
).all().result()
nodes_info = NodeInfo.from_result(results[0])
encoded = yaml.safe_dump({'nodes': map(lambda e: e.to_dict(), nodes_info)})
print(encoded)
print(NodeInfo.from_dict(yaml.safe_load(encoded)))
connection.close()
def test_connection(app):
gremlin_client = None
with app.app_context():
conn_string = current_app.config['JANUSGRAPH_YELP_CONN']
try:
gremlin_client = client.Client(conn_string, 'g')
gremlin_client.submit("1 + 1").next()
except Exception as e:
logging.warn(
'Could not connect to JanusGraph! JanusGraph will not be available for benchmarking. %s',
str(e))
return False
finally:
if gremlin_client is not None:
gremlin_client.close()
return True
def get_client(self, host, port, use_ssl) -> Client:
credentials = self.credentials_provider.get_iam_credentials()
request_params = make_signed_request('get', 'gremlin', '', host, port,
credentials.key,
credentials.secret,
credentials.region, use_ssl,
credentials.token)
ws_url = request_params['url'].strip('/').replace('http', 'ws')
signed_ws_request = httpclient.HTTPRequest(
ws_url, headers=request_params['headers'])
try:
c = client.Client(signed_ws_request, 'g')
return c
# TODO: handle exception explicitly
except Exception as e:
logger.error(f'error while creating client {e}')
raise e
def execute_query(query):
gremlin_client = None
with current_app.app_context():
conn_string = current_app.config['JANUSGRAPH_YELP_CONN']
try:
gremlin_client = client.Client(conn_string, 'g')
# this method blocks until the request is written to the server
result_set = gremlin_client.submit(query)
# the all method returns a concurrent.futures.Future
future_results = result_set.all()
# block until the script is evaluated and results are sent back by the server
return future_results.result()
except Exception as e:
logging.warn('Error while executing query! %s', str(e))
return str(e)
finally:
if gremlin_client is not None:
gremlin_client.close()
def __init__(self, account_name, account_key, database_name, graph_name):
self._gremlin_cleanup_graph = "g.V().drop()"
self._gremlin_insert_vertices = [
# partition 1
"g.addV('person').property('id', 'thomas').property('firstName', 'Thomas').property('age', 44).property('partitionKey', 1)",
"g.addV('person').property('id', 'mary').property('firstName', 'Mary').property('lastName', 'Andersen').property('age', 39).property('partitionKey', 1)",
"g.addV('person').property('id', 'peter').property('firstName', 'Peter').property('lastName', 'Smith').property('age', 25).property('partitionKey', 1)",
"g.addV('person').property('id', 'jenny').property('firstName', 'Jenny').property('lastName', 'Curran').property('age', 25).property('partitionKey', 1)",
# address
"g.addV('address').property('id', '8916_marvin_gardens').property('streetName', 'Marvin Gardens Rd').property('streetNumber', 8916).property('city', 'Atlanta').property('partitionKey', 1)",
"g.addV('address').property('id', '8917_marvin_gardens').property('streetName', 'Marvin Gardens Rd').property('streetNumber', 8917).property('city', 'Atlanta').property('partitionKey', 1)",
"g.addV('address').property('id', '1611_boardwalk').property('streetName', 'Boardwalk Blvd').property('streetNumber', 1611).property('city', 'New York').property('partitionKey', 1)",
# partition 2
"g.addV('person').property('id', 'ben').property('firstName', 'Ben').property('lastName', 'Miller').property('partitionKey', 2)",
"g.addV('person').property('id', 'robin').property('firstName', 'Robin').property('lastName', 'Wakefield').property('partitionKey', 2)",
"g.addV('person').property('id', 'forrest').property('firstName', 'Forrest').property('lastName', 'Gump').property('partitionKey', 2)",
"g.addV('person').property('id', 'john').property('firstName', 'John').property('lastName', 'McClane').property('age', 34).property('partitionKey', 2)",
# address
"g.addV('address').property('id', '926_parkplace').property('streetName', 'Park Place Ave').property('streetNumber', 926).property('city', 'New York').property('partitionKey', 2)",
"g.addV('address').property('id', '8917_illinois').property('streetName', 'Illinois Ave').property('streetNumber', 8917).property('city', 'Atlanta').property('partitionKey', 2)",
"g.addV('address').property('id', '4933_baltic').property('streetName', 'Baltic Ave').property('streetNumber', 4933).property('city', 'Vermont').property('partitionKey', 2)",
]
self._gremlin_insert_edges = [
"g.V('thomas').addE('knows').to(g.V('mary'))",
"g.V('thomas').addE('knows').to(g.V('ben'))",
"g.V('jenny').addE('knows').to(g.V('forrest'))",
"g.V('ben').addE('knows').to(g.V('robin'))",
"g.V('peter').addE('lives_on').to(g.V('1611_boardwalk'))",
"g.V('ben').addE('lives_on').to(g.V('926_parkplace'))",
"g.V('forrest').addE('lives_on').to(g.V('4933_baltic'))"
]
self.client = client.Client(
'wss://' + account_name + '.gremlin.cosmosdb.azure.com:443/',
'g',
username="******" + database_name + "/colls/" + graph_name,
password=account_key,
message_serializer=serializer.GraphSONSerializersV2d0())
def handler():
# Initialise client
print('Initialising client...')
gremlin_client = client.Client('wss://' + ENDPOINT + ':443/',
'g',
username="******" + DATABASE + "/colls/" +
COLLECTION,
password=PRIMARY_KEY)
print('Client initialised!')
# Purge graph
cleanup_graph(gremlin_client)
# Insert vertices (i.e. nodes)
insert_vertices(gremlin_client)
# Insert edges (i.e. nodes)
insert_edges(gremlin_client)
print('Finished!')
def get_client(self):
"""
Initializes the client for db interaction
:return:
"""
try:
c = client.Client(
'wss://topicgraph.gremlin.cosmosdb.azure.com:443/',
'g',
username="******",
password=
"******",
message_serializer=serializer.GraphSONSerializersV2d0())
self.client = c
except:
print("oops")
def connect_server():
"""Connect to azure cosmos DB.
Connet to azure cosmos DB.
Need to insert ID, DB name, table name, and key.
Args:
None
Returns:
clt(str): an instance for connecting to azure cosmos DB server
"""
clt = client.Client(
'wss://<YOURID>.gremlin.cosmos.azure.com:443/',
'g',
username="******",
password="******",
message_serializer=serializer.GraphSONSerializersV2d0())
return clt
def main():
parser = argparse.ArgumentParser()
parser.add_argument('--host', dest="host", type=str, required=True)
parser.add_argument('--port', dest="port", type=int, default=8182)
parser.add_argument('--username', dest="username", type=str, required=True)
parser.add_argument('--password', dest="password", type=str, required=True)
parser.add_argument('--threadCnt', dest="threadCnt", type=int, default=32)
parser.add_argument('--label',
dest="label",
type=str,
default=None,
help="drop element with specified label")
parser.add_argument('--edge',
dest="drop_edge_only",
action="store_true",
help="only drop edge")
parser.add_argument('--batch', dest="batch", type=int, default=128)
parser.add_argument('input',
metavar='input',
type=str,
nargs='+',
help="files including vertex/edge ids for deletion")
args = parser.parse_args()
gdb_client = client.Client('ws://%s:%d/gremlin' % (args.host, args.port),
'g',
username=args.username,
password=args.password)
gdb_data_remover = GdbParallelDataRemover(gdb_client, args.threadCnt,
args.batch)
signal.signal(signal.SIGINT, gdb_data_remover.quit)
signal.signal(signal.SIGTERM, gdb_data_remover.quit)
if len(args.input) > 0:
gdb_data_remover.drop_by_id(args.input, args.drop_edge_only)
else:
gdb_data_remover.drop(args.label, args.drop_edge_only)
def graph_connect(self):
"""Connects to graph db
Args:
Returns:
client connection
Raises:
Exception
"""
try:
return client.Client('wss://kq2.gremlin.cosmos.azure.com:443/', 'g',
username="******",
password="",
message_serializer=serializer.GraphSONSerializersV2d0()
)
except Exception as e:
print('There was an exception: {0}'.format(e))
traceback.print_exc(file=sys.stdout)
def handler():
#Initialise client
print("Initialising client ...")
gremlin_client = client.Client(
"wss://" + ENDPOINT + ":" + str(PORT) + "/",
"g",
message_serializer=serializer.GraphSONSerializersV2d0(),
username="******" + DATABASE + "/colls/" + COLLECTION,
password=PASSWORD)
print("client initlaised!")
#Pruge graph
cleanup_graph(gremlin_client)
#Insert vertices
insert_vertices(gremlin_client)
# Insert edges (nodes)
insert_edges(gremlin_client)
print("Finished !")
