def test_strategies(self):
statics.load_statics(globals())
connection = DriverRemoteConnection('ws://localhost:8182/gremlin', 'g')
#
g = Graph().traversal().withRemote(connection). \
withStrategies(TraversalStrategy("SubgraphStrategy",
{"vertices": __.hasLabel("person"),
"edges": __.hasLabel("created")}))
assert 4 == g.V().count().next()
assert 0 == g.E().count().next()
assert 1 == g.V().label().dedup().count().next()
assert "person" == g.V().label().dedup().next()
#
g = Graph().traversal().withRemote(connection). \
withStrategies(SubgraphStrategy(vertices=__.hasLabel("person"), edges=__.hasLabel("created")))
assert 4 == g.V().count().next()
assert 0 == g.E().count().next()
assert 1 == g.V().label().dedup().count().next()
assert "person" == g.V().label().dedup().next()
#
g = g.withoutStrategies(SubgraphStrategy). \
withComputer(workers=4, vertices=__.has("name", "marko"), edges=__.limit(0))
assert 1 == g.V().count().next()
assert 0 == g.E().count().next()
assert "person" == g.V().label().next()
assert "marko" == g.V().name.next()
#
g = Graph().traversal().withRemote(connection).withComputer()
assert 6 == g.V().count().next()
assert 6 == g.E().count().next()
connection.close()
def remote_connection_authenticated(request):
try:
if request.param == 'basic':
# turn off certificate verification for testing purposes only
ssl_opts = ssl.SSLContext(ssl.PROTOCOL_TLSv1_2)
ssl_opts.verify_mode = ssl.CERT_NONE
remote_conn = DriverRemoteConnection(
basic_url,
'gmodern',
username='******',
password='******',
message_serializer=serializer.GraphSONSerializersV2d0(),
transport_factory=lambda: AiohttpTransport(ssl_options=ssl_opts
))
elif request.param == 'kerberos':
remote_conn = DriverRemoteConnection(
kerberos_url,
'gmodern',
kerberized_service=kerberized_service,
message_serializer=serializer.GraphSONSerializersV2d0())
else:
raise ValueError("Invalid authentication option - " +
request.param)
except OSError:
pytest.skip('Gremlin Server is not running')
else:
def fin():
remote_conn.close()
request.addfinalizer(fin)
return remote_conn
def __init__(self):
self.b = Bindings()
self.graph = Graph()
self.connection = DriverRemoteConnection(
'ws://130.207.122.57:8182/gremlin', 'g')
self.g = self.graph.traversal().withRemote(self.connection)
logging.info("Connected")
def go():
conn = DriverRemoteConnection(
'ws://localhost:45940/gremlin', 'gmodern', pool_size=4)
g = traversal().withRemote(conn)
yield gen.sleep(0)
assert len(g.V().toList()) == 6
conn.close()
def test_traversals(self):
statics.load_statics(globals())
connection = DriverRemoteConnection('ws://localhost:8182/gremlin', 'g')
assert "remoteconnection[ws://localhost:8182/gremlin,g]" == str(connection)
g = Graph().traversal().withRemote(connection)
assert long(6) == g.V().count().toList()[0]
#
assert Vertex(1) == g.V(1).next()
assert 1 == g.V(1).id().next()
assert Traverser(Vertex(1)) == g.V(1).nextTraverser()
assert 1 == len(g.V(1).toList())
assert isinstance(g.V(1).toList(), list)
results = g.V().repeat(out()).times(2).name
results = results.toList()
assert 2 == len(results)
assert "lop" in results
assert "ripple" in results
#
assert 10 == g.V().repeat(both()).times(5)[0:10].count().next()
assert 1 == g.V().repeat(both()).times(5)[0].count().next()
assert 0 == g.V().repeat(both()).times(5)[0:0].count().next()
assert 4 == g.V()[2:].count().next()
assert 2 == g.V()[:2].count().next()
# todo: need a traversal metrics deserializer
g.V().out().profile().next()
connection.close()
def remote_connection(request):
try:
if request.param == 'graphbinaryv1':
remote_conn = DriverRemoteConnection(
anonymous_url,
'gmodern',
message_serializer=serializer.GraphBinarySerializersV1())
elif request.param == 'graphsonv2':
remote_conn = DriverRemoteConnection(
anonymous_url,
'gmodern',
message_serializer=serializer.GraphSONSerializersV2d0())
elif request.param == 'graphsonv3':
remote_conn = DriverRemoteConnection(
anonymous_url,
'gmodern',
message_serializer=serializer.GraphSONSerializersV3d0())
else:
raise ValueError("Invalid serializer option - " + request.param)
except OSError:
pytest.skip('Gremlin Server is not running')
else:
def fin():
remote_conn.close()
request.addfinalizer(fin)
return remote_conn
def test_strategies(self):
statics.load_statics(globals())
connection = DriverRemoteConnection('ws://localhost:8182/gremlin', 'g')
#
g = Graph().traversal().withRemote(connection). \
withStrategies(TraversalStrategy("SubgraphStrategy",
{"vertices": __.hasLabel("person"),
"edges": __.hasLabel("created")}))
assert 4 == g.V().count().next()
assert 0 == g.E().count().next()
assert 1 == g.V().label().dedup().count().next()
assert "person" == g.V().label().dedup().next()
#
g = Graph().traversal().withRemote(connection). \
withStrategies(SubgraphStrategy(vertices=__.hasLabel("person"), edges=__.hasLabel("created")))
assert 4 == g.V().count().next()
assert 0 == g.E().count().next()
assert 1 == g.V().label().dedup().count().next()
assert "person" == g.V().label().dedup().next()
#
g = g.withoutStrategies(SubgraphStrategy). \
withComputer(workers=4, vertices=__.has("name", "marko"), edges=__.limit(0))
assert 1 == g.V().count().next()
assert 0 == g.E().count().next()
assert "person" == g.V().label().next()
assert "marko" == g.V().name.next()
#
g = Graph().traversal().withRemote(connection).withComputer()
assert 6 == g.V().count().next()
assert 6 == g.E().count().next()
connection.close()
def test_side_effect_close(self):
connection = DriverRemoteConnection('ws://localhost:8182/gremlin', 'g')
g = Graph().traversal().withRemote(connection)
t = g.V().aggregate('a').aggregate('b')
t.toList()
# The 'a' key should return some side effects
results = t.side_effects.get('a')
assert results
# Close result is None
results = t.side_effects.close()
assert not results
# Shouldn't get any new info from server
# 'b' isn't in local cache
results = t.side_effects.get('b')
assert not results
# But 'a' should still be cached locally
results = t.side_effects.get('a')
assert results
# 'a' should have been added to local keys cache, but not 'b'
results = t.side_effects.keys()
assert len(results) == 1
a, = results
assert a == 'a'
# Try to get 'b' directly from server, should throw error
with pytest.raises(Exception):
t.side_effects.value_lambda('b')
connection.close()
def test_side_effects(self):
statics.load_statics(globals())
connection = DriverRemoteConnection('ws://localhost:45940/gremlin',
'g')
#
g = Graph().traversal().withRemote(connection)
###
t = g.V().hasLabel("project").name.iterate()
assert 0 == len(t.side_effects.keys())
try:
m = t.side_effects["m"]
raise Exception(
"Accessing a non-existent key should throw an error")
except KeyError:
pass
###
t = g.V().out("created").groupCount("m").by("name")
results = t.toSet()
assert 2 == len(results)
assert Vertex(3) in results
assert Vertex(5) in results
assert 1 == len(t.side_effects.keys())
assert "m" in t.side_effects.keys()
m = t.side_effects["m"]
assert isinstance(m, dict)
assert 2 == len(m)
assert 3 == m["lop"]
assert 1 == m["ripple"]
assert isinstance(m["lop"], long)
assert isinstance(m["ripple"], long)
###
t = g.V().out("created").groupCount("m").by("name").name.aggregate("n")
results = t.toSet()
assert 2 == len(results)
assert "lop" in results
assert "ripple" in results
assert 2 == len(t.side_effects.keys())
assert "m" in t.side_effects.keys()
assert "n" in t.side_effects.keys()
n = t.side_effects.get("n")
assert isinstance(n, dict)
assert 2 == len(n)
assert "lop" in n.keys()
assert "ripple" in n.keys()
assert 3 == n["lop"]
assert 1 == n["ripple"]
t = g.withSideEffect('m', 32).V().map(lambda: "x: x.sideEffects('m')")
results = t.toSet()
assert 1 == len(results)
assert 32 == list(results)[0]
assert 32 == t.side_effects['m']
assert 1 == len(t.side_effects.keys())
try:
x = t.side_effects["x"]
raise Exception(
"Accessing a non-existent key should throw an error")
except KeyError:
pass
connection.close()
def test_traversals(self):
statics.load_statics(globals())
connection = DriverRemoteConnection('ws://localhost:8182/gremlin', 'g')
assert "remoteconnection[ws://localhost:8182/gremlin,g]" == str(
connection)
#
g = Graph().traversal().withRemote(connection)
#
assert 6L == g.V().count().toList()[0]
#
assert Vertex(1) == g.V(1).next()
assert 1 == g.V(1).id().next()
assert Traverser(Vertex(1)) == g.V(1).nextTraverser()
assert 1 == len(g.V(1).toList())
assert isinstance(g.V(1).toList(), list)
#
results = g.V().repeat(out()).times(2).name.toList()
assert 2 == len(results)
assert "lop" in results
assert "ripple" in results
#
assert 10 == g.V().repeat(both()).times(5)[0:10].count().next()
assert 1 == g.V().repeat(both()).times(5)[0].count().next()
assert 0 == g.V().repeat(both()).times(5)[0:0].count().next()
assert 4 == g.V()[2:].count().next()
assert 2 == g.V()[:2].count().next()
# todo: need a traversal metrics deserializer
g.V().out().profile().next()
connection.close()
def lambda_handler(event, context):
graph = Graph()
uid1 = event["userId1"]
uid2 = event["userId2"]
remoteConn = DriverRemoteConnection('ws://neptunedbinstance-3f8bwqre3vsy.cft44vxyghsh.us-east-1.neptune.amazonaws.com:8182/gremlin','g')
g = graph.traversal().withRemote(remoteConn)
friends= g.V().hasLabel('User').has('uid', uid1).\
both('FRIEND').aggregate('friends'). \
valueMap().toList()
list2 = []
for item in friends:
uid = item["uid"]
list2.append(uid[0])
if uid2 in list2:
return {
'statusCode': 400
}
a=g.V().hasLabel('User').has('uid', uid1).next()
b=g.V().hasLabel('User').has('uid', uid2).next()
g.V(a).addE('FRIEND').to(b).iterate()
remoteConn.close()
# TODO implement
return {
'statusCode': 200
}
def go():
conn = DriverRemoteConnection(
'ws://localhost:45940/gremlin', 'gmodern', pool_size=4)
g = Graph().traversal().withRemote(conn)
yield gen.sleep(0)
assert len(g.V().toList()) == 6
conn.close()
def test_side_effect_close(self):
connection = DriverRemoteConnection('ws://localhost:8182/gremlin', 'g')
g = Graph().traversal().withRemote(connection)
t = g.V().aggregate('a').aggregate('b')
t.toList()
# The 'a' key should return some side effects
results = t.side_effects.get('a')
assert results
# Close result is None
results = t.side_effects.close()
assert not results
# Shouldn't get any new info from server
# 'b' isn't in local cache
results = t.side_effects.get('b')
assert not results
# But 'a' should still be cached locally
results = t.side_effects.get('a')
assert results
# 'a' should have been added to local keys cache, but not 'b'
results = t.side_effects.keys()
assert len(results) == 1
a, = results
assert a == 'a'
# Try to get 'b' directly from server, should throw error
with pytest.raises(Exception):
t.side_effects.value_lambda('b')
connection.close()
def remote_connection_authenticated(request):
try:
if request.param == 'basic':
remote_conn = DriverRemoteConnection(
basic_url,
'gmodern',
username='******',
password='******',
message_serializer=serializer.GraphSONSerializersV2d0())
elif request.param == 'kerberos':
remote_conn = DriverRemoteConnection(
kerberos_url,
'gmodern',
kerberized_service=kerberized_service,
message_serializer=serializer.GraphSONSerializersV2d0())
else:
raise ValueError("Invalid authentication option - " +
request.param)
except OSError:
pytest.skip('Gremlin Server is not running')
else:
def fin():
remote_conn.close()
request.addfinalizer(fin)
return remote_conn
def traversal_source(self):
"""Create a GraphTraversalSource and return.
Once `g` has been created using a connection, we can start to write
Gremlin traversals to query the remote graph.
Raises:
RuntimeError: If the interactive script is not running.
Examples:
.. code:: python
sess = graphscope.session()
graph = load_modern_graph(sess, modern_graph_data_dir)
interactive = sess.gremlin(graph)
g = interactive.traversal_source()
print(g.V().both()[1:3].toList())
print(g.V().both().name.toList())
Returns:
`GraphTraversalSource`
"""
if self._status != InteractiveQueryStatus.Running:
raise RuntimeError(
"Interactive query is unavailable with %s status.",
str(self._status))
if self._conn is None:
self._conn = DriverRemoteConnection(self._graph_url[0], "g")
return traversal().withRemote(self._conn)
def create_vertices(term: str, entities: list):
"""
Creates vertices to graph term to entities
"""
graph = Graph()
connection = DriverRemoteConnection(
f'wss://{os.environ["NEPTUNE_ENDPOINT"]}:8182/gremlin', 'g')
g = graph.traversal().withRemote(connection)
# Check if a vertex has been created for the term
term_vertex = g.V().has("term", "value", term)
term_vertex = term_vertex.next() if term_vertex.hasNext() else g.addV(
"term").property("value", term).next()
# Create an entity vertex for each and link to term
for e in entities:
entity_vertex = g.V().has("entity", "value", e)
entity_vertex = entity_vertex.next() if entity_vertex.hasNext() else \
g.addV("entity") \
.property("value", e["entity"]) \
.property("score", e["score"]) \
.property("type", e["type"]).next()
g.V(term_vertex).addE("has_entity").to(entity_vertex).iterate()
connection.close()
def start_up(self, **kwargs):
# it can cause exceptions when starting all at once!
# This solves it currently... ( TODO why does it not allow multiple connections consistently?)
sleep(random.randint(1, 15))
# This generates a connection to the gremlin server
self.conn = DriverRemoteConnection(kwargs["gremlin_url"], kwargs["gremlin_traversal_source"], pool_size=1)
self.remote_graph = traversal().withRemote(self.conn)
def main():
graph = Graph()
remote = DriverRemoteConnection('ws://' + os.environ['DB_ENDPOINT'] + ':8182/gremlin', 'g')
g = graph.traversal().withRemote(remote)
seed_users(g)
remote.close()
def test_side_effects(self):
statics.load_statics(globals())
connection = DriverRemoteConnection('ws://localhost:8182/gremlin', 'g')
#
g = Graph().traversal().withRemote(connection)
###
t = g.V().hasLabel("project").name.iterate()
assert 0 == len(t.side_effects.keys())
try:
m = t.side_effects["m"]
raise Exception("Accessing a non-existent key should throw an error")
except KeyError:
pass
###
t = g.V().out("created").groupCount("m").by("name")
results = t.toSet()
assert 2 == len(results)
assert Vertex(3) in results
assert Vertex(5) in results
assert 1 == len(t.side_effects.keys())
assert "m" in t.side_effects.keys()
m = t.side_effects["m"]
assert isinstance(m, dict)
assert 2 == len(m)
assert 3 == m["lop"]
assert 1 == m["ripple"]
assert isinstance(m["lop"], long)
assert isinstance(m["ripple"], long)
###
t = g.V().out("created").groupCount("m").by("name").name.aggregate("n")
results = t.toSet()
assert 2 == len(results)
assert "lop" in results
assert "ripple" in results
assert 2 == len(t.side_effects.keys())
assert "m" in t.side_effects.keys()
assert "n" in t.side_effects.keys()
n = t.side_effects.get("n")
assert isinstance(n, dict)
assert 2 == len(n)
assert "lop" in n.keys()
assert "ripple" in n.keys()
assert 3 == n["lop"]
assert 1 == n["ripple"]
t = g.withSideEffect('m', 32).V().map(lambda: "x: x.sideEffects('m')")
results = t.toSet()
assert 1 == len(results)
assert 32 == list(results)[0]
assert 32 == t.side_effects['m']
assert 1 == len(t.side_effects.keys())
try:
x = t.side_effects["x"]
raise Exception("Accessing a non-existent key should throw an error")
except KeyError:
pass
connection.close()
def handle_request():
try:
remote_connection = DriverRemoteConnection("ws://localhost:45940/gremlin", "g")
g = traversal().withRemote(remote_connection)
g.V().limit(1).toList()
remote_connection.close()
return True
except RuntimeError:
return False
def actors(request):
graph = Graph()
remoteConn = DriverRemoteConnection('ws://<neptune endpoint>:8182/gremlin',
'g')
g = graph.traversal().withRemote(remoteConn)
myList = g.V().has(
'title', request.POST['movie_name']).in_().limit(40).values().toList()
remoteConn.close()
context = {'movie': request.POST['movie_name'], 'actors': myList}
return render(request, 'polls/movie-results.html', context)
def lambda_handler(event, context):
print(event)
graph = Graph()
table = dynamodb_client.Table('user')
tmp = table.scan()
dict1 = {}
for item in tmp['Items']:
dict1[item['UID']] = []
pair = (item['firstName'], item['lastName'], item['pic_url'])
dict1[item['UID']].append(pair)
print(dict1)
remoteConn = DriverRemoteConnection(
'ws://neptunedbinstance-3f8bwqre3vsy.cft44vxyghsh.us-east-1.neptune.amazonaws.com:8182/gremlin',
'g')
g = graph.traversal().withRemote(remoteConn)
# a=g.V().hasLabel('User').has('uid', '1834389').next()
# b=g.V().hasLabel('User').has('uid', '594112').next()
# g.V(a).addE('Friend').to(b).iterate()
key = event["userId"]
friends= g.V().hasLabel('User').has('uid', key).\
both('FRIEND').aggregate('friends'). \
valueMap().toList()
list2 = []
for item in friends:
tmplist = []
uid = item["uid"]
tmplist.append(uid[0])
for tmp in dict1[uid[0]][0]:
tmplist.append(tmp)
list2.append(tmplist)
return {'statusCode': 200, 'body': list2}
count = 0
recommend_list = {
k: v
for k, v in sorted(recommend.items(), key=lambda item: -item[1])
}
list1 = []
for item in recommend_list:
if item != key:
data = dynamodb.get_item(TableName='user',
Key={'UID': {
'S': str(item)
}})
pair = (str(item), data['Item']['firstName']['S'],
data['Item']['lastName']['S'],
data['Item']['pic_url']['S'])
list1.append(pair)
count += 1
if (count == 3):
break
print(list1)
remoteConn.close()
# TODO implement
return {'statusCode': 200, 'body': list1}
def movies(request):
graph = Graph()
remoteConn = DriverRemoteConnection('ws://<path to neptune>:8182/gremlin',
'g')
g = graph.traversal().withRemote(remoteConn)
#print(g.V().limit(2).toList())
myList = g.V().has(
'name', request.POST['actor_name']).out().limit(40).values().toList()
remoteConn.close()
context = {'actor': request.POST['actor_name'], 'movies': myList}
return render(request, 'polls/movie-results.html', context)
def main():
graph = Graph()
remote = DriverRemoteConnection(
'ws://' + os.environ['DB_ENDPOINT'] + ':8182/gremlin', 'g')
g = graph.traversal().withRemote(remote)
print('Flushing existing vertices in local db...')
flush(g)
print('Done.')
remote.close()
def _executor(self, q, loop):
try:
connection = DriverRemoteConnection('ws://localhost:45940/gremlin',
'g',
loop=loop)
g = Graph().traversal().withRemote(connection)
assert len(g.V().toList()) == 6
except:
q.put(sys.exc_info()[0])
else:
q.put('success!')
connection.close()
def open(self):
'''
open the remote connection
Returns:
GraphTraversalSource: the remote graph traversal source
'''
self.graph = Graph()
self.url = 'ws://%s:%s/gremlin' % (self.server, self.port)
self.connection = DriverRemoteConnection(self.url, 'g')
# The connection should be closed on shut down to close open connections with connection.close()
self.g = self.graph.traversal().withRemote(self.connection)
return self.g
def separation(request):
statics.load_statics(globals())
inputs = [x.strip() for x in request.POST['actor_names'].split(',')]
graph = Graph()
remoteConn = DriverRemoteConnection(
'ws://<neptune endpoint>.com:8182/gremlin', 'g')
g = graph.traversal().withRemote(remoteConn)
myList = g.V().has(
'name', inputs[0]).repeat(out().in_().simplePath()).until(
has('name',
inputs[1])).path().by('name').by('title').limit(40).toList()
remoteConn.close()
context = {'actors': request.POST['actor_names'], 'separation': myList}
return render(request, 'polls/movie-results.html', context)
def kg_testing(inst=1, M=10, N=5, testing=False):
# number of data points and properties
m = M
p = N
if p > const.MAX_FEATURES:
p = const.MAX_FEATURES
# define the number of splits of each property
s = p if p <= const.MAX_SPLITS else const.MAX_SPLITS
# uniformly sample values between 0 and 1 as the data set
dat = np.random.sample(size=(m, p))
# create column names (normally obtained by var.dtype.names)
#
# use an explicit dict to make sure that the order is preserved
coln = [("col" + str(i), (i - 1)) for i in range(1, p + 1)]
# create the data for the sample knowledge graph (only one brain)
kgdat = create_kg(inst, dat, s, [[int(i) for i in np.asarray(coln)[:, 1]]])
# populate the knowledge graph into the remote DB
#
# instantiate a JanusGraph object
graph = Graph()
# connection to the remote server
conn = DriverRemoteConnection(url_kg(inst), 'g')
# get the remote graph traversal
g = graph.traversal().withRemote(conn)
# we only want to process the right brain
print(kg(const.V, inst, coln, kgdat, g, False, testing))
# after building the knowledge graph, use the output of ocr to test the GloVe write
#
# call cognitive to produce the ocr output
oret = ocr_testing()
# get the location of the glove file
src = cfg["instances"][inst]["src"]["index"]
typ = cfg["instances"][inst]["src"]["types"]["glove"]
gfl = cfg["instances"][inst]["sources"][src][typ]["connection"]["file"]
# call extendglove to produce the GloVe output and transform it to an array
# with the first term in each row being the key and all other terms are values
rdat = extendglove(oret[0][0], gfl[0])
rdat = [(k, v) for k, v in list(rdat.items())[0:M]]
# write the glove output to the knowledge graph
print(kg(const.ENTS, inst, coln, rdat, g, False, testing))
# get the ocr data ... using the real way to get the ocr data here
typ = cfg["instances"][inst]["src"]["types"]["ocrf"]
pdfs = cfg["instances"][inst]["sources"][src][typ]["connection"]["files"]
cdat = cognitive(const.OCR, pdfs, inst, False, testing)
# write the ocr data to the graph
print(kg(const.CONS, inst, coln, cdat[1:], g, True, testing))
# close the connection
conn.close()
# test the thought function with the default number of predictions 3
print(thought(inst, coln))
def remote_connection(request):
try:
if request.param == 'v2':
remote_conn = DriverRemoteConnection('ws://localhost:45940/gremlin', 'gmodern',
message_serializer=serializer.GraphSONSerializersV2d0())
else:
remote_conn = DriverRemoteConnection('ws://localhost:45940/gremlin', 'gmodern')
except OSError:
pytest.skip('Gremlin Server is not running')
else:
def fin():
remote_conn.close()
request.addfinalizer(fin)
return remote_conn
def __init__(
self,
*,
host: str,
port: Optional[int] = None,
user: Optional[str] = None,
password: Optional[str] = None,
traversal_source: 'str' = 'g',
key_property_name: str = 'key',
driver_remote_connection_options: Mapping[str, Any] = {}) -> None:
driver_remote_connection_options = dict(
driver_remote_connection_options)
# as others, we repurpose host a url
driver_remote_connection_options.update(url=host)
# port should be part of that url
if port is not None:
raise NotImplementedError(f'port is not allowed! port={port}')
if user is not None:
driver_remote_connection_options.update(username=user)
if password is not None:
driver_remote_connection_options.update(password=password)
driver_remote_connection_options.update(
traversal_source=traversal_source)
super().__init__(key_property_name=key_property_name,
remote_connection=DriverRemoteConnection(
**driver_remote_connection_options))
def __init__(self):
graph = Graph()
ip_pools = ['192.168.50.5', '192.168.50.6', '192.168.50.7']
request_ip = random.sample(ip_pools, 1)[0]
self.g = graph.traversal().withRemote(
DriverRemoteConnection('ws://' + request_ip + ':8182/gremlin',
'g'))
def main():
"""
Export graph data to CSV format.
"""
parser = argparse.ArgumentParser()
parser.add_argument("label", type=str, help="Please specify TYPE or LABEL")
parser.add_argument("-e",
"--edge",
action="store_true",
help="output edge")
args = parser.parse_args()
graph = Graph()
g = graph.traversal().withRemote(DriverRemoteConnection(URL, 'g'))
with open(args.label + ".csv", "w", newline="") as outf:
writer = csv.writer(outf,
delimiter=",",
quotechar='"',
quoting=csv.QUOTE_MINIMAL)
lines = get_lines(g, args.edge, args.label)
key_list = get_key_list(g, args.edge, lines[0])
key_list.append('label')
writer.writerow(key_list)
for line in lines:
tmp = get_line(g, args.edge, line)
tmp.append(args.label)
writer.writerow(tmp)
def lambda_handler(event, context):
g = graph.traversal().withRemote(DriverRemoteConnection(db, 'g')) ###!!!
accession = event['biosample_id']
record = xml.fromstring(
requests.get(
link.format(accession=accession,
api_key=api_key)).text)
biosample = record.find('.//SAMPLE_DESCRIPTOR/IDENTIFIERS/EXTERNAL_ID').text
for item in record.findall('.//Link/Id'):
link_id = item.text
record = xml.fromstring(
requests.get(
fetch.format(database='sra',
accession=accession,
api_key=api_key)).text)
for run in record.findall('.//RunSet/Run'):
sra_accession = run.attrib['accession']
if not list(g.E().hasLabel('NAMED_IN')
.filter(__.properties().values('name').is_(sra_accession))):
#this one is new
uri = f's3://edb/{biosample}/runs/{sra_accession}/'
ebd_id = load_record(run, record, biosample, uri, g)
sraq.send_message(MessageBody=json.dumps(dict(data=dict(accession=sra_accession,
biosample=event.get('biosample', ''),
bioproject=event.get('bioproject', '')
s3Bucket=uri,
edb_record_id = edb_id)),
results=dict()))
return ''
def data2janus(data: dict, args):
from gremlin_python.structure.graph import Graph
from gremlin_python.driver.driver_remote_connection import DriverRemoteConnection
address = args.output
graph = Graph()
connection = DriverRemoteConnection(address, 'g')
g = graph.traversal().withRemote(connection)
for name, vertex in data['vertex'].items():
vertex_count = len(vertex['data'])
for i, properties in enumerate(vertex['data']):
vv = g.addV(name)
for k, v in properties.items():
vv = vv.property(k, v)
vv.next()
logger.info("{}: {}/{}".format(name, i, vertex_count))
for name, edge in data['edge'].items():
edge_count = len(edge['data'])
for i, e in enumerate(edge['data']):
from_ = g.V().hasLabel(e['from']['tag'])
for k, v in e['from']['match'].items():
from_ = from_.has(k, v)
to_ = g.V().hasLabel(e['to']['tag'])
for k, v in e['to']['match'].items():
to_ = to_.has(k, v)
ee = g.addE(name).from_(from_).to(to_)
for k, v in e['data'].items():
ee = ee.property(k, v)
ee.next()
logger.info("{}: {}/{}".format(name, i, edge_count))
logger.info("total vertex count: {}, total edge count: {}".format(
g.V().count().next(),
g.E().count().next()))
def __init__(self,
*,
host: str,
port: Optional[int] = None,
user: str = None,
password: Optional[Union[str, Mapping[str, str]]] = None,
driver_remote_connection_options: Mapping[str, Any] = {},
aws4auth_options: Mapping[str, Any] = {},
websocket_options: Mapping[str, Any] = {}) -> None:
driver_remote_connection_options = dict(
driver_remote_connection_options)
# as others, we repurpose host a url
driver_remote_connection_options.update(url=host)
# port should be part of that url
if port is not None:
raise NotImplementedError(f'port is not allowed! port={port}')
# always g for Neptune
driver_remote_connection_options.update(traversal_source='g')
# for IAM auth, we need the triplet
if isinstance(password, Mapping):
if user or 'aws_access_key_id' not in password or 'aws_secret_access_key' not in password or \
'service_region' not in password:
raise NotImplementedError(
f'to use authentication, pass a Mapping with aws_access_key_id, '
f'aws_secret_access_key, service_region!')
aws_access_key_id = password['aws_access_key_id']
aws_secret_access_key = password['aws_secret_access_key']
service_region = password['service_region']
def factory() -> Aws4AuthWebsocketTransport:
return Aws4AuthWebsocketTransport(
aws_access_key_id=aws_access_key_id,
aws_secret_access_key=aws_secret_access_key,
service_region=service_region,
extra_aws4auth_options=aws4auth_options or {},
extra_websocket_options=websocket_options or {})
driver_remote_connection_options.update(transport_factory=factory)
elif password is not None:
raise NotImplementedError(
f'to use authentication, pass a Mapping with aws_access_key_id, '
f'aws_secret_access_key, service_region!')
else:
raise NotImplementedError(
f'to use authentication, pass a Mapping with aws_access_key_id, '
f'aws_secret_access_key, service_region!')
# we could use the default Transport, but then we'd have to take different options, which feels clumsier.
def factory() -> WebsocketClientTransport:
return WebsocketClientTransport(
extra_websocket_options=websocket_options or {})
driver_remote_connection_options.update(transport_factory=factory)
super().__init__(key_property_name='key',
remote_connection=DriverRemoteConnection(
**driver_remote_connection_options))
def _executor(q, conn):
close = False
if not conn:
# This isn't a fixture so close manually
close = True
conn = DriverRemoteConnection(
'ws://localhost:45940/gremlin', 'gmodern', pool_size=4)
try:
g = Graph().traversal().withRemote(conn)
future = g.V().promise()
t = future.result()
assert len(t.toList()) == 6
except:
q.put(sys.exc_info()[0])
else:
q.put('success!')
# Close conn
if close:
conn.close()
results = t.side_effects.get('b')
assert not results
# But 'a' should still be cached locally
results = t.side_effects.get('a')
assert results
# 'a' should have been added to local keys cache, but not 'b'
results = t.side_effects.keys()
assert len(results) == 1
a, = results
assert a == 'a'
# Try to get 'b' directly from server, should throw error
with pytest.raises(Exception):
t.side_effects.value_lambda('b')
connection.close()
if __name__ == '__main__':
test = False
try:
connection = DriverRemoteConnection('ws://localhost:8182/gremlin', 'g')
test = True
connection.close()
except:
print("GremlinServer is not running and this test case will not execute: " + __file__)
if test:
unittest.main()
