def test_bytecode(self):
g = Graph().traversal()
bytecode = g.V().out("created").bytecode
assert 0 == len(bytecode.bindings.keys())
assert 0 == len(bytecode.source_instructions)
assert 2 == len(bytecode.step_instructions)
assert "V" == bytecode.step_instructions[0][0]
assert "out" == bytecode.step_instructions[1][0]
assert "created" == bytecode.step_instructions[1][1]
assert 1 == len(bytecode.step_instructions[0])
assert 2 == len(bytecode.step_instructions[1])
##
bytecode = g.withSack(1).E().groupCount().by("weight").bytecode
assert 0 == len(bytecode.bindings.keys())
assert 1 == len(bytecode.source_instructions)
assert "withSack" == bytecode.source_instructions[0][0]
assert 1 == bytecode.source_instructions[0][1]
assert 3 == len(bytecode.step_instructions)
assert "E" == bytecode.step_instructions[0][0]
assert "groupCount" == bytecode.step_instructions[1][0]
assert "by" == bytecode.step_instructions[2][0]
assert "weight" == bytecode.step_instructions[2][1]
assert 1 == len(bytecode.step_instructions[0])
assert 1 == len(bytecode.step_instructions[1])
assert 2 == len(bytecode.step_instructions[2])
def test_strategies(self):
statics.load_statics(globals())
connection = DriverRemoteConnection('ws://localhost:8182/gremlin', 'g')
#
g = Graph().traversal().withRemote(connection). \
withStrategies(TraversalStrategy("SubgraphStrategy",
{"vertices": __.hasLabel("person"),
"edges": __.hasLabel("created")}))
assert 4 == g.V().count().next()
assert 0 == g.E().count().next()
assert 1 == g.V().label().dedup().count().next()
assert "person" == g.V().label().dedup().next()
#
g = Graph().traversal().withRemote(connection). \
withStrategies(SubgraphStrategy(vertices=__.hasLabel("person"), edges=__.hasLabel("created")))
assert 4 == g.V().count().next()
assert 0 == g.E().count().next()
assert 1 == g.V().label().dedup().count().next()
assert "person" == g.V().label().dedup().next()
#
g = g.withoutStrategies(SubgraphStrategy). \
withComputer(workers=4, vertices=__.has("name", "marko"), edges=__.limit(0))
assert 1 == g.V().count().next()
assert 0 == g.E().count().next()
assert "person" == g.V().label().next()
assert "marko" == g.V().name.next()
#
g = Graph().traversal().withRemote(connection).withComputer()
assert 6 == g.V().count().next()
assert 6 == g.E().count().next()
connection.close()
def test_side_effects(self):
statics.load_statics(globals())
connection = DriverRemoteConnection('ws://localhost:8182/gremlin', 'g')
#
g = Graph().traversal().withRemote(connection)
###
t = g.V().hasLabel("project").name.iterate()
assert 0 == len(t.side_effects.keys())
try:
m = t.side_effects["m"]
raise Exception("Accessing a non-existent key should throw an error")
except KeyError:
pass
###
t = g.V().out("created").groupCount("m").by("name")
results = t.toSet()
assert 2 == len(results)
assert Vertex(3) in results
assert Vertex(5) in results
assert 1 == len(t.side_effects.keys())
assert "m" in t.side_effects.keys()
m = t.side_effects["m"]
assert isinstance(m, dict)
assert 2 == len(m)
assert 3 == m["lop"]
assert 1 == m["ripple"]
assert isinstance(m["lop"], long)
assert isinstance(m["ripple"], long)
###
t = g.V().out("created").groupCount("m").by("name").name.aggregate("n")
results = t.toSet()
assert 2 == len(results)
assert "lop" in results
assert "ripple" in results
assert 2 == len(t.side_effects.keys())
assert "m" in t.side_effects.keys()
assert "n" in t.side_effects.keys()
n = t.side_effects.get("n")
assert isinstance(n, dict)
assert 2 == len(n)
assert "lop" in n.keys()
assert "ripple" in n.keys()
assert 3 == n["lop"]
assert 1 == n["ripple"]
t = g.withSideEffect('m', 32).V().map(lambda: "x: x.sideEffects('m')")
results = t.toSet()
assert 1 == len(results)
assert 32 == list(results)[0]
assert 32 == t.side_effects['m']
assert 1 == len(t.side_effects.keys())
try:
x = t.side_effects["x"]
raise Exception("Accessing a non-existent key should throw an error")
except KeyError:
pass
connection.close()
def test_datetime(self, remote_connection_v2):
g = Graph().traversal().withRemote(remote_connection_v2)
dt = datetime.datetime.fromtimestamp(1481750076295 / 1000)
resp = g.addV('test_vertex').property('dt', dt).toList()
vid = resp[0].id
try:
dt_prop = g.V(vid).properties('dt').toList()[0]
assert isinstance(dt_prop.value, datetime.datetime)
assert dt_prop.value == dt
finally:
g.V(vid).drop().iterate()
def test_uuid(self, remote_connection_v2):
g = Graph().traversal().withRemote(remote_connection_v2)
uid = uuid.UUID("41d2e28a-20a4-4ab0-b379-d810dede3786")
resp = g.addV('test_vertex').property('uuid', uid).toList()
vid = resp[0].id
try:
uid_prop = g.V(vid).properties('uuid').toList()[0]
assert isinstance(uid_prop.value, uuid.UUID)
assert uid_prop.value == uid
finally:
g.V(vid).drop().iterate()
def test_timestamp(self, remote_connection_v2):
g = Graph().traversal().withRemote(remote_connection_v2)
ts = timestamp(1481750076295 / 1000)
resp = g.addV('test_vertex').property('ts', ts)
resp = resp.toList()
vid = resp[0].id
try:
ts_prop = g.V(vid).properties('ts').toList()[0]
assert isinstance(ts_prop.value, timestamp)
assert ts_prop.value == ts
finally:
g.V(vid).drop().iterate()
def test_client_bytecode_options(client):
# smoke test to validate serialization of OptionsStrategy. no way to really validate this from an integration
# test perspective because there's no way to access the internals of the strategy via bytecode
g = Graph().traversal()
t = g.withStrategies(OptionsStrategy(options={"x": "test", "y": True})).V()
message = RequestMessage('traversal', 'bytecode', {'gremlin': t.bytecode, 'aliases': {'g': 'gmodern'}})
result_set = client.submit(message)
assert len(result_set.all().result()) == 6
##
t = g.with_("x", "test").with_("y", True).V()
message = RequestMessage('traversal', 'bytecode', {'gremlin': t.bytecode, 'aliases': {'g': 'gmodern'}})
result_set = client.submit(message)
assert len(result_set.all().result()) == 6
def test_big_result_set_secure(secure_client):
g = Graph().traversal()
t = g.inject(1).repeat(__.addV('person').property('name', __.loops())).times(20000).count()
message = RequestMessage('traversal', 'bytecode', {'gremlin': t.bytecode, 'aliases': {'g': 'g'}})
result_set = secure_client.submit(message)
results = []
for result in result_set:
results += result
assert len(results) == 1
t = g.V().limit(10)
message = RequestMessage('traversal', 'bytecode', {'gremlin': t.bytecode, 'aliases': {'g': 'g'}})
result_set = secure_client.submit(message)
results = []
for result in result_set:
results += result
assert len(results) == 10
t = g.V().limit(100)
message = RequestMessage('traversal', 'bytecode', {'gremlin': t.bytecode, 'aliases': {'g': 'g'}})
result_set = secure_client.submit(message)
results = []
for result in result_set:
results += result
assert len(results) == 100
t = g.V().limit(1000)
message = RequestMessage('traversal', 'bytecode', {'gremlin': t.bytecode, 'aliases': {'g': 'g'}})
result_set = secure_client.submit(message)
results = []
for result in result_set:
results += result
assert len(results) == 1000
t = g.V().limit(10000)
message = RequestMessage('traversal', 'bytecode', {'gremlin': t.bytecode, 'aliases': {'g': 'g'}})
result_set = secure_client.submit(message)
results = []
for result in result_set:
results += result
assert len(results) == 10000
def test_dsl(remote_connection):
social = Graph().traversal(SocialTraversalSource).withRemote(remote_connection)
assert social.persons("marko").knows("josh").next()
assert social.persons("marko").youngestFriendsAge().next() == 27
assert social.persons().count().next() == 4
assert social.persons("marko", "josh").count().next() == 2
assert social.persons().filter(__.createdAtLeast(2)).count().next() == 1
def test_side_effects(self):
statics.load_statics(globals())
connection = DriverRemoteConnection('ws://localhost:8182/gremlin', 'g')
#
g = Graph().traversal().withRemote(connection)
###
t = g.V().hasLabel("project").name.iterate()
assert 0 == len(t.side_effects.keys())
try:
m = t.side_effects["m"]
raise Exception(
"Accessing a non-existent key should throw an error")
except KeyError:
pass
###
t = g.V().out("created").groupCount("m").by("name")
results = t.toSet()
assert 2 == len(results)
assert Vertex(3) in results
assert Vertex(5) in results
assert 1 == len(t.side_effects.keys())
assert "m" in t.side_effects.keys()
m = t.side_effects["m"]
assert isinstance(m, dict)
assert 2 == len(m)
assert 3 == m["lop"]
assert 1 == m["ripple"]
assert isinstance(m["lop"], long)
assert isinstance(m["ripple"], long)
###
t = g.V().out("created").groupCount("m").by("name").name.aggregate("n")
results = t.toSet()
assert 2 == len(results)
assert "lop" in results
assert "ripple" in results
assert 2 == len(t.side_effects.keys())
assert "m" in t.side_effects.keys()
assert "n" in t.side_effects.keys()
n = t.side_effects.get("n")
assert isinstance(n, dict)
assert 2 == len(n)
assert "lop" in n.keys()
assert "ripple" in n.keys()
assert 3 == n["lop"]
assert 1 == n["ripple"]
#
t = g.withSideEffect('m', 32).V().map(lambda: "x: x.sideEffects('m')")
results = t.toSet()
assert 1 == len(results)
assert 32 == list(results)[0]
assert 32 == t.side_effects['m']
assert 1 == len(t.side_effects.keys())
try:
x = t.side_effects["x"]
raise Exception(
"Accessing a non-existent key should throw an error")
except KeyError:
pass
connection.close()
def test_bytecode(self):
g = Graph().traversal()
bytecode = g.V().out("created").bytecode
assert 0 == len(bytecode.bindings.keys())
assert 0 == len(bytecode.source_instructions)
assert 2 == len(bytecode.step_instructions)
assert "V" == bytecode.step_instructions[0][0]
assert "out" == bytecode.step_instructions[1][0]
assert "created" == bytecode.step_instructions[1][1]
assert 1 == len(bytecode.step_instructions[0])
assert 2 == len(bytecode.step_instructions[1])
##
bytecode = g.withSack(1).E().groupCount().by("weight").bytecode
assert 0 == len(bytecode.bindings.keys())
assert 1 == len(bytecode.source_instructions)
assert "withSack" == bytecode.source_instructions[0][0]
assert 1 == bytecode.source_instructions[0][1]
assert 3 == len(bytecode.step_instructions)
assert "E" == bytecode.step_instructions[0][0]
assert "groupCount" == bytecode.step_instructions[1][0]
assert "by" == bytecode.step_instructions[2][0]
assert "weight" == bytecode.step_instructions[2][1]
assert 1 == len(bytecode.step_instructions[0])
assert 1 == len(bytecode.step_instructions[1])
assert 2 == len(bytecode.step_instructions[2])
##
bytecode = g.V(Bindings.of('a', [1,2,3])) \
.out(Bindings.of('b','created')) \
.where(__.in_(Bindings.of('c','created'), Bindings.of('d','knows')) \
.count().is_(Bindings.of('e',P.gt(2)))).bytecode
assert 5 == len(bytecode.bindings.keys())
assert [1, 2, 3] == bytecode.bindings['a']
assert 'created' == bytecode.bindings['b']
assert 'created' == bytecode.bindings['c']
assert 'knows' == bytecode.bindings['d']
assert P.gt(2) == bytecode.bindings['e']
assert Binding('b', 'created') == bytecode.step_instructions[1][1]
assert 'binding[b=created]' == str(bytecode.step_instructions[1][1])
assert isinstance(hash(bytecode.step_instructions[1][1]), int)
def test_bytecode(self):
g = Graph().traversal()
bytecode = g.V().out("created").bytecode
assert 0 == len(bytecode.bindings.keys())
assert 0 == len(bytecode.source_instructions)
assert 2 == len(bytecode.step_instructions)
assert "V" == bytecode.step_instructions[0][0]
assert "out" == bytecode.step_instructions[1][0]
assert "created" == bytecode.step_instructions[1][1]
assert 1 == len(bytecode.step_instructions[0])
assert 2 == len(bytecode.step_instructions[1])
##
bytecode = g.withSack(1).E().groupCount().by("weight").bytecode
assert 0 == len(bytecode.bindings.keys())
assert 1 == len(bytecode.source_instructions)
assert "withSack" == bytecode.source_instructions[0][0]
assert 1 == bytecode.source_instructions[0][1]
assert 3 == len(bytecode.step_instructions)
assert "E" == bytecode.step_instructions[0][0]
assert "groupCount" == bytecode.step_instructions[1][0]
assert "by" == bytecode.step_instructions[2][0]
assert "weight" == bytecode.step_instructions[2][1]
assert 1 == len(bytecode.step_instructions[0])
assert 1 == len(bytecode.step_instructions[1])
assert 2 == len(bytecode.step_instructions[2])
##
bytecode = g.V(Bindings.of('a', [1,2,3])) \
.out(Bindings.of('b','created')) \
.where(__.in_(Bindings.of('c','created'), Bindings.of('d','knows')) \
.count().is_(Bindings.of('e',P.gt(2)))).bytecode
assert 5 == len(bytecode.bindings.keys())
assert [1,2,3] == bytecode.bindings['a']
assert 'created' == bytecode.bindings['b']
assert 'created' == bytecode.bindings['c']
assert 'knows' == bytecode.bindings['d']
assert P.gt(2) == bytecode.bindings['e']
assert Binding('b','created') == bytecode.step_instructions[1][1]
assert 'binding[b=created]' == str(bytecode.step_instructions[1][1])
assert isinstance(hash(bytecode.step_instructions[1][1]),int)
def prepare_traversal_source(scenario):
# some tests create data - create a fresh remote to the empty graph and clear that graph prior to each test
if not ("graphson" in world.config.user_data):
raise ValueError(
'test configuration requires setting of --user-data="graphson=*" to one of [v2,v3]'
)
if world.config.user_data["graphson"] == "v3":
s = serializer.GraphSONSerializersV3d0()
elif world.config.user_data["graphson"] == "v2":
s = serializer.GraphSONSerializersV2d0()
else:
raise ValueError(
'serializer set with --user-data="graphson=v2" must be one of [v2,v3]'
)
remote = DriverRemoteConnection('ws://localhost:45940/gremlin',
"ggraph",
message_serializer=s)
scenario.context.remote_conn["empty"] = remote
g = Graph().traversal().withRemote(remote)
g.V().drop().iterate()
def test_no_sugar_for_magic_methods(self):
g = traversal().withGraph(Graph())
t = g.V().age
assert 2 == len(t.bytecode.step_instructions)
try:
t = g.V().__len__
fail("can't do sugar with magic")
except AttributeError as err:
assert str(
err
) == 'Python magic methods or keys starting with double underscore cannot be used for Gremlin sugar - prefer values(__len__)'
def test_configurable(self):
g = Graph().traversal()
bytecode = g.withStrategies(MatchAlgorithmStrategy("greedy")).bytecode
assert 1 == len(bytecode.source_instructions)
assert 2 == len(bytecode.source_instructions[0])
assert "withStrategies" == bytecode.source_instructions[0][0]
assert MatchAlgorithmStrategy() == bytecode.source_instructions[0][1]
assert "MatchAlgorithmStrategy" == str(
bytecode.source_instructions[0][1])
assert hash(MatchAlgorithmStrategy()) == hash(
bytecode.source_instructions[0]
[1]) # even though different confs, same strategy
assert 0 == len(g.traversal_strategies.traversal_strategies
) # these strategies are proxies
###
bytecode = g.withStrategies(
SubgraphStrategy(vertices=__.has("name", "marko"))).bytecode
assert 1 == len(bytecode.source_instructions)
assert 2 == len(bytecode.source_instructions[0])
assert "withStrategies" == bytecode.source_instructions[0][0]
assert SubgraphStrategy() == bytecode.source_instructions[0][1]
strategy = bytecode.source_instructions[0][1]
assert 1 == len(strategy.configuration)
assert __.has("name", "marko") == strategy.configuration["vertices"]
###
bytecode = g.withStrategies(
OptionsStrategy(options={
"x": "test",
"y": True
})).bytecode
assert 1 == len(bytecode.source_instructions)
assert 2 == len(bytecode.source_instructions[0])
assert "withStrategies" == bytecode.source_instructions[0][0]
assert OptionsStrategy() == bytecode.source_instructions[0][1]
strategy = bytecode.source_instructions[0][1]
assert 2 == len(strategy.configuration)
assert "test" == strategy.configuration["x"]
assert strategy.configuration["y"]
def test_client_bytecode_options(client):
# smoke test to validate serialization of OptionsStrategy. no way to really validate this from an integration
# test perspective because there's no way to access the internals of the strategy via bytecode
g = Graph().traversal()
t = g.withStrategies(OptionsStrategy(options={"x": "test", "y": True})).V()
message = RequestMessage('traversal', 'bytecode', {
'gremlin': t.bytecode,
'aliases': {
'g': 'gmodern'
}
})
result_set = client.submit(message)
assert len(result_set.all().result()) == 6
##
t = g.with_("x", "test").with_("y", True).V()
message = RequestMessage('traversal', 'bytecode', {
'gremlin': t.bytecode,
'aliases': {
'g': 'gmodern'
}
})
result_set = client.submit(message)
assert len(result_set.all().result()) == 6
def test_iteration(self, remote_connection):
statics.load_statics(globals())
assert "remoteconnection[ws://localhost:45940/gremlin,gmodern]" == str(remote_connection)
g = Graph().traversal().withRemote(remote_connection)
t = g.V().count()
assert t.hasNext()
assert t.hasNext()
assert t.hasNext()
assert t.hasNext()
assert t.hasNext()
assert 6 == t.next()
assert not(t.hasNext())
assert not(t.hasNext())
assert not(t.hasNext())
assert not(t.hasNext())
assert not(t.hasNext())
t = g.V().has('name', P.within('marko', 'peter')).values('name').order()
assert "marko" == t.next()
assert t.hasNext()
assert t.hasNext()
assert t.hasNext()
assert t.hasNext()
assert t.hasNext()
assert "peter" == t.next()
assert not(t.hasNext())
assert not(t.hasNext())
assert not(t.hasNext())
assert not(t.hasNext())
assert not(t.hasNext())
try:
t.next()
assert False
except StopIteration:
assert True
def instance(cls):
if cls._instances['graph'] is None:
logger.info("BayesianGraph instance creating...")
graph = Graph()
g = graph.traversal().withRemote(
DriverRemoteConnection(config.GREMLIN_SERVER_URL_WEBSOCKET,
'g'))
# populate schema if not already done
if not cls.is_index_created():
logger.info(
"Index is not created as yet, checking schema creation")
if not cls.is_schema_defined():
logger.info("Schema is not yet created, creating now...")
cls.populate_schema()
## double check
schema_definition_success = cls.is_schema_defined()
logger.info("Double check: schema_definition_success %s" %
schema_definition_success)
if not schema_definition_success:
raise RuntimeError("Failed to setup graph schema")
cls._instances['graph'] = g
return cls._instances['graph']
def test_text_regex(self):
self.container.start()
docker_ip = Popen(["docker-machine", "ip"],
stdout=PIPE).communicate()[0]
docker_ip = docker_ip.strip().decode("utf-8")
client = JanusGraphClient()
client = client.connect(
host=str(docker_ip),
port="8182",
traversal_source="gods_traversal").get_connection()
g = Graph().traversal().withRemote(client)
truth = {"s.{2}": 2, "shouldNotBeFound": 0}
for k, v in truth.items():
count = g.V().has("name", Text.textRegex(k)).count().next()
self.assertEqual(count, v)
client.close()
self.container.stop()
def run_queries(cluster):
cluster_url = 'ws://localhost:%s/gremlin' % cluster['Port']
# test Client API
print('Connecting to Neptune Graph DB cluster URL: %s' % cluster_url)
graph_client = gremlin_client.Client(cluster_url, 'g')
values = '[1,2,3,4]'
print('Submitting values: %s' % values)
result_set = graph_client.submit(values)
future_results = result_set.all()
results = future_results.result()
print('Received values from cluster: %s' % results)
assert results == [1, 2, 3, 4]
future_result_set = graph_client.submitAsync('[1,2,3,4]')
result_set = future_result_set.result()
result = result_set.one()
assert result == [1, 2, 3, 4]
assert result_set.done.done()
graph_client.close()
# test DriverRemoteConnection API
graph = Graph()
conn = DriverRemoteConnection(cluster_url, 'g')
g = graph.traversal().withRemote(conn)
vertices_before = g.V().toList()
print('Existing vertices in the graph: %s' % vertices_before)
print('Adding new vertices "v1" and "v2" to the graph')
g.addV().property('id', 'v1').property('name', 'Vertex 1').next()
g.addV().property('id', 'v2').property('name', 'Vertex 2').next()
vertices_after = g.V().toList()
print('New list of vertices in the graph: %s' % vertices_after)
result = set(vertices_after) - set(vertices_before)
assert len(result) == 2
conn.close()
def test_clone_traversal(self):
g = traversal().withGraph(Graph())
original = g.V().out("created")
clone = original.clone().out("knows")
cloneClone = clone.clone().out("created")
assert 2 == len(original.bytecode.step_instructions)
assert 3 == len(clone.bytecode.step_instructions)
assert 4 == len(cloneClone.bytecode.step_instructions)
original.has("person", "name", "marko")
clone.V().out()
assert 3 == len(original.bytecode.step_instructions)
assert 5 == len(clone.bytecode.step_instructions)
assert 4 == len(cloneClone.bytecode.step_instructions)
def test_side_effects(self, remote_connection_v2):
statics.load_statics(globals())
#
g = Graph().traversal().withRemote(remote_connection_v2)
###
t = g.V().hasLabel("project").name.iterate()
assert 0 == len(t.side_effects.keys())
with pytest.raises(Exception):
m = t.side_effects["m"]
###
t = g.V().out("created").groupCount("m").by("name")
results = t.toSet()
assert 2 == len(results)
assert Vertex(3) in results
assert Vertex(5) in results
assert 1 == len(t.side_effects.keys())
assert "m" in t.side_effects.keys()
m = t.side_effects["m"]
assert isinstance(m, dict)
assert 2 == len(m)
assert 3 == m["lop"]
assert 1 == m["ripple"]
assert isinstance(m["lop"], long)
assert isinstance(m["ripple"], long)
##
t = g.V().out("created").groupCount("m").by("name").name.aggregate("n")
results = t.toSet()
assert 2 == len(results)
assert "lop" in results
assert "ripple" in results
assert 2 == len(t.side_effects.keys())
assert "m" in t.side_effects.keys()
assert "n" in t.side_effects.keys()
n = t.side_effects.get("n")
assert isinstance(n, dict)
assert 2 == len(n)
assert "lop" in n.keys()
assert "ripple" in n.keys()
assert 3 == n["lop"]
assert 1 == n["ripple"]
t = g.withSideEffect('m', 32).V().map(lambda: "x: x.sideEffects('m')")
results = t.toSet()
assert 1 == len(results)
assert 32 == list(results)[0]
assert 32 == t.side_effects['m']
assert 1 == len(t.side_effects.keys())
with pytest.raises(Exception):
x = t.side_effects["x"]
def test_big_result_set_secure(authenticated_client):
g = Graph().traversal()
t = g.inject(1).repeat(__.addV('person').property('name', __.loops())).times(20000).count()
message = RequestMessage('traversal', 'bytecode', {'gremlin': t.bytecode, 'aliases': {'g': 'g'}})
result_set = authenticated_client.submit(message)
results = []
for result in result_set:
results += result
assert len(results) == 1
t = g.V().limit(10)
message = RequestMessage('traversal', 'bytecode', {'gremlin': t.bytecode, 'aliases': {'g': 'g'}})
result_set = authenticated_client.submit(message)
results = []
for result in result_set:
results += result
assert len(results) == 10
t = g.V().limit(100)
message = RequestMessage('traversal', 'bytecode', {'gremlin': t.bytecode, 'aliases': {'g': 'g'}})
result_set = authenticated_client.submit(message)
results = []
for result in result_set:
results += result
assert len(results) == 100
t = g.V().limit(1000)
message = RequestMessage('traversal', 'bytecode', {'gremlin': t.bytecode, 'aliases': {'g': 'g'}})
result_set = authenticated_client.submit(message)
results = []
for result in result_set:
results += result
assert len(results) == 1000
t = g.V().limit(10000)
message = RequestMessage('traversal', 'bytecode', {'gremlin': t.bytecode, 'aliases': {'g': 'g'}})
result_set = authenticated_client.submit(message)
results = []
for result in result_set:
results += result
assert len(results) == 10000
def test_multi_thread_pool(client):
g = Graph().traversal()
traversals = [g.V(), g.V().count(), g.E(), g.E().count()]
results = [[] for _ in traversals]
# Use a condition variable to synchronise a group of threads, which should also inject some
# non-determinism into the run-time execution order
condition = threading.Condition()
def thread_run(tr, result_list):
message = RequestMessage('traversal', 'bytecode', {
'gremlin': tr.bytecode,
'aliases': {
'g': 'gmodern'
}
})
with condition:
condition.wait(5)
result_set = client.submit(message)
for result in result_set:
result_list.append(result)
threads = []
for i in range(len(results)):
thread = threading.Thread(target=thread_run,
args=(traversals[i], results[i]),
name="test_multi_thread_pool_%d" % i)
thread.daemon = True
threads.append(thread)
thread.start()
with condition:
condition.notify_all()
for t in threads:
t.join(5)
assert len(results[0][0]) == 6
assert results[1][0][0].object == 6
assert len(results[2][0]) == 6
assert results[3][0][0].object == 6
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
'''
from __future__ import print_function
from gremlin_python import statics
from gremlin_python.structure.graph import Graph
from gremlin_python.process.graph_traversal import __
from gremlin_python.process.strategies import *
from gremlin_python.driver.driver_remote_connection import \
DriverRemoteConnection
from gremlin_python.process.traversal import Order
from gremlin_python.process.traversal import P
statics.load_statics(globals())
graph = Graph()
g = graph.traversal().withRemote(DriverRemoteConnection(
'ws://mydb.cluster.us-east-1.neptune.amazonaws.com:8182/gremlin','g'))
#Add a vertex for a customer
c1 = g.addV('CUSTOMER').property('name', 'Bradley Russo').\
property('address', '486, 6221 Et St.,Barnstaple').\
property('country', 'Ukraine').\
property('zipcode', '10903').next()
#Add a vertex for a customer
c2 = g.addV('CUSTOMER').property('name', 'Jarrod Nieves').\
property('address', '198-550 At, Rd.,Hines Creek').\
property('country', 'Greece').\
property('zipcode', '20587').next()
# -------------------------------------------------------------------
# FUNCTION DELETE EVERYTHING
# -------------------------------------------------------------------
def delete_everything(traversal: GraphTraversalSource):
return traversal.V().drop().toList()
def get_property_pivot(data, prop):
customers = []
for vmx in data:
customers.append(vmx[prop])
return list(set(customers))
g = Graph().traversal().withRemote(
DriverRemoteConnection('ws://169.46.9.99:32344/gremlin',
'dataplatform_dev'))
parsed_data = []
with open('assets.csv') as f:
parsed_data = [{k: v
for k, v in row.items()}
for row in csv.DictReader(f, skipinitialspace=True)]
# for v in parsed_data:
# print(add_vertex(v, g, "VirtualMachineConfigurationItem"))
#
#
# for v in get_property_pivot(parsed_data, 'customer_id')[0:10]:
#
"""
Neptune DB cluster setup: https://docs.aws.amazon.com/neptune/latest/userguide/get-started-create-cluster.html
Graph Viz: https://github.com/bricaud/graphexp
"""
from __future__ import print_function # Python 2/3 compatibility
from gremlin_python import statics
from gremlin_python.driver.driver_remote_connection import DriverRemoteConnection
from gremlin_python.structure.graph import Graph
from tornado import httpclient
statics.load_statics(globals())
graph = Graph()
# connect to Neptune
my_req = httpclient.HTTPRequest(
'wss://localhost:8182/gremlin',
# headers={"Authorization": "Token AZX ..."},
validate_cert=False)
remoteConn = DriverRemoteConnection(my_req, 'g')
g = graph.traversal().withRemote(remoteConn)
# get who Lauren (003A000001fOaGAIA0) knows
r = g.V().has('~id', "003A000001fOaGAIA0").out('knows').last_name.toList()
print("Lauren's contacts")
for i in r:
print(i)
def test_client_bytecode(client):
g = Graph().traversal()
t = g.V()
message = RequestMessage('traversal', 'bytecode', {'gremlin': t.bytecode, 'aliases': {'g': 'gmodern'}})
result_set = client.submit(message)
assert len(result_set.all().result()) == 6
def with_remote(self, remote_connection):
return self.with_graph(Graph()).withRemote(remote_connection)
def __create_lookup_v(remote):
g = Graph().traversal().withRemote(remote)
# hold a map of name/vertex for use in asserting results
return g.V().group().by('name').by(tail()).next()
def prepare_traversal_source(scenario):
# some tests create data - create a fresh remote to the empty graph and clear that graph prior to each test
remote = DriverRemoteConnection('ws://localhost:45940/gremlin', "ggraph", message_serializer=serializer.GraphSONSerializersV3d0())
scenario.context.remote_conn["empty"] = remote
g = Graph().traversal().withRemote(remote)
g.V().drop().iterate()
def test_traversals(self):
statics.load_statics(globals())
connection = DriverRemoteConnection('ws://localhost:8182/gremlin', 'g')
assert "remoteconnection[ws://localhost:8182/gremlin,g]" == str(
connection)
#
g = Graph().traversal().withRemote(connection)
#
assert 6L == g.V().count().toList()[0]
#
assert Vertex(1) == g.V(1).next()
assert 1 == g.V(1).id().next()
assert Traverser(Vertex(1)) == g.V(1).nextTraverser()
assert 1 == len(g.V(1).toList())
assert isinstance(g.V(1).toList(), list)
#
results = g.V().repeat(out()).times(2).name.toList()
assert 2 == len(results)
assert "lop" in results
assert "ripple" in results
#
assert 10 == g.V().repeat(both()).times(5)[0:10].count().next()
assert 1 == g.V().repeat(both()).times(5)[0].count().next()
assert 0 == g.V().repeat(both()).times(5)[0:0].count().next()
assert 4 == g.V()[2:].count().next()
assert 2 == g.V()[:2].count().next()
# todo: need a traversal metrics deserializer
g.V().out().profile().next()
connection.close()
from gremlin_python.structure.graph import Graph
# creating the graph
graph = Graph()
# creating the traversal
g = graph.traversal()
# adding the vertex
g.addV("employee").property("name", "rico").property('position','data guy')
# running the query
my_stuff = g.V().has('name','rico').values()
# unfortunatelly, the result is not the same as the one gremlin-console provides...
# ??? how could we get simmilar result using python?
from gremlin_python import statics
from gremlin_python.structure.graph import Graph
from gremlin_python.process.graph_traversal import __
from gremlin_python.process.strategies import *
from gremlin_python.driver.driver_remote_connection import DriverRemoteConnection
graph = Graph()
g = graph.traversal().withRemote(
DriverRemoteConnection('ws://hdprd1-r01-edge-01:8182/gremlin', 'g'))
#g.addV("John")
g.addV('person').property('name', 'me').next()
#print(g.V().toList())
#g.addV('person').property('name', 'you').next()
#print(g.V().toList())
#print(g.V().toList())
from __future__ import print_function # Python 2/3 compatibility
from gremlin_python import statics
from gremlin_python.structure.graph import Graph
from gremlin_python.process.graph_traversal import __
from gremlin_python.process.strategies import *
from gremlin_python.driver.driver_remote_connection import DriverRemoteConnection
#initializing the graph object
graph = Graph()
#creating connection with the remote
remoteConn = DriverRemoteConnection('wss://<endpoint>:8182/gremlin','g')
g = graph.traversal().withRemote(DriverRemoteConnection('wss://<endpoint>:8182/gremlin','g'))
#clearing out all the vertices to start fresh
g.V().drop().iterate()
#Adding some vertices (nodes)
gerald = g.addV('person').property('age','81').property('first_name','Gerald').property('stays_in','Portland').next()
edith = g.addV('person').property('age','78').property('first_name','Edith').property('stays_in','Portland').next()
peter = g.addV('person').property('age','52').property('first_name','Shane').property('stays_in','Seattle').next()
mary = g.addV('person').property('age','50').property('first_name','Mary').property('stays_in','Seattle').next()
betty = g.addV('person').property('age','19').property('first_name','Betty').property('stays_in','Chicago').next()
#Adding relationships (edges)
edge = g.V().has('first_name', 'Gerald').addE('husband_of').to(g.V().has('first_name', 'Edith')).property('married_since','1947').next()
edge = g.V().has('first_name', 'Edith').addE('wife_of').to(g.V().has('first_name', 'Gerald')).property('married_since','1947').next()
edge = g.V().has('first_name', 'Shane').addE('son_of').to(g.V().has('first_name', 'Gerald')).property('known_since','1964').next()
edge = g.V().has('first_name', 'Gerald').addE('father_of').to(g.V().has('first_name', 'Shane')).property('known_since','1964').next()
edge = g.V().has('first_name', 'Shane').addE('son_of').to(g.V().has('first_name', 'Edith')).property('known_since','1964').next()
"""
Talking to JanusGraph from Python.
http://tinkerpop.apache.org/docs/current/reference/#gremlin-python
"""
from gremlin_python.structure.graph import Graph
from gremlin_python.driver.driver_remote_connection import DriverRemoteConnection
graph = Graph()
g = graph.traversal().withRemote(
DriverRemoteConnection('ws://localhost:8182/gremlin', 'g'))
vertices = g.V().toList()
print(vertices)
# Configure the environment so the Python Console can connect to # a Gremlin Server using gremlin-python and a web socket connection. # # To use this script start Python using: # python3 -i bootstrap-console.py from gremlin_python import statics from gremlin_python.structure.graph import Graph from gremlin_python.process.graph_traversal import __ from gremlin_python.process.strategies import * from gremlin_python.driver.driver_remote_connection import DriverRemoteConnection from gremlin_python.process.traversal import * import os # This script assumes that two environment variables have been defined # containing the DNS name of the Gremlin Server and the port it is listening on. # If the environment variables are not found, defaults will be used. skey = "GREMLIN_SERVER_NAME" pkey = "GREMLIN_SERVER_PORT" server = os.environ[skey] if skey in os.environ else 'localhost' port = os.environ[pkey] if pkey in os.environ else '8182' endpoint = 'ws://' + server + ':' + port + '/gremlin' print(endpoint) graph = Graph() connection = DriverRemoteConnection(endpoint, 'g') g = graph.traversal().withRemote(connection)
from gremlin_python.structure.graph import Graph
from gremlin_python.process.graph_traversal import __
from gremlin_python.process.strategies import *
from gremlin_python.process.traversal import T
from gremlin_python.process.traversal import Order
from gremlin_python.process.traversal import Cardinality
from gremlin_python.process.traversal import Column
from gremlin_python.process.traversal import Direction
from gremlin_python.process.traversal import Operator
from gremlin_python.process.traversal import P
from gremlin_python.process.traversal import Pop
from gremlin_python.process.traversal import Scope
from gremlin_python.process.traversal import Barrier
from gremlin_python.driver.driver_remote_connection import DriverRemoteConnection
statics.load_statics(globals())
graph = Graph()
class HWGremlinConnection:
def __init__(self, ip_address='localhost', port='8182'):
"""
Initialize a connection object to the gremlin server
"""
# 8182 is the default gremlin Server port
if type(port) != type(''):
port = str(port)
self.g = graph.traversal().withRemote(
DriverRemoteConnection(
'ws://' + ip_address + ':' + port + '/gremlin', 'g'))
def get_traversal_object(self):
from __future__ import print_function # Python 2/3 compatibility
from gremlin_python import statics
from gremlin_python.structure.graph import Graph
from gremlin_python.process.graph_traversal import __
from gremlin_python.process.strategies import *
from gremlin_python.driver.driver_remote_connection import DriverRemoteConnection
graph = Graph()
remoteConn = DriverRemoteConnection('wss://vpce-0e8ab31c183768604:8182/gremlin','g')
g = graph.traversal().withRemote(DriverRemoteConnection(remoteConn))
#print(g.V().limit(2).toList())
remoteConn.close()
def images_testing(inst=0, objd=True, lim=0, train=False, testing=False):
ret = {}
try:
# turn the punctuation into a list
punc = []
punc.extend(punctuation)
# getting constants from the JSON config file
src = cfg["instances"][inst]["src"]["index"]
typ = cfg["instances"][inst]["src"]["types"]["ocri"]
imgs = cfg["instances"][inst]["sources"][src][typ]["connection"][
"files"]
typ = cfg["instances"][inst]["src"]["types"]["pill"]
sel = cfg["instances"][inst]["sources"][src][typ]["connection"]["sel"]
typ = cfg["instances"][inst]["src"]["types"]["glove"]
gfl = cfg["instances"][inst]["sources"][src][typ]["connection"]["file"]
# get the ocr data ... using the real way to get the ocr data here
#
# first parameter is a list of images
# second parameter is an integer instance of this code base that is being run
# third parameter is a boolean value indicating whether (or not) we are testing
cdat = images(imgs, inst, objd, testing)
# perform the query against the NIH database with limit = 100 rows
sg = sodaget(inst, cdat, objd, lim, train)
# if we are training, then go ahead and get the wikipedia pages
# for the returns using the script medication in a post
if train:
# image keys and values
files = list(sg.keys())
vals = list(sg.values())
# number of clusters
nc = len(files)
# for each image file, get all wikipedia files using the scripts returned
rdat = None
inp = []
out = []
ent = []
cent = []
kimpr = {}
for fl in files:
if "error" not in list(sg[fl].keys()) and \
sel["splimprint"] in list(sg[fl].keys()) and \
(not type(sg[fl][sel["rxstring"]]) == type(None)):
imprs = []
impr = []
# the scripts
if type(sg[fl][sel["rxstring"]]) in [
type([]), type(np.asarray([]))
]:
for i in range(0, len(sg[fl][sel["rxstring"]])):
scrs = sg[fl][sel["rxstring"]][i].translate(
str.maketrans('', '',
punctuation)).lower().split()
# everything that will be added to the glove data set
simpr = "|".join(
p for p in punc
if p in sg[fl][sel["splimprint"]][i])
if not (len(simpr) == 0):
imprs = re.split(
simpr,
sg[fl][sel["splimprint"]][i].lower())
impr = "".join(imprs)
kimpr[impr] = sg[fl][sel["splimprint"]][i]
else:
scrs = sg[fl][sel["rxstring"]].translate(
str.maketrans('', '',
punctuation)).lower().split()
# everything that will be added to the glove data set
simpr = "|".join(p for p in punc
if p in sg[fl][sel["splimprint"]])
if not (len(simpr) == 0):
imprs = re.split(simpr,
sg[fl][sel["splimprint"]].lower())
impr = "".join(imprs)
kimpr[impr] = sg[fl][sel["splimprint"]]
ents = list(
np.append(
imprs,
np.append(
impr,
data.unique(
np.append(cdat[fl][const.IMG], scrs)))))
ent.extend(ents)
# each ents return is an array of entity arrays; search wikipedia for the entity
# tie each of the returns from the OCR imprint extraction to the entities in the script string
rdat = extendglove(ents,
rdat if not (rdat == None) else gfl[0])
# grab the wikipedia data
wikis = cognitive(const.WIK, scrs, inst, objd, testing)
# add the wikipedia data to the extended glove data set
rdat = extendglove(wikis,
rdat if not (rdat == None) else gfl[0])
# limit the data and revert the keys back to having the original imprints when possible
rdat = [(kimpr[k] if k in list(kimpr.keys()) else k,
list(np.asarray(v)[range(0, min(len(kimpr), len(v)))]))
for k, v in list(rdat.items()) if k in kimpr]
# write the extended glove data to a file for later recall
with open(gfl[1], "w+") as f:
for k, v in rdat:
f.write(str(k))
for i in range(0, len(v)):
f.write(" %lf" % v[i])
f.write("\n")
f.close()
# instantiate a JanusGraph object
graph = Graph()
# connection to the remote server
conn = DriverRemoteConnection(url_kg(inst), 'g')
# get the remote graph traversal
g = graph.traversal().withRemote(conn)
# write the glove output to the knowledge graph
#
# keys and values in the GloVe dataset
keys = list(list(rdat)[i][0] for i in range(0, len(rdat)))
vals = list(list(rdat)[i][1] for i in range(0, len(rdat)))
# use an explicit dict to make sure that the order is preserved
coln = [(keys[i], i) for i in range(0, len(keys))]
# create the data for the sample knowledge graph (only one brain)
perms = permutes(len(keys))
kgdat = create_kg(inst, vals, 2, perms)
# populate the knowledge graphs with the data
k1 = kg(const.V, inst, coln, kgdat, g, True, testing)
# see the extended glove data
ret = rdat
else:
# get the glove data
gdat = {}
fls = {"pills": gfl[1], "imprs": gfl[2]}
for key in fls:
with open(fls[key]) as f:
gdat[key] = [(list(wvec(line).keys())[0],
list(wvec(line).values())[0])
for line in f.readlines()]
f.close()
# keys and values for pills and imprints glove data
gpk = list(dict(gdat["pills"]).keys())
gpv = list(dict(gdat["pills"]).values())
gik = list(dict(gdat["imprs"]).keys())
giv = list(dict(gdat["imprs"]).values())
# for each file to be predicted
# get its imprints from cdat to be used to match the keys in gdat["imprs"].keys()
# get the row of data from gdat["imprs"].values() for inputs to the model
# then take the location of the maximum of the return to get the cluster
# take the key from pills that has row number matching the cluster number
ret = {}
for fl in cdat:
if "error" not in list(sg[fl].keys()) and \
sel["splimprint"] in list(sg[fl].keys()) and \
(not type(sg[fl][sel["rxstring"]]) == type(None)):
# start the return for this file
ret[fl] = {sel[key]: None for key in list(sel.keys())}
# if nothing else, we will return the most frequently appearing data
if not (type(sg[fl][sel["rxstring"]]) == type("")):
# string column of the imprints in the image
sret = list(sg[fl][sel["rxstring"]])
# most frequently appearing response from the DB
gm = glovemost(sret)
# row index of the most frequently appearing imprint
row = sret.index(gm)
# row of data corresponding to the most frequently appearing imprint
for key in sel:
if (type(sg[fl][sel[key]]) == type("")):
ret[fl][sel[key]] = sg[fl][sel[key]]
else:
ret[fl][sel[key]] = sg[fl][sel[key]][row]
# now we will start to see if we can make a prediction
#
# for each file predicted
cont = True
for i in cdat[fl][const.IMG]:
if cont:
# transformation of this imprint to match how it would have been stored
spl = "|".join(p for p in punc if p in i)
impr = i.lower()
if not (len(spl) == 0):
impr = re.split(spl, impr)
impr = "".join(impr)
# this could generate a key error if the imprint is not in the glove data set
# so we will just check gdat["imprs"].keys() first then go forward
for key in gik:
# need these next lines because some of the imprints are not stripped and lowered
nkey = key
nspl = "|".join(p for p in punc if p in nkey)
nimpr = nkey.lower()
if not (len(nspl) == 0):
nimpr = re.split(nspl, nimpr)
nimpr = "".join(nimpr)
# if this (possibly) partial imprint matches part
# of a transformed full imprint taken from the current imprint
if impr in nimpr:
# taking the constants just obtained, we can use the key to get the right
# model and make a prediction about the right cluster for this key that
# corresponds to an imprint ... if this imprint is noisy, we can take an
# imprint in the same cluster and get all parameters corresponding to that
# imprint by querying the knowledge graph for the other parameters in the
# the knowledge graph using "sel" from the JSON config file
#
# no need for a cluster return from thought ... simply get all columns
# starting with the one matching the key (use pop to get it first)
#
# read the data file to get the ordering, as the data will have been written
# to the file in the same order as the constants should appear
# then place the one matching key first and order all others in their original order
# and pass this as coln to "thought" ... make thought return the values to be
# matched with the values that were stored during training to get the key
#
# keys in gik are in the same ordering as when they were written during training
# so there is no need to try to figure out the ordering before constructing coln for thought
perms = permutes(len(gik))
pred = {}
for perm in perms:
if key in np.asarray(gik)[perm]:
coln = [(gik[k], k) for k in perm]
# call thought to get the data values for comparison
pred = thought(inst, coln, preds=0)
if not (len(pred) == 0
or "error" in pred):
break
# string column of the imprints in the image
sret = [
a.translate(
str.maketrans(
'', '', punctuation)).lower()
for a in list(sg[fl][
sel["splimprint"]])
]
# use GloVe to calculate the most frequently appearing imprint
gm = glovemost(sret)
# either we didn't get anything so that pred = {}
# or we got something that might be {"error":"error string"}
if not (len(pred) == 0 or "error" in pred):
if pred["pred"] in sret:
# row index of the most frequently appearing imprint
row = sret.index(pred["pred"])
else:
# use glove to find make a prediction using the sret data
row = sret.index(gm)
else:
# use glove to find make a prediction using the sret data
row = sret.index(gm)
# row of data corresponding to the most frequently appearing imprint
for k in sel:
if (type(sg[fl][sel[k]]) == type("")):
if sg[fl][sel["splshape_text"]] == ret[fl][sel["splshape_text"]] and \
sg[fl][sel["splcolor_text"]] == ret[fl][sel["splcolor_text"]]:
ret[fl][sel[k]] = sg[fl][
sel[k]]
else:
if sg[fl][sel["splshape_text"]][row] == ret[fl][sel["splshape_text"]] and \
sg[fl][sel["splcolor_text"]][row] == ret[fl][sel["splcolor_text"]]:
ret[fl][sel[k]] = sg[fl][
sel[k]][row]
# no need to go further just break
cont = False
break
else:
break
else:
ret[fl] = sg[fl]
# draw the image with the predicted medication
if not ("error" in ret[fl]):
if not (ret[fl][sel["rxstring"]] == None):
img = Image.open(fl)
draw = ImageDraw.Draw(img)
draw.text((10, 10), ret[fl][sel["rxstring"]],
(0, 0, 0))
img.save(fl[0:fl.rfind(".")] + "_PRED" +
fl[fl.rfind("."):len(fl)])
img.show()
except Exception as err:
ret["error"] = str(err)
return ret
