def __init__(self, session):
self.session = session
self.graph = DseGraph.traversal_source(
session=self.session,
graph_name='killrvideo_video_recommendations')
logging.debug('Graph traversal source: ' + str(self.graph) + ' verts' +
str(self.graph.V()))
self.suggested_videos_consumer = SuggestedVideosConsumer(self)
# At the time of this blog post, dse_graph only supports gremlinpython version 3.2.x
# This script was tested using gremlinpython version 3.2.6
from dse.cluster import Cluster, GraphExecutionProfile, EXEC_PROFILE_GRAPH_DEFAULT, EXEC_PROFILE_GRAPH_SYSTEM_DEFAULT
from dse.graph import GraphOptions
from dse_graph import DseGraph
from gremlin_python.process.graph_traversal import __
from gremlin_python.structure.graph import Vertex
graph_name = 'modern'
ep_schema = GraphExecutionProfile(graph_options=GraphOptions(graph_name=graph_name))
ep = DseGraph.create_execution_profile(graph_name)
cluster = Cluster(execution_profiles={'schema': ep_schema, EXEC_PROFILE_GRAPH_DEFAULT: ep})
session = cluster.connect()
# Define schema
session.execute_graph("system.graph(name).create()", { 'name': graph_name }, execution_profile = EXEC_PROFILE_GRAPH_SYSTEM_DEFAULT)
session.execute_graph("schema.propertyKey('neighborhood').Bigint().create()", execution_profile = 'schema')
session.execute_graph("schema.propertyKey('name').Text().create()", execution_profile = 'schema')
session.execute_graph("schema.propertyKey('age').Bigint().create()", execution_profile = 'schema')
session.execute_graph("schema.propertyKey('weight').Float().create()", execution_profile = 'schema')
session.execute_graph("schema.vertexLabel('person').partitionKey('neighborhood').clusteringKey('name').properties('age').create()", execution_profile = 'schema')
session.execute_graph("schema.edgeLabel('knows').properties('weight').connection('person', 'person').create()", execution_profile = 'schema')
# Execute batch
batch = DseGraph.batch()
batch.add(__.addV('person').property('neighborhood', 0).property('name', 'bob').property('age', 23))
batch.add(__.addV('person').property('neighborhood', 0).property('name', 'alice').property('age', 21))
batch.add(__.addE('knows')
.from_(Vertex({ 'neighborhood': 0, 'name': 'bob', '~label' : 'person' }))
def serve():
dse_username = os.getenv('KILLRVIDEO_DSE_USERNAME')
dse_password = os.getenv('KILLRVIDEO_DSE_PASSWORD')
dse_contact_points = os.getenv('KILLRVIDEO_DSE_CONTACT_POINTS', 'dse').split(',')
service_port = os.getenv('KILLRVIDEO_SERVICE_PORT', '50101')
file = open('config.json', 'r')
config = json.load(file)
default_consistency_level = config['DEFAULT_CONSISTENCY_LEVEL']
# Initialize Cassandra Driver and Mapper
load_balancing_policy = TokenAwarePolicy(DCAwareRoundRobinPolicy())
profile = ExecutionProfile(consistency_level=ConsistencyLevel.name_to_value[default_consistency_level],
load_balancing_policy=load_balancing_policy)
graph_profile = DseGraph.create_execution_profile('killrvideo_video_recommendations')
auth_provider = None
if dse_username:
auth_provider = PlainTextAuthProvider(username=dse_username, password=dse_password)
# Wait for Cassandra (DSE) to be up
session = None
while not session:
try:
session = Cluster(contact_points=dse_contact_points,
execution_profiles={EXEC_PROFILE_DEFAULT: profile, EXEC_PROFILE_GRAPH_DEFAULT: graph_profile},
auth_provider = auth_provider).connect("killrvideo")
except (NoHostAvailable):
logging.info('Waiting for Cassandra (DSE) to be available')
time.sleep(10)
# Additional retry loop to check if dummy keyspace exists
while True:
logging.info('Checking for schema to be created...')
result = session.execute('SELECT keyspace_name FROM system_schema.keyspaces WHERE keyspace_name=\'kv_init_done\'')
if result.one(): # any result indicates keyspace has been created
break
time.sleep(10)
dse.cqlengine.connection.set_session(session)
# Initialize GRPC Server
grpc_server = grpc.server(futures.ThreadPoolExecutor(max_workers=10))
# Initialize Services (GRPC servicers with reference to GRPC Server and appropriate service reference
CommentsServiceServicer(grpc_server, CommentsService(session=session))
RatingsServiceServicer(grpc_server, RatingsService())
SearchServiceServicer(grpc_server, SearchService(session=session))
StatisticsServiceServicer(grpc_server, StatisticsService())
SuggestedVideosServiceServicer(grpc_server, SuggestedVideosService(session=session))
#UploadsServiceServicer(grpc_server, UploadsService())
UserManagementServiceServicer(grpc_server, UserManagementService())
VideoCatalogServiceServicer(grpc_server, VideoCatalogService(session=session))
# Start GRPC Server
grpc_server.add_insecure_port('[::]:' + service_port)
grpc_server.start()
# Keep application alive
try:
while True:
time.sleep(_ONE_DAY_IN_SECONDS)
except KeyboardInterrupt:
grpc_server.stop(0)
def execute_traversal(self, traversal):
query = DseGraph.query_from_traversal(traversal)
#Use an ep that is configured with the correct row factory, and bytecode-json language flat set
result_set = self.session.execute_graph(query, execution_profile=self.ep)
return list(result_set)
def fetch_traversal_source(self):
return DseGraph().traversal_source(self.session, self.graph_name)
def setUpClass(self):
super(ExplicitSearchTest, self).setUpClass()
self.ep = DseGraph().create_execution_profile(self.graph_name)
self.cluster.add_execution_profile(self.graph_name, self.ep)
generate_address_book_graph(self.session, 0)
time.sleep(20)
def fetch_traversal_source(self):
return DseGraph().traversal_source(self.session, self.graph_name, execution_profile=self.ep)
def setUp(self):
super(ExplicitExecutionTest, self).setUp()
self.ep = DseGraph().create_execution_profile(self.graph_name)
self.cluster.add_execution_profile(self.graph_name, self.ep)
def __init__(self, session):
self.session = session
self.graph = DseGraph.traversal_source(session=self.session, graph_name='killrvideo_video_recommendations')
self.suggested_videos_consumer = SuggestedVideosConsumer(self)
def serve():
file = open('config.json', 'r')
config = json.load(file)
service_port = config['SERVICE_PORT']
service_host = config['SERVICE_HOST']
etcd_port = config['ETCD_PORT']
contact_points = config['CONTACT_POINTS']
default_consistency_level = config['DEFAULT_CONSISTENCY_LEVEL']
service_address = service_host + ":" + str(service_port)
etcd_client = etcd.Client(host=service_host, port=etcd_port)
# Wait for Cassandra (DSE) to be up, aka registered in etcd
while True:
try:
etcd_client.read('/killrvideo/services/cassandra')
break # if we get here, Cassandra is registered and should be available
except etcd.EtcdKeyNotFound:
logging.info('Waiting for Cassandra to be registered in etcd, sleeping 10s')
time.sleep(10)
# Initialize Cassandra Driver and Mapper
profile = ExecutionProfile(consistency_level =
ConsistencyLevel.name_to_value[default_consistency_level])
graph_profile = DseGraph.create_execution_profile('killrvideo_video_recommendations')
cluster = Cluster(contact_points=contact_points,
execution_profiles={EXEC_PROFILE_DEFAULT: profile, EXEC_PROFILE_GRAPH_DEFAULT: graph_profile})
session = cluster.connect("killrvideo")
dse.cqlengine.connection.set_session(session)
# Initialize GRPC Server
grpc_server = grpc.server(futures.ThreadPoolExecutor(max_workers=10))
# Initialize Services (GRPC servicers with reference to GRPC Server and appropriate service reference
CommentsServiceServicer(grpc_server, CommentsService(session=session))
RatingsServiceServicer(grpc_server, RatingsService())
SearchServiceServicer(grpc_server, SearchService(session=session))
StatisticsServiceServicer(grpc_server, StatisticsService())
SuggestedVideosServiceServicer(grpc_server, SuggestedVideosService(session=session))
#UploadsServiceServicer(grpc_server, UploadsService())
UserManagementServiceServicer(grpc_server, UserManagementService())
VideoCatalogServiceServicer(grpc_server, VideoCatalogService(session=session))
# Start GRPC Server
grpc_server.add_insecure_port('[::]:' + str(service_port))
grpc_server.start()
# Register Services with etcd
etcd_client.write('/killrvideo/services/CommentsService/killrvideo-python', service_address)
etcd_client.write('/killrvideo/services/RatingsService/killrvideo-python', service_address)
etcd_client.write('/killrvideo/services/SearchService/killrvideo-python', service_address)
etcd_client.write('/killrvideo/services/StatisticsService/killrvideo-python', service_address)
etcd_client.write('/killrvideo/services/SuggestedVideoService/killrvideo-python', service_address)
#etcd_client.write('/killrvideo/services/UploadsService/killrvideo-python', service_address)
etcd_client.write('/killrvideo/services/UserManagementService/killrvideo-python', service_address)
etcd_client.write('/killrvideo/services/VideoCatalogService/killrvideo-python', service_address)
# Keep application alive
try:
while True:
time.sleep(_ONE_DAY_IN_SECONDS)
except KeyboardInterrupt:
grpc_server.stop(0)