def use_cluster_with_graph(num_nodes):
"""
This is a work around to account for the fact that spark nodes will conflict over master assignment
when started all at once.
"""
# Create the cluster but don't start it.
use_singledc(start=False, workloads=['graph', 'spark'])
# Start first node.
get_node(1).start(wait_for_binary_proto=True)
# Wait binary protocol port to open
wait_for_node_socket(get_node(1), 120)
# Wait for spark master to start up
spark_master_http = ("localhost", 7080)
common.check_socket_listening(spark_master_http, timeout=60)
tmp_cluster = Cluster(protocol_version=PROTOCOL_VERSION)
# Start up remaining nodes.
try:
session = tmp_cluster.connect()
statement = "ALTER KEYSPACE dse_leases WITH REPLICATION = {'class': 'NetworkTopologyStrategy', 'dc1': '%d'}" % (
num_nodes)
session.execute(statement)
finally:
tmp_cluster.shutdown()
for i in range(1, num_nodes + 1):
if i is not 1:
node = get_node(i)
node.start(wait_for_binary_proto=True)
wait_for_node_socket(node, 120)
# Wait for workers to show up as Alive on master
wait_for_spark_workers(3, 120)
def use_cluster_with_graph(num_nodes):
"""
This is a work around to account for the fact that spark nodes will conflict over master assignment
when started all at once.
"""
# Create the cluster but don't start it.
use_singledc(start=False, workloads=['graph', 'spark'])
# Start first node.
get_node(1).start(wait_for_binary_proto=True)
# Wait binary protocol port to open
wait_for_node_socket(get_node(1), 120)
# Wait for spark master to start up
spark_master_http = ("localhost", 7080)
common.check_socket_listening(spark_master_http, timeout=60)
tmp_cluster = Cluster(protocol_version=PROTOCOL_VERSION)
# Start up remaining nodes.
try:
session = tmp_cluster.connect()
statement = "ALTER KEYSPACE dse_leases WITH REPLICATION = {'class': 'NetworkTopologyStrategy', 'dc1': '%d'}" % (num_nodes)
session.execute(statement)
finally:
tmp_cluster.shutdown()
for i in range(1, num_nodes+1):
if i is not 1:
node = get_node(i)
node.start(wait_for_binary_proto=True)
wait_for_node_socket(node, 120)
# Wait for workers to show up as Alive on master
wait_for_spark_workers(3, 120)
def use_singledc_wth_graph(start=True):
use_singledc(start=start, workloads=['graph'])
def use_singledc_wth_graph(start=True):
use_singledc(start=start, workloads=['graph'])
