def start_event_aggregator(
address,
event_publisher_pull_address,
event_publisher_pub_addresses,
environment = None
):
log = logging.getLogger("start_event_aggregator")
server_dir = identify_program_dir(u"event_aggregator")
server_path = os.path.join(server_dir, "event_aggregator_main.py")
args = [
sys.executable,
server_path,
]
if environment is None:
environment = {
"PYTHONPATH" : os.environ["PYTHONPATH"],
"NIMBUSIO_LOG_DIR" : os.environ["NIMBUSIO_LOG_DIR"],
"NIMBUSIO_EVENT_AGGREGATOR_PUB_ADDRESS" : address,
"NIMBUSIO_EVENT_PUBLISHER_PULL_ADDRESS" : \
event_publisher_pull_address,
"NIMBUSIO_EVENT_PUBLISHER_PUB_ADDRESSES" : \
" ".join(event_publisher_pub_addresses),
}
log.info("starting %s %s" % (args, environment, ))
return subprocess.Popen(args, stderr=subprocess.PIPE, env=environment)
def start_retrieve_source(node_name, environment, profile):
log = logging.getLogger("start_retrieve_source %s profile = %r" % (
node_name, profile,
))
server_dir = identify_program_dir(u"retrieve_source")
server_path = os.path.join(server_dir, "retrieve_source_main.py")
args = [
_python3,
server_path,
]
if profile:
profile_path = os.path.join(
environment["NIMBUSIO_PROFILE_DIR"],
"retrieve_source_%s.pstats" % (node_name, )
)
args[1:1] = [
"-m",
"cProfile",
"-o",
profile_path
]
log.info("starting %s" % (args, ))
return subprocess.Popen(args, stderr=subprocess.PIPE, env=environment)
def start_space_accounting_server(
node_name,
address,
pipeline_address,
event_publisher_pull_address,
environment = None,
):
log = logging.getLogger("_start_space_accounting_server%s" % (node_name, ))
server_dir = identify_program_dir(u"space_accounting_server")
server_path = os.path.join(
server_dir, "space_accounting_server_main.py"
)
args = [
sys.executable,
server_path,
]
if environment is None:
environment = {
"PYTHONPATH" : os.environ["PYTHONPATH"],
"NIMBUSIO_LOG_DIR" : os.environ["NIMBUSIO_LOG_DIR"],
"NIMBUSIO_NODE_NAME" : node_name,
"NIMBUSIO_SPACE_ACCOUNTING_SERVER_ADDRESS" : address,
"NIMBUSIO_SPACE_ACCOUNTING_PIPELINE_ADDRESS" : pipeline_address,
"NIMBUSIO_CENTRAL_USER_PASSWORD" : "pork",
"NIMBUSIO_EVENT_PUBLISHER_PULL_ADDRESS" : \
event_publisher_pull_address,
}
log.info("starting %s %s" % (args, environment, ))
return subprocess.Popen(args, stderr=subprocess.PIPE, env=environment)
def _install_schema_and_test_data():
log = logging.getLogger("_install_schema")
database_name = _node_database_name(_node_name)
user_name = _node_database_user(_node_name)
sql_path = identify_program_dir("sql")
schema_path = os.path.join(sql_path, "nimbusio_node.sql")
env = {"PGPASSWORD" : _database_password};
args = ["/usr/bin/psql",
"-h", _database_host,
"-p", str(_database_port),
"-d", database_name,
"-U", user_name,
"-f", schema_path]
log.debug(args)
process = subprocess.Popen(args, env=env)
process.wait()
assert process.returncode == 0, process.returncode
test_data_path = os.path.join(sql_path, "test_gc.sql")
env = {"PGPASSWORD" : _database_password};
args = ["/usr/bin/psql",
"-h", _database_host,
"-p", str(_database_port),
"-d", database_name,
"-U", user_name,
"-f", test_data_path]
log.debug(args)
process = subprocess.Popen(args, env=env)
process.wait()
assert process.returncode == 0, process.returncode
def start_data_reader(
node_name,
address,
event_publisher_pull_address,
repository_path,
environment=None
):
log = logging.getLogger("start_data_reader_%s" % (node_name, ))
server_dir = identify_program_dir(u"data_reader")
server_path = os.path.join(server_dir, "data_reader_main.py")
args = [
sys.executable,
server_path,
]
if environment is None:
environment = {
"PYTHONPATH" : os.environ["PYTHONPATH"],
"NIMBUSIO_LOG_DIR" : os.environ["NIMBUSIO_LOG_DIR"],
"NIMBUSIO_NODE_NAME" : node_name,
"NIMBUSIO_DATA_READER_ADDRESS" : address,
"NIMBUSIO_REPOSITORY_PATH" : repository_path,
"NIMBUSIO_EVENT_PUBLISHER_PULL_ADDRESS" : \
event_publisher_pull_address,
"NIMBUSIO_NODE_USER_PASSWORD" : "pork",
}
log.info("starting %s %s" % (args, environment, ))
return subprocess.Popen(args, stderr=subprocess.PIPE, env=environment)
def _start_subprocesses(halt_event):
"""
start subprocesses
"""
log = logging.getLogger("start_subprocesses")
node_generators = list()
anti_entropy_dir = identify_program_dir("anti_entropy")
subprocess_path = os.path.join(anti_entropy_dir,
"cluster_repair",
"node_data_reader_subprocess.py")
for index, node_name in enumerate(_node_names):
if halt_event.is_set():
log.info("halt_event set: exiting")
return node_generators
log.info("starting subprocess {0}".format(node_name))
args = [sys.executable, subprocess_path, str(index) ]
process = subprocess.Popen(args,
bufsize=_read_buffer_size,
stdout=subprocess.PIPE)
assert process is not None
node_generators.append(_node_generator(halt_event, node_name, process))
return node_generators
def start_event_aggregator(address,
event_publisher_pull_address,
event_publisher_pub_addresses,
environment=None):
log = logging.getLogger("start_event_aggregator")
server_dir = identify_program_dir(u"event_aggregator")
server_path = os.path.join(server_dir, "event_aggregator_main.py")
args = [
sys.executable,
server_path,
]
if environment is None:
environment = {
"PYTHONPATH" : os.environ["PYTHONPATH"],
"NIMBUSIO_LOG_DIR" : os.environ["NIMBUSIO_LOG_DIR"],
"NIMBUSIO_EVENT_AGGREGATOR_PUB_ADDRESS" : address,
"NIMBUSIO_EVENT_PUBLISHER_PULL_ADDRESS" : \
event_publisher_pull_address,
"NIMBUSIO_EVENT_PUBLISHER_PUB_ADDRESSES" : \
" ".join(event_publisher_pub_addresses),
}
log.info("starting %s %s" % (
args,
environment,
))
return subprocess.Popen(args, stderr=subprocess.PIPE, env=environment)
def start_event_publisher(node_name,
pull_address,
pub_address,
environment=None):
log = logging.getLogger("start_event_publisher_%s" % (node_name, ))
server_dir = identify_program_dir(u"event_publisher")
server_path = os.path.join(server_dir, "event_publisher_main.py")
args = [
sys.executable,
server_path,
]
if environment is None:
environment = {
"PYTHONPATH": os.environ["PYTHONPATH"],
"NIMBUSIO_LOG_DIR": os.environ["NIMBUSIO_LOG_DIR"],
"NIMBUSIO_NODE_NAME": node_name,
"NIMBUSIO_EVENT_PUBLISHER_PULL_ADDRESS": pull_address,
"NIMBUSIO_EVENT_PUBLISHER_PUB_ADDRESS": pub_address,
}
log.info("starting %s %s" % (
args,
environment,
))
return subprocess.Popen(args, stderr=subprocess.PIPE, env=environment)
def start_performance_packager(node_name,
event_aggregator_pub_address,
environment=None):
log = logging.getLogger("start_performance_packager_%s" % (node_name, ))
server_dir = identify_program_dir(u"performance_packager")
server_path = os.path.join(server_dir, "performance_packager_main.py")
args = [
sys.executable,
server_path,
]
if environment is None:
environment = {
"PYTHONPATH" : os.environ["PYTHONPATH"],
"NIMBUSIO_LOG_DIR" : os.environ["NIMBUSIO_LOG_DIR"],
"NIMBUSIO_NODE_NAME" : node_name,
"NIMBUSIO_EVENT_AGGREGATOR_PUB_ADDRESS" : \
event_aggregator_pub_address,
}
log.info("starting %s %s" % (
args,
environment,
))
return subprocess.Popen(args, stderr=subprocess.PIPE, env=environment)
def start_space_accounting_server(
node_name,
address,
pipeline_address,
event_publisher_pull_address,
environment=None,
):
log = logging.getLogger("_start_space_accounting_server%s" % (node_name, ))
server_dir = identify_program_dir(u"space_accounting_server")
server_path = os.path.join(server_dir, "space_accounting_server_main.py")
args = [
sys.executable,
server_path,
]
if environment is None:
environment = {
"PYTHONPATH" : os.environ["PYTHONPATH"],
"NIMBUSIO_LOG_DIR" : os.environ["NIMBUSIO_LOG_DIR"],
"NIMBUSIO_NODE_NAME" : node_name,
"NIMBUSIO_SPACE_ACCOUNTING_SERVER_ADDRESS" : address,
"NIMBUSIO_SPACE_ACCOUNTING_PIPELINE_ADDRESS" : pipeline_address,
"NIMBUSIO_CENTRAL_USER_PASSWORD" : "pork",
"NIMBUSIO_EVENT_PUBLISHER_PULL_ADDRESS" : \
event_publisher_pull_address,
}
log.info("starting %s %s" % (
args,
environment,
))
return subprocess.Popen(args, stderr=subprocess.PIPE, env=environment)
def _start_subprocesses(halt_event):
"""
start subprocesses
"""
log = logging.getLogger("start_subprocesses")
node_generators = list()
anti_entropy_dir = identify_program_dir("anti_entropy")
subprocess_path = os.path.join(anti_entropy_dir, "cluster_repair",
"node_data_reader_subprocess.py")
for index, node_name in enumerate(_node_names):
if halt_event.is_set():
log.info("halt_event set: exiting")
return node_generators
log.info("starting subprocess {0}".format(node_name))
args = [sys.executable, subprocess_path, str(index)]
process = subprocess.Popen(args,
bufsize=_read_buffer_size,
stdout=subprocess.PIPE)
assert process is not None
node_generators.append(_node_generator(halt_event, node_name, process))
return node_generators
def start_data_reader(node_name,
address,
event_publisher_pull_address,
repository_path,
environment=None):
log = logging.getLogger("start_data_reader_%s" % (node_name, ))
server_dir = identify_program_dir(u"data_reader")
server_path = os.path.join(server_dir, "data_reader_main.py")
args = [
sys.executable,
server_path,
]
if environment is None:
environment = {
"PYTHONPATH" : os.environ["PYTHONPATH"],
"NIMBUSIO_LOG_DIR" : os.environ["NIMBUSIO_LOG_DIR"],
"NIMBUSIO_NODE_NAME" : node_name,
"NIMBUSIO_DATA_READER_ADDRESS" : address,
"NIMBUSIO_REPOSITORY_PATH" : repository_path,
"NIMBUSIO_EVENT_PUBLISHER_PULL_ADDRESS" : \
event_publisher_pull_address,
"NIMBUSIO_NODE_USER_PASSWORD" : "pork",
}
log.info("starting %s %s" % (
args,
environment,
))
return subprocess.Popen(args, stderr=subprocess.PIPE, env=environment)
def _start_pullers(halt_event, work_dir):
"""
start puller subprocesses
"""
log = logging.getLogger("start_pullers")
pullers = dict()
anti_entropy_dir = identify_program_dir("anti_entropy")
puller_path = os.path.join(anti_entropy_dir,
"cluster_inspector",
"segment_puller_subprocess.py")
for index, node_name in enumerate(_node_names):
if halt_event.is_set():
log.info("halt_event set: exiting")
return pullers
log.info("starting subprocess {0}".format(node_name))
args = [sys.executable, puller_path, work_dir, str(index) ]
process = subprocess.Popen(args, stderr=subprocess.PIPE)
assert process is not None
pullers[node_name] = process
return pullers
def start_data_writer(node_name, environment, profile):
log = logging.getLogger("start_data_writer %s profile=%r" % (
node_name, profile
))
server_dir = identify_program_dir(u"data_writer")
server_path = os.path.join(server_dir, "data_writer_main.py")
args = [
sys.executable,
server_path,
]
if profile:
profile_path = os.path.join(
environment["NIMBUSIO_PROFILE_DIR"],
"data_writer_%s.pstats" % (node_name, )
)
args[1:1] = [
"-m",
"cProfile",
"-o",
profile_path
]
log.info("starting %s" % (args, ))
return subprocess.Popen(args, stderr=subprocess.PIPE, env=environment)
def _launch_database_pool_worker(worker_number):
log = logging.getLogger("launch_database_pool_worker")
module_dir = identify_program_dir("retrieve_source")
module_path = os.path.join(module_dir, "database_pool_worker.py")
args = [sys.executable, module_path, str(worker_number), ]
log.info("starting {0}".format(args))
return subprocess.Popen(args, stderr=subprocess.PIPE)
def _launch_io_controller():
log = logging.getLogger("launch_io_controller")
module_dir = identify_program_dir("retrieve_source")
module_path = os.path.join(module_dir, "io_controller.py")
args = [sys.executable, module_path]
log.info("starting {0}".format(args))
return subprocess.Popen(args, stderr=subprocess.PIPE)
def _start_write_subprocess():
anti_entropy_dir = identify_program_dir("anti_entropy")
subprocess_path = os.path.join(anti_entropy_dir,
"cluster_repair",
"node_data_writer.py")
args = [sys.executable, subprocess_path, ]
process = subprocess.Popen(args,
stdin=subprocess.PIPE)
assert process is not None
return process
def _launch_io_controller():
log = logging.getLogger("launch_io_controller")
module_dir = identify_program_dir("retrieve_source")
module_path = os.path.join(module_dir, "io_controller.py")
args = [
sys.executable,
module_path,
]
log.info("starting {0}".format(args))
return subprocess.Popen(args)
def start_event_aggregator(environment):
log = logging.getLogger("start_event_aggregator")
server_dir = identify_program_dir(u"event_aggregator")
server_path = os.path.join(server_dir, "event_aggregator_main.py")
args = [
sys.executable,
server_path,
]
log.info("starting %s" % (args, ))
return subprocess.Popen(args, stderr=subprocess.PIPE, env=environment)
def _start_write_subprocess():
anti_entropy_dir = identify_program_dir("anti_entropy")
subprocess_path = os.path.join(anti_entropy_dir, "cluster_repair",
"node_data_writer.py")
args = [
sys.executable,
subprocess_path,
]
process = subprocess.Popen(args, stdin=subprocess.PIPE)
assert process is not None
return process
def _start_worker_process(worker_id, args, rep_socket_uri):
module_dir = identify_program_dir("handoff_client")
module_path = os.path.join(module_dir, "worker.py")
args = [sys.executable,
module_path,
worker_id,
args.host_name,
str(args.base_port),
args.node_name,
rep_socket_uri, ]
return subprocess.Popen(args, stderr=subprocess.PIPE)
def start_data_writer(
cluster_name,
node_name,
address,
anti_entropy_address,
event_publisher_pull_address,
event_aggregator_pub_address,
repository_path,
central_db_pw="pork",
central_db_port=5432,
node_db_pw="pork",
node_db_port=5432,
environment=None,
):
log = logging.getLogger("start_data_writer_%s" % (node_name, ))
args = list()
data_writer_type = os.environ["NIMBUSIO_DATA_WRITER"]
if data_writer_type == "python":
server_dir = identify_program_dir(u"data_writer")
server_path = os.path.join(server_dir, "data_writer_main.py")
args = [
sys.executable,
server_path,
]
else:
raise AttrubuteError("unknown NIMBUSIO_DATA_WRITER '%s'",
(data_writer_type, ))
if environment is None:
environment = {
"PYTHONPATH" : os.environ["PYTHONPATH"],
"NIMBUSIO_LOG_DIR" : os.environ["NIMBUSIO_LOG_DIR"],
"NIMBUSIO_CLUSTER_NAME" : cluster_name,
"NIMBUSIO_NODE_NAME" : node_name,
"NIMBUSIO_DATA_WRITER_ADDRESS" : address,
"NIMBUSIO_REPOSITORY_PATH" : repository_path,
"NIMBUSIO_EVENT_PUBLISHER_PULL_ADDRESS" : \
event_publisher_pull_address,
"NIMBUSIO_DATA_WRITER_ANTI_ENTROPY_ADDRESS" : \
anti_entropy_address,
"NIMBUSIO_CENTRAL_USER_PASSWORD" : central_db_pw,
"NIMBUSIO_NODE_USER_PASSWORD" : node_db_pw,
"NIMBUSIO_EVENT_AGGREGATOR_PUB_ADDRESS": event_aggregator_pub_address,
}
log.info("starting %s %s" % (
args,
environment,
))
return subprocess.Popen(args, stderr=subprocess.PIPE, env=environment)
def _launch_database_pool_worker(worker_number):
log = logging.getLogger("launch_database_pool_worker")
module_dir = identify_program_dir("retrieve_source")
module_path = os.path.join(module_dir, "database_pool_worker.py")
args = [
sys.executable,
module_path,
str(worker_number),
]
log.info("starting {0}".format(args))
return subprocess.Popen(args, stderr=subprocess.PIPE)
def _start_worker_process(worker_id, args, rep_socket_uri):
module_dir = identify_program_dir("handoff_client")
module_path = os.path.join(module_dir, "worker.py")
args = [
sys.executable,
module_path,
worker_id,
args.host_name,
str(args.base_port),
args.node_name,
rep_socket_uri,
]
return subprocess.Popen(args, stderr=subprocess.PIPE)
def start_performance_packager(node_name, environment):
log = logging.getLogger("start_performance_packager_%s" % (node_name, ))
server_dir = identify_program_dir(u"performance_packager")
server_path = os.path.join(
server_dir, "performance_packager_main.py"
)
args = [
sys.executable,
server_path,
]
log.info("starting %s" % (args, ))
return subprocess.Popen(args, stderr=subprocess.PIPE, env=environment)
def start_stats_subscriber(environment):
log = logging.getLogger("start_stats_subscriber")
server_dir = identify_program_dir(u"test")
server_path = os.path.join(
server_dir, "stats_subscriber.py"
)
args = [
sys.executable,
server_path,
]
log.info("starting %s" % (args, ))
return subprocess.Popen(args, stderr=subprocess.PIPE, env=environment)
def prepare_profile_graphs(profile_dir):
log = logging.getLogger("prepare_profile_graphs")
tools_dir = identify_program_dir("tools")
if tools_dir is None:
raise ProfileGraphError("unable to locate tools directory")
gprof2dot_path = os.path.join(tools_dir, "gprof2dot.py")
if not os.path.isfile(gprof2dot_path):
raise ProfileGraphError("unable to locate gprof2dot.py")
for file_name in os.listdir(profile_dir):
if not file_name.endswith(".pstats"):
continue
log.debug(file_name)
pstats_path = os.path.join(profile_dir, file_name)
png_path = "".join([pstats_path[:-len("pstats")], "png"])
gprof2dot_args = [
sys.executable, gprof2dot_path, "-f", "pstats", "-e", "0.1", "-n",
"0.1", pstats_path
]
dot_args = ["/usr/bin/dot", "-T", "png", "-o", png_path]
gprof2dot_process = subprocess.Popen(gprof2dot_args,
stdout=subprocess.PIPE,
bufsize=_buffer_size)
dot_process = subprocess.Popen(dot_args,
stdin=gprof2dot_process.stdout,
stdout=subprocess.PIPE,
bufsize=_buffer_size)
dot_process.communicate()
if gprof2dot_process.returncode != 0:
if gprof2dot_process.returncode is None:
gprof2dot_process.terminate()
else:
message = "gprof2dot process failed %s %s" % (
gprof2dot_process.returncode,
gprof2dot_process.stderr,
)
log.error(message)
raise ProfileGraphError(message)
if dot_process.returncode != 0:
message = "dot process failed %s %s" % (
dot_process.returncode,
dot_process.stderr,
)
log.error(message)
raise ProfileGraphError(message)
def start_web_public_reader(node_name, environment):
log = logging.getLogger("start_performance_packager_%s" % (node_name, ))
server_dir = identify_program_dir(u"web_public_reader")
server_path = os.path.join(
server_dir, "web_public_reader_main.py"
)
args = [
sys.executable,
server_path,
]
log.info("starting %s %s" % (args, environment, ))
#return subprocess.Popen(args, stderr=subprocess.PIPE, env=environment)
#the webserver writes enough stderr/out to fill up the pipe and make the
#subprocess block.
return subprocess.Popen(args, stderr=None, env=environment)
def start_web_public_reader(node_name, environment):
log = logging.getLogger("start_performance_packager_%s" % (node_name, ))
server_dir = identify_program_dir(u"web_public_reader")
server_path = os.path.join(server_dir, "web_public_reader_main.py")
args = [
sys.executable,
server_path,
]
log.info("starting %s %s" % (
args,
environment,
))
#return subprocess.Popen(args, stderr=subprocess.PIPE, env=environment)
#the webserver writes enough stderr/out to fill up the pipe and make the
#subprocess block.
return subprocess.Popen(args, stderr=None, env=environment)
def _launch_ping_process(ping_process_desc, node_name, ping_uri):
log = logging.getLogger("launch_ping_process")
module_dir = identify_program_dir(ping_process_desc.module_dir)
module_path = os.path.join(module_dir, ping_process_desc.file_name)
args = [sys.executable, module_path,
"-u", ping_uri,
"-r", _pull_socket_uri,
"-m", str(_pull_socket_hwm), ]
log.info("starting {0}".format(args))
process = subprocess.Popen(args, bufsize=4096, stderr=subprocess.PIPE)
return _ping_process(service_name=ping_process_desc.service_name,
node_name=node_name,
process=process,
reachable_state=None)
def _launch_ping_process(ping_process_desc, node_name, ping_uri):
log = logging.getLogger("launch_ping_process")
module_dir = identify_program_dir(ping_process_desc.module_dir)
module_path = os.path.join(module_dir, ping_process_desc.file_name)
args = [sys.executable, module_path,
"-u", ping_uri,
"-r", _pull_socket_uri,
"-m", str(_pull_socket_hwm), ]
log.info("starting {0}".format(args))
process = subprocess.Popen(args, bufsize=4096, stderr=subprocess.PIPE)
return _ping_process(service_name=ping_process_desc.service_name,
node_name=node_name,
process=process,
reachable_state=None)
def start_data_writer(
cluster_name,
node_name,
address,
anti_entropy_address,
event_publisher_pull_address,
event_aggregator_pub_address,
repository_path,
central_db_pw = "pork",
central_db_port = 5432,
node_db_pw = "pork",
node_db_port = 5432,
environment = None,
):
log = logging.getLogger("start_data_writer_%s" % (node_name, ))
args = list()
data_writer_type = os.environ["NIMBUSIO_DATA_WRITER"]
if data_writer_type == "python":
server_dir = identify_program_dir(u"data_writer")
server_path = os.path.join(server_dir, "data_writer_main.py")
args = [sys.executable, server_path, ]
else:
raise AttrubuteError("unknown NIMBUSIO_DATA_WRITER '%s'", (data_writer_type, ))
if environment is None:
environment = {
"PYTHONPATH" : os.environ["PYTHONPATH"],
"NIMBUSIO_LOG_DIR" : os.environ["NIMBUSIO_LOG_DIR"],
"NIMBUSIO_CLUSTER_NAME" : cluster_name,
"NIMBUSIO_NODE_NAME" : node_name,
"NIMBUSIO_DATA_WRITER_ADDRESS" : address,
"NIMBUSIO_REPOSITORY_PATH" : repository_path,
"NIMBUSIO_EVENT_PUBLISHER_PULL_ADDRESS" : \
event_publisher_pull_address,
"NIMBUSIO_DATA_WRITER_ANTI_ENTROPY_ADDRESS" : \
anti_entropy_address,
"NIMBUSIO_CENTRAL_USER_PASSWORD" : central_db_pw,
"NIMBUSIO_NODE_USER_PASSWORD" : node_db_pw,
"NIMBUSIO_EVENT_AGGREGATOR_PUB_ADDRESS": event_aggregator_pub_address,
}
log.info("starting %s %s" % (args, environment, ))
return subprocess.Popen(args, stderr=subprocess.PIPE, env=environment)
def start_handoff_server(cluster_name,
local_node_name,
handoff_server_addresses,
pipeline_address,
data_reader_addresses,
data_writer_addresses,
event_publisher_pull_address,
repository_path,
environment=None):
log = logging.getLogger("start_handoff_server_%s" % (local_node_name, ))
server_dir = identify_program_dir(u"handoff_server")
server_path = os.path.join(server_dir, "handoff_server_main.py")
args = [
sys.executable,
server_path,
]
if environment is None:
environment = {
"PYTHONPATH" : os.environ["PYTHONPATH"],
"NIMBUSIO_LOG_DIR" : os.environ["NIMBUSIO_LOG_DIR"],
"NIMBUSIO_CLUSTER_NAME" : cluster_name,
"NIMBUSIO_NODE_NAME" : local_node_name,
"NIMBUSIO_HANDOFF_SERVER_ADDRESSES" : " ".join(
handoff_server_addresses
),
"NIMBUSIO_HANDOFF_SERVER_PIPELINE_ADDRESS": pipeline_address,
"NIMBUSIO_DATA_READER_ADDRESSES" :
" ".join(data_reader_addresses),
"NIMBUSIO_DATA_WRITER_ADDRESSES" :
" ".join(data_writer_addresses),
"NIMBUSIO_EVENT_PUBLISHER_PULL_ADDRESS" : \
event_publisher_pull_address,
"NIMBUSIO_REPOSITORY_PATH" : repository_path,
"NIMBUSIO_CENTRAL_USER_PASSWORD" : "pork",
"NIMBUSIO_NODE_USER_PASSWORD" : "pork",
}
log.info("starting %s %s" % (
args,
environment,
))
return subprocess.Popen(args, stderr=subprocess.PIPE, env=environment)
def start_handoff_server(
cluster_name,
local_node_name,
handoff_server_addresses,
pipeline_address,
data_reader_addresses,
data_writer_addresses,
event_publisher_pull_address,
repository_path,
environment=None
):
log = logging.getLogger("start_handoff_server_%s" % (local_node_name, ))
server_dir = identify_program_dir(u"handoff_server")
server_path = os.path.join(server_dir, "handoff_server_main.py")
args = [
sys.executable,
server_path,
]
if environment is None:
environment = {
"PYTHONPATH" : os.environ["PYTHONPATH"],
"NIMBUSIO_LOG_DIR" : os.environ["NIMBUSIO_LOG_DIR"],
"NIMBUSIO_CLUSTER_NAME" : cluster_name,
"NIMBUSIO_NODE_NAME" : local_node_name,
"NIMBUSIO_HANDOFF_SERVER_ADDRESSES" : " ".join(
handoff_server_addresses
),
"NIMBUSIO_HANDOFF_SERVER_PIPELINE_ADDRESS": pipeline_address,
"NIMBUSIO_DATA_READER_ADDRESSES" :
" ".join(data_reader_addresses),
"NIMBUSIO_DATA_WRITER_ADDRESSES" :
" ".join(data_writer_addresses),
"NIMBUSIO_EVENT_PUBLISHER_PULL_ADDRESS" : \
event_publisher_pull_address,
"NIMBUSIO_REPOSITORY_PATH" : repository_path,
"NIMBUSIO_CENTRAL_USER_PASSWORD" : "pork",
"NIMBUSIO_NODE_USER_PASSWORD" : "pork",
}
log.info("starting %s %s" % (args, environment, ))
return subprocess.Popen(args, stderr=subprocess.PIPE, env=environment)
def start_event_publisher(node_name, pull_address, pub_address,
environment=None):
log = logging.getLogger("start_event_publisher_%s" % (node_name, ))
server_dir = identify_program_dir(u"event_publisher")
server_path = os.path.join(server_dir, "event_publisher_main.py")
args = [
sys.executable,
server_path,
]
if environment is None:
environment = {
"PYTHONPATH" : os.environ["PYTHONPATH"],
"NIMBUSIO_LOG_DIR" : os.environ["NIMBUSIO_LOG_DIR"],
"NIMBUSIO_NODE_NAME" : node_name,
"NIMBUSIO_EVENT_PUBLISHER_PULL_ADDRESS" : pull_address,
"NIMBUSIO_EVENT_PUBLISHER_PUB_ADDRESS" : pub_address,
}
log.info("starting %s %s" % (args, environment, ))
return subprocess.Popen(args, stderr=subprocess.PIPE, env=environment)
def start_web_server(node_name, environment, profile):
log = logging.getLogger("start_web_server_%s" % (node_name, ))
server_dir = identify_program_dir(u"web_server")
server_path = os.path.join(
server_dir, "web_server_main.py"
)
# the gevent wsgi server spews out a lot of cruft onto stdout and stderr
# so we capture it in a separate log
log_dir = environment["NIMBUSIO_LOG_DIR"]
wsgi_log_path = "%s/nimbusio_web_server_%s_wsgi.log" % (
log_dir, node_name,
)
wsgi_log_file = open(wsgi_log_path, "w")
args = [
sys.executable,
server_path,
]
if profile:
profile_path = os.path.join(
environment["NIMBUSIO_PROFILE_DIR"],
"web_server_%s.pstats" % (node_name, )
)
args[1:1] = [
"-m",
"cProfile",
"-o",
profile_path
]
log.info("starting %s" % (args, ))
return subprocess.Popen(
args, stdout=wsgi_log_file, stderr=subprocess.STDOUT, env=environment
)
def _install_schema_and_test_data():
log = logging.getLogger("_install_schema")
database_name = _node_database_name(_node_name)
user_name = _node_database_user(_node_name)
sql_path = identify_program_dir("sql")
schema_path = os.path.join(sql_path, "nimbusio_node.sql")
env = {
"PGPASSWORD": _database_password
}
args = [
"/usr/bin/psql", "-h", _database_host, "-p",
str(_database_port), "-d", database_name, "-U", user_name, "-f",
schema_path
]
log.debug(args)
process = subprocess.Popen(args, env=env)
process.wait()
assert process.returncode == 0, process.returncode
test_data_path = os.path.join(sql_path, "test_gc.sql")
env = {
"PGPASSWORD": _database_password
}
args = [
"/usr/bin/psql", "-h", _database_host, "-p",
str(_database_port), "-d", database_name, "-U", user_name, "-f",
test_data_path
]
log.debug(args)
process = subprocess.Popen(args, env=env)
process.wait()
assert process.returncode == 0, process.returncode
def _start_pullers(halt_event, work_dir):
"""
start puller subprocesses
"""
log = logging.getLogger("start_pullers")
pullers = dict()
anti_entropy_dir = identify_program_dir("anti_entropy")
puller_path = os.path.join(anti_entropy_dir, "cluster_inspector",
"segment_puller_subprocess.py")
for index, node_name in enumerate(_node_names):
if halt_event.is_set():
log.info("halt_event set: exiting")
return pullers
log.info("starting subprocess {0}".format(node_name))
args = [sys.executable, puller_path, work_dir, str(index)]
process = subprocess.Popen(args, stderr=subprocess.PIPE)
assert process is not None
pullers[node_name] = process
return pullers
def start_performance_packager(node_name, event_aggregator_pub_address,
environment = None):
log = logging.getLogger("start_performance_packager_%s" % (node_name, ))
server_dir = identify_program_dir(u"performance_packager")
server_path = os.path.join(
server_dir, "performance_packager_main.py"
)
args = [
sys.executable,
server_path,
]
if environment is None:
environment = {
"PYTHONPATH" : os.environ["PYTHONPATH"],
"NIMBUSIO_LOG_DIR" : os.environ["NIMBUSIO_LOG_DIR"],
"NIMBUSIO_NODE_NAME" : node_name,
"NIMBUSIO_EVENT_AGGREGATOR_PUB_ADDRESS" : \
event_aggregator_pub_address,
}
log.info("starting %s %s" % (args, environment, ))
return subprocess.Popen(args, stderr=subprocess.PIPE, env=environment)
def prepare_profile_graphs(profile_dir):
log = logging.getLogger("prepare_profile_graphs")
tools_dir = identify_program_dir("tools")
if tools_dir is None:
raise ProfileGraphError("unable to locate tools directory")
gprof2dot_path = os.path.join(tools_dir, "gprof2dot.py")
if not os.path.isfile(gprof2dot_path):
raise ProfileGraphError("unable to locate gprof2dot.py")
for file_name in os.listdir(profile_dir):
if not file_name.endswith(".pstats"):
continue
log.debug(file_name)
pstats_path = os.path.join(profile_dir, file_name)
png_path = "".join([pstats_path[:-len("pstats")], "png"])
gprof2dot_args = [
sys.executable,
gprof2dot_path,
"-f",
"pstats",
"-e",
"0.1",
"-n",
"0.1",
pstats_path
]
dot_args = [
"/usr/bin/dot",
"-T",
"png",
"-o",
png_path
]
gprof2dot_process = subprocess.Popen(
gprof2dot_args, stdout=subprocess.PIPE, bufsize=_buffer_size
)
dot_process = subprocess.Popen(
dot_args,
stdin=gprof2dot_process.stdout,
stdout=subprocess.PIPE,
bufsize=_buffer_size
)
dot_process.communicate()
if gprof2dot_process.returncode != 0:
if gprof2dot_process.returncode is None:
gprof2dot_process.terminate()
else:
message = "gprof2dot process failed %s %s" % (
gprof2dot_process.returncode, gprof2dot_process.stderr,
)
log.error(message)
raise ProfileGraphError(message)
if dot_process.returncode != 0:
message = "dot process failed %s %s" % (
dot_process.returncode, dot_process.stderr,
)
log.error(message)
raise ProfileGraphError(message)
