def main():
"""
main entry point
return 0 for success (exit code)
"""
return_value = 0
initialize_logging(_log_path)
log = logging.getLogger("main")
log.info("program starts")
halt_event = Event()
set_signal_handler(halt_event)
node_generators = _start_subprocesses(halt_event)
merge_manager = heapq.merge(*node_generators)
try:
_manage_subprocesses(halt_event, merge_manager)
except Exception as instance:
log.exception(instance)
return_value = 1
return return_value
def main():
"""
main entry point
return 0 for success (exit code)
"""
return_value = 0
initialize_logging(_log_path)
log = logging.getLogger("main")
log.info("program starts")
halt_event = Event()
set_signal_handler(halt_event)
node_generators = _start_subprocesses(halt_event)
merge_manager = heapq.merge(*node_generators)
try:
_manage_subprocesses(halt_event, merge_manager)
except Exception as instance:
log.exception(instance)
return_value = 1
return return_value
def main():
"""
main entry point
return 0 for success (exit code)
"""
return_value = 0
initialize_logging(_log_path)
log = logging.getLogger("main")
log.info("program starts")
halt_event = Event()
set_signal_handler(halt_event)
node_subprocesses = _start_subprocesses(halt_event)
try:
halt_event.wait()
except Exception as instance:
log.exception(instance)
return_value = 1
for node_subprocess in node_subprocesses:
node_subprocess.terminate()
return return_value
def main():
"""
main entry point
return 0 for success (exit code)
"""
return_value = 0
initialize_logging(_log_path)
log = logging.getLogger("main")
log.info("program starts")
halt_event = Event()
set_signal_handler(halt_event)
node_subprocesses = _start_subprocesses(halt_event)
try:
halt_event.wait()
except Exception as instance:
log.exception(instance)
return_value = 1
for node_subprocess in node_subprocesses:
node_subprocess.terminate()
return return_value
def main():
"""
main entry point
"""
return_code = 0
args = _parse_command_line()
halt_event = Event()
set_signal_handler(halt_event)
zeromq_context = zmq.Context()
reporting_socket = zeromq_context.socket(zmq.PUSH)
reporting_socket.setsockopt(zmq.LINGER, 5000)
reporting_socket.setsockopt(zmq.SNDHWM, args.hwm)
reporting_socket.connect(args.reporting_url)
req_socket = None
ping_message = {"message-type" : "ping",
"message-id" : uuid.uuid1().hex, }
result_message = {"message-type" : "ping-result",
"url" : args.ping_url,
"result" : None,
"socket-reconnection-number" : 0,
"check-number" : 0, }
while not halt_event.is_set():
try:
req_socket = _process_one_ping(halt_event,
args,
zeromq_context,
ping_message,
result_message,
req_socket,
reporting_socket)
except zmq.ZMQError as zmq_error:
if is_interrupted_system_call(zmq_error) and halt_event.is_set():
pass
else:
logging.exception(str(zmq_error))
halt_event.set()
return_code = 1
except Exception as instance:
logging.exception(str(instance))
halt_event.set()
return_code = 1
else:
halt_event.wait(args.interval)
if req_socket is not None:
req_socket.close()
reporting_socket.close()
zeromq_context.term()
return return_code
def main():
"""
main entry point
returns 0 for normal termination (usually SIGTERM)
"""
return_value = 0
initialize_logging(_log_path)
log = logging.getLogger("main")
log.info("program starts")
halt_event = Event()
set_signal_handler(halt_event)
memcached_client = memcache.Client(_memcached_nodes)
zeromq_context = zmq.Context()
sub_socket = _create_sub_socket(zeromq_context)
expected_sequence = {
_cache_update_channel : None,
}
while not halt_event.is_set():
try:
topic = sub_socket.recv()
assert sub_socket.rcvmore
meta = sub_socket.recv()
if sub_socket.rcvmore:
data = sub_socket.recv()
else:
data = ""
_process_one_event(memcached_client,
expected_sequence,
topic,
meta,
data)
except KeyboardInterrupt: # convenience for testing
log.info("keyboard interrupt: terminating normally")
halt_event.set()
except zmq.ZMQError as zmq_error:
if is_interrupted_system_call(zmq_error) and halt_event.is_set():
log.info("interrupted system call - ok at shutdown")
else:
log.exception("zeromq error processing request")
return_value = 1
halt_event.set()
except Exception:
log.exception("error processing request")
return_value = 1
halt_event.set()
sub_socket.close()
zeromq_context.term()
log.info("program teminates: return value = {0}".format(return_value))
return return_value
def main():
"""
main entry point
return 0 for success (exit code)
"""
initialize_logging(_log_path)
log = logging.getLogger("main")
log.info("program starts")
args = parse_commandline()
halt_event = Event()
set_signal_handler(halt_event)
zeromq_context = zmq.Context()
event_push_client = EventPushClient(zeromq_context, "handoff_client")
event_push_client.info("program-start", "handoff_client starts")
return_code = 0
node_databases = None
try:
node_dict = get_node_ids(args.node_name)
node_databases = get_node_databases()
conjoined_rows, segment_rows = \
get_handoff_rows(node_databases, node_dict[args.node_name])
log.info("found {0} conjoined and {1} segment handoffs".format(
len(conjoined_rows), len(segment_rows)))
if len(conjoined_rows) > 0:
process_conjoined_rows(halt_event,
args,
node_databases,
conjoined_rows)
if len(segment_rows) > 0:
process_segment_rows(halt_event,
zeromq_context,
args,
node_dict,
node_databases,
segment_rows)
except Exception as instance:
log.exception("Uhandled exception {0}".format(instance))
event_push_client.exception(
unhandled_exception_topic,
str(instance),
exctype=instance.__class__.__name__
)
return_code = 1
if node_databases is not None:
for connection in node_databases.values():
connection.close()
event_push_client.close()
zeromq_context.term()
log.info("program terminates return_code = {0}".format(return_code))
return return_code
def main():
"""
main entry point
"""
return_code = 0
args = _parse_command_line()
halt_event = Event()
set_signal_handler(halt_event)
zeromq_context = zmq.Context()
reporting_socket = zeromq_context.socket(zmq.PUSH)
reporting_socket.setsockopt(zmq.LINGER, 5000)
reporting_socket.setsockopt(zmq.SNDHWM, args.hwm)
reporting_socket.connect(args.reporting_url)
req_socket = None
ping_message = {"message-type": "ping", "message-id": uuid.uuid1().hex}
result_message = {
"message-type": "ping-result",
"url": args.ping_url,
"result": None,
"socket-reconnection-number": 0,
"check-number": 0,
}
while not halt_event.is_set():
try:
req_socket = _process_one_ping(
halt_event, args, zeromq_context, ping_message, result_message, req_socket, reporting_socket
)
except zmq.ZMQError as zmq_error:
if is_interrupted_system_call(zmq_error) and halt_event.is_set():
pass
else:
logging.exception(str(zmq_error))
halt_event.set()
return_code = 1
except Exception as instance:
logging.exception(str(instance))
halt_event.set()
return_code = 1
else:
halt_event.wait(args.interval)
if req_socket is not None:
req_socket.close()
reporting_socket.close()
zeromq_context.term()
return return_code
def main():
"""
main entry point
returns 0 for normal termination (usually SIGTERM)
"""
return_value = 0
initialize_logging(_log_path)
log = logging.getLogger("main")
log.info("program starts")
halt_event = Event()
set_signal_handler(halt_event)
memcached_client = memcache.Client(_memcached_nodes)
zeromq_context = zmq.Context()
sub_socket = _create_sub_socket(zeromq_context)
expected_sequence = {
_cache_update_channel: None,
}
while not halt_event.is_set():
try:
topic = sub_socket.recv()
assert sub_socket.rcvmore
meta = sub_socket.recv()
if sub_socket.rcvmore:
data = sub_socket.recv()
else:
data = ""
_process_one_event(memcached_client, expected_sequence, topic,
meta, data)
except KeyboardInterrupt: # convenience for testing
log.info("keyboard interrupt: terminating normally")
halt_event.set()
except zmq.ZMQError as zmq_error:
if is_interrupted_system_call(zmq_error) and halt_event.is_set():
log.info("interrupted system call - ok at shutdown")
else:
log.exception("zeromq error processing request")
return_value = 1
halt_event.set()
except Exception:
log.exception("error processing request")
return_value = 1
halt_event.set()
sub_socket.close()
zeromq_context.term()
log.info("program teminates: return value = {0}".format(return_value))
return return_value
def _run_until_halt(
state,
pre_loop_actions,
post_loop_actions,
halt_event
):
log = logging.getLogger("_run_until_halt")
set_signal_handler(halt_event)
time_queue = TimeQueue()
# run any pre-loop actions.
# they have the opportunity to modify state
# and can return tasks to be added to the time queue
# if a pre-loop actions raises an exception, we must halt
log.debug("pre_loop_action(s)")
for pre_loop_action in pre_loop_actions:
if halt_event.is_set():
log.info("halt_event signaled during pre_loop")
return
try:
result_list = pre_loop_action(halt_event, state)
except Exception:
halt_event.set()
raise
if result_list is not None:
for task, start_time in result_list:
time_queue.put(task, start_time=start_time)
log.info("main loop starts")
# run until the time queue is empty. This means each task
# must watch the halt_event and behave correctly at shutdown.
while len(time_queue) > 0:
next_task_delay = time_queue.peek_time() - time.time()
if next_task_delay > 0.0:
halt_event.wait(next_task_delay)
next_task = time_queue.pop()
result_list = next_task(halt_event)
if result_list is not None:
for task, start_time in result_list:
time_queue.put(task, start_time=start_time)
log.info("main loop ends")
log.debug("post_loop_action(s)")
for post_loop_action in post_loop_actions:
post_loop_action(state)
log.info("program terminates")
def main():
"""
main entry point
return 0 for success (exit code)
"""
initialize_logging(_log_path)
log = logging.getLogger("main")
log.info("program starts")
halt_event = Event()
set_signal_handler(halt_event)
zmq_context = zmq.Context()
event_push_client = EventPushClient(zmq_context, "cluster_repair")
event_push_client.info("program-start", "cluster_repair starts")
zfec_server_req_socket = zmq_context.socket(zmq.REQ)
zfec_server_req_socket.setsockopt(zmq.LINGER, 1000)
log.info("connecting req socket to {0}".format(_zfec_server_address))
zfec_server_req_socket.connect(_zfec_server_address)
read_subprocess = _start_read_subprocess()
write_subprocess = _start_write_subprocess()
try:
_repair_cluster(halt_event,
zfec_server_req_socket,
read_subprocess,
write_subprocess)
except KeyboardInterrupt:
halt_event.set()
except Exception as instance:
log.exception(str(instance))
event_push_client.exception(
unhandled_exception_topic,
str(instance),
exctype=instance.__class__.__name__
)
return -3
finally:
read_subprocess.terminate()
write_subprocess.terminate()
event_push_client.close()
zfec_server_req_socket.close()
zmq_context.term()
log.info("program terminates normally")
return 0
def main():
"""
main entry point
return 0 for success (exit code)
"""
initialize_logging(_log_path)
log = logging.getLogger("main")
log.info("program starts")
args = parse_commandline()
halt_event = Event()
set_signal_handler(halt_event)
zeromq_context = zmq.Context()
event_push_client = EventPushClient(zeromq_context, "handoff_client")
event_push_client.info("program-start", "handoff_client starts")
return_code = 0
node_databases = None
try:
node_dict = get_node_ids(args.node_name)
node_databases = get_node_databases()
conjoined_rows, segment_rows = \
get_handoff_rows(node_databases, node_dict[args.node_name])
log.info("found {0} conjoined and {1} segment handoffs".format(
len(conjoined_rows), len(segment_rows)))
if len(conjoined_rows) > 0:
process_conjoined_rows(halt_event, args, node_databases,
conjoined_rows)
if len(segment_rows) > 0:
process_segment_rows(halt_event, zeromq_context, args, node_dict,
node_databases, segment_rows)
except Exception as instance:
log.exception("Uhandled exception {0}".format(instance))
event_push_client.exception(unhandled_exception_topic,
str(instance),
exctype=instance.__class__.__name__)
return_code = 1
if node_databases is not None:
for connection in node_databases.values():
connection.close()
event_push_client.close()
zeromq_context.term()
log.info("program terminates return_code = {0}".format(return_code))
return return_code
def main():
"""
main entry point
return 0 for success (exit code)
"""
initialize_logging(_log_path)
log = logging.getLogger("main")
log.info("program starts")
halt_event = Event()
set_signal_handler(halt_event)
zmq_context = zmq.Context()
event_push_client = EventPushClient(zmq_context, "cluster_inspector")
event_push_client.info("program-start", "cluster_inspector starts")
# if there's any wreckage from a previous run, clear it out
if os.path.exists(_work_dir):
log.info("removing old {0}".format(_work_dir))
shutil.rmtree(_work_dir)
os.mkdir(_work_dir)
try:
pull_segments_from_nodes(halt_event, _work_dir)
if halt_event.is_set():
log.info("halt_event set (1): exiting")
return -1
audit_segments(halt_event, _work_dir)
except KeyboardInterrupt:
halt_event.set()
except Exception as instance:
log.exception(str(instance))
event_push_client.exception(
unhandled_exception_topic,
str(instance),
exctype=instance.__class__.__name__
)
return -3
event_push_client.close()
zmq_context.term()
log.info("program terminates normally")
return 0
def _run_until_halt(state, pre_loop_actions, post_loop_actions, halt_event):
log = logging.getLogger("_run_until_halt")
set_signal_handler(halt_event)
time_queue = TimeQueue()
# run any pre-loop actions.
# they have the opportunity to modify state
# and can return tasks to be added to the time queue
# if a pre-loop actions raises an exception, we must halt
log.debug("pre_loop_action(s)")
for pre_loop_action in pre_loop_actions:
if halt_event.is_set():
log.info("halt_event signaled during pre_loop")
return
try:
result_list = pre_loop_action(halt_event, state)
except Exception:
halt_event.set()
raise
if result_list is not None:
for task, start_time in result_list:
time_queue.put(task, start_time=start_time)
log.info("main loop starts")
# run until the time queue is empty. This means each task
# must watch the halt_event and behave correctly at shutdown.
while len(time_queue) > 0:
next_task_delay = time_queue.peek_time() - time.time()
if next_task_delay > 0.0:
halt_event.wait(next_task_delay)
next_task = time_queue.pop()
result_list = next_task(halt_event)
if result_list is not None:
for task, start_time in result_list:
time_queue.put(task, start_time=start_time)
log.info("main loop ends")
log.debug("post_loop_action(s)")
for post_loop_action in post_loop_actions:
post_loop_action(state)
log.info("program terminates")
def main():
"""
main entry point
return 0 for success (exit code)
"""
initialize_logging(_log_path)
log = logging.getLogger("main")
log.info("program starts")
halt_event = Event()
set_signal_handler(halt_event)
zmq_context = zmq.Context()
event_push_client = EventPushClient(zmq_context, "cluster_inspector")
event_push_client.info("program-start", "cluster_inspector starts")
# if there's any wreckage from a previous run, clear it out
if os.path.exists(_work_dir):
log.info("removing old {0}".format(_work_dir))
shutil.rmtree(_work_dir)
os.mkdir(_work_dir)
try:
pull_segments_from_nodes(halt_event, _work_dir)
if halt_event.is_set():
log.info("halt_event set (1): exiting")
return -1
audit_segments(halt_event, _work_dir)
except KeyboardInterrupt:
halt_event.set()
except Exception as instance:
log.exception(str(instance))
event_push_client.exception(unhandled_exception_topic,
str(instance),
exctype=instance.__class__.__name__)
return -3
event_push_client.close()
zmq_context.term()
log.info("program terminates normally")
return 0
def main():
"""
main entry point
return 0 for success (exit code)
"""
initialize_logging(_log_path)
log = logging.getLogger("main")
log.info("program starts")
halt_event = Event()
set_signal_handler(halt_event)
zmq_context = zmq.Context()
event_push_client = EventPushClient(zmq_context, "cluster_repair")
event_push_client.info("program-start", "cluster_repair starts")
zfec_server_req_socket = zmq_context.socket(zmq.REQ)
zfec_server_req_socket.setsockopt(zmq.LINGER, 1000)
log.info("connecting req socket to {0}".format(_zfec_server_address))
zfec_server_req_socket.connect(_zfec_server_address)
read_subprocess = _start_read_subprocess()
write_subprocess = _start_write_subprocess()
try:
_repair_cluster(halt_event, zfec_server_req_socket, read_subprocess,
write_subprocess)
except KeyboardInterrupt:
halt_event.set()
except Exception as instance:
log.exception(str(instance))
event_push_client.exception(unhandled_exception_topic,
str(instance),
exctype=instance.__class__.__name__)
return -3
finally:
read_subprocess.terminate()
write_subprocess.terminate()
event_push_client.close()
zfec_server_req_socket.close()
zmq_context.term()
log.info("program terminates normally")
return 0
def main():
"""
main entry point
"""
returncode = 0
initialize_logging(_log_path)
log = logging.getLogger("main")
state = _create_state()
set_signal_handler(state["halt-event"])
try:
_setup(state)
except Exception:
instance = sys.exc_info()[1]
log.exception("unhandled exception in _setup")
log.critical("unhandled exception in _setup {0}".format(
instance))
state["halt-event"].set()
returncode = 1
log.debug("start halt_event loop")
while not state["halt-event"].is_set():
try:
state["pollster"].run(state["halt-event"])
except Exception:
instance = sys.exc_info()[1]
log.exception("unhandled exception in pollster")
log.critical("unhandled exception in pollster {0}".format(
instance))
state["halt-event"].set()
returncode = 1
log.debug("end halt_event loop")
try:
_tear_down(state)
except Exception:
instance = sys.exc_info()[1]
log.exception("unhandled exception in _tear_down")
log.critical("unhandled exception in _tear_down {0}".format(
instance))
returncode = 1
return returncode
def main():
"""
main entry point
returns 0 for normal termination (usually SIGTERM)
"""
return_value = 0
if len(sys.argv) == 1:
server_number = 0
else:
server_number = int(sys.argv[1])
log_path = _log_path_template.format(os.environ["NIMBUSIO_LOG_DIR"],
_local_node_name,
server_number)
initialize_logging(log_path)
log = logging.getLogger("main")
log.info("program starts")
halt_event = Event()
set_signal_handler(halt_event)
zeromq_context = zmq.Context()
rep_socket = _bind_rep_socket(zeromq_context)
try:
while not halt_event.is_set():
_process_one_request(rep_socket)
except ZfecServerInterrupedSystemCall:
if halt_event.is_set():
log.info("program teminates normally with interrupted system call")
return 0
log.exception("error processing request")
return_value = 1
except Exception:
log.exception("error processing request")
return_value = 1
else:
log.info("program teminates normally")
finally:
rep_socket.close()
zeromq_context.term()
return return_value
def main():
"""
main entry point
returns 0 for normal termination (usually SIGTERM)
"""
return_value = 0
if len(sys.argv) == 1:
server_number = 0
else:
server_number = int(sys.argv[1])
log_path = _log_path_template.format(os.environ["NIMBUSIO_LOG_DIR"],
_local_node_name, server_number)
initialize_logging(log_path)
log = logging.getLogger("main")
log.info("program starts")
halt_event = Event()
set_signal_handler(halt_event)
zeromq_context = zmq.Context()
rep_socket = _bind_rep_socket(zeromq_context)
try:
while not halt_event.is_set():
_process_one_request(rep_socket)
except ZfecServerInterrupedSystemCall:
if halt_event.is_set():
log.info("program teminates normally with interrupted system call")
return 0
log.exception("error processing request")
return_value = 1
except Exception:
log.exception("error processing request")
return_value = 1
else:
log.info("program teminates normally")
finally:
rep_socket.close()
zeromq_context.term()
return return_value
def main():
"""
main entry point
"""
return_code = 0
args = _parse_command_line()
halt_event = Event()
set_signal_handler(halt_event)
zeromq_context = zmq.Context()
reporting_socket = zeromq_context.socket(zmq.PUSH)
reporting_socket.setsockopt(zmq.LINGER, 5000)
reporting_socket.setsockopt(zmq.SNDHWM, args.hwm)
reporting_socket.connect(args.reporting_url)
result_message = {"message-type" : "ping-result",
"url" : args.ping_url,
"result" : None,
"socket-reconnection-number" : 0,
"check-number" : 0, }
while not halt_event.is_set():
try:
_process_one_ping(halt_event,
args,
result_message,
reporting_socket)
except Exception as instance:
logging.exception("fatal exception {0}".format(str(instance)))
halt_event.set()
return_code = 1
else:
halt_event.wait(args.interval)
reporting_socket.close()
zeromq_context.term()
return return_code
def main():
"""
main entry point
"""
return_code = 0
args = _parse_command_line()
halt_event = Event()
set_signal_handler(halt_event)
zeromq_context = zmq.Context()
reporting_socket = zeromq_context.socket(zmq.PUSH)
reporting_socket.setsockopt(zmq.LINGER, 5000)
reporting_socket.setsockopt(zmq.HWM, args.hwm)
reporting_socket.connect(args.reporting_url)
result_message = {"message-type" : "ping-result",
"url" : args.ping_url,
"result" : None,
"socket-reconnection-number" : 0,
"check-number" : 0, }
while not halt_event.is_set():
try:
_process_one_ping(halt_event,
args,
result_message,
reporting_socket)
except Exception as instance:
logging.exception("fatal exception {0}".format(str(instance)))
halt_event.set()
return_code = 1
else:
halt_event.wait(args.interval)
reporting_socket.close()
zeromq_context.term()
return return_code
def main():
"""
main entry point
returns 0 for normal termination (usually SIGTERM)
"""
return_value = 0
volume_name = sys.argv[1]
worker_number = int(sys.argv[2])
log_path = _log_path_template.format(os.environ["NIMBUSIO_LOG_DIR"],
volume_name.replace("/", "_"),
worker_number,
_local_node_name)
initialize_logging(log_path)
log = logging.getLogger("main")
log.info("program starts")
halt_event = Event()
set_signal_handler(halt_event)
zeromq_context = zmq.Context()
event_source_name = "rs_io_worker_{0}_{1}".format(volume_name,
worker_number)
resources = \
_resources_tuple(halt_event=halt_event,
zeromq_context=zeromq_context,
reply_push_sockets=dict(),
event_push_client=EventPushClient(zeromq_context,
event_source_name),
dealer_socket=zeromq_context.socket(zmq.DEALER),
file_cache=LRUCache(_max_file_cache_size))
resources.dealer_socket.setsockopt(zmq.LINGER, 1000)
log.debug("connecting to {0}".format(io_controller_router_socket_uri))
resources.dealer_socket.connect(io_controller_router_socket_uri)
last_close_pass_time = time.time()
try:
while not halt_event.is_set():
# an occasional pass that closes any open files that haven't
# been used
current_time = time.time()
elapsed_time = current_time - last_close_pass_time
if elapsed_time > _unused_file_close_interval:
_make_close_pass(resources, current_time)
last_close_pass_time = current_time
_send_work_request(resources, volume_name)
_process_request(resources)
except InterruptedSystemCall:
if halt_event.is_set():
log.info("program teminates normally with interrupted system call")
else:
log.exception("zeromq error processing request")
resources.event_push_client.exception(
unhandled_exception_topic,
"Interrupted zeromq system call",
exctype="InterruptedSystemCall")
return_value = 1
except Exception as instance:
log.exception("error processing request")
resources.event_push_client.exception(
unhandled_exception_topic,
str(instance),
exctype=instance.__class__.__name__)
return_value = 1
else:
log.info("program teminates normally")
finally:
resources.dealer_socket.close()
for push_socket in resources.reply_push_sockets.values():
push_socket.close()
resources.event_push_client.close()
resources.zeromq_context.term()
return return_value
def main():
"""
main entry point
returns 0 for normal termination (usually SIGTERM)
"""
return_value = 0
log_path = _log_path_template.format(os.environ["NIMBUSIO_LOG_DIR"],
_local_node_name)
initialize_logging(log_path)
log = logging.getLogger("main")
log.info("program starts")
for internal_socket_uri in internal_socket_uri_list:
prepare_ipc_path(internal_socket_uri)
halt_event = Event()
set_signal_handler(halt_event)
database_pool_controller = _launch_database_pool_controller()
io_controller = _launch_io_controller()
zeromq_context = zmq.Context()
rep_socket = _bind_rep_socket(zeromq_context)
db_controller_push_socket = \
_connect_db_controller_push_socket(zeromq_context)
event_push_client = EventPushClient(zeromq_context, "retrieve_source")
event_push_client.info("program-starts", "retrieve source starts")
# we poll the sockets for readability, we assume we can always
# write to the push client sockets
poller = zmq.Poller()
poller.register(rep_socket, zmq.POLLIN | zmq.POLLERR)
last_report_time = 0.0
request_count = 0
try:
while not halt_event.is_set():
poll_subprocess(database_pool_controller)
poll_subprocess(io_controller)
# we've only registered one socket, so we could use an 'if' here,
# but this 'for' works ok and it has the same form as the other
# places where we use poller
for active_socket, event_flags in poller.poll(_poll_timeout):
if event_flags & zmq.POLLERR:
error_message = \
"error flags from zmq {0}".format(active_socket)
log.error(error_message)
raise PollError(error_message)
assert active_socket is rep_socket
_process_one_request(rep_socket, db_controller_push_socket)
request_count += 1
current_time = time.time()
elapsed_time = current_time - last_report_time
if elapsed_time > _reporting_interval:
report_message = "{0:,} requests".format(request_count)
log.info(report_message)
event_push_client.info("request_count",
report_message,
request_count=request_count)
last_report_time = current_time
request_count = 0
except KeyboardInterrupt: # convenience for testing
log.info("keyboard interrupt: terminating normally")
except zmq.ZMQError as zmq_error:
if is_interrupted_system_call(zmq_error) and halt_event.is_set():
log.info("program teminates normally with interrupted system call")
else:
log.exception("zeromq error processing request")
event_push_client.exception(unhandled_exception_topic,
"zeromq_error",
exctype="ZMQError")
return_value = 1
except Exception as instance:
log.exception("error processing request")
event_push_client.exception(unhandled_exception_topic,
str(instance),
exctype=instance.__class__.__name__)
return_value = 1
else:
log.info("program teminates normally")
finally:
terminate_subprocess(database_pool_controller)
terminate_subprocess(io_controller)
rep_socket.close()
db_controller_push_socket.close()
event_push_client.close()
zeromq_context.term()
return return_value
def main():
"""
main entry point
return 0 for success (exit code)
"""
initialize_logging(_log_path)
log = logging.getLogger("main")
log.info("program starts")
halt_event = Event()
set_signal_handler(halt_event)
zmq_context = zmq.Context()
event_push_client = EventPushClient(zmq_context, "defragger")
event_push_client.info("program-start", "defragger starts")
connection = None
file_space_info = None
while not halt_event.is_set():
# if we don't have an open database connection, get one
if connection is None:
try:
connection = get_node_local_connection()
except Exception as instance:
exctype, value = sys.exc_info()[:2]
event_push_client.exception("database exception",
str(value),
exctype=exctype.__name__)
log.exception("Exception connecting to database")
halt_event.wait(_database_retry_interval)
continue
file_space_info = load_file_space_info(connection)
file_space_sanity_check(file_space_info, _repository_path)
# try one defrag pass
bytes_defragged = 0
connection.begin_transaction()
try:
bytes_defragged = _defrag_pass(connection, file_space_info,
event_push_client)
except KeyboardInterrupt:
halt_event.set()
connection.rollback()
except Exception as instance:
log.exception(str(instance))
event_push_client.exception(unhandled_exception_topic,
str(instance),
exctype=instance.__class__.__name__)
connection.rollback()
else:
connection.commit()
log.info("bytes defragged = {0:,}".format(bytes_defragged))
# if we didn't do anything on this pass...
if bytes_defragged == 0:
# exit if we're done and asked to do single pass
if int(os.environ.get('NIMBUSIO_EXIT_WHEN_DONE', '0')):
halt_event.set()
# close the database connection
if connection is not None:
connection.close()
connection = None
# wait and try again
try:
halt_event.wait(_defrag_check_interval)
except KeyboardInterrupt:
halt_event.set()
if connection is not None:
connection.close()
event_push_client.close()
zmq_context.term()
log.info("program terminates normally")
return 0
def main():
"""
main entry point
returns 0 for normal termination (usually SIGTERM)
"""
return_value = 0
log_path = _log_path_template.format(os.environ["NIMBUSIO_LOG_DIR"],
_local_node_name)
initialize_logging(log_path)
log = logging.getLogger("main")
log.info("program starts")
prepare_ipc_path(_pull_socket_uri)
halt_event = Event()
set_signal_handler(halt_event)
zeromq_context = zmq.Context()
pull_socket = _bind_pull_socket(zeromq_context)
event_push_client = EventPushClient(zeromq_context, "service_availability")
event_push_client.info("program-starts",
"service availability monitor starts")
message_count = 0
try:
ping_process_dict = _start_ping_processes(halt_event)
while not halt_event.is_set():
if message_count % len(ping_process_dict) == 0:
for ping_process in ping_process_dict.values():
poll_subprocess(ping_process.process)
message = pull_socket.recv_pyobj()
assert not pull_socket.rcvmore
_process_one_message(message, ping_process_dict, event_push_client)
message_count += 1
except KeyboardInterrupt: # convenience for testing
log.info("keyboard interrupt: terminating normally")
except zmq.ZMQError as zmq_error:
if is_interrupted_system_call(zmq_error) and halt_event.is_set():
log.info("program terminating normally; interrupted system call")
else:
log.exception("zeromq error processing request")
event_push_client.exception(unhandled_exception_topic,
"zeromq_error",
exctype="ZMQError")
return_value = 1
except Exception as instance:
log.exception("error processing request")
event_push_client.exception(unhandled_exception_topic,
str(instance),
exctype=instance.__class__.__name__)
return_value = 1
else:
log.info("program teminating normally")
log.debug("terminating subprocesses")
_terminate_ping_processes(ping_process_dict)
pull_socket.close()
event_push_client.close()
zeromq_context.term()
return return_value
def main():
"""
main entry point
return 0 for success (exit code)
"""
initialize_logging(_log_path)
log = logging.getLogger("main")
log.info("program starts")
halt_event = Event()
set_signal_handler(halt_event)
zeromq_context = zmq.Context()
event_push_client = EventPushClient(zeromq_context,
"redis_stats_collector")
event_push_client.info("program-start", "flush_stats_from_redis starts")
# don't flush anything newer than 1 minute ago
current_time = datetime.utcnow()
timestamp_cutoff = current_time - timedelta(minutes=1)
return_code = 0
central_db_connection = None
collection_ops_accounting_rows = list()
# values to be added to the dedupe table
new_dedupes = list()
# keys to be deleted (a list for each node
node_keys_processed = [list() for _ in _node_names]
try:
central_db_connection = get_central_connection()
# On startup, the program connects to the central database and tries
# to acquire a pg_advisory_lock appropriate for this program and the
# data center it is running in using the pg_try_advisory_lock function.
# If it cannot acquire the lock, it notes the status of the lock
# and exits. This central locking mechanism lets us avoid single points
# of failure by configuring the program to run on multiple nodes.
with advisory_lock(central_db_connection, "redis_stats_collector"):
node_dict = _retrieve_node_dict(central_db_connection)
for node_name, keys_processed in \
zip(_node_names, node_keys_processed):
node_id = node_dict[node_name]
log.debug("processing node {0} node_id={1}".format(node_name,
node_id))
# The program then selects into memory all recently collected
# keys from the central database table
# collection_ops_accounting_flush_dedupe and stores them in a
# dedupe set. This set allows runs of the collection/flush
# program to be idempotent across some time period (
# but we won't keep the list of old keys forever.)
dedupe_set = _retrieve_dedupe_set(central_db_connection,
node_id)
# The program then visits the Redis instance on every storage
# node in the local data center, collecting the data from all
# past stats keys -- aggregating it into the program's memory.
# The aggregation should involve buckets for each
# storage_node_id and redis key, corresponding to the columns
# in the database.
_process_one_node(node_name,
node_dict[node_name],
timestamp_cutoff,
dedupe_set,
collection_ops_accounting_rows,
new_dedupes,
keys_processed)
# After collecting past keys from every storage node,
# inside a central database transaction:
# 1. Insert the collected stats into the central database
# collection_ops_accounting
# 2. Insert collected keys into recently collected keys
# collection_ops_accounting_flush_dedupe.
# 3. commit transaction
log.debug("updating central database")
central_db_connection.begin_transaction()
try:
_insert_accounting_rows(central_db_connection,
collection_ops_accounting_rows)
_insert_dedupe_rows(central_db_connection,
timestamp_cutoff,
new_dedupes)
except Exception:
central_db_connection.rollback()
raise
else:
central_db_connection.commit()
# Then revisit the Redis nodes, and delete the keys we flushed
# into the database, and any keys we skipped because they were
# found in the dedupe set.
for node_name, keys_processed in zip(_node_names,
node_keys_processed):
_remove_processed_keys(node_name, keys_processed)
except Exception as instance:
log.exception("Uhandled exception {0}".format(instance))
event_push_client.exception(
unhandled_exception_topic,
str(instance),
exctype=instance.__class__.__name__
)
return_code = 1
if central_db_connection is not None:
central_db_connection.close()
event_push_client.close()
zeromq_context.term()
log.info("program terminates return_code = {0}".format(return_code))
return return_code
def main(worker_id, host_name, base_port, dest_node_name, rep_socket_uri):
"""
main entry point
return 0 on normal termination (exit code)
"""
log = logging.getLogger("main")
client_tag = _client_tag_template.format(worker_id)
halt_event = Event()
set_signal_handler(halt_event)
zeromq_context = zmq.Context()
log.debug("creating pull socket")
pull_socket_uri = "tcp://{0}:{1}".format(socket.gethostbyname(host_name),
base_port + worker_id)
pull_socket = zeromq_context.socket(zmq.PULL)
pull_socket.setsockopt(zmq.RCVHWM, _socket_high_water_mark)
log.info("binding pull socket to {0}".format(pull_socket_uri))
pull_socket.bind(pull_socket_uri)
req_socket = zeromq_context.socket(zmq.REQ)
req_socket.setsockopt(zmq.RCVHWM, _socket_high_water_mark)
req_socket.setsockopt(zmq.SNDHWM, _socket_high_water_mark)
req_socket.connect(rep_socket_uri)
log.info("sending 'start' message")
# notify our parent that we are ready to receive work
request = {"message-type": "start", "worker-id": worker_id}
req_socket.send_pyobj(request)
log.info("starting message loop")
node_dict = None
while not halt_event.is_set():
try:
message = req_socket.recv_pyobj()
except zmq.ZMQError as zmq_error:
if is_interrupted_system_call(zmq_error) and halt_event.is_set():
log.info("interrupted system call wiht halt_event_set")
break
log.exception(str(zmq_error))
sys.exit(1)
assert not req_socket.rcvmore
if message["message-type"] == "stop":
log.info("'stop' message received")
break
assert message["message-type"] == "work", message["message-type"]
# we expect our parent to send us the node dict in our first message
if "node-dict" in message:
node_dict = message["node-dict"]
assert node_dict is not None
# aliases for brevity
segment_row = message["segment-row"]
source_node_names = message["source-node-names"]
# send back enough information to purge the handoffs, if successful
request = {
"message-type": "handoff-complete",
"worker-id": worker_id,
"handoff-successful": False,
"unified-id": segment_row["unified_id"],
"collection-id": segment_row["collection_id"],
"key": segment_row["key"],
"conjoined-part": segment_row["conjoined_part"],
"handoff-node-id": segment_row["handoff_node_id"],
"source-node-names": source_node_names,
"error-message": ""
}
for source_node_name in source_node_names:
try:
_process_handoff(zeromq_context, halt_event, node_dict,
pull_socket, pull_socket_uri, client_tag,
source_node_name, dest_node_name, segment_row)
except Exception as instance:
log.exception(instance)
request["error-message"] = "".join(
[request["error-message"],
str(instance)])
else:
request["handoff-successful"] = True
break
req_socket.send_pyobj(request)
log.info("end message loop")
pull_socket.close()
req_socket.close()
zeromq_context.term()
log.info("program terminates")
return 0
def main():
"""
main entry point
return 0 for success (exit code)
"""
initialize_logging(_log_path)
log = logging.getLogger("main")
log.info("program starts")
halt_event = Event()
set_signal_handler(halt_event)
zmq_context = zmq.Context()
event_push_client = EventPushClient(zmq_context, "defragger")
event_push_client.info("program-start", "defragger starts")
connection = None
file_space_info = None
while not halt_event.is_set():
# if we don't have an open database connection, get one
if connection is None:
try:
connection = get_node_local_connection()
except Exception as instance:
exctype, value = sys.exc_info()[:2]
event_push_client.exception(
"database exception",
str(value),
exctype=exctype.__name__
)
log.exception("Exception connecting to database")
halt_event.wait(_database_retry_interval)
continue
file_space_info = load_file_space_info(connection)
file_space_sanity_check(file_space_info, _repository_path)
# try one defrag pass
bytes_defragged = 0
connection.begin_transaction()
try:
bytes_defragged = _defrag_pass(connection,
file_space_info,
event_push_client)
except KeyboardInterrupt:
halt_event.set()
connection.rollback()
except Exception as instance:
log.exception(str(instance))
event_push_client.exception(
unhandled_exception_topic,
str(instance),
exctype=instance.__class__.__name__
)
connection.rollback()
else:
connection.commit()
log.info("bytes defragged = {0:,}".format(bytes_defragged))
# if we didn't do anything on this pass...
if bytes_defragged == 0:
# exit if we're done and asked to do single pass
if int(os.environ.get('NIMBUSIO_EXIT_WHEN_DONE', '0')):
halt_event.set()
# close the database connection
if connection is not None:
connection.close()
connection = None
# wait and try again
try:
halt_event.wait(_defrag_check_interval)
except KeyboardInterrupt:
halt_event.set()
if connection is not None:
connection.close()
event_push_client.close()
zmq_context.term()
log.info("program terminates normally")
return 0
def main():
"""
main entry point
returns 0 for normal termination (usually SIGTERM)
"""
return_value = 0
log_path = _log_path_template.format(os.environ["NIMBUSIO_LOG_DIR"],
_local_node_name)
initialize_logging(log_path)
log = logging.getLogger("main")
log.info("program starts")
prepare_ipc_path(_pull_socket_uri)
halt_event = Event()
set_signal_handler(halt_event)
zeromq_context = zmq.Context()
pull_socket = _bind_pull_socket(zeromq_context)
event_push_client = EventPushClient(zeromq_context, "service_availability")
event_push_client.info("program-starts",
"service availability monitor starts")
message_count = 0
try:
ping_process_dict = _start_ping_processes(halt_event)
while not halt_event.is_set():
if message_count % len(ping_process_dict) == 0:
for ping_process in ping_process_dict.values():
poll_subprocess(ping_process.process)
message = pull_socket.recv_pyobj()
assert not pull_socket.rcvmore
_process_one_message(message, ping_process_dict, event_push_client)
message_count += 1
except KeyboardInterrupt: # convenience for testing
log.info("keyboard interrupt: terminating normally")
except zmq.ZMQError as zmq_error:
if is_interrupted_system_call(zmq_error) and halt_event.is_set():
log.info("program terminating normally; interrupted system call")
else:
log.exception("zeromq error processing request")
event_push_client.exception(unhandled_exception_topic,
"zeromq_error",
exctype="ZMQError")
return_value = 1
except Exception as instance:
log.exception("error processing request")
event_push_client.exception(unhandled_exception_topic,
str(instance),
exctype=instance.__class__.__name__)
return_value = 1
else:
log.info("program teminating normally")
log.debug("terminating subprocesses")
_terminate_ping_processes(ping_process_dict)
pull_socket.close()
event_push_client.close()
zeromq_context.term()
return return_value
def main():
"""
main entry point
returns 0 for normal termination (usually SIGTERM)
"""
return_value = 0
log_path = _log_path_template.format(os.environ["NIMBUSIO_LOG_DIR"],
_local_node_name)
initialize_logging(log_path)
log = logging.getLogger("main")
log.info("program starts")
halt_event = Event()
set_signal_handler(halt_event)
zeromq_context = zmq.Context()
resources = \
_resources_tuple(halt_event=Event(),
volume_by_space_id=_volume_name_by_space_id(),
pull_socket=zeromq_context.socket(zmq.PULL),
router_socket=zeromq_context.socket(zmq.ROUTER),
event_push_client=\
EventPushClient(zeromq_context,
"rs_io_controller"),
pending_work_by_volume=defaultdict(deque),
available_ident_by_volume=defaultdict(deque))
log.debug("binding to {0}".format(io_controller_pull_socket_uri))
resources.pull_socket.bind(io_controller_pull_socket_uri)
resources.router_socket.setsockopt(zmq.LINGER, 1000)
log.debug("binding to {0}".format(io_controller_router_socket_uri))
resources.router_socket.bind(io_controller_router_socket_uri)
# we poll the sockets for readability, we assume we can always
# write to the router socket
poller = zmq.Poller()
poller.register(resources.pull_socket, zmq.POLLIN | zmq.POLLERR)
poller.register(resources.router_socket, zmq.POLLIN | zmq.POLLERR)
worker_processes = list()
for volume_name in set(resources.volume_by_space_id.values()):
for index in range(_worker_count):
worker_processes.append(_launch_io_worker(volume_name, index + 1))
last_report_time = 0.0
try:
while not halt_event.is_set():
for worker_process in worker_processes:
poll_subprocess(worker_process)
for active_socket, event_flags in poller.poll(_poll_timeout):
if event_flags & zmq.POLLERR:
error_message = \
"error flags from zmq {0}".format(active_socket)
log.error(error_message)
raise PollError(error_message)
if active_socket is resources.pull_socket:
_read_pull_socket(resources)
elif active_socket is resources.router_socket:
_read_router_socket(resources)
else:
log.error("unknown socket {0}".format(active_socket))
current_time = time.time()
elapsed_time = current_time - last_report_time
if elapsed_time > _reporting_interval:
pending_work = 0
for volume_queue in resources.pending_work_by_volume.values():
pending_work += len(volume_queue)
report_message = \
"{0:,} pending_work entries".format(pending_work)
log.info(report_message)
resources.event_push_client.info("queue_sizes",
report_message,
pending_work=pending_work)
last_report_time = current_time
except zmq.ZMQError as zmq_error:
if is_interrupted_system_call(zmq_error) and halt_event.is_set():
log.info("program teminates normally with interrupted system call")
else:
log.exception("zeromq error processing request")
resources.event_push_client.exception(unhandled_exception_topic,
"zeromq_error",
exctype="ZMQError")
return_value = 1
except Exception as instance:
log.exception("error processing request")
resources.event_push_client.exception(
unhandled_exception_topic,
str(instance),
exctype=instance.__class__.__name__)
return_value = 1
else:
log.info("program teminates normally")
finally:
for worker_process in worker_processes:
terminate_subprocess(worker_process)
resources.pull_socket.close()
resources.router_socket.close()
resources.event_push_client.close()
zeromq_context.term()
return return_value
def main():
"""
main entry point
returns 0 for normal termination (usually SIGTERM)
"""
return_value = 0
volume_name = sys.argv[1]
worker_number = int(sys.argv[2])
log_path = _log_path_template.format(os.environ["NIMBUSIO_LOG_DIR"],
volume_name.replace("/", "_"),
worker_number, _local_node_name)
initialize_logging(log_path)
log = logging.getLogger("main")
log.info("program starts")
halt_event = Event()
set_signal_handler(halt_event)
zeromq_context = zmq.Context()
event_source_name = "rs_io_worker_{0}_{1}".format(volume_name,
worker_number)
resources = \
_resources_tuple(halt_event=halt_event,
zeromq_context=zeromq_context,
reply_push_sockets=dict(),
event_push_client=EventPushClient(zeromq_context,
event_source_name),
dealer_socket=zeromq_context.socket(zmq.DEALER),
file_cache=LRUCache(_max_file_cache_size))
resources.dealer_socket.setsockopt(zmq.LINGER, 1000)
log.debug("connecting to {0}".format(io_controller_router_socket_uri))
resources.dealer_socket.connect(io_controller_router_socket_uri)
last_close_pass_time = time.time()
try:
while not halt_event.is_set():
# an occasional pass that closes any open files that haven't
# been used
current_time = time.time()
elapsed_time = current_time - last_close_pass_time
if elapsed_time > _unused_file_close_interval:
_make_close_pass(resources, current_time)
last_close_pass_time = current_time
_send_work_request(resources, volume_name)
_process_request(resources)
except InterruptedSystemCall:
if halt_event.is_set():
log.info("program teminates normally with interrupted system call")
else:
log.exception("zeromq error processing request")
resources.event_push_client.exception(
unhandled_exception_topic,
"Interrupted zeromq system call",
exctype="InterruptedSystemCall")
return_value = 1
except Exception as instance:
log.exception("error processing request")
resources.event_push_client.exception(
unhandled_exception_topic,
str(instance),
exctype=instance.__class__.__name__)
return_value = 1
else:
log.info("program teminates normally")
finally:
resources.dealer_socket.close()
for push_socket in resources.reply_push_sockets.values():
push_socket.close()
resources.event_push_client.close()
resources.zeromq_context.term()
return return_value
def main():
"""
main entry point
returns 0 for normal termination (usually SIGTERM)
"""
return_value = 0
log_path = _log_path_template.format(os.environ["NIMBUSIO_LOG_DIR"], _local_node_name)
initialize_logging(log_path)
log = logging.getLogger("main")
log.info("program starts")
for internal_socket_uri in internal_socket_uri_list:
prepare_ipc_path(internal_socket_uri)
halt_event = Event()
set_signal_handler(halt_event)
database_pool_controller = _launch_database_pool_controller()
io_controller = _launch_io_controller()
zeromq_context = zmq.Context()
rep_socket = _bind_rep_socket(zeromq_context)
db_controller_push_socket = _connect_db_controller_push_socket(zeromq_context)
event_push_client = EventPushClient(zeromq_context, "retrieve_source")
event_push_client.info("program-starts", "retrieve source starts")
# we poll the sockets for readability, we assume we can always
# write to the push client sockets
poller = zmq.Poller()
poller.register(rep_socket, zmq.POLLIN | zmq.POLLERR)
last_report_time = 0.0
request_count = 0
try:
while not halt_event.is_set():
poll_subprocess(database_pool_controller)
poll_subprocess(io_controller)
# we've only registered one socket, so we could use an 'if' here,
# but this 'for' works ok and it has the same form as the other
# places where we use poller
for active_socket, event_flags in poller.poll(_poll_timeout):
if event_flags & zmq.POLLERR:
error_message = "error flags from zmq {0}".format(active_socket)
log.error(error_message)
raise PollError(error_message)
assert active_socket is rep_socket
_process_one_request(rep_socket, db_controller_push_socket)
request_count += 1
current_time = time.time()
elapsed_time = current_time - last_report_time
if elapsed_time > _reporting_interval:
report_message = "{0:,} requests".format(request_count)
log.info(report_message)
event_push_client.info("request_count", report_message, request_count=request_count)
last_report_time = current_time
request_count = 0
except KeyboardInterrupt: # convenience for testing
log.info("keyboard interrupt: terminating normally")
except zmq.ZMQError as zmq_error:
if is_interrupted_system_call(zmq_error) and halt_event.is_set():
log.info("program teminates normally with interrupted system call")
else:
log.exception("zeromq error processing request")
event_push_client.exception(unhandled_exception_topic, "zeromq_error", exctype="ZMQError")
return_value = 1
except Exception as instance:
log.exception("error processing request")
event_push_client.exception(unhandled_exception_topic, str(instance), exctype=instance.__class__.__name__)
return_value = 1
else:
log.info("program teminates normally")
finally:
terminate_subprocess(database_pool_controller)
terminate_subprocess(io_controller)
rep_socket.close()
db_controller_push_socket.close()
event_push_client.close()
zeromq_context.term()
return return_value
def main(worker_id, host_name, base_port, dest_node_name, rep_socket_uri):
"""
main entry point
return 0 on normal termination (exit code)
"""
log = logging.getLogger("main")
client_tag = _client_tag_template.format(worker_id)
halt_event = Event()
set_signal_handler(halt_event)
zeromq_context = zmq.Context()
log.debug("creating pull socket")
pull_socket_uri = "tcp://{0}:{1}".format(socket.gethostbyname(host_name),
base_port+worker_id)
pull_socket = zeromq_context.socket(zmq.PULL)
pull_socket.setsockopt(zmq.RCVHWM, _socket_high_water_mark)
log.info("binding pull socket to {0}".format(pull_socket_uri))
pull_socket.bind(pull_socket_uri)
req_socket = zeromq_context.socket(zmq.REQ)
req_socket.setsockopt(zmq.RCVHWM, _socket_high_water_mark)
req_socket.setsockopt(zmq.SNDHWM, _socket_high_water_mark)
req_socket.connect(rep_socket_uri)
log.info("sending 'start' message")
# notify our parent that we are ready to receive work
request = {"message-type" : "start",
"worker-id" : worker_id}
req_socket.send_pyobj(request)
log.info("starting message loop")
node_dict = None
while not halt_event.is_set():
try:
message = req_socket.recv_pyobj()
except zmq.ZMQError as zmq_error:
if is_interrupted_system_call(zmq_error) and halt_event.is_set():
log.info("interrupted system call wiht halt_event_set")
break
log.exception(str(zmq_error))
sys.exit(1)
assert not req_socket.rcvmore
if message["message-type"] == "stop":
log.info("'stop' message received")
break
assert message["message-type"] == "work", message["message-type"]
# we expect our parent to send us the node dict in our first message
if "node-dict" in message:
node_dict = message["node-dict"]
assert node_dict is not None
# aliases for brevity
segment_row = message["segment-row"]
source_node_names = message["source-node-names"]
# send back enough information to purge the handoffs, if successful
request = {"message-type" : "handoff-complete",
"worker-id" : worker_id,
"handoff-successful" : False,
"unified-id" : segment_row["unified_id"],
"collection-id" : segment_row["collection_id"],
"key" : segment_row["key"],
"conjoined-part" : segment_row["conjoined_part"],
"handoff-node-id" : segment_row["handoff_node_id"],
"source-node-names" : source_node_names,
"error-message" : ""}
for source_node_name in source_node_names:
try:
_process_handoff(zeromq_context,
halt_event,
node_dict,
pull_socket,
pull_socket_uri,
client_tag,
source_node_name,
dest_node_name,
segment_row)
except Exception as instance:
log.exception(instance)
request["error-message"] = "".join([request["error-message"],
str(instance)])
else:
request["handoff-successful"] = True
break
req_socket.send_pyobj(request)
log.info("end message loop")
pull_socket.close()
req_socket.close()
zeromq_context.term()
log.info("program terminates")
return 0
def main():
"""
main entry point
returns 0 for normal termination (usually SIGTERM)
"""
return_value = 0
worker_number = int(sys.argv[1])
log_path = _log_path_template.format(os.environ["NIMBUSIO_LOG_DIR"],
worker_number,
_local_node_name)
initialize_logging(log_path)
log = logging.getLogger("main")
log.info("program starts")
halt_event = Event()
set_signal_handler(halt_event)
zeromq_context = zmq.Context()
event_source_name = "rs_dbpool_worker_{0}".format(worker_number)
event_push_client = EventPushClient(zeromq_context, event_source_name)
dealer_socket = zeromq_context.socket(zmq.DEALER)
dealer_socket.setsockopt(zmq.LINGER, 1000)
log.debug("connecting to {0}".format(db_controller_router_socket_uri))
dealer_socket.connect(db_controller_router_socket_uri)
log.debug("opening local database connection")
database_connection = get_node_local_connection()
try:
_send_initial_work_request(dealer_socket)
while not halt_event.is_set():
_process_one_transaction(dealer_socket,
database_connection,
event_push_client)
except InterruptedSystemCall:
if halt_event.is_set():
log.info("program teminates normally with interrupted system call")
else:
log.exception("zeromq error processing request")
event_push_client.exception(unhandled_exception_topic,
"Interrupted zeromq system call",
exctype="InterruptedSystemCall")
return_value = 1
except Exception as instance:
log.exception("error processing request")
event_push_client.exception(unhandled_exception_topic,
str(instance),
exctype=instance.__class__.__name__)
return_value = 1
else:
log.info("program teminates normally")
finally:
database_connection.close()
dealer_socket.close()
event_push_client.close()
zeromq_context.term()
return return_value
def main():
"""
main entry point
returns 0 for normal termination (usually SIGTERM)
"""
return_value = 0
log_path = _log_path_template.format(os.environ["NIMBUSIO_LOG_DIR"],
_local_node_name)
initialize_logging(log_path)
log = logging.getLogger("main")
log.info("program starts")
halt_event = Event()
set_signal_handler(halt_event)
zeromq_context = zmq.Context()
resources = \
_resources_tuple(halt_event=Event(),
zeromq_context=zeromq_context,
reply_push_sockets=dict(),
pull_socket=zeromq_context.socket(zmq.PULL),
io_controller_push_socket=\
zeromq_context.socket(zmq.PUSH),
router_socket=zeromq_context.socket(zmq.ROUTER),
event_push_client=\
EventPushClient(zeromq_context,
"rs_db_pool_controller"),
active_retrieves=dict(),
pending_work_queue=deque(),
available_ident_queue=deque())
log.debug("binding to {0}".format(db_controller_pull_socket_uri))
resources.pull_socket.bind(db_controller_pull_socket_uri)
log.debug("connecting to {0}".format(io_controller_pull_socket_uri))
resources.io_controller_push_socket.connect(io_controller_pull_socket_uri)
resources.router_socket.setsockopt(zmq.LINGER, 1000)
log.debug("binding to {0}".format(db_controller_router_socket_uri))
resources.router_socket.bind(db_controller_router_socket_uri)
# we poll the sockets for readability, we assume we can always
# write to the router socket
poller = zmq.Poller()
poller.register(resources.pull_socket, zmq.POLLIN | zmq.POLLERR)
poller.register(resources.router_socket, zmq.POLLIN| zmq.POLLERR)
worker_processes = list()
for index in range(_worker_count):
worker_processes.append(_launch_database_pool_worker(index+1))
last_report_time = 0.0
try:
while not halt_event.is_set():
for worker_process in worker_processes:
poll_subprocess(worker_process)
for active_socket, event_flags in poller.poll(_poll_timeout):
if event_flags & zmq.POLLERR:
error_message = \
"error flags from zmq {0}".format(active_socket)
log.error(error_message)
raise PollError(error_message)
if active_socket is resources.pull_socket:
_read_pull_socket(resources)
elif active_socket is resources.router_socket:
_read_router_socket(resources)
else:
log.error("unknown socket {0}".format(active_socket))
current_time = time.time()
elapsed_time = current_time - last_report_time
if elapsed_time > _reporting_interval:
report_message = \
"{0:,} active_retrives, " \
"{1:,} pending_work_queue entries, " \
"{2:,} available_ident_queue entries" \
"".format(len(resources.active_retrieves),
len(resources.pending_work_queue),
len(resources.available_ident_queue))
log.info(report_message)
resources.event_push_client.info(
"queue_sizes",
report_message,
active_retrieves=len(resources.active_retrieves),
pending_work_queue=len(resources.pending_work_queue),
available_ident_queue=len(resources.available_ident_queue))
last_report_time = current_time
except zmq.ZMQError as zmq_error:
if is_interrupted_system_call(zmq_error) and halt_event.is_set():
log.info("program teminates normally with interrupted system call")
else:
log.exception("zeromq error processing request")
resources.event_push_client.exception(unhandled_exception_topic,
"zeromq_error",
exctype="ZMQError")
return_value = 1
except Exception as instance:
log.exception("error processing request")
resources.event_push_client.exception(unhandled_exception_topic,
str(instance),
exctype=instance.__class__.__name__)
return_value = 1
else:
log.info("program teminates normally")
finally:
for worker_process in worker_processes:
terminate_subprocess(worker_process)
resources.pull_socket.close()
resources.io_controller_push_socket.close()
resources.router_socket.close()
for push_socket in resources.reply_push_sockets.values():
push_socket.close()
resources.event_push_client.close()
zeromq_context.term()
return return_value
