def main():
"""
main entry point
"""
args = _parse_commandline()
_initialize_logging(args.verbose)
for address in [args.zmq_pub_socket_address, args.zmq_pull_socket_address]:
if is_ipc_protocol(address):
prepare_ipc_path(address)
topic_bytes = args.topic.encode("utf-8")
context = zmq.Context()
pub_socket = context.socket(zmq.PUB)
_log.info("binding pub_socket to {0}".format(args.zmq_pub_socket_address))
pub_socket.bind(args.zmq_pub_socket_address)
pub_socket.setsockopt(zmq.HWM, args.hwm)
pull_socket = context.socket(zmq.PULL)
_log.info("binding pull_socket to {0}".format(args.zmq_pull_socket_address))
pull_socket.bind(args.zmq_pull_socket_address)
poller = zmq.Poller()
poller.register(pull_socket, zmq.POLLIN)
halt_event = set_signal_handler()
while not halt_event.is_set():
try:
result = dict(poller.poll())
except zmq.ZMQError:
instance = sys.exc_info()[1]
if instance.errno == errno.EINTR and halt_event.is_set():
break
raise
_log.debug("poller received {0}".format(result))
if pull_socket in result and result[pull_socket] == zmq.POLLIN:
# we expect a compressed header followed by a compressed body
header = pull_socket.recv()
assert pull_socket.rcvmore
body = pull_socket.recv()
assert not pull_socket.rcvmore
# send out what we got in, preceded by the pub topic
pub_socket.send(topic_bytes, zmq.SNDMORE)
pub_socket.send(header, zmq.SNDMORE)
pub_socket.send(body)
_log.info("shutting down")
pub_socket.close()
pull_socket.close()
context.term()
return 0
def main():
"""
main entry point
"""
returncode = 0
args = _parse_commandline()
context = zmq.Context()
_initialize_logging()
if is_ipc_protocol(args.data_source_address):
prepare_ipc_path(args.data_source_address)
pull_socket = context.socket(zmq.PULL)
pull_socket.connect(args.data_source_address)
halt_event = set_signal_handler()
while not halt_event.is_set():
try:
line_number_bytes = pull_socket.recv()
except zmq.ZMQError:
instance = sys.exc_info()[1]
if instance.errno == errno.EINTR and halt_event.is_set():
break
raise
assert pull_socket.rcvmore
line_number = int(line_number_bytes.decode("utf-8"))
log_text_bytes = pull_socket.recv()
assert not pull_socket.rcvmore
# 2013-04-15 dougfort -- the logging StreamHandler really wants
# ascii, and it goes into an endless exception loop if it
# gets non ascii characters in utf-8
try:
log_text = log_text_bytes.decode("ascii")
except Exception:
log_text = "??? not ascii ???"
_log.info("{0:08} {1}".format(line_number, log_text))
pull_socket.close()
context.term()
return returncode
def main():
"""
main entry point
"""
args = _parse_commandline()
_initialize_logging(args.verbose)
_log.debug("program starts")
if is_ipc_protocol(args.push_socket_address):
prepare_ipc_path(args.push_socket_address)
context = zmq.Context()
push_socket = context.socket(zmq.PUSH)
_log.info("binding push socket to {0}".format(args.push_socket_address))
push_socket.bind(args.push_socket_address)
halt_event = set_signal_handler()
line_count = 0
_log.debug("opening {0}".format(args.source_path))
with open(args.source_path, "r") as input_file:
for line in input_file.readlines():
halt_event.wait(args.interval)
if halt_event.is_set():
_log.warn("breaking read loop: halt_event is set")
break
line_count += 1
try:
push_socket.send(str(line_count).encode("utf-8"), zmq.SNDMORE)
push_socket.send(line[:-1].encode("utf-8"))
except zmq.ZMQError:
instance = sys.exc_info()[1]
if is_interrupted_system_call(instance) and halt_event.is_set():
pass
else:
raise
_log.debug("shutting down: published {0} lines".format(line_count))
push_socket.close()
context.term()
return 0
def main():
"""
main entry point
"""
args = _parse_commandline()
zmq_context = zmq.Context()
log_line_pusher = LogLinePusher(zmq_context, args.log_path)
halt_event = set_signal_handler()
while not halt_event.is_set():
try:
line = sys.stdin.readline()
except IOError:
instance = sys.exc_info()[1]
if instance.errno == errno.EINTR and halt_event.is_set():
break
raise
log_line_pusher.push_log_line(line[:-1])
log_line_pusher.close()
zmq_context.term()
return 0
def main():
"""
main entry point
"""
args = _parse_commandline()
if not os.path.isdir(args.log_dir):
os.makedirs(args.log_dir)
_initialize_log(args)
log = logging.getLogger("main")
log.info("program starts")
header_filters = _create_header_filters(args)
body_filter = _create_body_filter(args)
if is_ipc_protocol(args.zmq_sub_address):
prepare_ipc_path(args.zmq_sub_address)
identity_bytes = args.zmq_identity.encode("utf-8")
context = zmq.Context()
sub_socket = context.socket(zmq.SUB)
sub_socket.connect(args.zmq_sub_address)
sub_socket.setsockopt(zmq.IDENTITY, identity_bytes)
sub_socket.setsockopt(zmq.SUBSCRIBE, "".encode("utf-8"))
log_handlers = dict()
halt_event = set_signal_handler()
while not halt_event.is_set():
try:
header, body = _get_one_message(sub_socket)
except zmq.ZMQError:
instance = sys.exc_info()[1]
if instance.errno == errno.EINTR and halt_event.is_set():
break
raise
log.debug("received {0}".format(header))
if not all([f(header) for f in header_filters]):
log.debug("header does not pass filters {0}".format(header))
continue
if not body_filter(body):
log.debug("body does not pass filters {0}".format(body))
continue
log_filename = _compute_log_filename(args, header)
if not log_filename in log_handlers:
log_handlers[log_filename] = _create_log_handler(args,
log_filename)
log_handler = log_handlers[log_filename]
log_record = DummyLogRecord(body)
log_handler.emit(log_record)
log_handler.flush()
log.info("program shutting down")
sub_socket.close()
context.term()
for log_handler in log_handlers.values():
log_handler.close()
return 0
def main():
"""
main entry point
"""
args = _parse_commandline()
_initialize_logging(args.verbose)
sub_address_list = _load_sub_list(args.sub_list_path)
for address in sub_address_list:
if is_ipc_protocol(address):
prepare_ipc_path(address)
for directory in [args.output_work_dir, args.output_complete_dir, ]:
if not os.path.isdir(directory):
_log.info("creating {0}".format(directory))
os.makedirs(directory)
context = zmq.Context()
poller = zmq.Poller()
sub_socket_list = list()
for sub_socket_address in sub_address_list:
sub_socket = context.socket(zmq.SUB)
sub_socket.setsockopt(zmq.SUBSCRIBE, "".encode("utf-8"))
_log.info("connecting sub_socket to {0}".format(sub_socket_address))
sub_socket.connect(sub_socket_address)
poller.register(sub_socket, zmq.POLLIN)
sub_socket_list.append(sub_socket)
stream_writer = LogStreamWriter(args.output_prefix,
args.output_suffix,
args.granularity,
args.output_work_dir,
args.output_complete_dir)
halt_event = set_signal_handler()
polling_intgerval_milliseconds = args.polling_interval * 1000
while not halt_event.is_set():
try:
result_list = poller.poll(polling_intgerval_milliseconds)
except zmq.ZMQError:
instance = sys.exc_info()[1]
if instance.errno == errno.EINTR and halt_event.is_set():
break
raise
if len(result_list) == 0:
stream_writer.check_for_rollover()
continue
for sub_socket, event in result_list:
assert event == zmq.POLLIN, event
# we expect topic, compressed header, compressed body
_topic = sub_socket.recv()
assert sub_socket.rcvmore
compressed_header = sub_socket.recv()
assert sub_socket.rcvmore
compressed_data = sub_socket.recv()
assert not sub_socket.rcvmore
# send out what we got in
header = zlib.decompress(compressed_header)
data = zlib.decompress(compressed_data)
stream_writer.write(header, data)
_log.debug("shutting down")
for sub_socket in sub_socket_list:
sub_socket.close()
context.term()
return 0
def main():
"""
main entry point
"""
args = _parse_commandline()
_initialize_logging(args.log_path)
log = logging.getLogger("main")
log.info("program starts")
config = _load_config_file(args.config_path)
# config is a dict with keys ['global', <node-name-1>...<node-name-n>]
# not necessarily in order
node_names = list(config.keys())
node_names.remove("global")
node_names.sort()
_constuct_aggregate_sub_socket_file(config, node_names)
processes = list()
# we start the processes from back to front
# optional wormhole to nimbus.io
if "directory_wormhole" in config["global"]:
process = _start_directory_wormhole(args.directory_wormhole_path,
config["global"])
processes.append(process)
# log stream writer
process = _start_zmq_log_stream_writer(args.old_log_inn_path,
config["global"])
processes.append(process)
# the aggregator
process = _start_zmq_subscription_aggregator(args.old_log_inn_path,
config["global"])
processes.append(process)
# optional file logger
if "zmq_log_file_logger" in config["global"]:
process = \
_start_zmq_log_file_logger(
args.old_log_inn_path,
config["global"]["host_name"],
config["global"]["aggregate_pub_socket_address"],
config["global"]["zmq_log_file_logger"])
processes.append(process)
data_source_address = \
config["global"]["data_source"]["push_socket_address"]
for node_name in node_names:
process = _start_zmq_push_pub_forwarder(args.old_log_inn_path,
node_name,
config[node_name])
processes.append(process)
if "log_spewers" in config[node_name]:
for log_spewer_config in config[node_name]["log_spewers"]:
process = _start_log_spewer(args.old_log_inn_path,
node_name,
config[node_name],
log_spewer_config,
data_source_address)
processes.append(process)
if "stdout_spewers" in config[node_name]:
for stdout_spewer_config in config[node_name]["stdout_spewers"]:
# start a stdout spewer pipelined to a stdin_to_zmq_log_proxy
spewer_process = \
_start_stdout_spewer(args.old_log_inn_path,
node_name,
stdout_spewer_config,
data_source_address)
proxy_process = \
_start_stdin_to_zmq_log_proxy(args.old_log_inn_path,
node_name,
config[node_name],
stdout_spewer_config,
spewer_process.stdout)
# The p1.stdout.close() call after starting the p2 is important
# in order for p1 to receive a SIGPIPE if p2 exits before p1.
spewer_process.stdout.close()
processes.append(spewer_process)
processes.append(proxy_process)
# start the data source last, hoping everyone has had time to hook up
process = _start_data_source(args.old_log_inn_path,
config["global"]["data_source"])
processes.append(process)
data_source_finished = False
halt_event = set_signal_handler()
start_time = time.time()
while not halt_event.is_set():
elapsed_time = int(time.time() - start_time)
if elapsed_time > args.duration:
log.info("time expired {0} seconds: stopping test".format(
elapsed_time))
halt_event.set()
break
terminated_subprocesses = _poll_processes(processes)
# if the data source is done, we are done
if "data_source" in terminated_subprocesses:
log.info("data_source has terminated")
data_source_finished = True
halt_event.wait(5.0)
log.info("shutting down")
if not data_source_finished:
log.warn("data source has not finished")
for process in processes:
if not process.active:
log.warn("node {0} process {1} already terminated {2}".format(
process.node_name,
process.program_name,
process.returncode))
else:
log.debug("terminating node {0} process {1}".format(
process.node_name,
process.program_name))
process.terminate()
_check_process_termination(process)
log.info("program terminates with return code 0")
return 0
def main():
"""
main entry point
"""
args = _parse_commandline()
_initialize_logging(args.verbose)
sub_address_list = _load_sub_list(args.sub_list_path)
for address in sub_address_list:
if is_ipc_protocol(address):
prepare_ipc_path(address)
if is_ipc_protocol(args.zmq_pub_socket_address):
prepare_ipc_path(args.zmq_pub_socket_address)
context = zmq.Context()
pub_socket = context.socket(zmq.PUB)
_log.info("binding pub socket to {0}".format(args.zmq_pub_socket_address))
pub_socket.bind(args.zmq_pub_socket_address)
pub_socket.setsockopt(zmq.HWM, args.hwm)
poller = zmq.Poller()
sub_socket_list = list()
for sub_socket_address in sub_address_list:
sub_socket = context.socket(zmq.SUB)
sub_socket.setsockopt(zmq.SUBSCRIBE, "".encode("utf-8"))
_log.info("connecting sub_socket to {0}".format(sub_socket_address))
sub_socket.connect(sub_socket_address)
poller.register(sub_socket, zmq.POLLIN)
sub_socket_list.append(sub_socket)
halt_event = set_signal_handler()
while not halt_event.is_set():
try:
result_list = poller.poll()
except zmq.ZMQError:
instance = sys.exc_info()[1]
if instance.errno == errno.EINTR and halt_event.is_set():
break
raise
for sub_socket, event in result_list:
assert event == zmq.POLLIN, event
_log.debug("traffic on socket {0}".format(sub_socket))
# we expect topic, compressed header, compressed body
topic = sub_socket.recv()
assert sub_socket.rcvmore
header = sub_socket.recv()
assert sub_socket.rcvmore
body = sub_socket.recv()
assert not sub_socket.rcvmore
# send out what we got in
pub_socket.send(topic, zmq.SNDMORE)
pub_socket.send(header, zmq.SNDMORE)
pub_socket.send(body)
_log.debug("shutting down")
pub_socket.close()
for sub_socket in sub_socket_list:
sub_socket.close()
context.term()
return 0