def main():
"""
main entry point
"""
args = _parse_commandline()
_initialize_logging(args.verbose)
for address in [args.zmq_pub_socket_address, args.zmq_pull_socket_address]:
if is_ipc_protocol(address):
prepare_ipc_path(address)
topic_bytes = args.topic.encode("utf-8")
context = zmq.Context()
pub_socket = context.socket(zmq.PUB)
_log.info("binding pub_socket to {0}".format(args.zmq_pub_socket_address))
pub_socket.bind(args.zmq_pub_socket_address)
pub_socket.setsockopt(zmq.HWM, args.hwm)
pull_socket = context.socket(zmq.PULL)
_log.info("binding pull_socket to {0}".format(args.zmq_pull_socket_address))
pull_socket.bind(args.zmq_pull_socket_address)
poller = zmq.Poller()
poller.register(pull_socket, zmq.POLLIN)
halt_event = set_signal_handler()
while not halt_event.is_set():
try:
result = dict(poller.poll())
except zmq.ZMQError:
instance = sys.exc_info()[1]
if instance.errno == errno.EINTR and halt_event.is_set():
break
raise
_log.debug("poller received {0}".format(result))
if pull_socket in result and result[pull_socket] == zmq.POLLIN:
# we expect a compressed header followed by a compressed body
header = pull_socket.recv()
assert pull_socket.rcvmore
body = pull_socket.recv()
assert not pull_socket.rcvmore
# send out what we got in, preceded by the pub topic
pub_socket.send(topic_bytes, zmq.SNDMORE)
pub_socket.send(header, zmq.SNDMORE)
pub_socket.send(body)
_log.info("shutting down")
pub_socket.close()
pull_socket.close()
context.term()
return 0
def main():
"""
main entry point
"""
returncode = 0
args = _parse_commandline()
context = zmq.Context()
_initialize_logging()
if is_ipc_protocol(args.data_source_address):
prepare_ipc_path(args.data_source_address)
pull_socket = context.socket(zmq.PULL)
pull_socket.connect(args.data_source_address)
halt_event = set_signal_handler()
while not halt_event.is_set():
try:
line_number_bytes = pull_socket.recv()
except zmq.ZMQError:
instance = sys.exc_info()[1]
if instance.errno == errno.EINTR and halt_event.is_set():
break
raise
assert pull_socket.rcvmore
line_number = int(line_number_bytes.decode("utf-8"))
log_text_bytes = pull_socket.recv()
assert not pull_socket.rcvmore
# 2013-04-15 dougfort -- the logging StreamHandler really wants
# ascii, and it goes into an endless exception loop if it
# gets non ascii characters in utf-8
try:
log_text = log_text_bytes.decode("ascii")
except Exception:
log_text = "??? not ascii ???"
_log.info("{0:08} {1}".format(line_number, log_text))
pull_socket.close()
context.term()
return returncode
def main():
"""
main entry point
"""
args = _parse_commandline()
_initialize_logging(args.verbose)
_log.debug("program starts")
if is_ipc_protocol(args.push_socket_address):
prepare_ipc_path(args.push_socket_address)
context = zmq.Context()
push_socket = context.socket(zmq.PUSH)
_log.info("binding push socket to {0}".format(args.push_socket_address))
push_socket.bind(args.push_socket_address)
halt_event = set_signal_handler()
line_count = 0
_log.debug("opening {0}".format(args.source_path))
with open(args.source_path, "r") as input_file:
for line in input_file.readlines():
halt_event.wait(args.interval)
if halt_event.is_set():
_log.warn("breaking read loop: halt_event is set")
break
line_count += 1
try:
push_socket.send(str(line_count).encode("utf-8"), zmq.SNDMORE)
push_socket.send(line[:-1].encode("utf-8"))
except zmq.ZMQError:
instance = sys.exc_info()[1]
if is_interrupted_system_call(instance) and halt_event.is_set():
pass
else:
raise
_log.debug("shutting down: published {0} lines".format(line_count))
push_socket.close()
context.term()
return 0
def __init__(self, zmq_context, log_path):
"""
zmq_context
zeromq context
used for creating sockets in init: we do not hold a reference
log_path
The filename that the program would log to if it were using normal
file based logging.
This may include slashes to indicate directories.
Used as a key to identify this log.
"""
self._log_path = log_path
self._nodename = os.environ.get("ZMQ_LOG_NODE_NAME")
self._uuid = uuid.uuid4()
self._sequence = 0
self._push_sockets = list()
for address in os.environ["PYTHON_ZMQ_LOG_HANDLER"].split():
if is_ipc_protocol(address):
prepare_ipc_path(address)
push_socket = zmq_context.socket(zmq.PUSH)
push_socket.setsockopt(zmq.LINGER, 5000)
push_socket.connect(address)
self._push_sockets.append(push_socket)
def main():
"""
main entry point
"""
args = _parse_commandline()
if not os.path.isdir(args.log_dir):
os.makedirs(args.log_dir)
_initialize_log(args)
log = logging.getLogger("main")
log.info("program starts")
header_filters = _create_header_filters(args)
body_filter = _create_body_filter(args)
if is_ipc_protocol(args.zmq_sub_address):
prepare_ipc_path(args.zmq_sub_address)
identity_bytes = args.zmq_identity.encode("utf-8")
context = zmq.Context()
sub_socket = context.socket(zmq.SUB)
sub_socket.connect(args.zmq_sub_address)
sub_socket.setsockopt(zmq.IDENTITY, identity_bytes)
sub_socket.setsockopt(zmq.SUBSCRIBE, "".encode("utf-8"))
log_handlers = dict()
halt_event = set_signal_handler()
while not halt_event.is_set():
try:
header, body = _get_one_message(sub_socket)
except zmq.ZMQError:
instance = sys.exc_info()[1]
if instance.errno == errno.EINTR and halt_event.is_set():
break
raise
log.debug("received {0}".format(header))
if not all([f(header) for f in header_filters]):
log.debug("header does not pass filters {0}".format(header))
continue
if not body_filter(body):
log.debug("body does not pass filters {0}".format(body))
continue
log_filename = _compute_log_filename(args, header)
if not log_filename in log_handlers:
log_handlers[log_filename] = _create_log_handler(args,
log_filename)
log_handler = log_handlers[log_filename]
log_record = DummyLogRecord(body)
log_handler.emit(log_record)
log_handler.flush()
log.info("program shutting down")
sub_socket.close()
context.term()
for log_handler in log_handlers.values():
log_handler.close()
return 0
def main():
"""
main entry point
"""
args = _parse_commandline()
_initialize_logging(args.verbose)
sub_address_list = _load_sub_list(args.sub_list_path)
for address in sub_address_list:
if is_ipc_protocol(address):
prepare_ipc_path(address)
for directory in [args.output_work_dir, args.output_complete_dir, ]:
if not os.path.isdir(directory):
_log.info("creating {0}".format(directory))
os.makedirs(directory)
context = zmq.Context()
poller = zmq.Poller()
sub_socket_list = list()
for sub_socket_address in sub_address_list:
sub_socket = context.socket(zmq.SUB)
sub_socket.setsockopt(zmq.SUBSCRIBE, "".encode("utf-8"))
_log.info("connecting sub_socket to {0}".format(sub_socket_address))
sub_socket.connect(sub_socket_address)
poller.register(sub_socket, zmq.POLLIN)
sub_socket_list.append(sub_socket)
stream_writer = LogStreamWriter(args.output_prefix,
args.output_suffix,
args.granularity,
args.output_work_dir,
args.output_complete_dir)
halt_event = set_signal_handler()
polling_intgerval_milliseconds = args.polling_interval * 1000
while not halt_event.is_set():
try:
result_list = poller.poll(polling_intgerval_milliseconds)
except zmq.ZMQError:
instance = sys.exc_info()[1]
if instance.errno == errno.EINTR and halt_event.is_set():
break
raise
if len(result_list) == 0:
stream_writer.check_for_rollover()
continue
for sub_socket, event in result_list:
assert event == zmq.POLLIN, event
# we expect topic, compressed header, compressed body
_topic = sub_socket.recv()
assert sub_socket.rcvmore
compressed_header = sub_socket.recv()
assert sub_socket.rcvmore
compressed_data = sub_socket.recv()
assert not sub_socket.rcvmore
# send out what we got in
header = zlib.decompress(compressed_header)
data = zlib.decompress(compressed_data)
stream_writer.write(header, data)
_log.debug("shutting down")
for sub_socket in sub_socket_list:
sub_socket.close()
context.term()
return 0
def main():
"""
main entry point
"""
args = _parse_commandline()
_initialize_logging(args.verbose)
sub_address_list = _load_sub_list(args.sub_list_path)
for address in sub_address_list:
if is_ipc_protocol(address):
prepare_ipc_path(address)
if is_ipc_protocol(args.zmq_pub_socket_address):
prepare_ipc_path(args.zmq_pub_socket_address)
context = zmq.Context()
pub_socket = context.socket(zmq.PUB)
_log.info("binding pub socket to {0}".format(args.zmq_pub_socket_address))
pub_socket.bind(args.zmq_pub_socket_address)
pub_socket.setsockopt(zmq.HWM, args.hwm)
poller = zmq.Poller()
sub_socket_list = list()
for sub_socket_address in sub_address_list:
sub_socket = context.socket(zmq.SUB)
sub_socket.setsockopt(zmq.SUBSCRIBE, "".encode("utf-8"))
_log.info("connecting sub_socket to {0}".format(sub_socket_address))
sub_socket.connect(sub_socket_address)
poller.register(sub_socket, zmq.POLLIN)
sub_socket_list.append(sub_socket)
halt_event = set_signal_handler()
while not halt_event.is_set():
try:
result_list = poller.poll()
except zmq.ZMQError:
instance = sys.exc_info()[1]
if instance.errno == errno.EINTR and halt_event.is_set():
break
raise
for sub_socket, event in result_list:
assert event == zmq.POLLIN, event
_log.debug("traffic on socket {0}".format(sub_socket))
# we expect topic, compressed header, compressed body
topic = sub_socket.recv()
assert sub_socket.rcvmore
header = sub_socket.recv()
assert sub_socket.rcvmore
body = sub_socket.recv()
assert not sub_socket.rcvmore
# send out what we got in
pub_socket.send(topic, zmq.SNDMORE)
pub_socket.send(header, zmq.SNDMORE)
pub_socket.send(body)
_log.debug("shutting down")
pub_socket.close()
for sub_socket in sub_socket_list:
sub_socket.close()
context.term()
return 0
