def main():
    """
    main entry point
    """
    args = _parse_commandline()

    _initialize_logging(args.verbose)

    for address in [args.zmq_pub_socket_address, args.zmq_pull_socket_address]:
        if is_ipc_protocol(address):
            prepare_ipc_path(address)

    topic_bytes = args.topic.encode("utf-8")

    context = zmq.Context()

    pub_socket = context.socket(zmq.PUB)
    _log.info("binding pub_socket to {0}".format(args.zmq_pub_socket_address))
    pub_socket.bind(args.zmq_pub_socket_address)
    pub_socket.setsockopt(zmq.HWM, args.hwm)

    pull_socket = context.socket(zmq.PULL)
    _log.info("binding pull_socket to {0}".format(args.zmq_pull_socket_address))
    pull_socket.bind(args.zmq_pull_socket_address)

    poller = zmq.Poller()
    poller.register(pull_socket, zmq.POLLIN)

    halt_event = set_signal_handler()
    while not halt_event.is_set():

        try:
            result = dict(poller.poll())
        except zmq.ZMQError:
            instance = sys.exc_info()[1]
            if instance.errno == errno.EINTR and halt_event.is_set():
                break
            raise
        _log.debug("poller received {0}".format(result))

        if pull_socket in result and result[pull_socket] == zmq.POLLIN:

            # we expect a compressed header followed by a compressed body
            header = pull_socket.recv()
            assert pull_socket.rcvmore
            body = pull_socket.recv()
            assert not pull_socket.rcvmore

            # send out what we got in, preceded by the pub topic
            pub_socket.send(topic_bytes, zmq.SNDMORE)
            pub_socket.send(header, zmq.SNDMORE)
            pub_socket.send(body)

    _log.info("shutting down")
    pub_socket.close()
    pull_socket.close()
    context.term()
    return 0
Пример #2
0
def main():
    """
    main entry point
    """
    returncode = 0
    args = _parse_commandline()
    context = zmq.Context()
    
    _initialize_logging()

    if is_ipc_protocol(args.data_source_address):
        prepare_ipc_path(args.data_source_address)

    pull_socket = context.socket(zmq.PULL)
    pull_socket.connect(args.data_source_address)

    halt_event = set_signal_handler()
    while not halt_event.is_set():

        try:
            line_number_bytes = pull_socket.recv()
        except zmq.ZMQError:
            instance = sys.exc_info()[1]
            if instance.errno == errno.EINTR and halt_event.is_set():
                break
            raise

        assert pull_socket.rcvmore
        line_number = int(line_number_bytes.decode("utf-8"))

        log_text_bytes = pull_socket.recv()
        assert not pull_socket.rcvmore

        # 2013-04-15 dougfort -- the logging StreamHandler really wants
        # ascii, and it goes into an endless exception loop if it
        # gets non ascii characters in utf-8
        try:
            log_text = log_text_bytes.decode("ascii")
        except Exception:
            log_text = "??? not ascii ???"
        
        _log.info("{0:08} {1}".format(line_number, log_text))

    pull_socket.close()
    context.term()

    return returncode
Пример #3
0
def main():
    """
    main entry point
    """
    args = _parse_commandline()

    _initialize_logging(args.verbose)
    _log.debug("program starts")

    if is_ipc_protocol(args.push_socket_address):
        prepare_ipc_path(args.push_socket_address)

    context = zmq.Context()

    push_socket = context.socket(zmq.PUSH)
    _log.info("binding push socket to {0}".format(args.push_socket_address))
    push_socket.bind(args.push_socket_address)

    halt_event = set_signal_handler()
    line_count = 0
    _log.debug("opening {0}".format(args.source_path))
    with open(args.source_path, "r") as input_file:
        for line in input_file.readlines():
            halt_event.wait(args.interval)
            if halt_event.is_set():
                _log.warn("breaking read loop: halt_event is set")
                break
            line_count += 1
            try:
                push_socket.send(str(line_count).encode("utf-8"), zmq.SNDMORE)
                push_socket.send(line[:-1].encode("utf-8"))
            except zmq.ZMQError:
                instance = sys.exc_info()[1]
                if is_interrupted_system_call(instance) and halt_event.is_set():
                    pass
                else:
                    raise

    _log.debug("shutting down: published {0} lines".format(line_count))
    push_socket.close()
    context.term()
    return 0
Пример #4
0
    def __init__(self, zmq_context, log_path):
        """
        zmq_context
            zeromq context
            used for creating sockets in init: we do not hold a reference

        log_path
            The filename that the program would log to if it were using normal 
            file based logging. 
            This may include slashes to indicate directories.
            Used as a key to identify this log.
        """
        self._log_path = log_path
        self._nodename = os.environ.get("ZMQ_LOG_NODE_NAME")
        self._uuid = uuid.uuid4()
        self._sequence = 0
        self._push_sockets = list()
        for address in os.environ["PYTHON_ZMQ_LOG_HANDLER"].split():
            if is_ipc_protocol(address):
                prepare_ipc_path(address)
            push_socket = zmq_context.socket(zmq.PUSH)
            push_socket.setsockopt(zmq.LINGER, 5000)
            push_socket.connect(address)
            self._push_sockets.append(push_socket)
Пример #5
0
def main():
    """
    main entry point
    """
    args = _parse_commandline()

    if not os.path.isdir(args.log_dir):
        os.makedirs(args.log_dir)

    _initialize_log(args)
    log = logging.getLogger("main")
    log.info("program starts")

    header_filters = _create_header_filters(args)
    body_filter = _create_body_filter(args)

    if is_ipc_protocol(args.zmq_sub_address):
        prepare_ipc_path(args.zmq_sub_address)

    identity_bytes = args.zmq_identity.encode("utf-8")

    context = zmq.Context()

    sub_socket = context.socket(zmq.SUB)
    sub_socket.connect(args.zmq_sub_address)
    sub_socket.setsockopt(zmq.IDENTITY, identity_bytes)
    sub_socket.setsockopt(zmq.SUBSCRIBE, "".encode("utf-8"))

    log_handlers = dict()
    halt_event = set_signal_handler()
    while not halt_event.is_set():

        try:
            header, body = _get_one_message(sub_socket)
        except zmq.ZMQError:
            instance = sys.exc_info()[1]
            if instance.errno == errno.EINTR and halt_event.is_set():
                break
            raise
        log.debug("received {0}".format(header))

        if not all([f(header) for f in header_filters]):
            log.debug("header does not pass filters {0}".format(header))
            continue

        if not body_filter(body):
            log.debug("body does not pass filters {0}".format(body))
            continue

        log_filename = _compute_log_filename(args, header)
        if not log_filename in log_handlers:
            log_handlers[log_filename] = _create_log_handler(args, 
                                                             log_filename)
        log_handler = log_handlers[log_filename]
        log_record = DummyLogRecord(body)
        log_handler.emit(log_record)
        log_handler.flush()

    log.info("program shutting down")
    sub_socket.close()
    context.term()

    for log_handler in log_handlers.values():
        log_handler.close()

    return 0
def main():
    """
    main entry point
    """
    args = _parse_commandline()

    _initialize_logging(args.verbose)

    sub_address_list = _load_sub_list(args.sub_list_path)
    for address in sub_address_list:
        if is_ipc_protocol(address):
            prepare_ipc_path(address)

    for directory in [args.output_work_dir, args.output_complete_dir, ]:
        if not os.path.isdir(directory):
            _log.info("creating {0}".format(directory))
            os.makedirs(directory) 

    context = zmq.Context()

    poller = zmq.Poller()

    sub_socket_list = list()
    for sub_socket_address in sub_address_list:
        sub_socket = context.socket(zmq.SUB)
        sub_socket.setsockopt(zmq.SUBSCRIBE, "".encode("utf-8"))
        _log.info("connecting sub_socket to {0}".format(sub_socket_address))
        sub_socket.connect(sub_socket_address)
        poller.register(sub_socket, zmq.POLLIN)
        sub_socket_list.append(sub_socket)

    stream_writer = LogStreamWriter(args.output_prefix,
                                    args.output_suffix,
                                    args.granularity,
                                    args.output_work_dir,
                                    args.output_complete_dir)

    halt_event = set_signal_handler()
    polling_intgerval_milliseconds = args.polling_interval * 1000
    while not halt_event.is_set():

        try:
            result_list = poller.poll(polling_intgerval_milliseconds)
        except zmq.ZMQError:
            instance = sys.exc_info()[1]
            if instance.errno == errno.EINTR and halt_event.is_set():
                break
            raise

        if len(result_list) == 0:
            stream_writer.check_for_rollover()
            continue

        for sub_socket, event in result_list: 
            assert event == zmq.POLLIN, event

            # we expect topic, compressed header, compressed body
            _topic = sub_socket.recv()
            assert sub_socket.rcvmore
            compressed_header = sub_socket.recv()
            assert sub_socket.rcvmore
            compressed_data = sub_socket.recv()
            assert not sub_socket.rcvmore

            # send out what we got in
            header = zlib.decompress(compressed_header)
            data = zlib.decompress(compressed_data)

            stream_writer.write(header, data)

    _log.debug("shutting down")
    for sub_socket in sub_socket_list:
        sub_socket.close()
    context.term()
    return 0
def main():
    """
    main entry point
    """
    args = _parse_commandline()

    _initialize_logging(args.verbose)

    sub_address_list = _load_sub_list(args.sub_list_path)
    for address in sub_address_list:
        if is_ipc_protocol(address):
            prepare_ipc_path(address)

    if is_ipc_protocol(args.zmq_pub_socket_address):
        prepare_ipc_path(args.zmq_pub_socket_address)

    context = zmq.Context()

    pub_socket = context.socket(zmq.PUB)
    _log.info("binding pub socket to {0}".format(args.zmq_pub_socket_address))
    pub_socket.bind(args.zmq_pub_socket_address)
    pub_socket.setsockopt(zmq.HWM, args.hwm)

    poller = zmq.Poller()

    sub_socket_list = list()
    for sub_socket_address in sub_address_list:
        sub_socket = context.socket(zmq.SUB)
        sub_socket.setsockopt(zmq.SUBSCRIBE, "".encode("utf-8"))
        _log.info("connecting sub_socket to {0}".format(sub_socket_address))
        sub_socket.connect(sub_socket_address)
        poller.register(sub_socket, zmq.POLLIN)
        sub_socket_list.append(sub_socket)

    halt_event = set_signal_handler()
    while not halt_event.is_set():

        try:
            result_list = poller.poll()
        except zmq.ZMQError:
            instance = sys.exc_info()[1]
            if instance.errno == errno.EINTR and halt_event.is_set():
                break
            raise

        for sub_socket, event in result_list: 
            assert event == zmq.POLLIN, event

            _log.debug("traffic on socket {0}".format(sub_socket))

            # we expect topic, compressed header, compressed body
            topic = sub_socket.recv()
            assert sub_socket.rcvmore
            header = sub_socket.recv()
            assert sub_socket.rcvmore
            body = sub_socket.recv()
            assert not sub_socket.rcvmore

            # send out what we got in
            pub_socket.send(topic, zmq.SNDMORE)
            pub_socket.send(header, zmq.SNDMORE)
            pub_socket.send(body)

    _log.debug("shutting down")
    pub_socket.close()
    for sub_socket in sub_socket_list:
        sub_socket.close()
    context.term()
    return 0