def scribe_get_from_time(self, scribe_env, stream_name, start_time, end_time): # Scribe connection details host_and_port = scribereader.get_env_scribe_host(scribe_env, True) host = host_and_port['host'] port = host_and_port['port'] # Annoyingly enough, scribe needs special handling if we're trying to retrieve logs from today. # There is a datetime.date.today but we can't use it because it isn't UTC today = datetime.datetime.utcnow().date() if start_time.date() == today or end_time.date() == today: # The reason we need a fake context here is because scribereader is a bit incosistent in its # returns. get_stream_reader returns a context that needs to be acquired for cleanup code but # get_stream_tailer simply returns an object that can be iterated over. We'd still like to have # the cleanup code for get_stream_reader to be executed by this function's caller and this is # one of the simpler ways to achieve it without having 2 if statements everywhere that calls # this method @contextmanager def fake_context(): log.info( "Running the equivalent of 'scribereader -f -e %s %s" % (scribe_env, stream_name)) yield scribereader.get_stream_tailer(stream_name, host, port, True, -1) return fake_context() log.info("Running the equivalent of 'scribereader -e %s %s" % (scribe_env, stream_name)) return scribereader.get_stream_reader(stream_name, host, port, start_time, end_time)
def scribe_tail(scribe_env, stream_name, service, levels, components, clusters, queue, filter_fn, parse_fn=None): """Creates a scribetailer for a particular environment. When it encounters a line that it should report, it sticks it into the provided queue. This code is designed to run in a thread as spawned by tail_paasta_logs(). """ try: log.debug("Going to tail %s scribe stream in %s" % (stream_name, scribe_env)) host_and_port = scribereader.get_env_scribe_host(scribe_env, True) host = host_and_port['host'] port = host_and_port['port'] tailer = scribereader.get_stream_tailer(stream_name, host, port) for line in tailer: if parse_fn: line = parse_fn(line, clusters) if filter_fn(line, levels, service, components, clusters): queue.put(line) except KeyboardInterrupt: # Die peacefully rather than printing N threads worth of stack # traces. pass except StreamTailerSetupError: log.error("Failed to setup stream tailing for %s in %s" % (stream_name, scribe_env)) log.error("Don't Panic! This can happen the first time a service is deployed because the log") log.error("doesn't exist yet. Please wait for the service to be deployed in %s and try again." % scribe_env) raise
def scribe_get_from_time(self, scribe_env, stream_name, start_time, end_time): # Scribe connection details host_and_port = scribereader.get_env_scribe_host(scribe_env, tail=False) host = host_and_port['host'] port = host_and_port['port'] # Recent logs might not be archived yet. Log warning message. warning_end_time = datetime.datetime.utcnow().replace( tzinfo=pytz.utc) - datetime.timedelta(hours=4) if end_time > warning_end_time: log.warn( "Recent logs might be incomplete. Consider tailing instead.") # scribereader, sadly, is not based on UTC timestamps. It uses YST # dates instead. start_date_yst = start_time.astimezone( pytz.timezone('America/Los_Angeles')).date() end_date_yst = end_time.astimezone( pytz.timezone('America/Los_Angeles')).date() log.debug( "Running the equivalent of 'scribereader -e %s %s --min-date %s --max-date %s" % (scribe_env, stream_name, start_date_yst, end_date_yst)) return scribereader.get_stream_reader( stream_name=stream_name, reader_host=host, reader_port=port, min_date=start_date_yst, max_date=end_date_yst, )
def scribe_get_from_time(self, scribe_env, stream_name, start_time, end_time): # Scribe connection details host_and_port = scribereader.get_env_scribe_host(scribe_env, True) host = host_and_port["host"] port = host_and_port["port"] # Annoyingly enough, scribe needs special handling if we're trying to retrieve logs from today. # There is a datetime.date.today but we can't use it because it isn't UTC today = datetime.datetime.utcnow().date() if start_time.date() == today or end_time.date() == today: # The reason we need a fake context here is because scribereader is a bit incosistent in its # returns. get_stream_reader returns a context that needs to be acquired for cleanup code but # get_stream_tailer simply returns an object that can be iterated over. We'd still like to have # the cleanup code for get_stream_reader to be executed by this function's caller and this is # one of the simpler ways to achieve it without having 2 if statements everywhere that calls # this method @contextmanager def fake_context(): log.info("Running the equivalent of 'scribereader -f -e %s %s" % (scribe_env, stream_name)) yield scribereader.get_stream_tailer(stream_name, host, port, True, -1) return fake_context() log.info("Running the equivalent of 'scribereader -e %s %s" % (scribe_env, stream_name)) return scribereader.get_stream_reader(stream_name, host, port, start_time, end_time)
def scribe_tail(self, scribe_env, stream_name, service, levels, components, clusters, queue, filter_fn, parse_fn=None): """Creates a scribetailer for a particular environment. When it encounters a line that it should report, it sticks it into the provided queue. This code is designed to run in a thread as spawned by tail_paasta_logs(). """ try: log.debug("Going to tail %s scribe stream in %s" % (stream_name, scribe_env)) host_and_port = scribereader.get_env_scribe_host(scribe_env, True) host = host_and_port['host'] port = host_and_port['port'] tailer = scribereader.get_stream_tailer(stream_name, host, port) for line in tailer: if parse_fn: line = parse_fn(line, clusters, service) if filter_fn(line, levels, service, components, clusters): queue.put(line) except KeyboardInterrupt: # Die peacefully rather than printing N threads worth of stack # traces. pass except StreamTailerSetupError as e: if 'No data in stream' in e.message: log.warning("Scribe stream %s is empty on %s" % (stream_name, scribe_env)) log.warning("Don't Panic! This may or may not be a problem depending on if you expect there to be") log.warning("output within this stream.") # Enter a wait so the process isn't considered dead. # This is just a large number, since apparently some python interpreters # don't like being passed sys.maxsize. sleep(2**16) else: raise
async def get_slack_events(): if scribereader is None: logging.error("Scribereader unavailable. Not tailing slack events.") return host_and_port = scribereader.get_env_scribe_host(SCRIBE_ENV, True) host = host_and_port["host"] port = host_and_port["port"] while True: reader, writer = await asyncio.open_connection(host=host, port=port) writer.write( construct_conn_msg(stream=SLACK_WEBHOOK_STREAM).encode("utf-8")) await writer.drain() while True: line = await reader.readline() log.debug(f"got log line {line.decode('utf-8')}") if not line: break event = parse_webhook_event_json(line) if is_relevant_event(event): yield event log.warning("Lost connection to scribe host; reconnecting")
def scribe_tail(self, scribe_env, stream_name, service, levels, components, clusters, instances, queue, filter_fn, parse_fn=None): """Creates a scribetailer for a particular environment. When it encounters a line that it should report, it sticks it into the provided queue. This code is designed to run in a thread as spawned by tail_paasta_logs(). """ try: log.debug("Going to tail %s scribe stream in %s" % (stream_name, scribe_env)) host_and_port = scribereader.get_env_scribe_host(scribe_env, True) host = host_and_port['host'] port = host_and_port['port'] tailer = scribereader.get_stream_tailer(stream_name, host, port) for line in tailer: if parse_fn: line = parse_fn(line, clusters, service) if filter_fn(line, levels, service, components, clusters, instances): queue.put(line) except KeyboardInterrupt: # Die peacefully rather than printing N threads worth of stack # traces. pass except StreamTailerSetupError as e: if 'No data in stream' in e.message: log.warning("Scribe stream %s is empty on %s" % (stream_name, scribe_env)) log.warning("Don't Panic! This may or may not be a problem depending on if you expect there to be") log.warning("output within this stream.") # Enter a wait so the process isn't considered dead. # This is just a large number, since apparently some python interpreters # don't like being passed sys.maxsize. sleep(2**16) else: raise
def scribe_tail(queue): host_and_port = scribereader.get_env_scribe_host(SCRIBE_ENV, True) host = host_and_port['host'] port = host_and_port['port'] tailer = scribereader.get_stream_tailer(SLACK_WEBHOOK_STREAM, host, port) for line in tailer: queue.put(line)
def scribe_get_last_n_lines(self, scribe_env, stream_name, line_count): # Scribe connection details host_and_port = scribereader.get_env_scribe_host(scribe_env, True) host = host_and_port["host"] port = host_and_port["port"] # Please read comment above in scribe_get_from_time as to why a fake # context here is necessary @contextmanager def fake_context(): yield scribereader.get_stream_tailer(stream_name, host, port, True, line_count) return fake_context()
def scribe_get_last_n_lines(self, scribe_env, stream_name, line_count): # Scribe connection details host_and_port = scribereader.get_env_scribe_host(scribe_env, True) host = host_and_port['host'] port = host_and_port['port'] # Please read comment above in scribe_get_from_time as to why a fake # context here is necessary @contextmanager def fake_context(): yield scribereader.get_stream_tailer(stream_name, host, port, True, line_count) return fake_context()
def scribe_get_last_n_lines(self, scribe_env, stream_name, line_count): # Scribe connection details host_and_port = scribereader.get_env_scribe_host(scribe_env, tail=True) host = host_and_port['host'] port = host_and_port['port'] # The reason we need a fake context here is because scribereader is a bit inconsistent in its # returns. get_stream_reader returns a context that needs to be acquired for cleanup code but # get_stream_tailer simply returns an object that can be iterated over. We'd still like to have # the cleanup code for get_stream_reader to be executed by this function's caller and this is # one of the simpler ways to achieve it without having 2 if statements everywhere that calls # this method @contextmanager def fake_context(): log.debug(f"Running the equivalent of 'scribereader -e {scribe_env} {stream_name}'") yield scribereader.get_stream_tailer(stream_name, host, port, True, line_count) return fake_context()
def scribe_tail(self, scribe_env, stream_name, service, levels, components, clusters, queue, filter_fn, parse_fn=None): """Creates a scribetailer for a particular environment. When it encounters a line that it should report, it sticks it into the provided queue. This code is designed to run in a thread as spawned by tail_paasta_logs(). """ try: log.debug("Going to tail %s scribe stream in %s" % (stream_name, scribe_env)) host_and_port = scribereader.get_env_scribe_host(scribe_env, True) host = host_and_port['host'] port = host_and_port['port'] tailer = scribereader.get_stream_tailer(stream_name, host, port) for line in tailer: if parse_fn: line = parse_fn(line, clusters, service) if filter_fn(line, levels, service, components, clusters): queue.put(line) except KeyboardInterrupt: # Die peacefully rather than printing N threads worth of stack # traces. pass except StreamTailerSetupError: log.error("Failed to setup stream tailing for %s in %s" % (stream_name, scribe_env)) log.error( "Don't Panic! This can happen the first time a service is deployed because the log" ) log.error( "doesn't exist yet. Please wait for the service to be deployed in %s and try again." % scribe_env) raise