def scribe_get_from_time(self, scribe_env, stream_name, start_time,
end_time):
# Scribe connection details
host_and_port = scribereader.get_env_scribe_host(scribe_env, True)
host = host_and_port['host']
port = host_and_port['port']
# Annoyingly enough, scribe needs special handling if we're trying to retrieve logs from today.
# There is a datetime.date.today but we can't use it because it isn't UTC
today = datetime.datetime.utcnow().date()
if start_time.date() == today or end_time.date() == today:
# The reason we need a fake context here is because scribereader is a bit incosistent in its
# returns. get_stream_reader returns a context that needs to be acquired for cleanup code but
# get_stream_tailer simply returns an object that can be iterated over. We'd still like to have
# the cleanup code for get_stream_reader to be executed by this function's caller and this is
# one of the simpler ways to achieve it without having 2 if statements everywhere that calls
# this method
@contextmanager
def fake_context():
log.info(
"Running the equivalent of 'scribereader -f -e %s %s" %
(scribe_env, stream_name))
yield scribereader.get_stream_tailer(stream_name, host, port,
True, -1)
return fake_context()
log.info("Running the equivalent of 'scribereader -e %s %s" %
(scribe_env, stream_name))
return scribereader.get_stream_reader(stream_name, host, port,
start_time, end_time)
def scribe_tail(scribe_env, stream_name, service, levels, components, clusters, queue, filter_fn, parse_fn=None):
"""Creates a scribetailer for a particular environment.
When it encounters a line that it should report, it sticks it into the
provided queue.
This code is designed to run in a thread as spawned by tail_paasta_logs().
"""
try:
log.debug("Going to tail %s scribe stream in %s" % (stream_name, scribe_env))
host_and_port = scribereader.get_env_scribe_host(scribe_env, True)
host = host_and_port['host']
port = host_and_port['port']
tailer = scribereader.get_stream_tailer(stream_name, host, port)
for line in tailer:
if parse_fn:
line = parse_fn(line, clusters)
if filter_fn(line, levels, service, components, clusters):
queue.put(line)
except KeyboardInterrupt:
# Die peacefully rather than printing N threads worth of stack
# traces.
pass
except StreamTailerSetupError:
log.error("Failed to setup stream tailing for %s in %s" % (stream_name, scribe_env))
log.error("Don't Panic! This can happen the first time a service is deployed because the log")
log.error("doesn't exist yet. Please wait for the service to be deployed in %s and try again." % scribe_env)
raise
def scribe_get_from_time(self, scribe_env, stream_name, start_time,
end_time):
# Scribe connection details
host_and_port = scribereader.get_env_scribe_host(scribe_env,
tail=False)
host = host_and_port['host']
port = host_and_port['port']
# Recent logs might not be archived yet. Log warning message.
warning_end_time = datetime.datetime.utcnow().replace(
tzinfo=pytz.utc) - datetime.timedelta(hours=4)
if end_time > warning_end_time:
log.warn(
"Recent logs might be incomplete. Consider tailing instead.")
# scribereader, sadly, is not based on UTC timestamps. It uses YST
# dates instead.
start_date_yst = start_time.astimezone(
pytz.timezone('America/Los_Angeles')).date()
end_date_yst = end_time.astimezone(
pytz.timezone('America/Los_Angeles')).date()
log.debug(
"Running the equivalent of 'scribereader -e %s %s --min-date %s --max-date %s"
% (scribe_env, stream_name, start_date_yst, end_date_yst))
return scribereader.get_stream_reader(
stream_name=stream_name,
reader_host=host,
reader_port=port,
min_date=start_date_yst,
max_date=end_date_yst,
)
def scribe_get_from_time(self, scribe_env, stream_name, start_time, end_time):
# Scribe connection details
host_and_port = scribereader.get_env_scribe_host(scribe_env, True)
host = host_and_port["host"]
port = host_and_port["port"]
# Annoyingly enough, scribe needs special handling if we're trying to retrieve logs from today.
# There is a datetime.date.today but we can't use it because it isn't UTC
today = datetime.datetime.utcnow().date()
if start_time.date() == today or end_time.date() == today:
# The reason we need a fake context here is because scribereader is a bit incosistent in its
# returns. get_stream_reader returns a context that needs to be acquired for cleanup code but
# get_stream_tailer simply returns an object that can be iterated over. We'd still like to have
# the cleanup code for get_stream_reader to be executed by this function's caller and this is
# one of the simpler ways to achieve it without having 2 if statements everywhere that calls
# this method
@contextmanager
def fake_context():
log.info("Running the equivalent of 'scribereader -f -e %s %s" % (scribe_env, stream_name))
yield scribereader.get_stream_tailer(stream_name, host, port, True, -1)
return fake_context()
log.info("Running the equivalent of 'scribereader -e %s %s" % (scribe_env, stream_name))
return scribereader.get_stream_reader(stream_name, host, port, start_time, end_time)
def scribe_tail(self, scribe_env, stream_name, service, levels, components, clusters, queue, filter_fn,
parse_fn=None):
"""Creates a scribetailer for a particular environment.
When it encounters a line that it should report, it sticks it into the
provided queue.
This code is designed to run in a thread as spawned by tail_paasta_logs().
"""
try:
log.debug("Going to tail %s scribe stream in %s" % (stream_name, scribe_env))
host_and_port = scribereader.get_env_scribe_host(scribe_env, True)
host = host_and_port['host']
port = host_and_port['port']
tailer = scribereader.get_stream_tailer(stream_name, host, port)
for line in tailer:
if parse_fn:
line = parse_fn(line, clusters, service)
if filter_fn(line, levels, service, components, clusters):
queue.put(line)
except KeyboardInterrupt:
# Die peacefully rather than printing N threads worth of stack
# traces.
pass
except StreamTailerSetupError as e:
if 'No data in stream' in e.message:
log.warning("Scribe stream %s is empty on %s" % (stream_name, scribe_env))
log.warning("Don't Panic! This may or may not be a problem depending on if you expect there to be")
log.warning("output within this stream.")
# Enter a wait so the process isn't considered dead.
# This is just a large number, since apparently some python interpreters
# don't like being passed sys.maxsize.
sleep(2**16)
else:
raise
async def get_slack_events():
if scribereader is None:
logging.error("Scribereader unavailable. Not tailing slack events.")
return
host_and_port = scribereader.get_env_scribe_host(SCRIBE_ENV, True)
host = host_and_port["host"]
port = host_and_port["port"]
while True:
reader, writer = await asyncio.open_connection(host=host, port=port)
writer.write(
construct_conn_msg(stream=SLACK_WEBHOOK_STREAM).encode("utf-8"))
await writer.drain()
while True:
line = await reader.readline()
log.debug(f"got log line {line.decode('utf-8')}")
if not line:
break
event = parse_webhook_event_json(line)
if is_relevant_event(event):
yield event
log.warning("Lost connection to scribe host; reconnecting")
def scribe_tail(self, scribe_env, stream_name, service, levels, components, clusters, instances, queue, filter_fn,
parse_fn=None):
"""Creates a scribetailer for a particular environment.
When it encounters a line that it should report, it sticks it into the
provided queue.
This code is designed to run in a thread as spawned by tail_paasta_logs().
"""
try:
log.debug("Going to tail %s scribe stream in %s" % (stream_name, scribe_env))
host_and_port = scribereader.get_env_scribe_host(scribe_env, True)
host = host_and_port['host']
port = host_and_port['port']
tailer = scribereader.get_stream_tailer(stream_name, host, port)
for line in tailer:
if parse_fn:
line = parse_fn(line, clusters, service)
if filter_fn(line, levels, service, components, clusters, instances):
queue.put(line)
except KeyboardInterrupt:
# Die peacefully rather than printing N threads worth of stack
# traces.
pass
except StreamTailerSetupError as e:
if 'No data in stream' in e.message:
log.warning("Scribe stream %s is empty on %s" % (stream_name, scribe_env))
log.warning("Don't Panic! This may or may not be a problem depending on if you expect there to be")
log.warning("output within this stream.")
# Enter a wait so the process isn't considered dead.
# This is just a large number, since apparently some python interpreters
# don't like being passed sys.maxsize.
sleep(2**16)
else:
raise
def scribe_tail(queue):
host_and_port = scribereader.get_env_scribe_host(SCRIBE_ENV, True)
host = host_and_port['host']
port = host_and_port['port']
tailer = scribereader.get_stream_tailer(SLACK_WEBHOOK_STREAM, host,
port)
for line in tailer:
queue.put(line)
def scribe_get_last_n_lines(self, scribe_env, stream_name, line_count):
# Scribe connection details
host_and_port = scribereader.get_env_scribe_host(scribe_env, True)
host = host_and_port["host"]
port = host_and_port["port"]
# Please read comment above in scribe_get_from_time as to why a fake
# context here is necessary
@contextmanager
def fake_context():
yield scribereader.get_stream_tailer(stream_name, host, port, True, line_count)
return fake_context()
def scribe_get_last_n_lines(self, scribe_env, stream_name, line_count):
# Scribe connection details
host_and_port = scribereader.get_env_scribe_host(scribe_env, True)
host = host_and_port['host']
port = host_and_port['port']
# Please read comment above in scribe_get_from_time as to why a fake
# context here is necessary
@contextmanager
def fake_context():
yield scribereader.get_stream_tailer(stream_name, host, port, True, line_count)
return fake_context()
def scribe_get_last_n_lines(self, scribe_env, stream_name, line_count):
# Scribe connection details
host_and_port = scribereader.get_env_scribe_host(scribe_env, tail=True)
host = host_and_port['host']
port = host_and_port['port']
# The reason we need a fake context here is because scribereader is a bit inconsistent in its
# returns. get_stream_reader returns a context that needs to be acquired for cleanup code but
# get_stream_tailer simply returns an object that can be iterated over. We'd still like to have
# the cleanup code for get_stream_reader to be executed by this function's caller and this is
# one of the simpler ways to achieve it without having 2 if statements everywhere that calls
# this method
@contextmanager
def fake_context():
log.debug(f"Running the equivalent of 'scribereader -e {scribe_env} {stream_name}'")
yield scribereader.get_stream_tailer(stream_name, host, port, True, line_count)
return fake_context()
def scribe_tail(self,
scribe_env,
stream_name,
service,
levels,
components,
clusters,
queue,
filter_fn,
parse_fn=None):
"""Creates a scribetailer for a particular environment.
When it encounters a line that it should report, it sticks it into the
provided queue.
This code is designed to run in a thread as spawned by tail_paasta_logs().
"""
try:
log.debug("Going to tail %s scribe stream in %s" %
(stream_name, scribe_env))
host_and_port = scribereader.get_env_scribe_host(scribe_env, True)
host = host_and_port['host']
port = host_and_port['port']
tailer = scribereader.get_stream_tailer(stream_name, host, port)
for line in tailer:
if parse_fn:
line = parse_fn(line, clusters, service)
if filter_fn(line, levels, service, components, clusters):
queue.put(line)
except KeyboardInterrupt:
# Die peacefully rather than printing N threads worth of stack
# traces.
pass
except StreamTailerSetupError:
log.error("Failed to setup stream tailing for %s in %s" %
(stream_name, scribe_env))
log.error(
"Don't Panic! This can happen the first time a service is deployed because the log"
)
log.error(
"doesn't exist yet. Please wait for the service to be deployed in %s and try again."
% scribe_env)
raise
