def debug_log(text):
'''
Helper function to help us trace our code.
We print a time stamp, a stack trace, and a /short/ summary of
what's going on.
This is not intended for programmatic debugging. We do change
format regularly (and you should feel free to do so too -- for
example, on narrower terminals, a `\n\t` can help)
'''
stack = inspect.stack()
stack_trace = "{s1}/{s2}/{s3}".format(
s1=stack[1].function,
s2=stack[2].function,
s3=stack[3].function,
)
message = "{time}: {st:60}\t{body}".format(
time=datetime.datetime.utcnow().isoformat(), st=stack_trace, body=text)
# Flip here to print / not print debug messages
if DEBUG:
print(message)
# Flip here to save / not save debug messages
# Ideally, we'd like to log these somewhere which won't cause cascading failures.
# If we e.g. have errors every 100ms, we don't want to create millions of debug files.
# There are services which handle this pretty well, I believe
if DEBUG:
debug_fp = open(paths.logs("debug.log"), "a")
debug_fp.write(message)
debug_fp.write("\n")
debug_fp.close()
async def pipeline(parsed_message):
'''
And this is the pipeline itself. It takes messages, processes them,
and informs consumers when there is new data.
'''
debug_log("Processing PubSub message {event} from {source}".format(
event=parsed_message["client"]["event"], source=client_source
))
# Try to run a message through all event processors.
#
# To do: Finer-grained exception handling. Right now, if we break, we don't run
# through remaining processors.
try:
print(event_processors)
processed_analytics = [await ep(parsed_message) for ep in event_processors]
except Exception as e:
traceback.print_exc()
filename = paths.logs("critical-error-{ts}-{rnd}.tb".format(
ts=datetime.datetime.now().isoformat(),
rnd=uuid.uuid4().hex
))
fp = open(filename, "w")
fp.write(json.dumps(parsed_message, sort_keys=True, indent=2))
fp.write("\nTraceback:\n")
fp.write(traceback.format_exc())
fp.close()
if settings.RUN_MODE == settings.RUN_MODES.DEV:
raise
if processed_analytics is None:
debug_log("No updates")
return []
# Transitional code.
#
# We'd eventually like to return only lists of outgoing
# events. No event means we send back [] For now, our
# modules return `None` to do nothing, events, or lists of
# events.
#
# That's a major refactor away. We'd like to pass in lists /
# iterators of incoming events so we can handle microbatches,
# and generate lists of outgoing events too.
if not isinstance(processed_analytics, list):
print("FIXME: Should return list")
processed_analytics = [processed_analytics]
return processed_analytics
def log_ajax(url, resp_json, request):
'''
This is primarily used to log the responses of AJAX requests made
TO Google and similar providers. This helps us understand the
context of classroom activity, debug, and recover from failures
'''
payload = {
'user': request['user'],
'url': url,
'response': resp_json,
'timestamp': datetime.datetime.utcnow().isoformat()
}
encoded_payload = encode_json_block(payload)
payload_hash = secure_hash(encoded_payload.encode('utf-8'))
filename = AJAX_FILENAME_TEMPLATE.format(
directory=paths.logs("ajax"),
time=datetime.datetime.utcnow().isoformat(),
payload_hash=payload_hash)
with open(filename, "w") as ajax_log_fp:
ajax_log_fp.write(encoded_payload)
def log_event(event, filename=None, preencoded=False, timestamp=False):
'''
This isn't done, but it's how we log events for now.
'''
if filename is None:
log_file_fp = mainlog
elif filename in files:
log_file_fp = files[filename]
else:
log_file_fp = open(paths.logs("" + filename + ".log"), "ab", 0)
files[filename] = log_file_fp
if not preencoded:
event = json.dumps(event, sort_keys=True)
log_file_fp.write(event.encode('utf-8'))
if timestamp:
log_file_fp.write("\t".encode('utf-8'))
log_file_fp.write(
datetime.datetime.utcnow().isoformat().encode('utf-8'))
log_file_fp.write("\n".encode('utf-8'))
log_file_fp.flush()
a dozen analyses, we'll want to know those happened and what those
were, but we might not keep terabytes of data around (just enough to
redo those analyses).
'''
import datetime
import inspect
import json
import hashlib
import learning_observer.filesystem_state
import learning_observer.paths as paths
import learning_observer.settings as settings
mainlog = open(paths.logs("main_log.json"), "ab", 0)
files = {}
# Do we make files for exceptions? Do we print extra stuff on the console?
#
# On deployed systems, this can make a mess. On dev systems, this is super-helpful
DEBUG = settings.RUN_MODE == settings.RUN_MODES.DEV or 'logging' in settings.settings[
'config']['debug']
def encode_json_line(line):
'''
For encoding short data, such as an event.
We use a helper function so we have the same encoding
everywhere. Our primary goal is replicability -- if
import hashlib
import os
import os.path
import shutil
import sys
import learning_observer.paths as paths
import learning_observer.module_loader as module_loader
# These are directories we'd like created on startup. At the moment,
# they're for different types of log files.
directories = {
'logs': {
'path': paths.logs()
},
'startup logs': {
'path': paths.logs('startup')
},
'AJAX logs': {
'path': paths.logs('ajax')
}
}
for d in directories:
dirpath = directories[d]['path']
if not os.path.exists(dirpath):
os.mkdir(dirpath)
print("Made {dirname} directory in {dirpath}".format(dirname=d,
dirpath=dirpath))