def main():
import catsoop.base_context as base_context
from catsoop.process import set_pdeathsig
# Make sure the checker database is set up
checker_db_loc = os.path.join(base_context.cs_data_root, "_logs",
"_checker")
for subdir in ("queued", "running", "results"):
os.makedirs(os.path.join(checker_db_loc, subdir), exist_ok=True)
procs = [
(scripts_dir, [sys.executable, "checker.py"], 0.1, "Checker"),
(scripts_dir, [sys.executable, "reporter.py"], 0.1, "Reporter"),
]
# set up WSGI options
if base_context.cs_wsgi_server == "cheroot":
wsgi_ports = base_context.cs_wsgi_server_port
if not isinstance(wsgi_ports, list):
wsgi_ports = [wsgi_ports]
for port in wsgi_ports:
procs.append((
scripts_dir,
[sys.executable, "wsgi_server.py",
str(port)],
0.1,
"WSGI Server at Port %d" % port,
))
elif base_context.cs_wsgi_server == "uwsgi":
if (base_context.cs_wsgi_server_min_processes >=
base_context.cs_wsgi_server_max_processes):
uwsgi_opts = [
"--processes",
str(base_context.cs_wsgi_server_min_processes)
]
else:
uwsgi_opts = [
"--cheaper",
str(base_context.cs_wsgi_server_min_processes),
"--workers",
str(base_context.cs_wsgi_server_max_processes),
"--cheaper-step",
"1",
"--cheaper-initial",
str(base_context.cs_wsgi_server_min_processes),
]
uwsgi_opts = [
"--http",
":%s" % base_context.cs_wsgi_server_port,
"--thunder-lock",
"--wsgi-file",
"wsgi.py",
"--touch-reload",
"wsgi.py",
] + uwsgi_opts
procs.append((base_dir, ["uwsgi"] + uwsgi_opts, 0.1, "WSGI Server"))
else:
raise ValueError("unsupported wsgi server: %r" %
base_context.cs_wsgi_server)
running = []
for (ix, (wd, cmd, slp, name)) in enumerate(procs):
print("Starting %s (cmd=%s)" % (name, cmd))
running.append(
subprocess.Popen(cmd,
cwd=wd,
preexec_fn=set_pdeathsig(signal.SIGTERM)))
time.sleep(slp)
def _kill_children():
for ix, i in enumerate(running):
os.kill(i.pid, signal.SIGTERM)
atexit.register(_kill_children)
while True:
time.sleep(1)
def do_check(row):
os.setpgrp() # make this part of its own process group
set_pdeathsig()() # but make it die if the parent dies. will this work?
context = loader.spoof_early_load(row['path'])
context['cs_course'] = row['path'][0]
context['cs_path_info'] = row['path']
context['cs_username'] = row['username']
context['cs_user_info'] = {'username': row['username']}
context['cs_user_info'] = auth.get_user_information(context)
context['cs_now'] = datetime.fromtimestamp(row['time'])
cfile = dispatch.content_file_location(context, row['path'])
loader.do_late_load(context, context['cs_course'], context['cs_path_info'], context, cfile)
namemap = collections.OrderedDict()
for elt in context['cs_problem_spec']:
if isinstance(elt, tuple):
m = elt[1]
namemap[m['csq_name']] = elt
# now, depending on the action we want, take the appropriate steps
names_done = set()
for name in row['names']:
if name.startswith('__'):
name = name[2:].rsplit('_', 1)[0]
if name in names_done:
continue
names_done.add(name)
question, args = namemap[name]
if row['action'] == 'submit':
try:
resp = question['handle_submission'](row['form'],
**args)
score = resp['score']
msg = resp['msg']
extra = resp.get('extra_data', None)
except:
resp = {}
score = 0.0
msg = exc_message(context)
extra = None
score_box = context['csm_tutor'].make_score_display(context, args,
name, score,
True)
elif row['action'] == 'check':
try:
msg = question['handle_check'](row['form'], **args)
except:
msg = exc_message(context)
score = None
score_box = ''
extra = None
row['score'] = score
row['score_box'] = score_box
row['response'] = language.handle_custom_tags(context, msg)
row['extra_data'] = extra
# make temporary file to write results to
_, temploc = tempfile.mkstemp()
with open(temploc, 'w') as f:
f.write(context['csm_cslog'].prep(row))
# move that file to results, close the handle to it.
magic = row['magic']
newloc = os.path.join(RESULTS, magic[0], magic[1], magic)
os.makedirs(os.path.dirname(newloc), exist_ok=True)
shutil.move(temploc, newloc)
try:
os.close(_)
except:
pass
# then remove from running
os.unlink(os.path.join(RUNNING, row['magic']))
# finally, update the appropriate log
lockname = context['csm_cslog'].get_log_filename(row['username'], row['path'], 'problemstate')
with context['csm_cslog'].FileLock(lockname) as lock:
x = context['csm_cslog'].most_recent(row['username'], row['path'], 'problemstate', {}, lock=False)
if row['action'] == 'submit':
x.setdefault('scores', {})[name] = row['score']
x.setdefault('score_displays', {})[name] = row['score_box']
x.setdefault('cached_responses', {})[name] = row['response']
x.setdefault('extra_data', {})[name] = row['extra_data']
context['csm_cslog'].overwrite_log(row['username'], row['path'], 'problemstate', x, lock=False)
def main():
import catsoop.base_context as base_context
import catsoop.loader as loader
from catsoop.process import set_pdeathsig
# Make sure the checker database is set up
checker_db_loc = os.path.join(base_context.cs_data_root, "_logs",
"_checker")
for subdir in ("queued", "running", "results", "staging"):
os.makedirs(os.path.join(checker_db_loc, subdir), exist_ok=True)
procs = [
(scripts_dir, [sys.executable, "checker.py"], 0.1, "Checker"),
(scripts_dir, [sys.executable, "reporter.py"], 0.1, "Reporter"),
]
# put plugin autostart scripts into the list
ctx = loader.generate_context([])
for plugin in loader.available_plugins(ctx, course=None):
script_dir = os.path.join(plugin, "autostart")
if os.path.isdir(script_dir):
for script in sorted(os.listdir(script_dir)):
if not script.endswith(".py"):
continue
procs.append((
script_dir,
[sys.executable, script],
0.1,
os.path.join(script_dir, script),
))
# set up WSGI options
if base_context.cs_wsgi_server == "cheroot":
print("[start_catsoop] Using cheroot for web service")
wsgi_ports = base_context.cs_wsgi_server_port
if not isinstance(wsgi_ports, list):
wsgi_ports = [wsgi_ports]
for port in wsgi_ports:
procs.append((
scripts_dir,
[sys.executable, "wsgi_server.py",
str(port)],
0.1,
"WSGI Server at Port %d" % port,
))
elif base_context.cs_wsgi_server == "uwsgi":
print("[start_catsoop] Using uwsgi for web service")
if (base_context.cs_wsgi_server_min_processes >=
base_context.cs_wsgi_server_max_processes):
uwsgi_opts = [
"--processes",
str(base_context.cs_wsgi_server_min_processes)
]
else:
uwsgi_opts = [
"--cheaper",
str(base_context.cs_wsgi_server_min_processes),
"--workers",
str(base_context.cs_wsgi_server_max_processes),
"--cheaper-step",
"1",
"--cheaper-initial",
str(base_context.cs_wsgi_server_min_processes),
]
uwsgi_opts = [
"--http",
":%s" % base_context.cs_wsgi_server_port,
"--thunder-lock",
"--wsgi-file",
"wsgi.py",
"--touch-reload",
"wsgi.py",
] + uwsgi_opts
procs.append((base_dir, ["uwsgi"] + uwsgi_opts, 0.1, "WSGI Server"))
else:
raise ValueError("unsupported wsgi server: %r" %
base_context.cs_wsgi_server)
running = []
for (ix, (wd, cmd, slp, name)) in enumerate(procs):
LOGGER.error("[start_catsoop] Starting %s (cmd=%s, wd=%s)" %
(name, cmd, wd))
running.append(
subprocess.Popen(cmd,
cwd=wd,
preexec_fn=set_pdeathsig(signal.SIGTERM)))
LOGGER.error("[start_catsoop] %s has pid=%s" %
(name, running[-1].pid))
time.sleep(slp)
def _kill_children():
for ix, i in enumerate(running):
os.kill(i.pid, signal.SIGTERM)
atexit.register(_kill_children)
while True:
for idx, (procinfo, proc) in enumerate(zip(
procs, running)): # restart running process if it has died
if proc.poll() is not None:
(wd, cmd, slp, name) = procinfo
LOGGER.error(
'[start_catsoop] %s (pid=%s) was killed, restarting it' %
(name, proc.pid))
running[idx] = subprocess.Popen(cmd,
cwd=wd,
preexec_fn=set_pdeathsig(
signal.SIGTERM))
LOGGER.error(
"[start_catsoop] Starting %s (cmd=%s, wd=%s) as pid=%s" %
(name, cmd, wd, running[idx].pid))
time.sleep(1)
'--cheaper-step', '1',
'--cheaper-initial', str(base_context.cs_wsgi_server_min_processes),
]
uwsgi_opts = [
'--http', ':%s' % base_context.cs_wsgi_server_port,
'--thunder-lock',
'--wsgi-file', os.path.join('catsoop', 'wsgi.py'),
'--touch-reload', os.path.join('catsoop', 'wsgi.py'),
] + uwsgi_opts
procs.append((base_dir, ['uwsgi'] + uwsgi_opts, 0.1, 'WSGI Server'))
else:
raise ValueError('unsupported wsgi server: %r' % base_context.cs_wsgi_server)
running = []
for (ix, (wd, cmd, slp, name)) in enumerate(procs):
print('Starting', name)
running.append(subprocess.Popen(cmd, cwd=wd,
preexec_fn=set_pdeathsig(signal.SIGTERM)))
time.sleep(slp)
def _kill_children():
for ix, i in enumerate(running):
os.kill(i.pid, signal.SIGTERM)
atexit.register(_kill_children)
while True:
time.sleep(1)
def do_check(row):
"""
Check submission, dispatching to appropriate question handler
row: (dict) action to take, with input data
"""
os.setpgrp() # make this part of its own process group
set_pdeathsig()() # but make it die if the parent dies. will this work?
context = loader.spoof_early_load(row["path"])
context["cs_course"] = row["path"][0]
context["cs_path_info"] = row["path"]
context["cs_username"] = row["username"]
context["cs_user_info"] = {"username": row["username"]}
context["cs_user_info"] = auth.get_user_information(context)
context["cs_now"] = datetime.fromtimestamp(row["time"])
have_lti = ("cs_lti_config" in context) and ("lti_data" in row)
if have_lti:
lti_data = row["lti_data"]
lti_handler = lti.lti4cs_response(
context, lti_data) # LTI response handler, from row['lti_data']
log("lti_handler.have_data=%s" % lti_handler.have_data)
if lti_handler.have_data:
log("lti_data=%s" % lti_handler.lti_data)
if not "cs_session_data" in context:
context["cs_session_data"] = {}
context["cs_session_data"][
"is_lti_user"] = True # so that course preload.py knows
cfile = dispatch.content_file_location(context, row["path"])
log("Loading grader python code course=%s, cfile=%s" %
(context["cs_course"], cfile))
loader.do_late_load(context, context["cs_course"], context["cs_path_info"],
context, cfile)
namemap = collections.OrderedDict()
cnt = 0
total_possible_npoints = 0
for elt in context["cs_problem_spec"]:
if isinstance(elt,
tuple): # each elt is (problem_context, problem_kwargs)
m = elt[1]
namemap[m["csq_name"]] = elt
csq_npoints = m.get("csq_npoints", 0)
total_possible_npoints += (
csq_npoints) # used to compute total aggregate score pct
if DEBUG:
question = elt[0]["handle_submission"]
dn = m.get("csq_display_name")
log("Map: %s (%s) -> %s" % (m["csq_name"], dn, question))
log("%s csq_npoints=%s, total_points=%s" %
(dn, csq_npoints, elt[0]["total_points"]()))
cnt += 1
if DEBUG:
log("Loaded %d procedures into question namemap (total_possible_npoints=%s)"
% (cnt, total_possible_npoints))
# now, depending on the action we want, take the appropriate steps
names_done = set()
for name in row["names"]:
if name.startswith("__"):
name = name[2:].rsplit("_", 1)[0]
if name in names_done:
continue
names_done.add(name)
question, args = namemap[name]
if row["action"] == "submit":
if DEBUG:
log("submit name=%s, row=%s" % (name, row))
try:
handler = question["handle_submission"]
if DEBUG:
log("handler=%s" % handler)
resp = handler(row["form"], **args)
score = resp["score"]
msg = resp["msg"]
extra = resp.get("extra_data", None)
except Exception as err:
resp = {}
score = 0.0
log("Failed to handle submission, err=%s" % str(err))
log("Traceback=%s" % traceback.format_exc())
msg = exc_message(context)
extra = None
if DEBUG:
log("submit resp=%s, msg=%s" % (resp, msg))
score_box = context["csm_tutor"].make_score_display(
context, args, name, score, True)
elif row["action"] == "check":
try:
msg = question["handle_check"](row["form"], **args)
except:
msg = exc_message(context)
score = None
score_box = ""
extra = None
if DEBUG:
log("check name=%s, msg=%s" % (name, msg))
row["score"] = score
row["score_box"] = score_box
row["response"] = language.handle_custom_tags(context, msg)
row["extra_data"] = extra
# make temporary file to write results to
_, temploc = tempfile.mkstemp()
with open(temploc, "wb") as f:
f.write(context["csm_cslog"].prep(row))
# move that file to results, close the handle to it.
magic = row["magic"]
newloc = os.path.join(RESULTS, magic[0], magic[1], magic)
os.makedirs(os.path.dirname(newloc), exist_ok=True)
shutil.move(temploc, newloc)
try:
os.close(_)
except:
pass
# then remove from running
os.unlink(os.path.join(RUNNING, row["magic"]))
# finally, update the appropriate log
cm = context["csm_cslog"].log_lock(
[row["username"], *row["path"], "problemstate"])
with cm as lock:
x = context["csm_cslog"].most_recent(row["username"],
row["path"],
"problemstate", {},
lock=False)
if row["action"] == "submit":
x.setdefault("scores", {})[name] = row["score"]
x.setdefault("score_displays", {})[name] = row["score_box"]
x.setdefault("cached_responses", {})[name] = row["response"]
x.setdefault("extra_data", {})[name] = row["extra_data"]
context["csm_cslog"].overwrite_log(row["username"],
row["path"],
"problemstate",
x,
lock=False)
# update LTI tool consumer with new aggregate score
if have_lti and lti_handler.have_data:
aggregate_score = 0
cnt = 0
for k, v in x["scores"].items(
): # e.g. 'scores': {'q000000': 1.0, 'q000001': True, 'q000002': 1.0}
aggregate_score += float(v)
cnt += 1
if total_possible_npoints == 0:
total_possible_npoints = 1.0
LOGGER.error("[checker] total_possible_npoints=0 ????")
aggregate_score_fract = (aggregate_score * 1.0 /
total_possible_npoints
) # LTI wants score in [0, 1.0]
log("Computed aggregate score from %d questions, aggregate_score=%s (fraction=%s)"
% (cnt, aggregate_score, aggregate_score_fract))
log("magic=%s sending aggregate_score_fract=%s to LTI tool consumer"
% (row["magic"], aggregate_score_fract))
try:
lti_handler.send_outcome(aggregate_score_fract)
except Exception as err:
LOGGER.error(
"[checker] failed to send outcome to LTI consumer, err=%s"
% str(err))
LOGGER.error("[checker] traceback=%s" %
traceback.format_exc())