def download_activity_files(request, course_slug, activity_slug):
offering = get_object_or_404(CourseOffering, slug=course_slug)
activity = get_object_or_404(offering.activity_set,
slug=activity_slug,
deleted=False)
submission_info = SubmissionInfo.for_activity(activity)
submission_info.get_all_components()
return submission_info.generate_activity_zip()
def all_code_submissions(activity: Activity) -> List[SubmittedCodefile]:
"""
Return a list of all files (as SubmittedCodefile instances) for Codefile submissions in this activity.
"""
si = SubmissionInfo.for_activity(activity)
si.get_all_components()
found, individual_subcomps, last_submission = si.most_recent_submissions()
print(individual_subcomps)
# flatten submitted component list: https://stackoverflow.com/a/952952
sub_comps = [c for sublist in si.all_submitted_components for c in sublist]
# keep only SubmittedCodefiles
sub_comps = [c for c in sub_comps if c is not None and isinstance(c, SubmittedCodefile)]
return sub_comps
def run_moss(main_activity: Activity, activities: List[Activity],
language: str, result: SimilarityResult) -> SimilarityResult:
"""
Run MOSS for the main_activity's submissions.
... comparing past submission from everything in the activities list.
... looking only at the given programming language.
... storing the results in result.
"""
assert language in MOSS_LANGUAGES
assert main_activity in activities
icon_url_path = reverse('dashboard:moss_icon', kwargs={'filename': ''})
tmpdir = tempfile.TemporaryDirectory()
tmp = tmpdir.name
code_dir = os.path.join(tmp, 'code')
moss_out_dir = os.path.join(tmp, 'moss')
# assemble tmp directory of submissions for MOSS
offering_slug = main_activity.offering.slug
extension = '.' + MOSS_LANGUAGES[language]
moss_files = [] # files that we will give to MOSS
file_submissions = {
} # MOSS input file to submission_id, so we can recover the source later
for a in activities:
si = SubmissionInfo.for_activity(a)
si.get_all_components()
_, individual_subcomps, _ = si.most_recent_submissions()
for userid, components in individual_subcomps.items():
prefix = os.path.join(code_dir, a.offering.slug, userid)
for comp, sub in components:
if not isinstance(sub, SubmittedCodefile):
# we can only deal with Codefile components
continue
if not isinstance(sub.code.storage, FileSystemStorage):
raise NotImplementedError(
'more work necessary to support non-filesystem file storage'
)
source_file = os.path.join(sub.code.storage.location,
sub.code.name)
moss_file = sub.file_filename(sub.code, prefix)
if not moss_file.endswith(extension):
# we only handle one language at a time
continue
dst_dir, _ = os.path.split(moss_file)
os.makedirs(dst_dir, exist_ok=True)
os.symlink(source_file, moss_file)
moss_files.append(moss_file)
file_submissions[moss_file] = sub.submission_id
if not moss_files:
raise MOSSError(
'No files found for that language to analyze with MOSS.')
# run MOSS
moss_pl = os.path.join(settings.MOSS_DISTRIBUTION_PATH, 'moss.pl')
cmd = [moss_pl, '-l', language, '-o', moss_out_dir] + moss_files
try:
res = subprocess.run(cmd, cwd=settings.MOSS_DISTRIBUTION_PATH)
except FileNotFoundError:
raise MOSSError(
'System not correctly configured with the MOSS executable.')
if res.returncode != 0:
raise MOSSError('MOSS command failed: ' + str(cmd))
# try to deal with MOSS' [profanity suppressed] HTML, and produce SimilarityData objects to represent everything
for f in os.listdir(moss_out_dir):
if f == 'index.html':
data = open(os.path.join(moss_out_dir, f), 'rt',
encoding='utf8').read()
soup = bs4.BeautifulSoup(data, 'lxml')
index_data = []
for tr in soup.find_all('tr'):
if tr.find('th'):
continue
m = []
for a in tr.find_all('a'):
label = a.get('href')
fn, perc = a.string.split(' ')
fn = _canonical_filename(fn, code_dir)
m.append((label, fn, perc))
# Only display if one side is from the main_activity: leave the past behind.
if any(fn.startswith(offering_slug + '/') for _, fn, _ in m):
index_data.append(m)
data = SimilarityData(result=result,
label='index.html',
file=None,
config={})
data.config['index_data'] = index_data
data.save()
elif match_base_re.match(f):
pass
elif match_top_re.match(f):
data = open(os.path.join(moss_out_dir, f), 'rt',
encoding='utf8').read()
soup = bs4.BeautifulSoup(data, 'lxml')
table = soup.find('table')
del table['bgcolor']
del table['border']
del table['cellspacing']
for th in table.find_all('th'):
if th.string is not None:
th.string = _canonical_filename(th.string, code_dir)
for img in table.find_all('img'):
src = img.get('src')
img['src'] = src.replace('../bitmaps/', icon_url_path)
file = File(file=io.BytesIO(str(table).encode('utf8')), name=f)
data = SimilarityData(result=result, label=f, file=file, config={})
data.save()
elif match_file_re.match(f):
try:
data = open(os.path.join(moss_out_dir, f),
'rt',
encoding='utf8').read()
except UnicodeDecodeError:
data = open(os.path.join(moss_out_dir, f),
'rt',
encoding='windows-1252').read()
soup = bs4.BeautifulSoup(data, 'lxml')
# find the input filename, which leads to the submission
for c in soup.find('body').children:
if isinstance(c, bs4.element.NavigableString):
c = str(c).strip()
if c.startswith(code_dir):
filename = c
break
submission_id = file_submissions[filename]
# the only <pre> is the real content we care about
pre = soup.find('pre')
for img in pre.find_all('img'):
src = img.get('src')
img['src'] = src.replace('../bitmaps/', icon_url_path)
file = File(file=io.BytesIO(str(pre).encode('utf8')), name=f)
data = SimilarityData(result=result,
label=f,
file=file,
submission_id=submission_id,
config={})
data.save()
else:
raise ValueError('unexpected file produced by MOSS')
result.config['complete'] = True
result.save()
return result
def download_activity_files(request, course_slug, activity_slug):
offering = get_object_or_404(CourseOffering, slug=course_slug)
activity = get_object_or_404(offering.activity_set, slug=activity_slug, deleted=False)
submission_info = SubmissionInfo.for_activity(activity)
submission_info.get_all_components()
return submission_info.generate_activity_zip()