def download_report(wd, project_name, report_url):
outs_dir = os.path.join(wd, "outs", project_name)
mosspy.download_report(report_url,
os.path.join(outs_dir, "report.html"),
connections=8)
def get_moss_results(userid, basefile_name, testfiles_path ,language, report_path):
real_language = None
if language == 'html':
real_language = 'javascript'
else:
real_language = language
moss_obj = mosspy.Moss(userid, real_language)
moss_obj.addBaseFile(basefile_name)
language_extensions = {
"javascript" : "js",
"python" : "py",
"c++" : "cpp",
"c" : "c",
"java" : "java",
'html' : 'html'
}
moss_obj.addFilesByWildcard(testfiles_path+"/*."+language_extensions[language])
url = moss_obj.send()
print ("Report Url: " + url)
mosspy.download_report(url, report_path+"/report", connections = 8)
def main():
"""
This allows us to send all of the diff bug reports that we generated into MOSS to compare them.
:return:
"""
with open(KEYS_PATH, "r") as stream:
moss_key = yaml.load(stream, Loader=yaml.BaseLoader)['moss_key']
# The MOSS instance
m = mosspy.Moss(moss_key, "C")
# This means that only files from separate directories will be compared to eachother
m.setDirectoryMode(1)
# The directories files will be taken from
compare_list = ['Arduino', 'ardupilot', 'elasticsearch']
# The number of files sent from each directory
n = 10
# Add's files to the MOSS request that will be send
for name in compare_list:
add_n_files_from_folder(m, name, n)
print('Sending files to MOSS')
url = m.send() # Submission Report URL
print("Report Url: " + url)
m.saveWebPage(url, "../submission/report.html")
mosspy.download_report(url, "../submission/report/", connections=8)
def download_moss_report(self) -> None:
if not os.path.isdir(
os.path.join(self.moss_report_dir,
self.MOSS_REPORT_DOWNLOAD)) or self.force:
self._print(f"Downloading MOSS report")
log_level = 10 if self.verbose else 0
mosspy.download_report(self.moss_report_url,
self.moss_report_download_dir,
log_level=log_level)
def sendToMoss():
for p in problems:
for l in config['languages']:
moss = mosspy.Moss(config['userid'], l)
moss.addFilesByWildcard('./{}/*.{}'.format(p['name'], getLangExtension(l)))
url = moss.send()
print "Moss url: ", url
print p['name'], l
# Seems buggy from mosspy project
mosspy.download_report(url, 'Plagiarism/{}/{}/'.format(p['name'], l), connections=8)
def plagcheck():
userid = 389750029
m = mosspy.Moss(userid, "python")
# Submission Files
m.addFilesByWildcard("submission/*.py")
url = m.send() # Submission Report URL
print("Report Url: " + url)
# Save report file
m.saveWebPage(url, "report.html")
# Download whole report locally including code diff links
mosspy.download_report(url, "submission/report/", connections=8)
webbrowser.open('report.html', new=2)
def generate_moss_report(force_refresh):
if config.MOSS_REPORT_FOLDER.is_dir():
print('Moss report already complete', end='')
if force_refresh:
print(' - Forcing refresh anyway')
shutil.rmtree(config.MOSS_REPORT_FOLDER)
else:
print()
return
print('Generating Moss report')
moss = mosspy.Moss(config.MOSS_USER_ID, 'python')
# Create folder for Moss report
config.MOSS_REPORT_FOLDER.mkdir()
concatenated_files_folder = config.MOSS_REPORT_FOLDER / 'concatenated_files'
concatenated_files_folder.mkdir()
print('\tConcatenating source files')
for unzipped_folder in config.UNZIPPED_FILES_FOLDER.iterdir():
concatenated_file_path = concatenated_files_folder / (
unzipped_folder.name + '.txt')
# Concatenate source files into a single file
with open(concatenated_file_path, 'w') as concatenated_file:
patterns_to_concatenate = ['*.cpp', '*.h', '*.inl']
source_file_paths_to_concatenate = []
for pattern in patterns_to_concatenate:
source_file_paths_to_concatenate.extend(
unzipped_folder.glob(pattern))
for source_file_path in source_file_paths_to_concatenate:
with open(source_file_path, errors='replace') as source_file:
concatenated_file.write(source_file.read())
# Only upload non-empty files
if concatenated_file_path.stat().st_size > 0:
moss.addFile(str(concatenated_file_path))
# Upload files
print('\tUploading files to Moss')
report_url = moss.send()
# Save generated report
print('\tDownloading Moss report')
mosspy.download_report(report_url,
config.MOSS_REPORT_FOLDER / 'report',
connections=8)
def run_mosspy(base_files, output_folder, ignore_limit):
plagiarism_output_folder = os.path.join(output_folder, 'plagiarism')
if not os.path.exists(plagiarism_output_folder):
os.makedirs(plagiarism_output_folder)
reports = []
# Run for each file separately --> better accuracy
for file in base_files:
filename = os.path.basename(file)
m = mosspy.Moss(MOSSPY_USERID, 'java')
m.setIgnoreLimit(int(ignore_limit))
m.addBaseFile(file)
for f in get_files([file], output_folder):
m.addFile(f)
print(f"Sending files for {filename}")
report_url = m.send() # Submission Report URL
if report_url.startswith('Error:'):
print(f"Error for {filename}: {report_url}")
continue
else:
print(f"Report Url for {filename}: {report_url}")
print(f"Downloading reports for {filename}... ", end='')
# report_file = os.path.join(output_folder, 'report.html')
# m.saveWebPage(report_url, report_file)
report_folder = os.path.join(
plagiarism_output_folder, f"reports_{filename}/")
mosspy.download_report(report_url, report_folder,
connections=8, log_level=logging.WARNING)
print('Done')
reports.append({
'file': filename,
'report': report_folder
})
return reports
def checkProgram(programName):
targetFilePaths = programs[programName]
for targetFilePath in targetFilePaths:
newBaseFile = baseProjectPath + targetFilePath
print("base file added:", newBaseFile)
m.addBaseFile(newBaseFile)
submissionPaths = os.listdir(baseDirPath)
for submissionPath in submissionPaths:
for targetFilePath in targetFilePaths:
newFile = baseDirPath + submissionPath + "/" + targetFilePath
if os.path.isfile(newFile):
print("file added:", newFile)
m.addFile(newFile)
else:
print("file missing:", newFile)
# exit()
# m.addFilesByWildcard("submission/a01-*.py")
url = m.send() # Submission Report URL
print("Report Url: " + url)
os.system("echo " + programName + ": " + url + " >> moss_history.txt")
pathlib.Path(reportPath + programName + "/report" + timestamp).mkdir(
parents=True, exist_ok=True)
#os.mkdir(reportPath+programName+"/report"+timestamp)
# Save report file
m.saveWebPage(
url, reportPath + programName + "/report" + timestamp + "/report.html")
# Download whole report locally including code diff links
mosspy.download_report(url,
reportPath + programName + "/report" + timestamp +
"/",
connections=8)
def moss_download(unzip_dir, result_buffer_dir, config):
language_list = ['cc', 'java', 'python']
buffer_dir_list = ['c++', 'java', 'python']
prefix_list = ['cpp', 'java', 'py']
for language, dir_lan, prefix in zip(language_list, buffer_dir_list,
prefix_list):
if language not in config.language:
continue
for problem in os.listdir(unzip_dir):
submission_dir = os.path.join(unzip_dir, problem)
buffer_dir = os.path.join(result_buffer_dir, dir_lan, problem)
wildcard = os.path.join(submission_dir, f"*.{prefix}")
if len(list(glob.glob(wildcard))) <= 1:
continue
m = mosspy.Moss(config.moss_userid, language)
m.setIgnoreLimit(config.maximal_match)
m.setNumberOfMatchingFiles(config.report_number)
if config.base_dir is not None:
moss_base_file = os.path.join(config.base_dir,
f"{problem}.cpp")
if os.path.exists(moss_base_file):
m.addBaseFile(moss_base_file, "base")
if config.add_dir is not None:
moss_add_dir = os.path.join(config.add_dir, problem)
moss_add_wildcard = os.path.join(moss_add_dir, "*")
if os.path.exists(moss_add_dir) and len(
glob.glob(moss_add_wildcard)) > 0:
m.addFilesByWildcard(moss_add_wildcard)
wildcard = os.path.join(submission_dir, f"*.{prefix}")
m.addFilesByWildcard(wildcard)
url = m.send()
print("Report Url: " + url)
if not url.startswith("http"):
continue
mosspy.download_report(url, buffer_dir + os.sep, 8, logging.INFO)
def check_plagiarism(c_id):
base_code = '''#include <stdio.h>
int main(int argc, char **argv){
printf("Hello C World!!\\n");
return 0;
}'''
'''
TODO: Database fetch
RETURN : A dictionary of all the codes :
key : USN
value : code as a string
'''
# return value ( (q_id:USN) -> code)
# adding a base file
with open("plagiarism_moss/submission/base.c", "w") as text_file:
text_file.write(base_code)
submissions = get_plagiarism_code(c_id)
print(submissions)
if (submissions == None):
res = set_plagiarism_report(c_id, [])
return
for usn in submissions.keys():
if not os.path.exists("submission/questions/" + str(usn[0])):
os.makedirs("submission/questions/" + str(usn[0]))
with open(
"submission/questions/" + str(usn[0]) + "/" + str(usn[1]) +
".c", "w") as text_file:
text_file.write(submissions[usn]['code'])
#for each directory
directory_list = [x[0] for x in os.walk("submission/questions")]
print(directory_list)
del directory_list[0]
final_list = []
for question in directory_list:
files_list = [x[2] for x in os.walk(question)]
print(files_list)
for file in files_list[0]:
with open(os.path.join(question, file), "r") as f:
if len(f.readlines()) < 2:
## If file exists, delete it ##
if os.path.isfile(os.path.join(question, file)):
print("removing files")
os.remove(os.path.join(question, file))
else: ## Show an error ##
print("Error: %s file not found" %
os.path.join(question, file))
for question in directory_list:
current_dict = {}
q_id = question.split()[-1]
current_dict['q_id'] = q_id
m = mosspy.Moss(userid, "c")
# can add a base file(skeleton code which need not be compared)
m.addBaseFile("plagiarism_moss/submission/base.c")
# Submission Files
print(question)
m.addFilesByWildcard(str(question) + "/*.c")
try:
url = m.send() # Submission Report URL
print("Report Url: " + url)
# Save report file
m.saveWebPage(url, "submission/report.html")
# Download whole report locally including code diff links
except:
print("caught exception")
continue
mosspy.download_report(url, "submission/report/", connections=8)
output = scrape_display(url)
current_dict['report'] = output
final_list.append(current_dict)
print(final_list)
res = set_plagiarism_report(c_id, final_list)
if (res == 1):
print("done checking")
import mosspy
userid = 987654321
m = mosspy.Moss(userid, "python")
#m.addBaseFile("submission/a01.py")
#m.addBaseFile("submission/test_student.py")
# Submission Files
m.addFile("submission/a01-sample.py")
m.addFilesByWildcard("submission/a01-*.py")
# progress function optional, run on every file uploaded
# result is submission URL
url = m.send(lambda file_path, display_name: print('*', end='', flush=True))
print()
print("Report URL: " + url)
# Save report file
m.saveWebPage(url, "submission/report.html")
mosspy.download_report(url,
"submission/report/",
connections=8,
log_level=10,
on_read=lambda url: print('*', end='', flush=True))
# log_level=logging.DEBUG (20 to disable)
# on_read function run for every downloaded file
def execute(submission, basefiles=None, task=None):
"""
This method is the main driver function. It does the following steps:
1. Extracts code files from submission zip file. Currently, it works for java files.
2. Adds basefiles if specified
3. Sends the folders to moss server
4. Once report is generated, provides the url and also downloads it locally
5. Once everything is complete (or if there's an error in between), it cleans
all the temporary folders that were created.
:param submission: Name of the submission zip file
:param basefiles: Base files, if any
:param task: Name of the task. If nothing specified, it extracts all java files.
:return:
"""
dir_list = list()
try:
print("Extracting code files from submissions")
dir_list = extract_code_files(submission, task)
Execute mos
m = mosspy.Moss(userid, "java")
m.setDirectoryMode(mode=1)
print("\nSetting MOSS parameters")
if basefiles is not None:
for file in basefiles:
m.addBaseFile(file)
m.addFilesByWildcard("submissions/*/*")
print("Sending to MOSS server")
print("Waiting for response from MOSS server")
url = m.send()
print("Report URL: {}".format(url))
print("Result received. Saving to {}".format("report_{}.html".format(task)))
m.saveWebPage(url, "report_{}.html".format(task))
report_dir = "report_{}".format(task)
if os.path.isdir(report_dir):
shutil.rmtree(report_dir)
print("Downloading whole report locally for {}".format(task))
mosspy.download_report(url, "report_{}/".format(task), connections=8, log_level=logging.ERROR)
print("\nDeleting directories")
# Delete created files
for directory in dir_list:
if os.path.isdir(directory):
shutil.rmtree(directory)
print("Finished")
except BaseException as e:
print("Exception thrown! {}".format(str(e)))
print("Deleting dirs")
# In case of any exception, delete the temporary directories which might have been created
for directory in dir_list:
if os.path.isdir(directory):
shutil.rmtree(directory)
import mosspy
userid = 987654321
m = mosspy.Moss(userid, "python")
#m.addBaseFile("submission/a01.py")
#m.addBaseFile("submission/test_student.py")
# Submission Files
m.addFile("submission/a01-sample.py")
m.addFilesByWildcard("submission/a01-*.py")
url = m.send()
print("Report URL: " + url)
# Save report file
m.saveWebPage(url, "submission/report.html")
mosspy.download_report(url, "submission/report/", connections=8,
log_level=10) # logging.DEBUG (20 to disable)
def moss(self):
try:
for assignment_part in self.assignment_parts:
m = mosspy.Moss(self.userid, "python")
m.setIgnoreLimit(50)
m.setNumberOfMatchingFiles(250) # should return 250 results
if self.base_file is not None:
m.addBaseFile(self.base_file)
for base in assignment_part['basefiles']:
m.addBaseFile(base)
for specific in assignment_part['files']:
m.addFile(specific)
temp_files = []
for wildcard in assignment_part['filesByWildcard']:
print("Uploading: {}".format(wildcard))
m.addFilesByWildcard(wildcard)
for file in glob.glob(wildcard):
temp_files.append((file, None))
print("Files: {}".format(len(temp_files)))
del temp_files
url = m.send() # Submission Report URL
self.urls.append((assignment_part['name'], url))
print("Moss finished {}: {}".format(assignment_part['name'],
url))
if assignment_part['name'] not in self.moss_urls:
self.moss_urls[assignment_part['name']] = {}
self.moss_urls[assignment_part['name']]['sub{}'.format(
assignment_part['submission'])] = url
# Save report file
# m.saveWebPage(url, "mosspy/report.html")
# Download whole report locally including code diff links
if not os.path.exists(
os.path.join(
self.output_dir, assignment_part['name'],
"{}".format(assignment_part['submission']))):
try:
os.makedirs(
os.path.join(
self.output_dir, assignment_part['name'],
"{}".format(assignment_part['submission'])))
except OSError as exc: # Guard against race condition
if exc.errno != errno.EEXIST:
raise
mosspy.download_report(
url,
"{}/{}/{}".format(self.output_dir, assignment_part['name'],
assignment_part['submission']),
connections=8)
for url in self.urls:
print("{}: {}".format(url[0], url[1]))
except:
return False
# return True if successful
return True
def getStudentGithubFilesAndMossCheck(gitOrg, mossUserId, studentsCSV, assignmentName, gitFileNamesString, gitFilePathsString):
repoNames = getRepoNamesForAssignment(studentsCSV, assignmentName)
# Separates the file names and file paths obtained from the env vars
# into separate file names and paths per the ";" delimiter
# Todo: Probably want to change it so path and filename are obtained from one env var
gitFileNamesList = gitFileNamesString.split('; ')
gitFilePathsList = gitFilePathsString.split('; ')
# Create a diretcory to store all of the copied files
createDirAndSetAsWorking(assignmentName + 'MossCheck')
for gitFilePath, gitFileName in zip(gitFilePathsList, gitFileNamesList):
# Create a diretcory to store all of the student files
createDirAndSetAsWorking(assignmentName + gitFileName)
# Assemble the path with the file name
if gitFilePath == '':
gitFilePathAndName = gitFileName
elif gitFilePath.endswith('/'):
gitFilePathAndName = gitFilePath + gitFileName
else:
gitFilePathAndName = gitFilePath + '/' + gitFileName
# Setup for a Moss check
m = mosspy.Moss(mossUserId, "python")
print('Getting ' + gitFilePathAndName + ' from each student...')
sys.stdout.flush()
for repoName in repoNames:
try:
repo = gitOrg.get_repo(repoName)
try:
file_contents = repo.get_file_contents(gitFilePathAndName)
fd = open(repoName + '-' + gitFileName,'w')
fd.write(file_contents.decoded_content.decode('utf-8-sig'))
fd.close()
except github.GithubException as e:
print('Error occurred getting ' + gitFilePathAndName + 'from ' + repoName + ':\n' + e)
except github.GithubException as e:
print('Error occurred while getting repository ' + repoName + ': ' + str(e.status))
print('Done getting ' + gitFilePathAndName + '\n')
# Add base file so Moss ignores all of the code that was originally
# provided to the students
#m.addBaseFile("submission/a01.py")
# Add all the students files in the current directory to the Moss session
print('Adding files to Moss session...')
m.addFilesByWildcard(assignmentName + '*')
# Send the files to be checked by Moss
print('Sending files to Moss server... Awaiting results (this may take a while)... ')
reportUrl = m.send()
# Save the HTML report in the current folder and download the webpage
m.saveWebPage(reportUrl, assignmentName + "Report.html")
mosspy.download_report(reportUrl, "report/", connections=8, log_level=20) # log_level=10 for debug (20 to disable)
# Leave the created directory
os.chdir('..')
print('Moss check complete!!!')
# Go back to the active working directory before the fn was called
os.chdir('..')
def save_files(self):
# Save report file
self.moss_obj.saveWebPage(self.url, "report.html")
# Download whole report locally including code diff links
mosspy.download_report(self.url, "report/", connections=8)
import mosspy
userid = 223762299
m = mosspy.Moss(userid, "javascript")
# Submission Files
m.addFile("files/d3.js")
m.addFilesByWildcard("files/map.js")
url = m.send() # Submission Report URL
print("Report Url: " + url)
# Save report file
m.saveWebPage(url, "report/report.html")
# Download whole report locally including code diff links
mosspy.download_report(url, "report/src/", connections=8)
def download_report(report_url):
'''
download all reports from the server for offline usage
'''
mosspy.download_report(report_url, "./judge/report/", connections=8)
except Exception as e:
print(i, e, end="")
print(" or File size is zero")
logger.info("-" * 50)
# Submission Report URL
# 下载报告
url = m.send()
logger.info("Report Url: " + url)
# Save report file
# 保存报告
os.makedirs("report", exist_ok=True)
REPORT_PATH = 'report'
time_f = datetime.now(
tz=pytz.timezone(LOCAL_TZ)).strftime('%Y-%m-%d_%H-%M-%S_%f')
PATH_PREFIX = f"{basename(sub_file_dir_path)}_{time_f}"
html_path = join(REPORT_PATH, PATH_PREFIX + '.html')
m.saveWebPage(url, html_path)
logger.info(f"Download report to {html_path}")
# Download whole report locally including code diff links
# 下载完全报告
whole_report_path = join(REPORT_PATH, PATH_PREFIX)
mosspy.download_report(url, whole_report_path, connections=8)
logger.info(f"Download whole report to {whole_report_path}")
# Convert html report to csv file, may need to modify the logic
# html报告转化为csv，根据需要自己修改函数逻辑
# result_html2csv.csv_report(html_path)
def _download_results_from_moss_url(moss: Moss, url: str, results_base_folder: str):
report_folder = os.path.join(results_base_folder, 'Reports')
summary_path = os.path.join(results_base_folder, 'report.html')
moss.saveWebPage(url, summary_path)
mosspy.download_report(url, report_folder, connections=8)
# add this file if a template/assistance was given
# m.addBaseFile(os.path.join(path, "base.py"))
print("Sending Report Of Size: ", count)
# Submission Report URL
url = m.send()
print("Report Url: " + url)
# Save report file
print(os.path.join(path, name))
m.saveWebPage(url, os.path.join(path, name))
print(os.path.join(path, "..", name))
# Download whole report locally including code diff links
mosspy.download_report(url, os.path.join(path, "..", name), connections=8)
# print main url again
print("Report Url: " + url)
# -----------------------------------------------------------
# scraping starts here
print("MOSS Complete, scraping")
page = os.path.abspath(os.path.join(path, "..", name, "index.html"))
soup = BeautifulSoup(open(page), 'html.parser')
data = []
names = []
for section in soup.find_all('tr'):
count = 0
# Submission Files
m.addFilesByWildcard("submissions/*/*.cpp")
m.addFilesByWildcard("submissions/*/*.h")
# Send the file to moss
url = m.send()
print()
print ("Report Url: " + url)
# Save report file
m.saveWebPage(url, "./report.html")
# Download whole report locally including code diff links
mosspy.download_report(url, "./report/", connections=8, log_level=10)
# log_level=logging.DEBUG (20 to disable)
# on_read function run for every downloaded file
########################################################################################################################
# Convert the downloaded report into csv #
########################################################################################################################
with open("report.html") as fp:
soup = BeautifulSoup(fp, 'html.parser')
links = soup.table.find_all('a')
# Create pairs of matches
pairs = []
import mosspy
userid = 987654321
m = mosspy.Moss(userid, "python")
m.addBaseFile("submission/cc.java")
# Submission Files
m.addFile("submission/cc.java")
url = m.send() # Submission Report URL
print("Report Url: " + url)
# Save report file
m.saveWebPage(url, "submission/report.html")
# Download whole report locally including code diff links
mosspy.download_report(url, "submission/report/", connections=8)
