def genAST(hwId,partId):
print('Generating starter code AST for homework ' + str(hwId) + ', part ' + str(partId))
tester = UnitTester(hwId,partId)
startercode = tester.unitTestFile
fname = 'starter_' + str(hwId) + '_' + str(partId)
fname_json = os.path.join(dataDir, fname + '.json')
fname_code = os.path.join(dataDir, fname + '.code')
fname_map = os.path.join(dataDir, fname + '.map')
astgenRunner = RunExternal([ASTCMD, startercode, fname_json,fname_code,fname_map],MAXTIME)
astgenRunner.go()
def trainTopicModel(self, topicModelParams, forceTrainOption):
if os.path.exists(self.modelParamsPath + '.twords') and not forceTrainOption:
logging.info(' skipping training: ' + self.currDB)
return
logging.info('trainTopicModel: ' + self.currDB)
LDAcmd = [self.trainExecutablePath, '-est', \
'-ntopics', str(topicModelParams['numTopics']), \
'-niters', str(topicModelParams['numIters']), \
'-twords', str(topicModelParams['twords']), \
'-dfile', self.tokensPath]
timeout = topicModelParams['maxRunTime']
runner = RunExternal(LDAcmd, timeout, pipeOption = False)
runner.run()
def run():
if len(sys.argv) > 2:
hw_id = int(sys.argv[1])
part_id = int(sys.argv[2])
logfilename = tmpdir + 'log_' + str(hw_id) + '_' + str(part_id) + '.log'
logging.basicConfig(filename=logfilename, format='%(asctime)s %(message)s', \
datefmt='%m/%d/%Y %I:%M:%S %p', level=logging.DEBUG)
# setup
errorcnt = 0
successcnt = 0
connection = opendb()
submissions = getAllSubmissionIDs(connection, hw_id, part_id)
ASTdict = {}
CodeExamples = {}
for submission_id, it in zip(submissions, range(len(submissions))):
filePrefix = tmpdir + astfilePrefix + identstr(hw_id, part_id,
submission_id)
# download submission from db and write to temporary file
src = getSource(connection, submission_id)
fname_tmp = filePrefix + octaveSuffix
fid = open(fname_tmp, 'wt')
fid.write(src)
fid.close()
# run astgen on temp file and write out more temp files
fname_json = filePrefix + JSONSuffix
fname_code = filePrefix + codeSuffix
fname_map = filePrefix + mapSuffix
#outputcode = os.system(ASTCMD + ' ' + fname_tmp + ' ' + fname_json \
# + ' ' + fname_code + ' ' + fname_map)
astgenRunner = RunExternal([ASTCMD, \
fname_tmp,fname_json,fname_code,fname_map],MAXTIME)
astgenRunner.go()
outputcode = astgenRunner.outputcode
if outputcode < 1 and not astgenRunner.killed:
# make sure json file is less than 1Mb
# read in json temp file and put it as key in dictionary,
# appending submissionid as well as string length of submission to the value
largeFile = os.path.getsize(fname_json) > 2**20
if not largeFile:
astkey = open(fname_json).read()
src = open(fname_code).read()
srcLen = len(src)
try:
ASTdict[astkey].append(submission_id)
if srcLen < CodeExamples[astkey][0]:
charmap = open(fname_map).read()
CodeExamples[astkey] = (srcLen, src, charmap)
except KeyError:
ASTdict[astkey] = [submission_id]
charmap = open(fname_map).read()
CodeExamples[astkey] = (srcLen, src, charmap)
successcnt += 1
else:
errorcnt += 1
os.system('rm ' + fname_tmp + ' ' + fname_json + ' ' + fname_code +
' ' + fname_map)
else:
os.system('rm ' + fname_tmp)
errorcnt += 1
if it % 50 == 0:
logging.debug(
report(hw_id, part_id, it, submissions, successcnt, errorcnt))
closedb(connection)
logging.debug('Finished.')
# sort ASTdict by number of submissions matching to each AST
sortedASTs = [
x[0]
for x in sorted(ASTdict.items(), key=lambda e: len(e[1]), reverse=True)
]
# For each AST in the dictionary find shortest submission in length
# write out the AST, submission, and character map
logging.debug('\n\nWriting ASTs to file.')
jsondirname = datadir + astfilePrefix + '_' + str(hw_id) + '_' + str(
part_id) + '/'
os.system('mkdir ' + jsondirname)
cnt = 0
for ast in sortedASTs:
filePrefix = jsondirname + astfilePrefix + '_' + str(cnt)
output_json = filePrefix + JSONSuffix
output_code = filePrefix + codeSuffix
output_map = filePrefix + mapSuffix
(srcLen, src, charmap) = CodeExamples[ast]
writeTextFile(ast, output_json)
writeTextFile(src, output_code)
writeTextFile(charmap, output_map)
cnt += 1
# write out number of distinct asts
# write out all submissionids and the number of submissions for each ast
numAST = cnt
output_ids = jsondirname + 'submissionids.dat'
fid = open(output_ids, 'wt')
fid.write('Number of ASTs:' + str(numAST) + '\n')
fid.write(
'astindex:Number of submissions:submission ids(comma separated)\n')
cnt = 0
for ast in sortedASTs:
idlist = ASTdict[ast]
fid.write(str(cnt) + ':' + str(len(idlist)) + ':')
for submission in idlist:
fid.write(str(submission) + ',')
fid.write('\n')
cnt += 1
fid.close()
logging.debug('Compressing results.')
astdirname = astfilePrefix + '_' + str(hw_id) + '_' + str(part_id)
tarfile = astdirname + tarSuffix
os.system('cd ' + datadir + '; ' + TARCMD + ' ' + tarfile + ' ' +
astdirname)
os.system('rm -rf ' + datadir + astdirname)
logging.debug('Done.')
# compress result and delete the things we don't need
print('\n\nReport (' + str(hw_id) + ', ' + str(part_id) + '):')
print('success: ' + str(successcnt))
print('errors: ' + str(errorcnt))
def zip(self, path):
logging.info('MakeVideoLogs.zip(), ' + self.currCourseName)
zipExec = RunExternal(['gzip', path], 3600)
zipExec.run()
def zip(self,path):
logging.info('MakeForumViewLogs.zip(), ' + self.currCourseName)
zipExec = RunExternal(['gzip',path],3600)
zipExec.run()
def unzip(self, path):
logging.info('MakeVideoLogs.unzip(), ' + self.currCourseName)
zipExec = RunExternal(['gunzip',path],3600)
zipExec.run()
def unzip(self, path):
logging.info('MakeForumViewLogs.unzip(), ' + self.currCourseName)
zipExec = RunExternal(['gunzip', path], 3600)
zipExec.run()
#! /usr/bin/env python
from PathImporter import PathImporter
PathImporter.importPythonPaths()
from RunExternal import RunExternal
runner = RunExternal(['sleep','3'],10)
runner2 = RunExternal(['sleep','3'],1)
print('This should take 3 seconds')
runner.go()
print('Done')
print('This should take 1 second')
runner2.go()
print('Done')
