def dumpShas(srcPath, destPath, fileSha, isBlame=False):
print srcPath, destPath, fileSha
git_repo = GitRepo(srcPath)
project_name = getProjName(srcPath)
print project_name
with open(fileSha, 'r') as csv_file:
csv_reader = csv.reader(csv_file)
for row in csv_reader:
proj, sha, file_name, commit_date = row[:]
if proj != project_name:
continue
print (',').join((proj,sha,file_name,commit_date))
fname, extension = os.path.splitext(file_name)
fname = fname.replace(os.sep, Util.SEP)
#print fname, extension
#copy_file = destPath + os.sep + fname + sha + ".java"
copy_file = fname + Util.SEP + sha + extension
copy_file = os.path.join(destPath,copy_file)
print copy_file
if isBlame == True:
git_repo.blameFile(file_name,sha,copy_file)
else:
#Todo: something else
pass
return
def __init__(self, projPath, snapshot, outDir, debug=False):
self.src_path = projPath + os.sep + snapshot
self.out_path = outDir + os.sep + snapshot
self.debug = debug
self.git_repo = GitRepo(self.src_path)
self.date = datetime.datetime.strptime(snapshot, '%Y-%m-%d').date()
self.out_dir = None
self.edits = []
self.test_files = set()
self.train_files = set()
def __init__(self, projPath, snapshotName, outDir, configInfo):
self.name = snapshotName
self.src_path = os.path.join(projPath, snapshotName)
self.config_info = configInfo
self.debug = configInfo.DEBUG
self.git_repo = GitRepo(self.src_path)
self.date = self.getSnapshotDate(
) #datetime.datetime.strptime(snapshot, '%Y-%m-%d').date
self.out_path = os.path.join(outDir, snapshotName)
self.out_dir = None
self.edits = []
self.changed_files = set()
self.nonbugfix_files = set()
self.bugfix_files = set()
self.nonchanged_files = set()
def __init__(self, projPath, snapshotName, outDir, configInfo):
self.name = snapshotName
self.src_path = os.path.join(projPath, snapshotName)
self.config_info = configInfo
self.debug = configInfo.DEBUG
self.git_repo = GitRepo(self.src_path)
self.date = self.getSnapshotDate() #datetime.datetime.strptime(snapshot, '%Y-%m-%d').date
self.out_path = os.path.join(outDir, snapshotName)
self.out_dir = None
self.edits = []
self.changed_files = set()
self.nonbugfix_files = set()
self.bugfix_files = set()
self.nonchanged_files = set()
else:
output_file.write(line)
output_file.close()
except OSError:
error_log.LogError("Unable to fix " + proj_name + " mgen file")
return
os.remove(temp_file_path)
# MAIN
error_log = ErrorLog()
subfolders = [f.path for f in os.scandir(main_dir_path) if f.is_dir()]
for dir in subfolders:
if dir.find(".Mapper") != -1 and dir.find("Blending") == -1:
proj_name = dir.split('\\')[-1].split('.')[0]
git_repo = GitRepo()
git_repo.SetProject(proj_name)
if SetupProject(git_repo):
# Remove unneccessary dirs
RemoveFile(join(main_dir_path, proj_name, 'Rebracer.xml'))
RemoveNugetExe(join(main_dir_path, proj_name))
# Add new dirs not added by setup bat
try:
os.mkdir(join(main_dir_path, proj_name, '%STAR_NUGET%'))
except OSError:
error_log.LogError("%STAR_NUGET% folder already exists for " +
proj_name + " Project")
# Setup src/Mapper folder by copying relevant mapping files from mapping repo
MoveInMappingFiles(proj_name)
# Add Mapper Specific csproj file
AddCustomFile(
class Workflow2XBlock(XBlock):
"""
这是学生回答习题的，需要保存每个学生的回答状态
"""
logger = Util.logger(Config.loggerConfig)
gitlabRepo = GitRepo(dict(Config.teacherGitlab, **{'logger': logger}))
# 这是xblock 的特殊fields 用于指定xblock的名字
display_name = String(display_name='Display Name', default=u'练习题工作流', scope=Scope.settings, help='Name of the component in the edxplatform')
# 学生能够回答该问题的最大尝试次数,0表示无限制
# 注意：下面的字段定义的Scope为user_state_summary，这样的设置让openedx允许从LMS修改这些字段
# 处于安全性的考虑，我个人不建议这样的修改，但是老师一定需要这样的功能。
maxTry = Integer(default=0, scope=Scope.content)
#maxTry = Integer(default=0, scope=Scope.user_state_summary)
# 当前block保存的题目
questionJson = Dict(default={}, scope=Scope.content)
#questionJson = Dict(default={}, scope=Scope.user_state_summary)
# 当前block保存的题题号
qNo = Integer(default=0, scope=Scope.content)
#qNo = Integer(default=0, scope=Scope.user_state_summary)
# 学生当前已经尝试的次数
tried = Integer(default=0, scope=Scope.user_state)
# 学生每次回答的记录
answerList = List(default=None, scope=Scope.user_state)
def resource_string(self, path):
"""Handy helper for getting resources from our kit."""
data = pkg_resources.resource_string(__name__, path)
return data.decode("utf8")
def student_view(self, context=None):
'''
学生页面
'''
if self.inStudio():
return self.author_view(context)
html = self.resource_string("static/html/workflow2.html")
frag = Fragment(html)
frag.add_css(self.resource_string("static/css/workflow2.css"))
frag.add_javascript_url('//cdn.bootcss.com/handlebars.js/4.0.5/handlebars.min.js')
frag.add_javascript_url('//cdn.bootcss.com/showdown/1.3.0/showdown.min.js')
frag.add_javascript(self.resource_string("static/js/src/workflow2.js"))
frag.initialize_js('Workflow2XBlock')
return frag
def author_view(self, context=None):
'''
Studio上的缩略页面
'''
content = {
'question': self.qNo,
'maxTry': self.maxTry
}
frag = Fragment(unicode(json.dumps(content)))
return frag
def studio_view(self, context=None):
'''
Studio 上的配置页面
'''
html = self.resource_string("static/html/workflow2_config.html")
frag = Fragment(unicode(html).format(qNo=self.qNo, maxTry=self.maxTry))
frag.add_javascript(self.resource_string('static/js/src/workflow2_config.js'))
frag.initialize_js('Workflow2XBlock')
return frag
def inStudio(self):
'''
检查当前是不是studio环境
'''
if hasattr(self.runtime, 'get_real_user'):
return self.runtime.get_real_user is None
else:
# 在测试环境
return False
def genCurrentStatus(self, needGradeInfo):
if not hasattr(self.runtime, "anonymous_student_id"):
# 测试环境
student = Test()
student.email = '*****@*****.**'
student.username = '******'
student.is_staff = True
graded, gradeInfo = (False, None)
else:
student = self.runtime.get_real_user(self.runtime.anonymous_student_id)
if needGradeInfo:
graded, gradeInfo = self.fetchGradeInfo(student, self.qNo)
self.tried, self.answerList = self.fetchAnswerInfo(student, self.qNo)
if self.answerList is None:
self.tried, self.answerList = self.fetchAnswerInfo(student, self.qNo)
studentEmail = student.email
studentUsername = student.username
studentIsStaff = student.is_staff
tried = self.tried
maxTry = self.maxTry
content = {
'maxTry': maxTry,
'tried': tried,
'student': {'email': studentEmail, 'username': studentUsername, 'is_staff': studentIsStaff},
'answer': self.answerList,
'question': self.questionJson
}
if needGradeInfo:
return dict(content, **{'graded': graded, 'gradeInfo': gradeInfo})
else:
return content
def fetchGradeInfo(self, student, qNo):
'''
获取学生该题的批改信息
'''
filepath = '%(emailHash)s/%(username)s/%(qNo)d/%(qNo)d.graded.json' % {
'emailHash': hashlib.new('md5', student.email).hexdigest()[-2:],
'username': student.username,
'qNo': qNo
}
gradeInfo = self.gitlabRepo.readContent(filepath)
if gradeInfo is None:
graded = False
else:
graded = True
return (graded, gradeInfo)
def fetchAnswerInfo(self, student, qNo):
'''
从gitlab获取学生的回答信息,并保存
'''
filepath = '%(emailHash)s/%(username)s/%(qNo)d/%(qNo)d.json' % {
'emailHash': hashlib.new('md5', student.email).hexdigest()[-2:],
'username': student.username,
'qNo': qNo
}
answerInfo = self.gitlabRepo.readContent(filepath)
if answerInfo is None:
return (0, [])
else:
self.logger.info('fetch answer info from gitlab')
return (answerInfo['tried'], answerInfo['answer'])
@XBlock.json_handler
def getCurrentStatus(self, data, suffix=''):
try:
status = self.genCurrentStatus(True)
return {'code': 0, 'desc': 'ok', 'result': status}
except Exception as e:
self.logger.exception('ERROR getCurrentStatus %s' % (str(e)))
return {'code': 1, 'dese': str(e)}
@XBlock.json_handler
def studentSubmit(self, data, suffix=''):
try:
student = self.runtime.get_real_user(self.runtime.anonymous_student_id)
t = datetime.datetime.now() + datetime.timedelta(hours=12)
createtime = t.strftime('%Y-%m-%d:%H:%M:%S')
answerItem = {'time': createtime, 'answer': data['answer']}
self.logger.debug('answerItem %s' % str(answerItem))
self.answerList.append(answerItem)
self.logger.debug('answerList %s' % str(self.answerList))
self.tried += 1
# 删除多余的历史数据
if len(self.answerList) > Config.maxSizeOfAnswerList:
self.answerList = self.answerList[-(Config.maxSizeOfAnswerList):]
content = self.genCurrentStatus(False)
# push to gitlab
filepath = '%(emailHash)s/%(username)s/%(qNo)d/%(qNo)d.json' % {
'emailHash': hashlib.new('md5', student.email).hexdigest()[-2:],
'username': student.username,
'qNo': self.qNo
}
oldContent = self.gitlabRepo.readContent(filepath)
if oldContent is None:
self.gitlabRepo.createContent(json.dumps(content, ensure_ascii=False, indent=4), filepath, 'create %s' % filepath)
else:
self.gitlabRepo.updateContent(json.dumps(content, ensure_ascii=False, indent=4), filepath, 'update %s' % filepath)
self.logger.info('studentSubmit [student=%s] [tried=%d] [maxTry=%d] [answer=%s] [qNo=%d]' % (
(student.email, student.username),
self.tried,
self.maxTry,
json.dumps(answerItem),
self.qNo
))
return {'code': 0, 'desc': 'ok', 'result': self.genCurrentStatus(False)}
except Exception as e:
self.logger.exception('ERROR student_submit %s' % str(e))
return {'code': 1, 'dese': str(e.args)}
@XBlock.json_handler
def studioSubmit(self, data, suffix=''):
'''
用于配置XBlock的题目，以及每个学生的回答次数
data.q_number 题号
data.max_try 最大尝试的次数
'''
try:
self.logger.info('studioSubmit data=%s' % str(data))
# 保存max_try
self.maxTry = int(data['maxTry'])
# 从github获取题号对应的题目json数据
q_number = int(data['qNo'])
self.qNo = q_number
url = Config.getQuestionJsonUrl % {
'qDir': ((q_number - 1) / 100) + 1,
'qNo': q_number,
}
self.logger.info('studioSubmint url=%s' % url)
res_data = urllib2.urlopen(url)
res = res_data.read()
res = json.loads(res)
if 'content' in res:
content = base64.b64decode(res['content'])
self.questionJson = json.loads(content)
self.logger.info('get question from remote [qNo=%s]' % (q_number))
return {'code': 0, 'desc': 'ok'}
else:
self.logger.warning('ERROR studioSubmit: Cannot read question json [qNo=%d] [msg=%s]' % (q_number, res['message']))
return {'code': 2, 'desc': res['message']}
except Exception as e:
self.logger.exception('ERROR')
return {'code': 1, 'dese': str(e.args)}
# workbench while developing your XBlock.
@staticmethod
def workbench_scenarios():
"""A canned scenario for display in the workbench."""
return [
("Workflow2XBlock",
"""<workflow2/>
"""),
("Workflow2XBlock-test",
"""
<workflow2 maxTry="5" questionJson='{"status":"error","knowledge":["文件系统"],"degree_of_difficulty":1,"explain":"解释\n","question":"文件的逻辑结构的基本形式有**(A)**，__(B)__和__(C)__。\\n```\\n$ pip install\\n```","source":"网络","answer":"解释\n","type":"fill_in_the_blank","options":["A.", "B.", "C."],"q_number":396}'/>
"""),
("Multiple Workflow2XBlock",
"""<vertical_demo>
<workflow2 maxTry="2" questionJson='{"status":"ok","knowledge":["操作系统概述"],"degree_of_difficulty":1,"explain":"B\n","question":"批处理系统的主要缺点是 。\n","source":"网络","answer":"B","type":"single_answer","options":["A.CPU的利用率不高","B.失去了交互性","C.不具备并行性","D.以上都不是"],"q_number":1002}'/>
<workflow2 maxTry="0" questionJson='{"status":"ok","knowledge":["调查问卷"],"degree_of_difficulty":1,"explain":"解释\n","question":"为什么要学这门课？\n","source":"网络","answer":"A","type":"multi_answer","options":["A.对内容有兴趣","B.内容与自己的目标相一致，结果有用","C.由于学分要求，必须选","D.其他，请注明原因"],"q_number":1137}' answerList='[{"time":"2012-01-01 13:20","answer":"A"}]'/>
</vertical_demo>
"""),
]
class WorkflowXBlock(XBlock):
"""
这是一个工作流测试的xblock
"""
# Fields are defined on the class. You can access them in your code as
# self.<fieldname>.
# TO-DO: delete count, and define your own fields.
logger = Util.logger(Config.loggerConfig)
gitlabRepo = GitRepo(dict(Config.teacherGitlab, **{'logger': logger}))
count = Integer(
default=0,
scope=Scope.user_state,
help="A simple counter, to show something happening",
)
display_name = String(
display_name="Display name",
help="This is a workflow test block",
scope=Scope.settings,
default='工作流',
)
def resource_string(self, path):
"""Handy helper for getting resources from our kit."""
data = pkg_resources.resource_string(__name__, path)
return data.decode("utf8")
# TO-DO: change this view to display your data your own way.
def student_view(self, context=None):
"""
The primary view of the WorkflowXBlock, shown to students
when viewing courses.
"""
html = self.resource_string("static/html/workflow.html")
frag = Fragment(html.format(self=self))
frag.add_css(self.resource_string("static/css/workflow.css"))
frag.add_javascript(self.resource_string("static/js/src/workflow.js"))
frag.initialize_js('WorkflowXBlock')
return frag
# TO-DO: change this handler to perform your own actions. You may need more
# than one handler, or you may not need any handlers at all.
@XBlock.json_handler
def submit(self, data, suffix=''):
"""提交本章所有练习"""
student = self.runtime.get_real_user(self.runtime.anonymous_student_id)
studentEmail = student.email
#get all quizzes qNo attribute of current chapter
#获取本章所有练习题号
chapter = self.get_parent().get_parent().get_parent()
subsections = chapter.get_children()
qNo_list = []
for subsection in subsections:
for unit in subsection.get_children():
for xblock in unit.get_children():
if hasattr(xblock, "qNo"):
qNo_list.append({
'qNo': xblock.qNo,
'subsection': subsection.display_name,
'url_name': subsection.url_name
})
self.logger.info(
"email=%s,chapter_display_name=%s,submit this chapter",
studentEmail, chapter.display_name)
#check if all of the quizzes of this chapter have already been submmited?
#fetch answer Info from gitlab
#从gialb获取本章所有未提交的练习题号
unSubmitList = []
for item in qNo_list:
tried, answerList = self.fetchAnswerInfo(student, item['qNo'])
item['tried'] = tried
if int(tried) == 0:
unSubmitList.append(item)
#self.logger.info('unSubmitList=%s',unSubmitList)
# all of the quizzes of this chapter have already been submmited
#如果所有题己提交
if len(unSubmitList) == 0:
cur_href = data['cur_href']
href_splited = cur_href.split('/')
for index, item in enumerate(href_splited):
if item == 'courseware':
url_name = href_splited[index + 1]
break
#connect to mongodb 'workflow'collection
conn = pymongo.Connection('localhost', 27017)
db = conn.test
db.authenticate("edxapp", "p@ssw0rd")
result = db.workflow.find_one({'email': studentEmail
}) #connect to collection
#更新MongoDB,开启下一章
if result:
for index, item in enumerate(result['workflow']):
if item['url_name'] == url_name:
result['workflow'][index + 1]['visible'] = True
next_url_name = result['workflow'][index +
1]['url_name']
break
db.workflow.update({"email": studentEmail},
{"$set": {
"workflow": result['workflow']
}})
next_url = cur_href.split(
'courseware')[0] + 'courseware/' + next_url_name
else:
next_url = cur_href
conn.disconnect()
#调用批改脚本
try:
status1 = os.system('/var/www/zyni/script/pullFromGitlab.sh')
status2 = os.system(
'python /var/www/data/answer/tool/grade.py -s ' +
student.username)
status3 = os.system(
'/var/www/zyni/script/pushToGitlab.sh grade')
self.logger.info(
"grade.py -s %s status1=%s,status2=%s,status3=%s",
studentEmail, status1, status2, status3)
except Exception as e:
self.logger.exception('ERROR: workflow.py %s' % (str(e)))
return {
'url_name': next_url,
'email': studentEmail,
'unSubmitList': unSubmitList
}
#还有题没有提交
else:
self.logger.info(
'submit chapter fail,because there are unSubmited quizzes.')
return {
'url_name': data['cur_href'],
'email': studentEmail,
'unSubmitList': unSubmitList
}
def fetchAnswerInfo(self, student, qNo):
'''
从gitlab获取学生的回答信息,并保存
'''
filepath = '%(emailHash)s/%(username)s/%(qNo)d/%(qNo)d.json' % {
'emailHash': hashlib.new('md5', student.email).hexdigest()[-2:],
'username': student.username,
'qNo': qNo
}
answerInfo = self.gitlabRepo.readContent(filepath)
if answerInfo is None:
return (0, [])
else:
self.logger.info('fetch answer info from gitlab')
return (answerInfo['tried'], answerInfo['answer'])
# TO-DO: change this to create the scenarios you'd like to see in the
# workbench while developing your XBlock.
@staticmethod
def workbench_scenarios():
"""A canned scenario for display in the workbench."""
return [
("WorkflowXBlock", """<workflow/>
"""),
("Multiple WorkflowXBlock", """<vertical_demo>
<workflow/>
<workflow/>
<workflow/>
</vertical_demo>
"""),
]
def createNewGitRepo(self, repoName):
self.gitRepo = GitRepo(repoName, self)
self.gitRepo.create()
class User(Maker):
DEFAULT_HOME = "/home/stagiaires"
DEFAULT_SHELL = "/usr/bin/git-shell"
DEFAULT_GROUP = "git"
DEFAULT_WEB_DIRECTORY = "web"
DEFAULT_GIT_DIRECTORY = "git"
DEFAULT_USER_DIRECTORIES = [DEFAULT_WEB_DIRECTORY, DEFAULT_GIT_DIRECTORY]
ACTION_TYPE = "user"
def __init__(self, username, password):
super().__init__()
self.setUsername(username)
self.setPassword(password)
def createDBUser(self):
DB.createUser(self)
def createNewGitRepo(self, repoName):
self.gitRepo = GitRepo(repoName, self)
self.gitRepo.create()
def createDBForGitRepo(self):
self.db = DB(self, self.gitRepo)
self.db.create()
def setUsername(self, username):
prog = re.compile("^[a-z][a-z0-9]{2,8}$")
result = prog.match(username)
if result is None:
raise Exception(
"L'identifiant ne peut contenir que des lettres minuscules, des chiffres et ne peut pas dépasser 8 caratères [a-z][a-z0-9]{2,7} "
)
self.username = username
def setPassword(self, password):
self.password = password
def getActionPerformedTo(self):
return self.username
def getCmdToCreate(self):
return [
self.__getUseradd(),
self.__getMkdirInUser(self.DEFAULT_USER_DIRECTORIES),
self.__getChownUserHome(),
# self.__getChownInUser(self.DEFAULT_USER_DIRECTORIES),
self.__getQuota()
]
def createChecker(self):
return (not self.isHomeDirectoryExists())
def getCmdToDelete(self):
return [
self.__getUserdel(),
# self.__getRmdirInUser(self.DEFAULT_USER_DIRECTORIES),
]
def deleteChecker(self):
return (self.isHomeDirectoryExists())
def isHomeDirectoryExists(self):
return os.path.exists(self.getHomePath())
def getHomePath(self):
return "{:s}/{:s}".format(self.DEFAULT_HOME, self.username)
def getWebPath(self):
return self.__getHomeDirsStr([self.DEFAULT_WEB_DIRECTORY]).strip()
def getGitPath(self):
return self.__getHomeDirsStr([self.DEFAULT_GIT_DIRECTORY]).strip()
def getUserAndGroup(self):
return "{:s}:{:s}".format(self.username, self.DEFAULT_GROUP)
def __getUseradd(self):
return "useradd -d {:s} -g {:s} -p {:s} -s {:s} {:s}".format(
self.getHomePath(), self.DEFAULT_GROUP,
crypt.crypt(self.password, self.password[:2]), self.DEFAULT_SHELL,
self.username)
def __getMkdirInUser(self, dirList):
return "mkdir -p{:s}".format(self.__getHomeDirsStr(dirList))
def __getChownUserHome(self):
return "chown -R {:s} {:s}".format(self.getUserAndGroup(),
self.getHomePath())
# def __getChownInUser(self, dirList):
# return "chown -R {:s}:{:s}{:s}".format(self.username, self.DEFAULT_GROUP, self.__getHomeDirsStr(dirList))
def __getChmodFor(self, dirList):
return "chmod ug+rwX {:s}:{:s}{:s}".format(
self.username, self.DEFAULT_GROUP, self.__getHomeDirsStr(dirList))
def __getHomeDirsStr(self, dirList):
dirs = ""
for name in dirList:
dirs += " {:s}/{:s}".format(self.getHomePath(), name)
return dirs
def __getQuota(self):
return "quotatool -u {:s} -bq 800M -l '950 Mb' /home".format(
self.username)
def __getUserdel(self):
return "userdel -r {:s}".format(self.username)
class SnapShot:
def __init__(self, projPath, snapshotName, outDir, configInfo):
self.name = snapshotName
self.src_path = os.path.join(projPath, snapshotName)
self.config_info = configInfo
self.debug = configInfo.DEBUG
self.git_repo = GitRepo(self.src_path)
self.date = self.getSnapshotDate() #datetime.datetime.strptime(snapshot, '%Y-%m-%d').date
self.out_path = os.path.join(outDir, snapshotName)
self.out_dir = None
self.edits = []
self.changed_files = set()
self.nonbugfix_files = set()
self.bugfix_files = set()
self.nonchanged_files = set()
def __str__(self):
retStr = self.name + "\n"
retStr += str(self.date) + "\n"
retStr += "========\n"
for e in self.edits:
retStr += str(e)
return retStr
def getSnapshotDate(self):
try:
snapshot_date = datetime.strptime(self.name, '%Y-%m-%d').date()
except:
with Util.cd(self.src_path):
print ">>>>>>> ", self.src_path
git_show = Util.runCmd('git show --date=short')[1]
for l in git_show.split('\n'):
if l.startswith('Date:'):
snapshot_date = l.split('Date: ')[1].strip()
snapshot_date = datetime.strptime(snapshot_date, '%Y-%m-%d').date()
return snapshot_date
def addEdit(self, edit):
if self.out_dir is None:
self.out_dir = OutDir(self.out_path)
self.out_dir.create_out_dir(self.out_path)
self.edits.append(edit)
def getChangedFiles(self, isBug=-1):
for e in self.edits:
file_name = e.file_name.replace('/',os.sep)
if e.isbug == 'False':
self.nonbugfix_files.add(file_name)
elif e.isbug == 'True':
self.bugfix_files.add(file_name)
if isBug == -1: #all
return self.nonbugfix_files | self.bugfix_files
elif isBug == 0: #only non-bug
return self.nonbugfix_files
elif isBug == 1: #only bug
return self.bugfix_files
def dumpTestFiles(self):
if self.out_dir is None:
# no edits corr to this snapshot
return
print('>>>> Dumping files in test dir for ' + path_leaf(self.src_path))
test_dirs = self.out_dir.get_test_dirs()
#print test_dirs
for e in self.edits:
#print e
if e.isbug == 'False':
'''
only considering bugfix files for time being
'''
continue
logging.debug(">>>> %s, %s" % (e.file_name, e.sha))
file_versions = self.git_repo.fetchFiles(e.file_name, e.sha)
for i, sha in enumerate(file_versions):
#print i, sha
file_name = e.file_name.replace('/', self.config_info.SEP)
file_name, extn = os.path.splitext(file_name)
if extn.lower() not in ['.c', '.cpp', '.cc', '.java']:
continue
file_name = file_name + self.config_info.SEP + e.sha + extn
dest_file = os.path.join(test_dirs[i], file_name)
#print file_name, dest_file
self.git_repo.dumpFile(e.file_name, sha, dest_file)
def getTrainFiles(self):
return self.nonchanged_files
def dumpTrainFiles(self):
if self.out_dir is None:
return
print('Dumping files in learn and change dirs for ' + path_leaf(self.src_path))
self.getChangedFiles(-1) #get all changed files
#all files under snapshot except test files
for root, dirs, files in os.walk(self.src_path):
for f in files:
src_file = os.path.join(root, f)
file_name = src_file.split(self.src_path)#[1].strip(os.sep)
if len(file_name) < 2:
logging.error(file_name)
continue
file_name = file_name[1].strip(os.sep)
# Condition to filter out all files except C source files. Added on 2014-09-25 1:32PM PDT by Saheel.
# Changed on 2015-02-01 by Saheel to consider C++ files as well.
extn = os.path.splitext(file_name)[1]
if extn.lower() not in ['.c', '.cpp', '.cc', '.java']:
logging.debug(".... Ignoring!! %s" % file_name)
continue
if file_name in self.nonbugfix_files:
logging.debug("NB %s" % file_name)
dest_file = self.out_dir.changed_dir + os.sep + file_name.replace(os.sep, self.config_info.SEP)
shutil.copyfile(src_file, dest_file)
elif file_name in self.bugfix_files:
logging.debug("BF %s" % file_name)
continue
else:
logging.debug("NC %s" % file_name)
self.nonchanged_files.add(file_name)
dest_file = os.path.join(self.out_dir.learn_dir , file_name.replace(os.sep, self.config_info.SEP))
shutil.copyfile(src_file, dest_file)
class SnapShot:
def __init__(self, projPath, snapshot, outDir, debug=False):
self.src_path = projPath + os.sep + snapshot
self.out_path = outDir + os.sep + snapshot
self.debug = debug
self.git_repo = GitRepo(self.src_path)
self.date = datetime.datetime.strptime(snapshot, '%Y-%m-%d').date()
self.out_dir = None
self.edits = []
self.test_files = set()
self.train_files = set()
def __str__(self):
retStr = str(self.date) + "\n"
retStr += "========\n"
for e in self.edits:
retStr += str(e)
return retStr
def addEdit(self, edit):
if self.out_dir is None:
self.out_dir = OutDir(self.out_path)
self.out_dir.create_out_dir(self.out_path)
self.edits.append(edit)
def getTestFiles(self):
for e in self.edits:
self.test_files.add(e.file_name)
return self.test_files
## def dumpTestFiles(self):
##
## if self.out_dir is None:
## # no edits corr to this snapshot
## return
##
## print('Dumping files in test dir for ' + path_leaf(self.src_path))
## test_dirs = self.out_dir.get_test_dirs()
##
## for e in self.edits:
## files = self.git_repo.fetchFiles(e.file_name, e.sha)
## for i, sha in enumerate(files):
## self.git_repo.checkFile(e.file_name, sha)
## src_file = self.git_repo.repo_path + os.sep + e.file_name
##
## if not os.path.exists(src_file):
## print "!!! " , src_file , " does not exist"
## continue
##
## file_name = e.file_name.replace(os.sep, Util.SEP)
##
## file_name, extn = os.path.splitext(file_name)
##
## # Condition to filter out all files except C source files. Added on 2014-09-25 by Saheel.
## # Changed on 2015-01-30 by Saheel to consider C++ files as well.
## if extn.lower() not in ['.c', '.cpp', '.cc', '.java']:
## continue
##
## file_name = file_name + Util.SEP + e.sha + extn
##
## dest_file = test_dirs[i] + os.sep + file_name
## #print src_file, dest_file
## shutil.copyfile(src_file, dest_file)
def dumpTestFiles(self):
if self.out_dir is None:
# no edits corr to this snapshot
return
print('Dumping files in test dir for ' + path_leaf(self.src_path))
test_dirs = self.out_dir.get_test_dirs()
for e in self.edits:
files = self.git_repo.fetchFiles(e.file_name, e.sha)
for i, sha in enumerate(files):
#copy_content = self.git_repo.showFile(e.file_name, sha)
file_name = e.file_name.replace(os.sep, Util.SEP)
file_name, extn = os.path.splitext(file_name)
if extn.lower() not in ['.c', '.cpp', '.cc', '.java']:
continue
file_name = file_name + Util.SEP + e.sha + extn
dest_file = test_dirs[i] + os.sep + file_name
#print file_name, dest_file
self.git_repo.dumpFile(e.file_name, sha, dest_file)
#copy_content = self.git_repo.showFile(e.file_name, sha)
#with codecs.open(dest_file, "w", encoding="utf-8") as f:
# f.write(copy_content)
def getTrainFiles(self):
return self.train_files
def dumpTrainFiles(self):
if self.out_dir is None:
return
print('Dumping files in learn and change dirs for ' +
path_leaf(self.src_path))
self.getTestFiles()
# self.git_repo.git.stash('-u')
#all files under snapshot except test files
for root, dirs, files in os.walk(self.src_path):
for f in files:
src_file = os.path.join(root, f)
file_name = src_file.split(self.src_path) #[1].strip(os.sep)
if len(file_name) < 2:
logger.error(file_name)
continue
file_name = file_name[1].strip(os.sep)
# Condition to filter out all files except C source files. Added on 2014-09-25 1:32PM PDT by Saheel.
# Changed on 2015-02-01 by Saheel to consider C++ files as well.
file_name_without_extn, extn = os.path.splitext(file_name)
if extn.lower() not in ['.c', '.cpp', '.cc', '.java']:
continue
if file_name in self.test_files:
dest_file = self.out_dir.changed_dir + os.sep + file_name.replace(
os.sep, Util.SEP)
shutil.copyfile(src_file, dest_file)
continue
self.train_files.add(file_name)
dest_file = self.out_dir.learn_dir + os.sep + file_name.replace(
os.sep, Util.SEP)
shutil.copyfile(src_file, dest_file)
def main(args):
sonar = Sonar()
if not sonar.isSonarQubeRunning():
sonar.startSonarQube()
print("SonarQube will be started. (This could take a while)")
api = API()
git_dir, *_ = os.path.basename(args.repo).rpartition('.git')
relative_git_path = os.path.join(args.o, git_dir)
if os.path.exists(relative_git_path):
print(
f'The path `{relative_git_path}` exists, can\'t load the repository',
file=sys.stderr)
print(f'To solve either:', file=sys.stderr)
print(' a) remove the path', file=sys.stderr)
print(' b) choose different save directory', file=sys.stderr)
exit(1)
try:
git = GitRepo(relative_git_path)
ok = git.pullRepoContents(args.repo)
assert ok
if os.path.isabs(args.o):
git_full_path = os.path.join(args.o, git_dir)
else:
git_full_path = os.path.join(os.getcwd(), args.o, git_dir)
print('git_dir:', git_full_path)
wait('sonarqube', sonar.isSonarQubeRunning, timeout=5)
before = set(api.projects())
print(before)
project_key = None
scan_started = False
try:
sonar.runSonarScanner(git_full_path, 'clean', 'verify',
'sonar:sonar')
scan_started = True
# NOTE: this is quite hacky and fragile
wait('analysis',
lambda: before.symmetric_difference(api.projects()),
timeout=1)
project_key = before.symmetric_difference(api.projects()).pop()
print('project_key:', project_key)
# NOTE: breaks if there is no issues
wait('issues',
lambda: any(api.issues(project=project_key)),
timeout=1)
issue_file = os.path.join(args.o, git_dir + '_issues.csv')
git_file = os.path.join(args.o, git_dir + '_git.csv')
issues = api.issues(project=project_key)
analysisParseList(
issues,
issue_file) # NOTE: NOT TESTED! Parse and write to file
git.gitParse(git_file) # NOTE: NOT TESTED!
finally:
if scan_started:
if project_key is None:
wait('analysis',
lambda: before.symmetric_difference(api.projects()),
timeout=0.2)
project_key = before.symmetric_difference(
api.projects()).pop()
# remove analysed project from sonarqube
api.delete_project(project_key)
finally:
# remove git repo
if os.path.isdir(relative_git_path):
shutil.rmtree(relative_git_path)
if p.hasActions("new", "user"):
user = User(options["username"], options["password"])
user.create()
user.createDBUser()
elif p.hasActions("new", "repo"):
user = User(options["username"], "")
user.createNewGitRepo(options["reponame"])
user.createDBForGitRepo()
elif p.hasActions("delete", "user"):
user = User(options["username"], "")
user.delete()
DB.deleteAllDBWithPrefix(options["username"])
DB.deleteUser(options["username"])
elif p.hasActions("delete", "repo"):
user = User(options["username"], "")
repo = GitRepo(options["reponame"], user)
db = DB(user, repo)
db.delete()
repo.delete()
elif p.hasActions("edit", "user"):
pass
elif p.hasActions("edit", "user", "username"):
pass
elif p.hasActions("edit", "user", "password"):
pass
elif p.hasActions("edit", "repo", "reponame"):
pass
elif p.hasActions("list", "user"):
pass
elif p.hasActions("list", "repo"):
pass
class SnapShot:
def __init__(self, projPath, snapshotName, outDir, configInfo):
self.name = snapshotName
self.src_path = os.path.join(projPath, snapshotName)
self.config_info = configInfo
self.debug = configInfo.DEBUG
self.git_repo = GitRepo(self.src_path)
self.date = self.getSnapshotDate(
) #datetime.datetime.strptime(snapshot, '%Y-%m-%d').date
self.out_path = os.path.join(outDir, snapshotName)
self.out_dir = None
self.edits = []
self.changed_files = set()
self.nonbugfix_files = set()
self.bugfix_files = set()
self.nonchanged_files = set()
def __str__(self):
retStr = self.name + "\n"
retStr += str(self.date) + "\n"
retStr += "========\n"
for e in self.edits:
retStr += str(e)
return retStr
def getSnapshotDate(self):
try:
snapshot_date = datetime.strptime(self.name, '%Y-%m-%d').date()
except:
with Util.cd(self.src_path):
print ">>>>>>> ", self.src_path
git_show = Util.runCmd('git show --date=short')[1]
for l in git_show.split('\n'):
if l.startswith('Date:'):
snapshot_date = l.split('Date: ')[1].strip()
snapshot_date = datetime.strptime(
snapshot_date, '%Y-%m-%d').date()
return snapshot_date
def addEdit(self, edit):
if self.out_dir is None:
self.out_dir = OutDir(self.out_path)
self.out_dir.create_out_dir(self.out_path)
self.edits.append(edit)
def getChangedFiles(self, isBug=-1):
for e in self.edits:
file_name = e.file_name.replace('/', os.sep)
if e.isbug == 'False':
self.nonbugfix_files.add(file_name)
elif e.isbug == 'True':
self.bugfix_files.add(file_name)
if isBug == -1: #all
return self.nonbugfix_files | self.bugfix_files
elif isBug == 0: #only non-bug
return self.nonbugfix_files
elif isBug == 1: #only bug
return self.bugfix_files
def dumpTestFiles(self):
if self.out_dir is None:
# no edits corr to this snapshot
return
print('>>>> Dumping files in test dir for ' + path_leaf(self.src_path))
test_dirs = self.out_dir.get_test_dirs()
#print test_dirs
for e in self.edits:
#print e
if e.isbug == 'False':
'''
only considering bugfix files for time being
'''
continue
logging.debug(">>>> %s, %s" % (e.file_name, e.sha))
file_versions = self.git_repo.fetchFiles(e.file_name, e.sha)
for i, sha in enumerate(file_versions):
#print i, sha
file_name = e.file_name.replace('/', self.config_info.SEP)
file_name, extn = os.path.splitext(file_name)
if extn.lower() not in ['.c', '.cpp', '.cc', '.java']:
continue
file_name = file_name + self.config_info.SEP + e.sha + extn
dest_file = os.path.join(test_dirs[i], file_name)
#print file_name, dest_file
self.git_repo.dumpFile(e.file_name, sha, dest_file)
def getTrainFiles(self):
return self.nonchanged_files
def dumpTrainFiles(self):
if self.out_dir is None:
return
print('Dumping files in learn and change dirs for ' +
path_leaf(self.src_path))
self.getChangedFiles(-1) #get all changed files
#all files under snapshot except test files
for root, dirs, files in os.walk(self.src_path):
for f in files:
src_file = os.path.join(root, f)
file_name = src_file.split(self.src_path) #[1].strip(os.sep)
if len(file_name) < 2:
logging.error(file_name)
continue
file_name = file_name[1].strip(os.sep)
# Condition to filter out all files except C source files. Added on 2014-09-25 1:32PM PDT by Saheel.
# Changed on 2015-02-01 by Saheel to consider C++ files as well.
extn = os.path.splitext(file_name)[1]
if extn.lower() not in ['.c', '.cpp', '.cc', '.java']:
logging.debug(".... Ignoring!! %s" % file_name)
continue
if file_name in self.nonbugfix_files:
logging.debug("NB %s" % file_name)
dest_file = self.out_dir.changed_dir + os.sep + file_name.replace(
os.sep, self.config_info.SEP)
shutil.copyfile(src_file, dest_file)
elif file_name in self.bugfix_files:
logging.debug("BF %s" % file_name)
continue
else:
logging.debug("NC %s" % file_name)
self.nonchanged_files.add(file_name)
dest_file = os.path.join(
self.out_dir.learn_dir,
file_name.replace(os.sep, self.config_info.SEP))
shutil.copyfile(src_file, dest_file)
def get_repo(name):
if name in repo_urls:
repo = GitRepo("../" + repo_urls[name])
return repo
else:
return None