def executeTests():
web_prediction_results = utilsConf.get_configuration().web_prediction_results
matrix_path = os.path.join(web_prediction_results, "matrix_{0}_{1}.matrix")
outcomes_path = os.path.join(web_prediction_results, "outcomes.json")
diagnoses_path = os.path.join(web_prediction_results, "diagnosis_{0}_{1}.matrix")
diagnoses_json_path = os.path.join(web_prediction_results, "diagnosis_{0}_{1}.json")
tested_repo = utilsConf.to_short_path(os.path.join(utilsConf.get_configuration().workingDir, "testedVer", "repo"))
test_runner = TestRunner(tested_repo, AmirTracer(tested_repo, utilsConf.get_configuration().amir_tracer, utilsConf.get_configuration().DebuggerTests))
test_runner.run()
tests = test_runner.get_tests()
json_observations = map(lambda test: test_runner.observations[test].as_dict(), tests)
with open(outcomes_path, "wb") as f:
f.write(json.dumps(json_observations))
for granularity in utilsConf.get_configuration().prediction_files:
for bugged_type in utilsConf.get_configuration().prediction_files[granularity]:
components_priors = get_components_probabilities(bugged_type, granularity, test_runner, tests)
tests_details = map(
lambda test_name: (test_name, list(set(test_runner.tracer.traces[test_name].get_trace(granularity)) & set(components_priors.keys())),
test_runner.observations[test_name].get_observation()),
tests)
matrix = matrix_path.format(granularity, bugged_type)
write_planning_file(matrix, [], filter(lambda test: len(test[1]) > 0, tests_details),
priors=components_priors)
inst = readPlanningFile(matrix)
inst.diagnose()
named_diagnoses = sorted(inst.get_named_diagnoses(), key=lambda d: d.probability, reverse=True)
with open(diagnoses_path.format(granularity, bugged_type), "wb") as diagnosis_file:
diagnosis_file.writelines("\n".join(map(lambda d: repr(d), named_diagnoses)))
with open(diagnoses_json_path.format(granularity, bugged_type), "wb") as diagnosis_json:
diagnosis_json.writelines(json.dumps(map(lambda d: dict([('_name', d[0])] + d[1].as_dict().items()), enumerate(named_diagnoses))))
return test_runner
def executeTests():
web_prediction_results = utilsConf.get_configuration().web_prediction_results
matrix_path = os.path.join(web_prediction_results, "matrix_{0}_{1}.matrix")
outcomes_path = os.path.join(web_prediction_results, "outcomes.json")
diagnoses_path = os.path.join(web_prediction_results, "diagnosis_{0}_{1}.matrix")
diagnoses_json_path = os.path.join(web_prediction_results, "diagnosis_{0}_{1}.json")
tested_repo = utilsConf.to_short_path(os.path.join(utilsConf.get_configuration().workingDir, "version_to_test_trace", "repo"))
utilsConf.open_subprocess(["git", "-C", tested_repo, 'checkout', '--', '.'], stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True).communicate()
test_runner = TestRunner(tested_repo, utilsConf.get_configuration().traces_dir)
test_runner.run()
tests = test_runner.get_tests()
json_observations = map(lambda test: test_runner.observations[test].as_dict(), tests)
with open(outcomes_path, "wb") as f:
f.write(json.dumps(json_observations))
for bugged_type in utilsConf.get_configuration().prediction_files:
for granularity in utilsConf.get_configuration().prediction_files[bugged_type]:
components_priors = get_components_probabilities(bugged_type, granularity, test_runner, tests)
tests_details = map(
lambda test_name: (test_name, list(set(test_runner.traces[test_name].get_trace(granularity)) & set(components_priors.keys())),
test_runner.observations[test_name].get_observation()),
tests)
matrix = matrix_path.format(granularity, bugged_type)
write_planning_file(matrix, [], filter(lambda test: len(test[1]) > 0, tests_details),
priors=components_priors)
inst = readPlanningFile(matrix)
inst.diagnose()
named_diagnoses = sorted(inst.get_named_diagnoses(), key=lambda d: round(d.probability, 2), reverse=True)
with open(diagnoses_path.format(granularity, bugged_type), "wb") as diagnosis_file:
diagnosis_file.writelines("\n".join(map(lambda d: repr(d), named_diagnoses)))
with open(diagnoses_json_path.format(granularity, bugged_type), "wb") as diagnosis_json:
diagnosis_json.writelines(json.dumps(map(lambda d: dict([('_name', d[0])] + d[1].as_dict().items()), enumerate(named_diagnoses))))
return test_runner
def createBuildMLModels():
for buggedType in ["All", "Most"]:
Bpath=os.path.join(utilsConf.get_configuration().workingDir, buggedType)
mkOneDir(Bpath)
FilesPath=os.path.join(Bpath, "Files")
methodsPath=os.path.join(Bpath, "Methods")
mkOneDir(FilesPath)
mkOneDir(methodsPath)
trainingFile,testingFile,NamesFile,Featuresnames,lensAttr=wekaMethods.articles.articlesAllpacks(FilesPath,
utilsConf.get_configuration().gitPath, utilsConf.get_configuration().weka_path, utilsConf.get_configuration().vers, utilsConf.get_configuration().vers_dirs, buggedType, utilsConf.get_configuration().db_dir)
trainingFile, testingFile, NamesFile, outCsv=BuildMLFiles(utilsConf.to_short_path(utilsConf.get_configuration().weka_path),buggedType,"files")
BuildWekaModel(utilsConf.get_configuration().weka_path,trainingFile,testingFile,NamesFile,outCsv,"files_"+buggedType, utilsConf.get_configuration().wekaJar)
# All_One_create.allFamilies(FilesPath,Featuresnames,lensAttr,trainingFile, testingFile,RemoveBat)
trainingFile,testingFile,NamesFile,Featuresnames,lensAttr=wekaMethods.articles.articlesAllpacksMethods(methodsPath,
utilsConf.get_configuration().gitPath, utilsConf.get_configuration().weka_path, utilsConf.get_configuration().vers, utilsConf.get_configuration().vers_dirs, buggedType, utilsConf.get_configuration().db_dir)
trainingFile, testingFile, NamesFile, outCsv=BuildMLFiles(utilsConf.to_short_path(utilsConf.get_configuration().weka_path),buggedType,"methods")
BuildWekaModel(utilsConf.get_configuration().weka_path,trainingFile,testingFile,NamesFile,outCsv,"methods_"+buggedType, utilsConf.get_configuration().wekaJar)
def BuildWekaModel(weka, training, testing, namesCsv, outCsv, name, wekaJar):
algorithm="weka.classifiers.trees.RandomForest -I 1000 -K 0 -S 1 -num-slots 1 "
#os.system("cd /d "+weka +" & java -Xmx2024m -cp \"C:\\Program Files\\Weka-3-7\\weka.jar\" weka.Run " +algorithm+ " -x 10 -d .\\model.model -t "+training+" > training"+name+".txt")
os.system("cd /d "+ utilsConf.to_short_path(weka) +" & java -Xmx2024m -cp "+utilsConf.to_short_path(wekaJar)+" weka.Run " +algorithm+ " -x 10 -d .\\model.model -t "+training+" > training"+name+".txt")
# run_commands = ['java', '-Xmx2024m', '-cp', wekaJar, 'weka.Run', algorithm, '-x', '10', '-d', '.\\model.model', 't']
# proc = subprocess.Popen(run_commands, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True, cwd=utilsConf.to_short_path(weka))
# proc.communicate()
algorithm="weka.classifiers.trees.RandomForest "
os.system("cd /d "+ utilsConf.to_short_path(weka) +" & java -Xmx2024m -cp "+ utilsConf.to_short_path(wekaJar) +" weka.Run " +algorithm+ " -l .\\model.model -T "+testing+" -classifications \"weka.classifiers.evaluation.output.prediction.CSV -file testing"+name+".csv\" ")
os.system("cd /d "+ utilsConf.to_short_path(weka) +" & java -Xmx2024m -cp "+ utilsConf.to_short_path(wekaJar) +" weka.Run " +algorithm+ " -l .\\model.model -T "+testing+" > testing"+name+".txt ")
wekaCsv = os.path.join(utilsConf.to_short_path(weka), "testing"+name+".csv")
wekaMethods.wekaAccuracy.priorsCreation(namesCsv, wekaCsv, outCsv)
def BuildWekaModel(weka, training, testing, namesCsv, outCsv, name, wekaJar):
algorithm="weka.classifiers.trees.RandomForest -I 1000 -K 0 -S 1 -num-slots 1 "
#os.system("cd /d "+weka +" & java -Xmx2024m -cp \"C:\\Program Files\\Weka-3-7\\weka.jar\" weka.Run " +algorithm+ " -x 10 -d .\\model.model -t "+training+" > training"+name+".txt")
os.system("cd /d "+ utilsConf.to_short_path(weka) +" & java -Xmx2024m -cp "+utilsConf.to_short_path(wekaJar)+" weka.Run " +algorithm+ " -x 10 -d .\\model.model -t "+training+" > training"+name+".txt")
# run_commands = ['java', '-Xmx2024m', '-cp', wekaJar, 'weka.Run', algorithm, '-x', '10', '-d', '.\\model.model', 't']
# proc = subprocess.Popen(run_commands, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True, cwd=utilsConf.to_short_path(weka))
# proc.communicate()
algorithm="weka.classifiers.trees.RandomForest "
os.system("cd /d "+ utilsConf.to_short_path(weka) +" & java -Xmx2024m -cp "+ utilsConf.to_short_path(wekaJar) +" weka.Run " +algorithm+ " -l .\\model.model -T "+testing+" -classifications \"weka.classifiers.evaluation.output.prediction.CSV -file testing"+name+".csv\" ")
os.system("cd /d "+ utilsConf.to_short_path(weka) +" & java -Xmx2024m -cp "+ utilsConf.to_short_path(wekaJar) +" weka.Run " +algorithm+ " -l .\\model.model -T "+testing+" > testing"+name+".txt ")
wekaCsv = os.path.join(utilsConf.to_short_path(weka), "testing"+name+".csv")
wekaMethods.wekaAccuracy.priorsCreation(namesCsv, wekaCsv, outCsv, "")
def create(vers, Path):
for ver in vers:
vPath = os.path.join(Path, ver)
commsPath = os.path.join(vPath, "commentsSpaces")
if not os.path.isdir(commsPath):
os.mkdir(commsPath)
java_files = []
for root, _, files in os.walk(os.path.join(vPath,"repo")):
java_files.extend(map(lambda file_name: utilsConf.to_short_path(os.path.join(root, file_name)),
filter(lambda file_name: file_name.endswith('.java'), files)))
for f in java_files:
if not os.path.isfile(f):
continue
outPath = os.path.join(commsPath, os.path.basename(f) + ".txt")
comm, spaces = commentsSpacesLines(f)
with open(outPath, "w") as out:
out.writelines(["comments\n", str(comm), "\nspaces\n", str(spaces)])
def SourceMonitorXml(workingDir, ver, sourceMonitorEXE):
verDir=os.path.join(utilsConf.to_short_path(workingDir), "vers", ver)
verREPO=os.path.join(verDir, "repo")
verP=os.path.join(verDir, ver)
xml="""
<!--?xml version="1.0" encoding="UTF-8" ?-->
<sourcemonitor_commands>
<write_log>true</write_log>
<command>
<project_file>verP.smp</project_file>
<project_language>Java</project_language>
<file_extensions>*.java</file_extensions>
<source_directory>verREPO</source_directory>
<include_subdirectories>true</include_subdirectories>
<checkpoint_name>Baseline</checkpoint_name>
<export>
<export_file>verP.csv</export_file>
<export_type>3 (Export project details in CSV)</export_type>
<export_option>1 (do not use any of the options set in the Options dialog)</export_option>
</export>
</command>
<command>
<project_file>verP.smp</project_file>
<project_language>Java</project_language>
<file_extensions>*.java</file_extensions>
<source_directory>verREPO</source_directory>
<include_subdirectories>true</include_subdirectories>
<checkpoint_name>Baseline</checkpoint_name>
<export>
<export_file>verP_methods.csv</export_file>
<export_type>6 (Export method metrics in CSV)</export_type>
<export_option>1 (do not use any of the options set in the Options dialog)</export_option>
</export>
</command>
</sourcemonitor_commands>""".replace("verP", verP).replace("verREPO", verREPO)
xmlPath = os.path.join(verDir, "sourceMonitor.xml")
with open(xmlPath, "wb") as f:
f.write(xml)
return [sourceMonitorEXE, "/C", xmlPath]
def SourceMonitorXml(workingDir, ver, sourceMonitorEXE):
verDir=os.path.join(utilsConf.to_short_path(workingDir), "vers", ver)
verREPO=os.path.join(verDir, "repo")
verP=os.path.join(verDir, ver)
xml="""
<!--?xml version="1.0" encoding="UTF-8" ?-->
<sourcemonitor_commands>
<write_log>true</write_log>
<command>
<project_file>verP.smp</project_file>
<project_language>Java</project_language>
<file_extensions>*.java</file_extensions>
<source_directory>verREPO</source_directory>
<include_subdirectories>true</include_subdirectories>
<checkpoint_name>Baseline</checkpoint_name>
<export>
<export_file>verP.csv</export_file>
<export_type>3 (Export project details in CSV)</export_type>
<export_option>1 (do not use any of the options set in the Options dialog)</export_option>
</export>
</command>
<command>
<project_file>verP.smp</project_file>
<project_language>Java</project_language>
<file_extensions>*.java</file_extensions>
<source_directory>verREPO</source_directory>
<include_subdirectories>true</include_subdirectories>
<checkpoint_name>Baseline</checkpoint_name>
<export>
<export_file>verP_methods.csv</export_file>
<export_type>6 (Export method metrics in CSV)</export_type>
<export_option>1 (do not use any of the options set in the Options dialog)</export_option>
</export>
</command>
</sourcemonitor_commands>""".replace("verP", verP).replace("verREPO", verREPO)
xmlPath = os.path.join(verDir, "sourceMonitor.xml")
with open(xmlPath, "wb") as f:
f.write(xml)
return [sourceMonitorEXE, "/C", xmlPath]
def checkStyleCreateDict(lines):
methods = {}
for o in lines:
if o == "":
continue
if not "@" in o:
continue
file, data = o.split(" ")
data=data.split("\n")[0]
file = file.split(".java")[0]+".java"
splitted = utilsConf.to_short_path(file).split(os.path.sep)
fileName = os.path.sep.join(splitted[splitted.index('repo') + 1:])
name, begin, end = data.split("@")
methodDir = fileName + "@" + name
methods.setdefault(methodDir, {})
methods[methodDir].setdefault("methodName", name)
methods[methodDir].setdefault("fileName", fileName)
methods[methodDir].setdefault("beginLine", begin)
methods[methodDir].setdefault("EndLine", end)
return methods
def checkStyleCreateDict(lines):
methods = {}
for o in lines:
if o == "":
continue
if not "@" in o:
continue
file, data = o.split(" ")
data = data.split("\n")[0]
file = file.split(".java")[0] + ".java"
splitted = utilsConf.to_short_path(file).split(os.path.sep)
fileName = os.path.sep.join(splitted[splitted.index('repo') + 1:])
name, begin, end = data.split("@")
methodDir = fileName + "@" + name
methods.setdefault(methodDir, {})
methods[methodDir].setdefault("methodName", name)
methods[methodDir].setdefault("fileName", fileName)
methods[methodDir].setdefault("beginLine", begin)
methods[methodDir].setdefault("EndLine", end)
return methods
def do_all():
patchD = os.path.join(utilsConf.get_configuration().LocalGitPath, "patch")
commitsFiles = os.path.join(utilsConf.get_configuration().LocalGitPath,
"commitsFiles")
changedFile = os.path.join(utilsConf.get_configuration().LocalGitPath,
"commitsFiles", "Ins_dels.txt")
mkdir(patchD)
mkdir(commitsFiles)
run_commands = "git format-patch --root -o patch --function-context --unified=9000".split(
)
proc = utilsConf.open_subprocess(
run_commands,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
shell=True,
cwd=utilsConf.to_short_path(
utilsConf.get_configuration().LocalGitPath))
proc.communicate()
buildPatchs(patchD, commitsFiles, changedFile)
checkOut = os.path.join(commitsFiles, "CheckStyle.txt")
RunCheckStyle(commitsFiles, checkOut,
utilsConf.get_configuration().checkStyle68,
utilsConf.get_configuration().methodsNamesXML)
def Extract_OO_features(versPath,vers,docletPath="..\..\\xml-doclet-1.0.4-jar-with-dependencies.jar"):
for x in vers:
verPath=os.path.join(versPath,x)
outPath=os.path.join(verPath,"Jdoc")
outPath=os.path.join(outPath,"javadoc.xml")
err=""
open(os.path.join(verPath,"JdocFunc.txt"),"wt").writelines([x for x in open(os.path.join(verPath,"javaFiles.txt"),"r").readlines()])
run_commands = ["javadoc", "-doclet", "com.github.markusbernhardt.xmldoclet.XmlDoclet","-docletpath ", docletPath, "-private","-d",".\Jdoc","@JdocFunc.txt"]
bads=[]
if (not os.path.exists(outPath)):
bads=bads+OO_features_error_analyze(err)
open(os.path.join(verPath,"JdocFunc.txt"),"wb").writelines([x for x in open(os.path.join(verPath,"javaFiles.txt"),"r").readlines() if x not in bads ])
proc = utilsConf.open_subprocess(run_commands, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True,cwd=utilsConf.to_short_path(verPath))
(out, err) = proc.communicate()
bads=bads+OO_features_error_analyze(err)
open(os.path.join(verPath,"JdocFunc.txt"),"wb").writelines([x for x in open(os.path.join(verPath,"javaFiles.txt"),"r").readlines() if x not in bads ])
proc = utilsConf.open_subprocess(run_commands, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True,cwd=utilsConf.to_short_path(verPath))
(out, err) = proc.communicate()
def Extract_OO_features_OLD():
for version in utilsConf.get_configuration().vers:
verPath = os.path.join(utilsConf.get_configuration().versPath, version_to_dir_name(version))
command = """cd /d """ + utilsConf.to_short_path(verPath) + " & for /R .\\repo %f in (*.java) do (call javadoc -doclet com.github.markusbernhardt.xmldoclet.XmlDoclet -docletpath "+utilsConf.to_short_path(utilsConf.get_configuration().docletPath)+" -filename %~nxf.xml -private -d .\Jdoc2 %f >NUL 2>NUL)"
os.system(command)
def labeling():
patchD = os.path.join(utilsConf.get_configuration().workingDir, "patch")
commitsFiles = os.path.join(utilsConf.get_configuration().workingDir, "commitsFiles")
mkdir(patchD)
mkdir(commitsFiles)
run_commands = "git format-patch --root -o ..\patch --function-context --unified=900000".split()
proc = utilsConf.open_subprocess(run_commands, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True, cwd=utilsConf.to_short_path(utilsConf.get_configuration().LocalGitPath))
proc.communicate()
buildPatchs(patchD, commitsFiles, utilsConf.get_configuration().changeFile)
checkOut = utilsConf.get_configuration().MethodsParsed
RunCheckStyle(commitsFiles, checkOut, utilsConf.get_configuration().checkStyle68, utilsConf.get_configuration().methodsNamesXML)
def blameExecute(path, pathRepo, version):
for root, dirs, files in os.walk(pathRepo):
for name in files:
if not os.path.splitext(name)[1].endswith("java"):
continue
git_file_path = os.path.join(root, name).replace(pathRepo + "\\", "")
blame_file_path = os.path.abspath(os.path.join(path, 'blame', name))
blame_commands = ['git', 'blame', '--show-stats', '--score-debug', '-p', '--line-porcelain', '-l', version, git_file_path]
proc = utilsConf.open_subprocess(blame_commands, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True, cwd=utilsConf.to_short_path(pathRepo))
(out, err) = proc.communicate()
if proc.returncode != 0:
raise RuntimeError('blame subprocess failed. args: {0}. err is {1}'.format(str(blame_commands), err))
with open(blame_file_path, "w") as f:
f.writelines(out)
run_commands = ["dir", "/b", "/s", "/A:-D", "*.java"]
proc = utilsConf.open_subprocess(run_commands, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True, cwd=utilsConf.to_short_path(pathRepo))
(out, err) = proc.communicate()
with open(os.path.join(path, "javaFiles.txt"), "wb") as f:
f.writelines(out)
def Extract_complexity_features():
processes = []
for version_path, version_name, git_version in zip(utilsConf.get_configuration().vers_paths, utilsConf.get_configuration().vers_dirs,
utilsConf.get_configuration().vers):
pathRepo=os.path.join(version_path,"repo")
run_commands = SourceMonitorXml(utilsConf.get_configuration().workingDir, version_name, utilsConf.get_configuration().sourceMonitorEXE)
proc = utilsConf.open_subprocess(run_commands, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)
processes.append((proc, run_commands))
run_commands = ["java", "-jar", utilsConf.get_configuration().checkStyle68, "-c", utilsConf.get_configuration().methodsNamesXML,"javaFile","-o","vers/checkAllMethodsData/"+version_name+".txt",pathRepo]
proc = utilsConf.open_subprocess(run_commands, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True, cwd=utilsConf.to_short_path(utilsConf.get_configuration().workingDir))
processes.append((proc, run_commands))
run_commands = ["java", "-jar", utilsConf.get_configuration().checkStyle57, "-c", utilsConf.get_configuration().allchecks,"-r",pathRepo,"-f","xml","-o","vers/checkAll/"+version_name+".xml"]
proc = utilsConf.open_subprocess(run_commands, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True, cwd=utilsConf.to_short_path(utilsConf.get_configuration().workingDir))
processes.append((proc, run_commands))
# with open(os.path.join(version_path, "javaFiles.txt"), "wb") as f:
# java_files = []
# for root, _, files in os.walk(pathRepo):
# java_files.extend(map(lambda file_name: utilsConf.to_short_path(os.path.join(root, file_name)) + "\n",
# filter(lambda file_name: file_name.endswith('.java'), files)))
# f.writelines(java_files)
# blameExecute(version_path, pathRepo, git_version)
for proc, run_commands in processes:
out, err = proc.communicate()
if proc.returncode != 0:
RuntimeWarning('subprocess execution failed. args are {0}. err is {1}'.format(str(run_commands), err))
def Extract_complexity_features():
processes = []
for version_path, version_name, git_version in zip(utilsConf.get_configuration().vers_paths, utilsConf.get_configuration().vers_dirs,
utilsConf.get_configuration().vers):
pathRepo=os.path.join(version_path,"repo")
run_commands = SourceMonitorXml(utilsConf.get_configuration().workingDir, version_name, utilsConf.get_configuration().sourceMonitorEXE)
proc = utilsConf.open_subprocess(run_commands, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)
processes.append((proc, run_commands))
run_commands = ["java", "-jar", utilsConf.get_configuration().checkStyle68, "-c", utilsConf.get_configuration().methodsNamesXML,"javaFile","-o","vers/checkAllMethodsData/"+version_name+".txt",pathRepo]
proc = utilsConf.open_subprocess(run_commands, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True, cwd=utilsConf.to_short_path(utilsConf.get_configuration().workingDir))
processes.append((proc, run_commands))
run_commands = ["java", "-jar", utilsConf.get_configuration().checkStyle57, "-c", utilsConf.get_configuration().allchecks,"-r",pathRepo,"-f","xml","-o","vers/checkAll/"+version_name+".xml"]
proc = utilsConf.open_subprocess(run_commands, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True, cwd=utilsConf.to_short_path(utilsConf.get_configuration().workingDir))
processes.append((proc, run_commands))
blameExecute(version_path, pathRepo, git_version)
for proc, run_commands in processes:
out, err = proc.communicate()
if proc.returncode != 0:
RuntimeWarning('subprocess execution failed. args are {0}. err is {1}'.format(str(run_commands), err))
def blameExecute(path, pathRepo, version):
for root, dirs, files in os.walk(pathRepo):
for name in files:
if not os.path.splitext(name)[1].endswith("java"):
continue
git_file_path = os.path.join(root, name).replace(pathRepo + "\\", "")
blame_file_path = os.path.abspath(os.path.join(path, 'blame', name))
blame_commands = ['git', 'blame', '--show-stats', '--score-debug', '-p', '--line-porcelain', '-l', version, git_file_path]
proc = utilsConf.open_subprocess(blame_commands, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True, cwd=utilsConf.to_short_path(pathRepo))
(out, err) = proc.communicate()
if proc.returncode != 0:
raise RuntimeError('blame subprocess failed. args: {0}. err is {1}'.format(str(blame_commands), err))
with open(blame_file_path, "w") as f:
f.writelines(out)
run_commands = ["dir", "/b", "/s", "*.java"]
proc = utilsConf.open_subprocess(run_commands, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True, cwd=utilsConf.to_short_path(pathRepo))
(out, err) = proc.communicate()
with open(os.path.join(path, "javaFiles.txt"), "wb") as f:
f.writelines(out)
def Extract_OO_features(versPath,vers,docletPath):
for x in vers:
verPath=os.path.join(versPath,x)
outPath=os.path.join(verPath,"Jdoc")
outPath=os.path.join(outPath,"javadoc.xml")
err=""
open(os.path.join(verPath,"JdocFunc.txt"),"wt").writelines([x for x in open(os.path.join(verPath,"javaFiles.txt"),"r").readlines()])
run_commands = ["javadoc", "-doclet", "com.github.markusbernhardt.xmldoclet.XmlDoclet","-docletpath ", docletPath, "-private","-d",".\Jdoc","@JdocFunc.txt"]
bads=[]
if (not os.path.exists(outPath)):
bads=bads+OO_features_error_analyze(err)
open(os.path.join(verPath,"JdocFunc.txt"),"wb").writelines([x for x in open(os.path.join(verPath,"javaFiles.txt"),"r").readlines() if x not in bads ])
proc = utilsConf.open_subprocess(run_commands, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True,cwd=utilsConf.to_short_path(verPath))
(out, err) = proc.communicate()
bads=bads+OO_features_error_analyze(err)
open(os.path.join(verPath,"JdocFunc.txt"),"wb").writelines([x for x in open(os.path.join(verPath,"javaFiles.txt"),"r").readlines() if x not in bads ])
proc = utilsConf.open_subprocess(run_commands, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True,cwd=utilsConf.to_short_path(verPath))
(out, err) = proc.communicate()
def Extract_OO_features_OLD():
for version in utilsConf.get_configuration().vers:
verPath = os.path.join(utilsConf.get_configuration().versPath, version_to_dir_name(version))
command = """cd /d """ + utilsConf.to_short_path(verPath) + " & for /R .\\repo %f in (*.java) do (call javadoc -doclet com.github.markusbernhardt.xmldoclet.XmlDoclet -docletpath "+utilsConf.to_short_path(utilsConf.get_configuration().docletPath)+" -filename %~nxf.xml -private -d .\Jdoc2 %f >NUL 2>NUL)"
os.system(command)