def __init__(self,
test_runner,
granularity,
bugged_type,
num_instances=50,
tests_per_instance=50,
bug_passing_probability=0.1):
self.test_runner = test_runner
self.granularity = granularity
self.bugged_type = bugged_type
self.num_instances = num_instances
self.tests_per_instance = tests_per_instance
self.bug_passing_probability = bug_passing_probability
self.components_priors = ExperimentGenerator.get_components_probabilities(
bugged_type, granularity, test_runner, test_runner.get_tests())
self.bugs = filter(
lambda bug: len(
bug.get_buggy_components(self.granularity, self.bugged_type)) >
0,
Bug.get_bugs_from_db(
os.path.join(utilsConf.get_configuration().db_dir,
utilsConf.get_configuration().vers_dirs[-2]) +
".db",
utilsConf.get_configuration().dates[-2],
utilsConf.get_configuration().dates[-1]))
assert len(self.bugs) > 0
def executeTests():
web_prediction_results = utilsConf.get_configuration().web_prediction_results
matrix_path = os.path.join(web_prediction_results, "matrix_{0}_{1}.matrix")
outcomes_path = os.path.join(web_prediction_results, "outcomes.json")
diagnoses_path = os.path.join(web_prediction_results, "diagnosis_{0}_{1}.matrix")
diagnoses_json_path = os.path.join(web_prediction_results, "diagnosis_{0}_{1}.json")
tested_repo = utilsConf.to_short_path(os.path.join(utilsConf.get_configuration().workingDir, "version_to_test_trace", "repo"))
utilsConf.open_subprocess(["git", "-C", tested_repo, 'checkout', '--', '.'], stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True).communicate()
test_runner = TestRunner(tested_repo, utilsConf.get_configuration().traces_dir)
test_runner.run()
tests = test_runner.get_tests()
json_observations = map(lambda test: test_runner.observations[test].as_dict(), tests)
with open(outcomes_path, "wb") as f:
f.write(json.dumps(json_observations))
for bugged_type in utilsConf.get_configuration().prediction_files:
for granularity in utilsConf.get_configuration().prediction_files[bugged_type]:
components_priors = get_components_probabilities(bugged_type, granularity, test_runner, tests)
tests_details = map(
lambda test_name: (test_name, list(set(test_runner.traces[test_name].get_trace(granularity)) & set(components_priors.keys())),
test_runner.observations[test_name].get_observation()),
tests)
matrix = matrix_path.format(granularity, bugged_type)
write_planning_file(matrix, [], filter(lambda test: len(test[1]) > 0, tests_details),
priors=components_priors)
inst = readPlanningFile(matrix)
inst.diagnose()
named_diagnoses = sorted(inst.get_named_diagnoses(), key=lambda d: round(d.probability, 2), reverse=True)
with open(diagnoses_path.format(granularity, bugged_type), "wb") as diagnosis_file:
diagnosis_file.writelines("\n".join(map(lambda d: repr(d), named_diagnoses)))
with open(diagnoses_json_path.format(granularity, bugged_type), "wb") as diagnosis_json:
diagnosis_json.writelines(json.dumps(map(lambda d: dict([('_name', d[0])] + d[1].as_dict().items()), enumerate(named_diagnoses))))
return test_runner
def get_components_probabilities(bugged_type, granularity, test_runner,
tests):
predictions = {}
with open(
os.path.join(
utilsConf.get_configuration().web_prediction_results,
utilsConf.get_configuration().prediction_files[granularity]
[bugged_type])) as f:
lines = list(csv.reader(f))[1:]
predictions = dict(
map(
lambda line: (line[0].replace(".java", "").replace(
os.path.sep, ".").lower().replace('$', '@'), line[1]),
lines))
components_priors = {}
components = set(
reduce(
list.__add__,
map(
lambda test_name: test_runner.tracer.traces[test_name].
get_trace(granularity), tests), []))
sorted_list = sorted(
map(lambda s: s[::-1],
list(components) + predictions.keys()))
for component in components:
reversed_component = component[::-1]
index = sorted_list.index(reversed_component) + 1
if index < len(sorted_list):
if sorted_list[index].startswith(reversed_component):
components_priors[component] = max(
float(predictions[sorted_list[index][::-1]]), 0.01)
return components_priors
def Extract_complexity_features():
processes = []
for version_path, version_name, git_version in zip(utilsConf.get_configuration().vers_paths, utilsConf.get_configuration().vers_dirs,
utilsConf.get_configuration().vers):
pathRepo=os.path.join(version_path,"repo")
run_commands = SourceMonitorXml(utilsConf.get_configuration().workingDir, version_name, utilsConf.get_configuration().sourceMonitorEXE)
proc = utilsConf.open_subprocess(run_commands, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)
processes.append((proc, run_commands))
run_commands = ["java", "-jar", utilsConf.get_configuration().checkStyle68, "-c", utilsConf.get_configuration().methodsNamesXML,"javaFile","-o","vers/checkAllMethodsData/"+version_name+".txt",pathRepo]
proc = utilsConf.open_subprocess(run_commands, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True, cwd=utilsConf.to_short_path(utilsConf.get_configuration().workingDir))
processes.append((proc, run_commands))
run_commands = ["java", "-jar", utilsConf.get_configuration().checkStyle57, "-c", utilsConf.get_configuration().allchecks,"-r",pathRepo,"-f","xml","-o","vers/checkAll/"+version_name+".xml"]
proc = utilsConf.open_subprocess(run_commands, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True, cwd=utilsConf.to_short_path(utilsConf.get_configuration().workingDir))
processes.append((proc, run_commands))
# with open(os.path.join(version_path, "javaFiles.txt"), "wb") as f:
# java_files = []
# for root, _, files in os.walk(pathRepo):
# java_files.extend(map(lambda file_name: utilsConf.to_short_path(os.path.join(root, file_name)) + "\n",
# filter(lambda file_name: file_name.endswith('.java'), files)))
# f.writelines(java_files)
# blameExecute(version_path, pathRepo, git_version)
for proc, run_commands in processes:
out, err = proc.communicate()
if proc.returncode != 0:
RuntimeWarning('subprocess execution failed. args are {0}. err is {1}'.format(str(run_commands), err))
def createBuildMLModels():
for granularity in wekaMethods.articles.BUG_QUERIES:
packages = wekaMethods.articles.PACKAGES[granularity]
for buggedType in wekaMethods.articles.BUG_QUERIES[granularity]:
print buggedType, granularity
generator = wekaMethods.articles.arffGenerator(buggedType, granularity)
generator.generate_features(utilsConf.get_configuration().weka_path, packages)
generator.BuildWekaModel(utilsConf.get_configuration().weka_path)
def create_experiment(test_runner, num_instances=50, tests_per_instance=50, bug_passing_probability=0.05):
results_path = os.path.join(utilsConf.get_configuration().experiments, "{GRANULARITY}_{BUGGED_TYPE}")
for bugged_type in utilsConf.get_configuration().prediction_files:
for granularity in utilsConf.get_configuration().prediction_files[bugged_type]:
eg = ExperimentGenerator(test_runner, granularity, bugged_type, num_instances, tests_per_instance, bug_passing_probability)
results = eg.create_instances()
with open(results_path.format(GRANULARITY=granularity, BUGGED_TYPE=bugged_type), 'wb') as f:
f.write(json.dumps(results))
def create_web_prediction_results():
for buggedType in utilsConf.get_configuration().prediction_files:
for granularity in utilsConf.get_configuration().prediction_files[buggedType]:
weka_csv = os.path.join(utilsConf.get_configuration().weka_path, "{buggedType}_out_{GRANULARITY}.csv".format(buggedType=buggedType, GRANULARITY=granularity))
prediction_csv = os.path.join(utilsConf.get_configuration().web_prediction_results,
utilsConf.get_configuration().prediction_files[buggedType][granularity])
weka_csv_to_readable_csv(weka_csv, prediction_csv)
save_json_watchers(prediction_csv)
def labeling():
patchD = os.path.join(utilsConf.get_configuration().workingDir, "patch")
commitsFiles = os.path.join(utilsConf.get_configuration().workingDir, "commitsFiles")
mkdir(patchD)
mkdir(commitsFiles)
run_commands = "git format-patch --root -o ..\patch --function-context --unified=900000".split()
proc = utilsConf.open_subprocess(run_commands, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True, cwd=utilsConf.to_short_path(utilsConf.get_configuration().LocalGitPath))
proc.communicate()
buildPatchs(patchD, commitsFiles, utilsConf.get_configuration().changeFile)
checkOut = utilsConf.get_configuration().MethodsParsed
RunCheckStyle(commitsFiles, checkOut, utilsConf.get_configuration().checkStyle68, utilsConf.get_configuration().methodsNamesXML)
def check_distribution():
wekaMethods.patchsBuild.labeling()
wekaMethods.buildDB.build_labels()
all_tags = sorted(git.Repo(utilsConf.get_configuration().gitPath).tags, key=lambda tag: tag.commit.committed_date)
majors, minors, micros = get_versions_by_type(all_tags)
vers_tags = filter(lambda tag: tag.name in utilsConf.get_configuration().vers, all_tags)
for tags, out_file in zip([all_tags, majors, minors, micros, vers_tags], [utilsConf.get_configuration().distribution_per_version_report, utilsConf.get_configuration().distribution_per_majors_report, utilsConf.get_configuration().distribution_per_minors_report, utilsConf.get_configuration().distribution_per_micros_report, utilsConf.get_configuration().distribution_report]):
rows_all_versions = distribution_for_tags(tags)
with open(out_file, "wb") as report:
writer = csv.writer(report)
writer.writerows(rows_all_versions)
def ml_all_one():
for granularity in wekaMethods.articles.BUG_QUERIES:
packages = wekaMethods.articles.PACKAGES[granularity]
for buggedType in wekaMethods.articles.BUG_QUERIES[granularity]:
generator = wekaMethods.articles.arffGenerator(buggedType, granularity)
generator.generate_features(utilsConf.get_configuration().weka_path, packages)
for ind in range(len(packages)):
one_feature_dir = os.path.join(utilsConf.get_configuration().one_dir, packages[ind])
mkOneDir(one_feature_dir)
all_feature_dir = os.path.join(utilsConf.get_configuration().all_but_one_dir, packages[ind])
mkOneDir(all_feature_dir)
generator.generate_features(one_feature_dir, [packages[ind]])
generator.generate_features(all_feature_dir, packages[:ind] + packages[ind + 1:])
def buildOneTimeCommits():
versPath = utilsConf.get_configuration().versPath
db_dir = utilsConf.get_configuration().db_dir
vers = utilsConf.get_configuration().vers_dirs
dates = utilsConf.get_configuration().dates
add = False
max = -1
CodeDir = "repo"
for version, date in zip(vers, dates):
gc.collect()
Path = os.path.join(versPath, version)
dbPath = os.path.join(db_dir, version + ".db")
JavaDocPath = os.path.join(Path, "Jdoc2")
sourceMonitorFiles = os.path.join(Path, version + ".csv")
sourceMonitorMethods = os.path.join(Path, version + "_methods.csv")
checkStyle = os.path.join(versPath, "checkAll", version + ".xml")
checkStyleMethods = os.path.join(versPath, "checkAllMethodsData",
version + ".txt")
blamePath = os.path.join(Path, "blame")
BuildAllOneTimeCommits(utilsConf.get_configuration().gitPath, dbPath,
JavaDocPath, sourceMonitorFiles,
sourceMonitorMethods, checkStyle,
checkStyleMethods, blamePath, date, add, max,
CodeDir)
createIndexes(dbPath)
buildBasicAllVers(vers, dates, versPath, CodeDir, db_dir,
utilsConf.get_configuration().bugsPath,
utilsConf.get_configuration().MethodsParsed,
utilsConf.get_configuration().changeFile)
def __init__(self, test_runner, granularity, bugged_type, num_instances=50, tests_per_instance=50, bug_passing_probability=0.1):
self.test_runner = test_runner
self.granularity = granularity
self.bugged_type = bugged_type
self.num_instances = num_instances
self.tests_per_instance = tests_per_instance
self.bug_passing_probability = bug_passing_probability
self.components_priors = ExperimentGenerator.get_components_probabilities(bugged_type, granularity, test_runner,
test_runner.get_tests())
self.bugs = filter(lambda bug: len(bug.get_buggy_components(self.granularity, self.bugged_type)) > 0,
Bug.get_bugs_from_db(os.path.join(utilsConf.get_configuration().db_dir,
utilsConf.get_configuration().vers_dirs[-2]) + ".db",
utilsConf.get_configuration().dates[-2],
utilsConf.get_configuration().dates[-1]))
assert len(self.bugs) > 0
def get_components_probabilities(bugged_type, granularity, test_runner, tests):
predictions = {}
with open(os.path.join(utilsConf.get_configuration().web_prediction_results, utilsConf.get_configuration().prediction_files[bugged_type][granularity])) as f:
lines = list(csv.reader(f))[1:]
predictions = dict(
map(lambda line: (line[0].replace(".java", "").replace(os.path.sep, ".").lower().replace('$', '.'), line[1]), lines))
components_priors = {}
for component in set(
reduce(list.__add__, map(lambda test_name: test_runner.traces[test_name].get_trace(granularity), tests), [])):
for prediction in predictions:
if prediction.endswith(component.split('(')[0]):
components_priors[component] = max(float(predictions[prediction]), 0.01)
else:
components_priors[component] = 0.01
return components_priors
def createBuildMLModels():
for buggedType in ["All", "Most"]:
Bpath=os.path.join(utilsConf.get_configuration().workingDir, buggedType)
mkOneDir(Bpath)
FilesPath=os.path.join(Bpath, "Files")
methodsPath=os.path.join(Bpath, "Methods")
mkOneDir(FilesPath)
mkOneDir(methodsPath)
trainingFile,testingFile,NamesFile,Featuresnames,lensAttr=wekaMethods.articles.articlesAllpacks(FilesPath,
utilsConf.get_configuration().gitPath, utilsConf.get_configuration().weka_path, utilsConf.get_configuration().vers, utilsConf.get_configuration().vers_dirs, buggedType, utilsConf.get_configuration().db_dir)
trainingFile, testingFile, NamesFile, outCsv=BuildMLFiles(utilsConf.to_short_path(utilsConf.get_configuration().weka_path),buggedType,"files")
BuildWekaModel(utilsConf.get_configuration().weka_path,trainingFile,testingFile,NamesFile,outCsv,"files_"+buggedType, utilsConf.get_configuration().wekaJar)
# All_One_create.allFamilies(FilesPath,Featuresnames,lensAttr,trainingFile, testingFile,RemoveBat)
trainingFile,testingFile,NamesFile,Featuresnames,lensAttr=wekaMethods.articles.articlesAllpacksMethods(methodsPath,
utilsConf.get_configuration().gitPath, utilsConf.get_configuration().weka_path, utilsConf.get_configuration().vers, utilsConf.get_configuration().vers_dirs, buggedType, utilsConf.get_configuration().db_dir)
trainingFile, testingFile, NamesFile, outCsv=BuildMLFiles(utilsConf.to_short_path(utilsConf.get_configuration().weka_path),buggedType,"methods")
BuildWekaModel(utilsConf.get_configuration().weka_path,trainingFile,testingFile,NamesFile,outCsv,"methods_"+buggedType, utilsConf.get_configuration().wekaJar)
def get_components_probabilities(bugged_type, granularity, test_runner, tests):
predictions = {}
with open(os.path.join(utilsConf.get_configuration().web_prediction_results,
utilsConf.get_configuration().prediction_files[bugged_type][granularity])) as f:
lines = list(csv.reader(f))[1:]
predictions = dict(
map(lambda line: (
line[0].replace(".java", "").replace(os.path.sep, ".").lower().replace('$', '@'), line[1]), lines))
components_priors = {}
components = set(reduce(list.__add__,
map(lambda test_name: test_runner.tracer.traces[test_name].get_trace(granularity), tests), []))
sorted_list = sorted(map(lambda s: s[::-1], list(components) + predictions.keys()))
for component in components:
reversed_component = component[::-1]
index = sorted_list.index(reversed_component) + 1
if index < len(sorted_list):
if sorted_list[index].startswith(reversed_component):
components_priors[component] = max(float(predictions[sorted_list[index][::-1]]), 0.01)
return components_priors
def buildOneTimeCommits():
versPath = utilsConf.get_configuration().versPath
db_dir = utilsConf.get_configuration().db_dir
vers = utilsConf.get_configuration().vers_dirs
dates = utilsConf.get_configuration().dates
CodeDir = "repo"
build_labels()
for version, date in zip(vers, dates):
gc.collect()
Path = os.path.join(versPath, version)
dbPath = os.path.join(db_dir, version + ".db")
JavaDocPath = os.path.join(Path, "Jdoc2")
sourceMonitorFiles = os.path.join(Path, version+".csv")
sourceMonitorMethods = os.path.join(Path, version+"_methods.csv")
checkStyle = os.path.join(versPath, "checkAll", version+".xml")
checkStyleMethods = os.path.join(versPath, "checkAllMethodsData", version+".txt")
blamePath = os.path.join(Path, "blame")
BuildAllOneTimeCommits(utilsConf.get_configuration().gitPath, dbPath, JavaDocPath,
sourceMonitorFiles, sourceMonitorMethods, checkStyle, checkStyleMethods,
blamePath, date, CodeDir)
def Extract_complexity_features():
processes = []
for version_path, version_name, git_version in zip(utilsConf.get_configuration().vers_paths, utilsConf.get_configuration().vers_dirs,
utilsConf.get_configuration().vers):
pathRepo=os.path.join(version_path,"repo")
run_commands = SourceMonitorXml(utilsConf.get_configuration().workingDir, version_name, utilsConf.get_configuration().sourceMonitorEXE)
proc = utilsConf.open_subprocess(run_commands, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)
processes.append((proc, run_commands))
run_commands = ["java", "-jar", utilsConf.get_configuration().checkStyle68, "-c", utilsConf.get_configuration().methodsNamesXML,"javaFile","-o","vers/checkAllMethodsData/"+version_name+".txt",pathRepo]
proc = utilsConf.open_subprocess(run_commands, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True, cwd=utilsConf.to_short_path(utilsConf.get_configuration().workingDir))
processes.append((proc, run_commands))
run_commands = ["java", "-jar", utilsConf.get_configuration().checkStyle57, "-c", utilsConf.get_configuration().allchecks,"-r",pathRepo,"-f","xml","-o","vers/checkAll/"+version_name+".xml"]
proc = utilsConf.open_subprocess(run_commands, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True, cwd=utilsConf.to_short_path(utilsConf.get_configuration().workingDir))
processes.append((proc, run_commands))
blameExecute(version_path, pathRepo, git_version)
for proc, run_commands in processes:
out, err = proc.communicate()
if proc.returncode != 0:
RuntimeWarning('subprocess execution failed. args are {0}. err is {1}'.format(str(run_commands), err))
def distribution_for_tags(tags):
tags_names = map(lambda tag: tag.name, tags)
tags_dates = [datetime.datetime(1900, 1, 1, 0, 0).strftime('%Y-%m-%d %H:%M:%S')] + map(
lambda tag: datetime.datetime.fromtimestamp(tag.commit.committed_date).strftime('%Y-%m-%d %H:%M:%S'), tags)
headers_per_version = ["granularity", "buggedType", "version_name", "valid", "bug"]
rows_per_version = [headers_per_version]
for granularity in wekaMethods.articles.BUG_QUERIES:
for buggedType in wekaMethods.articles.BUG_QUERIES[granularity]:
dbpath = os.path.join(utilsConf.get_configuration().db_dir,
str(utilsConf.get_configuration().vers_dirs[-1] + ".db"))
for i, version_name in list(enumerate(tags_names))[:-1]:
prev_date, start_date, end_date = tags_dates[i: i + 3]
counts = {'valid': 0, 'bugged': 0}
files_hasBug = wekaMethods.articles.get_arff_class(dbpath, start_date, end_date,
wekaMethods.articles.BUG_QUERIES[granularity][
buggedType],
wekaMethods.articles.COMPONENTS_QUERIES[granularity])
counts.update(Counter(map(itemgetter(0), files_hasBug.values())))
report_values = [counts['valid'], counts['bugged']]
rows_per_version.append([granularity, buggedType, version_name] + report_values)
return rows_per_version
def download_bugs():
bugsPath = utilsConf.get_configuration().bugsPath
issue_tracker_url = utilsConf.get_configuration().issue_tracker_url
issue_tracker_product = utilsConf.get_configuration().issue_tracker_product
if utilsConf.get_configuration().issue_tracker == "bugzilla":
wekaMethods.issuesExtract.python_bugzilla.write_bugs_csv(bugsPath, issue_tracker_url, issue_tracker_product)
elif utilsConf.get_configuration().issue_tracker == "jira":
wekaMethods.issuesExtract.jira_import.jiraIssues(bugsPath, issue_tracker_url, issue_tracker_product)
elif utilsConf.get_configuration().issue_tracker == "github":
wekaMethods.issuesExtract.github_import.GithubIssues(bugsPath, issue_tracker_url, issue_tracker_product)
def create_instances(self):
MATRIX_PATH = os.path.join(utilsConf.get_configuration().experiments,
"{ITERATION}_{BUG_ID}_{GRANULARITY}_{BUGGED_TYPE}.matrix")
DESCRIPTION = 'sample bug id {BUG_ID} with bug_passing_probability = {PROB} with garnularity of {GRANULARITY} and bugged type {BUGGED_TYPE}'
i = 0.0
results = AvgResults()
while i < self.num_instances:
bug = random.choice(self.bugs)
buggy_components = set()
for component in set(self.components_priors.keys()):
for buggy in bug.get_buggy_components(self.granularity, self.bugged_type):
if component in buggy:
buggy_components.add(component)
if len(buggy_components) == 0:
continue
tests = reduce(set.__or__,
map(lambda x: self.test_runner.get_packages_tests().get('.'.join(x[:random.randint(0, len(x))]), set()),
map(lambda file_name: file_name.replace('.java', '').split('.'), buggy_components)),
set())
if len(tests) < self.tests_per_instance:
continue
relevant_tests = random.sample(tests, self.tests_per_instance)
tests_details = []
for test_name in relevant_tests:
trace = list(set(self.test_runner.tracer.traces[test_name].get_trace(self.granularity)) & set(
self.components_priors.keys()))
tests_details.append((test_name, trace, self.sample_observation(trace, buggy_components)))
if sum(map(lambda x: x[2], tests_details)) == 0:
continue
matrix = MATRIX_PATH.format(ITERATION=i, BUG_ID=bug.bug_id, GRANULARITY=self.granularity, BUGGED_TYPE=self.bugged_type)
write_planning_file(matrix, list(buggy_components),
filter(lambda test: len(test[1]) > 0, tests_details),
priors=self.components_priors,
description=DESCRIPTION.format(BUG_ID=bug.bug_id, PROB=self.bug_passing_probability,
GRANULARITY=self.granularity, BUGGED_TYPE=self.bugged_type))
inst = readPlanningFile(matrix)
inst.diagnose()
res = Diagnosis_Results(inst.diagnoses, inst.initial_tests, inst.error)
results.update(res)
print "created instance num {ITERATION} with bugid {BUG_ID} for granularity {GRANULARITY} and type {BUGGED_TYPE}".\
format(ITERATION=i, BUG_ID=bug.bug_id, GRANULARITY=self.granularity, BUGGED_TYPE=self.bugged_type)
i += 1
return results.results
def do_all():
patchD = os.path.join(utilsConf.get_configuration().LocalGitPath, "patch")
commitsFiles = os.path.join(utilsConf.get_configuration().LocalGitPath,
"commitsFiles")
changedFile = os.path.join(utilsConf.get_configuration().LocalGitPath,
"commitsFiles", "Ins_dels.txt")
mkdir(patchD)
mkdir(commitsFiles)
run_commands = "git format-patch --root -o patch --function-context --unified=9000".split(
)
proc = utilsConf.open_subprocess(
run_commands,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
shell=True,
cwd=utilsConf.to_short_path(
utilsConf.get_configuration().LocalGitPath))
proc.communicate()
buildPatchs(patchD, commitsFiles, changedFile)
checkOut = os.path.join(commitsFiles, "CheckStyle.txt")
RunCheckStyle(commitsFiles, checkOut,
utilsConf.get_configuration().checkStyle68,
utilsConf.get_configuration().methodsNamesXML)
def featuresExtract():
Extract_complexity_features()
Extract_OO_features_OLD()
wekaMethods.commsSpaces.create(utilsConf.get_configuration().vers_dirs, os.path.join(utilsConf.get_configuration().workingDir, "vers"))
wekaMethods.patchsBuild.do_all()
lastVer=os.path.join(dbadd,vers[-1]+".db")
testsDB=os.path.join(workingDir,"testsBugsMethods.db")
report.report(reportCsv,LocalGitPath,lastVer,testsDB)
def create_experiment(test_runner, num_instances=50, tests_per_instance=50, bug_passing_probability=0.05):
results_path = os.path.join(utilsConf.get_configuration().experiments, "{GRANULARITY}_{BUGGED_TYPE}")
for granularity in utilsConf.get_configuration().prediction_files:
for bugged_type in utilsConf.get_configuration().prediction_files[granularity]:
eg = ExperimentGenerator(test_runner, granularity, bugged_type, num_instances, tests_per_instance, bug_passing_probability)
results = eg.create_instances()
with open(results_path.format(GRANULARITY=granularity, BUGGED_TYPE=bugged_type), 'wb') as f:
f.write(json.dumps(results))
@utilsConf.marker_decorator(utilsConf.ALL_DONE_FILE)
def wrapperAll():
wrapperLearner()
create_experiment(executeTests())
if __name__ == '__main__':
csv.field_size_limit(sys.maxint)
utilsConf.configure(sys.argv[1])
shutil.copyfile(sys.argv[1], utilsConf.get_configuration().configuration_path)
if utilsConf.copy_from_cache() is not None:
exit()
if len(sys.argv) == 2:
wrapperAll()
utilsConf.export_to_cache()
elif sys.argv[2] =="learn":
wrapperLearner()
results = eg.create_instances()
with open(results_path.format(GRANULARITY=granularity, BUGGED_TYPE=bugged_type), 'wb') as f:
f.write(json.dumps(results))
# @utilsConf.marker_decorator(utilsConf.ALL_DONE_FILE)
def wrapperAll():
wrapperLearner()
executeTests()
# create_experiment(executeTests())
if __name__ == '__main__':
csv.field_size_limit(sys.maxint)
utilsConf.configure(sys.argv[1])
if not os.path.exists(utilsConf.get_configuration().configuration_path):
try:
shutil.copyfile(sys.argv[1], utilsConf.get_configuration().configuration_path)
except:
pass
# utilsConf.copy_from_cache()
if len(utilsConf.get_configuration().vers) < 3:
exit()
if len(sys.argv) == 2:
wrapperAll()
utilsConf.export_to_cache()
elif sys.argv[2] == "learn":
wrapperLearner()
elif sys.argv[2] == "executeTests":
executeTests()
pass
def create_instances(self):
MATRIX_PATH = os.path.join(
utilsConf.get_configuration().experiments,
"{ITERATION}_{BUG_ID}_{GRANULARITY}_{BUGGED_TYPE}.matrix")
DESCRIPTION = 'sample bug id {BUG_ID} with bug_passing_probability = {PROB} with garnularity of {GRANULARITY} and bugged type {BUGGED_TYPE}'
i = 0.0
results = AvgResults()
while i < self.num_instances:
bug = random.choice(self.bugs)
buggy_components = set()
for component in set(self.components_priors.keys()):
for buggy in bug.get_buggy_components(self.granularity,
self.bugged_type):
if component in buggy:
buggy_components.add(component)
if len(buggy_components) == 0:
continue
tests = reduce(
set.__or__,
map(
lambda x: self.test_runner.get_packages_tests().get(
'.'.join(x[:random.randint(0, len(x))]), set()),
map(
lambda file_name: file_name.replace('.java', '').split(
'.'), buggy_components)), set())
if len(tests) < self.tests_per_instance:
continue
relevant_tests = random.sample(tests, self.tests_per_instance)
tests_details = []
for test_name in relevant_tests:
trace = list(
set(self.test_runner.tracer.traces[test_name].get_trace(
self.granularity))
& set(self.components_priors.keys()))
tests_details.append(
(test_name, trace,
self.sample_observation(trace, buggy_components)))
if sum(map(lambda x: x[2], tests_details)) == 0:
continue
matrix = MATRIX_PATH.format(ITERATION=i,
BUG_ID=bug.bug_id,
GRANULARITY=self.granularity,
BUGGED_TYPE=self.bugged_type)
write_planning_file(matrix,
list(buggy_components),
filter(lambda test: len(test[1]) > 0,
tests_details),
priors=self.components_priors,
description=DESCRIPTION.format(
BUG_ID=bug.bug_id,
PROB=self.bug_passing_probability,
GRANULARITY=self.granularity,
BUGGED_TYPE=self.bugged_type))
inst = readPlanningFile(matrix)
inst.diagnose()
res = Diagnosis_Results(inst.diagnoses, inst.initial_tests,
inst.error)
results.update(res)
print "created instance num {ITERATION} with bugid {BUG_ID} for granularity {GRANULARITY} and type {BUGGED_TYPE}".\
format(ITERATION=i, BUG_ID=bug.bug_id, GRANULARITY=self.granularity, BUGGED_TYPE=self.bugged_type)
i += 1
return results.results
def buildBasicAllVers(vers, dates, dbsPath, bugsPath, MethodsParsed, changeFile):
gitPath = utilsConf.get_configuration().LocalGitPath
commits, commitedFiles, allMethodsCommits, bugs, allFilesCommitsPatch = BuildRepo(gitPath, bugsPath, MethodsParsed, changeFile)
for ver, date in zip(vers,dates):
dbPath = os.path.join(dbsPath, ver+".db")
basicBuildOneTimeCommits(dbPath, commits, commitedFiles, allMethodsCommits, bugs)
def build_labels():
db_dir = utilsConf.get_configuration().db_dir
vers = utilsConf.get_configuration().vers_dirs
dates = utilsConf.get_configuration().dates
buildBasicAllVers(vers, dates, db_dir, utilsConf.get_configuration().bugsPath,
utilsConf.get_configuration().MethodsParsed, utilsConf.get_configuration().changeFile)
def test_version_create():
src = os.path.join(utilsConf.get_configuration().workingDir,"vers", utilsConf.get_configuration().vers_dirs[-2], "repo")
dst = os.path.join(utilsConf.get_configuration().workingDir,"testedVer", "repo")
if not os.path.exists(dst):
shutil.copytree(src, dst)
def Extract_OO_features_OLD():
for version in utilsConf.get_configuration().vers:
verPath = os.path.join(utilsConf.get_configuration().versPath, version_to_dir_name(version))
command = """cd /d """ + utilsConf.to_short_path(verPath) + " & for /R .\\repo %f in (*.java) do (call javadoc -doclet com.github.markusbernhardt.xmldoclet.XmlDoclet -docletpath "+utilsConf.to_short_path(utilsConf.get_configuration().docletPath)+" -filename %~nxf.xml -private -d .\Jdoc2 %f >NUL 2>NUL)"
os.system(command)
def featuresExtract():
Extract_complexity_features()
# Extract_OO_features_OLD()
wekaMethods.commsSpaces.create(utilsConf.get_configuration().vers_dirs, os.path.join(utilsConf.get_configuration().workingDir, "vers"))