def _create_problems(self, db, problemplugin,
report_min_count=0, speedup=False):
if speedup:
db_reports = get_reports_for_problems(db, problemplugin.name)
db_reports += get_unassigned_reports(db, problemplugin.name,
min_count=report_min_count)
else:
db_reports = get_reports_by_type(db, problemplugin.name,
min_count=report_min_count)
db_problems = get_problems(db)
# dict to get db_problem by problem_id
self.log_debug("Creating problem reuse dict")
problems_dict = {}
for db_problem in db_problems:
problems_dict[db_problem.id] = db_problem
# dict to get report_ids by problem_id
problem_report = defaultdict(list)
for db_report in db_reports:
if db_report.problem_id is not None:
problem_report[db_report.problem_id].append(db_report.id)
# create lookup dict for problems
reuse_problems = {}
for (problem_id, report_ids) in problem_report.items():
reuse_problems[tuple(sorted(report_ids))] = problem_id
invalid_report_ids_to_clean = []
problems = []
if not db_reports:
self.log_info("No reports found")
elif len(db_reports) == 1:
db_report = db_reports[0]
if db_report.problem is None:
problems.append([db_report])
else:
report_map = {}
_satyr_reports = []
i = 0
for db_report in db_reports:
i += 1
self.log_debug("[{0} / {1}] Loading report #{2}"
.format(i, len(db_reports), db_report.id))
_satyr_report = problemplugin._db_report_to_satyr(db_report)
if _satyr_report is None:
self.log_debug("Unable to create satyr report")
if db_report.problem_id is not None:
invalid_report_ids_to_clean.append(db_report.id)
else:
_satyr_reports.append(_satyr_report)
report_map[_satyr_report] = db_report
db.session.expire(db_report)
self.log_debug("Clustering")
clusters = self._create_clusters(_satyr_reports, 2000)
# Threads that share no function with another thread
unique_func_threads = set(_satyr_reports) - set().union(*clusters)
dendrograms = []
i = 0
for cluster in clusters:
i += 1
self.log_debug("[{0} / {1}] Computing distances"
.format(i, len(clusters)))
distances = satyr.Distances(cluster, len(cluster))
self.log_debug("Getting dendrogram")
dendrograms.append(satyr.Dendrogram(distances))
for dendrogram, cluster in zip(dendrograms, clusters):
problem = []
for dups in dendrogram.cut(0.3, 1):
reports = set(report_map[cluster[dup]] for dup in dups)
problem.append(reports)
problems.extend(problem)
# Unique threads form their own unique problems
for thread in unique_func_threads:
problems.append({report_map[thread]})
self.log_info("Creating problems from clusters")
if speedup:
for problem in problems:
if not problem:
continue
first_report = next(iter(problem))
if len(problem) > 1:
# Find assigned report
origin_report = None
for db_report in problem:
if db_report.problem_id:
origin_report = db_report
# Problem created only from new reports
comps = {}
if not origin_report:
new = Problem()
db.session.add(new)
db.session.flush()
first_occurrence = first_report.first_occurrence
last_occurrence = first_report.last_occurrence
for rep in problem:
rep.problem_id = new.id
if first_occurrence > rep.first_occurrence:
first_occurrence = rep.first_occurrence
if last_occurrence < rep.last_occurrence:
last_occurrence = rep.last_occurrence
if rep.component not in comps:
comps[rep.component] = 0
comps[rep.component] += 1
self.update_comps(db, comps, new)
new.last_occurrence = last_occurrence
new.first_occurrence = first_occurrence
else:
first_occurrence = origin_report.first_occurrence
last_occurrence = origin_report.last_occurrence
for rep in problem:
if not rep.problem_id:
rep.problem_id = origin_report.problem_id
if first_occurrence > rep.first_occurrence:
first_occurrence = rep.first_occurrence
if last_occurrence < rep.last_occurrence:
last_occurrence = rep.last_occurrence
if rep.component not in comps:
comps[rep.component] = 0
comps[rep.component] += 1
orig_p = get_problem_by_id(db, origin_report.problem_id)
self.update_comps(db, comps, orig_p)
orig_p.last_occurrence = last_occurrence
orig_p.first_occurrence = first_occurrence
else:
# The report is assigned
if first_report.problem_id:
continue
else:
# One report that wasn't matched with anything else
new = Problem()
new.first_occurrence = first_report.first_occurrence
new.last_occurrence = first_report.last_occurrence
db.session.add(new)
db.session.flush()
self.update_comps(db, {first_report.component: 1}, new)
first_report.problem_id = new.id
db.session.flush()
else:
for problem, db_problem, reports_changed in self._iter_problems(
db, problems, db_problems, problems_dict, reuse_problems):
comps = {}
problem_last_occurrence = None
problem_first_occurrence = None
for db_report in problem:
db_report.problem = db_problem
if (problem_last_occurrence is None or
problem_last_occurrence < db_report.last_occurrence):
problem_last_occurrence = db_report.last_occurrence
if (problem_first_occurrence is None or
problem_first_occurrence > db_report.first_occurrence):
problem_first_occurrence = db_report.first_occurrence
if db_report.component not in comps:
comps[db_report.component] = 0
comps[db_report.component] += 1
# In case nothing changed, we don't want to mark db_problem
# dirty which would cause another UPDATE
if db_problem.first_occurrence != problem_first_occurrence:
db_problem.first_occurrence = problem_first_occurrence
if db_problem.last_occurrence != problem_last_occurrence:
db_problem.last_occurrence = problem_last_occurrence
if reports_changed:
self.update_comps(db, comps, db_problem)
self.log_debug("Removing {0} invalid reports from problems"
.format(len(invalid_report_ids_to_clean)))
for report_id in invalid_report_ids_to_clean:
db_report = get_report_by_id(db, report_id)
if db_report is not None:
db_report.problem_id = None
db.session.add(db_report)
if report_min_count > 0:
self.log_debug("Removing problems from low count reports")
remove_problem_from_low_count_reports_by_type(db,
problemplugin.name,
min_count=report_min_count)
self.log_debug("Flushing session")
db.session.flush()
def _create_problems(self, db, problemplugin,
report_min_count=0, speedup=False):
if speedup:
db_reports = get_reports_for_problems(db, problemplugin.name)
db_reports += get_unassigned_reports(db, problemplugin.name,
min_count=report_min_count)
else:
db_reports = get_reports_by_type(db, problemplugin.name,
min_count=report_min_count)
db_problems = get_problems(db)
# dict to get db_problem by problem_id
self.log_debug("Creating problem reuse dict")
problems_dict = {}
for db_problem in db_problems:
problems_dict[db_problem.id] = db_problem
# dict to get report_ids by problem_id
problem_report = defaultdict(list)
for db_report in db_reports:
if db_report.problem_id is not None:
problem_report[db_report.problem_id].append(db_report.id)
# create lookup dict for problems
reuse_problems = {}
for (problem_id, report_ids) in problem_report.items():
reuse_problems[tuple(sorted(report_ids))] = problem_id
invalid_report_ids_to_clean = []
problems = []
if not db_reports:
self.log_info("No reports found")
elif len(db_reports) == 1:
db_report = db_reports[0]
if db_report.problem is None:
problems.append([db_report])
else:
report_map = {}
_satyr_reports = []
i = 0
for db_report in db_reports:
i += 1
self.log_debug("[{0} / {1}] Loading report #{2}"
.format(i, len(db_reports), db_report.id))
_satyr_report = problemplugin._db_report_to_satyr(db_report)
if _satyr_report is None:
self.log_debug("Unable to create satyr report")
if db_report.problem_id is not None:
invalid_report_ids_to_clean.append(db_report.id)
else:
_satyr_reports.append(_satyr_report)
report_map[_satyr_report] = db_report
db.session.expire(db_report)
self.log_debug("Clustering")
clusters = self._create_clusters(_satyr_reports, 2000)
# Threads that share no function with another thread
unique_func_threads = set(_satyr_reports) - set().union(*clusters)
dendrograms = []
i = 0
for cluster in clusters:
i += 1
self.log_debug("[{0} / {1}] Computing distances"
.format(i, len(clusters)))
distances = satyr.Distances(cluster, len(cluster))
self.log_debug("Getting dendrogram")
dendrograms.append(satyr.Dendrogram(distances))
for dendrogram, cluster in zip(dendrograms, clusters):
problem = []
for dups in dendrogram.cut(0.3, 1):
reports = set(report_map[cluster[dup]] for dup in dups)
problem.append(reports)
problems.extend(problem)
# Unique threads form their own unique problems
for thread in unique_func_threads:
problems.append({report_map[thread]})
self.log_info("Creating problems from clusters")
if speedup:
for problem in problems:
if not problem:
continue
first_report = next(iter(problem))
if len(problem) > 1:
# Find assigned report
origin_report = None
for db_report in problem:
if db_report.problem_id:
origin_report = db_report
# Problem created only from new reports
comps = {}
if not origin_report:
new = Problem()
db.session.add(new)
db.session.flush()
first_occurrence = first_report.first_occurrence
last_occurrence = first_report.last_occurrence
for rep in problem:
rep.problem_id = new.id
if first_occurrence > rep.first_occurrence:
first_occurrence = rep.first_occurrence
if last_occurrence < rep.last_occurrence:
last_occurrence = rep.last_occurrence
if rep.component not in comps:
comps[rep.component] = 0
comps[rep.component] += 1
self.update_comps(db, comps, new)
new.last_occurrence = last_occurrence
new.first_occurrence = first_occurrence
else:
first_occurrence = origin_report.first_occurrence
last_occurrence = origin_report.last_occurrence
for rep in problem:
if not rep.problem_id:
rep.problem_id = origin_report.problem_id
if first_occurrence > rep.first_occurrence:
first_occurrence = rep.first_occurrence
if last_occurrence < rep.last_occurrence:
last_occurrence = rep.last_occurrence
if rep.component not in comps:
comps[rep.component] = 0
comps[rep.component] += 1
orig_p = get_problem_by_id(db, origin_report.problem_id)
self.update_comps(db, comps, orig_p)
orig_p.last_occurrence = last_occurrence
orig_p.first_occurrence = first_occurrence
else:
# The report is assigned
if first_report.problem_id:
continue
else:
# One report that wasn't matched with anything else
new = Problem()
new.first_occurrence = first_report.first_occurrence
new.last_occurrence = first_report.last_occurrence
db.session.add(new)
db.session.flush()
self.update_comps(db, {first_report.component: 1}, new)
first_report.problem_id = new.id
db.session.flush()
else:
for problem, db_problem, reports_changed in self._iter_problems(
db, problems, db_problems, problems_dict, reuse_problems):
comps = {}
problem_last_occurrence = None
problem_first_occurrence = None
for db_report in problem:
db_report.problem = db_problem
if (problem_last_occurrence is None or
problem_last_occurrence < db_report.last_occurrence):
problem_last_occurrence = db_report.last_occurrence
if (problem_first_occurrence is None or
problem_first_occurrence > db_report.first_occurrence):
problem_first_occurrence = db_report.first_occurrence
if db_report.component not in comps:
comps[db_report.component] = 0
comps[db_report.component] += 1
# In case nothing changed, we don't want to mark db_problem
# dirty which would cause another UPDATE
if db_problem.first_occurrence != problem_first_occurrence:
db_problem.first_occurrence = problem_first_occurrence
if db_problem.last_occurrence != problem_last_occurrence:
db_problem.last_occurrence = problem_last_occurrence
if reports_changed:
self.update_comps(db, comps, db_problem)
self.log_debug("Removing {0} invalid reports from problems"
.format(len(invalid_report_ids_to_clean)))
for report_id in invalid_report_ids_to_clean:
db_report = get_report_by_id(db, report_id)
if db_report is not None:
db_report.problem_id = None
db.session.add(db_report)
if report_min_count > 0:
self.log_debug("Removing problems from low count reports")
remove_problem_from_low_count_reports_by_type(db,
problemplugin.name,
min_count=report_min_count)
self.log_debug("Flushing session")
db.session.flush()
def _create_problems(self, db, problemplugin):
db_reports = get_reports_by_type(db, problemplugin.name)
db_problems = get_problems(db)
# dict to get db_problem by problem_id
self.log_debug("Creating problem reuse dict")
problems_dict = {}
for db_problem in db_problems:
problems_dict[db_problem.id] = db_problem
# dict to get report_ids by problem_id
problem_report = defaultdict(list)
for db_report in db_reports:
if db_report.problem_id is not None:
problem_report[db_report.problem_id].append(db_report.id)
# create lookup dict for problems
reuse_problems = {}
for (problem_id, report_ids) in problem_report.items():
reuse_problems[tuple(sorted(report_ids))] = problem_id
invalid_report_ids_to_clean = []
problems = []
if len(db_reports) < 1:
self.log_info("No reports found")
elif len(db_reports) == 1:
db_report = db_reports[0]
if db_report.problem is None:
problems.append([db_report])
else:
report_map = {}
_satyr_reports = []
i = 0
for db_report in db_reports:
i += 1
self.log_debug("[{0} / {1}] Loading report #{2}"
.format(i, len(db_reports), db_report.id))
_satyr_report = problemplugin._db_report_to_satyr(db_report)
if _satyr_report is None:
self.log_debug("Unable to create satyr report")
if db_report.problem_id is not None:
invalid_report_ids_to_clean.append(db_report.id)
else:
_satyr_reports.append(_satyr_report)
report_map[_satyr_report] = db_report
db.session.expire(db_report)
self.log_debug("Clustering")
clusters = self._create_clusters(_satyr_reports, 2000)
unique_func_threads = set(_satyr_reports) - set().union(*clusters)
dendrograms = []
i = 0
for cluster in clusters:
i += 1
self.log_debug("[{0} / {1}] Computing distances"
.format(i, len(clusters)))
distances = satyr.Distances(cluster, len(cluster))
self.log_debug("Getting dendrogram")
dendrograms.append(satyr.Dendrogram(distances))
for dendrogram, cluster in zip(dendrograms, clusters):
problem = []
for dups in dendrogram.cut(0.3, 1):
reports = set(report_map[cluster[dup]] for dup in dups)
problem.append(reports)
problems.extend(problem)
for thread in unique_func_threads:
problems.append(set([report_map[thread]]))
self.log_info("Creating problems from clusters")
for problem, db_problem, reports_changed in self._iter_problems(
db, problems, db_problems, problems_dict, reuse_problems):
comps = {}
problem_last_occurrence = None
problem_first_occurrence = None
for db_report in problem:
db_report.problem = db_problem
if (problem_last_occurrence is None or
problem_last_occurrence < db_report.last_occurrence):
problem_last_occurrence = db_report.last_occurrence
if (problem_first_occurrence is None or
problem_first_occurrence > db_report.first_occurrence):
problem_first_occurrence = db_report.first_occurrence
if db_report.component not in comps:
comps[db_report.component] = 0
comps[db_report.component] += 1
# In case nothing changed, we don't want to mark db_problem dirty
# which would cause another UPDATE
if db_problem.first_occurrence != problem_first_occurrence:
db_problem.first_occurrence = problem_first_occurrence
if db_problem.last_occurrence != problem_last_occurrence:
db_problem.last_occurrence = problem_last_occurrence
if reports_changed:
db_comps = sorted(comps, key=lambda x: comps[x], reverse=True)
order = 0
for db_component in db_comps:
order += 1
db_pcomp = get_problem_component(db, db_problem, db_component)
if db_pcomp is None:
db_pcomp = ProblemComponent()
db_pcomp.problem = db_problem
db_pcomp.component = db_component
db_pcomp.order = order
db.session.add(db_pcomp)
self.log_debug("Removing {0} invalid reports from problems"
.format(len(invalid_report_ids_to_clean)))
for report_id in invalid_report_ids_to_clean:
db_report = get_report_by_id(db, report_id)
if db_report is not None:
db_report.problem_id = None
db.session.add(db_report)
self.log_debug("Flushing session")
db.session.flush()