def reports(self, count=100):
comps = self.ses.query(OpSysComponent).all()
releases = self.ses.query(OpSysRelease).all()
arches = self.ses.query(Arch).all()
symbols = self.ses.query(SymbolSource).all()
bugs = self.ses.query(BzBug).all()
for rel in releases:
self.begin('Reports for %s %s' % (rel.opsys.name, rel.version))
since = rel.releasedate
if since is None:
since = datetime.now().date() + fuzzy_timedelta(
months=random.randrange(-6, 0))
till = self.get_release_end_date(since, rel.opsys)
for _ in range(count):
report = Report()
report.type = 'USERSPACE'
report.count = random.randrange(1, 20)
occ_date = self.get_occurrence_date(since, till)
if occ_date > datetime.now():
# skipping reports from the future
continue
report.first_occurrence = report.last_occurrence = occ_date
report.component = random.choice(comps)
self.add(report)
report_bt = ReportBacktrace()
report_bt.report = report
self.add(report_bt)
bthash = ReportBtHash()
bthash.type = 'NAMES'
bthash.hash = randutils.randhash()
bthash.backtrace = report_bt
self.add(bthash)
for j in range(random.randrange(1, 40)):
btframe = ReportBtFrame()
btframe.backtrace = report_bt
btframe.order = j
btframe.symbolsource = random.choice(symbols)
report_bt.crashfn = btframe.symbolsource.symbol.name
current = []
last_occ = occ_date
for j in range(report.count):
if j > 1:
occ_date = self.get_occurrence_date(since, till)
if occ_date > datetime.now():
continue
if occ_date > last_occ:
last_occ = occ_date
arch = random.choice(arches)
day = occ_date.date()
week = day - timedelta(days=day.weekday())
month = day.replace(day=1)
smode = random.choice(['DISABLED', 'PERMISSIVE', 'ENFORCING'])
stat_map = [(ReportArch, [('arch', arch)]),
(ReportOpSysRelease, [('opsysrelease', rel)]),
(ReportHistoryMonthly, [('opsysrelease', rel),
('month', month)]),
(ReportHistoryWeekly, [('opsysrelease', rel),
('week', week)]),
(ReportHistoryDaily, [('opsysrelease', rel),
('day', day)]),
(ReportSelinuxMode, [('mode', smode)]),
]
# add crashed and related packages
for typ in ['CRASHED', 'RELATED']:
for pkg in locals()['{0}_pkgs'.format(typ.lower())]:
if randutils.toss():
stat_map.append(
(ReportPackage,
[('installed_package', pkg),
('running_package', pkg),
('type', typ)])
)
if randutils.tosshigh():
stat_map.append((ReportUptime, [('uptime_exp',
int(math.log(random.randrange(1, 100000))))]))
if randutils.toss():
stat_map.append((ReportOpSysRelease,
[('opsysrelease', random.choice(releases))]))
if randutils.tosslow():
stat_map.append((ReportBz,
[('bzbug', random.choice(bugs))]))
for table, cols in stat_map:
for report_stat in current:
if not isinstance(report_stat, table):
continue
matching = True
for name, value in cols:
if getattr(report_stat, name) != value:
matching = False
if matching:
report_stat.count += 1
break
else:
report_stat = table()
report_stat.report = report
for name, value in cols:
setattr(report_stat, name, value)
report_stat.count = 1
current.append(report_stat)
self.extend(current)
report.last_occurrence = last_occ
self.commit()
def reports(self, count=100):
comps = self.ses.query(OpSysComponent).all()
releases = self.ses.query(OpSysRelease).all()
arches = self.ses.query(Arch).all()
symbols = self.ses.query(SymbolSource).all()
packages = self.ses.query(Package).all()
bugs = self.ses.query(RhbzBug).all()
for rel in releases:
self.begin('Reports for %s %s' % (rel.opsys.name, rel.version))
since = rel.releasedate
if since is None:
since = datetime.now().date() + fuzzy_timedelta(
months=random.randrange(-6, 0))
till = self.get_release_end_date(since, rel.opsys)
for i in range(count):
report = Report()
report.type = 'USERSPACE'
report.count = random.randrange(1, 20)
occ_date = self.get_occurence_date(since, till)
if occ_date > datetime.now():
# skipping reports from the future
continue
report.first_occurence = report.last_occurence = occ_date
report.component = random.choice(comps)
self.add(report)
report_bt = ReportBacktrace()
report_bt.report = report
self.add(report_bt)
bthash = ReportBtHash()
bthash.type = 'NAMES'
bthash.hash = randutils.randhash()
bthash.backtrace = report_bt
self.add(bthash)
for j in range(random.randrange(1, 40)):
btframe = ReportBtFrame()
btframe.backtrace = report_bt
btframe.order = j
btframe.symbolsource = random.choice(symbols)
report_bt.crashfn = btframe.symbolsource.symbol.name
current = []
last_occ = occ_date
crashed_pkgs = random.sample(packages, random.randrange(1,4))
related_pkgs = random.sample(packages, random.randrange(1,4))
for j in range(report.count):
if j > 1:
occ_date = self.get_occurence_date(since, till)
if occ_date > datetime.now():
continue
if occ_date > last_occ:
last_occ = occ_date
arch = random.choice(arches)
day = occ_date.date()
week = day - timedelta(days=day.weekday())
month = day.replace(day=1)
smode = random.choice(['DISABLED', 'PERMISSIVE', 'ENFORCING'])
stat_map = [(ReportArch, [('arch', arch)]),
(ReportOpSysRelease, [('opsysrelease', rel)]),
(ReportHistoryMonthly, [('opsysrelease', rel),
('month', month)]),
(ReportHistoryWeekly, [('opsysrelease', rel),
('week', week)]),
(ReportHistoryDaily, [('opsysrelease', rel),
('day', day)]),
(ReportSelinuxMode, [('mode', smode)]),
]
# add crashed and related packages
for typ in ['CRASHED', 'RELATED']:
for pkg in locals()['{0}_pkgs'.format(typ.lower())]:
if randutils.toss():
stat_map.append(
(ReportPackage,
[('installed_package', pkg),
('running_package', pkg),
('type', typ)])
)
if randutils.tosshigh():
stat_map.append((ReportUptime, [('uptime_exp',
int(math.log(random.randrange(1, 100000))))]))
if randutils.toss():
stat_map.append((ReportOpSysRelease,
[('opsysrelease', random.choice(releases))]))
if randutils.tosslow():
stat_map.append((ReportRhbz,
[('rhbzbug', random.choice(bugs))]))
for table, cols in stat_map:
fn = lambda x: type(x) == table
for report_stat in filter(fn, current):
matching = True
for name, value in cols:
if getattr(report_stat, name) != value:
matching = False
if matching:
report_stat.count += 1
break
else:
report_stat = table()
report_stat.report = report
for name, value in cols:
setattr(report_stat, name, value)
report_stat.count = 1
current.append(report_stat)
self.extend(current)
report.last_occurence = last_occ
self.commit()
def add_report(ureport, db, utctime=None, count=1, only_check_if_known=False, return_report=False):
if not utctime:
utctime = datetime.datetime.utcnow()
flip_corebt_if_necessary(ureport)
if "component" in ureport:
component = get_component(ureport["component"], ureport["os"], db)
else:
component = guess_component(ureport["installed_package"], ureport["os"], db)
if component is None:
raise Exception, "Unknown component."
for frame in ureport["core_backtrace"]:
if not "path" in frame and "executable" in ureport:
frame["path"] = ureport["executable"]
if ureport["type"].lower() != "kerneloops":
frame["path"] = os.path.abspath(frame["path"])
hash_type, hash_hash = get_report_hash(ureport, component.name)
# Find a report with matching hash and component.
report = (
db.session.query(Report)
.join(ReportBacktrace)
.join(ReportBtHash)
.filter((ReportBtHash.hash == hash_hash) & (ReportBtHash.type == hash_type) & (Report.component == component))
.first()
)
if only_check_if_known:
# check whether the report has a BZ associated
# if it does not, proclaim it unknown
if report:
reportbz = db.session.query(ReportRhbz).filter(ReportRhbz.report_id == report.id).first()
solution = find_ureport_kb_solution(ureport, db, opsys_id=report.component.opsys.id)
if not reportbz and not solution:
report = None
if return_report:
return report
return bool(report)
# Create a new report if not found.
if not report:
# do not process reports with empty
# core-backtrace, except for selinux
if ureport["type"].lower() != "selinux" and len(ureport["core_backtrace"]) < 1:
raise Exception, "empty core_backtrace"
report = Report()
report.type = ureport["type"].upper()
report.first_occurence = report.last_occurence = utctime
report.count = count
report.component = component
db.session.add(report)
report_backtrace = ReportBacktrace()
report_backtrace.report = report
db.session.add(report_backtrace)
report_bthash = ReportBtHash()
report_bthash.type = hash_type
report_bthash.hash = hash_hash
report_bthash.backtrace = report_backtrace
db.session.add(report_bthash)
# Add frames, symbols, hashes and sources.
for frame in get_crash_thread(ureport):
report_btframe = ReportBtFrame()
report_btframe.backtrace = report_backtrace
report_btframe.order = frame["frame"]
if not "buildid" in frame:
frame["buildid"] = None
# Check if there was already such symbol added, but first check
# the current session before doing a query.
for symbolsource in (x for x in db.session.new if type(x) is SymbolSource):
if (
symbolsource.build_id == frame["buildid"]
and symbolsource.path == frame["path"]
and symbolsource.offset == frame["offset"]
):
break
else:
symbolsource = (
db.session.query(SymbolSource)
.filter(
(SymbolSource.build_id == frame["buildid"])
& (SymbolSource.path == frame["path"])
& (SymbolSource.offset == frame["offset"])
)
.first()
)
# Create a new symbolsource if not found.
if not symbolsource:
symbolsource = SymbolSource()
symbolsource.build_id = frame["buildid"]
symbolsource.path = frame["path"]
symbolsource.offset = frame["offset"]
if ureport["type"].lower() == "python":
symbolsource.source_path = frame["path"]
symbolsource.line_number = frame["offset"]
if "funchash" in frame:
symbolsource.hash = frame["funchash"]
if "funcname" in frame:
normalized_path = get_libname(frame["path"])
for symbol in (x for x in db.session.new if type(x) is Symbol):
if symbol.name == frame["funcname"] and symbol.normalized_path == normalized_path:
break
else:
symbol = (
db.session.query(Symbol)
.filter((Symbol.name == frame["funcname"]) & (Symbol.normalized_path == normalized_path))
.first()
)
# Create a new symbol if not found.
if not symbol:
symbol = Symbol()
symbol.name = frame["funcname"]
demangled = cpp_demangle(symbol.name)
if demangled != symbol.name:
symbol.nice_name = demangled
symbol.normalized_path = normalized_path
db.session.add(symbol)
symbolsource.symbol = symbol
db.session.add(symbolsource)
report_btframe.symbolsource = symbolsource
db.session.add(report_btframe)
else:
report.count += count
if report.last_occurence < utctime:
report.last_occurence = utctime
elif report.first_occurence > utctime:
report.first_occurence = utctime
if report.problem:
if report.problem.last_occurence < report.last_occurence:
report.problem.last_occurence = report.last_occurence
if report.problem.first_occurence > report.first_occurence:
report.problem.first_occurence = report.first_occurence
db.session.flush()
if report.type == "KERNELOOPS":
if not report.get_lob_fd("oops") and "oops" in ureport:
report.save_lob("oops", ureport["oops"])
# Update various stats.
opsysrelease = (
db.session.query(OpSysRelease)
.join(OpSys)
.filter((OpSysRelease.version == ureport["os"]["version"]) & (OpSys.name == ureport["os"]["name"]))
.one()
)
arch = db.session.query(Arch).filter_by(name=ureport["architecture"]).one()
day = utctime.date()
week = day - datetime.timedelta(days=day.weekday())
month = day.replace(day=1)
stat_map = [
(ReportArch, [("arch", arch)]),
(ReportOpSysRelease, [("opsysrelease", opsysrelease)]),
(ReportReason, [("reason", ureport["reason"])]),
(ReportHistoryMonthly, [("opsysrelease", opsysrelease), ("month", month)]),
(ReportHistoryWeekly, [("opsysrelease", opsysrelease), ("week", week)]),
(ReportHistoryDaily, [("opsysrelease", opsysrelease), ("day", day)]),
]
if "executable" in ureport:
stat_map.append((ReportExecutable, [("path", ureport["executable"])]))
if "uptime" in ureport:
if ureport["uptime"] < 0.1:
uptime_exp = -1
else:
uptime_exp = int(math.log(ureport["uptime"], 10))
stat_map.append((ReportUptime, [("uptime_exp", uptime_exp)]))
# Add the reported package (installed and running).
stat_map.append(get_package_stat("CRASHED", ureport, ureport["os"], db))
# Similarly add related packages.
if "related_packages" in ureport:
for related_package in ureport["related_packages"]:
stat_map.append(get_package_stat("RELATED", related_package, ureport["os"], db))
# Add selinux fields to stat_map
if "selinux" in ureport:
stat_map.append((ReportSelinuxMode, [("mode", ureport["selinux"]["mode"].upper())]))
if "context" in ureport["selinux"]:
stat_map.append((ReportSelinuxContext, [("context", ureport["selinux"]["context"])]))
if "policy_package" in ureport["selinux"]:
stat_map.append(
get_package_stat(
"SELINUX_POLICY", {"installed_package": ureport["selinux"]["policy_package"]}, ureport["os"], db
)
)
# Add kernel taint state fields to stat_map.
if "kernel_taint_state" in ureport:
stat_map.append((ReportKernelTaintState, [("state", ureport["kernel_taint_state"])]))
# Create missing stats and increase counters.
for table, cols in stat_map:
report_stat_query = db.session.query(table).join(Report).filter(Report.id == report.id)
for name, value in cols:
report_stat_query = report_stat_query.filter(getattr(table, name) == value)
report_stat = report_stat_query.first()
if not report_stat:
report_stat = table()
report_stat.report = report
for name, value in cols:
setattr(report_stat, name, value)
report_stat.count = 0
db.session.add(report_stat)
report_stat.count += count