def reports(self, count=100): comps = self.ses.query(OpSysComponent).all() releases = self.ses.query(OpSysRelease).all() arches = self.ses.query(Arch).all() symbols = self.ses.query(SymbolSource).all() packages = self.ses.query(Package).all() bugs = self.ses.query(RhbzBug).all() for rel in releases: self.begin('Reports for %s %s' % (rel.opsys.name, rel.version)) since = rel.releasedate if since is None: since = datetime.now().date() + fuzzy_timedelta( months=random.randrange(-6, 0)) till = self.get_release_end_date(since, rel.opsys) for i in range(count): report = Report() report.type = 'USERSPACE' report.count = random.randrange(1, 20) occ_date = self.get_occurence_date(since, till) if occ_date > datetime.now(): # skipping reports from the future continue report.first_occurence = report.last_occurence = occ_date report.component = random.choice(comps) self.add(report) report_bt = ReportBacktrace() report_bt.report = report self.add(report_bt) bthash = ReportBtHash() bthash.type = 'NAMES' bthash.hash = randutils.randhash() bthash.backtrace = report_bt self.add(bthash) for j in range(random.randrange(1, 40)): btframe = ReportBtFrame() btframe.backtrace = report_bt btframe.order = j btframe.symbolsource = random.choice(symbols) report_bt.crashfn = btframe.symbolsource.symbol.name current = [] last_occ = occ_date crashed_pkgs = random.sample(packages, random.randrange(1,4)) related_pkgs = random.sample(packages, random.randrange(1,4)) for j in range(report.count): if j > 1: occ_date = self.get_occurence_date(since, till) if occ_date > datetime.now(): continue if occ_date > last_occ: last_occ = occ_date arch = random.choice(arches) day = occ_date.date() week = day - timedelta(days=day.weekday()) month = day.replace(day=1) smode = random.choice(['DISABLED', 'PERMISSIVE', 'ENFORCING']) stat_map = [(ReportArch, [('arch', arch)]), (ReportOpSysRelease, [('opsysrelease', rel)]), (ReportHistoryMonthly, [('opsysrelease', rel), ('month', month)]), (ReportHistoryWeekly, [('opsysrelease', rel), ('week', week)]), (ReportHistoryDaily, [('opsysrelease', rel), ('day', day)]), (ReportSelinuxMode, [('mode', smode)]), ] # add crashed and related packages for typ in ['CRASHED', 'RELATED']: for pkg in locals()['{0}_pkgs'.format(typ.lower())]: if randutils.toss(): stat_map.append( (ReportPackage, [('installed_package', pkg), ('running_package', pkg), ('type', typ)]) ) if randutils.tosshigh(): stat_map.append((ReportUptime, [('uptime_exp', int(math.log(random.randrange(1, 100000))))])) if randutils.toss(): stat_map.append((ReportOpSysRelease, [('opsysrelease', random.choice(releases))])) if randutils.tosslow(): stat_map.append((ReportRhbz, [('rhbzbug', random.choice(bugs))])) for table, cols in stat_map: fn = lambda x: type(x) == table for report_stat in filter(fn, current): matching = True for name, value in cols: if getattr(report_stat, name) != value: matching = False if matching: report_stat.count += 1 break else: report_stat = table() report_stat.report = report for name, value in cols: setattr(report_stat, name, value) report_stat.count = 1 current.append(report_stat) self.extend(current) report.last_occurence = last_occ self.commit()
def setUp(self): super(MPFTestCase, self).setUp() self.basic_fixtures() # Fixed problem 1 problem = Problem() self.db.session.add(problem) report = Report() report.type = "type" report.count = 2 report.problem = problem report.component = self.comp_systemd self.db.session.add(report) with open("sample_reports/ureport_systemd2", "r") as file: self.ureport_systemd2 = json.load(file) with open("sample_reports/ureport_systemd77", "r") as file: self.ureport_systemd77 = json.load(file) problemplugin = problemtypes[self.ureport_systemd2["problem"]["type"]] report_hash = problemplugin.hash_ureport( self.ureport_systemd2["problem"]) db_hash = ReportHash() db_hash.hash = report_hash db_hash.report = report self.db.session.add(db_hash) report_unknown = ReportUnknownPackage() report_unknown.report = report report_unknown.name = "systemd-test" report_unknown.epoch = 0 report_unknown.version = 2 report_unknown.release = 1 report_unknown.semver = "2.0.0" report_unknown.semrel = "1.0.0" report_unknown.arch = self.arch_i686 report_unknown.count = 1 report_unknown.type = "CRASHED" self.db.session.add(report_unknown) report_unknown = ReportUnknownPackage() report_unknown.report = report report_unknown.name = "faf" report_unknown.epoch = 0 report_unknown.version = 1 report_unknown.release = 1 report_unknown.semver = "1.0.0" report_unknown.semrel = "1.0.0" report_unknown.arch = self.arch_i686 report_unknown.count = 1 report_unknown.type = "RELATED" self.db.session.add(report_unknown) report_osr = ReportOpSysRelease() report_osr.count = 77 report_osr.opsysrelease = self.release_20 report_osr.report = report self.db.session.add(report_osr) report_osr = ReportOpSysRelease() report_osr.count = 77 report_osr.opsysrelease = self.release_19 report_osr.report = report self.db.session.add(report_osr) self.problem_fixed1 = problem # Fixed problem 2 problem = Problem() self.db.session.add(problem) report = Report() report.type = "type" report.count = 2 report.problem = problem report.component = self.comp_faf self.db.session.add(report) build = Build() build.base_package_name = "faf" build.epoch = 0 build.version = "3.4" build.release = "5" self.db.session.add(build) pkg = Package() pkg.build = build pkg.arch = self.arch_i686 pkg.name = "faf-test" self.db.session.add(pkg) rpkg = ReportPackage() rpkg.report = report rpkg.type = "CRASHED" rpkg.installed_package = pkg rpkg.count = 66 self.db.session.add(rpkg) report_osr = ReportOpSysRelease() report_osr.count = 77 report_osr.opsysrelease = self.release_20 report_osr.report = report self.db.session.add(report_osr) self.problem_fixed2 = problem # Unfixed problem 1 problem = Problem() self.db.session.add(problem) report = Report() report.type = "type" report.count = 2 report.problem = problem report.component = self.comp_systemd self.db.session.add(report) report_unknown = ReportUnknownPackage() report_unknown.report = report report_unknown.name = "systemd" report_unknown.epoch = 0 report_unknown.version = 3 report_unknown.release = 1 report_unknown.semver = "3.0.0" report_unknown.semrel = "1.0.0" report_unknown.arch = self.arch_i686 report_unknown.count = 1 report_unknown.type = "CRASHED" self.db.session.add(report_unknown) report_osr = ReportOpSysRelease() report_osr.count = 77 report_osr.opsysrelease = self.release_20 report_osr.report = report self.db.session.add(report_osr) self.problem_unfixed1 = problem # Unfixed problem 2 problem = Problem() self.db.session.add(problem) report = Report() report.type = "type" report.count = 2 report.problem = problem report.component = self.comp_systemd self.db.session.add(report) report_unknown = ReportUnknownPackage() report_unknown.report = report report_unknown.name = "systemd" report_unknown.epoch = 0 report_unknown.version = 3 report_unknown.release = 0 report_unknown.semver = "3.0.0" report_unknown.semrel = "0.0.0" report_unknown.arch = self.arch_i686 report_unknown.count = 1 report_unknown.type = "CRASHED" self.db.session.add(report_unknown) report_osr = ReportOpSysRelease() report_osr.count = 77 report_osr.opsysrelease = self.release_20 report_osr.report = report self.db.session.add(report_osr) self.problem_unfixed2 = problem # Unfixed problem 3 problem = Problem() self.db.session.add(problem) report = Report() report.type = "type" report.count = 2 report.problem = problem report.component = self.comp_faf self.db.session.add(report) build = Build() build.base_package_name = "faf" build.epoch = 0 build.version = "3.110" build.release = "5" self.db.session.add(build) pkg = Package() pkg.build = build pkg.arch = self.arch_i686 pkg.name = "faf-test" self.db.session.add(pkg) rpkg = ReportPackage() rpkg.report = report rpkg.type = "CRASHED" rpkg.installed_package = pkg rpkg.count = 66 self.db.session.add(rpkg) report_osr = ReportOpSysRelease() report_osr.count = 77 report_osr.opsysrelease = self.release_20 report_osr.report = report self.db.session.add(report_osr) self.problem_unfixed3 = problem self.db.session.flush() systems['fedora'].get_released_builds = get_released_builds_mock systems['fedora'].build_aging_days = 7
def reports(self, count=100): comps = self.ses.query(OpSysComponent).all() releases = self.ses.query(OpSysRelease).all() arches = self.ses.query(Arch).all() symbols = self.ses.query(SymbolSource).all() bugs = self.ses.query(BzBug).all() for rel in releases: self.begin('Reports for %s %s' % (rel.opsys.name, rel.version)) since = rel.releasedate if since is None: since = datetime.now().date() + fuzzy_timedelta( months=random.randrange(-6, 0)) till = self.get_release_end_date(since, rel.opsys) for _ in range(count): report = Report() report.type = 'USERSPACE' report.count = random.randrange(1, 20) occ_date = self.get_occurrence_date(since, till) if occ_date > datetime.now(): # skipping reports from the future continue report.first_occurrence = report.last_occurrence = occ_date report.component = random.choice(comps) self.add(report) report_bt = ReportBacktrace() report_bt.report = report self.add(report_bt) bthash = ReportBtHash() bthash.type = 'NAMES' bthash.hash = randutils.randhash() bthash.backtrace = report_bt self.add(bthash) for j in range(random.randrange(1, 40)): btframe = ReportBtFrame() btframe.backtrace = report_bt btframe.order = j btframe.symbolsource = random.choice(symbols) report_bt.crashfn = btframe.symbolsource.symbol.name current = [] last_occ = occ_date for j in range(report.count): if j > 1: occ_date = self.get_occurrence_date(since, till) if occ_date > datetime.now(): continue if occ_date > last_occ: last_occ = occ_date arch = random.choice(arches) day = occ_date.date() week = day - timedelta(days=day.weekday()) month = day.replace(day=1) smode = random.choice(['DISABLED', 'PERMISSIVE', 'ENFORCING']) stat_map = [(ReportArch, [('arch', arch)]), (ReportOpSysRelease, [('opsysrelease', rel)]), (ReportHistoryMonthly, [('opsysrelease', rel), ('month', month)]), (ReportHistoryWeekly, [('opsysrelease', rel), ('week', week)]), (ReportHistoryDaily, [('opsysrelease', rel), ('day', day)]), (ReportSelinuxMode, [('mode', smode)]), ] # add crashed and related packages for typ in ['CRASHED', 'RELATED']: for pkg in locals()['{0}_pkgs'.format(typ.lower())]: if randutils.toss(): stat_map.append( (ReportPackage, [('installed_package', pkg), ('running_package', pkg), ('type', typ)]) ) if randutils.tosshigh(): stat_map.append((ReportUptime, [('uptime_exp', int(math.log(random.randrange(1, 100000))))])) if randutils.toss(): stat_map.append((ReportOpSysRelease, [('opsysrelease', random.choice(releases))])) if randutils.tosslow(): stat_map.append((ReportBz, [('bzbug', random.choice(bugs))])) for table, cols in stat_map: for report_stat in current: if not isinstance(report_stat, table): continue matching = True for name, value in cols: if getattr(report_stat, name) != value: matching = False if matching: report_stat.count += 1 break else: report_stat = table() report_stat.report = report for name, value in cols: setattr(report_stat, name, value) report_stat.count = 1 current.append(report_stat) self.extend(current) report.last_occurrence = last_occ self.commit()
def test_get_packages_and_their_reports_unknown_packages(self): """ """ # add required stuff to db arch = Arch() arch.name = "noarch" self.db.session.add(arch) arch1 = Arch() arch1.name = "x86_64" self.db.session.add(arch1) build = Build() build.base_package_name = "sample" build.version = "1" build.release = "1" build.epoch = "0" self.db.session.add(build) pkg = Package() pkg.name = "sample" pkg.pkgtype = "rpm" pkg.arch = arch pkg.build = build self.db.session.add(pkg) #print(pkg.nevra()) #different arch pkg2 = Package() pkg2.name = "sample" pkg2.pkgtype = "rpm" pkg2.arch = arch1 pkg2.build = build self.db.session.add(pkg2) #print(pkg2.nevra()) build2 = Build() build2.base_package_name = "sample" build2.version = "1" build2.release = "2" build2.epoch = "0" self.db.session.add(build2) pkg3 = Package() pkg3.name = "sample" pkg3.pkgtype = "rpm" pkg3.arch = arch pkg3.build = build2 self.db.session.add(pkg3) #print(pkg3.nevra()) problem = Problem() self.db.session.add(problem) opsys = OpSys() opsys.name = "Fedora" self.db.session.add(opsys) opsys_component = OpSysComponent() opsys_component.name = "core" opsys_component.opsys = opsys self.db.session.add(opsys_component) report = Report() report.type = "type" report.count = 2 report.problem = problem report.component = opsys_component self.db.session.add(report) report_unknown = ReportUnknownPackage() report_unknown.report = report report_unknown.name = pkg.name report_unknown.epoch = pkg.build.epoch report_unknown.version = pkg.build.version report_unknown.release = pkg.build.release report_unknown.arch = pkg.arch report_unknown.count = 1 self.db.session.add(report_unknown) report_unknown2 = ReportUnknownPackage() report_unknown2.report = report report_unknown2.name = pkg2.name report_unknown2.epoch = pkg2.build.epoch report_unknown2.version = pkg2.build.version report_unknown2.release = pkg2.build.release report_unknown2.arch = pkg2.arch report_unknown2.count = 1 self.db.session.add(report_unknown2) report_unknown3 = ReportUnknownPackage() report_unknown3.report = report report_unknown3.name = "nonsense" report_unknown3.epoch = pkg.build.epoch report_unknown3.version = pkg.build.version report_unknown3.release = pkg.build.release report_unknown3.arch = pkg.arch report_unknown3.count = 1 self.db.session.add(report_unknown3) self.db.session.flush() packages_and_their_reports_unknown_packages = \ get_packages_and_their_reports_unknown_packages(self.db).all() self.assertEqual(len(packages_and_their_reports_unknown_packages), 2) self.assertIn((pkg, report_unknown), packages_and_their_reports_unknown_packages) self.assertIn((pkg2, report_unknown2), packages_and_their_reports_unknown_packages)
def test_get_packages_and_their_reports_unknown_packages(self): """ """ # add required stuff to db arch = Arch() arch.name = "noarch" self.db.session.add(arch) arch1 = Arch() arch1.name = "x86_64" self.db.session.add(arch1) build = Build() build.base_package_name = "sample" build.version = "1" build.release = "1" build.epoch = "0" self.db.session.add(build) pkg = Package() pkg.name = "sample" pkg.pkgtype = "rpm" pkg.arch = arch pkg.build = build self.db.session.add(pkg) #print(pkg.nevra()) #different arch pkg2 = Package() pkg2.name = "sample" pkg2.pkgtype = "rpm" pkg2.arch = arch1 pkg2.build = build self.db.session.add(pkg2) #print(pkg2.nevra()) build2 = Build() build2.base_package_name = "sample" build2.version = "1" build2.release = "2" build2.epoch = "0" self.db.session.add(build2) pkg3 = Package() pkg3.name = "sample" pkg3.pkgtype = "rpm" pkg3.arch = arch pkg3.build = build2 self.db.session.add(pkg3) #print(pkg3.nevra()) problem = Problem() self.db.session.add(problem) opsys = OpSys() opsys.name = "Fedora" self.db.session.add(opsys) opsys_component = OpSysComponent() opsys_component.name = "core" opsys_component.opsys = opsys self.db.session.add(opsys_component) report = Report() report.type = "type" report.count = 2 report.problem = problem report.component = opsys_component self.db.session.add(report) report_unknown = ReportUnknownPackage() report_unknown.report = report report_unknown.name = pkg.name report_unknown.epoch = pkg.build.epoch report_unknown.version = pkg.build.version report_unknown.release = pkg.build.release report_unknown.arch = pkg.arch report_unknown.count = 1 self.db.session.add(report_unknown) report_unknown2 = ReportUnknownPackage() report_unknown2.report = report report_unknown2.name = pkg2.name report_unknown2.epoch = pkg2.build.epoch report_unknown2.version = pkg2.build.version report_unknown2.release = pkg2.build.release report_unknown2.arch = pkg2.arch report_unknown2.count = 1 self.db.session.add(report_unknown2) report_unknown3 = ReportUnknownPackage() report_unknown3.report = report report_unknown3.name = "nonsense" report_unknown3.epoch = pkg.build.epoch report_unknown3.version = pkg.build.version report_unknown3.release = pkg.build.release report_unknown3.arch = pkg.arch report_unknown3.count = 1 self.db.session.add(report_unknown3) self.db.session.flush() packages_and_their_reports_unknown_packages = \ get_packages_and_their_reports_unknown_packages(self.db).all() self.assertEqual(len(packages_and_their_reports_unknown_packages), 2) self.assertIn( (pkg, report_unknown), packages_and_their_reports_unknown_packages) self.assertIn( (pkg2, report_unknown2), packages_and_their_reports_unknown_packages)
def add_report(ureport, db, utctime=None, count=1, only_check_if_known=False, return_report=False): if not utctime: utctime = datetime.datetime.utcnow() flip_corebt_if_necessary(ureport) if "component" in ureport: component = get_component(ureport["component"], ureport["os"], db) else: component = guess_component(ureport["installed_package"], ureport["os"], db) if component is None: raise Exception, "Unknown component." for frame in ureport["core_backtrace"]: if not "path" in frame and "executable" in ureport: frame["path"] = ureport["executable"] if ureport["type"].lower() != "kerneloops": frame["path"] = os.path.abspath(frame["path"]) hash_type, hash_hash = get_report_hash(ureport, component.name) # Find a report with matching hash and component. report = ( db.session.query(Report) .join(ReportBacktrace) .join(ReportBtHash) .filter((ReportBtHash.hash == hash_hash) & (ReportBtHash.type == hash_type) & (Report.component == component)) .first() ) if only_check_if_known: # check whether the report has a BZ associated # if it does not, proclaim it unknown if report: reportbz = db.session.query(ReportRhbz).filter(ReportRhbz.report_id == report.id).first() solution = find_ureport_kb_solution(ureport, db, opsys_id=report.component.opsys.id) if not reportbz and not solution: report = None if return_report: return report return bool(report) # Create a new report if not found. if not report: # do not process reports with empty # core-backtrace, except for selinux if ureport["type"].lower() != "selinux" and len(ureport["core_backtrace"]) < 1: raise Exception, "empty core_backtrace" report = Report() report.type = ureport["type"].upper() report.first_occurence = report.last_occurence = utctime report.count = count report.component = component db.session.add(report) report_backtrace = ReportBacktrace() report_backtrace.report = report db.session.add(report_backtrace) report_bthash = ReportBtHash() report_bthash.type = hash_type report_bthash.hash = hash_hash report_bthash.backtrace = report_backtrace db.session.add(report_bthash) # Add frames, symbols, hashes and sources. for frame in get_crash_thread(ureport): report_btframe = ReportBtFrame() report_btframe.backtrace = report_backtrace report_btframe.order = frame["frame"] if not "buildid" in frame: frame["buildid"] = None # Check if there was already such symbol added, but first check # the current session before doing a query. for symbolsource in (x for x in db.session.new if type(x) is SymbolSource): if ( symbolsource.build_id == frame["buildid"] and symbolsource.path == frame["path"] and symbolsource.offset == frame["offset"] ): break else: symbolsource = ( db.session.query(SymbolSource) .filter( (SymbolSource.build_id == frame["buildid"]) & (SymbolSource.path == frame["path"]) & (SymbolSource.offset == frame["offset"]) ) .first() ) # Create a new symbolsource if not found. if not symbolsource: symbolsource = SymbolSource() symbolsource.build_id = frame["buildid"] symbolsource.path = frame["path"] symbolsource.offset = frame["offset"] if ureport["type"].lower() == "python": symbolsource.source_path = frame["path"] symbolsource.line_number = frame["offset"] if "funchash" in frame: symbolsource.hash = frame["funchash"] if "funcname" in frame: normalized_path = get_libname(frame["path"]) for symbol in (x for x in db.session.new if type(x) is Symbol): if symbol.name == frame["funcname"] and symbol.normalized_path == normalized_path: break else: symbol = ( db.session.query(Symbol) .filter((Symbol.name == frame["funcname"]) & (Symbol.normalized_path == normalized_path)) .first() ) # Create a new symbol if not found. if not symbol: symbol = Symbol() symbol.name = frame["funcname"] demangled = cpp_demangle(symbol.name) if demangled != symbol.name: symbol.nice_name = demangled symbol.normalized_path = normalized_path db.session.add(symbol) symbolsource.symbol = symbol db.session.add(symbolsource) report_btframe.symbolsource = symbolsource db.session.add(report_btframe) else: report.count += count if report.last_occurence < utctime: report.last_occurence = utctime elif report.first_occurence > utctime: report.first_occurence = utctime if report.problem: if report.problem.last_occurence < report.last_occurence: report.problem.last_occurence = report.last_occurence if report.problem.first_occurence > report.first_occurence: report.problem.first_occurence = report.first_occurence db.session.flush() if report.type == "KERNELOOPS": if not report.get_lob_fd("oops") and "oops" in ureport: report.save_lob("oops", ureport["oops"]) # Update various stats. opsysrelease = ( db.session.query(OpSysRelease) .join(OpSys) .filter((OpSysRelease.version == ureport["os"]["version"]) & (OpSys.name == ureport["os"]["name"])) .one() ) arch = db.session.query(Arch).filter_by(name=ureport["architecture"]).one() day = utctime.date() week = day - datetime.timedelta(days=day.weekday()) month = day.replace(day=1) stat_map = [ (ReportArch, [("arch", arch)]), (ReportOpSysRelease, [("opsysrelease", opsysrelease)]), (ReportReason, [("reason", ureport["reason"])]), (ReportHistoryMonthly, [("opsysrelease", opsysrelease), ("month", month)]), (ReportHistoryWeekly, [("opsysrelease", opsysrelease), ("week", week)]), (ReportHistoryDaily, [("opsysrelease", opsysrelease), ("day", day)]), ] if "executable" in ureport: stat_map.append((ReportExecutable, [("path", ureport["executable"])])) if "uptime" in ureport: if ureport["uptime"] < 0.1: uptime_exp = -1 else: uptime_exp = int(math.log(ureport["uptime"], 10)) stat_map.append((ReportUptime, [("uptime_exp", uptime_exp)])) # Add the reported package (installed and running). stat_map.append(get_package_stat("CRASHED", ureport, ureport["os"], db)) # Similarly add related packages. if "related_packages" in ureport: for related_package in ureport["related_packages"]: stat_map.append(get_package_stat("RELATED", related_package, ureport["os"], db)) # Add selinux fields to stat_map if "selinux" in ureport: stat_map.append((ReportSelinuxMode, [("mode", ureport["selinux"]["mode"].upper())])) if "context" in ureport["selinux"]: stat_map.append((ReportSelinuxContext, [("context", ureport["selinux"]["context"])])) if "policy_package" in ureport["selinux"]: stat_map.append( get_package_stat( "SELINUX_POLICY", {"installed_package": ureport["selinux"]["policy_package"]}, ureport["os"], db ) ) # Add kernel taint state fields to stat_map. if "kernel_taint_state" in ureport: stat_map.append((ReportKernelTaintState, [("state", ureport["kernel_taint_state"])])) # Create missing stats and increase counters. for table, cols in stat_map: report_stat_query = db.session.query(table).join(Report).filter(Report.id == report.id) for name, value in cols: report_stat_query = report_stat_query.filter(getattr(table, name) == value) report_stat = report_stat_query.first() if not report_stat: report_stat = table() report_stat.report = report for name, value in cols: setattr(report_stat, name, value) report_stat.count = 0 db.session.add(report_stat) report_stat.count += count