def save_report_dict(self, report): ureport.validate(report) mtime = datetime.datetime.utcnow() ureport.save(self.db, report, timestamp=mtime) self.db.session.flush()
def test_ureport_saving(self): """ Check if ureport saving works correctly. """ # save raises FafError on failure for report_name in self.sample_report_names: save(self.db, self.sample_reports[report_name])
def _save_reports(self, db, pattern="*"): self.log_info("Saving reports") report_filenames = glob.glob( os.path.join(self.dir_report_incoming, pattern)) i = 0 for filename in sorted(report_filenames): i += 1 fname = os.path.basename(filename) self.log_info("[{0} / {1}] Processing file '{2}'".format( i, len(report_filenames), filename)) try: with open(filename, "r") as fil: ureport = json.load(fil) except (OSError, ValueError) as ex: self.log_warn("Failed to load uReport: {0}".format(str(ex))) self._move_report_to_deferred(fname) continue try: validate(ureport) except FafError as ex: self.log_warn("uReport is invalid: {0}".format(str(ex))) if ("os" in ureport and "name" in ureport["os"] and ureport["os"]["name"] not in systems and ureport["os"]["name"].lower() not in systems): self._save_unknown_opsys(db, ureport["os"]) self._move_report_to_deferred(fname) continue mtime = os.path.getmtime(filename) timestamp = datetime.datetime.fromtimestamp(mtime) try: save(db, ureport, create_component=self.create_components, timestamp=timestamp) except FafError as ex: self.log_warn("Failed to save uReport: {0}".format(str(ex))) self._move_report_to_deferred(fname) continue self._move_report_to_saved(fname)
def test_comment_saving(self): """ Check if comment attachment is added to report. """ save(self.db, self.sample_reports['ureport2']) report = self.db.session.query(Report).first() # update hash locally reporthash = report.hashes[0].hash com_attachment = self.comment_attachment com_attachment["bthash"] = reporthash save_attachment(self.db, com_attachment) self.assertEqual(len(report.comments), 1)
def _save_reports(self, db): self.log_info("Saving reports") report_filenames = os.listdir(self.dir_report_incoming) i = 0 for fname in sorted(report_filenames): i += 1 filename = os.path.join(self.dir_report_incoming, fname) self.log_info("[{0} / {1}] Processing file '{2}'" .format(i, len(report_filenames), filename)) try: with open(filename, "r") as fil: ureport = json.load(fil) except (OSError, ValueError) as ex: self.log_warn("Failed to load uReport: {0}".format(str(ex))) self._move_report_to_deferred(fname) continue try: validate(ureport) except FafError as ex: self.log_warn("uReport is invalid: {0}".format(str(ex))) if ("os" in ureport and "name" in ureport["os"] and ureport["os"]["name"] not in systems and ureport["os"]["name"].lower() not in systems): self._save_unknown_opsys(db, ureport["os"]) self._move_report_to_deferred(fname) continue mtime = os.path.getmtime(filename) timestamp = datetime.datetime.fromtimestamp(mtime) try: save(db, ureport, create_component=self.create_components, timestamp=timestamp) except FafError as ex: self.log_warn("Failed to save uReport: {0}".format(str(ex))) self._move_report_to_deferred(fname) continue self._move_report_to_saved(fname)
def save_report(self, filename): """ Save report located in sample_reports directory with `filename`. """ path = os.path.join(self.reports_path, filename) with open(path) as file: report = json.load(file) ureport.validate(report) mtime = datetime.datetime.utcnow() ureport.save(self.db, report, timestamp=mtime) self.db.session.flush()
def test_url_saving(self): """ Check if URL attachment is added to report. """ save(self.db, self.sample_reports['ureport2']) report = self.db.session.query(Report).first() # update hash locally reporthash = report.hashes[0].hash url_attachment = self.url_attachment url_attachment["bthash"] = reporthash save_attachment(self.db, url_attachment) self.assertEqual(len(report.urls), 1) self.assertEqual(report.urls[0].url, 'http://example.org') self.assertIsNotNone(report.urls[0].saved)
def test_contact_email_saving(self): """ Check if comment attachment is added to report. """ save(self.db, self.sample_reports['ureport2']) report = self.db.session.query(Report).first() # update hash locally reporthash = report.hashes[0].hash contact_email_attachment = self.contact_email_attachment contact_email_attachment["bthash"] = reporthash save_attachment(self.db, contact_email_attachment) self.assertEqual(len(report.report_contact_emails), 1) self.assertEqual(len(self.db.session.query(ContactEmail).all()), 1) self.assertEqual(report.report_contact_emails[0].contact_email.email_address, contact_email_attachment["data"]) # saving it twice should have no effect save_attachment(self.db, contact_email_attachment) self.assertEqual(len(report.report_contact_emails), 1) self.assertEqual(len(self.db.session.query(ContactEmail).all()), 1)
def test_attachment_saving(self): """ Check if bugzilla attachment is added to report. """ save(self.db, self.sample_reports['ureport2']) report = self.db.session.query(Report).first() # update hash locally reporthash = report.hashes[0].hash bz_attachment = self.bugzilla_attachment bz_attachment["bthash"] = reporthash class MockBugtracker(object): def download_bug_to_storage(db, bug_id): return self.bug old_fedora_bt = bugtrackers["fedora-bugzilla"] bugtrackers["fedora-bugzilla"] = MockBugtracker() save_attachment(self.db, bz_attachment) self.assertEqual(len(report.bz_bugs), 1) bugtrackers["fedora-bugzilla"] = old_fedora_bt
def _save_reports_speedup(self, db): self.log_info("Saving reports (--speedup)") # This creates a lock file and only works on file modified between the # last lock file and this new lock file. This way a new process can # be run while the older is still running. now = time.time() lock_name = ".sr-speedup-{0}-{1}.lock".format(os.getpid(), int(now)) self.lock_filename = os.path.join(self.dir_report_incoming, lock_name) open(self.lock_filename, "w").close() os.utime(self.lock_filename, (int(now), int(now))) self.log_debug("Created lock %s", self.lock_filename) # Remove lock on SIGTERM and Ctrl-C def handle_term(_, __): self.log_debug("Signal caught, removing lock %s", self.lock_filename) os.remove(self.lock_filename) sys.exit(0) signal.signal(signal.SIGTERM, handle_term) signal.signal(signal.SIGINT, handle_term) locks = glob.glob( os.path.join(self.dir_report_incoming, ".sr-speedup-*.lock")) newest_older_ctime = 0 for lock in locks: stat = os.stat(lock) if int(stat.st_ctime) > int(now) and not lock.endswith(lock_name): self.log_info("Newer lock found. Exiting.") os.remove(self.lock_filename) return if stat.st_ctime > newest_older_ctime and int( stat.st_ctime) < int(now): newest_older_ctime = stat.st_ctime report_filenames = [] with os.scandir(self.dir_report_incoming) as iterator: for entry in iterator: if not entry.name.startswith('.') and entry.is_file(): stat = entry.stat() if stat.st_mtime > newest_older_ctime and stat.st_mtime <= now: report_filenames.append(entry.name) # We create a dict of SHA1 unique reports and then treat them as one # with appropriate count. reports = {} for i, fname in enumerate(sorted(report_filenames), start=1): filename = os.path.join(self.dir_report_incoming, fname) self.log_info("[{0} / {1}] Loading file '{2}'".format( i, len(report_filenames), filename)) try: with open(filename, "rb") as fil: stat = os.stat(filename) contents = fil.read() h = hashlib.sha1() h.update(contents) h.update( datetime.date.fromtimestamp( stat.st_mtime).isoformat().encode("utf-8")) digest = h.digest() if digest in reports: reports[digest]["filenames"].append(fname) if reports[digest]["mtime"] < stat.st_mtime: reports[digest]["mtime"] = stat.st_mtime self.log_debug("Duplicate") else: reports[digest] = { "ureport": json.loads(contents), "filenames": [fname], "mtime": stat.st_mtime, } self.log_debug("Original") except (OSError, ValueError) as ex: self.log_warn("Failed to load uReport: {0}".format(str(ex))) self._move_report_to_deferred(fname) continue for i, unique in enumerate(reports.values(), start=1): self.log_info("[{0} / {1}] Processing unique file '{2}'".format( i, len(reports), unique["filenames"][0])) ureport = unique["ureport"] try: validate(ureport) except FafError as ex: self.log_warn("uReport is invalid: {0}".format(str(ex))) if ("os" in ureport and "name" in ureport["os"] and ureport["os"]["name"] not in systems and ureport["os"]["name"].lower() not in systems): self._save_unknown_opsys(db, ureport["os"]) self._move_reports_to_deferred(unique["filenames"]) continue mtime = unique["mtime"] timestamp = datetime.datetime.fromtimestamp(mtime) try: save(db, ureport, create_component=self.create_components, timestamp=timestamp, count=len(unique["filenames"])) except FafError as ex: self.log_warn("Failed to save uReport: {0}".format(str(ex))) self._move_reports_to_deferred(unique["filenames"]) continue self._move_reports_to_saved(unique["filenames"]) self.log_debug("Removing lock %s", self.lock_filename) os.remove(self.lock_filename)
def _save_reports_speedup(self, db): self.log_info("Saving reports (--speedup)") # This creates a lock file and only works on file modified between the # last lock file and this new lock file. This way a new process can # be run while the older is still running. now = time.time() lock_name = ".sr-speedup-{0}-{1}.lock".format(os.getpid(), int(now)) self.lock_filename = os.path.join(self.dir_report_incoming, lock_name) open(self.lock_filename, "w").close() os.utime(self.lock_filename, (int(now), int(now))) self.log_debug("Created lock {0}".format(self.lock_filename)) # Remove lock on SIGTERM and Ctrl-C def handle_term(_, __): self.log_debug("Signal caught, removing lock {0}".format(self.lock_filename)) os.remove(self.lock_filename) sys.exit(0) signal.signal(signal.SIGTERM, handle_term) signal.signal(signal.SIGINT, handle_term) locks = glob.glob(os.path.join(self.dir_report_incoming, ".sr-speedup-*.lock")) newest_older_ctime = 0 for lock in locks: stat = os.stat(lock) if int(stat.st_ctime) > int(now) and not lock.endswith(lock_name): self.log_info("Newer lock found. Exiting.") os.remove(self.lock_filename) return if stat.st_ctime > newest_older_ctime and int(stat.st_ctime) < int(now): newest_older_ctime = stat.st_ctime report_filenames = [] for fname in os.listdir(self.dir_report_incoming): stat = os.stat(os.path.join(self.dir_report_incoming, fname)) if fname[0] != "." and stat.st_mtime > newest_older_ctime and stat.st_mtime <= now: report_filenames.append(fname) # We create a dict of SHA1 unique reports and then treat them as one # with appropriate count. reports = {} i = 0 for fname in sorted(report_filenames): i += 1 filename = os.path.join(self.dir_report_incoming, fname) self.log_info("[{0} / {1}] Loading file '{2}'" .format(i, len(report_filenames), filename)) try: with open(filename, "rb") as fil: stat = os.stat(filename) contents = fil.read() h = hashlib.sha1() h.update(contents) h.update(datetime.date.fromtimestamp(stat.st_mtime) .isoformat().encode("utf-8")) digest = h.digest() if digest in reports: reports[digest]["filenames"].append(fname) if reports[digest]["mtime"] < stat.st_mtime: reports[digest]["mtime"] = stat.st_mtime self.log_debug("Duplicate") else: reports[digest] = { "ureport": json.loads(contents), "filenames": [fname], "mtime": stat.st_mtime, } self.log_debug("Original") except (OSError, ValueError) as ex: self.log_warn("Failed to load uReport: {0}".format(str(ex))) self._move_report_to_deferred(fname) continue i = 0 for unique in reports.values(): i += 1 self.log_info("[{0} / {1}] Processing unique file '{2}'" .format(i, len(reports), unique["filenames"][0])) ureport = unique["ureport"] try: validate(ureport) except FafError as ex: self.log_warn("uReport is invalid: {0}".format(str(ex))) if ("os" in ureport and "name" in ureport["os"] and ureport["os"]["name"] not in systems and ureport["os"]["name"].lower() not in systems): self._save_unknown_opsys(db, ureport["os"]) self._move_reports_to_deferred(unique["filenames"]) continue mtime = unique["mtime"] timestamp = datetime.datetime.fromtimestamp(mtime) try: save(db, ureport, create_component=self.create_components, timestamp=timestamp, count=len(unique["filenames"])) except FafError as ex: self.log_warn("Failed to save uReport: {0}".format(str(ex))) self._move_reports_to_deferred(unique["filenames"]) continue self._move_reports_to_saved(unique["filenames"]) self.log_debug("Removing lock {0}".format(self.lock_filename)) os.remove(self.lock_filename)