def _save_ccs(self, db, ccs, new_bug_id):
"""
Save CC"ed users to the database.
Expects list of emails (`ccs`) and ID of the bug as `new_bug_id`.
"""
total = len(ccs)
for num, user_email in enumerate(ccs, start=1):
self.log_debug("Processing CC: %d/%d", num, total)
cc = (db.session.query(BzBugCc).join(
BzUser).filter((BzUser.email == user_email)
& (BzBugCc.bug_id == new_bug_id)).first())
if cc:
self.log_debug("CC'ed user %s already exists", user_email)
continue
cced = queries.get_bz_user(db, user_email)
if not cced:
self.log_debug("CC'ed user %s not found, adding.", user_email)
downloaded = self._download_user(user_email)
if not downloaded:
self.log_error("Unable to download user, skipping.")
continue
cced = self._save_user(db, downloaded)
new = BzBugCc()
new.bug_id = new_bug_id
new.user = cced
db.session.add(new)
db.session.flush()
def delete_user_data():
usermail = g.user.mail
fas_user = queries.get_user_by_mail(db, usermail).first()
bz_user = queries.get_bz_user(db, usermail)
contact_email = queries.get_contact_email(db, usermail)
if bz_user is not None:
anonymous = create_anonymous_bzuser(db, uid=-1)
delete_bugzilla_user(db, bz_user.id, anonymous.id)
db.session.delete(bz_user)
if fas_user is not None:
queries.get_reportarchives_by_username(db, fas_user.username).delete(False)
queries.get_problemreassigns_by_username(db, fas_user.username).delete(False)
if contact_email is not None:
queries.get_reportcontactmails_by_id(db, contact_email.id).delete(False)
db.session.delete(contact_email)
# Sign out user before deleting his account
session.pop("openid", None)
flash(u"You were signed out.", category='info')
db.session.delete(fas_user)
db.session.commit()
return redirect(oid.get_next_url())
def _save_attachments(self, db, attachments, new_bug_id):
"""
Save bug attachments to the database.
Expects list of `attachments` and ID of the bug as `new_bug_id`.
"""
total = len(attachments)
for num, attachment in enumerate(attachments):
self.log_debug("Processing attachment {0}/{1}".format(num + 1,
total))
if queries.get_bz_attachment(db, attachment["id"]):
self.log_debug("Skipping existing attachment #{0}".format(
attachment["id"]))
continue
user_email = attachment["attacher"]
user = queries.get_bz_user(db, user_email)
if not user:
self.log_debug("Attachment from unknown user {0}".format(
user_email))
downloaded = self._download_user(user_email)
if not downloaded:
self.log_error("Unable to download user, skipping.")
continue
user = self._save_user(db, downloaded)
new = BzAttachment()
new.id = attachment["id"]
new.bug_id = new_bug_id
new.mimetype = attachment["content_type"]
new.description = attachment["description"]
new.filename = attachment["file_name"]
new.is_private = bool(attachment["is_private"])
new.is_patch = bool(attachment["is_patch"])
new.is_obsolete = bool(attachment["is_obsolete"])
new.creation_time = self._convert_datetime(
attachment["creation_time"])
new.last_change_time = self._convert_datetime(
attachment["last_change_time"])
new.user = user
db.session.add(new)
self._connect()
data = self.bz.openattachment(attachment["id"])
# save_lob is inherited method which cannot be seen by pylint
# because of sqlalchemy magic
# pylint: disable=E1101
new.save_lob("content", data, truncate=True, overwrite=True)
data.close()
db.session.flush()
def _save_attachments(self, db, attachments, new_bug_id):
"""
Save bug attachments to the database.
Expects list of `attachments` and ID of the bug as `new_bug_id`.
"""
total = len(attachments)
for num, attachment in enumerate(attachments):
self.log_debug("Processing attachment {0}/{1}".format(num + 1,
total))
if queries.get_bz_attachment(db, attachment["id"]):
self.log_debug("Skipping existing attachment #{0}".format(
attachment["id"]))
continue
user_email = attachment["attacher"]
user = queries.get_bz_user(db, user_email)
if not user:
self.log_debug("Attachment from unknown user {0}".format(
user_email))
downloaded = self._download_user(user_email)
if not downloaded:
self.log_error("Unable to download user, skipping.")
continue
user = self._save_user(db, downloaded)
new = BzAttachment()
new.id = attachment["id"]
new.bug_id = new_bug_id
new.mimetype = attachment["content_type"]
new.description = attachment["description"]
new.filename = attachment["file_name"]
new.is_private = bool(attachment["is_private"])
new.is_patch = bool(attachment["is_patch"])
new.is_obsolete = bool(attachment["is_obsolete"])
new.creation_time = self._convert_datetime(
attachment["creation_time"])
new.last_change_time = self._convert_datetime(
attachment["last_change_time"])
new.user = user
db.session.add(new)
self.connect()
data = self.bz.openattachment(attachment["id"])
# save_lob is inherited method which cannot be seen by pylint
# because of sqlalchemy magic
# pylint: disable=E1101
new.save_lob("content", data, truncate=True, overwrite=True)
data.close()
db.session.flush()
def _save_history(self, db, events, new_bug_id):
"""
Save bug history to the database.
Expects list of `events` and ID of the bug as `new_bug_id`.
"""
total = len(events)
for num, event in enumerate(events):
self.log_debug("Processing history event {0}/{1}".format(num + 1,
total))
user_email = event["who"]
user = queries.get_bz_user(db, user_email)
if not user:
self.log_debug("History changed by unknown user #{0}".format(
user_email))
downloaded = self._download_user(user_email)
if not downloaded:
self.log_error("Unable to download user, skipping.")
continue
user = self._save_user(db, downloaded)
for change in event["changes"]:
chtime = self._convert_datetime(event["when"])
ch = (
db.session.query(BzBugHistory)
.filter((BzBugHistory.user == user) &
(BzBugHistory.time == chtime) &
(BzBugHistory.field == change["field_name"]) &
(BzBugHistory.added == change["added"]) &
(BzBugHistory.removed == change["removed"]))
.first())
if ch:
self.log_debug("Skipping existing history event "
"#{0}".format(ch.id))
continue
new = BzBugHistory()
new.bug_id = new_bug_id
new.user = user
new.time = chtime
new.field = change["field_name"]
new.added = change["added"][:column_len(BzBugHistory, "added")]
new.removed = change["removed"][:column_len(BzBugHistory, "removed")]
db.session.add(new)
db.session.flush()
def _save_history(self, db, events, new_bug_id):
"""
Save bug history to the database.
Expects list of `events` and ID of the bug as `new_bug_id`.
"""
total = len(events)
for num, event in enumerate(events):
self.log_debug("Processing history event {0}/{1}".format(num + 1,
total))
user_email = event["who"]
user = queries.get_bz_user(db, user_email)
if not user:
self.log_debug("History changed by unknown user #{0}".format(
user_email))
downloaded = self._download_user(user_email)
if not downloaded:
self.log_error("Unable to download user, skipping.")
continue
user = self._save_user(db, downloaded)
for change in event["changes"]:
chtime = self._convert_datetime(event["when"])
ch = (
db.session.query(BzBugHistory)
.filter((BzBugHistory.user == user) &
(BzBugHistory.time == chtime) &
(BzBugHistory.field == change["field_name"]) &
(BzBugHistory.added == change["added"]) &
(BzBugHistory.removed == change["removed"]))
.first())
if ch:
self.log_debug("Skipping existing history event "
"#{0}".format(ch.id))
continue
new = BzBugHistory()
new.bug_id = new_bug_id
new.user = user
new.time = chtime
new.field = change["field_name"]
new.added = change["added"][:column_len(BzBugHistory, "added")]
new.removed = change["removed"][:column_len(BzBugHistory, "removed")]
db.session.add(new)
db.session.flush()
def bugzillas(self):
bz_user = queries.get_bz_user(self.db, self.mail)
if not bz_user:
return {}
user_bugzillas = queries.get_bugzillas_by_uid(self.db, bz_user.id)
bz_data = {
"Mail": bz_user.email,
"Name": bz_user.name,
"Real Name": bz_user.real_name
}
bz_data["Created Bugzillas"] = [{
"Bug ID": bz.id,
"Summary": bz.summary,
"Status": bz.status,
"Resolution": bz.resolution,
"Creation Date": str(bz.creation_time)
} for bz in user_bugzillas]
attachments = queries.get_bzattachments_by_uid(self.db,
bz_user.id).all()
ccs = queries.get_bzbugccs_by_uid(self.db, bz_user.id).all()
comments = queries.get_bzcomments_by_uid(self.db, bz_user.id).all()
history = queries.get_bzbughistory_by_uid(self.db, bz_user.id).all()
bz_data["Attachments"] = [{
"Bug ID": a.bug_id,
"Description": a.description,
"Creation Date": str(a.creation_time),
"Filename": a.filename
} for a in attachments]
bz_data["CCs"] = [cc.bug_id for cc in ccs]
bz_data["Comments"] = [{
"Bug ID": c.bug_id,
"Comment #": c.number,
"Creation Date": str(c.creation_time)
} for c in comments]
bz_data["History"] = [{
"Bug ID": h.bug_id,
"Time": str(h.time),
"Field": h.field,
"Added": h.added,
"Removed": h.removed
} for h in history]
return bz_data
def bugzillas(self):
bz_user = queries.get_bz_user(self.db, self.mail)
if not bz_user:
return {}
user_bugzillas = queries.get_bugzillas_by_uid(self.db, bz_user.id)
bz_data = {"Mail": bz_user.email,
"Name": bz_user.name,
"Real Name": bz_user.real_name}
bz_data["Created Bugzillas"] = [{"Bug ID": bz.id,
"Summary": bz.summary,
"Status": bz.status,
"Resolution": bz.resolution,
"Creation Date": str(bz.creation_time)}
for bz in user_bugzillas]
attachments = queries.get_bzattachments_by_uid(self.db, bz_user.id).all()
ccs = queries.get_bzbugccs_by_uid(self.db, bz_user.id).all()
comments = queries.get_bzcomments_by_uid(self.db, bz_user.id).all()
history = queries.get_bzbughistory_by_uid(self.db, bz_user.id).all()
bz_data["Attachments"] = [{"Bug ID": a.bug_id,
"Description": a.description,
"Creation Date": str(a.creation_time),
"Filename": a.filename}
for a in attachments]
bz_data["CCs"] = [cc.bug_id for cc in ccs]
bz_data["Comments"] = [{"Bug ID": c.bug_id,
"Comment #": c.number,
"Creation Date": str(c.creation_time)}
for c in comments]
bz_data["History"] = [{"Bug ID": h.bug_id,
"Time": str(h.time),
"Field": h.field,
"Added": h.added,
"Removed": h.removed}
for h in history]
return bz_data
def _save_ccs(self, db, ccs, new_bug_id):
"""
Save CC"ed users to the database.
Expects list of emails (`ccs`) and ID of the bug as `new_bug_id`.
"""
total = len(ccs)
for num, user_email in enumerate(ccs):
self.log_debug("Processing CC: {0}/{1}".format(num + 1, total))
cc = (
db.session.query(BzBugCc)
.join(BzUser)
.filter((BzUser.email == user_email) &
(BzBugCc.bug_id == new_bug_id)).first())
if cc:
self.log_debug("CC'ed user {0} already"
" exists".format(user_email))
continue
cced = queries.get_bz_user(db, user_email)
if not cced:
self.log_debug("CC'ed user {0} not found,"
" adding.".format(user_email))
downloaded = self._download_user(user_email)
if not downloaded:
self.log_error("Unable to download user, skipping.")
continue
cced = self._save_user(db, downloaded)
new = BzBugCc()
new.bug_id = new_bug_id
new.user = cced
db.session.add(new)
db.session.flush()
def _save_comments(self, db, comments, new_bug_id):
"""
Save bug comments to the database.
Expects list of `comments` and ID of the bug as `new_bug_id`.
"""
total = len(comments)
for num, comment in enumerate(comments):
self.log_debug("Processing comment {0}/{1}".format(num + 1,
total))
if queries.get_bz_comment(db, comment["id"]):
self.log_debug("Skipping existing comment #{0}".format(
comment["id"]))
continue
self.log_debug("Downloading comment #{0}".format(comment["id"]))
user_email = comment["creator"]
user = queries.get_bz_user(db, user_email)
if not user:
self.log_debug("History changed by unknown user #{0}".format(
user_email))
downloaded = self._download_user(user_email)
if not downloaded:
self.log_error("Unable to download user, skipping.")
continue
user = self._save_user(db, downloaded)
new = BzComment()
new.id = comment["id"]
new.bug_id = new_bug_id
new.creation_time = self._convert_datetime(comment["time"])
new.is_private = comment["is_private"]
if "attachment_id" in comment:
attachment = queries.get_bz_attachment(
db, comment["attachment_id"])
if attachment:
new.attachment = attachment
else:
self.log_warn("Comment is referencing an attachment"
" which is not accessible.")
new.number = num
new.user = user
db.session.add(new)
if not isinstance(comment["text"], six.string_types):
comment["text"] = str(comment["text"])
# save_lob is inherited method which cannot
# be seen by pylint because of sqlalchemy magic
# pylint: disable=E1101
new.save_lob("content", comment["text"].encode("utf-8"),
overwrite=True)
db.session.flush()
def _save_bug(self, db, bug):
"""
Save bug represented by `bug_dict` to the database.
If bug is marked as duplicate, the duplicate bug is downloaded
as well.
"""
bug_dict = self._preprocess_bug(bug)
if not bug_dict:
self.log_error("Bug pre-processing failed")
raise FafError("Bug pre-processing failed")
self.log_debug("Saving bug #{0}: {1}".format(bug_dict["bug_id"],
bug_dict["summary"]))
bug_id = bug_dict["bug_id"]
# check if we already have this bug up-to-date
old_bug = (
db.session.query(BzBug)
.filter(BzBug.id == bug_id)
.filter(BzBug.last_change_time == bug_dict["last_change_time"])
.first())
if old_bug:
self.log_info("Bug already up-to-date")
return old_bug
tracker = queries.get_bugtracker_by_name(db, self.name)
if not tracker:
self.log_error("Tracker with name '{0}' is not installed"
.format(self.name))
raise FafError("Tracker with name '{0}' is not installed"
.format(self.name))
opsysrelease = queries.get_osrelease(db, bug_dict["product"],
bug_dict["version"])
if not opsysrelease:
self.log_error("Unable to save this bug due to unknown "
"release '{0} {1}'".format(bug_dict["product"],
bug_dict["version"]))
raise FafError("Unable to save this bug due to unknown "
"release '{0} {1}'".format(bug_dict["product"],
bug_dict["version"]))
relcomponent = queries.get_component_by_name_release(
db, opsysrelease, bug_dict["component"])
if not relcomponent:
self.log_error("Unable to save this bug due to unknown "
"component '{0}'".format(bug_dict["component"]))
raise FafError("Unable to save this bug due to unknown "
"component '{0}'".format(bug_dict["component"]))
component = relcomponent.component
reporter = queries.get_bz_user(db, bug_dict["reporter"])
if not reporter:
self.log_debug("Creator {0} not found".format(
bug_dict["reporter"]))
downloaded = self._download_user(bug_dict["reporter"])
if not downloaded:
self.log_error("Unable to download user, skipping.")
raise FafError("Unable to download user, skipping.")
reporter = self._save_user(db, downloaded)
new_bug = BzBug()
new_bug.id = bug_dict["bug_id"]
new_bug.summary = bug_dict["summary"]
new_bug.status = bug_dict["status"]
new_bug.creation_time = bug_dict["creation_time"]
new_bug.last_change_time = bug_dict["last_change_time"]
new_bug.private = True if bug_dict["groups"] else False
if bug_dict["status"] == "CLOSED":
new_bug.resolution = bug_dict["resolution"]
if bug_dict["resolution"] == "DUPLICATE":
if not queries.get_bz_bug(db, bug_dict["dupe_id"]):
self.log_debug("Duplicate #{0} not found".format(
bug_dict["dupe_id"]))
dup = self.download_bug_to_storage(db, bug_dict["dupe_id"])
if dup:
new_bug.duplicate = dup.id
new_bug.tracker_id = tracker.id
new_bug.component_id = component.id
new_bug.opsysrelease_id = opsysrelease.id
new_bug.creator_id = reporter.id
new_bug.whiteboard = bug_dict["status_whiteboard"]
# the bug itself might be downloaded during duplicate processing
# exit in this case - it would cause duplicate database entry
if queries.get_bz_bug(db, bug_dict["bug_id"]):
self.log_debug("Bug #{0} already exists in storage,"
" updating".format(bug_dict["bug_id"]))
bugdict = {}
for col in new_bug.__table__._columns:
bugdict[col.name] = getattr(new_bug, col.name)
(db.session.query(BzBug)
.filter(BzBug.id == bug_id).update(bugdict))
new_bug = queries.get_bz_bug(db, bug_dict["bug_id"])
else:
db.session.add(new_bug)
db.session.flush()
self._save_ccs(db, bug_dict["cc"], new_bug.id)
self._save_history(db, bug_dict["history"], new_bug.id)
if self.save_attachments:
self._save_attachments(db, bug_dict["attachments"], new_bug.id)
if self.save_comments:
self._save_comments(db, bug_dict["comments"], new_bug.id)
return new_bug
def _save_comments(self, db, comments, new_bug_id):
"""
Save bug comments to the database.
Expects list of `comments` and ID of the bug as `new_bug_id`.
"""
total = len(comments)
for num, comment in enumerate(comments):
self.log_debug("Processing comment {0}/{1}".format(num + 1,
total))
if queries.get_bz_comment(db, comment["id"]):
self.log_debug("Skipping existing comment #{0}".format(
comment["id"]))
continue
self.log_debug("Downloading comment #{0}".format(comment["id"]))
user_email = comment["creator"]
user = queries.get_bz_user(db, user_email)
if not user:
self.log_debug("History changed by unknown user #{0}".format(
user_email))
downloaded = self._download_user(user_email)
if not downloaded:
self.log_error("Unable to download user, skipping.")
continue
user = self._save_user(db, downloaded)
new = BzComment()
new.id = comment["id"]
new.bug_id = new_bug_id
new.creation_time = self._convert_datetime(comment["time"])
new.is_private = comment["is_private"]
if "attachment_id" in comment:
attachment = queries.get_bz_attachment(
db, comment["attachment_id"])
if attachment:
new.attachment = attachment
else:
self.log_warn("Comment is referencing an attachment"
" which is not accessible.")
new.number = num
new.user = user
db.session.add(new)
if not isinstance(comment["text"], str):
comment["text"] = str(comment["text"])
# save_lob is inherited method which cannot
# be seen by pylint because of sqlalchemy magic
# pylint: disable=E1101
new.save_lob("content", comment["text"].encode("utf-8"),
overwrite=True)
db.session.flush()
def _save_bug(self, db, bug):
"""
Save bug represented by `bug_dict` to the database.
If bug is marked as duplicate, the duplicate bug is downloaded
as well.
"""
bug_dict = self.preprocess_bug(bug)
if not bug_dict:
self.log_error("Bug pre-processing failed")
raise FafError("Bug pre-processing failed")
self.log_debug("Saving bug #{0}: {1}".format(bug_dict["bug_id"],
bug_dict["summary"]))
bug_id = bug_dict["bug_id"]
# check if we already have this bug up-to-date
old_bug = (
db.session.query(BzBug)
.filter(BzBug.id == bug_id)
.filter(BzBug.last_change_time == bug_dict["last_change_time"])
.first())
if old_bug:
self.log_info("Bug already up-to-date")
return old_bug
tracker = queries.get_bugtracker_by_name(db, self.name)
if not tracker:
self.log_error("Tracker with name '{0}' is not installed"
.format(self.name))
raise FafError("Tracker with name '{0}' is not installed"
.format(self.name))
opsysrelease = queries.get_osrelease(db, bug_dict["product"],
bug_dict["version"])
if not opsysrelease:
self.log_error("Unable to save this bug due to unknown "
"release '{0} {1}'".format(bug_dict["product"],
bug_dict["version"]))
raise FafError("Unable to save this bug due to unknown "
"release '{0} {1}'".format(bug_dict["product"],
bug_dict["version"]))
relcomponent = queries.get_component_by_name_release(
db, opsysrelease, bug_dict["component"])
if not relcomponent:
self.log_error("Unable to save this bug due to unknown "
"component '{0}'".format(bug_dict["component"]))
raise FafError("Unable to save this bug due to unknown "
"component '{0}'".format(bug_dict["component"]))
component = relcomponent.component
reporter = queries.get_bz_user(db, bug_dict["reporter"])
if not reporter:
self.log_debug("Creator {0} not found".format(
bug_dict["reporter"]))
downloaded = self._download_user(bug_dict["reporter"])
if not downloaded:
self.log_error("Unable to download user, skipping.")
raise FafError("Unable to download user, skipping.")
reporter = self._save_user(db, downloaded)
new_bug = BzBug()
new_bug.id = bug_dict["bug_id"]
new_bug.summary = bug_dict["summary"]
new_bug.status = bug_dict["status"]
new_bug.creation_time = bug_dict["creation_time"]
new_bug.last_change_time = bug_dict["last_change_time"]
new_bug.private = bool(bug_dict["groups"])
if bug_dict["status"] == "CLOSED":
new_bug.resolution = bug_dict["resolution"]
if bug_dict["resolution"] == "DUPLICATE":
if not queries.get_bz_bug(db, bug_dict["dupe_id"]):
self.log_debug("Duplicate #{0} not found".format(
bug_dict["dupe_id"]))
dup = self.download_bug_to_storage(db, bug_dict["dupe_id"])
if dup:
new_bug.duplicate = dup.id
new_bug.tracker_id = tracker.id
new_bug.component_id = component.id
new_bug.opsysrelease_id = opsysrelease.id
new_bug.creator_id = reporter.id
new_bug.whiteboard = bug_dict["status_whiteboard"]
# the bug itself might be downloaded during duplicate processing
# exit in this case - it would cause duplicate database entry
if queries.get_bz_bug(db, bug_dict["bug_id"]):
self.log_debug("Bug #{0} already exists in storage,"
" updating".format(bug_dict["bug_id"]))
bugdict = {}
for col in new_bug.__table__._columns: #pylint: disable=protected-access
bugdict[col.name] = getattr(new_bug, col.name)
(db.session.query(BzBug)
.filter(BzBug.id == bug_id).update(bugdict))
new_bug = queries.get_bz_bug(db, bug_dict["bug_id"])
else:
db.session.add(new_bug)
db.session.flush()
self._save_ccs(db, bug_dict["cc"], new_bug.id)
self._save_history(db, bug_dict["history"], new_bug.id)
if self.save_attachments:
self._save_attachments(db, bug_dict["attachments"], new_bug.id)
if self.save_comments:
self._save_comments(db, bug_dict["comments"], new_bug.id)
return new_bug