def run(self):
snapshot_db = snapshots.SnapshotDB(self.args)
tag_db = tags.TagsDB(self.args)
a_handle, a = snapshots.get(self.args, snapshot_db, tag_db, self.args.a_ref)
if a_handle is None:
logger.critical("Invalid baseline reference (-a --from)")
return 5
if a is None:
logger.critical("Error retrieving baseline content")
return 5
b_handle, b = snapshots.get(self.args, snapshot_db, tag_db, self.args.b_ref)
if b_handle is None:
logger.critical("Invalid target reference (-b --to)")
return 5
if b is None:
logger.critical("Error retrieving target content")
return 5
logger.debug("Diffing %s and %s" % (a_handle, b_handle))
# Unless marshal=True is given, jsondiff.diff() produces dicts with non-string
# keys which json.dump() does not like at all.
diff = jsondiff.diff(a, b, syntax="symmetric", marshal=True)
# Filter out obvious changelings
if "meta" in diff and "snapshot_time" in diff["meta"]:
del diff["meta"]["snapshot_time"]
if len(diff["meta"]) == 0:
del diff["meta"]
if len(diff) == 0:
return 0
if self.args.format == "jdiff":
print diff
elif self.args.format == "json":
diff_str = json.dumps(diff, indent=4, sort_keys=True)
if sys.stdout.isatty():
print highlight(diff_str, JsonLexer(), Terminal256Formatter())
else:
print diff_str
if self.args.format == "pretty":
# py3 knows os.get_terminal_size(), but we need to guesstimate a width in py2
pp(indent=1, width=120).pprint(diff)
return 1
def run(self):
snapshot_db = snapshots.SnapshotDB(self.args)
tag_db = tags.TagsDB(self.args)
if self.args.show is None:
handle_list = snapshot_db.list()
length = len(handle_list)
for number in xrange(length):
ref = length - number
handle = handle_list[number]
tag_list = tag_db.handle_to_tags(handle)
if len(tag_list) == 0:
print "%d: %s" % (ref, handle)
else:
print "%d: %s [%s]" % (ref, handle, ",".join(tag_list))
else:
handle, content = snapshots.get(self.args, snapshot_db, tag_db,
self.args.show)
if handle is None:
logger.critical("Invalid snapshot reference (-s --show)")
return 5
if content is None:
logger.critical("Error retrieving snapshot content")
return 5
content_str = json.dumps(content, indent=4, sort_keys=True)
if sys.stdout.isatty():
print highlight(content_str, JsonLexer(),
Terminal256Formatter())
else:
print content_str
return 0
def run(self):
tag_db = tags.TagsDB(self.args)
snapshot_db = snapshots.SnapshotDB(self.args)
_, snapshot = snapshots.get(self.args, snapshot_db, tag_db, "0") # handle 0 is current online state
if self.args.dump:
snapshots.json_highlight_print(snapshot)
else:
snapshots.store(snapshot_db, snapshot)
return 0
def run(self):
snapshot_db = snapshots.SnapshotDB(self.args)
tag_db = tags.TagsDB(self.args)
if self.args.delete:
handle = snapshots.match(snapshot_db, tag_db, self.args.delete)
if handle is None:
logger.critical("Unknown reference for deletion")
return 10
elif handle == "online":
logger.critical("Deleting the Internet...")
logger.warn("Done. You are on your own now. Please start over.")
return 42
else:
snapshot_db.delete(handle)
# TODO: Also delete associated tags
return 0
if self.args.show is None:
handle_list = snapshot_db.list()
length = len(handle_list)
for number in xrange(length):
ref = length - number
handle = handle_list[number]
tag_list = tag_db.handle_to_tags(handle)
if len(tag_list) == 0:
print "%d: %s" % (ref, handle)
else:
print "%d: %s [%s]" % (ref, handle, ",".join(tag_list))
else:
handle, content = snapshots.get(self.args, snapshot_db, tag_db, self.args.show)
if handle is None:
logger.critical("Invalid snapshot reference (-s --show)")
return 5
if content is None:
logger.critical("Error retrieving snapshot content")
return 5
snapshots.json_highlight_print(content)
return 0
def run(self):
snapshot_db = snapshots.SnapshotDB(self.args)
tag_db = tags.TagsDB(self.args)
handle, content = snapshots.get(self.args, snapshot_db, tag_db, self.args.snapshot)
if handle is None:
logger.critical("Invalid snapshot reference (-s --show)")
return 5
if content is None:
logger.critical("Error retrieving snapshot content")
return 5
if self.args.id is None:
logger.critical("Please specify ID to query with `-i`")
return 10
tid = str(self.args.id)
logger.debug("Looking for ID pattern `%s`" % tid)
result = {}
for p in content:
for k in content[p]:
logger.debug("Searching in `%s`" % k)
if tid in content[p][k]:
logger.debug("Direct hit for `%s` in `%s`" % (tid, k))
result = {p: {k: {tid: content[p][k][tid]}}}
break
for kk in content[p][k]:
logger.debug("Checking against `%s`" % kk)
if tid in kk:
logger.debug("Matched `%s` in %s `%s`" % (tid, k, kk))
if p not in result:
result[p] = {}
if k not in result[p]:
result[p][k] = {}
result[p][k][kk] = content[p][k][kk]
snapshots.json_highlight_print(result)
return 0
def run(self):
snapshot_db = snapshots.SnapshotDB(self.args)
tag_db = tags.TagsDB(self.args)
trello_token = read_token(self.args.workdir, token_type="trello")
if trello_token is None:
logger.critical(
"No Trello access token configured. Use `setup` command first")
raise Exception("Unable to continue without token")
tr = FirefoxTrello(user_token=trello_token)
bz_token = read_token(self.args.workdir, token_type="bugzilla")
if bz_token is None:
logger.critical(
"No Bugzilla access token configured. Use `setup` command first"
)
raise Exception("Unable to continue without token")
bz = BugzillaClient(token=bz_token)
snapshot = snapshots.get(self.args, snapshot_db, tag_db, "1")
embed()
return 0
def run(self):
snapshot_db = snapshots.SnapshotDB(self.args)
tag_db = tags.TagsDB(self.args)
bz_token = token.read_token(self.args.workdir, token_type="bugzilla")
if bz_token is None:
logger.critical(
"No Bugzilla access token configured. Use `setup` command first"
)
return 10
bz = BugzillaClient(bz_token)
tr_token = token.read_token(self.args.workdir, token_type="trello")
if tr_token is None:
logger.critical(
"No Trello access token configured. Use `setup` command first")
return 10
tr = FirefoxTrello(user_token=tr_token)
a_handle, a = snapshots.get(self.args, snapshot_db, tag_db,
self.args.a_ref)
if a_handle is None:
if self.args.a_ref == "triaged":
logger.critical(
"You might want to tag the base snapshot to compare against as `triaged` first"
)
else:
logger.critical("Invalid baseline reference (-a --from)")
return 5
if a is None:
logger.critical("Error retrieving baseline content")
return 5
b_handle, b = snapshots.get(self.args, snapshot_db, tag_db,
self.args.b_ref)
if b_handle is None:
logger.critical("Invalid target reference (-b --to)")
return 5
if b is None:
logger.critical("Error retrieving target content")
return 5
# Just a few shortcuts for terser code
aft = b["firefox_trello"]
bft = b["firefox_trello"]
abz = b["bugzilla"]
bbz = b["bugzilla"]
if self.args.mode == "json":
from IPython import embed
embed()
return 0
# FIXME: Internal state is not updated during operations. Hence full syncronization
# may require multiple passes.
# Create a new bug for every (relevant) card that is not associated to one, yet.
for cid, card in bft["cards"].iteritems():
# Check whether there are strange security notes
sec_note = extract_security_info(card, tr.security_notes_id)
if sec_note is not None and not sec_note.startswith("bug "):
logger.warning(
"Card %s has strange security note format: `%s`" %
(card["shortUrl"], sec_note))
if cid in aft["cards"]:
from_card = aft["cards"][cid]
from_list = aft["lists"][from_card["idList"]]
else:
from_card = None
from_list = None
to_list = bft["lists"][card["idList"]]
labels = [l["name"] for l in card["labels"]]
bug_id = extract_bugzilla_bug(card, tr.security_notes_id)
if bug_id is not None and bug_id not in bbz["bugs"]:
logger.warning(
"Bug http://bugzil.la/%s referenced by Trello card %s, but not among bug list"
% (bug_id, card["shortUrl"]))
firefox_version = parse_firefox_version(to_list["name"])
# Don't add bugs for cards that were around last triage, unless called with --all
if cid in aft["cards"] and not self.args.all:
logger.debug(
"Skipping card `%s` which was already triaged last time" %
card["shortUrl"])
continue
card_info = {
"id": card["id"],
"name": card["name"],
"labels": labels,
"description": card["desc"][:300] + "...",
"list_from": from_list["name"],
"list_in": to_list["name"],
"card_url": card["shortUrl"],
"bug": bug_id
}
if bug_id is None:
if to_list["name"] == "About:This Board":
continue
if to_list["name"] == "Backlog" and not self.args.all:
# Skip if card is not relevant
logger.debug("Ignoring card %s" % card["shortUrl"])
else:
# Create new bug
if firefox_version is None:
print "Bugless card %s in list `%s`:" % (
card["shortUrl"], to_list["name"])
else:
print "Bugless card %s for Firefox version %s:" % (
card["shortUrl"], firefox_version)
snapshots.json_highlight_print([card_info])
print "The corresponding bug data is:"
bug = bz.create_bug(cid, bft)
bug_preview = bug.preview_verbose()
snapshots.json_highlight_print([bug_preview])
yes_or_no = raw_input(
"Do you want to create a bug with that data in Bugzilla? (y/N) "
)
if yes_or_no.lower().startswith("y"):
bug_id = bug.submit()
logger.critical("Created bug http://bugzil.la/%s" %
bug_id)
logger.info(
"Updating security note and label on Trello card %s"
% card["shortUrl"])
tr.set_security_notes(card["id"], "bug %s" % bug_id)
tr.set_security_action_required_label(card["id"])
else:
logger.warning("Skipping bug creation")
# Syncronize Bugzilla version / target milestone to Trello state.
# Trello state is authoritative.
short_url_map = {None: None, "": None}
for cid, card in bft["cards"].iteritems():
short_url_map[card["shortUrl"]] = card
for bid, bug in bbz["bugs"].iteritems():
# Sometimes bugs still have long Trello URLs
card_url = None
m = re.match(r"""(https://trello.com/c/[A-Za-z0-9]+)(/.*)*""",
bug["url"])
if m is not None:
card_url = m.group(1)
if m.group(2) is not None:
logger.debug(
"Bug http://bugzil.la/%s `%s` has long Trello URL" %
(bid, bug["summary"]))
card = short_url_map[card_url]
if card is None:
logger.warn(
"Bug http://bugzil.la/%s `%s` is not associated with a Trello card"
% (bid, bug["summary"]))
# snapshots.json_highlight_print(bug)
# Look through Trello cards and see if one links to this bug
associated_cards = []
for card in bft["cards"].itervalues():
if extract_bugzilla_bug(card,
tr.security_notes_id) == str(bid):
associated_cards.append(card)
if len(associated_cards) == 1:
logger.info(
"Trello card `%s` %s is associated with https://bugzil.la/%s"
% (associated_cards[0]["name"],
associated_cards[0]["shortUrl"], bid))
yes_or_no = raw_input(
"Do you want to update the bug's URL field? (y/N) ")
if yes_or_no.lower().startswith("y"):
bz.update_url(bid, card["shortUrl"])
else:
logger.warn("https://bugzil.la/%s not updated" % bid)
elif len(associated_cards) > 1:
associated_urls = [c["shortUrl"] for c in associated_cards]
logger.warn(
"Multiple cards point to https://bugzil.la/%s: %s" %
(bid, associated_urls))
raw_input("Press return to continue.")
continue
card_bug = bz.create_bug(card["id"], bft)
old_version = bug["version"]
old_milestone = bug["target_milestone"]
new_version = card_bug["version"]
new_milestone = card_bug["target_milestone"]
if old_version != new_version or old_milestone != new_milestone:
logger.warn("Version mismatch for bug http://bugzil.la/%s" %
bid)
print "Bugzilla state:"
snapshots.json_highlight_print([bug])
print "Bug according to Trello state:"
snapshots.json_highlight_print([card_bug.bugdata])
yes_or_no = raw_input(
"Do you want to update the bug from `%s / %s` to `%s / %s`? (y/N) "
% (old_version, old_milestone, new_version, new_milestone))
if yes_or_no.lower().startswith("y"):
logger.debug("Updating bug %s to version %s" %
(bid, card_bug.firefox_version))
bz.update_version(bid, card_bug.firefox_version)
else:
logger.info(
"Skipping version update for bug http://bugzil.la/%s" %
bid)
# TODO: Syncronize Trello labels to Bugzilla bug state.
# Bugzilla state is authoritative.
for cid, card in bft["cards"].iteritems():
bid = extract_bugzilla_bug(card, tr.security_notes_id)
if bid is None:
continue
try:
bug = bbz["bugs"][bid]
except KeyError:
logger.warn(
"Card `%s` references unfetched bug http://bugzil.la/%s" %
(card["shortUrl"], bid))
continue
if bug["resolution"] == "DUPLICATE":
logger.debug("Skipping duplicate bug http://bugzil.la/%s" %
bid)
continue
label_is = extract_security_labels(
card, tr.security_action_required_label, tr.security_ok_label)
label_should = security_label_should_be(
bug, tr.security_action_required_label, tr.security_ok_label)
if label_is == [label_should]:
continue
print card[
"shortUrl"], "http://bugzil.la/%s" % bid, label_is, label_should
if label_should == tr.security_action_required_label:
if label_is == [tr.security_ok_label]:
logger.warn(
"Card %s has triage label `OK`, but should have `Action required`"
% card["shortUrl"])
else:
logger.warn(
"Card %s has no triage label and should have `Action required`"
% card["shortUrl"])
yes_or_no = raw_input("Do you want to update the card? (y/N) ")
if yes_or_no.lower().startswith("y"):
tr.set_security_action_required_label(card["id"])
elif label_should == tr.security_ok_label:
if label_is == [tr.security_action_required_label]:
logger.warn(
"Card %s has triage label `Action required`, but should have `OK`"
% card["shortUrl"])
else:
logger.warn(
"Card %s has no triage label and should have `OK`" %
card["shortUrl"])
yes_or_no = raw_input("Do you want to update the card? (y/N) ")
if yes_or_no.lower().startswith("y"):
tr.set_security_ok_label(card["id"])
else:
logger.error(
"Internal error with card %s / bug http://bugzil.la/%s" %
(card["shortUrl"], bug["id"]))
raise Exception("Internal error")
return 0
def run(self):
snapshot_db = snapshots.SnapshotDB(self.args)
tag_db = tags.TagsDB(self.args)
a_handle, a = snapshots.get(self.args, snapshot_db, tag_db,
self.args.a_ref)
if a_handle is None:
logger.critical("Invalid baseline reference (-a --from)")
return 5
if a is None:
logger.critical("Error retrieving baseline content")
return 5
b_handle, b = snapshots.get(self.args, snapshot_db, tag_db,
self.args.b_ref)
if b_handle is None:
logger.critical("Invalid target reference (-b --to)")
return 5
if b is None:
logger.critical("Error retrieving target content")
return 5
logger.debug("Diffing %s and %s" % (a_handle, b_handle))
# Unless marshal=True is given, jsondiff.diff() produces dicts with non-string
# keys which json.dump() does not like at all.
diff = jsondiff.diff(a, b, syntax="symmetric", marshal=True)
# TODO: Adapt filtering to new trello + bugzilla combo snapshots
# Filter out obvious changelings
if "meta" in diff and "snapshot_time" in diff["meta"]:
del diff["meta"]["snapshot_time"]
if len(diff["meta"]) == 0:
del diff["meta"]
if len(diff) == 0:
return 0
if not self.args.everything:
# Filter out irrelevant and noisy changes
if "labels" in diff:
for l in diff["labels"].keys():
if "uses" in diff["labels"][l]:
del diff["labels"][l]["uses"]
if len(diff["labels"][l]) == 0:
del diff["labels"][l]
if len(diff["labels"]) == 0:
del diff["labels"]
if "cards" in diff:
for c in diff["cards"].keys():
if "desc" in diff["cards"][c]:
del diff["cards"][c]["desc"]
if "labels" in diff["cards"][c]:
del diff["cards"][c]["labels"]
if "badges" in diff["cards"][c]:
del diff["cards"][c]["badges"]
if "uses" in diff["cards"][c]:
del diff["cards"][c]["uses"]
if "idLabels" in diff["cards"][c]:
del diff["cards"][c]["idLabels"]
if "dateLastActivity" in diff["cards"][c]:
del diff["cards"][c]["dateLastActivity"]
if len(diff["cards"][c]) == 0:
del diff["cards"][c]
if len(diff["cards"]) == 0:
del diff["cards"]
if len(diff) == 0:
logger.warning(
"Irrelevant changes hidden. Use `--everything` to see them.")
return 0
if self.args.format == "jdiff":
print diff
elif self.args.format == "json":
snapshots.json_highlight_print(diff)
if self.args.format == "pretty":
# py3 knows os.get_terminal_size(), but we need to guesstimate a width in py2
try:
width = os.get_terminal_size().columns
except AttributeError:
width = 120
except OSError:
width = 120
pp(indent=1, width=str(width)).pprint(diff)
return 1
def run(self):
snapshot_db = snapshots.SnapshotDB(self.args)
tag_db = tags.TagsDB(self.args)
handle, content = snapshots.get(self.args, snapshot_db, tag_db,
self.args.snapshot)
if handle is None:
logger.critical("Invalid snapshot reference (-s --show)")
return 5
if content is None:
logger.critical("Error retrieving snapshot content")
return 5
result = {}
# Just Focusing on trello for now
# TODO: implement Bugzilla stats
content = content["firefox_trello"]
for lid in content["lists"]:
if content["lists"][lid]["closed"] and not self.args.all:
continue
if content["lists"][lid]["name"].startswith("About:"):
continue
sec_label_counts = {}
num_cards = 0
num_inactive_cards = 0
for cid in content["cards"]:
if content["cards"][cid]["idList"] != lid:
continue
if content["cards"][cid]["closed"]:
num_inactive_cards += 1
continue
num_cards += 1
c = content["cards"][cid]
has_sec_label = False
for l in c["labels"]:
if l["name"].startswith("Security Triage:"):
has_sec_label = True
if l["name"] not in sec_label_counts:
sec_label_counts[l["name"]] = 1
else:
sec_label_counts[l["name"]] += 1
if l["name"] not in [
"Security Triage: OK",
"Security Triage: Action required"
]:
logger.warning("Unknown label `%s` on card `%s`" %
(l["name"], c["shortUrl"]))
if not has_sec_label:
logger.warning(
"Card without security label in `%s`: `%s` `%s`" %
(content["lists"][lid]["name"][:20] + "...", c["name"],
c["shortUrl"]))
sec_ok = sec_label_counts[
"Security Triage: OK"] if "Security Triage: OK" in sec_label_counts else 0
sec_action = sec_label_counts["Security Triage: Action required"] \
if "Security Triage: Action required" in sec_label_counts else 0
sec_missing = num_cards - sec_ok - sec_action
l = content["lists"][lid]
result[lid] = {
"__name": l["name"],
"_closed": l["closed"],
"active_cards": num_cards,
"inactive_cards": num_inactive_cards,
"security_label_ok": sec_ok,
"security_label_action": sec_action,
"security_label_missing": sec_missing
}
snapshots.json_highlight_print(result)
return 0