def test_private_bugs_not_leaking(self):
bad_news = False
# FOR ALL BUG BLOCKS
for min_id, max_id in self.blocks_of_bugs():
results = get(
self.private,
{"and": [
{"match_all": {}},
{"and": [
{"range": {"bug_id": {"gte": min_id, "lt": max_id}}},
{"exists": {"field": "bug_group"}},
{"range": {"expires_on": {"gte": NOW}}}, #CURRENT RECORDS
{"range": {"modified_ts": {"lt": A_WHILE_AGO}}}, #OF A MINIMUM AGE
]}
]},
["bug_id", "bug_group", "modified_ts"]
)
private_ids = {b.bug_id: b.bug_group for b in results}
Log.note("Ensure {{num}} bugs did not leak", {
"num": len(private_ids.keys())
})
# VERIFY NONE IN PUBLIC
leaked_bugs = get(
self.public,
{"and": [
{"terms": {"bug_id": private_ids.keys()}},
{"range": {"expires_on": {"gte": NOW}}} # SOME BUGS WILL LEAK FOR A LITTLE WHILE
]}
)
if leaked_bugs:
bad_news = True
if self.settings.param.delete:
self.public.delete_record(
{"terms":{"bug_id":leaked_bugs.bug_id}}
)
Log.note("{{num}} leaks!! {{bugs}}", {
"num": len(leaked_bugs),
"bugs": Q.run({
"from":leaked_bugs,
"select":["bug_id", "bug_version_num", {"name":"modified_ts", "value":lambda d: CNV.datetime2string(CNV.milli2datetime(d.modified_ts))}],
"sort":"bug_id"
})
})
for b in leaked_bugs:
Log.note("{{bug_id}} has bug groups {{bug_group}}\n{{version|indent}}", {
"bug_id": b.bug_id,
"bug_group": private_ids[b.bug_id],
"version": milli2datetime(b)
})
#CHECK FOR LEAKED COMMENTS, BEYOND THE ONES LEAKED BY BUG
leaked_comments = get(
self.public_comments,
{"terms": {"bug_id": private_ids.keys()}},
limit=20
)
if leaked_comments:
bad_news = True
if self.settings.param.delete:
self.public_comments.delete_record(
{"terms":{"bug_id":leaked_comments.bug_id}}
)
Log.warning("{{num}} comments marked private have leaked!\n{{comments|indent}}", {
"num": len(leaked_comments),
"comments": leaked_comments
})
if bad_news:
Log.error("Bugs have leaked!")
def test_private_bugs_not_leaking(self):
bad_news = False
# FOR ALL BUG BLOCKS
for min_id, max_id in self.blocks_of_bugs():
results = get(
self.private,
{
"and": [
{
"match_all": {}
},
{
"and": [
{
"range": {
"bug_id": {
"gte": min_id,
"lt": max_id
}
}
},
{
"exists": {
"field": "bug_group"
}
},
{
"range": {
"expires_on": {
"gte": NOW
}
}
}, #CURRENT RECORDS
{
"range": {
"modified_ts": {
"lt": A_WHILE_AGO
}
}
}, #OF A MINIMUM AGE
]
}
]
},
["bug_id", "bug_group", "modified_ts"])
private_ids = {b.bug_id: b.bug_group for b in results}
Log.note("Ensure {{num}} bugs did not leak",
{"num": len(private_ids.keys())})
# VERIFY NONE IN PUBLIC
leaked_bugs = get(
self.public,
{
"and": [
{
"terms": {
"bug_id": private_ids.keys()
}
},
{
"range": {
"expires_on": {
"gte": NOW
}
}
} # SOME BUGS WILL LEAK FOR A LITTLE WHILE
]
})
if leaked_bugs:
bad_news = True
if self.settings.param.delete:
self.public.delete_record(
{"terms": {
"bug_id": leaked_bugs.bug_id
}})
Log.note(
"{{num}} leaks!! {{bugs}}", {
"num":
len(leaked_bugs),
"bugs":
Q.run({
"from":
leaked_bugs,
"select": [
"bug_id", "bug_version_num", {
"name":
"modified_ts",
"value":
lambda d: CNV.datetime2string(
CNV.milli2datetime(d.modified_ts))
}
],
"sort":
"bug_id"
})
})
for b in leaked_bugs:
Log.note(
"{{bug_id}} has bug groups {{bug_group}}\n{{version|indent}}",
{
"bug_id": b.bug_id,
"bug_group": private_ids[b.bug_id],
"version": milli2datetime(b)
})
#CHECK FOR LEAKED COMMENTS, BEYOND THE ONES LEAKED BY BUG
leaked_comments = get(self.public_comments,
{"terms": {
"bug_id": private_ids.keys()
}},
limit=20)
if leaked_comments:
bad_news = True
if self.settings.param.delete:
self.public_comments.delete_record(
{"terms": {
"bug_id": leaked_comments.bug_id
}})
Log.warning(
"{{num}} comments marked private have leaked!\n{{comments|indent}}",
{
"num": len(leaked_comments),
"comments": leaked_comments
})
if bad_news:
Log.error("Bugs have leaked!")
def test_private_attachments_not_leaking(self):
for min_id, max_id in self.blocks_of_bugs():
# FIND ALL PRIVATE ATTACHMENTS
bugs_w_private_attachments = get(
self.private,
{"and": [
{"range": {"bug_id": {"gte": min_id, "lt": max_id}}},
{"range": {"expires_on": {"gte": NOW}}}, #CURRENT RECORDS
{"range": {"modified_ts": {"lt": A_WHILE_AGO}}}, #OF A MINIMUM AGE
{"nested": { #HAS ATTACHMENT.
"path": "attachments",
"query": {"filtered": {
"query": {"match_all": {}},
"filter": {"exists": {"field":"attachments.attach_id"}}
}}
}},
{"or":[
{"nested": { #PRIVATE ATTACHMENT, OR...
"path": "attachments",
"query": {"filtered": {
"query": {"match_all": {}},
"filter": {"term": {"attachments.isprivate": 1}}
}}
}},
{"exists":{"field":"bug_group"}} # ...PRIVATE BUG
]}
]},
fields=["bug_id", "bug_group", "attachments", "modified_ts"]
)
private_attachments = Q.run({
"from": bugs_w_private_attachments,
"select": "attachments.attach_id",
"where": {"or": [
{"exists": "bug_group"},
{"terms": {"attachments.isprivate": ['1', True, 1]}}
]}
})
try:
private_attachments = [int(v) for v in private_attachments]
except Exception, e:
private_attachments = Q.run({
"from": bugs_w_private_attachments,
"select": "attachments.attach_id",
"where": {"or": [
{"exists": "bug_group"},
{"terms": {"attachments.isprivate": ['1', True, 1]}}
]}
})
Log.note("Ensure {{num}} attachments did not leak", {
"num": len(private_attachments)
})
#VERIFY NONE IN PUBLIC
leaked_bugs = get(
self.public,
{"and": [
{"range": {"bug_id": {"gte": min_id, "lt": max_id}}},
{"range": {"expires_on": {"gte": NOW}}}, # CURRENT BUGS
{"nested": {
"path": "attachments",
"query": {"filtered": {
"query": {"match_all": {}},
"filter": {"terms": {"attach_id": private_attachments}}
}}
}}
]}
# fields=["bug_id", "attachments"]
)
#
if leaked_bugs:
if self.settings.param.delete:
self.public.delete_record(
{"terms":{"bug_id":leaked_bugs.bug_id}}
)
Log.note("{{num}} bugs with private attachments have leaked!", {"num": len(leaked_bugs)})
for b in leaked_bugs:
Log.note("{{bug_id}} has private_attachment\n{{version|indent}}", {
"bug_id": b.bug_id,
"version": b
})
Log.error("Attachments have leaked!")
def test_private_attachments_not_leaking(self):
for min_id, max_id in self.blocks_of_bugs():
# FIND ALL PRIVATE ATTACHMENTS
bugs_w_private_attachments = get(
self.private,
{
"and": [
{
"range": {
"bug_id": {
"gte": min_id,
"lt": max_id
}
}
},
{
"range": {
"expires_on": {
"gte": NOW
}
}
}, #CURRENT RECORDS
{
"range": {
"modified_ts": {
"lt": A_WHILE_AGO
}
}
}, #OF A MINIMUM AGE
{
"nested": { #HAS ATTACHMENT.
"path": "attachments",
"query": {
"filtered": {
"query": {
"match_all": {}
},
"filter": {
"exists": {
"field":
"attachments.attach_id"
}
}
}
}
}
},
{
"or": [
{
"nested": { #PRIVATE ATTACHMENT, OR...
"path": "attachments",
"query": {
"filtered": {
"query": {
"match_all": {}
},
"filter": {
"term": {
"attachments.isprivate":
1
}
}
}
}
}
},
{
"exists": {
"field": "bug_group"
}
} # ...PRIVATE BUG
]
}
]
},
fields=["bug_id", "bug_group", "attachments", "modified_ts"])
private_attachments = Q.run({
"from": bugs_w_private_attachments,
"select": "attachments.attach_id",
"where": {
"or": [{
"exists": "bug_group"
}, {
"terms": {
"attachments.isprivate": ['1', True, 1]
}
}]
}
})
try:
private_attachments = [int(v) for v in private_attachments]
except Exception, e:
private_attachments = Q.run({
"from": bugs_w_private_attachments,
"select": "attachments.attach_id",
"where": {
"or": [{
"exists": "bug_group"
}, {
"terms": {
"attachments.isprivate": ['1', True, 1]
}
}]
}
})
Log.note("Ensure {{num}} attachments did not leak",
{"num": len(private_attachments)})
#VERIFY NONE IN PUBLIC
leaked_bugs = get(
self.public,
{
"and": [
{
"range": {
"bug_id": {
"gte": min_id,
"lt": max_id
}
}
},
{
"range": {
"expires_on": {
"gte": NOW
}
}
}, # CURRENT BUGS
{
"nested": {
"path": "attachments",
"query": {
"filtered": {
"query": {
"match_all": {}
},
"filter": {
"terms": {
"attach_id":
private_attachments
}
}
}
}
}
}
]
}
# fields=["bug_id", "attachments"]
)
#
if leaked_bugs:
if self.settings.param.delete:
self.public.delete_record(
{"terms": {
"bug_id": leaked_bugs.bug_id
}})
Log.note("{{num}} bugs with private attachments have leaked!",
{"num": len(leaked_bugs)})
for b in leaked_bugs:
Log.note(
"{{bug_id}} has private_attachment\n{{version|indent}}",
{
"bug_id": b.bug_id,
"version": b
})
Log.error("Attachments have leaked!")
