def test_confidential_whiteboard_is_screened(self):
leaked_whiteboard = get(
self.private,
{
"and": [
{
"terms": {
"bug_group": SCREENED_WHITEBOARD_BUG_GROUPS
}
},
{
"exists": {
"field": "status_whiteboard"
}
},
{
"not": {
"terms": {
"status_whiteboard": ["", "[screened]"]
}
}
},
{
"range": {
"expires_on": {
"gte": NOW
}
}
}, #CURRENT RECORDS
{
"range": {
"modified_ts": {
"lt": A_WHILE_AGO
}
}
}, #OF A MINIMUM AGE
]
},
fields=[
"bug_id", "product", "component", "status_whiteboard",
"bug_group", "modified_ts"
],
limit=100)
if leaked_whiteboard:
for l in leaked_whiteboard:
l.modified_ts = CNV.datetime2string(
CNV.milli2datetime(l.modified_ts))
Log.error("Whiteboard leaking:\b{{leak}}",
{"leak": leaked_whiteboard})
def test_confidential_whiteboard_is_screened(self):
leaked_whiteboard = get(
self.private,
{"and": [
{"terms": {"bug_group": SCREENED_WHITEBOARD_BUG_GROUPS}},
{"exists": {"field": "status_whiteboard"}},
{"not": {"terms": {"status_whiteboard": ["", "[screened]"]}}},
{"range": {"expires_on": {"gte": NOW}}}, #CURRENT RECORDS
{"range": {"modified_ts": {"lt": A_WHILE_AGO}}}, #OF A MINIMUM AGE
]},
fields=["bug_id", "product", "component", "status_whiteboard", "bug_group", "modified_ts"],
limit=100
)
if leaked_whiteboard:
for l in leaked_whiteboard:
l.modified_ts=CNV.datetime2string(CNV.milli2datetime(l.modified_ts))
Log.error("Whiteboard leaking:\b{{leak}}", {"leak": leaked_whiteboard})
def milli2datetime(r):
"""
CONVERT ANY longs INTO TIME STRINGS
"""
try:
if r == None:
return None
elif isinstance(r, basestring):
return r
elif Math.is_number(r):
if CNV.value2number(r) > 800000000000:
return CNV.datetime2string(CNV.milli2datetime(r),
"%Y-%m-%d %H:%M:%S")
else:
return r
elif isinstance(r, dict):
output = {}
for k, v in r.items():
v = milli2datetime(v)
if v != None:
output[k.lower()] = v
return output
elif hasattr(r, '__iter__'):
output = []
for v in r:
v = milli2datetime(v)
if v != None:
output.append(v)
if not output:
return None
try:
return Q.sort(output)
except Exception:
return output
else:
return r
except Exception, e:
Log.warning("Can not scrub: {{json}}", {"json": r}, e)
def milli2datetime(r):
"""
CONVERT ANY longs INTO TIME STRINGS
"""
try:
if r == None:
return None
elif isinstance(r, basestring):
return r
elif Math.is_number(r):
if CNV.value2number(r) > 800000000000:
return CNV.datetime2string(CNV.milli2datetime(r), "%Y-%m-%d %H:%M:%S")
else:
return r
elif isinstance(r, dict):
output = {}
for k, v in r.items():
v = milli2datetime(v)
if v != None:
output[k.lower()] = v
return output
elif hasattr(r, '__iter__'):
output = []
for v in r:
v = milli2datetime(v)
if v != None:
output.append(v)
if not output:
return None
try:
return Q.sort(output)
except Exception:
return output
else:
return r
except Exception, e:
Log.warning("Can not scrub: {{json}}", {"json": r}, e)
def test_private_bugs_not_leaking(self):
bad_news = False
# FOR ALL BUG BLOCKS
for min_id, max_id in self.blocks_of_bugs():
results = get(
self.private,
{"and": [
{"match_all": {}},
{"and": [
{"range": {"bug_id": {"gte": min_id, "lt": max_id}}},
{"exists": {"field": "bug_group"}},
{"range": {"expires_on": {"gte": NOW}}}, #CURRENT RECORDS
{"range": {"modified_ts": {"lt": A_WHILE_AGO}}}, #OF A MINIMUM AGE
]}
]},
["bug_id", "bug_group", "modified_ts"]
)
private_ids = {b.bug_id: b.bug_group for b in results}
Log.note("Ensure {{num}} bugs did not leak", {
"num": len(private_ids.keys())
})
# VERIFY NONE IN PUBLIC
leaked_bugs = get(
self.public,
{"and": [
{"terms": {"bug_id": private_ids.keys()}},
{"range": {"expires_on": {"gte": NOW}}} # SOME BUGS WILL LEAK FOR A LITTLE WHILE
]}
)
if leaked_bugs:
bad_news = True
if self.settings.param.delete:
self.public.delete_record(
{"terms":{"bug_id":leaked_bugs.bug_id}}
)
Log.note("{{num}} leaks!! {{bugs}}", {
"num": len(leaked_bugs),
"bugs": Q.run({
"from":leaked_bugs,
"select":["bug_id", "bug_version_num", {"name":"modified_ts", "value":lambda d: CNV.datetime2string(CNV.milli2datetime(d.modified_ts))}],
"sort":"bug_id"
})
})
for b in leaked_bugs:
Log.note("{{bug_id}} has bug groups {{bug_group}}\n{{version|indent}}", {
"bug_id": b.bug_id,
"bug_group": private_ids[b.bug_id],
"version": milli2datetime(b)
})
#CHECK FOR LEAKED COMMENTS, BEYOND THE ONES LEAKED BY BUG
leaked_comments = get(
self.public_comments,
{"terms": {"bug_id": private_ids.keys()}},
limit=20
)
if leaked_comments:
bad_news = True
if self.settings.param.delete:
self.public_comments.delete_record(
{"terms":{"bug_id":leaked_comments.bug_id}}
)
Log.warning("{{num}} comments marked private have leaked!\n{{comments|indent}}", {
"num": len(leaked_comments),
"comments": leaked_comments
})
if bad_news:
Log.error("Bugs have leaked!")
def test_private_bugs_not_leaking(self):
bad_news = False
# FOR ALL BUG BLOCKS
for min_id, max_id in self.blocks_of_bugs():
results = get(
self.private,
{
"and": [
{
"match_all": {}
},
{
"and": [
{
"range": {
"bug_id": {
"gte": min_id,
"lt": max_id
}
}
},
{
"exists": {
"field": "bug_group"
}
},
{
"range": {
"expires_on": {
"gte": NOW
}
}
}, #CURRENT RECORDS
{
"range": {
"modified_ts": {
"lt": A_WHILE_AGO
}
}
}, #OF A MINIMUM AGE
]
}
]
},
["bug_id", "bug_group", "modified_ts"])
private_ids = {b.bug_id: b.bug_group for b in results}
Log.note("Ensure {{num}} bugs did not leak",
{"num": len(private_ids.keys())})
# VERIFY NONE IN PUBLIC
leaked_bugs = get(
self.public,
{
"and": [
{
"terms": {
"bug_id": private_ids.keys()
}
},
{
"range": {
"expires_on": {
"gte": NOW
}
}
} # SOME BUGS WILL LEAK FOR A LITTLE WHILE
]
})
if leaked_bugs:
bad_news = True
if self.settings.param.delete:
self.public.delete_record(
{"terms": {
"bug_id": leaked_bugs.bug_id
}})
Log.note(
"{{num}} leaks!! {{bugs}}", {
"num":
len(leaked_bugs),
"bugs":
Q.run({
"from":
leaked_bugs,
"select": [
"bug_id", "bug_version_num", {
"name":
"modified_ts",
"value":
lambda d: CNV.datetime2string(
CNV.milli2datetime(d.modified_ts))
}
],
"sort":
"bug_id"
})
})
for b in leaked_bugs:
Log.note(
"{{bug_id}} has bug groups {{bug_group}}\n{{version|indent}}",
{
"bug_id": b.bug_id,
"bug_group": private_ids[b.bug_id],
"version": milli2datetime(b)
})
#CHECK FOR LEAKED COMMENTS, BEYOND THE ONES LEAKED BY BUG
leaked_comments = get(self.public_comments,
{"terms": {
"bug_id": private_ids.keys()
}},
limit=20)
if leaked_comments:
bad_news = True
if self.settings.param.delete:
self.public_comments.delete_record(
{"terms": {
"bug_id": leaked_comments.bug_id
}})
Log.warning(
"{{num}} comments marked private have leaked!\n{{comments|indent}}",
{
"num": len(leaked_comments),
"comments": leaked_comments
})
if bad_news:
Log.error("Bugs have leaked!")
