def test_943478(url):
#https://bugzilla.mozilla.org/show_bug.cgi?id=943478
response = request("POST",
url + "/" + NOT_WHITELISTED + "/_search",
data="""
{
"query":{"filtered":{
"query":{"match_all":{}}
}},
"from":0,
"size":0,
"sort":[],
"facets":{"default":{"terms":{"field":"info.OS","size":200000}}}
}""")
if response.status_code != 400:
Log.error("should not allow")
# VERIFY ALLOWED INDEX GETS THROUGH
response = request("POST",
url + "/" + WHITELISTED + "/_search",
data="""
{
"query":{"filtered":{
"query":{"match_all":{}},
"filter":{"range":{"bug_id":{"lt":700000}}}
}},
"from":0,
"size":0,
"sort":[],
"facets":{"default":{"terms":{"field":"product","size":200000}}}
}""")
if response.status_code != 200:
Log.error("query should work")
def test_allow_3path_mapping(url):
#WE SHOULD ALLOW -mapping WITH INDEX AND TYPE IN PATH
#http://klahnakoski-es.corp.tor1.mozilla.com:9204/bugs/bug_version/_mapping
response = request("GET",
url + "/" + WHITELISTED + "/bug_version/_mapping")
if response.status_code != 200:
Log.error("should be allowed")
def test_943478(url):
#https://bugzilla.mozilla.org/show_bug.cgi?id=943478
response = request("POST", url + "/" + NOT_WHITELISTED + "/_search", data="""
{
"query":{"filtered":{
"query":{"match_all":{}}
}},
"from":0,
"size":0,
"sort":[],
"facets":{"default":{"terms":{"field":"info.OS","size":200000}}}
}""")
if response.status_code != 400:
Log.error("should not allow")
# VERIFY ALLOWED INDEX GETS THROUGH
response = request("POST", url + "/" + WHITELISTED + "/_search", data="""
{
"query":{"filtered":{
"query":{"match_all":{}},
"filter":{"range":{"bug_id":{"lt":700000}}}
}},
"from":0,
"size":0,
"sort":[],
"facets":{"default":{"terms":{"field":"product","size":200000}}}
}""")
if response.status_code != 200:
Log.error("query should work")
def test_943465(url):
#https://bugzilla.mozilla.org/show_bug.cgi?id=943465
response = request("GET", url + "/_cluster/nodes/_local")
if response.status_code != 400:
Log.error("should not allow")
response = request("GET", url + "/_cluster/nodes/stats")
if response.status_code != 400:
Log.error("should not allow")
def test_943465(url):
#https://bugzilla.mozilla.org/show_bug.cgi?id=943465
response = request("GET", url + "/_cluster/nodes/_local")
if response.status_code != 400:
Log.error("should not allow")
response = request("GET", url + "/_cluster/nodes/stats")
if response.status_code != 400:
Log.error("should not allow")
def all_tests(url):
test_allow_head_request(url)
test_943465(url)
test_943472(url)
test_943478(url)
test_allow_3path_mapping(url)
test_slow_streaming()
Log.note("ALL TESTS PASS")
def main():
url = "http://localhost:9292"
thread = Thread.run("run app", run_app)
try:
server_is_ready.wait_for_go()
all_tests(url)
finally:
thread.please_stop.go()
Log.stop()
def main():
url = "http://localhost:9292"
thread = Thread.run("run app", run_app)
try:
server_is_ready.wait_for_go()
all_tests(url)
finally:
thread.please_stop.go()
Log.stop()
def all_tests(url):
test_allow_head_request(url)
test_943465(url)
test_943472(url)
test_943478(url)
test_allow_3path_mapping(url)
test_slow_streaming()
Log.note("ALL TESTS PASS")
def main():
settings = startup.read_settings(defs=[{
"name": ["--file", "--dir"],
"help": "provide a file or directory",
"dest": "files"
}])
Log.start(settings.debug)
try:
process(File(settings.args.files))
except Exception, e:
Log.warning("Problem converting", e)
def request(type, url, data=None, **kwargs):
Log.note("CLIENT REQUEST: {{type}} {{url}} data={{data|newline|indent}}", {
"type": type,
"url": url,
"data": data,
"args": kwargs
})
response = requests.request(type, url, data=data, **kwargs)
Log.note("CLIENT GOT RESPONSE: {{status_code}}\n{{text|indent}}", {
"status_code": response.status_code,
"text": response.text
})
return response
def request(type, url, data=None, **kwargs):
Log.note("CLIENT REQUEST: {{type}} {{url}} data={{data|newline|indent}}", {
"type": type,
"url": url,
"data": data,
"args": kwargs
})
response = requests.request(type, url, data=data, **kwargs)
Log.note("CLIENT GOT RESPONSE: {{status_code}}\n{{text|indent}}", {
"status_code": response.status_code,
"text": response.text
})
return response
def test_slow_streaming():
"""
TEST THAT THE app ACTUALLY STREAMS. WE SHOULD GET A RESPONSE BEFORE THE SERVER
FINISHES DELIVERING
"""
slow_server_thread = Thread.run("run slow server", run_slow_server)
proxy_thread = Thread.run("run proxy", run_proxy)
try:
proxy_is_ready.wait_for_go()
server_is_ready.wait_for_go()
start = time.clock()
response = requests.get("http://localhost:" + str(PROXY_PORT) + PATH,
stream=True)
for i, data in enumerate(stream(response.raw)):
Log.note("CLIENT GOT RESPONSE:\n{{data|indent}}", {"data": data})
end = time.clock()
if i == 0 and end - start > 10: # IF WE GET DATA BEFORE 10sec, THEN WE KNOW WE ARE STREAMING
Log.error("should have something by now")
if response.status_code != 200:
Log.error("Expecting a positive response")
except Exception, e:
Log.error("Not expected", e)
def process(file, depth=0):
if file.name=="util": #SPECIFIC TO ALL THAT USE THE pyLibrary
return 0
if file.is_directory():
loc=0
for f in file.children:
try:
loc += process(f, depth+1)
except Exception, e:
pass
if loc>0:
Log.note(indent("{{lines|right_align(8)}} loc in {{filename}}", indent=depth), {
"lines": loc,
"filename": file.abspath
})
return loc
def run_slow_server(please_stop):
proc = subprocess.Popen(["python", "tests\\test_slow_server.py"],
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
bufsize=-1,
creationflags=CREATE_NEW_PROCESS_GROUP)
while not please_stop:
line = proc.stdout.readline()
if not line:
continue
if line.find(" * Running on") >= 0:
server_is_ready.go()
Log.note("SLOW SERVER: " + line)
proc.send_signal(signal.CTRL_C_EVENT)
def run_app(please_stop):
proc = subprocess.Popen(
["python", "esFrontLine\\app.py", "--settings", "tests/resources/test_settings.json"],
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
bufsize=-1,
creationflags=CREATE_NEW_PROCESS_GROUP
)
while not please_stop:
line = proc.stdout.readline()
if not line:
continue
if line.find(" * Running on") >= 0:
server_is_ready.go()
Log.note("SERVER: {{line}}", {"line": line.strip()})
proc.send_signal(signal.CTRL_C_EVENT)
def run_slow_server(please_stop):
proc = subprocess.Popen(
["python", "tests\\test_slow_server.py"],
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
bufsize=-1,
creationflags=CREATE_NEW_PROCESS_GROUP,
)
while not please_stop:
line = proc.stdout.readline()
if not line:
continue
if line.find(" * Running on") >= 0:
server_is_ready.go()
Log.note("SLOW SERVER: " + line)
proc.send_signal(signal.CTRL_C_EVENT)
def run_app(please_stop):
proc = subprocess.Popen([
"python", "esFrontLine\\app.py", "--settings",
"tests/resources/test_settings.json"
],
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
bufsize=-1,
creationflags=CREATE_NEW_PROCESS_GROUP)
while not please_stop:
line = proc.stdout.readline()
if not line:
continue
if line.find(" * Running on") >= 0:
server_is_ready.go()
Log.note("SERVER: {{line}}", {"line": line.strip()})
proc.send_signal(signal.CTRL_C_EVENT)
def test_allow_head_request(url):
#WE SHOULD ALLOW HEAD REQUESTS TO /
response = request("HEAD", url + "/")
if response.status_code != 200:
Log.error("should be allowed")
response = request("HEAD", url)
if response.status_code != 200:
Log.error("should be allowed")
# ENVEN HEAD REQUESTS TO WHITELISTED INDEXES WILL BE DENIED
response = request("HEAD", url + "/" + WHITELISTED + "/bug_version/_mapping")
if response.status_code == 200:
Log.error("should NOT be allowed")
def test_allow_head_request(url):
#WE SHOULD ALLOW HEAD REQUESTS TO /
response = request("HEAD", url + "/")
if response.status_code != 200:
Log.error("should be allowed")
response = request("HEAD", url)
if response.status_code != 200:
Log.error("should be allowed")
# ENVEN HEAD REQUESTS TO WHITELISTED INDEXES WILL BE DENIED
response = request("HEAD",
url + "/" + WHITELISTED + "/bug_version/_mapping")
if response.status_code == 200:
Log.error("should NOT be allowed")
def test_slow_streaming():
"""
TEST THAT THE app ACTUALLY STREAMS. WE SHOULD GET A RESPONSE BEFORE THE SERVER
FINISHES DELIVERING
"""
slow_server_thread = Thread.run("run slow server", run_slow_server)
proxy_thread = Thread.run("run proxy", run_proxy)
try:
proxy_is_ready.wait_for_go()
server_is_ready.wait_for_go()
start = time.clock()
response = requests.get("http://localhost:" + str(PROXY_PORT) + PATH, stream=True)
for i, data in enumerate(stream(response.raw)):
Log.note("CLIENT GOT RESPONSE:\n{{data|indent}}", {"data": data})
end = time.clock()
if i == 0 and end - start > 10: # IF WE GET DATA BEFORE 10sec, THEN WE KNOW WE ARE STREAMING
Log.error("should have something by now")
if response.status_code != 200:
Log.error("Expecting a positive response")
except Exception, e:
Log.error("Not expected", e)
from tests_by_bug_id import all_tests
from util.env.logs import Log
url = "http://klahnakoski-es.corp.tor1.mozilla.com:9201"
try:
all_tests(url)
finally:
Log.stop()
def test_allow_3path_mapping(url):
#WE SHOULD ALLOW -mapping WITH INDEX AND TYPE IN PATH
#http://klahnakoski-es.corp.tor1.mozilla.com:9204/bugs/bug_version/_mapping
response = request("GET", url + "/" + WHITELISTED + "/bug_version/_mapping")
if response.status_code != 200:
Log.error("should be allowed")
def test_943472(url):
#https://bugzilla.mozilla.org/show_bug.cgi?id=943472
response = request("GET", url + "/" + WHITELISTED + "/_stats/")
if response.status_code != 400:
Log.error("should not allow")
def test_943472(url):
#https://bugzilla.mozilla.org/show_bug.cgi?id=943472
response = request("GET", url + "/" + WHITELISTED + "/_stats/")
if response.status_code != 400:
Log.error("should not allow")
code = file.read()
code = replace(r'\"\"\".*?\"\"\"', r'', code) # REMOVE MULTILINE COMMENTS
code = replace(r'#.*?\n', r'', code) # REMOVE EOL COMMENTS
old_code = None
while (code != old_code):
old_code = code
code = replace(r'\n\s*?\n', r'\n', code) # REMOVE BLANK LINES
loc = len(code.split("\n"))
if loc == 1 and len(code.strip()) == 0:
loc = 0
Log.note(indent("{{lines|right_align(8)}} loc in {{filename}}", indent=depth), {
"lines": loc,
"filename": file.abspath
})
return loc
def main():
settings = startup.read_settings(defs=[{
"name": ["--file", "--dir"],
"help": "provide a file or directory",
"dest": "files"
}])
Log.start(settings.debug)
try:
process(File(settings.args.files))
except Exception, e:
Log.warning("Problem converting", e)
finally:
