def _get_and_retry(self, url, **kwargs):
"""
requests 2.5.0 HTTPS IS A LITTLE UNSTABLE
"""
kwargs = set_default(kwargs, {"timeout": self.timeout.seconds})
try:
return http.get(url, **kwargs)
except Exception, e:
try:
Thread.sleep(seconds=5)
return http.get(url.replace("https://", "http://"), **kwargs)
except Exception, f:
Log.error("Tried {{url}} twice. Both failed.", {"url": url}, cause=[e, f])
def _get_and_retry(self, url, **kwargs):
"""
requests 2.5.0 HTTPS IS A LITTLE UNSTABLE
"""
kwargs = set_default(kwargs, {"timeout": self.timeout.seconds})
try:
return http.get(url, **kwargs)
except Exception, e:
try:
Thread.sleep(seconds=5)
return http.get(url.replace("https://", "http://"), **kwargs)
except Exception, f:
Log.error("Tried {{url}} twice. Both failed.", {"url": url},
cause=[e, f])
def test_branch_count(self):
if self.not_real_service():
return
test = wrap({"query": {
"from": {
"type": "elasticsearch",
"settings": {
"host": ES_CLUSTER_LOCATION,
"index": "unittest",
"type": "test_result"
}
},
"select": [
{"aggregate": "count"},
],
"edges": [
"build.branch"
],
"where": {"or": [
{"missing": "build.id"}
# {"gte": {"timestamp": Date.floor(Date.now() - (Duration.DAY * 7), Duration.DAY).milli / 1000}}
]},
"format": "table"
}})
query = convert.unicode2utf8(convert.value2json(test.query))
# EXECUTE QUERY
with Timer("query"):
response = http.get(self.service_url, data=query)
if response.status_code != 200:
error(response)
result = convert.json2value(convert.utf82unicode(response.all_content))
Log.note("result\n{{result|indent}}", {"result": result})
def verify_blobber_file(line_number, name, url):
"""
:param line_number: for debugging
:param name: for debugging
:param url: TO BE READ
:return: RETURNS BYTES **NOT** UNICODE
"""
if name in ["emulator-5554.log", "qemu.log"] or any(map(name.endswith, [".png", ".html"])):
return None, 0
with Timer("Read {{name}}: {{url}}", {"name": name, "url": url}, debug=DEBUG):
response = http.get(url)
try:
logs = response.all_lines
except Exception, e:
if name.endswith("_raw.log"):
Log.error(
"Line {{line}}: {{name}} = {{url}} is NOT structured log",
line=line_number,
name=name,
url=url,
cause=e
)
if DEBUG:
Log.note(
"Line {{line}}: {{name}} = {{url}} is NOT structured log",
line=line_number,
name=name,
url=url
)
return None, 0
def verify_blobber_file(line_number, name, url):
"""
:param line_number: for debugging
:param name: for debugging
:param url: TO BE READ
:return: RETURNS BYTES **NOT** UNICODE
"""
if name in ["emulator-5554.log", "qemu.log"] or any(
map(name.endswith, [".png", ".html"])):
return None, 0
with Timer("Read {{name}}: {{url}}", {
"name": name,
"url": url
},
debug=DEBUG):
response = http.get(url)
try:
logs = response.all_lines
except Exception, e:
if name.endswith("_raw.log"):
Log.error(
"Line {{line}}: {{name}} = {{url}} is NOT structured log",
line=line_number,
name=name,
url=url,
cause=e)
if DEBUG:
Log.note(
"Line {{line}}: {{name}} = {{url}} is NOT structured log",
line=line_number,
name=name,
url=url)
return None, 0
def test_multiple_agg_on_same_field(self):
if self.not_real_service():
return
test = wrap({
"query": {
"from": {
"type": "elasticsearch",
"settings": {
"host": ES_CLUSTER_LOCATION,
"index": "unittest",
"type": "test_result"
}
},
"select": [{
"name": "max_bytes",
"value": "run.stats.bytes",
"aggregate": "max"
}, {
"name": "count",
"value": "run.stats.bytes",
"aggregate": "count"
}]
}
})
query = unicode2utf8(convert.value2json(test.query))
# EXECUTE QUERY
with Timer("query"):
response = http.get(self.testing.query, data=query)
if response.status_code != 200:
error(response)
result = json2value(utf82unicode(response.all_content))
Log.note("result\n{{result|indent}}", {"result": result})
def _get_branches_from_hg(settings):
# GET MAIN PAGE
response = http.get(settings.url)
doc = BeautifulSoup(response.all_content)
all_repos = doc("table")[1]
branches = UniqueIndex(["name", "locale"], fail_on_dup=False)
for i, r in enumerate(all_repos("tr")):
dir, name = [v.text.strip() for v in r("td")]
b = _get_single_branch_from_hg(settings, name, dir.lstrip("/"))
branches.extend(b)
# branches.add(set_default({"name": "release-mozilla-beta"}, branches["mozilla-beta", DEFAULT_LOCALE]))
for b in list(branches["mozilla-beta", ]):
branches.add(set_default({"name": "release-mozilla-beta"}, b)) # THIS IS THE l10n "name"
b.url = "https://hg.mozilla.org/releases/mozilla-beta" # THIS IS THE
for b in list(branches["mozilla-release", ]):
branches.add(set_default({"name": "release-mozilla-release"}, b))
for b in list(branches["mozilla-aurora", ]):
if b.locale == "en-US":
continue
branches.add(set_default({"name": "comm-aurora"}, b))
# b.url = "https://hg.mozilla.org/releases/mozilla-aurora"
return branches
def get_branches(settings):
# GET MAIN PAGE
response = http.get(settings.url)
doc = BeautifulSoup(response.all_content)
all_repos = doc("table")[1]
branches = UniqueIndex(["name", "locale"], fail_on_dup=False)
for i, r in enumerate(all_repos("tr")):
dir, name = [v.text.strip() for v in r("td")]
b = get_branch(settings, name, dir.lstrip("/"))
branches.extend(b)
# branches.add(set_default({"name": "release-mozilla-beta"}, branches["mozilla-beta", DEFAULT_LOCALE]))
for b in list(branches["mozilla-beta", ]):
branches.add(set_default({"name": "release-mozilla-beta"}, b))
for b in list(branches["mozilla-release", ]):
branches.add(set_default({"name": "release-mozilla-release"}, b))
for b in list(branches["mozilla-aurora", ]):
if b.locale == "en-US":
continue
branches.add(set_default({"name": "comm-aurora"}, b))
return branches
def _get_url(url, branch, **kwargs):
with Explanation("get push from {{url}}", url=url):
response = http.get(url, **kwargs)
data = convert.json2value(response.content.decode("utf8"))
if isinstance(data, basestring) and data.startswith("unknown revision"):
Log.error("Unknown push {{revision}}", revision=strings.between(data, "'", "'"))
branch.url = _trim(url) #RECORD THIS SUCCESS IN THE BRANCH
return data
def _get_url(url, branch, **kwargs):
with Explanation("get push from {{url}}", url=url, debug=DEBUG):
response = http.get(url, **kwargs)
data = json2value(response.content.decode("utf8"))
if isinstance(data, (text_type, str)) and data.startswith("unknown revision"):
Log.error(UNKNOWN_PUSH, revision=strings.between(data, "'", "'"))
branch.url = _trim(url) # RECORD THIS SUCCESS IN THE BRANCH
return data
def _get_url(url, branch, **kwargs):
with Explanation("get push from {{url}}", url=url, debug=DEBUG):
response = http.get(url, **kwargs)
data = json2value(response.content.decode("utf8"))
if isinstance(data, (text_type, str)) and data.startswith("unknown revision"):
Log.error("Unknown push {{revision}}", revision=strings.between(data, "'", "'"))
branch.url = _trim(url) # RECORD THIS SUCCESS IN THE BRANCH
return data
def more():
xml = http.get(self.url + "?" + value2url_param(state)).content
data = BeautifulSoup(xml, 'xml')
state.get_more = data.find("istruncated").contents[0] == "true"
contents = data.findAll("contents")
state.marker = contents[-1].find("key").contents[0]
return [{k: t(d.find(k).contents[0]) for k, t in content_keys.items()} for d in contents]
def more():
xml = http.get(self.url + "?" + value2url_param(state)).content
data = BeautifulSoup(xml, 'xml')
state.get_more = data.find("istruncated").contents[0] == "true"
contents = data.findAll("contents")
state.marker = contents[-1].find("key").contents[0]
return [{k: t(d.find(k).contents[0]) for k, t in content_keys.items()} for d in contents]
def _find(b, please_stop):
try:
url = b.url + "rev/" + revision
response = http.get(url)
if response.status_code == 200:
Log.note("{{revision}} found at {{url}}", url=url, revision=revision)
except Exception, e:
pass
def _open(self):
""" DO NOT USE THIS UNLESS YOU close() FIRST"""
if self.settings.host.startswith("mysql://"):
# DECODE THE URI: mysql://username:password@host:optional_port/database_name
up = strings.between(self.settings.host, "mysql://", "@")
if ":" in up:
self.settings.username, self.settings.password = unquote(up).split(":")
else:
self.settings.username = up
url = strings.between(self.settings.host, "@", None)
hp, self.settings.schema = url.split("/", 1)
if ":" in hp:
self.settings.host, self.settings.port = hp.split(":")
self.settings.port = int(self.settings.port)
else:
self.settings.host = hp
# SSL PEM
if self.settings.host in ("localhost", "mysql"):
ssl_context = None
else:
if self.settings.ssl and not self.settings.ssl.pem:
Log.error("Expecting 'pem' property in ssl")
# ssl_context = ssl.create_default_context(**get_ssl_pem_file(self.settings.ssl.pem))
filename = File(".pem") / URL(self.settings.ssl.pem).host
filename.write_bytes(http.get(self.settings.ssl.pem).content)
ssl_context = {"ca": filename.abspath}
try:
self.db = connect(
host=self.settings.host,
port=self.settings.port,
user=coalesce(self.settings.username, self.settings.user),
passwd=coalesce(self.settings.password, self.settings.passwd),
db=coalesce(self.settings.schema, self.settings.db),
read_timeout=coalesce(self.settings.read_timeout, (EXECUTE_TIMEOUT / 1000) - 10 if EXECUTE_TIMEOUT else None, 5*60),
charset=u"utf8",
use_unicode=True,
ssl=ssl_context,
cursorclass=cursors.SSCursor
)
except Exception as e:
if self.settings.host.find("://") == -1:
Log.error(
u"Failure to connect to {{host}}:{{port}}",
host=self.settings.host,
port=self.settings.port,
cause=e
)
else:
Log.error(u"Failure to connect. PROTOCOL PREFIX IS PROBABLY BAD", e)
self.cursor = None
self.partial_rollback = False
self.transaction_level = 0
self.backlog = [] # accumulate the write commands so they are sent at once
if self.readonly:
self.begin()
def get_http(ref, url):
from pyLibrary.env import http
params = url.query
new_value = convert.json2value(http.get(ref),
params=params,
flexible=True,
paths=True)
return new_value
def test_timing(self):
if self.not_real_service():
return
test = wrap({
"query": {
"from": {
"type": "elasticsearch",
"settings": {
"host": ES_CLUSTER_LOCATION,
"index": "unittest",
"type": "test_result"
}
},
"select": [{
"name": "count",
"value": "run.duration",
"aggregate": "count"
}, {
"name": "total",
"value": "run.duration",
"aggregate": "sum"
}],
"edges": [{
"name": "chunk",
"value": ["run.suite", "run.chunk"]
}, "result.ok"],
"where": {
"and": [{
"lt": {
"timestamp": Date.floor(Date.now()).milli / 1000
}
}, {
"gte": {
"timestamp":
Date.floor(Date.now() - (Duration.DAY * 7),
Duration.DAY).milli / 1000
}
}]
},
"format":
"cube",
"samples": {
"limit": 30
}
}
})
query = unicode2utf8(convert.value2json(test.query))
# EXECUTE QUERY
with Timer("query"):
response = http.get(self.testing.query, data=query)
if response.status_code != 200:
error(response)
result = json2value(utf82unicode(response.all_content))
Log.note("result\n{{result|indent}}", {"result": result})
def _get_source_code_from_hg(self, revision, file_path):
response = http.get(
expand_template(
FILE_URL, {
"location": revision.branch.url,
"rev": revision.changeset.id,
"path": file_path
}))
return response.content.decode("utf8", "replace")
def _find(b, please_stop):
try:
url = b.url + "rev/" + revision
response = http.get(url)
if response.status_code == 200:
Log.note("{{revision}} found at {{url}}",
url=url,
revision=revision)
except Exception, e:
pass
def test_coverage_parser(self):
diff = http.get(
'https://hg.mozilla.org/mozilla-central/raw-rev/14dc6342ec5'
).content.decode('utf8')
moves = diff_to_moves(diff)
Log.note("{{files}}",
files=[
m.old.name if m.new.name == 'dev/null' else m.new.name
for m in moves
])
def get_branch(settings, description, dir):
if dir == "users":
return []
response = http.get(settings.url + "/" + dir)
doc = BeautifulSoup(response.all_content)
output = []
try:
all_branches = doc("table")[0]
except Exception, _:
return []
def _get_single_branch_from_hg(settings, description, dir):
if dir == "users":
return []
response = http.get(settings.url + "/" + dir)
doc = BeautifulSoup(response.all_content)
output = []
try:
all_branches = doc("table")[0]
except Exception, _:
return []
def _find(please_stop):
for b in queue:
if please_stop:
return
try:
url = b.url + "json-info?node=" + revision
response = http.get(url, timeout=30)
if response.status_code == 200:
with locker:
output.append(b)
Log.note("{{revision}} found at {{url}}", url=url, revision=revision)
except Exception, f:
problems.append(f)
def test_simple_query(self):
if self.not_real_service():
return
query = convert.unicode2utf8(convert.value2json({"from": "unittest"}))
# EXECUTE QUERY
with Timer("query"):
response = http.get(self.service_url, data=query)
if response.status_code != 200:
error(response)
result = convert.json2value(convert.utf82unicode(response.all_content))
Log.note("result\n{{result|indent}}", {"result": result})
def inner(changeset_id):
if self.es.cluster.version.startswith("1.7."):
query = {
"query": {"filtered": {
"query": {"match_all": {}},
"filter": {"and": [
{"prefix": {"changeset.id": changeset_id}},
{"range": {"etl.timestamp": {"gt": MIN_ETL_AGE}}}
]}
}},
"size": 1
}
else:
query = {
"query": {"bool": {"must": [
{"prefix": {"changeset.id": changeset_id}},
{"range": {"etl.timestamp": {"gt": MIN_ETL_AGE}}}
]}},
"size": 1
}
try:
# ALWAYS TRY ES FIRST
with self.es_locker:
response = self.es.search(query)
json_diff = response.hits.hits[0]._source.changeset.diff
if json_diff:
return json_diff
except Exception as e:
pass
url = expand_template(DIFF_URL, {"location": revision.branch.url, "rev": changeset_id})
if DEBUG:
Log.note("get unified diff from {{url}}", url=url)
try:
response = http.get(url)
diff = response.content.decode("utf8", "replace")
json_diff = diff_to_json(diff)
num_changes = _count(c for f in json_diff for c in f.changes)
if json_diff:
if num_changes < MAX_DIFF_SIZE:
return json_diff
elif revision.changeset.description.startswith("merge "):
return None # IGNORE THE MERGE CHANGESETS
else:
Log.warning("Revision at {{url}} has a diff with {{num}} changes, ignored", url=url, num=num_changes)
for file in json_diff:
file.changes = None
return json_diff
except Exception as e:
Log.warning("could not get unified diff", cause=e)
def get(self, path, **kwargs):
url = self.settings.host + ":" + unicode(self.settings.port) + path
try:
response = http.get(url, **kwargs)
if response.status_code not in [200]:
Log.error(response.reason+": "+response.all_content)
if self.debug:
Log.note("response: {{response}}", response=strings.limit(utf82unicode(response.all_content), 130))
details = wrap(convert.json2value(utf82unicode(response.all_content)))
if details.error:
Log.error(details.error)
return details
except Exception, e:
Log.error("Problem with call to {{url}}", url=url, cause=e)
def get(self, path, **kwargs):
url = self.settings.host + ":" + unicode(self.settings.port) + path
try:
response = http.get(url, **kwargs)
if response.status_code not in [200]:
Log.error(response.reason+": "+response.all_content)
if self.debug:
Log.note("response: {{response}}", response=strings.limit(utf82unicode(response.all_content), 130))
details = wrap(convert.json2value(utf82unicode(response.all_content)))
if details.error:
Log.error(details.error)
return details
except Exception, e:
Log.error("Problem with call to {{url}}", url=url, cause=e)
def inner(changeset_id):
if self.es.cluster.version.startswith("1.7."):
query = {
"query": {"filtered": {
"query": {"match_all": {}},
"filter": {"and": [
{"prefix": {"changeset.id": changeset_id}},
{"range": {"etl.timestamp": {"gt": MIN_ETL_AGE}}}
]}
}},
"size": 1
}
else:
query = {
"query": {"bool": {"must": [
{"prefix": {"changeset.id": changeset_id}},
{"range": {"etl.timestamp": {"gt": MIN_ETL_AGE}}}
]}},
"size": 1
}
try:
# ALWAYS TRY ES FIRST
with self.es_locker:
response = self.es.search(query)
json_diff = response.hits.hits[0]._source.changeset.diff
if json_diff:
return json_diff
except Exception as e:
pass
url = expand_template(DIFF_URL, {"location": revision.branch.url, "rev": changeset_id})
DEBUG and Log.note("get unified diff from {{url}}", url=url)
try:
response = http.get(url)
diff = response.content.decode("utf8")
json_diff = diff_to_json(diff)
num_changes = _count(c for f in json_diff for c in f.changes)
if json_diff:
if revision.changeset.description.startswith("merge "):
return None # IGNORE THE MERGE CHANGESETS
elif num_changes < MAX_DIFF_SIZE:
return json_diff
else:
Log.warning("Revision at {{url}} has a diff with {{num}} changes, ignored", url=url, num=num_changes)
for file in json_diff:
file.changes = None
return json_diff
except Exception as e:
Log.warning("could not get unified diff from {{url}}", url=url, cause=e)
def _get_branches_from_hg(kwarg):
# GET MAIN PAGE
response = http.get(kwarg.url)
doc = BeautifulSoup(response.all_content)
all_repos = doc("table")[1]
branches = UniqueIndex(["name", "locale"], fail_on_dup=False)
for i, r in enumerate(all_repos("tr")):
dir, name = [v.text.strip() for v in r("td")]
b = _get_single_branch_from_hg(kwarg, name, dir.lstrip("/"))
branches.extend(b)
# branches.add(set_default({"name": "release-mozilla-beta"}, branches["mozilla-beta", DEFAULT_LOCALE]))
for b in list(branches["mozilla-beta", ]):
branches.add(set_default({"name": "release-mozilla-beta"},
b)) # THIS IS THE l10n "name"
b.url = "https://hg.mozilla.org/releases/mozilla-beta" # THIS IS THE
for b in list(branches["mozilla-release", ]):
branches.add(set_default({"name": "release-mozilla-release"}, b))
for b in list(branches["mozilla-aurora", ]):
if b.locale == "en-US":
continue
branches.add(set_default({"name": "comm-aurora"}, b))
# b.url = "https://hg.mozilla.org/releases/mozilla-aurora"
for b in list(branches):
if b.name.startswith("mozilla-esr"):
branches.add(set_default({"name": "release-" + b.name},
b)) # THIS IS THE l10n "name"
b.url = "https://hg.mozilla.org/releases/" + b.name
#CHECKS
for b in branches:
if b.name != b.name.lower():
Log.error("Expecting lowercase name")
if not b.locale:
Log.error("Not expected")
if not b.url.startswith("http"):
Log.error("Expecting a valid url")
if not b.etl.timestamp:
Log.error("Expecting a timestamp")
return branches
def test_longest_running_tests(self):
test = wrap({
"query": {
"sort": {
"sort": -1,
"field": "avg"
},
"from": {
"from":
"unittest",
"where": {
"and": [{
"gt": {
"build.date": "1439337600"
}
}]
},
"groupby": [
"build.platform", "build.type", "run.suite",
"result.test"
],
"select": [{
"aggregate": "avg",
"name": "avg",
"value": "result.duration"
}],
"format":
"table",
"limit":
100
},
"limit": 100,
"format": "list"
}
})
query = unicode2utf8(convert.value2json(test.query))
# EXECUTE QUERY
with Timer("query"):
response = http.get(self.testing.query, data=query)
if response.status_code != 200:
error(response)
result = json2value(utf82unicode(response.all_content))
Log.note("result\n{{result|indent}}", {"result": result})
def _get_branches_from_hg(settings):
# GET MAIN PAGE
response = http.get(settings.url)
doc = BeautifulSoup(response.all_content)
all_repos = doc("table")[1]
branches = UniqueIndex(["name", "locale"], fail_on_dup=False)
for i, r in enumerate(all_repos("tr")):
dir, name = [v.text.strip() for v in r("td")]
b = _get_single_branch_from_hg(settings, name, dir.lstrip("/"))
branches.extend(b)
# branches.add(set_default({"name": "release-mozilla-beta"}, branches["mozilla-beta", DEFAULT_LOCALE]))
for b in list(branches["mozilla-beta", ]):
branches.add(set_default({"name": "release-mozilla-beta"}, b)) # THIS IS THE l10n "name"
b.url = "https://hg.mozilla.org/releases/mozilla-beta" # THIS IS THE
for b in list(branches["mozilla-release", ]):
branches.add(set_default({"name": "release-mozilla-release"}, b))
for b in list(branches["mozilla-aurora", ]):
if b.locale == "en-US":
continue
branches.add(set_default({"name": "comm-aurora"}, b))
# b.url = "https://hg.mozilla.org/releases/mozilla-aurora"
for b in list(branches):
if b.name.startswith("mozilla-esr"):
branches.add(set_default({"name": "release-" + b.name}, b)) # THIS IS THE l10n "name"
b.url = "https://hg.mozilla.org/releases/" + b.name
#CHECKS
for b in branches:
if b.name != b.name.lower():
Log.error("Expecting lowercase name")
if not b.locale:
Log.error("Not expected")
if not b.url.startswith("http"):
Log.error("Expecting a valid url")
if not b.etl.timestamp:
Log.error("Expecting a timestamp")
return branches
def test_failures_by_directory(self):
if self.not_real_service():
return
test = wrap({"query": {
"from": {
"type": "elasticsearch",
"settings": {
"host": ES_CLUSTER_LOCATION,
"index": "unittest",
"type": "test_result"
}
},
"select": [
{
"aggregate": "count"
}
],
"edges": [
"result.test",
"result.ok"
],
"where": {
"prefix": {
"result.test": "/"
}
},
"format": "table"
}})
query = convert.unicode2utf8(convert.value2json(test.query))
# EXECUTE QUERY
with Timer("query"):
response = http.get(self.service_url, data=query)
if response.status_code != 200:
error(response)
result = convert.json2value(convert.utf82unicode(response.all_content))
Log.note("result\n{{result|indent}}", {"result": result})
def inner(changeset_id):
if self.es.cluster.version.startswith("1.7."):
query = {
"query": {"filtered": {
"query": {"match_all": {}},
"filter": {"and": [
{"prefix": {"changeset.id": changeset_id}},
{"range": {"etl.timestamp": {"gt": MIN_ETL_AGE}}}
]}
}},
"size": 1
}
else:
query = {
"query": {"bool": {"must": [
{"prefix": {"changeset.id": changeset_id}},
{"range": {"etl.timestamp": {"gt": MIN_ETL_AGE}}}
]}},
"size": 1
}
try:
# ALWAYS TRY ES FIRST
with self.es_locker:
response = self.es.search(query)
moves = response.hits.hits[0]._source.changeset.moves
if moves:
return moves
except Exception as e:
pass
url = expand_template(DIFF_URL, {"location": revision.branch.url, "rev": changeset_id})
DEBUG and Log.note("get unified diff from {{url}}", url=url)
try:
moves = http.get(url).content.decode('latin1') # THE ENCODING DOES NOT MATTER BECAUSE WE ONLY USE THE '+', '-' PREFIXES IN THE DIFF
return diff_to_moves(text_type(moves))
except Exception as e:
Log.warning("could not get unified diff from {{url}}", url=url, cause=e)
def _open(self):
""" DO NOT USE THIS UNLESS YOU close() FIRST"""
if self.settings.ssl.ca.startswith("https://"):
self.pemfile_url = self.settings.ssl.ca
self.pemfile = File("./resources/pem") / self.settings.host
self.pemfile.write_bytes(http.get(self.pemfile_url).content)
self.settings.ssl.ca = self.pemfile.abspath
try:
self.db = connect(
host=self.settings.host,
port=self.settings.port,
user=coalesce(self.settings.username, self.settings.user),
passwd=coalesce(self.settings.password, self.settings.passwd),
db=coalesce(self.settings.schema, self.settings.db),
read_timeout=coalesce(self.settings.read_timeout,
(EXECUTE_TIMEOUT / 1000) -
10 if EXECUTE_TIMEOUT else None, 5 * 60),
charset=u"utf8",
use_unicode=True,
ssl=coalesce(self.settings.ssl, None),
cursorclass=cursors.SSCursor)
except Exception as e:
if self.settings.host.find("://") == -1:
Log.error(u"Failure to connect to {{host}}:{{port}}",
host=self.settings.host,
port=self.settings.port,
cause=e)
else:
Log.error(
u"Failure to connect. PROTOCOL PREFIX IS PROBABLY BAD", e)
self.cursor = None
self.partial_rollback = False
self.transaction_level = 0
self.backlog = [
] # accumulate the write commands so they are sent at once
if self.readonly:
self.begin()
def get_http(ref, url):
from pyLibrary.env import http
params = url.query
new_value = _convert.json2value(http.get(ref), params=params, flexible=True, leaves=True)
return new_value
def test_empty_query(config, app):
url = "http://localhost:" + text_type(config.flask.port) + "/tuid"
response = http.get(url)
assert response.status_code == 400
assert response.content == EXPECTING_QUERY
def test_query_error(config, app):
url = "http://localhost:" + text_type(config.flask.port) + "/tuid"
http.get(url)
def test_query_too_big(config, app):
url = "http://localhost:" + text_type(config.flask.port) + "/tuid/"
name = "a"*10000000
response = http.get(url, json={"from": name})
assert response.status_code == 400
assert response.content == b"request too large"
def _get_single_branch_from_hg(settings, description, dir):
if dir == "users":
return []
response = http.get(settings.url + "/" + dir)
doc = BeautifulSoup(response.all_content, "html.parser")
output = []
try:
all_branches = doc("table")[0]
except Exception:
return []
for i, b in enumerate(all_branches("tr")):
if i == 0:
continue # IGNORE HEADER
columns = b("td")
try:
path = columns[0].a.get('href')
if path == "/":
continue
name, desc, last_used = [c.text.strip() for c in columns][0:3]
if last_used.startswith('at'):
last_used = last_used[2:]
detail = Data(
name=name.lower(),
locale=DEFAULT_LOCALE,
parent_name=description,
url=settings.url + path,
description=desc,
last_used=Date(last_used),
etl={"timestamp": Date.now()}
)
if detail.description == "unknown":
detail.description = None
# SOME BRANCHES HAVE NAME COLLISIONS, IGNORE LEAST POPULAR
if path in [
"/projects/dxr/", # moved to webtools
"/build/compare-locales/", # ?build team likes to clone?
"/build/puppet/", # ?build team likes to clone?
"/SeaMonkey/puppet/", # looses the popularity contest
"/releases/gaia-l10n/v1_2/en-US/", # use default branch
"/releases/gaia-l10n/v1_3/en-US/", # use default branch
"/releases/gaia-l10n/v1_4/en-US/", # use default branch
"/releases/gaia-l10n/v2_0/en-US/", # use default branch
"/releases/gaia-l10n/v2_1/en-US/", # use default branch
"/build/autoland/"
]:
continue
# MARKUP BRANCH IF LOCALE SPECIFIC
if path.startswith("/l10n-central"):
_path = path.strip("/").split("/")
detail.locale = _path[-1]
detail.name = "mozilla-central"
elif path.startswith("/releases/l10n/"):
_path = path.strip("/").split("/")
detail.locale = _path[-1]
detail.name = _path[-2].lower()
elif path.startswith("/releases/gaia-l10n/"):
_path = path.strip("/").split("/")
detail.locale = _path[-1]
detail.name = "gaia-" + _path[-2][1::]
elif path.startswith("/weave-l10n"):
_path = path.strip("/").split("/")
detail.locale = _path[-1]
detail.name = "weave"
if BRANCH_WHITELIST is not None:
found = False
for br in BRANCH_WHITELIST:
if br in str(detail.name):
found = True
break
if not found:
continue
Log.note("Branch {{name}} {{locale}}", name=detail.name, locale=detail.locale)
output.append(detail)
except Exception as e:
Log.warning("branch digestion problem", cause=e)
return output
def test_shutdown(config, app):
# SHOULD NOT ACCEPT A SHUTDOWN COMMAND, WILL BREAK OTHER TESTS
url = "http://localhost:" + text_type(config.flask.port) + "/shutdown"
http.get(url)
Log.alert("/shutdown sent, should have no effect")
def read_lines(self, key):
url = self.url + "/" + key
return http.get(url).all_lines
def test_query_too_big(config, app):
url = "http://localhost:" + text_type(config.flask.port) + "/tuid/"
name = "a" * 10000000
response = http.get(url, json={"from": name})
assert response.status_code == 400
assert response.content == b"request too large"
def test_shutdown(config, app):
# SHOULD NOT ACCEPT A SHUTDOWN COMMAND, WILL BREAK OTHER TESTS
url = "http://localhost:" + text_type(config.flask.port) + "/shutdown"
http.get(url)
Log.alert("/shutdown sent, should have no effect")
def inner(changeset_id):
if self.es.cluster.version.startswith("1.7."):
query = {
"query": {
"filtered": {
"query": {
"match_all": {}
},
"filter": {
"and": [{
"prefix": {
"changeset.id": changeset_id
}
}, {
"range": {
"etl.timestamp": {
"gt": MIN_ETL_AGE
}
}
}]
}
}
},
"size": 1
}
else:
query = {
"query": {
"bool": {
"must": [{
"prefix": {
"changeset.id": changeset_id
}
}, {
"range": {
"etl.timestamp": {
"gt": MIN_ETL_AGE
}
}
}]
}
},
"size": 1
}
try:
# ALWAYS TRY ES FIRST
with self.es_locker:
response = self.es.search(query)
moves = response.hits.hits[0]._source.changeset.moves
if moves:
return moves
except Exception as e:
pass
url = expand_template(DIFF_URL, {
"location": revision.branch.url,
"rev": changeset_id
})
DEBUG and Log.note("get unified diff from {{url}}", url=url)
try:
moves = http.get(url).content.decode(
'latin1'
) # THE ENCODING DOES NOT MATTER BECAUSE WE ONLY USE THE '+', '-' PREFIXES IN THE DIFF
return diff_to_moves(text_type(moves))
except Exception as e:
Log.warning("could not get unified diff from {{url}}",
url=url,
cause=e)
def read_lines(self, key):
url = self.url + "/" + key
return http.get(url).all_lines
def process(source_key, source, dest_bucket, resources, please_stop=None):
"""
SIMPLE CONVERT pulse_block INTO TALOS, IF ANY
"""
etl_head_gen = EtlHeadGenerator(source_key)
stats = Dict()
counter = 0
output = set()
for i, pulse_line in enumerate(source.read_lines()):
pulse_record = scrub_pulse_record(source_key, i, pulse_line, stats)
if not pulse_record:
continue
if not pulse_record.payload.talos:
continue
all_talos = []
etl_file = wrap({
"id": counter,
"file": pulse_record.payload.logurl,
"timestamp": Date.now().unix,
"source": pulse_record.etl,
"type": "join"
})
with Timer("Read {{url}}", {"url": pulse_record.payload.logurl}, debug=DEBUG) as timer:
try:
response = http.get(pulse_record.payload.logurl)
if response.status_code == 404:
Log.alarm("Talos log missing {{url}}", url=pulse_record.payload.logurl)
k = source_key + "." + unicode(counter)
try:
# IF IT EXISTS WE WILL ASSUME SOME PAST PROCESS TRANSFORMED THE MISSING DATA ALREADY
dest_bucket.get_key(k)
output |= {k} # FOR DENSITY CALCULATIONS
except Exception:
_, dest_etl = etl_head_gen.next(etl_file, "talos")
dest_etl.error = "Talos log missing"
output |= dest_bucket.extend([{
"id": etl2key(dest_etl),
"value": {
"etl": dest_etl,
"pulse": pulse_record.payload
}
}])
continue
all_log_lines = response.all_lines
for log_line in all_log_lines:
s = log_line.find(TALOS_PREFIX)
if s < 0:
continue
log_line = strings.strip(log_line[s + len(TALOS_PREFIX):])
talos = convert.json2value(convert.utf82unicode(log_line))
for t in talos:
_, dest_etl = etl_head_gen.next(etl_file, "talos")
t.etl = dest_etl
t.pulse = pulse_record.payload
all_talos.extend(talos)
except Exception, e:
Log.error("Problem processing {{url}}", {
"url": pulse_record.payload.logurl
}, e)
finally:
def test_query_error(config, app):
url = "http://localhost:" + text_type(config.flask.port) + "/tuid"
response = http.get(url, json={"from": "files"})
error = json2value(utf82unicode(response.content))
assert response.status_code == 400
assert "expecting a simple where clause with following structure" in error.template
def _get_source_code_from_hg(self, revision, file_path):
response = http.get(expand_template(FILE_URL, {"location": revision.branch.url, "rev": revision.changeset.id, "path": file_path}))
return response.content.decode("utf8", "replace")
def get_branches(self):
response = http.get(self.settings.branches.url, timeout=coalesce(self.settings.timeout, 30))
branches = convert.json2value(convert.utf82unicode(response.all_content))
return wrap({branch.name: unwrap(branch) for branch in branches})
def get_ssl_pem_file(url):
filename = File(".pem") / URL(url).host
filename.write_bytes(http.get(url).content)
return {"cafile": filename.abspath}
def test_empty_query(config, app):
url = "http://localhost:" + text_type(config.flask.port) + "/tuid"
response = http.get(url)
assert response.status_code == 400
assert response.content == b"expecting query"
def _get_single_branch_from_hg(settings, description, dir):
if dir == "users":
return []
response = http.get(settings.url + "/" + dir)
doc = BeautifulSoup(response.all_content, "html.parser")
output = []
try:
all_branches = doc("table")[0]
except Exception:
return []
for i, b in enumerate(all_branches("tr")):
if i == 0:
continue # IGNORE HEADER
columns = b("td")
try:
path = columns[0].a.get('href')
if path == "/":
continue
name, desc, last_used = [c.text.strip() for c in columns][0:3]
if last_used.startswith('at'):
last_used = last_used[2:]
detail = Data(
name=name.lower(),
locale=DEFAULT_LOCALE,
parent_name=description,
url=settings.url + path,
description=desc,
last_used=Date(last_used),
etl={"timestamp": Date.now()}
)
if detail.description == "unknown":
detail.description = None
# SOME BRANCHES HAVE NAME COLLISIONS, IGNORE LEAST POPULAR
if path in [
"/projects/dxr/", # moved to webtools
"/build/compare-locales/", # ?build team likes to clone?
"/build/puppet/", # ?build team likes to clone?
"/SeaMonkey/puppet/", # looses the popularity contest
"/releases/gaia-l10n/v1_2/en-US/", # use default branch
"/releases/gaia-l10n/v1_3/en-US/", # use default branch
"/releases/gaia-l10n/v1_4/en-US/", # use default branch
"/releases/gaia-l10n/v2_0/en-US/", # use default branch
"/releases/gaia-l10n/v2_1/en-US/", # use default branch
"/build/autoland/"
]:
continue
# MARKUP BRANCH IF LOCALE SPECIFIC
if path.startswith("/l10n-central"):
_path = path.strip("/").split("/")
detail.locale = _path[-1]
detail.name = "mozilla-central"
elif path.startswith("/releases/l10n/"):
_path = path.strip("/").split("/")
detail.locale = _path[-1]
detail.name = _path[-2].lower()
elif path.startswith("/releases/gaia-l10n/"):
_path = path.strip("/").split("/")
detail.locale = _path[-1]
detail.name = "gaia-" + _path[-2][1::]
elif path.startswith("/weave-l10n"):
_path = path.strip("/").split("/")
detail.locale = _path[-1]
detail.name = "weave"
Log.note("Branch {{name}} {{locale}}", name=detail.name, locale=detail.locale)
output.append(detail)
except Exception as e:
Log.warning("branch digestion problem", cause=e)
return output
def test_query_error(config, app):
url = "http://localhost:" + text_type(config.flask.port) + "/tuid"
http.get(url)
