def __init__(
self,
host,
index,
port=9200,
type="log",
queue_size=1000,
batch_size=100,
kwargs=None,
):
"""
settings ARE FOR THE ELASTICSEARCH INDEX
"""
kwargs.timeout = Duration(coalesce(kwargs.timeout, "30second")).seconds
kwargs.retry.times = coalesce(kwargs.retry.times, 3)
kwargs.retry.sleep = Duration(coalesce(kwargs.retry.sleep, MINUTE)).seconds
kwargs.host = Random.sample(listwrap(host), 1)[0]
schema = json2value(value2json(SCHEMA), leaves=True)
schema.mappings[type].properties["~N~"].type = "nested"
self.es = Cluster(kwargs).get_or_create_index(
schema=schema,
limit_replicas=True,
typed=True,
kwargs=kwargs,
)
self.batch_size = batch_size
self.es.add_alias(coalesce(kwargs.alias, kwargs.index))
self.queue = Queue("debug logs to es", max=queue_size, silent=True)
self.worker = Thread.run("add debug logs to es", self._insert_loop)
def fix(source_key, rownum, line, source, sample_only_filter, sample_size):
"""
:param rownum:
:param line:
:param source:
:param sample_only_filter:
:param sample_size:
:return: (row, no_more_data) TUPLE WHERE row IS {"value":<data structure>} OR {"json":<text line>}
"""
value = json2value(line)
if rownum == 0:
if len(line) > MAX_RECORD_LENGTH:
_shorten(source_key, value, source)
value = _fix(value)
if sample_only_filter and Random.int(int(1.0/coalesce(sample_size, 0.01))) != 0 and jx.filter([value], sample_only_filter):
# INDEX etl.id==0, BUT NO MORE
if value.etl.id != 0:
Log.error("Expecting etl.id==0")
row = {"value": value}
return row, True
elif len(line) > MAX_RECORD_LENGTH:
_shorten(source_key, value, source)
value = _fix(value)
elif '"resource_usage":' in line:
value = _fix(value)
row = {"value": value}
return row, False
def output(*args):
with cache_store.locker:
if using_self:
self = args[0]
args = args[1:]
else:
self = cache_store
now = Date.now()
try:
_cache = getattr(self, attr_name)
except Exception, _:
_cache = {}
setattr(self, attr_name, _cache)
if Random.int(100) == 0:
# REMOVE OLD CACHE
_cache = {
k: v
for k, v in _cache.items() if v[0] == None or v[0] > now
}
setattr(self, attr_name, _cache)
timeout, key, value, exception = _cache.get(
args, (Null, Null, Null, Null))
def _send_email(self):
try:
if not self.accumulation:
return
with Emailer(self.settings) as emailer:
# WHO ARE WE SENDING TO
emails = Data()
for template, params in self.accumulation:
content = expand_template(template, params)
emails[literal_field(
self.settings.to_address)] += [content]
for c in self.cc:
if any(d in params.params.error for d in c.contains):
emails[literal_field(c.to_address)] += [content]
# SEND TO EACH
for to_address, content in emails.items():
emailer.send_email(from_address=self.settings.from_address,
to_address=listwrap(to_address),
subject=self.settings.subject,
text_data="\n\n".join(content))
self.accumulation = []
except Exception as e:
Log.warning("Could not send", e)
finally:
self.next_send = Date.now() + self.settings.average_interval * (
2 * Random.float())
def commit(self):
with self.lock:
if self.closed:
Log.error("Queue is closed, commit not allowed")
try:
self._add_pending({"add": {"status.start": self.start}})
for i in range(self.db.status.start, self.start):
self._add_pending({"remove": str(i)})
if self.db.status.end - self.start < 10 or Random.range(
0, 1000) == 0: # FORCE RE-WRITE TO LIMIT FILE SIZE
# SIMPLY RE-WRITE FILE
if DEBUG:
Log.note(
"Re-write {{num_keys}} keys to persistent queue",
num_keys=self.db.status.end - self.start)
for k in self.db.keys():
if k == "status" or int(k) >= self.db.status.start:
continue
Log.error("Not expecting {{key}}", key=k)
self._commit()
self.file.write(
mo_json.value2json({"add": self.db}) + "\n")
else:
self._commit()
except Exception as e:
raise e
def _send_email(self):
try:
if not self.accumulation:
return
with Emailer(self.settings) as emailer:
# WHO ARE WE SENDING TO
emails = Data()
for template, params in self.accumulation:
content = expand_template(template, params)
emails[literal_field(self.settings.to_address)] += [content]
for c in self.cc:
if any(d in params.params.error for d in c.contains):
emails[literal_field(c.to_address)] += [content]
# SEND TO EACH
for to_address, content in emails.items():
emailer.send_email(
from_address=self.settings.from_address,
to_address=listwrap(to_address),
subject=self.settings.subject,
text_data="\n\n".join(content)
)
self.accumulation = []
except Exception as e:
Log.warning("Could not send", e)
finally:
self.next_send = Date.now() + self.settings.average_interval * (2 * Random.float())
def fix(source_key, rownum, line, source, sample_only_filter, sample_size):
"""
:param rownum:
:param line:
:param source:
:param sample_only_filter:
:param sample_size:
:return: (row, no_more_data) TUPLE WHERE row IS {"value":<data structure>} OR {"json":<text line>}
"""
value = json2value(line)
if rownum == 0:
if len(line) > MAX_RECORD_LENGTH:
_shorten(source_key, value, source)
value = _fix(value)
if sample_only_filter and Random.int(int(1.0/coalesce(sample_size, 0.01))) != 0 and jx.filter([value], sample_only_filter):
# INDEX etl.id==0, BUT NO MORE
if value.etl.id != 0:
Log.error("Expecting etl.id==0")
row = {"value": value}
return row, True
elif len(line) > MAX_RECORD_LENGTH:
_shorten(source_key, value, source)
value = _fix(value)
elif '"resource_usage":' in line:
value = _fix(value)
row = {"value": value}
return row, False
def _create_new_shard(self):
primary_shard = self.container.create_table(
table=self.short_name + "_" + "".join(Random.sample(ALLOWED, 20)),
sharded=False,
schema=self._flake.schema,
kwargs=self.config,
)
self.shard = primary_shard.shard
def put(self, local, remote, use_sudo=False):
if use_sudo:
filename = "/tmp/" + Random.string(20, string.digits + 'ABCDEF')
self.conn.put(File(local).abspath, filename)
self.sudo("cp " + filename + " " + remote)
self.sudo("rm " + filename)
else:
self.conn.put(File(local).abspath, remote)
def test_wrap_2():
Log.alert("Random types")
switch = [
lambda: Random.int(20),
lambda: Random.string(20),
lambda: {"i": Random.int(2000)},
lambda: Data(i=Random.int(2000)),
lambda: FlatList([{"i": Random.int(2000)}]),
lambda: [{"i": Random.int(2000)}]
]
inputs = [switch[min(len(switch) - 1, int(floor(-log(Random.float(), 2))))]() for i in range(NUM_INPUT)]
for i in range(NUM_REPEAT):
results = []
gc.collect()
with Timer("more string: to_data"):
for v in inputs:
results.append(to_data(v))
results = []
gc.collect()
with Timer("more string: baseline"):
for v in inputs:
results.append(baseline(v))
Log.note("Done {{i}} of {{num}}", i=i, num=NUM_REPEAT)
def output(*args, **kwargs):
if kwargs:
Log.error(
"Sorry, caching only works with ordered parameter, not keyword arguments"
)
with cache_store.locker:
if using_self:
self = args[0]
args = args[1:]
else:
self = cache_store
now = Date.now()
try:
_cache = getattr(self, attr_name)
except Exception:
_cache = {}
setattr(self, attr_name, _cache)
if Random.int(100) == 0:
# REMOVE OLD CACHE
_cache = {
k: v
for k, v in _cache.items()
if v.timeout == None or v.timeout > now
}
setattr(self, attr_name, _cache)
timeout, key, value, exception = _cache.get(
args, (Null, Null, Null, Null))
if now >= timeout:
value = func(self, *args)
with cache_store.locker:
_cache[args] = CacheElement(now + cache_store.timeout, args,
value, None)
return value
if value == None:
if exception == None:
try:
value = func(self, *args)
with cache_store.locker:
_cache[args] = CacheElement(now + cache_store.timeout,
args, value, None)
return value
except Exception as e:
e = Except.wrap(e)
with cache_store.locker:
_cache[args] = CacheElement(now + cache_store.timeout,
args, None, e)
raise e
else:
raise exception
else:
return value
def __enter__(self):
if self.sample and Random.int(100) == 0:
_Log.warning("acquire lock {{name|quote}}", name=self.name)
self.debug and _Log.note("acquire lock {{name|quote}}",
name=self.name)
self.lock.acquire()
self.debug and _Log.note("acquired lock {{name|quote}}",
name=self.name)
return self
def _daemon(self, please_stop):
while not please_stop:
with Explanation("looking for work"):
try:
branch, revisions, after = self.todo.pop(till=please_stop)
except Exception as e:
if please_stop:
break
else:
raise e
if branch.name in DAEMON_DO_NO_SCAN:
continue
revisions = set(revisions)
# FIND THE REVSIONS ON THIS BRANCH
for r in list(revisions):
try:
rev = self.get_revision(
Revision(branch=branch, changeset={"id": r}),
None, # local
False, # get_diff
True, # get_moves
)
if after and after > rev.etl.timestamp:
rev = self._get_from_hg(revision=rev)
if DAEMON_DEBUG:
Log.note(
"found revision with push date {{date|datetime}}",
date=rev.push.date,
)
revisions.discard(r)
if rev.etl.timestamp > Date.now() - (
DAEMON_RECENT_HG_PULL * SECOND):
# SOME PUSHES ARE BIG, RUNNING THE RISK OTHER MACHINES ARE
# ALSO INTERESTED AND PERFORMING THE SAME SCAN. THIS DELAY
# WILL HAVE SMALL EFFECT ON THE MAJORITY OF SMALL PUSHES
# https://bugzilla.mozilla.org/show_bug.cgi?id=1417720
Till(seconds=Random.float(DAEMON_HG_INTERVAL *
2)).wait()
except Exception as e:
Log.warning(
"Scanning {{branch}} {{revision|left(12)}}",
branch=branch.name,
revision=r,
cause=e,
)
if "Read timed out" in e:
Till(seconds=DAEMON_WAIT_AFTER_TIMEOUT).wait()
# FIND ANY BRANCH THAT MAY HAVE THIS REVISION
for r in list(revisions):
self._find_revision(r)
def put(self, local, remote, use_sudo=False):
if self.conn.command_cwds and not remote.startswith(("/", "~")):
remote = self.conn.command_cwds[-1].rstrip("/'") + "/" + remote
if use_sudo:
filename = "/tmp/" + Random.hex(20)
self.conn.put(File(local).abspath, filename)
self.sudo("cp " + filename + " " + remote)
self.sudo("rm " + filename)
else:
self.conn.put(File(local).abspath, remote)
def __init__(
self,
host,
index,
port=9200,
type="log",
queue_size=1000,
batch_size=100,
kwargs=None,
):
"""
settings ARE FOR THE ELASTICSEARCH INDEX
"""
kwargs.timeout = Duration(coalesce(kwargs.timeout, "30second")).seconds
kwargs.retry.times = coalesce(kwargs.retry.times, 3)
kwargs.retry.sleep = Duration(coalesce(kwargs.retry.sleep,
MINUTE)).seconds
kwargs.host = Random.sample(listwrap(host), 1)[0]
rollover_interval = coalesce(kwargs.rollover.interval,
kwargs.rollover.max, "year")
rollover_max = coalesce(kwargs.rollover.max, kwargs.rollover.interval,
"year")
schema = set_default(kwargs.schema, {
"mappings": {
kwargs.type: {
"properties": {
"~N~": {
"type": "nested"
}
}
}
}
}, json2value(value2json(SCHEMA), leaves=True))
self.es = RolloverIndex(
rollover_field={"get": [{
"first": "."
}, {
"literal": "timestamp"
}]},
rollover_interval=rollover_interval,
rollover_max=rollover_max,
schema=schema,
limit_replicas=True,
typed=True,
read_only=False,
kwargs=kwargs,
)
self.batch_size = batch_size
self.queue = Queue("debug logs to es", max=queue_size, silent=True)
self.worker = Thread.run("add debug logs to es", self._insert_loop)
def fix(rownum, line, source, sample_only_filter, sample_size):
# ES SCHEMA IS STRICTLY TYPED, USE "code" FOR TEXT IDS
line = line.replace('{"id": "bb"}',
'{"code": "bb"}').replace('{"id": "tc"}',
'{"code": "tc"}')
# ES SCHEMA IS STRICTLY TYPED, THE SUITE OBJECT CAN NOT BE HANDLED
if source.name.startswith("active-data-test-result"):
# "suite": {"flavor": "plain-chunked", "name": "mochitest"}
found = strings.between(line, '"suite": {', '}')
if found:
suite_json = '{' + found + "}"
if suite_json:
suite = mo_json.json2value(suite_json)
suite = convert.value2json(coalesce(suite.fullname,
suite.name))
line = line.replace(suite_json, suite)
if source.name.startswith("active-data-codecoverage"):
d = convert.json2value(line)
if d.source.file.total_covered > 0:
return {"id": d._id, "json": line}, False
else:
return None, False
if rownum == 0:
value = mo_json.json2value(line)
if len(line) > MAX_RECORD_LENGTH:
_shorten(value, source)
_id, value = _fix(value)
row = {"id": _id, "value": value}
if sample_only_filter and Random.int(
int(1.0 / coalesce(sample_size, 0.01))) != 0 and jx.filter(
[value], sample_only_filter):
# INDEX etl.id==0, BUT NO MORE
if value.etl.id != 0:
Log.error("Expecting etl.id==0")
return row, True
elif len(line) > MAX_RECORD_LENGTH:
value = mo_json.json2value(line)
_shorten(value, source)
_id, value = _fix(value)
row = {"id": _id, "value": value}
elif line.find('"resource_usage":') != -1:
value = mo_json.json2value(line)
_id, value = _fix(value)
row = {"id": _id, "value": value}
else:
# FAST
_id = strings.between(line, "\"_id\": \"", "\"") # AVOID DECODING JSON
row = {"id": _id, "json": line}
return row, False
def get(self, remote, local, use_sudo=False):
if self.conn.command_cwds and not remote.startswith(("/", "~")):
remote = self.conn.command_cwds[-1].rstrip("/'") + "/" + remote
if use_sudo:
filename = "/tmp/" + Random.filename()
self.sudo("cp " + remote + " " + filename)
self.sudo("chmod a+r " + filename)
self.conn.get(filename, File(local).abspath)
self.sudo("rm " + filename)
else:
self.conn.get(remote, File(local).abspath)
def _get_from_elasticsearch(self, revision, locale=None, get_diff=False, get_moves=True):
rev = revision.changeset.id
if self.es.cluster.version.startswith("1.7."):
query = {
"query": {"filtered": {
"query": {"match_all": {}},
"filter": {"and": [
{"term": {"changeset.id12": rev[0:12]}},
{"term": {"branch.name": revision.branch.name}},
{"term": {"branch.locale": coalesce(locale, revision.branch.locale, DEFAULT_LOCALE)}},
{"range": {"etl.timestamp": {"gt": MIN_ETL_AGE}}}
]}
}},
"size": 20
}
else:
query = {
"query": {"bool": {"must": [
{"term": {"changeset.id12": rev[0:12]}},
{"term": {"branch.name": revision.branch.name}},
{"term": {"branch.locale": coalesce(locale, revision.branch.locale, DEFAULT_LOCALE)}},
{"range": {"etl.timestamp": {"gt": MIN_ETL_AGE}}}
]}},
"size": 20
}
for attempt in range(3):
try:
with self.es_locker:
docs = self.es.search(query).hits.hits
if len(docs) == 0:
return None
best = docs[0]._source
if len(docs) > 1:
for d in docs:
if d._id.endswith(d._source.branch.locale):
best = d._source
Log.warning("expecting no more than one document")
return best
except Exception as e:
e = Except.wrap(e)
if "EsRejectedExecutionException[rejected execution (queue capacity" in e:
(Till(seconds=Random.int(30))).wait()
continue
else:
Log.warning("Bad ES call, waiting for {{num}} seconds", num=WAIT_AFTER_NODE_FAILURE, cause=e)
Till(seconds=WAIT_AFTER_NODE_FAILURE).wait()
continue
Log.warning("ES did not deliver, fall back to HG")
return None
def output(*args):
with cache_store.locker:
if using_self:
self = args[0]
args = args[1:]
else:
self = cache_store
now = Date.now()
try:
_cache = getattr(self, attr_name)
except Exception:
_cache = {}
setattr(self, attr_name, _cache)
if Random.int(100) == 0:
# REMOVE OLD CACHE
_cache = {
k: v
for k, v in _cache.items() if v[0] == None or v[0] > now
}
setattr(self, attr_name, _cache)
timeout, key, value, exception = _cache.get(
args, (Null, Null, Null, Null))
if now >= timeout:
value = func(self, *args)
with cache_store.locker:
_cache[args] = (now + cache_store.timeout, args, value, None)
return value
if value == None:
if exception == None:
try:
value = func(self, *args)
with cache_store.locker:
_cache[args] = (now + cache_store.timeout, args, value,
None)
return value
except Exception as e:
e = Except.wrap(e)
with cache_store.locker:
_cache[args] = (now + cache_store.timeout, args, None,
e)
raise e
else:
raise exception
else:
return value
def test_wrap_2():
switch = [
lambda: {
"i": Random.int(2000)
}, lambda: Data(i=Random.int(2000)),
lambda: FlatList([{
"i": Random.int(2000)
}]), lambda: [{
"i": Random.int(2000)
}]
]
inputs = [
switch[min(len(switch) - 1, int(floor(-log(Random.float(), 2))))]()
for i in range(NUM_INPUT)
]
for i in range(NUM_REPEAT):
results = []
gc.collect()
with Profiler("more dict: slow_wrap"):
for v in inputs:
results.append(slow_wrap(v))
results = []
gc.collect()
with Profiler("more dict: wrap"):
for v in inputs:
results.append(wrap(v))
results = []
gc.collect()
with Profiler("more dict: baseline"):
for v in inputs:
results.append(baseline(v))
Log.note("Done {{i}} of {{num}}", {"i": i, "num": NUM_REPEAT})
def fix(rownum, line, source, sample_only_filter, sample_size):
value = json2value(line)
if value._id.startswith(("tc.97", "96", "bb.27")):
# AUG 24, 25 2017 - included full diff with repo; too big to index
try:
data = json2value(line)
repo = data.repo
repo.etl = None
repo.branch.last_used = None
repo.branch.description = None
repo.branch.etl = None
repo.branch.parent_name = None
repo.children = None
repo.parents = None
if repo.changeset.diff or data.build.repo.changeset.diff:
Log.error("no diff allowed")
else:
assertAlmostEqual(minimize_repo(repo), repo)
except Exception as e:
if CAN_NOT_DECODE_JSON in e:
raise e
data.repo = minimize_repo(repo)
data.build.repo = minimize_repo(data.build.repo)
line = value2json(data)
else:
pass
if rownum == 0:
if len(line) > MAX_RECORD_LENGTH:
_shorten(value, source)
value = _fix(value)
if sample_only_filter and Random.int(
int(1.0 / coalesce(sample_size, 0.01))) != 0 and jx.filter(
[value], sample_only_filter):
# INDEX etl.id==0, BUT NO MORE
if value.etl.id != 0:
Log.error("Expecting etl.id==0")
row = {"value": value}
return row, True
elif len(line) > MAX_RECORD_LENGTH:
_shorten(value, source)
value = _fix(value)
elif line.find('"resource_usage":') != -1:
value = _fix(value)
row = {"value": value}
return row, False
def test_id_vs_id(self):
ops = [Op() for _ in range(200)]
lang1 = {id(o): o for o in ops}
sample = Random.sample(ops, 1000 * 1000)
with Timer("using id()"):
result1 = [lang1[id(o)] for o in sample]
lang2 = [None] * (max(o.id for o in ops) + 1)
for o in ops:
lang2[o.id] = o
# lang2 = tuple(lang2)
with Timer("using o.id"):
result2 = [lang2[o.id] for o in sample]
def encrypt(text, _key, salt=None):
"""
RETURN {"salt":s, "length":l, "data":d} -> JSON -> UTF8
"""
if is_text(text):
encoding = 'utf8'
data = bytearray(text.encode("utf8"))
elif is_binary(text):
encoding = None
if PY2:
data = bytearray(text)
else:
data = text
if _key is None:
Log.error("Expecting a key")
if is_binary(_key):
_key = bytearray(_key)
if salt is None:
salt = Random.bytes(16)
# Initialize encryption using key and iv
key_expander_256 = key_expander.KeyExpander(256)
expanded_key = key_expander_256.expand(_key)
aes_cipher_256 = aes_cipher.AESCipher(expanded_key)
aes_cbc_256 = cbc_mode.CBCMode(aes_cipher_256, 16)
aes_cbc_256.set_iv(salt)
output = Data()
output.type = "AES256"
output.salt = bytes2base64(salt)
output.length = len(data)
output.encoding = encoding
encrypted = bytearray()
for _, d in _groupby16(data):
encrypted.extend(aes_cbc_256.encrypt_block(d))
output.data = bytes2base64(encrypted)
json = get_module("mo_json").value2json(output, pretty=True).encode('utf8')
if DEBUG:
test = decrypt(json, _key)
if test != text:
Log.error("problem with encryption")
return json
def output(*args, **kwargs):
if kwargs:
Log.error("Sorry, caching only works with ordered parameter, not keyword arguments")
with cache_store.locker:
if using_self:
self = args[0]
args = args[1:]
else:
self = cache_store
now = Date.now()
try:
_cache = getattr(self, attr_name)
except Exception:
_cache = {}
setattr(self, attr_name, _cache)
if Random.int(100) == 0:
# REMOVE OLD CACHE
_cache = {k: v for k, v in _cache.items() if v.timeout == None or v.timeout > now}
setattr(self, attr_name, _cache)
timeout, key, value, exception = _cache.get(args, (Null, Null, Null, Null))
if now >= timeout:
value = func(self, *args)
with cache_store.locker:
_cache[args] = CacheElement(now + cache_store.timeout, args, value, None)
return value
if value == None:
if exception == None:
try:
value = func(self, *args)
with cache_store.locker:
_cache[args] = CacheElement(now + cache_store.timeout, args, value, None)
return value
except Exception as e:
e = Except.wrap(e)
with cache_store.locker:
_cache[args] = CacheElement(now + cache_store.timeout, args, None, e)
raise e
else:
raise exception
else:
return value
def encrypt(text, _key, salt=None):
"""
RETURN {"salt":s, "length":l, "data":d} -> JSON -> UTF8
"""
if isinstance(text, text_type):
encoding = 'utf8'
data = bytearray(text.encode("utf8"))
elif isinstance(text, binary_type):
encoding = None
if PY2:
data = bytearray(text)
else:
data = text
if _key is None:
Log.error("Expecting a key")
if isinstance(_key, binary_type):
_key = bytearray(_key)
if salt is None:
salt = Random.bytes(16)
# Initialize encryption using key and iv
key_expander_256 = key_expander.KeyExpander(256)
expanded_key = key_expander_256.expand(_key)
aes_cipher_256 = aes_cipher.AESCipher(expanded_key)
aes_cbc_256 = cbc_mode.CBCMode(aes_cipher_256, 16)
aes_cbc_256.set_iv(salt)
output = Data()
output.type = "AES256"
output.salt = bytes2base64(salt)
output.length = len(data)
output.encoding = encoding
encrypted = bytearray()
for _, d in _groupby16(data):
encrypted.extend(aes_cbc_256.encrypt_block(d))
output.data = bytes2base64(encrypted)
json = get_module("mo_json").value2json(output, pretty=True).encode('utf8')
if DEBUG:
test = decrypt(json, _key)
if test != text:
Log.error("problem with encryption")
return json
def _daemon(self, please_stop):
while not please_stop:
with Explanation("looking for work"):
try:
branch, revisions = self.todo.pop(till=please_stop)
except Exception as e:
if please_stop:
break
else:
raise e
if branch.name in DAEMON_DO_NO_SCAN:
continue
revisions = set(revisions)
# FIND THE REVSIONS ON THIS BRANCH
for r in list(revisions):
try:
rev = self.get_revision(Revision(branch=branch, changeset={"id": r}))
if DAEMON_DEBUG:
Log.note("found revision with push date {{date|datetime}}", date=rev.push.date)
revisions.discard(r)
if rev.etl.timestamp > Date.now() - (DAEMON_RECENT_HG_PULL * SECOND):
# SOME PUSHES ARE BIG, RUNNING THE RISK OTHER MACHINES ARE
# ALSO INTERESTED AND PERFORMING THE SAME SCAN. THIS DELAY
# WILL HAVE SMALL EFFECT ON THE MAJORITY OF SMALL PUSHES
# https://bugzilla.mozilla.org/show_bug.cgi?id=1417720
Till(seconds=Random.float(DAEMON_HG_INTERVAL*2)).wait()
except Exception as e:
Log.warning(
"Scanning {{branch}} {{revision|left(12)}}",
branch=branch.name,
revision=r,
cause=e
)
if "Read timed out" in e:
Till(seconds=DAEMON_WAIT_AFTER_TIMEOUT).wait()
# FIND ANY BRANCH THAT MAY HAVE THIS REVISION
for r in list(revisions):
self._find_revision(r)
def fix(rownum, line, source, sample_only_filter, sample_size):
# ES SCHEMA IS STRICTLY TYPED, USE "code" FOR TEXT IDS
line = line.replace('{"id": "bb"}', '{"code": "bb"}').replace('{"id": "tc"}', '{"code": "tc"}')
# ES SCHEMA IS STRICTLY TYPED, THE SUITE OBJECT CAN NOT BE HANDLED
if source.name.startswith("active-data-test-result"):
# "suite": {"flavor": "plain-chunked", "name": "mochitest"}
found = strings.between(line, '"suite": {', '}')
if found:
suite_json = '{' + found + "}"
if suite_json:
suite = mo_json.json2value(suite_json)
suite = convert.value2json(coalesce(suite.fullname, suite.name))
line = line.replace(suite_json, suite)
if rownum == 0:
value = mo_json.json2value(line)
if len(line) > MAX_RECORD_LENGTH:
_shorten(value, source)
_id, value = _fix(value)
row = {"id": _id, "value": value}
if sample_only_filter and Random.int(int(1.0/coalesce(sample_size, 0.01))) != 0 and jx.filter([value], sample_only_filter):
# INDEX etl.id==0, BUT NO MORE
if value.etl.id != 0:
Log.error("Expecting etl.id==0")
return row, True
elif len(line) > MAX_RECORD_LENGTH:
value = mo_json.json2value(line)
_shorten(value, source)
_id, value = _fix(value)
row = {"id": _id, "value": value}
elif line.find('"resource_usage":') != -1:
value = mo_json.json2value(line)
_id, value = _fix(value)
row = {"id": _id, "value": value}
else:
# FAST
_id = strings.between(line, "\"_id\": \"", "\"") # AVOID DECODING JSON
row = {"id": _id, "json": line}
return row, False
def output(*args):
with cache_store.locker:
if using_self:
self = args[0]
args = args[1:]
else:
self = cache_store
now = Date.now()
try:
_cache = getattr(self, attr_name)
except Exception, _:
_cache = {}
setattr(self, attr_name, _cache)
if Random.int(100) == 0:
# REMOVE OLD CACHE
_cache = {k: v for k, v in _cache.items() if v[0]==None or v[0] > now}
setattr(self, attr_name, _cache)
timeout, key, value, exception = _cache.get(args, (Null, Null, Null, Null))
def device_register(self, path=None):
"""
EXPECTING A SIGNED REGISTRATION REQUEST
RETURN JSON WITH url FOR LOGIN
"""
now = Date.now().unix
request_body = request.get_data().strip()
signed = json2value(request_body.decode("utf8"))
command = json2value(base642bytes(signed.data).decode("utf8"))
session.public_key = command.public_key
rsa_crypto.verify(signed, session.public_key)
self.session_manager.setup_session(session)
session.expires = now + parse("10minute").seconds
session.state = bytes2base64URL(Random.bytes(32))
with self.device.db.transaction() as t:
t.execute(
sql_insert(
self.device.table,
{
"state": session.state,
"session_id": session.session_id
},
))
response = value2json(
Data(
session_id=session.session_id,
interval="5second",
expiry=session.expires,
url=URL(
self.device.home,
path=self.device.endpoints.login,
query={"state": session.state},
),
))
return Response(response,
headers={"Content-Type": "application/json"},
status=200)
def encrypt(text, _key, salt=None):
"""
RETURN JSON OF ENCRYPTED DATA {"salt":s, "length":l, "data":d}
"""
if not isinstance(text, unicode):
Log.error("only unicode is encrypted")
if _key is None:
Log.error("Expecting a key")
if isinstance(_key, str):
_key = bytearray(_key)
if salt is None:
salt = Random.bytes(16)
data = bytearray(text.encode("utf8"))
# Initialize encryption using key and iv
key_expander_256 = key_expander.KeyExpander(256)
expanded_key = key_expander_256.expand(_key)
aes_cipher_256 = aes_cipher.AESCipher(expanded_key)
aes_cbc_256 = cbc_mode.CBCMode(aes_cipher_256, 16)
aes_cbc_256.set_iv(salt)
output = Data()
output.type = "AES256"
output.salt = bytes2base64(salt)
output.length = len(data)
encrypted = bytearray()
for _, d in _groupby16(data):
encrypted.extend(aes_cbc_256.encrypt_block(d))
output.data = bytes2base64(encrypted)
json = get_module("mo_json").value2json(output)
if DEBUG:
test = decrypt(json, _key)
if test != text:
Log.error("problem with encryption")
return json
def encrypt(text, _key, salt=None):
"""
RETURN JSON OF ENCRYPTED DATA {"salt":s, "length":l, "data":d}
"""
if not isinstance(text, text_type):
Log.error("only unicode is encrypted")
if _key is None:
Log.error("Expecting a key")
if isinstance(_key, str):
_key = bytearray(_key)
if salt is None:
salt = Random.bytes(16)
data = bytearray(text.encode("utf8"))
# Initialize encryption using key and iv
key_expander_256 = key_expander.KeyExpander(256)
expanded_key = key_expander_256.expand(_key)
aes_cipher_256 = aes_cipher.AESCipher(expanded_key)
aes_cbc_256 = cbc_mode.CBCMode(aes_cipher_256, 16)
aes_cbc_256.set_iv(salt)
output = Data()
output.type = "AES256"
output.salt = bytes2base64(salt)
output.length = len(data)
encrypted = bytearray()
for _, d in _groupby16(data):
encrypted.extend(aes_cbc_256.encrypt_block(d))
output.data = bytes2base64(encrypted)
json = get_module("mo_json").value2json(output)
if DEBUG:
test = decrypt(json, _key)
if test != text:
Log.error("problem with encryption")
return json
def es_bulksetop(esq, frum, query):
abs_limit = MIN([query.limit, MAX_DOCUMENTS])
guid = Random.base64(32, extra="-_")
schema = query.frum.schema
query_path = schema.query_path[0]
new_select, split_select = get_selects(query)
split_wheres = split_expression_by_path(query.where, schema, lang=ES52)
es_query = es_query_proto(query_path, split_select, split_wheres, schema)
es_query.size = MIN([query.chunk_size, MAX_CHUNK_SIZE])
es_query.sort = jx_sort_to_es_sort(query.sort, schema)
if not es_query.sort:
es_query.sort = ["_doc"]
formatter = formatters[query.format](abs_limit, new_select, query)
Thread.run(
"Download " + guid,
extractor,
guid,
abs_limit,
esq,
es_query,
formatter,
parent_thread=Null,
).release()
output = wrap({
"url": URL_PREFIX / (guid + ".json"),
"status": URL_PREFIX / (guid + ".status.json"),
"meta": {
"format": query.format,
"es_query": es_query,
"limit": abs_limit
},
})
return output
def test_compare_isinstance_to_class_checks(self):
num = 1 * 1000 * 1000
options = {
0: lambda: {},
1: lambda: Data(),
2: lambda: Null,
3: lambda: 6,
4: lambda: "string",
}
data = [options[Random.int(len(options))]() for _ in range(num)]
with Timer("isinstance check") as i_time:
i_result = [isinstance(d, Mapping) for d in data]
with Timer("set check") as s_time:
s_result = [d.__class__ in MAPPING_TYPES for d in data]
with Timer("eq check") as e_time:
e_result = [
d.__class__ is Data or d.__class__ is dict for d in data
]
with Timer("name check") as n_time:
n_result = [
is_instance(d, Data) or is_instance(d, dict) for d in data
]
with Timer("check w method") as m_time:
m_result = [is_mapping(d) for d in data]
self.assertEqual(s_result, i_result)
self.assertEqual(m_result, i_result)
self.assertEqual(e_result, i_result)
self.assertEqual(n_result, i_result)
self.assertGreater(i_time.duration, s_time.duration)
self.assertGreater(m_time.duration, s_time.duration)
def commit(self):
with self.lock:
if self.closed:
Log.error("Queue is closed, commit not allowed")
try:
self._add_pending({"add": {"status.start": self.start}})
for i in range(self.db.status.start, self.start):
self._add_pending({"remove": str(i)})
if self.db.status.end - self.start < 10 or Random.range(0, 1000) == 0: # FORCE RE-WRITE TO LIMIT FILE SIZE
# SIMPLY RE-WRITE FILE
if DEBUG:
Log.note("Re-write {{num_keys}} keys to persistent queue", num_keys=self.db.status.end - self.start)
for k in self.db.keys():
if k == "status" or int(k) >= self.db.status.start:
continue
Log.error("Not expecting {{key}}", key=k)
self._commit()
self.file.write(mo_json.value2json({"add": self.db}) + "\n")
else:
self._commit()
except Exception as e:
raise e
def complex_job(
transactional_db, generic_reference_data, test_repository, extract_job_settings, now
):
fc = FailureClassification.objects.create(id=1, name="not classified")
repository_group = RepositoryGroup.objects.create(name="common")
repo = Repository.objects.create(name="autoland", repository_group=repository_group)
push = Push.objects.create(
**{
"author": "*****@*****.**",
"repository": repo,
"revision": "ae6bb3a1066959a8c43d003a3caab0af769455bf",
"time": unix2datetime(1578427105).replace(tzinfo=None),
}
)
Commit.objects.create(
push=push,
revision="ae6bb3a1066959a8c43d003a3caab0af769455bf",
author="*****@*****.**",
comments="no comment",
)
Commit.objects.create(
push=push,
revision="0123456789012345678901234567890123456789",
author="*****@*****.**",
comments="no comment2",
)
debug = Option.objects.create(name="debug")
oc = OptionCollection.objects.create(option_collection_hash=Random.base64(5), option=debug)
job = Job.objects.create(
autoclassify_status=1,
guid=Random.base64(20),
repository=test_repository,
push_id=push.id,
signature=generic_reference_data.signature,
build_platform=generic_reference_data.build_platform,
machine_platform=generic_reference_data.machine_platform,
machine=generic_reference_data.machine,
option_collection_hash=oc.option_collection_hash,
job_type=generic_reference_data.job_type,
job_group=generic_reference_data.job_group,
product=generic_reference_data.product,
failure_classification_id=fc.id,
who="*****@*****.**",
reason="scheduled",
result="success",
state="completed",
submit_time=unix2datetime(1578427253).replace(tzinfo=None),
start_time=unix2datetime(1578430841).replace(tzinfo=None),
last_modified=unix2datetime(1578432686.364459).replace(tzinfo=None),
end_time=unix2datetime(1578432680).replace(tzinfo=None),
tier=1,
)
text_log_step = TextLogStep.objects.create(
job=job,
**{
"finished_line_number": 88739,
"name": "Unnamed step",
"result": 7,
"started_line_number": 0,
},
)
TextLogError.objects.create(
step=text_log_step, line="line contents here", line_number=619845839
)
TextLogError.objects.create(step=text_log_step, line="ERROR! more line contents", line_number=6)
TaskclusterMetadata.objects.create(job=job, retry_id=0, task_id="WWb9ExAvQUa78ku0DIxdSQ")
JobLog.objects.create(
**{
"job_id": job.id,
"name": "builds-4h",
"status": 1,
"url": "https://example.com/api/queue/v1/task/WWb9ExAvQUa78ku0DIxdSQ/runs/0/artifacts/public/logs/live_backing.log",
}
)
job_logs1 = JobLog.objects.create(
**{
"job_id": job.id,
"name": "errorsummary_json",
"status": 1,
"url": "https://example.com/api/queue/v1/task/WWb9ExAvQUa78ku0DIxdSQ/runs/0/artifacts/public/test_info/wpt_errorsummary.log",
}
)
bcf = ClassifiedFailure.objects.create(**{"bug_number": 1234567,})
bcf.created = Date("2020-01-17 12:00:00").datetime
bcf.save()
FailureLine.objects.create(
job_log=job_logs1,
**{
"action": "test_groups",
"best_classification": bcf,
"best_is_verified": True,
"repository": repo,
"job_guid": job.guid,
"line": 15,
"modified": 0,
"stackwalk_stderr": 1578432686,
"stackwalk_stdout": 1578432686,
},
)
FailureLine.objects.create(
job_log=job_logs1,
**{
"action": "crash",
"best_classification": bcf,
"best_is_verified": False,
"repository": repo,
"job_guid": job.guid,
"line": 24031,
"modified": 0,
"signature": "@ mozilla::dom::CustomElementData::SetCustomElementDefinition(mozilla::dom::CustomElementDefinition*)",
"stackwalk_stderr": 1578432686,
"stackwalk_stdout": 1578432686,
"test": "/custom-elements/upgrading.html",
},
)
return job
def unique_name():
return Random.string(20)
def _scan_database(self):
# GET ALL RELATIONS
raw_relations = self.db.query(
"""
SELECT
table_schema,
table_name,
referenced_table_schema,
referenced_table_name,
referenced_column_name,
constraint_name,
column_name,
ordinal_position
FROM
information_schema.key_column_usage
WHERE
referenced_column_name IS NOT NULL
""",
param=self.settings.database,
)
if not raw_relations:
Log.error("No relations in the database")
for r in self.settings.add_relations:
try:
lhs, rhs = map(strings.trim, r.split("->"))
lhs = lhs.split(".")
if len(lhs) == 2:
lhs = [self.settings.database.schema] + lhs
rhs = rhs.split(".")
if len(rhs) == 2:
rhs = [self.settings.database.schema] + rhs
to_add = Data(
ordinal_position=1, # CAN ONLY HANDLE 1-COLUMN RELATIONS
table_schema=lhs[0],
table_name=lhs[1],
column_name=lhs[2],
referenced_table_schema=rhs[0],
referenced_table_name=rhs[1],
referenced_column_name=rhs[2],
)
# CHECK IF EXISTING
if jx.filter(raw_relations, {"eq": to_add}):
Log.note("Relation {{relation}} already exists",
relation=r)
continue
to_add.constraint_name = Random.hex(20)
raw_relations.append(to_add)
except Exception as e:
Log.error("Could not parse {{line|quote}}", line=r, cause=e)
relations = jx.select(
raw_relations,
[
{
"name": "constraint.name",
"value": "constraint_name"
},
{
"name": "table.schema",
"value": "table_schema"
},
{
"name": "table.name",
"value": "table_name"
},
{
"name": "column.name",
"value": "column_name"
},
{
"name": "referenced.table.schema",
"value": "referenced_table_schema"
},
{
"name": "referenced.table.name",
"value": "referenced_table_name"
},
{
"name": "referenced.column.name",
"value": "referenced_column_name"
},
{
"name": "ordinal_position",
"value": "ordinal_position"
},
],
)
# GET ALL TABLES
raw_tables = self.db.query("""
SELECT
t.table_schema,
t.table_name,
c.constraint_name,
c.constraint_type,
k.column_name,
k.ordinal_position
FROM
information_schema.tables t
LEFT JOIN
information_schema.table_constraints c on c.table_name=t.table_name AND c.table_schema=t.table_schema and (constraint_type='UNIQUE' or constraint_type='PRIMARY KEY')
LEFT JOIN
information_schema.key_column_usage k on k.constraint_name=c.constraint_name AND k.table_name=t.table_name and k.table_schema=t.table_schema
ORDER BY
t.table_schema,
t.table_name,
c.constraint_name,
k.ordinal_position,
k.column_name
""")
# ORGANIZE, AND PICK ONE UNIQUE CONSTRAINT FOR LINKING
tables = UniqueIndex(keys=["name", "schema"])
for t, c in jx.groupby(raw_tables, ["table_name", "table_schema"]):
c = wrap(list(c))
best_index = Null
is_referenced = False
is_primary = False
for g, w in jx.groupby(c, "constraint_name"):
if not g.constraint_name:
continue
w = list(w)
ref = False
for r in relations:
if (r.table.name == t.table_name
and r.table.schema == t.table_schema
and r.constraint.name == g.constraint_name):
ref = True
is_prime = w[0].constraint_type == "PRIMARY"
reasons_this_one_is_better = [
best_index == None, # WE DO NOT HAVE A CANDIDATE YET
is_prime
and not is_primary, # PRIMARY KEYS ARE GOOD TO HAVE
is_primary == is_prime and ref and
not is_referenced, # REFERENCED UNIQUE TUPLES ARE GOOD TOO
is_primary == is_prime and ref == is_referenced and len(w)
< len(best_index), # THE SHORTER THE TUPLE, THE BETTER
]
if any(reasons_this_one_is_better):
is_primary = is_prime
is_referenced = ref
best_index = w
tables.add({
"name": t.table_name,
"schema": t.table_schema,
"id": [b.column_name for b in best_index],
})
fact_table = tables[self.settings.fact_table,
self.settings.database.schema]
ids_table = {
"alias": "t0",
"name": "__ids__",
"schema": fact_table.schema,
"id": fact_table.id,
}
relations.extend(
wrap({
"constraint": {
"name": "__link_ids_to_fact_table__"
},
"table": ids_table,
"column": {
"name": c
},
"referenced": {
"table": fact_table,
"column": {
"name": c
}
},
"ordinal_position": i,
}) for i, c in enumerate(fact_table.id))
tables.add(ids_table)
# GET ALL COLUMNS
raw_columns = self.db.query("""
SELECT
column_name,
table_schema,
table_name,
ordinal_position,
data_type
FROM
information_schema.columns
""")
reference_only_tables = [
r.split(".")[0] for r in self.settings.reference_only
if len(r.split(".")) == 2
]
reference_all_tables = [
r.split(".")[0] for r in self.settings.reference_only
if len(r.split(".")) == 1
]
foreign_column_table_schema_triples = {(r.column.name, r.table.name,
r.table.schema)
for r in relations}
referenced_column_table_schema_triples = {(
r.referenced.column.name,
r.referenced.table.name,
r.referenced.table.schema,
)
for r in relations}
related_column_table_schema_triples = (
foreign_column_table_schema_triples
| referenced_column_table_schema_triples)
columns = UniqueIndex(["column.name", "table.name", "table.schema"])
for c in raw_columns:
if c.table_name in reference_only_tables:
if c.table_name + "." + c.column_name in self.settings.reference_only:
include = True
reference = True
foreign = False
elif c.column_name in tables[(c.table_name,
c.table_schema)].id:
include = self.settings.show_foreign_keys
reference = False
foreign = False
else:
include = False
reference = False
foreign = False
elif c.table_name in reference_all_tables:
# TABLES USED FOR REFERENCE, NO NESTED DOCUMENTS EXPECTED
if c.column_name in tables[(c.table_name, c.table_schema)].id:
include = self.settings.show_foreign_keys
reference = True
foreign = False
elif (
c.column_name,
c.table_name,
c.table_schema,
) in foreign_column_table_schema_triples:
include = False
reference = False
foreign = True
else:
include = True
reference = False
foreign = False
elif c.column_name in tables[(c.table_name, c.table_schema)].id:
include = self.settings.show_foreign_keys
reference = False
foreign = False
elif (
c.column_name,
c.table_name,
c.table_schema,
) in foreign_column_table_schema_triples:
include = False
reference = False
foreign = True
elif (
c.column_name,
c.table_name,
c.table_schema,
) in referenced_column_table_schema_triples:
include = self.settings.show_foreign_keys
reference = False
foreign = False
else:
include = True
reference = False
foreign = False
rel = {
"column": {
"name": c.column_name,
"type": c.data_type
},
"table": {
"name": c.table_name,
"schema": c.table_schema
},
"ordinal_position": c.ordinal_position,
"is_id": c.column_name
in tables[(c.table_name, c.table_schema)].id,
"include": include, # TRUE IF THIS COLUMN IS OUTPUTTED
"reference":
reference, # TRUE IF THIS COLUMN REPRESENTS THE ROW
"foreign":
foreign, # TRUE IF THIS COLUMN POINTS TO ANOTHER ROW
}
columns.add(rel)
# ITERATE OVER ALL PATHS
todo = FlatList()
output_columns = FlatList()
nested_path_to_join = {}
all_nested_paths = [["."]]
def follow_paths(position, path, nested_path, done_relations,
no_nested_docs):
if position.name in self.settings.exclude:
return
if self.path_not_allowed(path):
return
if DEBUG:
Log.note("Trace {{path}}", path=path)
if position.name != "__ids__":
# USED TO CONFIRM WE CAN ACCESS THE TABLE (WILL THROW ERROR WHEN IF IT FAILS)
self.db.query(
ConcatSQL(
SQL_SELECT,
SQL_STAR,
SQL_FROM,
quote_column(position.schema, position.name),
SQL_LIMIT,
SQL_ONE,
))
if position.name in reference_all_tables:
no_nested_docs = True
if position.name in reference_only_tables:
return
curr_join_list = copy(nested_path_to_join[nested_path[0]])
###############################################################################
# INNER OBJECTS
###############################################################################
referenced_tables = list(
sort_using_key(
jx.groupby(
jx.filter(
relations,
{
"eq": {
"table.name": position.name,
"table.schema": position.schema,
}
},
),
"constraint.name",
),
key=lambda p: first(p[1]).column.name,
))
for g, constraint_columns in referenced_tables:
g = unwrap(g)
constraint_columns = deepcopy(constraint_columns)
if g["constraint.name"] in done_relations:
continue
if any(cc for cc in constraint_columns
if cc.referenced.table.name in self.settings.exclude):
continue
done_relations.add(g["constraint.name"])
many_to_one_joins = nested_path_to_join[nested_path[0]]
index = len(many_to_one_joins)
alias = "t" + text(index)
for c in constraint_columns:
c.referenced.table.alias = alias
c.table = position
many_to_one_joins.append({
"join_columns": constraint_columns,
"path": path,
"nested_path": nested_path,
})
# HANDLE THE COMMON *id SUFFIX
name = []
for cname, tname in zip(
constraint_columns.column.name,
constraint_columns.referenced.table.name,
):
if cname.startswith(tname):
name.append(tname)
elif cname.endswith("_id"):
name.append(cname[:-3])
else:
name.append(cname)
relation_string = many_to_one_string(constraint_columns[0])
step = "/".join(name)
if len(constraint_columns) == 1:
step = self.name_relations.get(relation_string, step)
referenced_column_path = concat_field(path, step)
if self.path_not_allowed(referenced_column_path):
continue
if referenced_column_path in reference_only_tables:
continue
col_pointer_name = relative_field(referenced_column_path,
nested_path[0])
for col in columns:
if (col.table.name
== constraint_columns[0].referenced.table.name
and col.table.schema
== constraint_columns[0].referenced.table.schema):
col_full_name = concat_field(
col_pointer_name, literal_field(col.column.name))
if (col.is_id and len(nested_path) == 1
and col.table.name == fact_table.name
and col.table.schema == fact_table.schema):
# ALWAYS SHOW THE ID OF THE FACT
c_index = len(output_columns)
output_columns.append({
"table_alias":
alias,
"column_alias":
"c" + text(c_index),
"column":
col,
"sort":
True,
"path":
referenced_column_path,
"nested_path":
nested_path,
"put":
col_full_name,
})
elif col.column.name == constraint_columns[
0].column.name:
c_index = len(output_columns)
output_columns.append({
"table_alias":
alias,
"column_alias":
"c" + text(c_index),
"column":
col,
"sort":
False,
"path":
referenced_column_path,
"nested_path":
nested_path,
"put":
col_full_name
if self.settings.show_foreign_keys else None,
})
elif col.is_id:
c_index = len(output_columns)
output_columns.append({
"table_alias":
alias,
"column_alias":
"c" + text(c_index),
"column":
col,
"sort":
False,
"path":
referenced_column_path,
"nested_path":
nested_path,
"put":
col_full_name
if self.settings.show_foreign_keys else None,
})
elif col.reference:
c_index = len(output_columns)
output_columns.append({
"table_alias":
alias,
"column_alias":
"c" + text(c_index),
"column":
col,
"sort":
False,
"path":
referenced_column_path,
"nested_path":
nested_path,
"put":
col_pointer_name
if not self.settings.show_foreign_keys else
col_full_name, # REFERENCE FIELDS CAN REPLACE THE WHOLE OBJECT BEING REFERENCED
})
elif col.include:
c_index = len(output_columns)
output_columns.append({
"table_alias":
alias,
"column_alias":
"c" + text(c_index),
"column":
col,
"sort":
False,
"path":
referenced_column_path,
"nested_path":
nested_path,
"put":
col_full_name,
})
if position.name in reference_only_tables:
continue
todo.append(
Data(
position=copy(constraint_columns[0].referenced.table),
path=referenced_column_path,
nested_path=nested_path,
done_relations=copy(done_relations),
no_nested_docs=no_nested_docs,
))
###############################################################################
# NESTED OBJECTS
###############################################################################
if not no_nested_docs:
nesting_tables = list(
sort_using_key(
jx.groupby(
jx.filter(
relations,
{
"eq": {
"referenced.table.name": position.name,
"referenced.table.schema":
position.schema,
}
},
),
"constraint.name",
),
key=lambda p: [(r.table.name, r.column.name)
for r in [first(p[1])]][0],
))
for g, constraint_columns in nesting_tables:
g = unwrap(g)
constraint_columns = deepcopy(constraint_columns)
if g["constraint.name"] in done_relations:
continue
done_relations.add(g["constraint.name"])
many_table = set(constraint_columns.table.name)
if not (many_table - self.settings.exclude):
continue
relation_string = one_to_many_string(constraint_columns[0])
step = "/".join(many_table)
if len(constraint_columns) == 1:
step = self.name_relations.get(relation_string, step)
referenced_column_path = concat_field(path, step)
if self.path_not_allowed(referenced_column_path):
continue
new_nested_path = [referenced_column_path] + nested_path
all_nested_paths.append(new_nested_path)
if referenced_column_path in nested_path_to_join:
Log.error(
"{{path}} already exists, try adding entry to name_relations",
path=referenced_column_path,
)
one_to_many_joins = nested_path_to_join[
referenced_column_path] = copy(curr_join_list)
index = len(one_to_many_joins)
alias = "t" + text(index)
for c in constraint_columns:
c.table.alias = alias
c.referenced.table = position
one_to_many_joins.append(
set_default(
{},
g,
{
"children": True,
"join_columns": constraint_columns,
"path": path,
"nested_path": nested_path,
},
))
for col in columns:
if (col.table.name == constraint_columns[0].table.name
and col.table.schema
== constraint_columns[0].table.schema):
col_full_name = join_field(
split_field(referenced_column_path)
[len(split_field(new_nested_path[0])):] +
[literal_field(col.column.name)])
if col.column.name == constraint_columns[
0].column.name:
c_index = len(output_columns)
output_columns.append({
"table_alias":
alias,
"column_alias":
"c" + text(c_index),
"column":
col,
"sort":
col.is_id,
"path":
referenced_column_path,
"nested_path":
new_nested_path,
"put":
col_full_name if
self.settings.show_foreign_keys else None,
})
elif col.is_id:
c_index = len(output_columns)
output_columns.append({
"table_alias":
alias,
"column_alias":
"c" + text(c_index),
"column":
col,
"sort":
col.is_id,
"path":
referenced_column_path,
"nested_path":
new_nested_path,
"put":
col_full_name if
self.settings.show_foreign_keys else None,
})
else:
c_index = len(output_columns)
output_columns.append({
"table_alias":
alias,
"column_alias":
"c" + text(c_index),
"column":
col,
"sort":
col.is_id,
"path":
referenced_column_path,
"nested_path":
new_nested_path,
"put":
col_full_name if col.include else None,
})
todo.append(
Data(
position=constraint_columns[0].table,
path=referenced_column_path,
nested_path=new_nested_path,
done_relations=copy(done_relations),
no_nested_docs=no_nested_docs,
))
path = "."
nested_path = [path]
nested_path_to_join["."] = [{
"path":
path,
"join_columns": [{
"referenced": {
"table": ids_table
}
}],
"nested_path":
nested_path,
}]
todo.append(
Data(
position=ids_table,
path=path,
nested_path=nested_path,
done_relations=set(),
no_nested_docs=False,
))
while todo:
item = todo.pop(0)
follow_paths(**item)
self.all_nested_paths = all_nested_paths
self.nested_path_to_join = nested_path_to_join
self.columns = output_columns
def _get_from_elasticsearch(
self,
revision,
locale=None,
get_diff=False,
get_moves=True,
after=None, # RETURN RECORDS ETLed AFTER GIVEN TIME
):
"""
MAKE CALL TO ES
"""
rev = revision.changeset.id
if self.repo.cluster.version.startswith("1.7."):
query = {
"query": {
"filtered": {
"query": {
"match_all": {}
},
"filter": {
"and": [
{
"term": {
"changeset.id12": rev[0:12]
}
},
{
"term": {
"branch.name": revision.branch.name
}
},
{
"term": {
"branch.locale":
coalesce(
locale,
revision.branch.locale,
DEFAULT_LOCALE,
)
}
},
{
"range": {
"etl.timestamp": {
"gt": Date.max(after, MIN_ETL_AGE)
}
}
},
]
},
}
},
"size": 20,
}
else:
query = {
"query": {
"bool": {
"must": [
{
"term": {
"changeset.id12": rev[0:12]
}
},
{
"term": {
"branch.name": revision.branch.name
}
},
{
"term": {
"branch.locale":
coalesce(locale, revision.branch.locale,
DEFAULT_LOCALE)
}
},
{
"range": {
"etl.timestamp": {
"gt": Date.max(after, MIN_ETL_AGE)
}
}
},
]
}
},
"size": 20,
}
for attempt in range(3):
try:
if get_moves:
with self.moves_locker:
docs = self.moves.search(query).hits.hits
else:
with self.repo_locker:
docs = self.repo.search(query).hits.hits
if len(docs) == 0:
return None
best = docs[0]._source
if len(docs) > 1:
for d in docs:
if d._id.endswith(d._source.branch.locale):
best = d._source
Log.warning("expecting no more than one document")
return best
except Exception as e:
e = Except.wrap(e)
if ("EsRejectedExecutionException[rejected execution (queue capacity"
in e):
(Till(seconds=Random.int(30))).wait()
continue
else:
Log.warning(
"Bad ES call, waiting for {{num}} seconds",
num=WAIT_AFTER_NODE_FAILURE,
cause=e,
)
Till(seconds=WAIT_AFTER_NODE_FAILURE).wait()
continue
Log.warning("ES did not deliver, fall back to HG")
return None
def es_bulkaggsop(esq, frum, query):
query = query.copy() # WE WILL MARK UP THIS QUERY
chunk_size = min(coalesce(query.chunk_size, MAX_CHUNK_SIZE), MAX_CHUNK_SIZE)
schema = frum.schema
query_path = first(schema.query_path)
selects = listwrap(query.select)
variable = first(query.groupby).value
# FIND CARDINALITY
cardinality_check = Timer(
"Get cardinality for {{column}}", param={"column": variable.var}
)
with cardinality_check:
columns = schema.leaves(variable.var)
if len(columns) != 1:
Log.error(
"too many columns to bulk groupby:\n{{columns|json}}", columns=columns
)
column = first(columns)
if query.where is TRUE:
cardinality = column.cardinality
if cardinality == None:
esq.namespace._update_cardinality(column)
cardinality = column.cardinality
else:
cardinality = esq.query(
{
"select": {
"name": "card",
"value": variable,
"aggregate": "cardinality",
},
"from": frum.name,
"where": query.where,
"format": "cube",
}
).card
num_partitions = (cardinality + chunk_size - 1) // chunk_size
if num_partitions > MAX_PARTITIONS:
Log.error("Requesting more than {{num}} partitions", num=num_partitions)
acc, decoders, es_query = build_es_query(selects, query_path, schema, query)
guid = Random.base64(32, extra="-_")
abs_limit = mo_math.MIN((query.limit, first(query.groupby).domain.limit))
formatter = formatters[query.format](abs_limit)
Thread.run(
"extract to " + guid + ".json",
extractor,
guid,
num_partitions,
esq,
query,
selects,
query_path,
schema,
chunk_size,
cardinality,
abs_limit,
formatter,
parent_thread=Null,
)
output = wrap(
{
"url": URL_PREFIX / (guid + ".json"),
"status": URL_PREFIX / (guid + ".status.json"),
"meta": {
"format": query.format,
"timing": {"cardinality_check": cardinality_check.duration},
"es_query": es_query,
"num_partitions": num_partitions,
"cardinality": cardinality,
},
}
)
return output
def get_revision(self, revision, locale=None, get_diff=False, get_moves=True):
"""
EXPECTING INCOMPLETE revision OBJECT
RETURNS revision
"""
rev = revision.changeset.id
if not rev:
return Null
elif rev == "None":
return Null
elif revision.branch.name == None:
return Null
locale = coalesce(locale, revision.branch.locale, DEFAULT_LOCALE)
output = self._get_from_elasticsearch(revision, locale=locale, get_diff=get_diff)
if output:
if not get_diff: # DIFF IS BIG, DO NOT KEEP IT IF NOT NEEDED
output.changeset.diff = None
if not get_moves:
output.changeset.moves = None
DEBUG and Log.note("Got hg ({{branch}}, {{locale}}, {{revision}}) from ES", branch=output.branch.name, locale=locale, revision=output.changeset.id)
if output.push.date >= Date.now()-MAX_TODO_AGE:
self.todo.add((output.branch, listwrap(output.parents)))
self.todo.add((output.branch, listwrap(output.children)))
if output.push.date:
return output
# RATE LIMIT CALLS TO HG (CACHE MISSES)
next_cache_miss = self.last_cache_miss + (Random.float(WAIT_AFTER_CACHE_MISS * 2) * SECOND)
self.last_cache_miss = Date.now()
if next_cache_miss > self.last_cache_miss:
Log.note("delaying next hg call for {{seconds|round(decimal=1)}}", seconds=next_cache_miss - self.last_cache_miss)
Till(till=next_cache_miss.unix).wait()
found_revision = copy(revision)
if isinstance(found_revision.branch, (text_type, binary_type)):
lower_name = found_revision.branch.lower()
else:
lower_name = found_revision.branch.name.lower()
if not lower_name:
Log.error("Defective revision? {{rev|json}}", rev=found_revision.branch)
b = found_revision.branch = self.branches[(lower_name, locale)]
if not b:
b = found_revision.branch = self.branches[(lower_name, DEFAULT_LOCALE)]
if not b:
Log.warning("can not find branch ({{branch}}, {{locale}})", branch=lower_name, locale=locale)
return Null
if Date.now() - Date(b.etl.timestamp) > _OLD_BRANCH:
self.branches = _hg_branches.get_branches(kwargs=self.settings)
push = self._get_push(found_revision.branch, found_revision.changeset.id)
url1 = found_revision.branch.url.rstrip("/") + "/json-info?node=" + found_revision.changeset.id[0:12]
url2 = found_revision.branch.url.rstrip("/") + "/json-rev/" + found_revision.changeset.id[0:12]
with Explanation("get revision from {{url}}", url=url1, debug=DEBUG):
raw_rev2 = Null
try:
raw_rev1 = self._get_raw_json_info(url1, found_revision.branch)
raw_rev2 = self._get_raw_json_rev(url2, found_revision.branch)
except Exception as e:
if "Hg denies it exists" in e:
raw_rev1 = Data(node=revision.changeset.id)
else:
raise e
output = self._normalize_revision(set_default(raw_rev1, raw_rev2), found_revision, push, get_diff, get_moves)
if output.push.date >= Date.now()-MAX_TODO_AGE:
self.todo.add((output.branch, listwrap(output.parents)))
self.todo.add((output.branch, listwrap(output.children)))
if not get_diff: # DIFF IS BIG, DO NOT KEEP IT IF NOT NEEDED
output.changeset.diff = None
if not get_moves:
output.changeset.moves = None
return output
def _install_es(self, gigabytes):
volumes = self.instance.markup.drives
if not fabric_files.exists("/usr/local/elasticsearch"):
with cd("/home/ec2-user/"):
run("mkdir -p temp")
if not File(LOCAL_JRE).exists:
Log.error(
"Expecting {{file}} on manager to spread to ES instances",
file=LOCAL_JRE)
with cd("/home/ec2-user/temp"):
run('rm -f ' + JRE)
put("resources/" + JRE, JRE)
sudo("rpm -i " + JRE)
sudo(
"alternatives --install /usr/bin/java java /usr/java/default/bin/java 20000"
)
run("export JAVA_HOME=/usr/java/default")
with cd("/home/ec2-user/"):
run('wget https://download.elasticsearch.org/elasticsearch/elasticsearch/elasticsearch-1.7.1.tar.gz'
)
run('tar zxfv elasticsearch-1.7.1.tar.gz')
sudo('mkdir /usr/local/elasticsearch')
sudo('cp -R elasticsearch-1.7.1/* /usr/local/elasticsearch/')
with cd('/usr/local/elasticsearch/'):
# BE SURE TO MATCH THE PLUGLIN WITH ES VERSION
# https://github.com/elasticsearch/elasticsearch-cloud-aws
sudo(
'bin/plugin -install elasticsearch/elasticsearch-cloud-aws/2.7.1'
)
#REMOVE THESE FILES, WE WILL REPLACE THEM WITH THE CORRECT VERSIONS AT THE END
sudo("rm -f /usr/local/elasticsearch/config/elasticsearch.yml")
sudo("rm -f /usr/local/elasticsearch/bin/elasticsearch.in.sh")
self.conn = self.instance.connection
# MOUNT AND FORMAT THE EBS VOLUMES (list with `lsblk`)
for i, k in enumerate(volumes):
if not fabric_files.exists(k.path):
with fabric_settings(warn_only=True):
sudo('sudo umount ' + k.device)
sudo('yes | sudo mkfs -t ext4 ' + k.device)
sudo('mkdir ' + k.path)
sudo('sudo mount ' + k.device + ' ' + k.path)
#ADD TO /etc/fstab SO AROUND AFTER REBOOT
sudo("sed -i '$ a\\" + k.device + " " + k.path +
" ext4 defaults,nofail 0 2' /etc/fstab")
# TEST IT IS WORKING
sudo('mount -a')
# INCREASE THE FILE HANDLE LIMITS
with cd("/home/ec2-user/"):
File("./results/temp/sysctl.conf").delete()
get("/etc/sysctl.conf",
"./results/temp/sysctl.conf",
use_sudo=True)
lines = File("./results/temp/sysctl.conf").read()
if lines.find("fs.file-max = 100000") == -1:
lines += "\nfs.file-max = 100000"
lines = lines.replace("net.bridge.bridge-nf-call-ip6tables = 0",
"")
lines = lines.replace("net.bridge.bridge-nf-call-iptables = 0", "")
lines = lines.replace("net.bridge.bridge-nf-call-arptables = 0",
"")
File("./results/temp/sysctl.conf").write(lines)
put("./results/temp/sysctl.conf",
"/etc/sysctl.conf",
use_sudo=True)
sudo("sysctl -p")
# INCREASE FILE HANDLE PERMISSIONS
sudo("sed -i '$ a\\root soft nofile 50000' /etc/security/limits.conf")
sudo("sed -i '$ a\\root hard nofile 100000' /etc/security/limits.conf")
sudo("sed -i '$ a\\root memlock unlimited' /etc/security/limits.conf")
sudo(
"sed -i '$ a\\ec2-user soft nofile 50000' /etc/security/limits.conf"
)
sudo(
"sed -i '$ a\\ec2-user hard nofile 100000' /etc/security/limits.conf"
)
sudo(
"sed -i '$ a\\ec2-user memlock unlimited' /etc/security/limits.conf"
)
# EFFECTIVE LOGIN TO LOAD CHANGES TO FILE HANDLES
# sudo("sudo -i -u ec2-user")
if not fabric_files.exists("/data1/logs"):
sudo('mkdir /data1/logs')
sudo('mkdir /data1/heapdump')
#INCREASE NUMBER OF FILE HANDLES
# sudo("sysctl -w fs.file-max=64000")
# COPY CONFIG FILE TO ES DIR
if not fabric_files.exists(
"/usr/local/elasticsearch/config/elasticsearch.yml"):
yml = File("./examples/config/es_config.yml").read().replace(
"\r", "")
yml = expand_template(
yml, {
"id":
Random.hex(length=8),
"data_paths":
",".join("/data" + unicode(i + 1)
for i, _ in enumerate(volumes))
})
File("./results/temp/elasticsearch.yml").write(yml)
put("./results/temp/elasticsearch.yml",
'/usr/local/elasticsearch/config/elasticsearch.yml',
use_sudo=True)
# FOR SOME REASON THE export COMMAND DOES NOT SEEM TO WORK
# THIS SCRIPT SETS THE ES_MIN_MEM/ES_MAX_MEM EXPLICITLY
if not fabric_files.exists(
"/usr/local/elasticsearch/bin/elasticsearch.in.sh"):
sh = File("./examples/config/es_run.sh").read().replace("\r", "")
sh = expand_template(sh, {"memory": unicode(int(gigabytes / 2))})
File("./results/temp/elasticsearch.in.sh").write(sh)
with cd("/home/ec2-user"):
put("./results/temp/elasticsearch.in.sh",
'./temp/elasticsearch.in.sh',
use_sudo=True)
sudo(
"cp -f ./temp/elasticsearch.in.sh /usr/local/elasticsearch/bin/elasticsearch.in.sh"
)
def _scan_database(self):
# GET ALL RELATIONS
raw_relations = self.db.query("""
SELECT
table_schema,
table_name,
referenced_table_schema,
referenced_table_name,
referenced_column_name,
constraint_name,
column_name,
ordinal_position
FROM
information_schema.key_column_usage
WHERE
referenced_column_name IS NOT NULL
""",
param=self.settings.database)
if not raw_relations:
Log.error("No relations in the database")
for r in self.settings.add_relations:
try:
a, b = map(strings.trim, r.split("->"))
a = a.split(".")
b = b.split(".")
raw_relations.append(
Data(table_schema=a[0],
table_name=a[1],
referenced_table_schema=b[0],
referenced_table_name=b[1],
referenced_column_name=b[2],
constraint_name=Random.hex(20),
column_name=a[2],
ordinal_position=1))
except Exception as e:
Log.error("Could not parse {{line|quote}}", line=r, cause=e)
relations = jx.select(raw_relations,
[{
"name": "constraint.name",
"value": "constraint_name"
}, {
"name": "table.schema",
"value": "table_schema"
}, {
"name": "table.name",
"value": "table_name"
}, {
"name": "column.name",
"value": "column_name"
}, {
"name": "referenced.table.schema",
"value": "referenced_table_schema"
}, {
"name": "referenced.table.name",
"value": "referenced_table_name"
}, {
"name": "referenced.column.name",
"value": "referenced_column_name"
}, {
"name": "ordinal_position",
"value": "ordinal_position"
}])
# GET ALL TABLES
raw_tables = self.db.query("""
SELECT
t.table_schema,
t.table_name,
c.constraint_name,
c.constraint_type,
k.column_name,
k.ordinal_position
FROM
information_schema.tables t
LEFT JOIN
information_schema.table_constraints c on c.table_name=t.table_name AND c.table_schema=t.table_schema and (constraint_type='UNIQUE' or constraint_type='PRIMARY KEY')
LEFT JOIN
information_schema.key_column_usage k on k.constraint_name=c.constraint_name AND k.table_name=t.table_name and k.table_schema=t.table_schema
ORDER BY
t.table_schema,
t.table_name,
c.constraint_name,
k.ordinal_position,
k.column_name
""",
param=self.settings.database)
# ORGANIZE, AND PICK ONE UNIQUE CONSTRAINT FOR LINKING
tables = UniqueIndex(keys=["name", "schema"])
for t, c in jx.groupby(raw_tables, ["table_name", "table_schema"]):
c = wrap(list(c))
best_index = Null
is_referenced = False
is_primary = False
for g, w in jx.groupby(c, "constraint_name"):
if not g.constraint_name:
continue
w = list(w)
ref = False
for r in relations:
if r.table.name == t.table_name and r.table.schema == t.table_schema and r.constraint.name == g.constraint_name:
ref = True
is_prime = w[0].constraint_type == "PRIMARY"
reasons_this_one_is_better = [
best_index == None, # WE DO NOT HAVE A CANDIDATE YET
is_prime
and not is_primary, # PRIMARY KEYS ARE GOOD TO HAVE
is_primary == is_prime and ref and
not is_referenced, # REFERENCED UNIQUE TUPLES ARE GOOD TOO
is_primary == is_prime and ref == is_referenced and len(w)
< len(best_index) # THE SHORTER THE TUPLE, THE BETTER
]
if any(reasons_this_one_is_better):
is_primary = is_prime
is_referenced = ref
best_index = w
tables.add({
"name": t.table_name,
"schema": t.table_schema,
"id": [b.column_name for b in best_index]
})
fact_table = tables[self.settings.fact_table,
self.settings.database.schema]
ids_table = {
"alias": "t0",
"name": "__ids__",
"schema": fact_table.schema,
"id": fact_table.id
}
relations.extend(
wrap({
"constraint": {
"name": "__link_ids_to_fact_table__"
},
"table": ids_table,
"column": {
"name": c
},
"referenced": {
"table": fact_table,
"column": {
"name": c
}
},
"ordinal_position": i
}) for i, c in enumerate(fact_table.id))
tables.add(ids_table)
# GET ALL COLUMNS
raw_columns = self.db.query("""
SELECT
column_name,
table_schema,
table_name,
ordinal_position,
data_type
FROM
information_schema.columns
""",
param=self.settings.database)
reference_only_tables = [
r.split(".")[0] for r in self.settings.reference_only
if len(r.split(".")) == 2
]
reference_all_tables = [
r.split(".")[0] for r in self.settings.reference_only
if len(r.split(".")) == 1
]
foreign_column_table_schema_triples = {(r.column.name, r.table.name,
r.table.schema)
for r in relations}
referenced_column_table_schema_triples = {
(r.referenced.column.name, r.referenced.table.name,
r.referenced.table.schema)
for r in relations
}
related_column_table_schema_triples = foreign_column_table_schema_triples | referenced_column_table_schema_triples
columns = UniqueIndex(["column.name", "table.name", "table.schema"])
for c in raw_columns:
if c.table_name in reference_only_tables:
if c.table_name + "." + c.column_name in self.settings.reference_only:
include = True
reference = True
foreign = False
elif c.column_name in tables[(c.table_name,
c.table_schema)].id:
include = self.settings.show_foreign_keys
reference = False
foreign = False
else:
include = False
reference = False
foreign = False
elif c.table_name in reference_all_tables:
# TABLES USED FOR REFERENCE, NO NESTED DOCUMENTS EXPECTED
if c.column_name in tables[(c.table_name, c.table_schema)].id:
include = self.settings.show_foreign_keys
reference = True
foreign = False
elif (c.column_name, c.table_name,
c.table_schema) in foreign_column_table_schema_triples:
include = False
reference = False
foreign = True
else:
include = True
reference = False
foreign = False
elif c.column_name in tables[(c.table_name, c.table_schema)].id:
include = self.settings.show_foreign_keys
reference = False
foreign = False
elif (c.column_name, c.table_name,
c.table_schema) in foreign_column_table_schema_triples:
include = False
reference = False
foreign = True
elif (c.column_name, c.table_name,
c.table_schema) in referenced_column_table_schema_triples:
include = self.settings.show_foreign_keys
reference = False
foreign = False
else:
include = True
reference = False
foreign = False
rel = {
"column": {
"name": c.column_name,
"type": c.data_type
},
"table": {
"name": c.table_name,
"schema": c.table_schema
},
"ordinal_position": c.ordinal_position,
"is_id": c.column_name
in tables[(c.table_name, c.table_schema)].id,
"include": include, # TRUE IF THIS COLUMN IS OUTPUTTED
"reference":
reference, # TRUE IF THIS COLUMN REPRESENTS THE ROW
"foreign": foreign # TRUE IF THIS COLUMN POINTS TO ANOTHER ROW
}
columns.add(rel)
# ITERATE OVER ALL PATHS
todo = FlatList()
output_columns = FlatList()
nested_path_to_join = {}
all_nested_paths = [["."]]
def follow_paths(position, path, nested_path, done_relations,
no_nested_docs):
if position.name in self.settings.exclude:
return
if DEBUG:
Log.note("Trace {{path}}", path=path)
if position.name != "__ids__":
# USED TO CONFIRM WE CAN ACCESS THE TABLE (WILL THROW ERROR WHEN IF IT FAILS)
self.db.query("SELECT * FROM " +
quote_column(position.name, position.schema) +
" LIMIT 1")
if position.name in reference_all_tables:
no_nested_docs = True
if position.name in reference_only_tables:
return
curr_join_list = copy(nested_path_to_join[nested_path[0]])
# INNER OBJECTS
referenced_tables = list(
jx.groupby(
jx.filter(
relations, {
"eq": {
"table.name": position.name,
"table.schema": position.schema
}
}), "constraint.name"))
for g, constraint_columns in referenced_tables:
g = unwrap(g)
constraint_columns = deepcopy(constraint_columns)
if g["constraint.name"] in done_relations:
continue
if any(cc for cc in constraint_columns
if cc.referenced.table.name in self.settings.exclude):
continue
done_relations.add(g["constraint.name"])
many_to_one_joins = nested_path_to_join[nested_path[0]]
index = len(many_to_one_joins)
alias = "t" + text_type(index)
for c in constraint_columns:
c.referenced.table.alias = alias
c.table = position
many_to_one_joins.append({
"join_columns": constraint_columns,
"path": path,
"nested_path": nested_path
})
# referenced_table_path = join_field(split_field(path) + ["/".join(constraint_columns.referenced.table.name)])
# HANDLE THE COMMON *id SUFFIX
name = []
for a, b in zip(constraint_columns.column.name,
constraint_columns.referenced.table.name):
if a.startswith(b):
name.append(b)
elif a.endswith("_id"):
name.append(a[:-3])
else:
name.append(a)
referenced_column_path = join_field(
split_field(path) + ["/".join(name)])
col_pointer_name = relative_field(referenced_column_path,
nested_path[0])
# insert into nested1 VALUES (100, 10, 'aaa', -1);
# id.about.time.nested1 .ref=10
# id.about.time.nested1 .ref.name
for col in columns:
if col.table.name == constraint_columns[
0].referenced.table.name and col.table.schema == constraint_columns[
0].referenced.table.schema:
col_full_name = concat_field(
col_pointer_name, literal_field(col.column.name))
if col.is_id and col.table.name == fact_table.name and col.table.schema == fact_table.schema:
# ALWAYS SHOW THE ID OF THE FACT
c_index = len(output_columns)
output_columns.append({
"table_alias":
alias,
"column_alias":
"c" + text_type(c_index),
"column":
col,
"sort":
True,
"path":
referenced_column_path,
"nested_path":
nested_path,
"put":
col_full_name
})
elif col.column.name == constraint_columns[
0].column.name:
c_index = len(output_columns)
output_columns.append({
"table_alias":
alias,
"column_alias":
"c" + text_type(c_index),
"column":
col,
"sort":
False,
"path":
referenced_column_path,
"nested_path":
nested_path,
"put":
col_full_name
if self.settings.show_foreign_keys else None
})
elif col.is_id:
c_index = len(output_columns)
output_columns.append({
"table_alias":
alias,
"column_alias":
"c" + text_type(c_index),
"column":
col,
"sort":
False,
"path":
referenced_column_path,
"nested_path":
nested_path,
"put":
col_full_name
if self.settings.show_foreign_keys else None
})
elif col.reference:
c_index = len(output_columns)
output_columns.append({
"table_alias":
alias,
"column_alias":
"c" + text_type(c_index),
"column":
col,
"sort":
False,
"path":
referenced_column_path,
"nested_path":
nested_path,
"put":
col_pointer_name
if not self.settings.show_foreign_keys else
col_full_name # REFERENCE FIELDS CAN REPLACE THE WHOLE OBJECT BEING REFERENCED
})
elif col.include:
c_index = len(output_columns)
output_columns.append({
"table_alias":
alias,
"column_alias":
"c" + text_type(c_index),
"column":
col,
"sort":
False,
"path":
referenced_column_path,
"nested_path":
nested_path,
"put":
col_full_name
})
if position.name in reference_only_tables:
continue
todo.append(
Data(position=copy(constraint_columns[0].referenced.table),
path=referenced_column_path,
nested_path=nested_path,
done_relations=copy(done_relations),
no_nested_docs=no_nested_docs))
# NESTED OBJECTS
if not no_nested_docs:
for g, constraint_columns in jx.groupby(
jx.filter(
relations, {
"eq": {
"referenced.table.name": position.name,
"referenced.table.schema": position.schema
}
}), "constraint.name"):
g = unwrap(g)
constraint_columns = deepcopy(constraint_columns)
if g["constraint.name"] in done_relations:
continue
done_relations.add(g["constraint.name"])
many_table = set(constraint_columns.table.name)
if not (many_table - self.settings.exclude):
continue
referenced_column_path = join_field(
split_field(path) + ["/".join(many_table)])
new_nested_path = [referenced_column_path] + nested_path
all_nested_paths.append(new_nested_path)
# if new_path not in self.settings.include:
# Log.note("Exclude nested path {{path}}", path=new_path)
# continue
one_to_many_joins = nested_path_to_join[
referenced_column_path] = copy(curr_join_list)
index = len(one_to_many_joins)
alias = "t" + text_type(index)
for c in constraint_columns:
c.table.alias = alias
c.referenced.table = position
one_to_many_joins.append(
set_default({}, g, {
"children": True,
"join_columns": constraint_columns,
"path": path,
"nested_path": nested_path
}))
# insert into nested1 VALUES (100, 10, 'aaa', -1); # id.about.time.nested1 .ref=10# id.about.time.nested1 .ref.name
for col in columns:
if col.table.name == constraint_columns[
0].table.name and col.table.schema == constraint_columns[
0].table.schema:
col_full_name = join_field(
split_field(referenced_column_path)
[len(split_field(new_nested_path[0])):] +
[literal_field(col.column.name)])
if col.column.name == constraint_columns[
0].column.name:
c_index = len(output_columns)
output_columns.append({
"table_alias":
alias,
"column_alias":
"c" + text_type(c_index),
"column":
col,
"sort":
col.is_id,
"path":
referenced_column_path,
"nested_path":
new_nested_path,
"put":
col_full_name if
self.settings.show_foreign_keys else None
})
elif col.is_id:
c_index = len(output_columns)
output_columns.append({
"table_alias":
alias,
"column_alias":
"c" + text_type(c_index),
"column":
col,
"sort":
col.is_id,
"path":
referenced_column_path,
"nested_path":
new_nested_path,
"put":
col_full_name if
self.settings.show_foreign_keys else None
})
else:
c_index = len(output_columns)
output_columns.append({
"table_alias":
alias,
"column_alias":
"c" + text_type(c_index),
"column":
col,
"sort":
col.is_id,
"path":
referenced_column_path,
"nested_path":
new_nested_path,
"put":
col_full_name if col.include else None
})
todo.append(
Data(position=constraint_columns[0].table,
path=referenced_column_path,
nested_path=new_nested_path,
done_relations=copy(done_relations),
no_nested_docs=no_nested_docs))
path = "."
nested_path = [path]
nested_path_to_join["."] = [{
"path":
path,
"join_columns": [{
"referenced": {
"table": ids_table
}
}],
"nested_path":
nested_path
}]
todo.append(
Data(position=ids_table,
path=path,
nested_path=nested_path,
done_relations=set(),
no_nested_docs=False))
while todo:
item = todo.pop(0)
follow_paths(**item)
self.all_nested_paths = all_nested_paths
self.nested_path_to_join = nested_path_to_join
self.columns = output_columns
def random_id():
return Random.hex(40)
def _install_es(self, gigabytes):
volumes = self.instance.markup.drives
if not fabric_files.exists("/usr/local/elasticsearch"):
with cd("/home/ec2-user/"):
run("mkdir -p temp")
if not File(LOCAL_JRE).exists:
Log.error("Expecting {{file}} on manager to spread to ES instances", file=LOCAL_JRE)
with cd("/home/ec2-user/temp"):
run('rm -f '+JRE)
put("resources/"+JRE, JRE)
sudo("rpm -i "+JRE)
sudo("alternatives --install /usr/bin/java java /usr/java/default/bin/java 20000")
run("export JAVA_HOME=/usr/java/default")
with cd("/home/ec2-user/"):
run('wget https://download.elasticsearch.org/elasticsearch/elasticsearch/elasticsearch-1.7.1.tar.gz')
run('tar zxfv elasticsearch-1.7.1.tar.gz')
sudo('mkdir /usr/local/elasticsearch')
sudo('cp -R elasticsearch-1.7.1/* /usr/local/elasticsearch/')
with cd('/usr/local/elasticsearch/'):
# BE SURE TO MATCH THE PLUGLIN WITH ES VERSION
# https://github.com/elasticsearch/elasticsearch-cloud-aws
sudo('bin/plugin -install elasticsearch/elasticsearch-cloud-aws/2.7.1')
#REMOVE THESE FILES, WE WILL REPLACE THEM WITH THE CORRECT VERSIONS AT THE END
sudo("rm -f /usr/local/elasticsearch/config/elasticsearch.yml")
sudo("rm -f /usr/local/elasticsearch/bin/elasticsearch.in.sh")
self.conn = self.instance.connection
# MOUNT AND FORMAT THE EBS VOLUMES (list with `lsblk`)
for i, k in enumerate(volumes):
if not fabric_files.exists(k.path):
with fabric_settings(warn_only=True):
sudo('sudo umount '+k.device)
sudo('yes | sudo mkfs -t ext4 '+k.device)
sudo('mkdir '+k.path)
sudo('sudo mount '+k.device+' '+k.path)
#ADD TO /etc/fstab SO AROUND AFTER REBOOT
sudo("sed -i '$ a\\"+k.device+" "+k.path+" ext4 defaults,nofail 0 2' /etc/fstab")
# TEST IT IS WORKING
sudo('mount -a')
# INCREASE THE FILE HANDLE LIMITS
with cd("/home/ec2-user/"):
File("./results/temp/sysctl.conf").delete()
get("/etc/sysctl.conf", "./results/temp/sysctl.conf", use_sudo=True)
lines = File("./results/temp/sysctl.conf").read()
if lines.find("fs.file-max = 100000") == -1:
lines += "\nfs.file-max = 100000"
lines = lines.replace("net.bridge.bridge-nf-call-ip6tables = 0", "")
lines = lines.replace("net.bridge.bridge-nf-call-iptables = 0", "")
lines = lines.replace("net.bridge.bridge-nf-call-arptables = 0", "")
File("./results/temp/sysctl.conf").write(lines)
put("./results/temp/sysctl.conf", "/etc/sysctl.conf", use_sudo=True)
sudo("sysctl -p")
# INCREASE FILE HANDLE PERMISSIONS
sudo("sed -i '$ a\\root soft nofile 50000' /etc/security/limits.conf")
sudo("sed -i '$ a\\root hard nofile 100000' /etc/security/limits.conf")
sudo("sed -i '$ a\\root memlock unlimited' /etc/security/limits.conf")
sudo("sed -i '$ a\\ec2-user soft nofile 50000' /etc/security/limits.conf")
sudo("sed -i '$ a\\ec2-user hard nofile 100000' /etc/security/limits.conf")
sudo("sed -i '$ a\\ec2-user memlock unlimited' /etc/security/limits.conf")
# EFFECTIVE LOGIN TO LOAD CHANGES TO FILE HANDLES
# sudo("sudo -i -u ec2-user")
if not fabric_files.exists("/data1/logs"):
sudo('mkdir /data1/logs')
sudo('mkdir /data1/heapdump')
#INCREASE NUMBER OF FILE HANDLES
# sudo("sysctl -w fs.file-max=64000")
# COPY CONFIG FILE TO ES DIR
if not fabric_files.exists("/usr/local/elasticsearch/config/elasticsearch.yml"):
yml = File("./examples/config/es_config.yml").read().replace("\r", "")
yml = expand_template(yml, {
"id": Random.hex(length=8),
"data_paths": ",".join("/data"+unicode(i+1) for i, _ in enumerate(volumes))
})
File("./results/temp/elasticsearch.yml").write(yml)
put("./results/temp/elasticsearch.yml", '/usr/local/elasticsearch/config/elasticsearch.yml', use_sudo=True)
# FOR SOME REASON THE export COMMAND DOES NOT SEEM TO WORK
# THIS SCRIPT SETS THE ES_MIN_MEM/ES_MAX_MEM EXPLICITLY
if not fabric_files.exists("/usr/local/elasticsearch/bin/elasticsearch.in.sh"):
sh = File("./examples/config/es_run.sh").read().replace("\r", "")
sh = expand_template(sh, {"memory": unicode(int(gigabytes/2))})
File("./results/temp/elasticsearch.in.sh").write(sh)
with cd("/home/ec2-user"):
put("./results/temp/elasticsearch.in.sh", './temp/elasticsearch.in.sh', use_sudo=True)
sudo("cp -f ./temp/elasticsearch.in.sh /usr/local/elasticsearch/bin/elasticsearch.in.sh")
