def test_wrap_2():
switch = [
lambda: {"i": Random.int(2000)},
lambda: Data(i=Random.int(2000)),
lambda: FlatList([{"i": Random.int(2000)}]),
lambda: [{"i": Random.int(2000)}]
]
inputs = [switch[min(len(switch) - 1, int(floor(-log(Random.float(), 2))))]() for i in range(NUM_INPUT)]
for i in range(NUM_REPEAT):
results = []
gc.collect()
with Profiler("more dict: slow_wrap"):
for v in inputs:
results.append(slow_wrap(v))
results = []
gc.collect()
with Profiler("more dict: wrap"):
for v in inputs:
results.append(wrap(v))
results = []
gc.collect()
with Profiler("more dict: baseline"):
for v in inputs:
results.append(baseline(v))
Log.note("Done {{i}} of {{num}}", {"i": i, "num": NUM_REPEAT})
def test_aes(self):
crypto.DEBUG = True
crypto.encrypt("this is a test", Random.bytes(32))
crypto.encrypt("this is a longer test with more than 16bytes", Random.bytes(32))
crypto.encrypt("", Random.bytes(32))
crypto.encrypt(convert.latin12unicode(b"testing accented char àáâãäåæçèéêëìíîïðñòóôõö÷øùúûüýþÿ"), Random.bytes(32))
crypto.encrypt("testing accented char àáâãäåæçèéêëìíîïðñòóôõö÷øùúûüýþÿ", Random.bytes(32))
def commit(self):
with self.lock:
if self.closed:
Log.error("Queue is closed, commit not allowed")
try:
self._add_pending({"add": {"status.start": self.start}})
for i in range(self.db.status.start, self.start):
self._add_pending({"remove": str(i)})
if (
self.db.status.end - self.start < 10 or Random.range(0, 1000) == 0
): # FORCE RE-WRITE TO LIMIT FILE SIZE
# SIMPLY RE-WRITE FILE
if DEBUG:
Log.note(
"Re-write {{num_keys}} keys to persistent queue", num_keys=self.db.status.end - self.start
)
for k in self.db.keys():
if k == "status" or int(k) >= self.db.status.start:
continue
Log.error("Not expecting {{key}}", key=k)
self._commit()
self.file.write(convert.value2json({"add": self.db}) + "\n")
else:
self._commit()
except Exception, e:
raise e
def output(*args):
with cache_store.locker:
if using_self:
self = args[0]
args = args[1:]
else:
self = cache_store
now = Date.now()
try:
_cache = getattr(self, attr_name)
except Exception, _:
_cache = {}
setattr(self, attr_name, _cache)
if Random.int(100) == 0:
# REMOVE OLD CACHE
_cache = {
k: v
for k, v in _cache.items() if v[0] == None or v[0] > now
}
setattr(self, attr_name, _cache)
timeout, key, value, exception = _cache.get(
args, (Null, Null, Null, Null))
def test_request(self):
# MAKE SOME DATA
data = {
"constant": "this is a test",
"random-data": convert.bytes2base64(Random.bytes(100))
}
client = Client(settings.url, unwrap(settings.hawk)) # unwrap() DUE TO BUG https://github.com/kumar303/mohawk/issues/21
link, id = client.send(data)
Log.note("Success! Located at {{link}} id={{id}}", link=link, id=id)
# FILL THE REST OF THE FILE
Log.note("Add ing {{num}} more...", num=99-id)
for i in range(id + 1, storage.BATCH_SIZE):
l, k = client.send(data)
if l != link:
Log.error("Expecting rest of data to have same link")
# TEST LINK HAS DATA
raw_content = requests.get(link).content
content = convert.zip2bytes(raw_content)
for line in convert.utf82unicode(content).split("\n"):
data = convert.json2value(line)
if data.etl.id == id:
Log.note("Data {{id}} found", id=id)
break
else:
Log.error("Expecting to find data at link")
def test_public_request_too_big(self):
# MAKE SOME DATA
data = {
"a": { # MATCHES SERVER PATTERN
"b": "good",
"c": [
{"k": "good", "m": 1},
{"k": 2, "m": 2}
]
},
"constant": "this is a test",
"random-data": convert.bytes2base64(Random.bytes(500))
}
content = json.dumps(data)
def poster():
response = requests.post(
url=settings.url,
data=content,
headers={
'Content-Type': CONTENT_TYPE
}
)
self.assertEqual(response.status_code, 200, "Expecting 200")
self.assertRaises(Exception, poster)
def test_public_request(self):
# MAKE SOME DATA
data = {
"a": { # MATCHES SERVER PATTERN
"b": "good",
"c": [
{"k": "good", "m": 1},
{"k": 2, "m": 2}
]
},
"constant": "this is a test",
"random-data": convert.bytes2base64(Random.bytes(100))
}
content = json.dumps(data)
response = requests.post(
url=settings.url,
data=content,
headers={
'Content-Type': CONTENT_TYPE
}
)
self.assertEqual(response.status_code, 200, "Expecting 200")
about = json.loads(response.content)
return about['link'], about['etl']['id']
Log.note("Data located at {{link}} id={{id}}", link=link, id=id)
def extend(self, records):
"""
records - MUST HAVE FORM OF
[{"value":value}, ... {"value":value}] OR
[{"json":json}, ... {"json":json}]
OPTIONAL "id" PROPERTY IS ALSO ACCEPTED
"""
lines = []
try:
for r in records:
id = r.get("id")
if id == None:
id = Random.hex(40)
if "json" in r:
json = r["json"]
elif "value" in r:
json = convert.value2json(r["value"])
else:
json = None
Log.error("Expecting every record given to have \"value\" or \"json\" property")
lines.append('{"index":{"_id": ' + convert.value2json(id) + '}}')
lines.append(json)
del records
if not lines:
return
try:
data_bytes = "\n".join(lines) + "\n"
data_bytes = data_bytes.encode("utf8")
except Exception, e:
Log.error("can not make request body from\n{{lines|indent}}", lines= lines, cause=e)
response = self.cluster._post(
self.path + "/_bulk",
data=data_bytes,
headers={"Content-Type": "text"},
timeout=self.settings.timeout
)
items = response["items"]
for i, item in enumerate(items):
if self.cluster.version.startswith("0.90."):
if not item.index.ok:
Log.error("{{error}} while loading line:\n{{line}}",
error= item.index.error,
line= lines[i * 2 + 1])
elif self.cluster.version.startswith("1.4."):
if item.index.status not in [200, 201]:
Log.error("{{error}} while loading line:\n{{line}}",
error= item.index.error,
line= lines[i * 2 + 1])
else:
Log.error("version not supported {{version}}", version=self.cluster.version)
if self.debug:
Log.note("{{num}} documents added", num= len(items))
def commit(self):
with self.lock:
if self.closed:
Log.error("Queue is closed, commit not allowed")
try:
self._add_pending({"add": {"status.start": self.start}})
for i in range(self.db.status.start, self.start):
self._add_pending({"remove": str(i)})
if self.db.status.end - self.start < 10 or Random.range(
0, 1000) == 0: # FORCE RE-WRITE TO LIMIT FILE SIZE
# SIMPLY RE-WRITE FILE
if DEBUG:
Log.note(
"Re-write {{num_keys}} keys to persistent queue",
num_keys=self.db.status.end - self.start)
for k in self.db.keys():
if k == "status" or int(k) >= self.db.status.start:
continue
Log.error("Not expecting {{key}}", key=k)
self._commit()
self.file.write(
convert.value2json({"add": self.db}) + "\n")
else:
self._commit()
except Exception, e:
raise e
def main():
settings = startup.read_settings()
Log.start(settings.debug)
try:
with Multithread(update_repo, threads=10, outbound=False) as multi:
for repo in Random.combination(settings.param.repos):
multi.execute([{"repos": repo, "settings": settings}])
finally:
Log.stop()
def test_missing_auth(self):
# MAKE SOME DATA
data = {
"constant": "this is a test",
"random-data": convert.bytes2base64(Random.bytes(100))
}
response = requests.post(settings.bad_url, data=convert.unicode2utf8(convert.value2json(data)))
self.assertEqual(response.status_code, 403)
def main():
settings = startup.read_settings()
Log.start(settings.debug)
try:
with Multithread(update_repo, threads=10, outbound=False) as multi:
for repo in Random.combination(settings.param.repos):
multi.execute([{"repos": repo, "settings": settings}])
finally:
Log.stop()
def test_floor_mod_identity_w_ints(self):
for i in range(100):
x = Random.float()*200 - 100.0
m = floor(abs(random.gauss(0, 5)))
if m == 0:
self.assertEqual(Math.floor(x, m), None)
self.assertEqual(Math.mod(x, m), None)
else:
self.assertAlmostEqual(Math.floor(x, m)+Math.mod(x, m), x, places=7)
def fix(rownum, line, source, sample_only_filter, sample_size):
# ES SCHEMA IS STRICTLY TYPED, USE "code" FOR TEXT IDS
line = line.replace('{"id": "bb"}',
'{"code": "bb"}').replace('{"id": "tc"}',
'{"code": "tc"}')
# ES SCHEMA IS STRICTLY TYPED, THE SUITE OBJECT CAN NOT BE HANDLED
if source.name.startswith("active-data-test-result"):
# "suite": {"flavor": "plain-chunked", "name": "mochitest"}
found = strings.between(line, '"suite": {', '}')
if found:
suite_json = '{' + found + "}"
if suite_json:
suite = convert.json2value(suite_json)
suite = convert.value2json(suite.name)
line = line.replace(suite_json, suite)
if rownum == 0:
value = convert.json2value(line)
if len(line) > 100000:
value.result.subtests = [
s for s in value.result.subtests if s.ok is False
]
value.result.missing_subtests = True
_id, value = _fix(value)
row = {"id": _id, "value": value}
if sample_only_filter and Random.int(
int(1.0 / coalesce(sample_size, 0.01))) != 0 and jx.filter(
[value], sample_only_filter):
# INDEX etl.id==0, BUT NO MORE
if value.etl.id != 0:
Log.error("Expecting etl.id==0")
return row, True
elif len(line) > 100000:
value = convert.json2value(line)
value.result.subtests = [
s for s in value.result.subtests if s.ok is False
]
value.result.missing_subtests = True
_id, value = _fix(value)
row = {"id": _id, "value": value}
elif line.find("\"resource_usage\":") != -1:
value = convert.json2value(line)
_id, value = _fix(value)
row = {"id": _id, "value": value}
else:
# FAST
_id = strings.between(line, "\"_id\": \"", "\"") # AVOID DECODING JSON
row = {"id": _id, "json": line}
return row, False
def seen_nonce(sender_id, nonce, timestamp):
global seen
key = '{id}:{nonce}:{ts}'.format(
id=sender_id,
nonce=nonce,
ts=timestamp
)
if Random.int(1000) == 0:
old = (Date.now() - HOUR).unix
seen = {k: v for k, v in seen.items() if v["timestamp"] >= old}
if seen.get(key):
return True
else:
seen[key] = {"timestamp": timestamp}
return False
def fix(rownum, line, source, sample_only_filter, sample_size):
# ES SCHEMA IS STRICTLY TYPED, USE "code" FOR TEXT IDS
line = line.replace('{"id": "bb"}', '{"code": "bb"}').replace('{"id": "tc"}', '{"code": "tc"}')
# ES SCHEMA IS STRICTLY TYPED, THE SUITE OBJECT CAN NOT BE HANDLED
if source.name.startswith("active-data-test-result"):
# "suite": {"flavor": "plain-chunked", "name": "mochitest"}
found = strings.between(line, '"suite": {', '}')
if found:
suite_json = '{' + found + "}"
if suite_json:
suite = convert.json2value(suite_json)
suite = convert.value2json(suite.name)
line = line.replace(suite_json, suite)
if rownum == 0:
value = convert.json2value(line)
if len(line) > 100000:
value.result.subtests = [s for s in value.result.subtests if s.ok is False]
value.result.missing_subtests = True
_id, value = _fix(value)
row = {"id": _id, "value": value}
if sample_only_filter and Random.int(int(1.0/coalesce(sample_size, 0.01))) != 0 and jx.filter([value], sample_only_filter):
# INDEX etl.id==0, BUT NO MORE
if value.etl.id != 0:
Log.error("Expecting etl.id==0")
return row, True
elif len(line) > 100000:
value = convert.json2value(line)
value.result.subtests = [s for s in value.result.subtests if s.ok is False]
value.result.missing_subtests = True
_id, value = _fix(value)
row = {"id": _id, "value": value}
elif line.find("\"resource_usage\":") != -1:
value = convert.json2value(line)
_id, value = _fix(value)
row = {"id": _id, "value": value}
else:
# FAST
_id = strings.between(line, "\"_id\": \"", "\"") # AVOID DECODING JSON
row = {"id": _id, "json": line}
return row, False
def copy(self, keys, source, sample_only_filter=None, sample_size=None):
num_keys = 0
for key in keys:
try:
for rownum, line in enumerate(
source.read_lines(strip_extension(key))):
if rownum == 0:
value = convert.json2value(line)
if len(line) > 1000000:
# Log.warning("Line {{num}} for key {{key}} is too long ({{length|comma}} bytes, {{num_tests}} subtests)", key=key, length=len(line), num=rownum, num_tests=len(value.result.subtests))
value.result.subtests = None
value.result.missing_subtests = True
_id, value = _fix(value)
row = {"id": _id, "value": value}
if sample_only_filter and Random.int(
int(1.0 / coalesce(
sample_size, 0.01))) != 0 and qb.filter(
[value], sample_only_filter):
# INDEX etl.id==0, BUT NO MORE
if value.etl.id != 0:
Log.error("Expecting etl.id==0")
num_keys += 1
self.queue.add(row)
break
elif len(line) > 1000000:
value = convert.json2value(line)
# Log.warning("Line {{num}} for key {{key}} is too long ({{length|comma}} bytes, {{num_tests}} subtests).", key=key, length=len(line), num=rownum, num_tests=len(value.result.subtests))
value.result.subtests = None
value.result.missing_subtests = True
_id, value = _fix(value)
row = {"id": _id, "value": value}
else:
#FAST
_id = strings.between(line, "_id\": \"",
"\"") # AVOID DECODING JSON
row = {"id": _id, "json": line}
num_keys += 1
self.queue.add(row)
except Exception, e:
Log.warning("Could not get queue for {{key}}",
key=key,
cause=e)
def encrypt(text, _key, salt=None):
"""
RETURN JSON OF ENCRYPTED DATA {"salt":s, "length":l, "data":d}
"""
from pyLibrary.queries import jx
if not isinstance(text, unicode):
Log.error("only unicode is encrypted")
if _key is None:
Log.error("Expecting a key")
if isinstance(_key, str):
_key = bytearray(_key)
if salt is None:
salt = Random.bytes(16)
data = bytearray(text.encode("utf8"))
# Initialize encryption using key and iv
key_expander_256 = key_expander.KeyExpander(256)
expanded_key = key_expander_256.expand(_key)
aes_cipher_256 = aes_cipher.AESCipher(expanded_key)
aes_cbc_256 = cbc_mode.CBCMode(aes_cipher_256, 16)
aes_cbc_256.set_iv(salt)
output = Data()
output.type = "AES256"
output.salt = convert.bytes2base64(salt)
output.length = len(data)
encrypted = bytearray()
for _, d in jx.groupby(data, size=16):
encrypted.extend(aes_cbc_256.encrypt_block(d))
output.data = convert.bytes2base64(encrypted)
json = convert.value2json(output)
if DEBUG:
test = decrypt(json, _key)
if test != text:
Log.error("problem with encryption")
return json
def encrypt(text, _key, salt=None):
"""
RETURN JSON OF ENCRYPTED DATA {"salt":s, "length":l, "data":d}
"""
from pyLibrary.queries import jx
if not isinstance(text, unicode):
Log.error("only unicode is encrypted")
if _key is None:
Log.error("Expecting a key")
if isinstance(_key, str):
_key = bytearray(_key)
if salt is None:
salt = Random.bytes(16)
data = bytearray(text.encode("utf8"))
# Initialize encryption using key and iv
key_expander_256 = key_expander.KeyExpander(256)
expanded_key = key_expander_256.expand(_key)
aes_cipher_256 = aes_cipher.AESCipher(expanded_key)
aes_cbc_256 = cbc_mode.CBCMode(aes_cipher_256, 16)
aes_cbc_256.set_iv(salt)
output = Dict()
output.type = "AES256"
output.salt = convert.bytes2base64(salt)
output.length = len(data)
encrypted = bytearray()
for _, d in jx.groupby(data, size=16):
encrypted.extend(aes_cbc_256.encrypt_block(d))
output.data = convert.bytes2base64(encrypted)
json = convert.value2json(output)
if DEBUG:
test = decrypt(json, _key)
if test != text:
Log.error("problem with encryption")
return json
def output(*args):
with cache_store.locker:
if using_self:
self = args[0]
args = args[1:]
else:
self = cache_store
now = Date.now()
try:
_cache = getattr(self, attr_name)
except Exception, _:
_cache = {}
setattr(self, attr_name, _cache)
if Random.int(100) == 0:
# REMOVE OLD CACHE
_cache = {k: v for k, v in _cache.items() if v[0]==None or v[0] > now}
setattr(self, attr_name, _cache)
timeout, key, value, exception = _cache.get(args, (Null, Null, Null, Null))
def output(*args):
with cache_store.locker:
if using_self:
self = args[0]
args = args[1:]
else:
self = cache_store
now = Date.now()
try:
_cache = getattr(self, attr_name)
except Exception, _:
_cache = {}
setattr(self, attr_name, _cache)
if Random.int(100) == 0:
# REMOVE OLD CACHE
_cache = {k: v for k, v in _cache.items() if v[0]==None or v[0] < now}
setattr(self, attr_name, _cache)
timeout, key, value, exception = _cache.get(args, (Null, Null, Null, Null))
if now > timeout:
value = func(self, *args)
_cache[args] = (now + cache_store.timeout, args, value, None)
return value
if value == None:
if exception == None:
try:
value = func(self, *args)
_cache[args] = (now + cache_store.timeout, args, value, None)
return value
except Exception, e:
e = Except.wrap(e)
_cache[args] = (now + cache_store.timeout, args, None, e)
raise e
else:
raise exception
def copy(self, keys, source, sample_only_filter=None, sample_size=None):
num_keys = 0
for key in keys:
try:
for rownum, line in enumerate(source.read_lines(strip_extension(key))):
if rownum == 0:
value = convert.json2value(line)
if len(line) > 1000000:
# Log.warning("Line {{num}} for key {{key}} is too long ({{length|comma}} bytes, {{num_tests}} subtests)", key=key, length=len(line), num=rownum, num_tests=len(value.result.subtests))
value.result.subtests = None
value.result.missing_subtests = True
_id, value = _fix(value)
row = {"id": _id, "value": value}
if sample_only_filter and Random.int(int(1.0/coalesce(sample_size, 0.01))) != 0 and qb.filter([value], sample_only_filter):
# INDEX etl.id==0, BUT NO MORE
if value.etl.id != 0:
Log.error("Expecting etl.id==0")
num_keys += 1
self.queue.add(row)
break
elif len(line) > 1000000:
value = convert.json2value(line)
# Log.warning("Line {{num}} for key {{key}} is too long ({{length|comma}} bytes, {{num_tests}} subtests).", key=key, length=len(line), num=rownum, num_tests=len(value.result.subtests))
value.result.subtests = None
value.result.missing_subtests = True
_id, value = _fix(value)
row = {"id": _id, "value": value}
else:
#FAST
_id = strings.between(line, "_id\": \"", "\"") # AVOID DECODING JSON
row = {"id": _id, "json": line}
num_keys += 1
self.queue.add(row)
except Exception, e:
Log.warning("Could not get queue for {{key}}", key=key, cause=e)
def test_floor_mod_identity(self):
for i in range(100):
x = Random.float()*200 - 100.0
m = abs(random.gauss(0, 5))
self.assertAlmostEqual(Math.floor(x, m)+Math.mod(x, m), x, places=7)
def unique_name():
return Random.string(20)
def unique_name():
return Random.string(20)
def random_id():
return Random.hex(40)
def random_id():
return Random.hex(40)
import requests
from pyLibrary import convert, jsons
from pyLibrary.debugs.logs import Log
from pyLibrary.dot import unwrap
from pyLibrary.maths.randoms import Random
from modatasubmission import Client
settings = jsons.ref.get("file://~/MoDataSubmissionClient.json")
data={
"constant": "this is a test",
"random-data": convert.bytes2base64(Random.bytes(100))
}
link, id = Client(settings.url, unwrap(settings.hawk)).send(data)
Log.note("Success! Located at {{link}} id={{id}}", link=link, id=id)
data = convert.unicode2utf8(convert.value2json(settings.example))
response = requests.post(
settings.url,
data=data,
headers={
'Content-Type': b'application/json'
}
)
if response.status_code != 200:
def extend(self, records):
"""
records - MUST HAVE FORM OF
[{"value":value}, ... {"value":value}] OR
[{"json":json}, ... {"json":json}]
OPTIONAL "id" PROPERTY IS ALSO ACCEPTED
"""
lines = []
try:
for r in records:
id = r.get("id")
if id == None:
id = Random.hex(40)
if "json" in r:
# if id != coalesce(wrap(convert.json2value(r["json"])).value._id, id):
# Log.error("expecting _id to match")
json = r["json"]
elif "value" in r:
# if id != coalesce(wrap(r).value._id, id):
# Log.error("expecting _id to match")
json = convert.value2json(r["value"])
else:
json = None
Log.error(
"Expecting every record given to have \"value\" or \"json\" property"
)
lines.append('{"index":{"_id": ' + convert.value2json(id) +
'}}')
lines.append(json)
del records
if not lines:
return
try:
data_bytes = "\n".join(lines) + "\n"
data_bytes = data_bytes.encode("utf8")
except Exception, e:
Log.error("can not make request body from\n{{lines|indent}}",
lines=lines,
cause=e)
response = self.cluster._post(self.path + "/_bulk",
data=data_bytes,
headers={"Content-Type": "text"},
timeout=self.settings.timeout)
items = response["items"]
for i, item in enumerate(items):
if self.cluster.version.startswith("0.90."):
if not item.index.ok:
Log.error("{{error}} while loading line:\n{{line}}",
error=item.index.error,
line=lines[i * 2 + 1])
elif any(map(self.cluster.version.startswith,
["1.4.", "1.5."])):
if item.index.status not in [200, 201]:
Log.error(
"{{num}} {{error}} while loading line into {{index}}:\n{{line}}",
num=item.index.status,
error=item.index.error,
line=lines[i * 2 + 1],
index=self.settings.index)
else:
Log.error("version not supported {{version}}",
version=self.cluster.version)
if self.debug:
Log.note("{{num}} documents added", num=len(items))
