def test_create_output_path_pathlib():
ext = rand_str()
base = rand_str()
file_type = random.choice(list(FileType))
path = pathlib.Path(f"{ext}.{base}")
gold = f"{ext}.{file_type}"
assert create_output_path(path, file_type) == gold
def __init__(self, server='127.0.0.1', min_retry_time=2, max_retry_time=60, node_name=None):
self.id = "{}-{}".format(get_mac(), rand_str(10))
self.connected = False
self.min_retry_time = min_retry_time
self.max_retry_time = max_retry_time
self.retry_time = self.min_retry_time
self.server = server
if not node_name:
node_name = rand_str(10)
banned = "/#+"
self.node_name = "".join(["-" if ch in banned else ch for ch in node_name])
self.path = "home/{name}/{mac}".format(name=self.node_name, mac=get_mac())
self.mqttc = mqtt.Client(client_id=self.id, clean_session=True, userdata=self)
self.mqttc.will_set(
topic=self.path + "/info/status",
payload="lost",
qos=1,
retain=True)
self.mqttc.on_message = self._dispatch
self.mqttc.on_connect = self._on_connect
self.mqttc.on_disconnect = self._on_disconnect
self.dispatch_table = dict()
self.dtlock = Lock()
def test_find_space():
gold = rand_str()
gold_offset = len(gold.encode("utf-8")) + 1
extra = rand_str()
text = f"{gold} {extra}".encode("utf-8")
word, offset = find_space(text, offset=0)
assert word == gold
assert offset == gold_offset
def test_find_space_with_offset():
before = rand_str()
start = len(before.encode("utf-8")) + 1
gold = rand_str()
gold_offset = start + len(gold.encode("utf-8")) + 1
after = rand_str()
text = f"{before} {gold} {after}".encode("utf-8")
word, offset = find_space(text, offset=start)
def test_add_vertex_label():
label2idx = {rand_str(): i for i in range(random.randint(1, 10))}
graph = Graph(deepcopy(label2idx))
gold_label, idx = rand_str(), len(label2idx)
label2idx[gold_label] = idx
idx = graph._add_vertex(gold_label)
assert graph.label2idx == label2idx
assert graph.idx2label == {i: l for l, i in label2idx.items()}
def test_graph_contains_etr():
g = Graph({})
node = n2 = rand_str()
while n2 == node:
n2 = rand_str()
g.label2idx = {node: None}
assert node in g
assert n2 not in g
def test_create_output_path_open_bytes():
ext = rand_str()
base = rand_str()
file_type = random.choice(list(FileType))
path = f"{ext}.{base}"
path_file = io.BytesIO(path.encode("utf-8"))
path_file.name = path
gold = f"{ext}.{file_type}"
assert create_output_path(path_file, file_type) == gold
def test_create_output_path_open():
ext = rand_str()
base = rand_str()
file_type = random.choice(list(FileType))
path = f"{ext}.{base}"
path_file = io.StringIO(path)
path_file.name = path
gold = f"{ext}.{file_type}"
assert create_output_path(path_file, file_type) == gold
def test_add_vertex_list():
verts = [rand_str() for _ in range(random.randint(1, 10))]
graph = AdjacencyList(verts)
gold_position = len(verts)
graph._add_vertex = MagicMock(return_value=gold_position)
gold_label = rand_str()
idx = graph.add_vertex(gold_label)
graph._add_vertex.assert_called_once_with(gold_label)
assert idx == gold_position
assert graph.vertices[gold_position] == Vertex(gold_label)
def test_add_node_matrix():
verts = [rand_str() for _ in range(random.randint(1, 10))]
graph = AdjacencyMatrix(verts)
graph._adjacency_matrix = np.random.rand(*graph._adjacency_matrix.shape)
gold_position = len(verts)
graph._add_vertex = MagicMock(return_value=gold_position)
gold_label = rand_str()
idx = graph.add_vertex(gold_label)
graph._add_vertex.assert_called_once_with(gold_label)
assert idx == gold_position
assert (graph.adjacency_matrix[:, idx] == 0).all()
assert (graph.adjacency_matrix[idx, :] == 0).all()
def test_SUID_sandbox(self):
log_file = self.new_temp_file('/tmp/chrome.log')
url = cm.BASE_TEST_URL + "font-face/font-face-names.html"
unexpected_strs = ('FATAL:browser_main_loop.cc', 'Running without the SUID sandbox')
expected_strs = ('FPLOG',)
cmd='timeout 10 xvfb-run --auto-servernum %s --disable-setuid-sandbox --enable-logging=stderr --v=1 --vmodule=frame=1 \
--user-data-dir=/tmp/temp_profile%s --disk-cache-dir=/tmp/tmp_cache%s %s 2>&1 | tee %s' %\
(cm.CHROME_MOD_BINARY, ut.rand_str(), ut.rand_str(), url, log_file)
ut.run_cmd(cmd)
self.assert_all_patterns_not_in_file(log_file, unexpected_strs)
self.assert_all_patterns_in_file(log_file, expected_strs)
def test_SUID_sandbox(self):
log_file = self.new_temp_file('/tmp/chrome.log')
url = cm.BASE_TEST_URL + "font-face/font-face-names.html"
unexpected_strs = ('FATAL:browser_main_loop.cc',
'Running without the SUID sandbox')
expected_strs = ('FPLOG', )
cmd='timeout 10 xvfb-run --auto-servernum %s --disable-setuid-sandbox --enable-logging=stderr --v=1 --vmodule=frame=1 \
--user-data-dir=/tmp/temp_profile%s --disk-cache-dir=/tmp/tmp_cache%s %s 2>&1 | tee %s' %\
(cm.CHROME_MOD_BINARY, ut.rand_str(), ut.rand_str(), url, log_file)
ut.run_cmd(cmd)
self.assert_all_patterns_not_in_file(log_file, unexpected_strs)
self.assert_all_patterns_in_file(log_file, expected_strs)
def test_output_list_sum_to_one():
v = random.randint(2, 10)
verts = [rand_str() for _ in range(v)]
graph = MagicMock(vertices=[Vertex(v) for v in verts])
ws = np.random.rand(v)
scores = [x[1] for x in text_rank_output_list(graph, ws)]
assert math.isclose(sum(scores), 1)
def test_output_matrix_sum_to_one():
v = random.randint(2, 10)
verts = [rand_str() for _ in range(v)]
graph = MagicMock(label2idx={v: None for v in verts})
ws = np.random.rand(v)
scores = [x[1] for x in text_rank_output_matrix(graph, ws)]
assert math.isclose(sum(scores), 1)
def test_rand_str(self):
# Test default parameters
random_str = ut.rand_str()
self.assertEqual(ut.DEFAULT_RAND_STR_SIZE, len(random_str),
"rand_str does not return string with default size!")
self.failIf(set(random_str) - set(ut.DEFAULT_RAND_STR_CHARS), "Unexpected characters in string!")
# Test with different sizes and charsets
sizes = [1, 2, 10, 100]
charsets = (ut.DEFAULT_RAND_STR_CHARS, ut.DIGITS)
for size in sizes:
for charset in charsets:
random_str = ut.rand_str(size, charset)
self.assertEqual(len(random_str), size,
"Random string size is different than expected!")
self.failIf(set(random_str) - set(ut.DEFAULT_RAND_STR_CHARS),
"Unexpected characters in string!")
def test_norm_sentence():
sentence = [rand_str() for _ in range(random.randint(1, 10))]
with patch("text_rank.utils.norm_token") as norm_patch:
norm_patch.side_effect = sentence
res = norm_sentence(" ".join(sentence))
for token in sentence:
assert call(token) in norm_patch.call_args_list
assert res == " ".join(sentence)
def test_add_node_matrix_out_of_order():
graph = AdjacencyMatrix({})
gold_position = random.randint(10, 100)
graph._add_vertex = MagicMock(return_value=gold_position)
label = rand_str()
with pytest.raises(ValueError):
graph.add_vertex(label)
graph._add_vertex.assert_called_once_with(label)
def test_hash_file(self):
filename = self.new_temp_file('hash_test.txt')
random_str = ut.rand_str(1000)
fu.write_to_file(filename, random_str)
self.assertEqual(fu.hash_file(filename, 'sha1'),
ut.hash_text(random_str, 'sha1'),
'SHA1 hashes don\'t match')
self.assertEqual(fu.hash_file(filename), ut.hash_text(random_str),
'Hashes with default algo don\'t match')
def test_hash_file(self):
filename = self.new_temp_file('hash_test.txt')
random_str = ut.rand_str(1000)
fu.write_to_file(filename, random_str)
self.assertEqual(fu.hash_file(filename, 'sha1'),
ut.hash_text(random_str, 'sha1'),
'SHA1 hashes don\'t match')
self.assertEqual(fu.hash_file(filename), ut.hash_text(random_str),
'Hashes with default algo don\'t match')
def test_graph_forces_contiguous_idx():
pop = random.randint(100, 200)
vertex_idxs = [1, 2, 3]
while all(p != c - 1 for p, c in zip(vertex_idxs, vertex_idxs[1:])):
vertex_idxs = sorted(
random.sample(range(pop), k=pop // random.randint(2, 10)))
vertices = {rand_str(): vi for vi in vertex_idxs}
with pytest.raises(ValueError):
g = Graph(vertices)
def sendBox(self, addr, data, tx_id=None):
box = BoxContainer()
box.name = data.__class__.__name__
box.value = token.node_data(data.SerializeToString(), self.fingerprint)
# generate random tx id if not given
if not tx_id:
tx_id = utils.rand_str(8)
box.tx_id = tx_id
self.sendString(addr, box.SerializeToString())
return tx_id
def sendBox(self, data, tx_id = None):
box = BoxContainer()
box.name = data.__class__.__name__
box.value = data.SerializeToString()
# generate random tx id if not given
if not tx_id:
tx_id = utils.rand_str(8)
box.tx_id = tx_id
self.sendString(box.SerializeToString())
return tx_id
async def share(self, file_id, duration):
key = rand_str()
expires = datetime.utcnow() + relativedelta(hours=duration)
q = self.insert().columns(
'file_id', 'hash', 'expires'
).insert(
file_id, key, expires,
)
await self.exec(q)
return key
def sendBox(self, addr, data, tx_id = None):
box = BoxContainer()
box.name = data.__class__.__name__
box.value = token.node_data(data.SerializeToString(), self.fingerprint)
# generate random tx id if not given
if not tx_id:
tx_id = utils.rand_str(8)
box.tx_id = tx_id
self.sendString(addr, box.SerializeToString())
return tx_id
def test_add_edge_greater_than_zero():
graph = AdjacencyList([rand_str() for _ in range(random.randint(10, 100))])
source = target = 0
while source == target:
source = random.randint(0, graph.vertex_count - 1)
target = random.randint(0, graph.vertex_count - 1)
weight = (random.random() + 0.01) * -1
with pytest.raises(ValueError):
graph.add_edge(source, target, weight)
def sendBox(self, data, tx_id=None):
box = BoxContainer()
box.name = data.__class__.__name__
box.value = data.SerializeToString()
# generate random tx id if not given
if not tx_id:
tx_id = utils.rand_str(8)
box.tx_id = tx_id
self.sendString(box.SerializeToString())
return tx_id
def test():
intersection = random.randint(0, 10)
len_1 = random.randint(intersection,
intersection + random.randint(1, 5))
len_2 = random.randint(intersection,
intersection + random.randint(1, 5))
len_1 = max(1, len_1)
len_2 = max(1 if len_1 != 1 else 2, len_2)
union = math.log(len_1) + math.log(len_2)
union = 1 if union == 0.0 else union
gold_score = intersection / union
set1 = set()
set2 = set()
while len(set1) < intersection:
t = rand_str()
set1.add(t)
set2.add(t)
while len(set1) < len_1:
t = rand_str()
if t not in set1:
set1.add(t)
while len(set2) < len_2:
t = rand_str()
if t not in set1 and t not in set2:
set2.add(t)
ex1 = list(chain(*[[t] * random.randint(1, 5) for t in set1]))
ex2 = list(chain(*[[t] * random.randint(1, 5) for t in set2]))
random.shuffle(ex1)
random.shuffle(ex2)
ex1 = " ".join(ex1)
ex2 = " ".join(ex2)
score = overlap(ex1, ex2)
assert math.isclose(score, gold_score)
def test_005_add_user(self):
name = rand_str(5)
self.chrome.by_xpath("//div[text()='添加用户']").click()
self.chrome.by_xpath("//input[@placeholder='请输入用户']").send_keys(name)
self.chrome.by_xpath("//input[@placeholder='请输入邮箱地址']").send_keys(
email)
self.chrome.by_xpath("//input[@placeholder='请输入手机号']").send_keys(phone)
self.chrome.by_xpath("//input[@placeholder='请输入密码']").send_keys(
login_pwd)
self.chrome.by_xpath("//span[text()='完 成']").click()
assert (self.chrome.is_has("添加用户成功！"))
def update(self, userid, code=False):
"""Add/replace a validation record."""
if not code:
random = True
code = utils.rand_str(utils.VALIDATION_CODE_LENGTH, utils.CHARSBOX_NUMBERS)
else:
random = False
# TODO retry with a different code on error (limiting attempts of course)
fields = (userid, code)
return (self.execute_update("REPLACE INTO validations VALUES (?, ?, sysdate())", fields), code)
def __init__(self, A):
self.node_name = "ReLU{}".format(rand_str())
if type(A) is not Parameter:
A = A.get_output()
self.A = A
NeuralNetwork.NeuralNetwork.add_node(self)
NeuralNetwork.NeuralNetwork.add_param(self.A)
self.out = Parameter(is_placeholder=True, name=self.node_name + "_out")
def test_rand_str(self):
# Test default parameters
random_str = ut.rand_str()
self.assertEqual(ut.DEFAULT_RAND_STR_SIZE, len(random_str),
"rand_str does not return string with default size!")
self.failIf(
set(random_str) - set(ut.DEFAULT_RAND_STR_CHARS),
"Unexpected characters in string!")
# Test with different sizes and charsets
sizes = [1, 2, 10, 100]
charsets = (ut.DEFAULT_RAND_STR_CHARS, ut.DIGITS)
for size in sizes:
for charset in charsets:
random_str = ut.rand_str(size, charset)
self.assertEqual(
len(random_str), size,
"Random string size is different than expected!")
self.failIf(
set(random_str) - set(ut.DEFAULT_RAND_STR_CHARS),
"Unexpected characters in string!")
def test_output_matrix():
v = random.randint(2, 10)
verts = [rand_str() for _ in range(v)]
graph = MagicMock(label2idx={v: None for v in verts})
ws = np.random.rand(v)
gold_labels = [verts[i] for i in np.argsort(ws)[::-1]]
gold_score = np.sort(ws)[::-1]
gold_score = gold_score / np.sum(gold_score)
res = text_rank_output_matrix(graph, ws)
for gl, gs, (l, s) in zip(gold_labels, gold_score, res):
assert l == gl
assert math.isclose(s, gs)
def test_vertex_equal_mismatch_values():
label = label2 = rand_str()
while label2 == label:
label2 = rand_str()
edges_in = {
random.randint(0, 100): random.random()
for _ in range(random.randint(0, 10))
}
edges_out = {
random.randint(0, 100): random.random()
for _ in range(random.randint(0, 10))
}
v1 = Vertex(label)
v1._edges_out = edges_out
v1._edges_in = edges_in
v2 = Vertex(label2)
v2._edges_out = edges_out
v2._edges_in = edges_in
assert v1 != v2
def update(self, userid, code=False):
'''Add/replace a validation record.'''
if not code:
random = True
code = utils.rand_str(utils.VALIDATION_CODE_LENGTH,
utils.CHARSBOX_NUMBERS)
else:
random = False
# TODO retry with a different code on error (limiting attempts of course)
fields = (userid, code)
return (self.execute_update(
'REPLACE INTO validations VALUES (?, ?, sysdate())', fields), code)
def test_convert_with_output_pathlib():
data = random.choice([GLOVE, W2V, W2V_TEXT, LEADER])
output_type = random.choice(list(FileType))
output = pathlib.Path(rand_str())
input_path = DATA / data
with patch("word_vectors.convert_module.write") as write_patch:
w, wv = read(input_path)
convert(input_path, output, output_file_type=output_type)
call_file, call_w, call_wv, call_type = write_patch.call_args_list[0][0]
assert call_file == output
assert call_w == w
assert call_type == output_type
np.testing.assert_allclose(call_wv, wv)
def test():
gold = rand_str()
token = []
for char in gold:
if random.choice([True, False]):
token.append(char.upper())
else:
token.append(char)
while random.random() < 0.3:
token.append(random.choice(string.punctuation))
token = "".join(token)
res = norm_token(token)
assert res == gold
def copy_file(self, src_file_path):
temp_file_name = os.path.join(self.temp_dir, rand_str())
result = self.source.download(src_file_path, temp_file_name)
if result is None:
self.log(ERROR_MESSAGES['down'] + src_file_path)
else:
result = self.dest.upload(temp_file_name, src_file_path)
if result != error_codes.OK:
self.log(ERROR_MESSAGES['up'] + src_file_path)
try:
os.remove(temp_file_name)
except:
self.log(ERROR_MESSAGES['local_rm'] + temp_file_name)
def gen_decompile_swf(swf_path, out_dir=''):
"""Decompile a Flash file with the available decompilers."""
if not os.path.isfile(swf_path):
return
if not out_dir:
base_dir = os.path.dirname(swf_path)
out_dir = ut.rand_str() + '-src'
out_dir = os.path.join(base_dir, out_dir)
if not os.path.isdir(out_dir):
os.mkdir(out_dir)
cmd_ffdec = 'java -jar ' + FFDEC_PATH + ' -export as "' + out_dir + '" "' + swf_path + '"'
timeout_prefx = 'timeout -k 5 30 ' # linux specific
ffdec_status, ffdec_output = cmds.getstatusoutput(timeout_prefx + cmd_ffdec)
write_decomp_log(swf_path, ffdec_status, ffdec_output, '', '')
for flash_src_file in fu.gen_find_files(AS_SOURCE_FILE_PATTERN, out_dir):
yield flash_src_file
def test_write_to_file(self):
filename = self.new_temp_file('write_test.txt')
random_str = ut.rand_str(100)
fu.write_to_file(filename, random_str)
self.assertEqual(random_str, fu.read_file(filename))
def get_visit_cmd(agent_cfg, proxy_opt, stdout_log, url):
"""Return command for a visit."""
xvfb = ''
caps_name = ''
clientjs = ''
if 'chrome' in agent_cfg['type']:
proxy_opt = '--proxy-server=%s' % proxy_opt
redir_str = '2>&1 | tee %s' % stdout_log # chrome logs to stderr, we redirect it to stdout
cmd_line_options = agent_cfg['cmd_line_options'] + \
' --disk-cache-dir=/tmp/tmp_cache%s --user-data-dir=/tmp/temp_profile%s' % (ut.rand_str(), ut.rand_str())
xvfb = 'xvfb-run --auto-servernum' # use xvfb for chrome
else:
proxy_opt = '--proxy=%s' % proxy_opt
redir_str = ' > %s' % stdout_log # redirect all output to log file
cmd_line_options = agent_cfg['cmd_line_options']
caps_name = stdout_log[:-4] + '.png' if agent_cfg['screenshot'] else 'NO_SCREENSHOT'
clientjs = agent_cfg['casper_client_js']
# TODO separate cmd construction for chrome and phantomjs
cmd = 'export FC_DEBUG=%s; %s timeout -k %s %s %s %s %s %s %s %s %s %s' \
% (agent_cfg['fc_fontdebug'], xvfb, KILL_PROC_AFTER_TIMEOUT, agent_cfg['timeout'], agent_cfg['binary_path'],
cmd_line_options, proxy_opt, agent_cfg['main_js'], url,
caps_name, clientjs, redir_str)
return cmd