def evaluate_model(net, encode, samples, question_map):
samples = filter(lambda s: len(s.similar) > 0, samples)
criterion = nn.CosineSimilarity()
print
print 'Evaluating',
results_matrix = []
scores_matrix = []
for i in range(len(samples)):
sample = samples[i]
encoded = encode(net, sample.id, question_map)
sys.stdout.write('.')
sys.stdout.flush()
results = []
for candidate_id in sample.candidate_map:
similar_indicator = sample.candidate_map[candidate_id]
candidate_title, candidate_body = question_map[candidate_id]
candidate_encoded = encode(net, candidate_id, question_map)
# Compare similarity
similarity = criterion(encoded.unsqueeze(0),
candidate_encoded.unsqueeze(0)).data[0]
results.append((1.0 - similarity, candidate_id))
results.sort()
scores = map(lambda x: 1.0 - x[0], results)
results = map(lambda x: x[1], results)
results_matrix.append(results)
scores_matrix.append(scores)
MAP = mean_average_precision(samples, results_matrix)
MRR = mean_reciprocal_rank(samples, results_matrix)
MPK1 = mean_precision_at_k(samples, results_matrix, 1)
MPK5 = mean_precision_at_k(samples, results_matrix, 5)
MAUC = mean_area_under_curve(samples, results_matrix)
AUC05 = area_under_curve_fpr(samples, results_matrix, scores_matrix, 0.05)
print
print 'MAP:', MAP
print 'MRR:', MRR
print 'MP@1:', MPK1
print 'MP@5:', MPK5
print 'MAUC:', MAUC
print 'AUC(0.05):', AUC05
print
return MAP, MRR, MPK1, MPK5, MAUC, AUC05
def message_handle(client, task_queue, task_lock, response_queue):
while True:
try:
size_byte = client.recv(SIZE_CNT)
if (len(size_byte) == 0):
break
size = decode_size(size_byte)
packet_size = 1024
task_byte = b''
while size > packet_size:
data = client.recv(packet_size)
task_byte += data
size -= packet_size
task_byte += client.recv(size)
task = decode_data(task_byte)
name = task[0]
with task_lock:
while not response_queue.empty():
response_queue.get_nowait()
task_queue.put(task)
res_name, resp = response_queue.get(True)
assert (res_name == name)
size_byte, resp_byte = encode(resp)
client.send(size_byte)
client.send(resp_byte)
except:
return
def recv_data(s, name, bufname, n, head, p = False):
at = AvgTime()
print(bufname)
buf = Buffer(bufname)
idx = 0
node = head
s.settimeout(0)
while idx < n:
now = time.time()
if idx%128 == 0:
size_byte, data_byte = encode((name,2))
s.send(size_byte)
s.send(data_byte)
next_node = buf.get_next(node)
while next_node == -1:
# time.sleep(0.001)
next_node = buf.get_next(node)
# print(name,idx, "read from", node)
node = next_node
data = buf.read(node)
# print("read", len(buf.read(node)), time.time()-now)
# print(node)
at.add(time.time()-now)
idx += 1
print("[%s,%d]avg time %f"%(name, n, at.avg()))
def execute(self, cmd, callback=None):
print cmd
if cmd[0] in ["subscribe", "psubscribe", "unsubscribe", "punsubscribe"]:
for i in cmd[1:]:
self._pending.append(callback)
else:
raise RedisError("command not supported")
self.conn.stream.write(encode(cmd))
def execute(self, cmd, callback=None):
print cmd
if cmd[0] in [
"subscribe", "psubscribe", "unsubscribe", "punsubscribe"
]:
for i in cmd[1:]:
self._pending.append(callback)
else:
raise RedisError("command not supported")
self.conn.stream.write(encode(cmd))
def __call__(self, data):
self.min_key = 1e90
def get_vals():
for key, value in data:
self.min_key = min(self.min_key, key)
print >>sys.stderr, "key", key
print >>sys.stderr, value[0:10], value.split("\t", 1)[1][0:10]
yield value
tmp_handle = open(self.tmp_file, "w")
for l in decode(get_vals()):
print >>sys.stderr, "print"
print >> tmp_handle, l
tmp_handle.close()
# Oracle-ize the forst
print >>sys.stderr, os.getenv("PYTHONPATH")
self.shell.call(
"export PYTHONPATH=%s;cd $TRANSFOREST; cat $TMP_FILE | $PYTHON $TRANSFOREST/Features/oracle.py -w $TRANSFOREST/example/config.ini --lm $TRANSFOREST/example/lm.3.sri --order 3 $LOCAL_TMPDIR/oracle"
% os.getenv("PPATH")
)
# Add Features to oracle forst
self.shell.call(
"export PYTHONPATH=%s;cd $TRANSFOREST; $PYTHON $TRANSFOREST/Features/add_features.py $LOCAL_TMPDIR/oracle $LOCAL_TMPDIR/oracle_features"
% os.getenv("PPATH")
)
# Add features to the main sentence
self.shell.call(
"export PYTHONPATH=%s;cd $TRANSFOREST; $PYTHON $TRANSFOREST/Features/add_features.py $TMP_FILE $LOCAL_TMPDIR/features"
% os.getenv("PPATH")
)
for i, (l1, l2) in enumerate(
izip(
encode(self.shell.open("$LOCAL_TMPDIR/features")),
encode(self.shell.open("$LOCAL_TMPDIR/oracle_features")),
)
):
yield int(l.split("\t", 1)[0]), l1 + "****" + l2
def send_idx(name, n, s):
idx = list(range(n))
size_byte, data_byte = encode((name,0,idx))
s.send(size_byte)
s.sendall(data_byte)
size_byte = s.recv(SIZE_CNT)
size = decode_size(size_byte)
resp_byte = s.recv(size)
resp = decode_data(resp_byte)
return resp
def execute(self, cmd, callback=None):
with _manager.connect(self.address) as conn:
yield gen.Task(conn.stream.write, encode(cmd))
s = yield gen.Task(conn.stream.read_until, "\r\n")
g = decode(s[:-2])
reply = g.next()
while True:
if isinstance(reply, Reply):
break
s = yield gen.Task(conn.stream.read_until, "\r\n")
reply = g.send(s[:-2])
if callback:
callback(reply)
def execute(self, cmd, callback=None):
with _manager.connect(self.address) as conn:
yield gen.Task(conn.stream.write, encode(cmd))
s = yield gen.Task(conn.stream.read_until, "\r\n")
g = decode(s[:-2])
reply = g.next()
while True:
if isinstance(reply, Reply):
break
s = yield gen.Task(conn.stream.read_until, "\r\n")
reply = g.send(s[:-2])
if callback:
callback(reply)
def predict(input):
testX = [sentenceToIndex(input)]
testX = pad_sequences(testX, maxlen=100, value=0.)
y = model.predict(testX)
y_ = np.argmax(y, axis=1)
confidence = y[0][y_]
y_ = [mapNumberToIntent(z) for z in y_]
print('Intent: {}'.format(y_))
print('Confidence level: {}'.format(confidence))
return {
'intent': y_[0],
'confidence': encode(confidence[0])
}
def run_test(data, code_type, count_results):
result = {'True': [], 'False': []}
if code_type == 'encode':
for pair in data:
res = encode(pair[0])
if res == pair[1]:
lst = result['True']
lst.append(pair)
result['True'] = lst
else:
lst = result['False']
lst.append(pair)
result['False'] = lst
elif code_type == 'decode':
for pair in data:
res = decode(pair[1])
if res == pair[0]:
lst = result['True']
lst.append(pair)
result['True'] = lst
else:
lst = result['False']
lst.append(pair)
result['False'] = lst
if count_results == True:
count_true = result['True']
count_true = len(count_true)
result['True'] = count_true
count_false = result['False']
count_false = len(count_false)
result['False'] = count_false
return [code_type, result]
return [code_type, result]
def __call__(self, data):
ref_handle = open(self.tmp_ref, 'w')
parse_handle = open(self.tmp_parse, 'w')
min_key = 1e90
for key, value in data:
(parse, ref) = value.strip().split("\t")
print >>parse_handle, parse
print >>ref_handle, ref
min_key = min(min_key, key)
ref_handle.close()
parse_handle.close()
# Follow README file
# 1) convert a parse tree to a trivial parse forest
self.shell.call("cat $PARSE_FILE | $PYTHON $P_DIR/tree.py --toforest > $PFOREST")
# 2.1) filter the large rule set and output the small rule set, which is only used in current parse forest.
self.shell.call("cat $PFOREST | $PYTHON $P_DIR/forest.py --rulefilter $RULE_FILE -w $P_DIR/example/config.ini --max_height 3 > $LOCAL_TMPDIR/rules_count2_rhs50")
# SKIP assume it is done previously
# 2.2) filter the count=1 rules
#self.shell.call("grep -v \"count1=1 \" $LOCAL_TMPDIR/rules >$LOCAL_TMPDIR/rules_count2")
# 2.3) filter max(lhs)<=k
#self.shell.call("cat $LOCAL_TMPDIR/rules_count2 | $PYTHON $P_DIR/rulefilter.py 50 > $LOCAL_TMPDIR/rules_count2_rhs50")
# 2.4) convert a parse forest into a translation forest. NOTE -w won't be used
self.shell.call("cat $PFOREST | $PYTHON $P_DIR/forest.py -r $LOCAL_TMPDIR/rules_count2_rhs50 --max_height 3 -w \"gt_prob=-1\" $REF_FILE 1> $LOCAL_TMPDIR/processed.tforest")
# 3) prune a translation forest
self.shell.call("cat $LOCAL_TMPDIR/processed.tforest | $PYTHON $P_DIR/prune.py --lm $P_DIR/example/lm.3.sri -r15 -w $P_DIR/example/config.ini > $LOCAL_TMPDIR/processed.pruned.tforest")
#output without last \n
for i,l in enumerate(encode(self.shell.open("$LOCAL_TMPDIR/processed.pruned.tforest"))):
yield int(min_key), l
import sys
from encode import *
from decode import *
if __name__ == '__main__':
result = ''
if sys.argv[1] == 'encode':
result = encode(sys.argv[2])
else:
result = decode(sys.argv[2])
print(result)
#!/usr/bin/env/python
import sys
from optparse import OptionParser
from extract import *
from encode import *
extract = extract()
encode = encode()
# Main function called to begin execution
def main(options):
if options.extract is True:
if extract.extractFile(options.inputFile) != -1:
exit("Extracting complete!")
else:
exit("Error upon extraction")
else:
encode.encodeFile(options.inputFile, options.metaFile, options.gifFile, options.outputFile)
exit("Encoding complete!")
# Exits the program
def fail(message):
if message is not None:
print message
sys.exit()
parser = OptionParser()
parser.add_option("-e", "--encode", action="store_false", dest="extract", default=False, help="Flag set to encode the input file into destination")
parser.add_option("-g", "--gif", dest="gifFile", metavar="GIF", help="The gif to encode into")
def encodeNTimes(input, n):
for i in range(n):
input = encode(input)
# print(input)
return input
#对图片进行缩放
original_change = original
if width > change_size:
#缩小
original_change = cv.resize(original_change,
(change_size, change_size),
interpolation=cv.INTER_AREA)
elif width < change_size:
#放大
original_change = cv.resize(original_change,
(change_size, change_size),
interpolation=cv.INTER_CUBIC)
#编码
img_no_location = encode(original_change, message, k, Lambda, encode_type)
#添加定位图案
image_result = add_location(img_no_location, k)
if image_result.shape[0] > original_size:
#缩小
image_result = cv.resize(image_result, (original_size, original_size),
interpolation=cv.INTER_AREA)
elif image_result.shape[0] < original_size:
#放大
image_result = cv.resize(image_result, (original_size, original_size),
interpolation=cv.INTER_CUBIC)
#保存
cv.imwrite("./result.jpg", image_result)
>>> a.encode('base64','strict')
Traceback (most recent call last):
File "<pyshell#129>", line 1, in <module>
a.encode('base64','strict')
LookupError: 'base64' is not a text encoding; use codecs.encode() to handle arbitrary codecs
>>> from codecs import encode, decode
>>> d=a.encode('base64','strict')
Traceback (most recent call last):
File "<pyshell#131>", line 1, in <module>
d=a.encode('base64','strict')
LookupError: 'base64' is not a text encoding; use codecs.encode() to handle arbitrary codecs
>>> d=a.encode(hex)
Traceback (most recent call last):
File "<pyshell#132>", line 1, in <module>
d=a.encode(hex)
TypeError: encode() argument 1 must be str, not builtin_function_or_method
>>> d=a.encode('hex')
Traceback (most recent call last):
File "<pyshell#133>", line 1, in <module>
d=a.encode('hex')
LookupError: 'hex' is not a text encoding; use codecs.encode() to handle arbitrary codecs
>>> d=encode('a','hex')
Traceback (most recent call last):
File "C:\Python34\lib\encodings\hex_codec.py", line 15, in hex_encode
return (binascii.b2a_hex(input), len(input))
TypeError: 'str' does not support the buffer interface
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "<pyshell#134>", line 1, in <module>
def encode(in_str, ID):
if not isinstance(in_str, bytes):
ord_str = [ord(l) for l in in_str]
else:
ord_str = [int(x) for x in in_str]
b3_str = "".join(a2b3(ord_str))
print(b3_str)
dna_str = build_initial_dna_string2(b3_str)
#split_str = split_and_index(dna_str)
#out_str = polish_ends(add_parity_trit(split_str, ID))
return dna_str
dna_str = encode(test_str, "12")
dna0 = dna_str[:100]
dna2base4 = str.maketrans({'A': '0', 'C': '1', 'G': '2', 'T': '3'})
base4_2_dna = str.maketrans({'0': 'A', '1': 'C', '2': 'G', '3': 'T'})
print(dna0)
base4_0 = [int(x) for x in dna0.translate(dna2base4)]
temp = list()
temp2 = list()
kstrits = [int(x) for x in keystream[0]]
print(len(base4_0))
print(len(kstrits))
L = len(kstrits)
# this code adapted from the author's perl script, it did not translate well, but it works
# need to change keystream based on chunk_id tho
def delete(name, s):
size_byte, data_byte = encode((name,-1))
s.send(size_byte)
s.send(data_byte)
resp = s.recv(1)
return resp
