def test255_13(self):
coder = rs.RSCoder(255, 13)
m = "Hello, world!"
code = coder.encode(m)
self.assertTrue(coder.check(code))
self.assertEqual(m, coder.decode(code)[0])
self.assertEqual(255, len(code))
# Change 121 bytes. This code should tolerate up to 121 bytes changed
changes = [
1, 4, 5, 6, 9, 10, 14, 15, 19, 20, 21, 24, 26, 30, 32, 34, 38, 39,
40, 42, 43, 44, 45, 47, 49, 50, 53, 59, 60, 62, 65, 67, 68, 69, 71,
73, 74, 79, 80, 81, 85, 89, 90, 93, 94, 95, 100, 101, 105, 106,
107, 110, 112, 117, 120, 121, 123, 126, 127, 132, 133, 135, 136,
138, 143, 149, 150, 152, 154, 158, 159, 161, 162, 163, 165, 166,
168, 169, 170, 174, 176, 177, 178, 179, 182, 186, 191, 192, 193,
196, 197, 198, 200, 203, 206, 208, 209, 210, 211, 212, 216, 219,
222, 224, 225, 226, 228, 230, 232, 234, 235, 237, 238, 240, 242,
244, 245, 248, 249, 250, 253
]
c = list(ord(x) for x in code)
for pos in changes:
c[pos] = (c[pos] + 50) % 255
c = "".join(chr(x) for x in c)
decode, _ = coder.decode(c)
self.assertEqual(m, decode)
def _threaded_coder(self, message, to_encode):
blocks = self._chunk(message, to_encode)
to_code = [(i, to_encode, block) for i, block in enumerate(blocks)]
num_blocks = int(math.ceil(len(to_code) / (1. * self.num_threads)))
to_code_list = [
to_code[i:i + num_blocks]
for i in range(0, len(to_code), num_blocks)
]
coded_queue = Queue.Queue()
threads = []
for i in range(FLAGS.threads):
new_encoder = rs.RSCoder(self.codeword_length,
self.message_byte_length)
threads.append(
ECCodeRunner(new_encoder, to_code_list[i], coded_queue))
[t.start() for t in threads]
[t.join() for t in threads]
coded_list = []
# print coded_queue
while True:
try:
dequeued = coded_queue.get_nowait()
except:
break
coded_list.append(dequeued)
coded_list.sort()
thread_coded = ''.join([e[1] for e in coded_list])
return thread_coded
def rs_fix(data):
data = data[4:]
def unterleave(intLev):
output = []
for i in xrange(intLev, len(data), 4):
output.append(data[i])
return output
rsc = rs.RSCoder(255, 223)
print "DATA = "
print hexstr(data)
for il in xrange(4):
#print "Data from %d to %d:" % ( 4 + il * 223, 228 + il * 223 )
#d = data[4+il*223:4+il*223+223]
d = unterleave(il)
print "Unterleved data"
print hexstr(d)
#print "Parity from %d to %d:" % ( 896 + il * 32, 928 + il * 32 )
p = d[223:]
print "Encoded string would be:"
temp = d[:223]
print hexstr(rsc.encode("".join(map(chr, temp))))
print "Parity:"
print hexstr(p)
raw = "".join(map(chr, d))
fixed = rsc.decode(str(raw))
if raw != fixed:
print "Fixed error!"
def setUp(self):
self.coder = rs.RSCoder(255, 223)
self.string = "Hello, world! This is a long string"
codestr = self.coder.encode(self.string)
self.code = codestr
def encode(input, output_filename):
#print privkey
#signa=rsa.sign(r,privkey,'SHA-1')
coder = rs.RSCoder(255, 223)
#sig='2'*200+signa[-55:]
#signa_hex=repr(signa)
output.append(signa)
#print "hello"
#a=1
block = 1
count = 1
while block:
block = input.read(223)
#print "hello"
#if not block: break
code = coder.encode(block)
"""if count==1:
code=code[:250]+'\0'*5
count+=1"""
output.append(code)
sys.stderr.write(".")
sys.stderr.write("\n")
n = len(output)
print n
n = n * 255
si(n)
shuf1(a, n, output_filename)
out = Image.new("L", (rowstride, len(output)))
out.putdata("".join(output))
out.save('monk-r.png')
def encode(input, output_filename):
"""Encodes the input data with reed-solomon error correction in 223 byte
blocks, and outputs each block along with 32 parity bytes to a new file by
the given filename.
input is a file-like object
The outputted image will be in png format, and will be 255 by x pixels with
one color channel. X is the number of 255 byte blocks from the input. Each
block of data will be one row, therefore, the data can be recovered if no
more than 16 pixels per row are altered.
"""
coder = rs.RSCoder(255,223)
output = []
while True:
block = input.read(223)
if not block: break
code = coder.encode(block)
output.append(code)
sys.stderr.write(".")
sys.stderr.write("\n")
out = Image.new("L", (rowstride,len(output)))
out.putdata("".join(output))
out.save(output_filename)
def test_synd_shift(self):
'''Test if syndrome computation do not trim automatically the 0 coefficients. This is an edge case (that sometimes the syndromes generate leading 0 coefficients), but it's very important that we don't trim them (at least for the syndromes polynomial) because we need to know the full length of the syndromes polynomial to calculate the correct syndrome shift in Berlekamp-Massey'''
orig_mes = bytearray("hello world")
n = len(orig_mes) * 2
k = len(orig_mes)
fcr = 1
prim = 0xfd
generator = 3
c_exponent = 7
erasenb = 5
rsman = rs.RSCoder(n,
k,
fcr=fcr,
prim=prim,
generator=generator,
c_exp=c_exponent)
rmesecc = bytearray(rsman.encode(orig_mes, k=k))
rmesecc[-erasenb:] = "\x00" * erasenb
rmesecc_poly = rsman._list2gfpoly(rmesecc)
# Compute the syndromes and check that 0 coefficients aren't removed
synd = rsman._syndromes(rmesecc_poly, k=k)
self.assertEqual([int(x) for x in synd],
[0, 75, 112, 28, 95, 77, 73, 113, 22, 29, 100, 0
]) # ground truth comparison
self.assertEqual(
len(synd), 12
) # the syndrome's length should be 12, if it's less (because it was trimmed of the leading non significant 0 coefficient), then can't calculate the syndrome shift in BM
def test_small_k(self):
coder = rs.RSCoder(5, 2)
mes = [140, 128]
ecc_good = [182, 242, 0]
messtr = "".join(chr(x) for x in mes)
self.assertEqual(
hashlib.md5(messtr).hexdigest(),
"8052809d008df30342a22e7910d05600")
mesandecc = coder.encode(messtr)
mesandeccstr = [ord(x) for x in mesandecc]
if mesandeccstr == [140, 54, 199, 92, 175]:
print(
"Error in polynomial __divmod__ (probably in the stopping criterion)."
)
self.assertEqual(mesandeccstr, mes + ecc_good)
def decode(input_filename):
coder = rs.RSCoder(255,223)
input = Image.open(input_filename)
data = "".join(chr(x) for x in input.getdata())
del input
blocknum = 0
while True:
if blocknum*255 > len(data):
break
rowdata = data[blocknum*255:(blocknum+1)*255]
decoded = coder.decode(rowdata)
blocknum += 1
sys.stdout.write(str(decoded))
sys.stderr.write(".")
sys.stderr.write("\n")
def test_decode_above_bound(self):
'''Particular case where a message with too many errors is wrongly decoded (as expected), but the check methods won't report that the decoding failed!
Maybe we cannot do anything about it (because the decoded message is formally correct and thus it's one of the peculiar cases where RS can't know it even failed)
or maybe we can implement other checking methods like the ones described in the works of Dilip Sarwate:
http://www.dsprelated.com/showthread/comp.dsp/15148-1.php
http://www.ifp.uiuc.edu/~sarwate/malfunction.ps and
http://www.ifp.uiuc.edu/~sarwate/pubs/Sarwate90Decoder.pdf
'''
mes = bytearray('hello world')
n = len(mes) * 2
k = len(mes)
erase_char = "\x00"
erasenb = 12 # total number of erratas
errnb = 1 # total number of errors
istart = 0
self.assertTrue(
erasenb - errnb + errnb * 2 > (n - k)
) # check if we are outside of the bound (too many errors), this should be true in this case because we have 2e+v=2+11=13 > n-k = 11.
# init the codec
rsman = rs.RSCoder(
n, k
) # every set of parameters for generator/prim/fcr will give the same issue (even if the output is different, but in the end, the message won't be decoded correctly but check won't detect the error)
# encode the message
rmesecc = bytearray(rsman.encode(mes, k=k))
rmesecc_orig = bytearray(
rmesecc) # copy the original untampered message for later checking
# tamper the message
rmesecc[istart:istart +
erasenb] = erase_char * erasenb # introduce erratas
erase_pos = [
x for x in xrange(len(rmesecc)) if rmesecc[x] == ord(erase_char)
] # compute erasures positions
if errnb > 0:
erase_pos = erase_pos[:
-errnb] # remove the errors positions (so we only have erasures)
# decode the message
rmes, recc = rsman.decode(rmesecc, k=k, erasures_pos=erase_pos)
# This check, comparing the original message with the decoded message, clearly shows that the decoding failed
self.assertFalse((rmes + recc) == rmesecc)
# But the check methods won't see any problem!
self.assertTrue(rsman.check(rmes + recc, k=k))
self.assertTrue(rsman.check_fast(rmes + recc, k=k))
def test30_10(self):
'''Tests the RS(30,10) code'''
coder = rs.RSCoder(30, 10)
m = "Hello, wor"
code = coder.encode(m)
self.assertTrue(coder.check(code))
self.assertEqual(m, coder.decode(code)[0])
self.assertEqual(30, len(code))
# Change 10 bytes. This code should tolerate up to 10 bytes changed
changes = [0, 1, 2, 4, 7, 10, 14, 18, 22, 27]
c = list(ord(x) for x in code)
for pos in changes:
c[pos] = (c[pos] + 50) % 255
c = "".join(chr(x) for x in c)
decode, _ = coder.decode(c)
self.assertEqual(m, decode)
def test_fast_encode(self):
'''Test if the fast encoding method is correct'''
coder = rs.RSCoder(255, 180)
mes = 'hello world'
mesecc = coder.encode_fast(mes, k=180).lstrip("\0")
mesecclist = [ord(x) for x in mesecc]
# Compare to ground truth
good_res = [
104, 101, 108, 108, 111, 32, 119, 111, 114, 108, 100, 52, 234, 152,
75, 39, 171, 122, 196, 96, 245, 151, 167, 164, 13, 207, 148, 5,
112, 192, 124, 46, 134, 198, 32, 49, 75, 204, 217, 71, 148, 43, 66,
94, 210, 201, 128, 80, 185, 30, 219, 33, 53, 174, 183, 121, 191,
69, 203, 2, 206, 194, 109, 221, 51, 207, 4, 129, 37, 255, 237, 174,
104, 199, 28, 33, 90, 10, 74, 125, 113, 70, 59, 150, 197, 157
]
self.assertEqual(mesecclist, good_res)
# Compare with the normal encoding method
nmesecc = coder.encode_fast(mes, k=180).lstrip("\0")
nmesecclist = [ord(x) for x in nmesecc]
self.assertEqual(mesecclist, nmesecclist)
def test30_10(self):
"""Tests the RS(30,10) code"""
coder = rs.RSCoder(30, 10)
m = "01010101"
code = coder.encode(m)
print m
self.assertTrue(coder.verify(code))
self.assertEqual(m, coder.decode(code))
self.assertEqual(30, len(code))
# Change 10 bytes. This code should tolerate up to 10 bytes changed
changes = [0, 1, 2, 4, 7, 10, 14, 18, 22, 27]
c = list(ord(x) for x in code)
for pos in changes:
c[pos] = (c[pos] + 50) % 255
c = "".join(chr(x) for x in c)
decode = coder.decode(c)
self.assertEqual(m, decode)
print c
print decode
def decode(input_filename):
data1 = []
input = Image.open(input_filename)
data = "".join(chr(x) for x in input.getdata())
del input
blocknum = 0
while True:
if blocknum * 255 >= len(data):
break
rowdata = data[blocknum * 255:(blocknum + 1) * 255]
#sys.stderr.write('\nrowdata is------> '+str(rowdata))
blocknum += 1
#sys.stdout.write(str(decoded))
#sys.stderr.write(".")
data1.append(rowdata)
n = len(data1) * 255
sys.stderr.write('\n' + str(len(data1)))
"""f=open("n.txt","w")
f.write(str(data1))
f.close()"""
si(n)
perm_index_a = shuf2(a, n)
sys.stderr.write('\nlength of perm_index in decode--> ' +
str(len(perm_index_a)))
data1_break = []
for i in data1:
for j in i:
data1_break.append(j)
#sys.stderr.write('\n'+str(data1_break))
#sys.stderr.write('\nlength of data1_break--> '+str(len(data1_break)))
for z in range(0, n):
decode_shuf.insert(z, data1_break[perm_index_a[z]])
#print "decoded array-----------------"
sys.stderr.write('\nlength of decode_shuf--> ' + str(len(decode_shuf)))
final_decrypt = []
blocknum = 0
while True:
if blocknum * 255 >= len(decode_shuf):
break
rowdata = decode_shuf[blocknum * 255:(blocknum + 1) * 255]
#sys.stderr.write('\nrowdata is------> '+str(rowdata))
x = "".join(rowdata)
blocknum += 1
final_decrypt.append(x)
#sys.stderr.write(x)
#sys.stderr.write('\nlength of x is------>'+str(len(x)))
#break
sys.stderr.write('\nlength of final_decrypt is------> ' +
str(len(final_decrypt)))
coder = rs.RSCoder(255, 223)
sign = final_decrypt[0]
t = open("original_signa_decoded.txt", "w")
t.write(str(sign))
t.close()
#sys.stderr.write(str(sign))
data = final_decrypt[1:]
#sys.stderr.write(str(data))
for i in data:
decoded = coder.decode(i)
sys.stdout.write(str(decoded))
sys.stderr.write(".")
#f=open("chksign.txt","w")
sys.stderr.write("\n")
#u.close()"""
"""val=rsa.verify(r,sign,pubkey)
def __init__(self, n, k, num_threads=1):
self.codeword_length = n
self.message_byte_length = k
self.num_threads = num_threads
self.coder = rs.RSCoder(self.codeword_length, self.message_byte_length)
import ff16, ff64, ff125, ff343, ff125, ff256, ff3481, rs
import itertools, string
### fix barcode
barcode_rs_coder = rs.RSCoder(GFint=ff16.GFint, k=7, n=8)
ff16_trantab = {
''.join(vv): i
for i, vv in enumerate(itertools.product('ACGT', 'ACGT'))
}
ff16_rev_trantab = {i: vv for vv, i in ff16_trantab.items()}
def barcode_rs(barcode):
message = barcode[2:]
message_int = [
ff16_trantab[''.join(message[i:i + 2])] for i in range(0, 14, 2)
]
codeword = barcode_rs_coder.encode(message_int)
coded_barcode = [vv for v in codeword for vv in ff16_rev_trantab[v]]
return coded_barcode
def barcode_rs_decode(barcode, verify_only=True):
received_int = [
ff16_trantab[''.join(barcode[i:i + 2])] for i in range(0, 16, 2)
]
if barcode_rs_coder.verify(received_int):
return ['A', 'A'] + barcode[0:14]
elif not verify_only:
decoded_int = barcode_rs_coder.decode(received_int)
decoded_message = [
def setUp(self):
self.coder = rs.RSCoder(255, 223)
def test_main(self):
def cartesian_product_dict_items(dicts):
return (dict(itertools.izip(dicts, x))
for x in itertools.product(*dicts.itervalues()))
debugg = False # if one or more tests don't pass, you can enable this flag to True to get verbose output to debug
orig_mes = bytearray("hello world")
n = len(orig_mes) * 2
k = len(orig_mes)
nsym = n - k
istart = 0
params = {
"count": 5,
"fcr": [120, 0, 1, 1, 1],
"prim": [0x187, 0x11d, 0x11b, 0xfd, 0xfd],
"generator": [2, 2, 3, 3, 2],
"c_exponent": [8, 8, 8, 7, 7],
}
cases = {
"errmode": [1, 2, 3, 4],
"erratasnb_errorsnb_onlyeras":
[[8, 3, False], [6, 5, False], [5, 5, False], [11, 0, True],
[11, 0, False], [0, 0, False]
], # errata number (errors+erasures), erasures number and only_erasures: the last item is the value for only_erasures (True/False)
}
############################$
results_br = []
results_rs = []
it = 0
for p in xrange(params["count"]):
fcr = params["fcr"][p]
prim = params["prim"][p]
generator = params["generator"][p]
c_exponent = params["c_exponent"][p]
for case in cartesian_product_dict_items(cases):
errmode = case["errmode"]
erratanb = case["erratasnb_errorsnb_onlyeras"][0]
errnb = case["erratasnb_errorsnb_onlyeras"][1]
only_erasures = case["erratasnb_errorsnb_onlyeras"][2]
it += 1
if debugg:
print "it ", it
print "param", p
print case
# BROWNANRS
# Init the RS codec
rsman = rs.RSCoder(n,
k,
fcr=fcr,
prim=prim,
generator=generator,
c_exp=c_exponent)
# Encode the message
rmesecc = bytearray(rsman.encode(orig_mes, k=k))
rmesecc_orig = bytearray(
rmesecc
) # make a copy of the original message to check later if fully corrected (because the syndrome may be wrong sometimes)
# Tamper the message
if erratanb > 0:
if errmode == 1:
sl = slice(istart, istart + erratanb)
elif errmode == 2:
sl = slice(-istart - erratanb - (n - k), -(n - k))
elif errmode == 3:
sl = slice(-istart - erratanb - 1, -1)
elif errmode == 4:
sl = slice(-istart - erratanb, None)
if debugg:
print "Removed slice:", list(rmesecc[sl]), rmesecc[sl]
rmesecc[sl] = "\x00" * erratanb
# Generate the erasures positions (if any)
erase_pos = [
x for x in xrange(len(rmesecc)) if rmesecc[x] == 0
]
if errnb > 0:
erase_pos = erase_pos[:
-errnb] # remove the errors positions (must not be known by definition)
if debugg:
print "erase_pos", erase_pos
print "coef_pos", [
len(rmesecc) - 1 - pos for pos in erase_pos
]
print "Errata total: ", erratanb - errnb + errnb * 2, " -- Correctable? ", (
erratanb - errnb + errnb * 2 <= nsym)
# Decoding the corrupted codeword
# -- fast method
try:
rmes, recc = rsman.decode_fast(rmesecc,
k=k,
erasures_pos=erase_pos,
only_erasures=only_erasures)
results_br.append(
rsman.check(rmes + recc, k=k)
) # check if correct by syndrome analysis (can be wrong)
results_br.append(
rmesecc_orig == (rmes + recc)
) # check if correct by comparing to the original message (always correct)
if debugg and not rsman.check(rmes + recc, k=k) or not (
rmesecc_orig == (rmes + recc)):
raise RSCodecError("False!!!!!")
except rs.RSCodecError as exc:
results_br.append(False)
results_br.append(False)
if debugg:
print "===="
print "ERROR! Details:"
print "param", p
print case
print erase_pos
print "original_msg", rmesecc_orig
print "tampered_msg", rmesecc
print "decoded_msg", rmes + recc
print "checks: ", rsman.check(
rmes + recc, k=k), rmesecc_orig == (rmes + recc)
print "===="
raise exc
# -- normal method
try:
rmes, recc = rsman.decode(rmesecc,
k=k,
erasures_pos=erase_pos,
only_erasures=only_erasures)
results_br.append(rsman.check(rmes + recc, k=k))
results_br.append(rmesecc_orig == (rmes + recc))
except rs.RSCodecError as exc:
results_br.append(False)
results_br.append(False)
if debugg: print "-----"
self.assertTrue(results_br.count(True) == len(results_br))
def test_bm(self):
'''Test Berlekamp-Massey decoding and also the analytical equations to calculate directly the erasure/errata locator and evaluator
The trick here is that the errata locator should always be the same, whether it's solely computed from erasures using the analytical equation, or using Berlekamp-Massey, it doesn't matter.
'''
orig_mes = bytearray("hello world")
n = len(orig_mes) * 2
k = len(orig_mes)
fcr = 1
prim = 0x11b
generator = 3
c_exponent = 8
erasenb = 4
############################
# Encode the message and tamper it
rsman = rs.RSCoder(n,
k,
fcr=fcr,
prim=prim,
generator=generator,
c_exp=c_exponent)
rmesecc = bytearray(rsman.encode(orig_mes, k=k))
rmesecc_orig = bytearray(rmesecc)
rmesecc[:erasenb] = "\x00" * erasenb
# Prepare the erasures positions
erasures_pos = [x for x in xrange(len(rmesecc)) if rmesecc[x] == 0]
####
# Start of decoding
rp = rsman._list2gfpoly(rmesecc)
sz = rsman._syndromes(rp, k=k)
self.assertEqual(list(sz),
[206, 253, 1, 78, 232, 253, 96, 190, 77, 125, 20, 0])
coeff_pos = [len(rmesecc) - 1 - x for x in erasures_pos]
erasures_count = len(erasures_pos)
# Compute the erasure locator polynomial
erasures_loc = rsman._find_erasures_locator(coeff_pos)
# Compute the erasure evaluator polynomial
erasures_eval = rsman._find_error_evaluator(sz, erasures_loc, k=k)
# Compute error locator polynomial without providing the erasures
sigma, omega = rsman._berlekamp_massey(sz, k=k)
self.assertEqual(erasures_loc, sigma)
# same but provide the erasures (so normally BM should not modify the locator polynomial, the output should be equal to the erasures_loc in input)
sigma, omega = rsman._berlekamp_massey(sz,
k=k,
erasures_loc=erasures_loc,
erasures_eval=erasures_eval,
erasures_count=erasures_count)
self.assertEqual(erasures_loc, sigma)
# same but with fast BM
sigma, omega = rsman._berlekamp_massey_fast(sz, k=k)
self.assertEqual(erasures_loc, sigma)
sigma, omega = rsman._berlekamp_massey_fast(
sz,
k=k,
erasures_loc=erasures_loc,
erasures_eval=erasures_eval,
erasures_count=erasures_count)
self.assertEqual(erasures_loc, sigma)
# Finally, check against ground truth
sigma_groundtruth = [120, 210, 102, 219, 1]
self.assertEqual(list(sigma), sigma_groundtruth)
# Bonus: check that the evaluator polynomial is the same, whether:
self.assertEqual(
erasures_eval, omega
) # ... it's computed only from the erasures vs computed from the output of BM simultaneously computed with the errata locator polynomial
omega2 = rsman._find_error_evaluator(
sz, sigma,
k=k) # analytical calculation from BM's final errata_locator sigma
self.assertEqual(
omega, omega2
) # ... it's computed simultaneously in BM vs analytically calculated from BM's final errata_locator sigma
