def testCreationFromDataWithOffset(self):
s1 = Bits(bytes=b'\x0b\x1c\x2f', offset=0, length=20)
s2 = Bits(bytes=b'\xa0\xb1\xC2', offset=4)
self.assertEqual((s2.len, s2.hex), (20, '0b1c2'))
self.assertEqual((s1.len, s1.hex), (20, '0b1c2'))
self.assertTrue(s1 == s2)
def bits(self):
if len(self.operands) != 2:
print('BrOp got %i arguments, 2 expected!' % len(self.operands))
if self.operation != 'jcc':
print('BrOp got unknown operation (%s), jcc expected!' % self.operation)
if self.operands[1].type != OP_TYPE_IMM:
print('BrOp needs immediate as second operand type!')
cc = next((cc for cc in k10_cc if cc['mnem'].lower() == self.operands[0].mnem), None)
if cc == None:
print('BrOp first operand needs to be one of k10_cc!')
r = Bits(1) # unkn
r+= Bits(bin='0101') # cond ucode branch
r+= self.operands[0].bits() # cc
r+= Bits(bin='11') # xchg src/dst, 3 op mode
r+= Bits(bin='111001') # always zero register, TODO: verify and add to k10_regs
r+= Bits(bin='101') # branch target rom/ram indicator
r+= self.commitPZSBit # commit p, z and s flag
r+= self.commitCBit # commit c flag
r+= Bits(1) # unkn
r+= Bits(bin='000') # RegOp/SpecOp
r+= Bits(bin='1111') # segment
r+= Bits(bin='011') # size
r+= Bits(bin='111011') # always zero register, TODO: verify and add to k10_regs
r+= Bits(bin='0') # immediate mode
r+= Bits(bin='000000') # unkn
r+= self.operands[1].bits(bitlen=17)
if len(r) != 64:
print('BrOp has size of %i bit, 64 bit expected!' % len(r))
return r
def bits(self):
if len(self.operands) != 2:
print('LdStOp got %i arguments, 2 expected!' % len(self.operands))
is_load_op = False
if self.operation == 'ld':
is_load_op = True
elif self.operation == 'st':
is_load_op = False
else:
print('LdStOp got unknown operation (%s), ld or st expected!' % self.operation)
if is_load_op:
if self.operands[0].type != OP_TYPE_REG:
print('Load needs register as first operand type!')
if self.operands[1].type != OP_TYPE_MEM:
print('Load needs memory as second operand type!')
else:
if self.operands[0].type != OP_TYPE_MEM:
print('Store needs memory as first operand type!')
if self.operands[1].type != OP_TYPE_REG:
print('Store needs register as second operand type!')
r = Bits(1) # unkn
r+= self.operation_bits() # operation
if is_load_op:
r+= Bits(bin='1') # swap src and dst
else:
r+= Bits(bin='0') # swap src and dst
r+= Bits(1) # 2/3 operand mode
if is_load_op:
r+= self.operands[1].bits() # memory type src
r+= Bits(bin='000') # unknown
else:
r+= self.operands[0].bits() # memory type dst
r+= Bits(bin='100') # unknown
r+= self.commitPZSBit # commit p, z and s flag
r+= self.commitCBit # commit c flag
r+= Bits(1) # unkn
if is_load_op:
r+= Bits(bin='001') # LdStOp (load)
else:
r+= Bits(bin='010') # LdStOp (store)
if is_load_op:
r+= self.operands[1].seg_bits() # memory type src segment reg
else:
r+= self.operands[0].seg_bits() # memory type dst segment reg
r+= self.sizeMSB
r+= Bits(uint=int(math.log(self.operands[1].size, 2) - 3), length=2) # size
r+= Bits(bin='111111') # always zero register, TODO: verify and add to k10_regs
r+= Bits(bin='1') # register mode
r+= Bits(7) # unkn
r+= Bits(1) # unkn
if is_load_op:
r+= self.operands[0].bits() # register type dst
else:
r+= self.operands[1].bits() # register type src
r+= Bits(9) # unkn
if len(r) != 64:
print('LdStOp has size of %i bit, 64 bit expected!' % len(r))
return r
def test_float_container(self):
ci = FloatContainerItem(3.1415)
self.assertEqual(ci.prepare_bitstream(),
Bits(b'\4\0\1\0\0\0\0\0\0\0\0\0\0\0\0\0') + pack('floatle:32', 3.1415)
)
self.assertEqual(ci.get_all_pointers(), [])
def sample_sequences(positions, buildname, basedir, options):
"""
"""
rpt_err = options.rpt_err
gc_err = options.gc_err
max_trys = options.max_trys
norpt = options.norpt
nogc = options.nogc
chrnames = sorted(set(map(lambda p: p[0], positions)))
profiles = make_profile(positions, buildname, basedir)
excluded = []
if options.skipfile:
excluded = read_bed_file(options.skipfile, chr)
f = open(options.output,"w")
for chrom in chrnames:
print chrom
idxf_na = os.path.join(basedir, '.'.join([buildname, chrom, 'na', 'out']))
idxf_gc = os.path.join(basedir, '.'.join([buildname, chrom, 'gc', 'out']))
idxf_rpt = os.path.join(basedir, '.'.join([buildname, chrom, 'rpt', 'out']))
bits_gc = Bits(filename=idxf_gc)
bits_rpt = Bits(filename=idxf_rpt)
#this bit array is used to mark positions that are excluded from sampling
#this will be updated as we sample more sequences in order to prevent sampled sequences from overlapping
bits_na = BitArray(filename=idxf_na)
#mark excluded regions
for pos in excluded:
if pos[0] != chrom:
continue
bits_na.set(True, range(pos[1], pos[2]))
npos+=1
npos = 0
#mark positive regions
for pos in positions:
if pos[0] != chrom:
continue
bits_na.set(True, range(pos[1], pos[2]))
npos+=1
if options.count == 0:
count = options.fold*npos
sampled_cnt = 0
while sampled_cnt < count:
sampled_prof = random.choice(profiles)
sampled_len = sampled_prof[1]
sampled_gc = sampled_prof[2]
sampled_rpt = sampled_prof[3]
rpt_err_allowed = int(rpt_err*sampled_len)
gc_err_allowed = int(gc_err*sampled_len)
trys = 0
while trys < max_trys:
trys += 1
pos = random.randint(1, bits_na.length - sampled_len)
pos_e = pos+sampled_len
#if bits_na.any(True, range(pos, pos_e)):
# continue
if bits_na[pos:pos_e].count(True) > 0:
continue
if not norpt:
pos_rpt = bits_rpt[pos:pos_e].count(True)
if abs(sampled_rpt - pos_rpt) > rpt_err_allowed:
continue
if not nogc:
pos_gc = bits_gc[pos:pos_e].count(True)
if abs(sampled_gc - pos_gc) > gc_err_allowed:
continue
#accept the sampled position
#mark the sampled regions
bits_na.set(True, range(pos, pos_e))
f.write('\t'.join([chrom, str(pos), str(pos_e)]) + '\n')
sampled_cnt += 1
print trys, chrom, pos, pos_e, sampled_len, pos_rpt, sampled_rpt, pos_gc, sampled_gc
break
else:
print "maximum trys reached"
f.close()
def twocomplements(val):
"""
2dml qhtncjfl translate to korean
"""
s = Bits(unit=val, length=16)
return s.int # signed 2's complement representation
def test_int_container(self):
ci = IntContainerItem(0xbaadf00d)
self.assertEqual(ci.prepare_bitstream(), Bits(b'\2\0\1\0\0\0\0\0\0\0\0\0\0\0\0\0\x0d\xf0\xad\xba'))
self.assertEqual(ci.get_all_pointers(), [])
def testCreationFromOctErrors(self):
s = Bits('0b00011')
self.assertRaises(bitstring.InterpretError, s._getoct)
self.assertRaises(bitstring.CreationError, s._setoct, '8')
def testCreationFromSe(self):
for i in range(-100, 10):
s = Bits(se=i)
self.assertEqual(s.se, i)
def testCreationFromHexWithWhitespace(self):
s = Bits(hex=' \n0 X a 4e \r3 \n')
self.assertEqual(s.hex, 'a4e3')
def testCreationFromBinWithWhitespace(self):
s = Bits(bin=' \r\r\n0 B 00 1 1 \t0 ')
self.assertEqual(s.bin, '00110')
def testCreationFromHex(self):
s = Bits(hex='0xA0ff')
self.assertEqual((s.len, s.hex), (16, 'a0ff'))
s = Bits(hex='0x0x0X')
self.assertEqual((s.length, s.hex), (0, ''))
def setUp(self):
self.a = Bits(filename='smalltestfile')
self.b = Bits(filename='smalltestfile', offset=16)
self.c = Bits(filename='smalltestfile', offset=20, length=16)
self.d = Bits(filename='smalltestfile', offset=20, length=4)
def testErrors(self):
for f in ['sie=100, 0b1001', '0b00', 'uie=100, 0b1001']:
s = Bits(f)
self.assertRaises(bitstring.InterpretError, s._getsie)
self.assertRaises(bitstring.InterpretError, s._getuie)
self.assertRaises(ValueError, Bits, 'uie=-10')
def calculate(self, field):
value = field.render()
digest = self.hasher(value.bytes).digest()
return Bits(bytes=digest)
def img_dct(img, q_table, data, lsb_qty=1):
assert q_table.shape == (8, 8), "q_table parameter should be a 8x8 matrix"
# Center values around 0
img = np.subtract(img.astype(int), 128)
# We will work on 8x8 blocks of the image
# Thus, we will pad the image so it is divisible by 8 in both dimensions
m, n = img.shape[:2]
n_channels = img.shape[2] if len(img.shape) > 2 else 1
# Padding image so we can work with a divisible 8x8 image
h_pad = 8 - (m % 8)
v_pad = 8 - (n % 8)
padding = ((0, v_pad),
(0, h_pad)) if n_channels == 1 else ((0, v_pad), (0, h_pad),
(0, 0))
pad_img = np.pad(img, padding, "constant", constant_values=0)
m, n = pad_img.shape[:2]
# preparing data to be embed
data = list(Bits(data).bin)
data.reverse()
G = np.zeros(pad_img.shape)
for ch in range(n_channels):
cur_channel = pad_img if n_channels == 1 else pad_img[:, :, ch]
for i in range(0, m, 8):
for j in range(0, n, 8):
# apply DCT for every 8x8 block
b = cur_channel[i:i + 8, j:j + 8]
b_dct = dctn(b, norm='ortho')
# Quantize using the quantization table provided, rounding values to integers
b_qntz = np.round(np.divide(b_dct, q_table)).astype(int)
# Embeding data
# `data` is a bitarray
if len(data) > 0:
for s, row in enumerate(b_qntz):
if len(data) <= 0: break
for t, c in enumerate(row):
if len(data) <= 0: break
else:
if c != 0 and c != 1:
c_bin = BitArray(int=c, length=8)
lsb = []
for _ in range(lsb_qty):
if len(data) == 0: break
lsb.append(data.pop())
lsb = ''.join(lsb)
c_bin.overwrite(f'0b{lsb}', 8 - len(lsb))
b_qntz[s, t] = c_bin.int
if n_channels == 1:
G[i:i + 8, j:j + 8] = b_qntz
else:
G[i:i + 8, j:j + 8, ch] = b_qntz
return G.astype(int)
def func(data):
return Bits()
def _mutate(self):
new_val = BitArray(self._default_value).copy()
start, end = self._start_end()
new_val.invert(range(start, end))
self.set_current_value(Bits(new_val))
def test_argument_container(self):
ci = ArgumentContainerItem('arg0')
self.assertEqual(ci.prepare_bitstream(), Bits(b'\0\0\1\0\0\0\0\0\0\0\0\0\0\0\0\0\x18\0\0\0\0\0\0\0\4\0arg0\0'))
self.assertEqual(ci.get_all_pointers(), [16])
def read(self):
'''Reads 32 bits of the SPI bus for processing and stores as 32-bit bitstring.'''
raw_spi = self.spi.transaction(Spibus.reading(4))
self.data = Bits(bytes=raw_spi[0], length=32)
self.checkErrors()
def test_bool_container(self):
ci = BoolContainerItem(True)
self.assertEqual(ci.prepare_bitstream(), Bits(b'\3\0\1\0\0\0\0\0\0\0\0\0\0\0\0\0\1\0\0\x80'))
self.assertEqual(ci.get_all_pointers(), [])
def _val_to_bits(conv, val, length):
if val == -1:
return Bits(int=-1, length=length)
return Bits(bytes=conv(val), length=length)
def test_string_container(self):
ci = StringContainerItem('FooBar')
self.assertEqual(ci.prepare_bitstream(),
Bits(b'\5\0\1\0\0\0\0\0\0\0\0\0\0\0\0\0\x18\0\0\0\0\0\0\0\6\0FooBar\0')
)
self.assertEqual(ci.get_all_pointers(), [16])
def __init__(self, name=None, citation=None, bits=None):
super(Terminal, self).__init__(name, citation)
self.bits = bits if bits is not None else Bits(256)
def to_bits(self, val):
return Bits(self.str_to_bytes(val))
def octet_range(min_, max_):
"""Create a terminal that accepts bytes from `min_` to `max_` inclusive."""
bits = BitArray(256)
for i in range(min_, max_ + 1):
bits[i] = True
return Terminal(bits=Bits(bits))
def operation_bits(self): op = next((op for op in k10_operations if op['mnem'] == self.operation), None) return Bits(bin=op['bits'])
def func(s):
return Bits('')
def assemble(self):
uop_pos = 0
uops = [None, None, None]
sw = SW('next')
stream = Bits(length=0)
for line in self.asm.split('\n'):
line = line.strip()
if len(line) == 0:
continue
if line.startswith('//'):
continue
if line.startswith('.start'):
continue
if line.startswith('.sw_complete'):
sw = SW('complete')
continue
if line.startswith('.test'):
sw = SW('test')
continue
if line.startswith('.sw_branch'):
sw = SW('branch', addr=int(line[11:], 0))
continue
if line.startswith('.sw_dbg'):
sw = SW('dbg', dbg=line[8:])
continue
opcode = line.split(' ', 1)[0].lower()
operands = line.split(' ', 1)[1].lower()
if not self.__asm_is_op(opcode):
raise ValueError('Invalid opcode: %s' % opcode)
continue
if opcode in op_constraints.keys():
constraint = op_constraints[opcode]
if constraint < uop_pos:
for i in range(3):
if uops[i] == None:
uops[i] = NoOp()
stream += Triad(uops[0], uops[1], uops[2], sw).bits()
uop_pos = 0
uops = [None, None, None]
sw = SW('next')
if constraint > uop_pos:
while(constraint > uop_pos):
uops[uop_pos] = NoOp()
uop_pos += 1
if uop_pos < 3:
uops[uop_pos] = self.__asm_get_op(opcode, operands)
uop_pos += 1
else:
stream += Triad(uops[0], uops[1], uops[2], sw).bits()
uop_pos = 1
uops = [self.__asm_get_op(opcode, operands), None, None]
sw = SW('next')
if uop_pos > 0 and uop_pos <= 3:
for i in range(3):
if uops[i] == None:
uops[i] = NoOp()
stream += Triad(uops[0], uops[1], uops[2], sw).bits()
return stream
def testInterpretation(self):
for x in range(101):
self.assertEqual(Bits(uie=x).uie, x)
for x in range(-100, 100):
self.assertEqual(Bits(sie=x).sie, x)
