def get_frame(self): if self.pickle_files is None or len(self.pickle_files) == 0: print('No file Loaded') return None #pickle_files = list_pickles2read(cpc=cpc, maxfiles=maxfiles, debug=debug, from_datetime=from_datetime, to_datetime=to_datetime) self.attacks = pd.DataFrame() with IncrementalBar('Loading files:', max=len(self.pickle_files)) as bar: for filename in self.pickle_files: try: with open(filename, 'rb') as f: new_attaks = pickle.load(f) self.attacks = self.attacks.append(new_attaks) bar.next() except: continue self.attacks.reset_index(inplace=True, drop=True) if self.attacks.empty: print('empty attacks...') return None self.attacks['err_vec'] = self.attacks['err_vec'].apply(lambda x: frozenbitarray(x)) self.attacks['write_data'] = self.attacks['write_data'].apply(lambda x: frozenbitarray(x)) self.attacks['read_data'] = self.attacks['read_data'].apply(lambda x: frozenbitarray(x)) self.attacks['mem_before'] = self.attacks['mem_before'].apply(lambda x: frozenbitarray(x)) # TODO move to parse self.set_defaults() print(f'Loaded {len(self.attacks)} attacks in cpc{self.cpc} from {len(self.pickle_files)} files') return self.attacks
def test_lr_justification(): unsafe = gen_random_string(15) safe = gen_random_safe_from_unsafe(unsafe) fillchar_trusted = safe_string._new_trusted("-") for width in (10, 15, 20, 25): assert safe.ljust(width) == unsafe.ljust(width) assert safe.ljust(width, fillchar_trusted) == unsafe.ljust(width, "-") assert safe.rjust(width) == unsafe.rjust(width) assert safe.rjust(width, fillchar_trusted) == unsafe.rjust(width, "-") for width in (0, 10, 15): assert safe.ljust(width)._trusted == safe._trusted assert safe.ljust(width, fillchar_trusted)._trusted == safe._trusted assert safe.rjust(width)._trusted == safe._trusted assert safe.rjust(width, fillchar_trusted)._trusted == safe._trusted for width in (16, 20, 25): false_array = frozenbitarray([False] * (width - 15)) true_array = frozenbitarray([True] * (width - 15)) assert safe.ljust(width)._trusted == safe._trusted + false_array assert safe.rjust(width)._trusted == false_array + safe._trusted assert safe.ljust( width, fillchar_trusted)._trusted == safe._trusted + true_array assert safe.rjust( width, fillchar_trusted)._trusted == true_array + safe._trusted
def __init__(self, filename): self.filename = filename self.logfile_hash = hash(self.filename) with open(self.filename, 'r') as file: _ = file.readline() # _="Original data:" write_data_lst = file.readline().split(',') write_data_lst = [frozenbitarray(wd.strip()[10:-1]) for wd in write_data_lst] _ = file.readline() # _="Original data:" mem_before_write_logfile = re.split(',|, ', file.readline()) mem_before_write_logfile = [frozenbitarray(mbw.strip()[10:-1]) for mbw in mem_before_write_logfile] attack_params_keys = file.readline() attack_params_keys = ''.join(attack_params_keys.split()) attack_params_keys = attack_params_keys.split(',,')[0].split(',') lines = file.readlines() if(len(lines) < 1): self.Empty = True self.df = pd.DataFrame() #print(f'\nempty file: {self.filename}') return #print(f'\n{len(lines)} lines in {self.filename}') dfs = [] drop_count = 0 for i, line in enumerate(lines): line = Logline(line, attack_params_keys, write_data_lst, self.logfile_hash, i) if line.is_dropped: #print(f'line {i} dropped. msg: {line.drop_reason}') drop_count += 1 continue dfs.append(line.to_frame()) self.df = pd.DataFrame().append(dfs) self.Empty = False self.lines = len(lines) self.dropped_lines = drop_count
def test_split(): unsafe = "011110011010101011101010" safe = safe_string(unsafe, frozenbitarray(unsafe)) for maxsplit in range(10): for chunk, chunk_unsafe in zip(safe.split("0", maxsplit), unsafe.split("0", maxsplit)): assert chunk == chunk_unsafe assert len(chunk) == len(chunk._trusted) for char in chunk: assert (char == "1") == bool(char._trusted[0]) for chunk, chunk_unsafe in zip(safe.rsplit("0", maxsplit), unsafe.rsplit("0", maxsplit)): assert chunk == chunk_unsafe assert len(chunk) == len(chunk._trusted) for char in chunk: assert (char == "1") == bool(char._trusted[0]) unsafe = " 11\t222 \n\r333 \n" trusted = frozenbitarray([not char.isspace() for char in unsafe]) safe = safe_string(unsafe, trusted) for maxsplit in range(5): for chunk, chunk_unsafe in zip(safe.split(None, maxsplit), unsafe.split(None, maxsplit)): assert chunk == chunk_unsafe assert len(chunk) == len(chunk._trusted) for char in chunk: assert not char.isspace() == bool(char._trusted[0]) for chunk, chunk_unsafe in zip(safe.rsplit(None, maxsplit), unsafe.rsplit(None, maxsplit)): assert chunk == chunk_unsafe assert len(chunk) == len(chunk._trusted) for i, char in enumerate(chunk): assert not char.isspace() == bool(char._trusted[0]) for _ in range(10): unsafe = gen_random_string(50) maxsplit = random.randint(5, 10) safe = gen_random_safe_from_unsafe(unsafe) assert safe.split(None, maxsplit) == unsafe.split(None, maxsplit) assert safe.split(unsafe[:2], maxsplit) == unsafe.split(unsafe[:2], maxsplit) assert safe.split(safe[:2], maxsplit) == unsafe.split(safe[:2], maxsplit) assert safe.split(unsafe[-2:], maxsplit) == unsafe.split(unsafe[-2:], maxsplit) assert safe.split(safe[-2:], maxsplit) == unsafe.split(safe[-2:], maxsplit) assert safe.rsplit(None, maxsplit) == unsafe.rsplit(None, maxsplit) assert safe.rsplit(unsafe[:2], maxsplit) == unsafe.rsplit(unsafe[:2], maxsplit) assert safe.rsplit(safe[:2], maxsplit) == unsafe.rsplit(safe[:2], maxsplit) assert safe.rsplit(unsafe[-2:], maxsplit) == unsafe.rsplit(unsafe[-2:], maxsplit) assert safe.rsplit(safe[-2:], maxsplit) == unsafe.rsplit(safe[-2:], maxsplit)
def getOpcode(ba): """ Creates frozenbitarray of opcode, used as a key in the instructionTable """ if len(ba) == 32: # 32 bit instruction opcode return frozenbitarray(ba[-7:]) elif len(ba) == 16: # compressed instruction opcode return frozenbitarray(ba[-2:])
def __repr__(self): """ __repr__ override for safe_string objects. `str.__repr__` has a complicated escaping logic. It escapes backslashes, special escape characters (\n, \t, ...) but also unicode codepoints outside the printable range. There might be more to it. For now, a slow but safe approach is to use the trusted value of each character as the trusted of its repr, and concat them together. Examples: repr("\\") == "'\\\\'" repr("\n") == "'\\n'" repr("\x0c") == "'\\x0c'" repr("\x1c") == "'\\x1c'" repr("\x2c") == "','" == "'\x2c'" repr("'") == "'" repr('"') == '"' repr("'\"") == '\'"' """ # XXX uncomment for debugging # return "\n" + repr(self._to_unsafe_str()) + "\n" + repr(self._trusted.to01()) repr_string = super().__repr__() if len(repr_string) - len(self) == 2: # nothing has been escaped; only quotes have been added # TODO: should these quotes be trusted? since this is performed by the developer repr_trusted = frozenbitarray( [False]) + self._trusted + frozenbitarray([False]) else: # some characters have been escaped # we have no way to know which without looping through the string repr_trusted = bitarray([False]) # single quote is only escaped if the repr string # is surrounded with single quotes, but this doesn't # show up if you call repr on it by itself (since it wraps # it in double quotes). single_quote_escaped = (repr_string[0] == "'") for char in self: if char == "'" and single_quote_escaped: len_repr = 2 else: len_repr = len(str.__repr__(char)) - 2 repr_trusted.extend(char._trusted * len_repr) repr_trusted.append(False) # freeze to make immutable repr_trusted = frozenbitarray(repr_trusted) return safe_string(repr_string, repr_trusted)
def zfill(self, width): # zfill has special behaviour if the string starts with +/- string = super().zfill(width) if not string: return self num_extra = len(string) - len(self) if string[0] == self[0]: trusted = self._trusted[0:1] + frozenbitarray( [False]) * num_extra + self._trusted[1:] else: trusted = frozenbitarray([False]) * num_extra + self._trusted return safe_string(string, trusted)
def test_ba2int(self): self.assertEqual(ba2int(bitarray('0')), 0) self.assertEqual(ba2int(bitarray('1')), 1) self.assertEqual(ba2int(bitarray('00101', 'big')), 5) self.assertEqual(ba2int(bitarray('00101', 'little')), 20) self.assertEqual(ba2int(frozenbitarray('11')), 3) self.assertRaises(ValueError, ba2int, bitarray()) self.assertRaises(ValueError, ba2int, frozenbitarray()) self.assertRaises(TypeError, ba2int, '101') a = bitarray('111') b = a.copy() self.assertEqual(ba2int(a), 7) # ensure original object wasn't altered self.assertEQUAL(a, b)
def test_count(): unsafe = "abABabAB" unsafe_old1 = "AB" unsafe_old2 = "cd" safe = safe_string(unsafe, trusted=frozenbitarray([ True, True, False, False, True, True, False, False ])) old_str1 = safe_string(unsafe_old1, frozenbitarray([False] * 2)) old_str2 = safe_string(unsafe_old2, frozenbitarray([False] * 2)) assert safe.count(old_str1) == unsafe.count(unsafe_old1) assert safe.count(old_str2) == unsafe.count(unsafe_old2)
def get_max_prob_collision(self, env, scaling_factors): """ - called after add_point and update - now we have everything to compute all possible X(k+1) and intersect those with the associated environment """ know = self.k % (self.kmax + 1) n_extr = len(self.Sn[know]) Xks = self.get_Xks() p = 0.0 for configID in range(2 ** n_extr): bconfig = PlanUtils.configID2bitarray( configID, self.Sn[know], self._nlines_tot ) for rectangle_idx in self.rectangles_seen_now: # Get the corresponding rectangle configuration config = bconfig[rectangle_idx * 4 : (rectangle_idx + 1) * 4] p = max( p, self.get_prob_collision( Xks[configID], env.rectangles[rectangle_idx].to_zonotope( frozenbitarray(config) ), scaling_factors, ), ) return p
def expandtabs(self, tabsize=8): """ Override for expandtabs. This function is a lot more complicated than our initial assumption - we thought it would just expand each "\t" to tabsize " " characters. This turns out to be completely false. The function does a much more useful job of aligning the segment following a "\t" to the next tabstop. The calculation for this is somewhat nuanced, and is mostly derived from pypy3.6's implementation. """ if not self: return self string = super().expandtabs(tabsize) splitted = self.split("\t") # keep track of idx to get trust value of \t oldtoken = splitted.pop(0) final_trusted = bitarray(oldtoken._trusted) start_idx = len(final_trusted) for token in splitted: dist = self._tabindent(oldtoken, tabsize) final_trusted.extend(self._trusted[start_idx:start_idx + 1] * dist) final_trusted.extend(token._trusted) start_idx += 1 + len(token) oldtoken = token final_trusted = frozenbitarray(final_trusted) return safe_string(string, final_trusted)
def test_from_frozen(self): a = frozenbitarray('1101111', 'big') b = make_endian(a, 'big') self.assertTrue(b is a) c = make_endian(a, 'little') self.assertTrue(c == a) self.assertEqual(c.endian(), 'little')
def test_rindex(self): self.assertRaises(TypeError, rindex) self.assertRaises(TypeError, rindex, None) self.assertRaises(TypeError, rindex, bitarray(), 1, 2) for endian in 'big', 'little': a = bitarray('00010110000', endian) self.assertEqual(rindex(a), 6) self.assertEqual(rindex(a, 1), 6) self.assertEqual(rindex(a, 'A'), 6) self.assertEqual(rindex(a, True), 6) a = bitarray('00010110111', endian) self.assertEqual(rindex(a, 0), 7) self.assertEqual(rindex(a, None), 7) self.assertEqual(rindex(a, False), 7) a = frozenbitarray('00010110111', endian) self.assertEqual(rindex(a, 0), 7) self.assertEqual(rindex(a, None), 7) self.assertEqual(rindex(a, False), 7) for v in 0, 1: self.assertRaises(ValueError, rindex, bitarray(0, endian), v) self.assertRaises(ValueError, rindex, bitarray('000', endian), 1) self.assertRaises(ValueError, rindex, bitarray('11111', endian), 0)
def test_repr_same_length(): examples = [ "abc", "\\", "\n", b"Ni\xc3\xb10".decode("utf-8"), string.printable, "\x0c", "\x1c", "\x2c", "\xff", "\xf8", "\x7f\x8f\x9f\xff", "\f", "\t", " ", "", ] for unsafe in examples: # safe = gen_random_safe_from_unsafe(unsafe) safe = safe_string( unsafe, frozenbitarray([True, False] * (len(unsafe) // 2) + [True] * (len(unsafe) % 2))) safe_repr = repr(safe) assert len(safe_repr) == len(repr(unsafe)) if len(safe_repr) != len(safe_repr._trusted): safe._debug_repr() safe_repr._debug_repr() assert False
def to_dict(self): dict2ret = self.attack_params dict2ret['address'] = range(len(self.read_data_lst)) dict2ret['read_data'] = [barr.to01() for barr in self.read_data_lst] dict2ret['detected by HW'] = [int(data[-1]) for data in self.line[1]] dict2ret['detected by SW'] = [self.cpc.is_faulty(rd) for rd in self.read_data_lst] dict2ret['err_vec'] = [e.to01() for e in self.errvecs] dict2ret['#bitflips'] = [err_vec.count(1) for err_vec in self.errvecs] dict2ret['#bitflips data'] = [err_vec[:self.cpc.k].count(1) for err_vec in self.errvecs] dict2ret['#bitflips redundancy'] = [err_vec[self.cpc.k:].count(1) for err_vec in self.errvecs] dict2ret['try_id'] = self.try_id dict2ret['mem_before'] = self.mem_before_given dict2ret['write_data'] = self.write_data max_const_data = int(dict2ret['index']) dict2ret['write_data_consistent_count'] = sum(a == b for a,b in zip(self.write_data[:max_const_data], self.read_data_lst[:max_const_data])) dict2ret['write_data_consistent_normalized'] = dict2ret['write_data_consistent_count'] / max_const_data if max_const_data > 0 else np.inf dict2ret['write_data_consistent_tuple'] = frozenbitarray(a == b for a,b in zip(self.write_data[:max_const_data], self.read_data_lst[:max_const_data])).to01() dict2ret['mem_before_consistent_data'] = [a[:self.cpc.k] == b[:self.cpc.k] for a, b in zip(self.mem_before, self.mem_before_given)] dict2ret['line_mem_before_consistent_data'] = any(dict2ret['mem_before_consistent_data']) dict2ret['mem_before_consistent_red'] = [a[-self.cpc.r:] == b[-self.cpc.r:] for a, b in zip(self.mem_before, self.mem_before_given)] dict2ret['line_mem_before_consistent_red'] = any(dict2ret['mem_before_consistent_red']) return dict2ret
def cont_mat(searcher, queries): """ Ugly slow hack alert. A better way would be to give dep-search an OR like query and get it to give as back answers as well as a boolean vector of whether each term was a hit. """ results = [] frame_dict = {} frame_idx = 0 print("queries", queries) for sid, group in groupby(searcher.multi_query(queries), itemgetter(0)): col = [] for _, idx, hittoken in group: tok_idx, pb_frame, is_verb = parse_hittoken(hittoken) if not is_verb: continue col.append(((idx, tok_idx), sid, pb_frame)) if pb_frame not in frame_dict: frame_dict[pb_frame] = frame_idx frame_idx += 1 col.sort() results.append(col) # Regroup by sent_tok_id and get bit vector of queries bitvec_dict = {} bitvec_idx = 0 counters = [] frame_marginal = [0] * frame_idx query_marginal = [] hits = 0 for sent_tok_id, group in groupby(merge(*results, key=itemgetter(0)), key=itemgetter(0)): bitvec = bitarray(len(queries)) bitvec.setall(0) pb_frame = None for sent_tok_id, sid, g_pb_frame in group: bitvec[sid] = 1 if pb_frame is None: pb_frame = g_pb_frame else: assert pb_frame == g_pb_frame bitvec = frozenbitarray(bitvec) if bitvec not in bitvec_dict: cur_bitvec_idx = bitvec_idx bitvec_dict[bitvec] = bitvec_idx bitvec_idx += 1 counters.append([0] * frame_idx) query_marginal.append(0) else: cur_bitvec_idx = bitvec_dict[bitvec] counters[cur_bitvec_idx][frame_dict[pb_frame]] += 1 frame_marginal[frame_dict[pb_frame]] += 1 query_marginal[cur_bitvec_idx] += 1 hits += 1 # Join results into contingency matrix return counters, frame_dict, bitvec_dict, frame_marginal, query_marginal, hits
def test_replace(): unsafe = "abABabAB" unsafe_old = "AB" unsafe_new = "ab" unsafe_old2 = "cd" safe = safe_string(unsafe, trusted=frozenbitarray([ True, True, False, False, True, True, False, False ])) old_str = safe_string(unsafe_old, frozenbitarray([False] * 2)) old_str2 = safe_string(unsafe_old2, frozenbitarray([False] * 2)) new_str = safe_string(unsafe_new, trusted=frozenbitarray([True] * 2)) assert unsafe.replace(unsafe_old, unsafe_new) == safe.replace(old_str, new_str) assert safe.replace(old_str, new_str)._trusted == frozenbitarray([True] * 8) assert unsafe.replace(unsafe_old, unsafe_new, 1) == safe.replace(old_str, new_str, 1) assert safe.replace(old_str, new_str, 1)._trusted == frozenbitarray( [True, True, True, True, True, True, False, False]) assert unsafe.replace(unsafe_old, unsafe_new, 10) == safe.replace(old_str, new_str, 10) assert safe.replace(old_str, new_str, 10)._trusted == frozenbitarray([True] * 8) assert unsafe.replace(unsafe_old2, unsafe_new) == safe.replace(old_str2, new_str) assert safe.replace(old_str2, new_str)._trusted == safe._trusted
def __init__(self, z_mean: np.ndarray): """Creates a median hash from numpy ndarray""" z_mean = z_mean.flatten() self._num_bits = len(z_mean) median = np.median(z_mean) one_hot = [str(int(x > median)) for x in z_mean] str_code = "".join(one_hot) self._code = frozenbitarray(str_code)
def set_values(self, Sn, Sk): """ Set values from the same Sn used for the constructor and Sk. w is the half-width of the uncertain mean. R is the variance for all measurements. """ assert Sn.size == self._nlines # incomplete check but still useful self._values = {} # reinitialize idx_extr = np.in1d(Sn, Sk) # True for elements in Sn also in Sk, size of Sn # number of lines that need to be considered as extrema, those are both in Sn and Sk n_extr = np.where(idx_extr)[0].size n_config = 2 ** n_extr # bits is used to keep track which extremum has been considered # for each line considered as extrema bits = bitarray(n_extr) bitstring_format = "{" + "0:0%db" % n_extr + "}" centers = np.zeros((Sn.size,)) generators = np.zeros((Sn.size,)) # final key of size the total number of lines in the environment key = PlanUtils.init_bitarray(self._nlines_tot) for i_config in range(n_config): # set a bitmask for the current config in consideration bits = bitarray(bitstring_format.format(i_config)[::-1]) centers.fill(0) generators.fill(0) key.setall(False) i = 0 for iSn in range(Sn.size): if idx_extr[iSn]: # a line in Sn that is also in Sk if bits[i]: # choose + half width centers[iSn] = -self._w[iSn] key[Sn[iSn]] = True # Sn[iSn] = this line ID else: # choose - half width centers[iSn] = self._w[iSn] i += 1 else: # a line in Sn that is not in Sk, consider the full range generators[iSn] = self._w[iSn] # all extrema must have been considered assert i == n_extr self._values[frozenbitarray(key)] = deepcopy( Zonotope(centers, generators, self._cov * np.eye(Sn.size)) )
def test_subset(self): a = frozenbitarray('0101') b = bitarray('0111') self.assertTrue(subset(a, b)) self.assertFalse(subset(b, a)) self.assertRaises(TypeError, subset) self.assertRaises(TypeError, subset, a, '') self.assertRaises(TypeError, subset, '1', b) self.assertRaises(TypeError, subset, a, 4) b.append(1) self.assertRaises(ValueError, subset, a, b)
def __add__(self, other): # call str method first to get same behaviour on error string = orig_str_add(self, other) if isinstance(other, safe_string): trusted = self._trusted + other._trusted return safe_string(string, trusted) elif isinstance(other, str): trusted = self._trusted + (frozenbitarray([False] * len(other))) return safe_string(string, trusted) else: raise TypeError("argument must be a str or safe_string")
def rjust(self, width, fillchar=" "): string = super().rjust(width, fillchar) if isinstance(fillchar, safe_string): fillchar_trust = fillchar._trusted[0] else: fillchar_trust = False trusted = max(0, width - len(self)) * frozenbitarray( [fillchar_trust]) + self._trusted return safe_string(string, trusted)
def test_all_perm_1(self): n, k = 10, 5 c = 0 s = set() for a in all_perm(n, k, 'little'): self.assertIsType(a, 'bitarray') self.assertEqual(len(a), n) self.assertEqual(a.count(), k) s.add(frozenbitarray(a)) c += 1 self.assertEqual(c, binomial(n, k)) self.assertEqual(len(s), binomial(n, k))
def join(self, iterable): seq = iter(iterable) try: first_elem = next(seq) final = first_elem except StopIteration: return safe_string("", trusted=frozenbitarray()) for elem in seq: final += self + elem return final
def at_config(self, Sk, configID): """ Return the zonotope associated with a config ID. configID can be between 0 and 2^|S_k|-1. Can only be used with the most recent Sk. """ n_extr = len(Sk) assert configID < 2 ** n_extr bconfig = PlanUtils.configID2bitarray(configID, Sk, self._nlines_tot) # filter it by the bitmask of lines seen at n and return return self._values[frozenbitarray(self._bitmask & bconfig)]
def test_count_n1_frozen(self): a = frozenbitarray('001111101111101111101111100111100') self.assertEqual(len(a), 33) self.assertEqual(a.count(), 24) self.assertRaises(TypeError, count_n, '', 0) self.assertEqual(count_n(a, 0), 0) self.assertEqual(count_n(a, 10), 13) self.assertEqual(count_n(a, 24), 31) self.assertRaises(ValueError, count_n, a, -1) # n < 0 self.assertRaises(ValueError, count_n, a, 25) # n > a.count() self.assertRaises(ValueError, count_n, a, 34) # n > len(a) self.assertRaises(TypeError, count_n, a, "7")
def test_frozen(self): a = frozenbitarray('001111101111101111101111100111100') self.assertEqual(len(a), 33) self.assertEqual(a.count(), 24) self.assertEqual(count_n(a, 0), 0) self.assertEqual(count_n(a, 10), 13) self.assertEqual(count_n(a, 24), 31) self.assertRaises(ValueError, count_n, a, -1) # n < 0 self.assertRaises(ValueError, count_n, a, 25) # n > a.count() self.assertRaises(ValueError, count_n, a, 34) # n > len(a) for n in range(0, 25): self.check_result(a, n, count_n(a, n))
def pad_to_len(self, desired_length): """ Add 0s to the internal representation of this Symbol, so that the number of bits used matches desired_length :param desired_length: the number of bits we want this Symbol to be represented on :return: self """ mutable_repr = bitarray.bitarray(self._internal_repr) while len(mutable_repr) < desired_length: mutable_repr.insert(0, 0) self._internal_repr = bitarray.frozenbitarray(mutable_repr) return self
def __iadd__(self, other): """ Append the bits of the other Symbol at the end of those of this Symbol. After this the internal representation will change :param other: Symbol whose bits we want to append to this one :return: self """ if not isinstance(other, Symbol): raise TypeError("unsupporetd += with type %s" % type(other).__name__) self._internal_repr = bitarray.frozenbitarray(self._internal_repr + other._internal_repr) return self
def test_zeros(self): for n in range(10): for mode in 'left', 'right', 'both': a = zeros(n) c = strip(a, mode) self.assertIsType(c, 'bitarray') self.assertEqual(c, bitarray()) self.assertEqual(a, zeros(n)) b = frozenbitarray(a) c = strip(b, mode) self.assertIsType(c, 'frozenbitarray') self.assertEqual(c, bitarray())