def get_patterns(size: int, error_size: int) -> List[List[int]]: patterns = [] for i in range(2**size): pattern = BitArray(uint=i, length=size) if pattern.count(1) == error_size: patterns.append(to_list(pattern)) return patterns
def floatable(mem_addr: bitstring.BitArray, mask: bitstring.BitArray): amount_of_trues = mask.count(True) for possible_setting in range(2**amount_of_trues): possible_setting = get_bitsstring(possible_setting) possible_setting_index = -1 for index in mask.findall([1]): mem_addr[index] = possible_setting[possible_setting_index] possible_setting_index -= 1 yield mem_addr
def encode(data: list, encoding: str) -> List[BitArray]: """encodes elements of the list into packets and generates parity bit for each packet""" ready_packets = [] for d in data: tmp = BitArray(bytearray(d, encoding)) if tmp.count(True) % 2 == 1: tmp.append(BitArray(uint=1, length=1)) else: tmp.append(BitArray(uint=0, length=1)) ready_packets.append(tmp) return ready_packets
class mpt1327_state: def __init__(self): self.data = BitArray(uint=0, length=64) self.cnt = 0 self.codeword = 0 self.prev = 0 self.base_freq = 170.8 self.step_freq = 0.0125 def crc(self): data, checksum = self.data.unpack('uint:48, uint:15') return CRC.calcWord(data, 48) == checksum and self.data.count(1) % 2 == 0 # Even parity
def test_valid_replace_bits(num, start, stop, bit_idx, count): ga = RealGA(fitness_test_sin_func, cross_prob=1) ga.interval = (-numpy.inf, numpy.inf) source_bstr = BitArray(floatbe=num, length=ga._bin_length).bin new_chromosome = ga._replace_bits(num, 0, start, stop) actual_bstr = BitArray(floatbe=new_chromosome, length=ga._bin_length).bin for i in bit_idx: assert actual_bstr[i] == source_bstr[i] assert actual_bstr.count('1') == count
def test_dirty_bit_pos(self): ''' Test ``bbc.dirty_bit_pos()`` against all possible bytes. ''' for bits in it.product(*it.repeat(['0', '1'], 8)): byte = BitArray(bin=''.join(bits)) if byte.count(1) == 1: # offset byte found self.assertEqual(bbc.dirty_bit_pos(byte), byte.find('0b1')[0]) else: # non-offset byte found self.assertEqual(bbc.dirty_bit_pos(byte), -1)
def monobits(s, v): res = True chkvals = [] b = BitArray(s) c = b.count(True) if v == 1: chkvals = [9654, 10346] elif v == 2: chkvals = [9725, 10275] if c <chkvals[0] or c>chkvals[1]: res = False return res, c
def get_distance(self): """ Assuming that the two bit strings are the same length: 1. XOR the two bit strings. This will calculate which positions are different. 2. Count the number of set bits from the result of step 1. """ self._pre_process() bit_str_len = self.bit_str1.len resulting_diff_stream = BitArray(bin='0') for i in range(0, bit_str_len): result = self.bit_str1[i] ^ self.bit_str2[i] resulting_diff_stream.append(BitArray(bool=result)) return resulting_diff_stream.count(1)
def set_hammingArray(character): #make a bitarray for the input character char_bitarray = BitArray('int:8=' + str(ord(character))) #make a bit array to hold the channel coded version of the character. char_hammingArray = BitArray('0b0000000000000') #transfer all bits from the char_bitarray to the char_hammingArray skipping the parity bit positions i = 0 j = 0 length = len(char_hammingArray) while (i < length): if i in parity_positions: i += 1 continue if char_bitarray[j]: char_hammingArray[i] = 1 else: char_hammingArray[i] = 0 i += 1 j += 1 #set each parity bit for the hamming code: 1 if checksum is odd, 0 if checksum is even for bit in parity_positions: #calculate the value of the check sum for the current parity bit check_sum = calc_checksum(char_hammingArray, bit) #if the checksum is odd, set the parity bit to 1, otherwise leave at 0 if (check_sum % 2 != 0): char_hammingArray[bit] = 1 char_hammingArray[-1] = 0 total_parity = char_hammingArray.count(1) if (total_parity % 2 != 0): char_hammingArray[-1] = 1 return char_hammingArray
def set_hammingArray(character): #make a bitarray for the input character char_bitarray = BitArray('int:8='+str(ord(character))) #make a bit array to hold the channel coded version of the character. char_hammingArray = BitArray('0b0000000000000') #transfer all bits from the char_bitarray to the char_hammingArray skipping the parity bit positions i = 0 j = 0 length = len(char_hammingArray) while (i < length): if i in parity_positions: i += 1 continue if char_bitarray[j]: char_hammingArray[i] = 1 else: char_hammingArray[i] = 0 i += 1 j += 1 #set each parity bit for the hamming code: 1 if checksum is odd, 0 if checksum is even for bit in parity_positions: #calculate the value of the check sum for the current parity bit check_sum = calc_checksum(char_hammingArray, bit) #if the checksum is odd, set the parity bit to 1, otherwise leave at 0 if(check_sum%2!=0): char_hammingArray[bit] = 1 char_hammingArray[-1] = 0 total_parity = char_hammingArray.count(1) if(total_parity%2!=0): char_hammingArray[-1] = 1 return char_hammingArray
class KalmanFilterPID(Parser): """ generated source for class KalmanFilterPID """ # sampling rate def __init__(self, param): """ generated source for method __init__ """ Parser.__init__(self) self.param = param self.differ = Differential(self.param.Seed) self.predict = [] self.interval = None # Kalman Filter params self.P = 100 # estimation error covariance (over all time instance) self.Q = 1000 # process noise synthetic data self.R = 1000000 # measurement noise optimal for alpha = 1, synthetic data self.K = 0 # kalman gain # PID control params - default self.Cp = 0.9 # proportional gain, to keep output proportional to current error self.Ci = 0.1 # integral gain, to eliminate offset self.Cd = 0.0 # derivative gain, to ensure stability - prevent large error in future # fixed internally self.theta = 1 # magnitude of changes self.xi = 0.2 # gamma (10%) self.minIntvl = 1 # make sure the interval is greater than 1 self.windowPID = 5 # I(integration) window self.ratioM = 0.2 # sampling rate # self.isSampling = False def adjustParams(self): # adjust params if self.ratioM < 0.1: self.theta = 20 if 0.1 <= self.ratioM < 0.2: self.theta = 14 if 0.2 <= self.ratioM < 0.3: self.theta = 2 if 0.3 <= self.ratioM < 0.4: self.theta = 0.5 if 0.4 <= self.ratioM < 0.5: self.theta = 0.3 if 0.5 <= self.ratioM: self.theta = 0.1 # test @classmethod def main(self, args): """ generated source for method main """ if len(args) < 5: print "Usage: python KalmanFilterPID.py input output privacy-budget process-variance Cp(optional) Ci(optional) Cd(optional)" sys.exit() output = open(args[2], "w") budget = eval(args[3]) Q = float(args[4]) if budget <= 0 or Q <= 0: print "Usage: privacy-budget AND process-variance are positive values" sys.exit() p = Params(1000) kfPID = KalmanFilterPID(p) kfPID.setTotalBudget(budget) kfPID.setQ(Q) kfPID.orig = Parser.getData(args[1]) kfPID.publish = [None] * len(kfPID.orig) # adjust R based on T and alpha kfPID.setR(len(kfPID.orig) * len(kfPID.orig) / (0.0 + budget * budget)) # set optional control gains if len(args) >= 6: d = args[5] if d > 1: d = 1 kfPID.setCp(d) if len(args) >= 7: d = args[6] if d + kfPID.Cp > 1: d = 1 - kfPID.Cp kfPID.setCi(d) else: kfPID.setCi(1 - kfPID.Cp) if len(args) >= 8: d = args[7] if d + kfPID.Cp + kfPID.Ci > 1: d = 1 - kfPID.Cp - kfPID.Ci kfPID.setCd(d) else: kfPID.setCd(1 - kfPID.Cp - kfPID.Ci) # kfPID.adjustParams() start = time.time() kfPID.publishCounts() end = time.time() Parser.outputData(output, kfPID.publish) print "Method:\tKalman Filter with Adaptive Sampling" print "Data Series Length:\t" + str(len(kfPID.orig)) print "Queries Issued:\t" + str(kfPID.query.count(1)) print "Privacy Budget Used:\t" + str( kfPID.query.count(1) * kfPID.epsilon) print "Average Relative Error:\t" + str(kfPID.getRelError()) print "Time Used (in second):\t" + str(end - start) def kalmanFilter(self, orig, budget, samplingRate=None): self.totalBudget = budget self.orig = orig if samplingRate is not None: self.isSampling = True self.ratioM = samplingRate else: self.isSampling = False # self.adjustParams() self.publish = [None] * len(self.orig) # adjust R based on T and alpha self.setR(len(self.orig) * len(self.orig) / (0.0 + budget * budget)) self.publishCounts() return self.publish def getCount(self, value, epsilon): """ return true count or noisy count of a node, depending on epsilon. Note that the noisy count can be negative """ if epsilon < 10**(-8): return value else: return value + self.differ.getNoise(1, epsilon) # sensitivity is 1 # data publication procedure def publishCounts(self): """ generated source for method publish """ self.query = BitArray(len(self.orig)) self.predict = [None] * len(self.orig) # recalculate individual budget based on M if (self.isSampling): M = int(self.ratioM * (len(self.orig))) # 0.25 optimal percentile else: M = len(self.orig) if M <= 0: M = 1 self.epsilon = (self.totalBudget + 0.0) / M # error = 0 self.interval = 1 nextQuery = max(1, self.windowPID) + self.interval - 1 for i in range(len(self.orig)): if i == 0: # the first time instance self.publish[i] = self.getCount(self.orig[i], self.epsilon) self.query[i] = 1 self.correctKF(i, 0) else: predct = self.predictKF(i) self.predict[i] = predct if self.query.count(1) < self.windowPID and self.query.count( 1) < M: # i is NOT the sampling point self.publish[i] = self.getCount(self.orig[i], self.epsilon) self.query[i] = 1 # update count using observation self.correctKF(i, predct) elif i == nextQuery and self.query.count(1) < M: # if i is the sampling point # query self.publish[i] = self.getCount(self.orig[i], self.epsilon) self.query[i] = 1 # update count using observation self.correctKF(i, predct) # update freq if (self.isSampling): ratio = self.PID(i) frac = min(20, (ratio - self.xi) / self.xi) deltaI = self.theta * (1 - math.exp(frac)) deltaI = int(deltaI) + (random.random() < deltaI - int(deltaI)) self.interval += deltaI else: self.interval = 1 if self.interval < self.minIntvl: self.interval = self.minIntvl nextQuery += self.interval # nextQuery is ns in the paper else: # --> predict self.publish[i] = predct # del self.orig # del self.predict # del self.query # if self.isPostProcessing: # self.postProcessing() # def postProcessing(self): # print len(self.samples), self.samples # remainedEps = self.totalBudget - len(self.samples) * self.epsilon # self.epsilon = self.epsilon + remainedEps/len(self.samples) # # # recompute noisy counts # prev = 0 # for i in self.samples: # self.publish[i] = self.getCount(self.orig[i], self.epsilon) # if i > prev + 1: # self.publish[prev + 1 : i] = [self.publish[prev]] * (i - prev - 1) # prev = i def setR(self, r): """ generated source for method setR """ self.R = r def setQ(self, q): """ generated source for method setQ """ self.Q = q def setCp(self, cp): """ generated source for method setCp """ self.Cp = cp def setCi(self, ci): """ generated source for method setCi """ self.Ci = ci def setCd(self, cd): """ generated source for method setCd """ self.Cd = cd # prediction step def predictKF(self, curr): """ generated source for method predictKF """ # predict using Kalman Filter lastValue = self.getLastQuery(curr) # project estimation error self.P += self.Q # Q is gaussian noise return lastValue # correction step def correctKF(self, curr, predict): """ generated source for method correctKF """ self.K = (self.P + 0.0) / (self.P + self.R) correct = predict + self.K * (self.publish[curr] - predict) # publish[curr] = Math.max((int) correct, 0) if curr > 0: # only correct from 2nd values self.publish[curr] = correct # print correct, "\t", self.publish[curr], self.K, self.P # update estimation error variance self.P *= (1 - self.K) def getLastQuery(self, curr): """ generated source for method getLastQuery """ for i in reversed(range(curr)): if self.query[i]: break return self.publish[i] # adaptive sampling - return feedback error def PID(self, curr): """ generated source for method PID """ sum = 0 lastValue = 0 change = 0 timeDiff = 0 next = curr for j in reversed(range(self.windowPID - 1)): index = next while index >= 0: if self.query[index]: next = index - 1 # the last nextQuery break index -= 1 if j == self.windowPID - 1: lastValue = abs(self.publish[index] - self.predict[index]) / ( 0.0 + max(self.publish[index], 1)) change = abs(self.publish[index] - self.predict[index]) / ( 0.0 + max(self.publish[index], 1)) timeDiff = index if j == self.windowPID - 2: change -= abs(self.publish[index] - self.predict[index]) / ( 0.0 + max(self.publish[index], 1)) timeDiff -= index sum += (abs(self.publish[index] - self.predict[index]) / (0.0 + max(self.publish[index], 1))) ratio = self.Cp * lastValue + self.Ci * sum + self.Cd * change / ( 0.0 + timeDiff) return ratio
class Client(object): def __init__(self, torrent): self.torrent = torrent self.torrent_state = 'random' self.reactor = Reactor() self.reactor_activated = False self.peer_id = '-TZ-0000-00000000000' self.peers = [] self.decode_torrent_and_setup_pieces() self.handshake = self.build_handshake() self.setup_tracker() self.stitcher = Stitcher(self) self.setup_peers() def decode_torrent_and_setup_pieces(self): f = open(self.torrent, 'r') metainfo = B.bdecode(f.read()) data = metainfo['info'] # Un-bencoded dictionary self.info_hash = H.sha1(B.bencode(data)).digest() self.announce_url = self.find_http_announce_url(metainfo) #self.announce_url = 'http://tracker.ccc.de:6969/announce' self.file_name = data['name'] # Dir name if multi, otherwise file name self.piece_length = data['piece length'] if 'files' in data: # Multifile torrent self.setup_multi_file_info(data) else: self.setup_single_file_info(data) self.setup_download_directory() self.check_if_dload_file_exists() self.setup_pieces(self.piece_length, data['pieces']) def find_http_announce_url(self, metainfo): print metainfo.keys() # print metainfo['announce-list'] if self.is_http_url(metainfo['announce']): return metainfo['announce'] elif 'announce-list' in metainfo.keys(): for url in metainfo['announce-list']: url = url[0] if self.is_http_url(url): print url return url raise SystemExit('UDP announce urls are not supported. Currently only HTTP is supported.') def is_http_url(self, url): return 'http://' in url def setup_multi_file_info(self, metainfo): self.is_multi_file = True self.files = metainfo['files'] # dictionary of file lengths + paths self.file_length = sum([file_dict['length'] for file_dict in self.files]) # file_length = total # bytes to dload def setup_single_file_info(self, metainfo): self.is_multi_file = False self.file_length = metainfo['length'] def build_handshake(self): logging.info('Building handshake') pstr = 'BitTorrent protocol' handshake = struct.pack('B' + str(len(pstr)) + 's8x20s20s', # 8x => reserved null bytes len(pstr), pstr, self.info_hash, self.peer_id ) assert handshake != None assert len(handshake) == 49 + len(pstr) logging.info('Handshake constructed.') return handshake def setup_tracker(self): self.tracker = Tracker(self, self.announce_url) def setup_peers(self): peer_ips = self.tracker.send_request_and_parse_response() self.connect_to_peers(peer_ips) def connect_to_peers(self, peer_tuples): peers = [Peer(ip, port, self) for ip, port in peer_tuples] logging.debug('Attempting to connect to peers %s', peer_tuples) for peer in peers: try: if peer.ip == self.get_self_ip(): logging.info('Skipping peer; cannot connect to self') continue peer.connect() peer_handshake = peer.send_and_receive_handshake(self.handshake) logging.debug('Handshake returned.') if peer.verify_handshake(peer_handshake, self.info_hash): logging.debug('Handshake verified. Adding peer to peer list') self.add_peer(peer) if not self.reactor_activated: self.activate_reactor() self.reactor_activated = True except IOError as e: logging.warning('Error in construct_peers! %s', e) self.manage_requests(5) def get_self_ip(self): # http://stackoverflow.com/questions/166506/finding-local-ip-addresses-using-pythons-stdlib/166520#166520 s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) s.connect(('8.8.8.8', 80)) ip = s.getsockname()[0] s.close() return ip def add_peer(self, peer): logging.info('Adding peer %s to peer list (in add_peer)', peer) self.peers.append(peer) self.reactor.add_peer_socket(peer) def activate_reactor(self): logging.debug('Activating reactor.') self.reactor.get_data() def process_raw_hash_list(self, hash_list, size): tmp_hash = '' piece_hashes = [] for char in hash_list: if len(tmp_hash) < size: tmp_hash = tmp_hash + char else: piece_hashes.append(tmp_hash) tmp_hash = char piece_hashes.append(tmp_hash) return piece_hashes def setup_pieces(self, length, hash_bytes): hash_list = self.process_raw_hash_list(hash_bytes, 20) logging.info('setting up pieces for file length, %s', length) pieces = [] self.num_pieces = len(hash_list) logging.info('dividing up file into %s pieces', self.num_pieces) self.bitfield = BitArray(self.num_pieces) last_piece_length = self.file_length - (self.num_pieces - 1) * length for i in range(self.num_pieces): if i == self.num_pieces - 1: length = last_piece_length pieces.append(Piece(self, i, length, hash_list[i], self.dload_dir)) self.pieces = pieces self.piece_queue = PieceQueue(pieces) def setup_download_directory(self): dir_name = self.torrent if dir_name.endswith('.torrent'): dir_name = dir_name[:-8] self.dload_dir = os.path.join(os.path.abspath(os.curdir), dir_name) try: os.makedirs(self.dload_dir) except OSError: if not os.path.isdir(self.dload_dir): raise SystemExit('Cannot create directory to download torrent files into. Please check if a file named ' + dir_name + ' exists') # raise OSError('Cannot create directory to download torrent files to.') def check_if_dload_file_exists(self): file_path = os.path.join(self.dload_dir, self.file_name) if os.path.exists(file_path): raise SystemExit('This file has already been downloaded.') # Do something to cancel the rest of the setup def add_piece_to_queue(self, piece): self.piece_queue.put(piece) def add_piece_to_bitfield(self, index): if not self.bitfield[index]: self.bitfield.invert(index) self.manage_requests() else: logging.warning('Should never get save same piece more than once!') def add_peer_to_piece_peer_list(self, piece_index, peer): # print 'Adding piece', piece_index, 'to peer', peer self.pieces[piece_index].add_peer_to_peer_list(peer) def manage_requests(self, num_pieces=1): logging.info('Sending more piece requests') logging.info('Piece queue has %s pieces', self.piece_queue.length()) if not self.piece_queue.empty(): self.manage_piece_queue_state(); for i in xrange(num_pieces): self.request_next_piece(); logging.info('Cleaning up piece queue') else: # Count outstanding requests to decide when to go into endgame self.torrent_state = 'endgame' self.start_endgame() def start_endgame(self): self.blasted_requests = [] for i in xrange(ENDGAME_MAX_BLASTS): self.send_endgame_request() def send_endgame_request(self): block_info = self.select_outstanding_request() if block_info: self.blasted_requests.append(block_info) self.pieces(block_info[0]).request_block_endgame(block_info) def select_outstanding_request(self): # TODO: Use filter instead of picking at random peers_with_requests = filter(lambda peer: len(peer.outstanding_requests) > 0, self.peers) if len(peers_with_requests): peer = random.choice(peers_with_requests) block_info = random.choice(peer.outstanding_requests) return block_info def manage_piece_queue_state(self): # This should probably only get called occasionally logging.debug('Have received %s pieces, need %s more', self.bitfield.count(1), self.bitfield.count(0)) if self.bitfield.count(1) > PIECE_THRESHOLD and self.piece_queue.length() > PIECE_THRESHOLD: self.piece_queue.update_piece_order() self.torrent_state = 'rarest_first' # DISPATCHES TO PIECE def request_block(self, block_info): piece_index = block_info[0] self.pieces[piece_index].request_block(block_info) def request_next_piece(self): next_piece = self.piece_queue.get_next_piece(self.torrent_state) logging.info('Requesting piece %s', next_piece) if next_piece: try: next_piece.request_all_blocks() except IndexError as e: self.piece_queue.put(next_piece) logging.error(e) def add_block(self, block_info, block): (piece_index, begin, block_length) = block_info logging.info('Writing block of length %s at index %s for piece %s', block_length, begin, piece_index) piece = self.pieces[piece_index] logging.info('Piece has index %s', piece.index) piece.add_block(begin, block) self.tracker.update_download_stats(block_length) if self.torrent_state == 'endgame' and block_info in self.blasted_requests: piece = self.pieces[block_info[0]] piece.cancel_block(block_info, self) if self.bitfield.count(0) > 0: self.send_endgame_request() if self.num_pieces - self.bitfield.count(1) == 0: self.finalize_download() def finalize_download(self): logging.info('Finalizing download') if not self.tracker.is_download_complete(): raise SystemExit('Download didnt complete. Shutting down.') self.stitch_files() self.tracker.send_completed_msg_to_tracker_server() logging.info('Shutting down connection with peers') for peer in self.peers: peer.close() print 'Quitting client' logging.info('Download completed. Quitting client.') sys.exit() def stitch_files(self): print 'stitching...' logging.info('Wrote all pieces, stitching them together') self.stitcher.stitch() logging.info('Stitching completed.') def finalize_piece(self, piece): if piece.check_info_hash(): logging.debug('Yay! Correct info hash!') self.add_piece_to_bitfield(piece_index) else: logging.debug('Incorrect infohash, starting over with piece %s', piece_index) piece.reset()
if not os.path.isdir("data"): os.mkdir("data") for (index_bits, fill_perc) in sets: size = pow(2, index_bits) fill_count = round(fill_perc * size / 100) inv_count = size - fill_count if fill_perc > 50 else fill_count # slower method # registry.invert(range(0, fill_count)) # random.shuffle(registry) if fill_perc == 50: registry = BitArray(os.urandom(size // 8)) filled = registry.count(1) if filled > fill_count: registry.invert() filled = size - filled else: filled = 0 registry = BitArray(length=size) for _ in range(index_bits): registry.set(1, random.choices(range(0, size), k=(inv_count - filled))) filled = registry.count(1) if inv_count - filled < 10: break while filled < inv_count: pos = random.randrange(size)
def test1(s): b = BitArray(s) c = b.count(True) return c
class KalmanFilterPID(Parser): """ generated source for class KalmanFilterPID """ # sampling rate def __init__(self, param): """ generated source for method __init__ """ Parser.__init__(self) self.param = param self.differ = Differential(self.param.Seed) self.predict = [] self.interval = None # Kalman Filter params self.P = 100 # estimation error covariance (over all time instance) self.Q = 1000 # process noise synthetic data self.R = 1000000 # measurement noise optimal for alpha = 1, synthetic data self.K = 0 # kalman gain # PID control params - default self.Cp = 0.9 # proportional gain, to keep output proportional to current error self.Ci = 0.1 # integral gain, to eliminate offset self.Cd = 0.0 # derivative gain, to ensure stability - prevent large error in future # fixed internally self.theta = 1 # magnitude of changes self.xi = 0.2 # gamma (10%) self.minIntvl = 1 # make sure the interval is greater than 1 self.windowPID = 5 # I(integration) window self.ratioM = 0.2 # sampling rate # self.isSampling = False def adjustParams(self): # adjust params if self.ratioM < 0.1: self.theta = 20 if 0.1 <= self.ratioM < 0.2: self.theta = 14 if 0.2 <= self.ratioM < 0.3: self.theta = 2 if 0.3 <= self.ratioM < 0.4: self.theta = 0.5 if 0.4 <= self.ratioM < 0.5: self.theta = 0.3 if 0.5 <= self.ratioM: self.theta = 0.1 # test @classmethod def main(self, args): """ generated source for method main """ if len(args) < 5: print "Usage: python KalmanFilterPID.py input output privacy-budget process-variance Cp(optional) Ci(optional) Cd(optional)" sys.exit() output = open(args[2], "w") budget = eval(args[3]) Q = float(args[4]) if budget <= 0 or Q <= 0: print "Usage: privacy-budget AND process-variance are positive values" sys.exit() p = Params(1000) kfPID = KalmanFilterPID(p) kfPID.setTotalBudget(budget) kfPID.setQ(Q) kfPID.orig = Parser.getData(args[1]) kfPID.publish = [None] * len(kfPID.orig) # adjust R based on T and alpha kfPID.setR(len(kfPID.orig) * len(kfPID.orig) / (0.0 + budget * budget)) # set optional control gains if len(args) >= 6: d = args[5] if d > 1: d = 1 kfPID.setCp(d) if len(args) >= 7: d = args[6] if d + kfPID.Cp > 1: d = 1 - kfPID.Cp kfPID.setCi(d) else: kfPID.setCi(1 - kfPID.Cp) if len(args) >= 8: d = args[7] if d + kfPID.Cp + kfPID.Ci > 1: d = 1 - kfPID.Cp - kfPID.Ci kfPID.setCd(d) else: kfPID.setCd(1 - kfPID.Cp - kfPID.Ci) # kfPID.adjustParams() start = time.time() kfPID.publishCounts() end = time.time() Parser.outputData(output, kfPID.publish) print "Method:\tKalman Filter with Adaptive Sampling" print "Data Series Length:\t" + str(len(kfPID.orig)) print "Queries Issued:\t" + str(kfPID.query.count(1)) print "Privacy Budget Used:\t" + str(kfPID.query.count(1) * kfPID.epsilon) print "Average Relative Error:\t" + str(kfPID.getRelError()) print "Time Used (in second):\t" + str(end - start) def kalmanFilter(self, orig, budget, samplingRate=None): self.totalBudget = budget self.orig = orig if samplingRate is not None: self.isSampling = True self.ratioM = samplingRate else: self.isSampling = False # self.adjustParams() self.publish = [None] * len(self.orig) # adjust R based on T and alpha self.setR(len(self.orig) * len(self.orig) / (0.0 + budget * budget)) self.publishCounts() return self.publish def getCount(self, value, epsilon): """ return true count or noisy count of a node, depending on epsilon. Note that the noisy count can be negative """ if epsilon < 10 ** (-8): return value else: return value + self.differ.getNoise(1, epsilon) # sensitivity is 1 # data publication procedure def publishCounts(self): """ generated source for method publish """ self.query = BitArray(len(self.orig)) self.predict = [None] * len(self.orig) # recalculate individual budget based on M if (self.isSampling): M = int(self.ratioM * (len(self.orig))) # 0.25 optimal percentile else: M = len(self.orig) if M <= 0: M = 1 self.epsilon = (self.totalBudget + 0.0) / M # error = 0 self.interval = 1 nextQuery = max(1, self.windowPID) + self.interval - 1 for i in range(len(self.orig)): if i == 0: # the first time instance self.publish[i] = self.getCount(self.orig[i], self.epsilon) self.query[i] = 1 self.correctKF(i, 0) else: predct = self.predictKF(i) self.predict[i] = predct if self.query.count(1) < self.windowPID and self.query.count(1) < M: # i is NOT the sampling point self.publish[i] = self.getCount(self.orig[i], self.epsilon) self.query[i] = 1 # update count using observation self.correctKF(i, predct) elif i == nextQuery and self.query.count(1) < M: # if i is the sampling point # query self.publish[i] = self.getCount(self.orig[i], self.epsilon) self.query[i] = 1 # update count using observation self.correctKF(i, predct) # update freq if (self.isSampling): ratio = self.PID(i) frac = min(20, (ratio - self.xi) / self.xi) deltaI = self.theta * (1 - math.exp(frac)) deltaI = int(deltaI) + (random.random() < deltaI - int(deltaI)) self.interval += deltaI else: self.interval = 1 if self.interval < self.minIntvl: self.interval = self.minIntvl nextQuery += self.interval # nextQuery is ns in the paper else: # --> predict self.publish[i] = predct # del self.orig # del self.predict # del self.query # if self.isPostProcessing: # self.postProcessing() # def postProcessing(self): # print len(self.samples), self.samples # remainedEps = self.totalBudget - len(self.samples) * self.epsilon # self.epsilon = self.epsilon + remainedEps/len(self.samples) # # # recompute noisy counts # prev = 0 # for i in self.samples: # self.publish[i] = self.getCount(self.orig[i], self.epsilon) # if i > prev + 1: # self.publish[prev + 1 : i] = [self.publish[prev]] * (i - prev - 1) # prev = i def setR(self, r): """ generated source for method setR """ self.R = r def setQ(self, q): """ generated source for method setQ """ self.Q = q def setCp(self, cp): """ generated source for method setCp """ self.Cp = cp def setCi(self, ci): """ generated source for method setCi """ self.Ci = ci def setCd(self, cd): """ generated source for method setCd """ self.Cd = cd # prediction step def predictKF(self, curr): """ generated source for method predictKF """ # predict using Kalman Filter lastValue = self.getLastQuery(curr) # project estimation error self.P += self.Q # Q is gaussian noise return lastValue # correction step def correctKF(self, curr, predict): """ generated source for method correctKF """ self.K = (self.P + 0.0) / (self.P + self.R) correct = predict + self.K * (self.publish[curr] - predict) # publish[curr] = Math.max((int) correct, 0) if curr > 0: # only correct from 2nd values self.publish[curr] = correct # print correct, "\t", self.publish[curr], self.K, self.P # update estimation error variance self.P *= (1 - self.K) def getLastQuery(self, curr): """ generated source for method getLastQuery """ for i in reversed(range(curr)): if self.query[i]: break return self.publish[i] # adaptive sampling - return feedback error def PID(self, curr): """ generated source for method PID """ sum = 0 lastValue = 0 change = 0 timeDiff = 0 next = curr for j in reversed(range(self.windowPID - 1)): index = next while index >= 0: if self.query[index]: next = index - 1 # the last nextQuery break index -= 1 if j == self.windowPID - 1: lastValue = abs(self.publish[index] - self.predict[index]) / (0.0 + max(self.publish[index], 1)) change = abs(self.publish[index] - self.predict[index]) / (0.0 + max(self.publish[index], 1)) timeDiff = index if j == self.windowPID - 2: change -= abs(self.publish[index] - self.predict[index]) / (0.0 + max(self.publish[index], 1)) timeDiff -= index sum += (abs(self.publish[index] - self.predict[index]) / (0.0 + max(self.publish[index], 1))) ratio = self.Cp * lastValue + self.Ci * sum + self.Cd * change / (0.0 + timeDiff) return ratio
from Crypto.Cipher import DES from bitstring import BitArray import itertools import sys import copy L = ['0','1','2','3','4','5','6','7','8','9','A','B','C','D','E','F'] odd_parity = dict() for i in range(pow(2,7)): k = BitArray(uint=i, length=7) p = copy.copy(k) if k.count(1)%2 == 0: p.append('0b1') else: p.append('0b0') odd_parity[k.bin] = p.tobytes() def replace(indices, elements, keylist): for i in range(len(indices)): keylist[indices[i]] = elements[i] def DES_crypt(keybits, plainbytes, mode): rev_keybytes = '' for k in range(0, 56, 7): rev_keybytes += (odd_parity[keybits[k:k+7]]) obj = DES.new(rev_keybytes, DES.MODE_ECB) if mode == True: cipherbytes = obj.encrypt(plainbytes) else: cipherbytes = obj.decrypt(plainbytes)
class Sidewalk: """Models a sidewalk aas a MxM grid with BitArray Note that for efficiency, the grid is alternatively set to 0 or 1 The filling algorithm is varied based on dot size and binning """ def __init__( self, mesh, dot ): self.mesh_side = mesh self.mesh_size = mesh*mesh self.dot_side = dot self.dot_size = dot*dot self.safe_side = self.mesh_side - self.dot_side self.sidewalk = BitArray( length=self.mesh_size ) self.current = False self.nex = True self.Header() def SingleDot( self ): self.sidewalk[random.randrange(self.mesh_size)] = self.nex def MultiDot( self ): iy = random.randrange(self.mesh_side) ix = random.randrange(self.mesh_side) for jy in range(self.dot_side): ky = ((iy+jy) % self.mesh_side)*self.mesh_side if ix <= self.safe_side: self.sidewalk.set( self.nex, range( ix+ky, ix+ky+self.dot_side ) ) else: # split range for jx in range(self.dot_side): self.sidewalk[ ky + (iy+jy)%self.mesh_side] = self.nex def Cover( self ): """One pass covering sidewalk, return number needed""" cover_count = 0 if self.dot_side == 1: #simpler single dot while True: left = self.mesh_size - self.sidewalk.count(self.nex) if left == 0: break ; cover_count += left for i in range(left): self.SingleDot() else: while True: left = self.mesh_size - self.sidewalk.count(self.nex) if left == 0: break ; for i in range(0,left,self.dot_size): cover_count += 1 self.MultiDot() self.current = self.nex self.nex = not self.nex return cover_count def Coverbin( self, binwidth ): """One pass covering sidewalk, return bins needed""" bin_number = 0 if self.dot_side == 1: #simpler single dot while True: if self.sidewalk.count(self.current) == 0 : break ; bin_number += 1 for i in range(binwidth): self.SingleDot() else: while True: if self.sidewalk.count(self.current) == 0 : break ; for i in range(binwidth): bin_number += 1 self.MultiDot() self.current = self.nex self.nex = not self.nex return bin_number def Header(self): if Globals.quiet<2: print( "Sidewalk coverage\n\tMesh {:d} X {:d} = {:d}\n\tDot {:d} X {:d} = {:d}\n".format(self.mesh_side,self.mesh_side,self.mesh_size,self.dot_side,self.dot_side,self.dot_size))
def split(self, A): gain_list = [] A_count = float(A.count(True)) d = self.mean(A) if A_count <= self.count_thresh: #print "Node has few elements:", A_count return None,None,d,None,None mse_A = self.mse([A]) #if mse_A <= 15: # print "Node has small MSE:", mse_A # return None,None,d,None,None print "Elem, MSE:",A_count,mse_A F = self.X.T for f in range(len(F)): #print f # Sort the relevant column and keep indices indices = F[f].argsort() pairs = zip(indices, F[f][indices]) s = len(pairs)*'0' B = BitArray(bin=s) C = A.copy() i = 0 gain_prev = 0.0 # Threshold emulator loop while i < len(pairs)-1: if A[pairs[i][0]]: B[pairs[i][0]] = 1 C[pairs[i][0]] = 0 if pairs[i][1] < pairs[i+1][1]: t = pairs[i+1][1] # Calculate MSE for the split mse_curr = self.mse([B,C]) gain_curr = mse_A - mse_curr if gain_curr < 0: print gain_curr #print mse_curr if gain_curr < gain_prev: pass #break else: gain_prev = gain_curr # Check if entropy for any branches is below thresh if B.count(True) == 0 or C.count(True) == 0: pass else: gain_list.append((gain_curr,f,t,d,B.copy(),C.copy(),mse_A,mse_curr)) i += 1 if gain_list == []: print "mse_list empty" return None,None,d,None,None else: max_val = max(mse[0] for mse in gain_list) gain,f,t,d,B,C,mse_A,mse_curr = [mse for mse in gain_list if mse[0] == max_val][0] if B.count(True) == 0 or C.count(True) == 0: print "Count of B or C = 0:",B.count(True), C.count(True) for elem in gain_list: print elem[4].count(True), elem[5].count(True) print return None,None,d,None,None else: return f,t,d,B,C