def get_patterns(size: int, error_size: int) -> List[List[int]]:
patterns = []
for i in range(2**size):
pattern = BitArray(uint=i, length=size)
if pattern.count(1) == error_size:
patterns.append(to_list(pattern))
return patterns
def floatable(mem_addr: bitstring.BitArray, mask: bitstring.BitArray):
amount_of_trues = mask.count(True)
for possible_setting in range(2**amount_of_trues):
possible_setting = get_bitsstring(possible_setting)
possible_setting_index = -1
for index in mask.findall([1]):
mem_addr[index] = possible_setting[possible_setting_index]
possible_setting_index -= 1
yield mem_addr
def encode(data: list, encoding: str) -> List[BitArray]:
"""encodes elements of the list into packets and generates parity bit for each packet"""
ready_packets = []
for d in data:
tmp = BitArray(bytearray(d, encoding))
if tmp.count(True) % 2 == 1:
tmp.append(BitArray(uint=1, length=1))
else:
tmp.append(BitArray(uint=0, length=1))
ready_packets.append(tmp)
return ready_packets
class mpt1327_state:
def __init__(self):
self.data = BitArray(uint=0, length=64)
self.cnt = 0
self.codeword = 0
self.prev = 0
self.base_freq = 170.8
self.step_freq = 0.0125
def crc(self):
data, checksum = self.data.unpack('uint:48, uint:15')
return CRC.calcWord(data, 48) == checksum and self.data.count(1) % 2 == 0 # Even parity
def test_valid_replace_bits(num, start, stop, bit_idx, count):
ga = RealGA(fitness_test_sin_func, cross_prob=1)
ga.interval = (-numpy.inf, numpy.inf)
source_bstr = BitArray(floatbe=num, length=ga._bin_length).bin
new_chromosome = ga._replace_bits(num, 0, start, stop)
actual_bstr = BitArray(floatbe=new_chromosome, length=ga._bin_length).bin
for i in bit_idx:
assert actual_bstr[i] == source_bstr[i]
assert actual_bstr.count('1') == count
def test_valid_replace_bits(num, start, stop, bit_idx, count):
ga = RealGA(fitness_test_sin_func, cross_prob=1)
ga.interval = (-numpy.inf, numpy.inf)
source_bstr = BitArray(floatbe=num, length=ga._bin_length).bin
new_chromosome = ga._replace_bits(num, 0, start, stop)
actual_bstr = BitArray(floatbe=new_chromosome, length=ga._bin_length).bin
for i in bit_idx:
assert actual_bstr[i] == source_bstr[i]
assert actual_bstr.count('1') == count
def test_dirty_bit_pos(self):
'''
Test ``bbc.dirty_bit_pos()`` against all possible bytes.
'''
for bits in it.product(*it.repeat(['0', '1'], 8)):
byte = BitArray(bin=''.join(bits))
if byte.count(1) == 1:
# offset byte found
self.assertEqual(bbc.dirty_bit_pos(byte), byte.find('0b1')[0])
else:
# non-offset byte found
self.assertEqual(bbc.dirty_bit_pos(byte), -1)
def monobits(s, v):
res = True
chkvals = []
b = BitArray(s)
c = b.count(True)
if v == 1:
chkvals = [9654, 10346]
elif v == 2:
chkvals = [9725, 10275]
if c <chkvals[0] or c>chkvals[1]:
res = False
return res, c
def get_distance(self):
"""
Assuming that the two bit strings are the same length:
1. XOR the two bit strings. This will calculate which positions are different.
2. Count the number of set bits from the result of step 1.
"""
self._pre_process()
bit_str_len = self.bit_str1.len
resulting_diff_stream = BitArray(bin='0')
for i in range(0, bit_str_len):
result = self.bit_str1[i] ^ self.bit_str2[i]
resulting_diff_stream.append(BitArray(bool=result))
return resulting_diff_stream.count(1)
def set_hammingArray(character):
#make a bitarray for the input character
char_bitarray = BitArray('int:8=' + str(ord(character)))
#make a bit array to hold the channel coded version of the character.
char_hammingArray = BitArray('0b0000000000000')
#transfer all bits from the char_bitarray to the char_hammingArray skipping the parity bit positions
i = 0
j = 0
length = len(char_hammingArray)
while (i < length):
if i in parity_positions:
i += 1
continue
if char_bitarray[j]:
char_hammingArray[i] = 1
else:
char_hammingArray[i] = 0
i += 1
j += 1
#set each parity bit for the hamming code: 1 if checksum is odd, 0 if checksum is even
for bit in parity_positions:
#calculate the value of the check sum for the current parity bit
check_sum = calc_checksum(char_hammingArray, bit)
#if the checksum is odd, set the parity bit to 1, otherwise leave at 0
if (check_sum % 2 != 0):
char_hammingArray[bit] = 1
char_hammingArray[-1] = 0
total_parity = char_hammingArray.count(1)
if (total_parity % 2 != 0):
char_hammingArray[-1] = 1
return char_hammingArray
def set_hammingArray(character):
#make a bitarray for the input character
char_bitarray = BitArray('int:8='+str(ord(character)))
#make a bit array to hold the channel coded version of the character.
char_hammingArray = BitArray('0b0000000000000')
#transfer all bits from the char_bitarray to the char_hammingArray skipping the parity bit positions
i = 0
j = 0
length = len(char_hammingArray)
while (i < length):
if i in parity_positions:
i += 1
continue
if char_bitarray[j]:
char_hammingArray[i] = 1
else:
char_hammingArray[i] = 0
i += 1
j += 1
#set each parity bit for the hamming code: 1 if checksum is odd, 0 if checksum is even
for bit in parity_positions:
#calculate the value of the check sum for the current parity bit
check_sum = calc_checksum(char_hammingArray, bit)
#if the checksum is odd, set the parity bit to 1, otherwise leave at 0
if(check_sum%2!=0):
char_hammingArray[bit] = 1
char_hammingArray[-1] = 0
total_parity = char_hammingArray.count(1)
if(total_parity%2!=0):
char_hammingArray[-1] = 1
return char_hammingArray
class KalmanFilterPID(Parser):
""" generated source for class KalmanFilterPID """
# sampling rate
def __init__(self, param):
"""
generated source for method __init__
"""
Parser.__init__(self)
self.param = param
self.differ = Differential(self.param.Seed)
self.predict = []
self.interval = None
# Kalman Filter params
self.P = 100
# estimation error covariance (over all time instance)
self.Q = 1000
# process noise synthetic data
self.R = 1000000
# measurement noise optimal for alpha = 1, synthetic data
self.K = 0
# kalman gain
# PID control params - default
self.Cp = 0.9 # proportional gain, to keep output proportional to current error
self.Ci = 0.1 # integral gain, to eliminate offset
self.Cd = 0.0 # derivative gain, to ensure stability - prevent large error in future
# fixed internally
self.theta = 1 # magnitude of changes
self.xi = 0.2 # gamma (10%)
self.minIntvl = 1 # make sure the interval is greater than 1
self.windowPID = 5 # I(integration) window
self.ratioM = 0.2 # sampling rate
#
self.isSampling = False
def adjustParams(self):
# adjust params
if self.ratioM < 0.1:
self.theta = 20
if 0.1 <= self.ratioM < 0.2:
self.theta = 14
if 0.2 <= self.ratioM < 0.3:
self.theta = 2
if 0.3 <= self.ratioM < 0.4:
self.theta = 0.5
if 0.4 <= self.ratioM < 0.5:
self.theta = 0.3
if 0.5 <= self.ratioM:
self.theta = 0.1
# test
@classmethod
def main(self, args):
""" generated source for method main """
if len(args) < 5:
print "Usage: python KalmanFilterPID.py input output privacy-budget process-variance Cp(optional) Ci(optional) Cd(optional)"
sys.exit()
output = open(args[2], "w")
budget = eval(args[3])
Q = float(args[4])
if budget <= 0 or Q <= 0:
print "Usage: privacy-budget AND process-variance are positive values"
sys.exit()
p = Params(1000)
kfPID = KalmanFilterPID(p)
kfPID.setTotalBudget(budget)
kfPID.setQ(Q)
kfPID.orig = Parser.getData(args[1])
kfPID.publish = [None] * len(kfPID.orig)
# adjust R based on T and alpha
kfPID.setR(len(kfPID.orig) * len(kfPID.orig) / (0.0 + budget * budget))
# set optional control gains
if len(args) >= 6:
d = args[5]
if d > 1:
d = 1
kfPID.setCp(d)
if len(args) >= 7:
d = args[6]
if d + kfPID.Cp > 1:
d = 1 - kfPID.Cp
kfPID.setCi(d)
else:
kfPID.setCi(1 - kfPID.Cp)
if len(args) >= 8:
d = args[7]
if d + kfPID.Cp + kfPID.Ci > 1:
d = 1 - kfPID.Cp - kfPID.Ci
kfPID.setCd(d)
else:
kfPID.setCd(1 - kfPID.Cp - kfPID.Ci)
# kfPID.adjustParams()
start = time.time()
kfPID.publishCounts()
end = time.time()
Parser.outputData(output, kfPID.publish)
print "Method:\tKalman Filter with Adaptive Sampling"
print "Data Series Length:\t" + str(len(kfPID.orig))
print "Queries Issued:\t" + str(kfPID.query.count(1))
print "Privacy Budget Used:\t" + str(
kfPID.query.count(1) * kfPID.epsilon)
print "Average Relative Error:\t" + str(kfPID.getRelError())
print "Time Used (in second):\t" + str(end - start)
def kalmanFilter(self, orig, budget, samplingRate=None):
self.totalBudget = budget
self.orig = orig
if samplingRate is not None:
self.isSampling = True
self.ratioM = samplingRate
else:
self.isSampling = False
# self.adjustParams()
self.publish = [None] * len(self.orig)
# adjust R based on T and alpha
self.setR(len(self.orig) * len(self.orig) / (0.0 + budget * budget))
self.publishCounts()
return self.publish
def getCount(self, value, epsilon):
"""
return true count or noisy count of a node, depending on epsilon.
Note that the noisy count can be negative
"""
if epsilon < 10**(-8):
return value
else:
return value + self.differ.getNoise(1, epsilon) # sensitivity is 1
# data publication procedure
def publishCounts(self):
""" generated source for method publish """
self.query = BitArray(len(self.orig))
self.predict = [None] * len(self.orig)
# recalculate individual budget based on M
if (self.isSampling):
M = int(self.ratioM * (len(self.orig))) # 0.25 optimal percentile
else:
M = len(self.orig)
if M <= 0:
M = 1
self.epsilon = (self.totalBudget + 0.0) / M
# error = 0
self.interval = 1
nextQuery = max(1, self.windowPID) + self.interval - 1
for i in range(len(self.orig)):
if i == 0:
# the first time instance
self.publish[i] = self.getCount(self.orig[i], self.epsilon)
self.query[i] = 1
self.correctKF(i, 0)
else:
predct = self.predictKF(i)
self.predict[i] = predct
if self.query.count(1) < self.windowPID and self.query.count(
1) < M:
# i is NOT the sampling point
self.publish[i] = self.getCount(self.orig[i], self.epsilon)
self.query[i] = 1
# update count using observation
self.correctKF(i, predct)
elif i == nextQuery and self.query.count(1) < M:
# if i is the sampling point
# query
self.publish[i] = self.getCount(self.orig[i], self.epsilon)
self.query[i] = 1
# update count using observation
self.correctKF(i, predct)
# update freq
if (self.isSampling):
ratio = self.PID(i)
frac = min(20, (ratio - self.xi) / self.xi)
deltaI = self.theta * (1 - math.exp(frac))
deltaI = int(deltaI) + (random.random() <
deltaI - int(deltaI))
self.interval += deltaI
else:
self.interval = 1
if self.interval < self.minIntvl:
self.interval = self.minIntvl
nextQuery += self.interval # nextQuery is ns in the paper
else:
# --> predict
self.publish[i] = predct
# del self.orig
# del self.predict
# del self.query
# if self.isPostProcessing:
# self.postProcessing()
# def postProcessing(self):
# print len(self.samples), self.samples
# remainedEps = self.totalBudget - len(self.samples) * self.epsilon
# self.epsilon = self.epsilon + remainedEps/len(self.samples)
#
# # recompute noisy counts
# prev = 0
# for i in self.samples:
# self.publish[i] = self.getCount(self.orig[i], self.epsilon)
# if i > prev + 1:
# self.publish[prev + 1 : i] = [self.publish[prev]] * (i - prev - 1)
# prev = i
def setR(self, r):
""" generated source for method setR """
self.R = r
def setQ(self, q):
""" generated source for method setQ """
self.Q = q
def setCp(self, cp):
""" generated source for method setCp """
self.Cp = cp
def setCi(self, ci):
""" generated source for method setCi """
self.Ci = ci
def setCd(self, cd):
""" generated source for method setCd """
self.Cd = cd
# prediction step
def predictKF(self, curr):
""" generated source for method predictKF """
# predict using Kalman Filter
lastValue = self.getLastQuery(curr)
# project estimation error
self.P += self.Q # Q is gaussian noise
return lastValue
# correction step
def correctKF(self, curr, predict):
""" generated source for method correctKF """
self.K = (self.P + 0.0) / (self.P + self.R)
correct = predict + self.K * (self.publish[curr] - predict)
# publish[curr] = Math.max((int) correct, 0)
if curr > 0:
# only correct from 2nd values
self.publish[curr] = correct
# print correct, "\t", self.publish[curr], self.K, self.P
# update estimation error variance
self.P *= (1 - self.K)
def getLastQuery(self, curr):
""" generated source for method getLastQuery """
for i in reversed(range(curr)):
if self.query[i]:
break
return self.publish[i]
# adaptive sampling - return feedback error
def PID(self, curr):
""" generated source for method PID """
sum = 0
lastValue = 0
change = 0
timeDiff = 0
next = curr
for j in reversed(range(self.windowPID - 1)):
index = next
while index >= 0:
if self.query[index]:
next = index - 1 # the last nextQuery
break
index -= 1
if j == self.windowPID - 1:
lastValue = abs(self.publish[index] - self.predict[index]) / (
0.0 + max(self.publish[index], 1))
change = abs(self.publish[index] - self.predict[index]) / (
0.0 + max(self.publish[index], 1))
timeDiff = index
if j == self.windowPID - 2:
change -= abs(self.publish[index] - self.predict[index]) / (
0.0 + max(self.publish[index], 1))
timeDiff -= index
sum += (abs(self.publish[index] - self.predict[index]) /
(0.0 + max(self.publish[index], 1)))
ratio = self.Cp * lastValue + self.Ci * sum + self.Cd * change / (
0.0 + timeDiff)
return ratio
class Client(object):
def __init__(self, torrent):
self.torrent = torrent
self.torrent_state = 'random'
self.reactor = Reactor()
self.reactor_activated = False
self.peer_id = '-TZ-0000-00000000000'
self.peers = []
self.decode_torrent_and_setup_pieces()
self.handshake = self.build_handshake()
self.setup_tracker()
self.stitcher = Stitcher(self)
self.setup_peers()
def decode_torrent_and_setup_pieces(self):
f = open(self.torrent, 'r')
metainfo = B.bdecode(f.read())
data = metainfo['info'] # Un-bencoded dictionary
self.info_hash = H.sha1(B.bencode(data)).digest()
self.announce_url = self.find_http_announce_url(metainfo)
#self.announce_url = 'http://tracker.ccc.de:6969/announce'
self.file_name = data['name'] # Dir name if multi, otherwise file name
self.piece_length = data['piece length']
if 'files' in data: # Multifile torrent
self.setup_multi_file_info(data)
else:
self.setup_single_file_info(data)
self.setup_download_directory()
self.check_if_dload_file_exists()
self.setup_pieces(self.piece_length, data['pieces'])
def find_http_announce_url(self, metainfo):
print metainfo.keys()
# print metainfo['announce-list']
if self.is_http_url(metainfo['announce']):
return metainfo['announce']
elif 'announce-list' in metainfo.keys():
for url in metainfo['announce-list']:
url = url[0]
if self.is_http_url(url):
print url
return url
raise SystemExit('UDP announce urls are not supported. Currently only HTTP is supported.')
def is_http_url(self, url):
return 'http://' in url
def setup_multi_file_info(self, metainfo):
self.is_multi_file = True
self.files = metainfo['files'] # dictionary of file lengths + paths
self.file_length = sum([file_dict['length'] for file_dict in self.files]) # file_length = total # bytes to dload
def setup_single_file_info(self, metainfo):
self.is_multi_file = False
self.file_length = metainfo['length']
def build_handshake(self):
logging.info('Building handshake')
pstr = 'BitTorrent protocol'
handshake = struct.pack('B' + str(len(pstr)) + 's8x20s20s',
# 8x => reserved null bytes
len(pstr),
pstr,
self.info_hash,
self.peer_id
)
assert handshake != None
assert len(handshake) == 49 + len(pstr)
logging.info('Handshake constructed.')
return handshake
def setup_tracker(self):
self.tracker = Tracker(self, self.announce_url)
def setup_peers(self):
peer_ips = self.tracker.send_request_and_parse_response()
self.connect_to_peers(peer_ips)
def connect_to_peers(self, peer_tuples):
peers = [Peer(ip, port, self) for ip, port in peer_tuples]
logging.debug('Attempting to connect to peers %s', peer_tuples)
for peer in peers:
try:
if peer.ip == self.get_self_ip():
logging.info('Skipping peer; cannot connect to self')
continue
peer.connect()
peer_handshake = peer.send_and_receive_handshake(self.handshake)
logging.debug('Handshake returned.')
if peer.verify_handshake(peer_handshake, self.info_hash):
logging.debug('Handshake verified. Adding peer to peer list')
self.add_peer(peer)
if not self.reactor_activated:
self.activate_reactor()
self.reactor_activated = True
except IOError as e:
logging.warning('Error in construct_peers! %s', e)
self.manage_requests(5)
def get_self_ip(self):
# http://stackoverflow.com/questions/166506/finding-local-ip-addresses-using-pythons-stdlib/166520#166520
s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
s.connect(('8.8.8.8', 80))
ip = s.getsockname()[0]
s.close()
return ip
def add_peer(self, peer):
logging.info('Adding peer %s to peer list (in add_peer)', peer)
self.peers.append(peer)
self.reactor.add_peer_socket(peer)
def activate_reactor(self):
logging.debug('Activating reactor.')
self.reactor.get_data()
def process_raw_hash_list(self, hash_list, size):
tmp_hash = ''
piece_hashes = []
for char in hash_list:
if len(tmp_hash) < size:
tmp_hash = tmp_hash + char
else:
piece_hashes.append(tmp_hash)
tmp_hash = char
piece_hashes.append(tmp_hash)
return piece_hashes
def setup_pieces(self, length, hash_bytes):
hash_list = self.process_raw_hash_list(hash_bytes, 20)
logging.info('setting up pieces for file length, %s', length)
pieces = []
self.num_pieces = len(hash_list)
logging.info('dividing up file into %s pieces', self.num_pieces)
self.bitfield = BitArray(self.num_pieces)
last_piece_length = self.file_length - (self.num_pieces - 1) * length
for i in range(self.num_pieces):
if i == self.num_pieces - 1:
length = last_piece_length
pieces.append(Piece(self, i, length, hash_list[i], self.dload_dir))
self.pieces = pieces
self.piece_queue = PieceQueue(pieces)
def setup_download_directory(self):
dir_name = self.torrent
if dir_name.endswith('.torrent'):
dir_name = dir_name[:-8]
self.dload_dir = os.path.join(os.path.abspath(os.curdir), dir_name)
try:
os.makedirs(self.dload_dir)
except OSError:
if not os.path.isdir(self.dload_dir):
raise SystemExit('Cannot create directory to download torrent files into. Please check if a file named ' + dir_name + ' exists')
# raise OSError('Cannot create directory to download torrent files to.')
def check_if_dload_file_exists(self):
file_path = os.path.join(self.dload_dir, self.file_name)
if os.path.exists(file_path):
raise SystemExit('This file has already been downloaded.')
# Do something to cancel the rest of the setup
def add_piece_to_queue(self, piece):
self.piece_queue.put(piece)
def add_piece_to_bitfield(self, index):
if not self.bitfield[index]:
self.bitfield.invert(index)
self.manage_requests()
else:
logging.warning('Should never get save same piece more than once!')
def add_peer_to_piece_peer_list(self, piece_index, peer):
# print 'Adding piece', piece_index, 'to peer', peer
self.pieces[piece_index].add_peer_to_peer_list(peer)
def manage_requests(self, num_pieces=1):
logging.info('Sending more piece requests')
logging.info('Piece queue has %s pieces', self.piece_queue.length())
if not self.piece_queue.empty():
self.manage_piece_queue_state();
for i in xrange(num_pieces):
self.request_next_piece();
logging.info('Cleaning up piece queue')
else:
# Count outstanding requests to decide when to go into endgame
self.torrent_state = 'endgame'
self.start_endgame()
def start_endgame(self):
self.blasted_requests = []
for i in xrange(ENDGAME_MAX_BLASTS):
self.send_endgame_request()
def send_endgame_request(self):
block_info = self.select_outstanding_request()
if block_info:
self.blasted_requests.append(block_info)
self.pieces(block_info[0]).request_block_endgame(block_info)
def select_outstanding_request(self):
# TODO: Use filter instead of picking at random
peers_with_requests = filter(lambda peer: len(peer.outstanding_requests) > 0, self.peers)
if len(peers_with_requests):
peer = random.choice(peers_with_requests)
block_info = random.choice(peer.outstanding_requests)
return block_info
def manage_piece_queue_state(self):
# This should probably only get called occasionally
logging.debug('Have received %s pieces, need %s more', self.bitfield.count(1), self.bitfield.count(0))
if self.bitfield.count(1) > PIECE_THRESHOLD and self.piece_queue.length() > PIECE_THRESHOLD:
self.piece_queue.update_piece_order()
self.torrent_state = 'rarest_first'
# DISPATCHES TO PIECE
def request_block(self, block_info):
piece_index = block_info[0]
self.pieces[piece_index].request_block(block_info)
def request_next_piece(self):
next_piece = self.piece_queue.get_next_piece(self.torrent_state)
logging.info('Requesting piece %s', next_piece)
if next_piece:
try:
next_piece.request_all_blocks()
except IndexError as e:
self.piece_queue.put(next_piece)
logging.error(e)
def add_block(self, block_info, block):
(piece_index, begin, block_length) = block_info
logging.info('Writing block of length %s at index %s for piece %s',
block_length, begin, piece_index)
piece = self.pieces[piece_index]
logging.info('Piece has index %s', piece.index)
piece.add_block(begin, block)
self.tracker.update_download_stats(block_length)
if self.torrent_state == 'endgame' and block_info in self.blasted_requests:
piece = self.pieces[block_info[0]]
piece.cancel_block(block_info, self)
if self.bitfield.count(0) > 0:
self.send_endgame_request()
if self.num_pieces - self.bitfield.count(1) == 0:
self.finalize_download()
def finalize_download(self):
logging.info('Finalizing download')
if not self.tracker.is_download_complete():
raise SystemExit('Download didnt complete. Shutting down.')
self.stitch_files()
self.tracker.send_completed_msg_to_tracker_server()
logging.info('Shutting down connection with peers')
for peer in self.peers:
peer.close()
print 'Quitting client'
logging.info('Download completed. Quitting client.')
sys.exit()
def stitch_files(self):
print 'stitching...'
logging.info('Wrote all pieces, stitching them together')
self.stitcher.stitch()
logging.info('Stitching completed.')
def finalize_piece(self, piece):
if piece.check_info_hash():
logging.debug('Yay! Correct info hash!')
self.add_piece_to_bitfield(piece_index)
else:
logging.debug('Incorrect infohash, starting over with piece %s', piece_index)
piece.reset()
if not os.path.isdir("data"):
os.mkdir("data")
for (index_bits, fill_perc) in sets:
size = pow(2, index_bits)
fill_count = round(fill_perc * size / 100)
inv_count = size - fill_count if fill_perc > 50 else fill_count
# slower method
# registry.invert(range(0, fill_count))
# random.shuffle(registry)
if fill_perc == 50:
registry = BitArray(os.urandom(size // 8))
filled = registry.count(1)
if filled > fill_count:
registry.invert()
filled = size - filled
else:
filled = 0
registry = BitArray(length=size)
for _ in range(index_bits):
registry.set(1, random.choices(range(0, size), k=(inv_count - filled)))
filled = registry.count(1)
if inv_count - filled < 10:
break
while filled < inv_count:
pos = random.randrange(size)
def test1(s):
b = BitArray(s)
c = b.count(True)
return c
class KalmanFilterPID(Parser):
""" generated source for class KalmanFilterPID """
# sampling rate
def __init__(self, param):
"""
generated source for method __init__
"""
Parser.__init__(self)
self.param = param
self.differ = Differential(self.param.Seed)
self.predict = []
self.interval = None
# Kalman Filter params
self.P = 100
# estimation error covariance (over all time instance)
self.Q = 1000
# process noise synthetic data
self.R = 1000000
# measurement noise optimal for alpha = 1, synthetic data
self.K = 0
# kalman gain
# PID control params - default
self.Cp = 0.9 # proportional gain, to keep output proportional to current error
self.Ci = 0.1 # integral gain, to eliminate offset
self.Cd = 0.0 # derivative gain, to ensure stability - prevent large error in future
# fixed internally
self.theta = 1 # magnitude of changes
self.xi = 0.2 # gamma (10%)
self.minIntvl = 1 # make sure the interval is greater than 1
self.windowPID = 5 # I(integration) window
self.ratioM = 0.2 # sampling rate
#
self.isSampling = False
def adjustParams(self):
# adjust params
if self.ratioM < 0.1:
self.theta = 20
if 0.1 <= self.ratioM < 0.2:
self.theta = 14
if 0.2 <= self.ratioM < 0.3:
self.theta = 2
if 0.3 <= self.ratioM < 0.4:
self.theta = 0.5
if 0.4 <= self.ratioM < 0.5:
self.theta = 0.3
if 0.5 <= self.ratioM:
self.theta = 0.1
# test
@classmethod
def main(self, args):
""" generated source for method main """
if len(args) < 5:
print "Usage: python KalmanFilterPID.py input output privacy-budget process-variance Cp(optional) Ci(optional) Cd(optional)"
sys.exit()
output = open(args[2], "w")
budget = eval(args[3])
Q = float(args[4])
if budget <= 0 or Q <= 0:
print "Usage: privacy-budget AND process-variance are positive values"
sys.exit()
p = Params(1000)
kfPID = KalmanFilterPID(p)
kfPID.setTotalBudget(budget)
kfPID.setQ(Q)
kfPID.orig = Parser.getData(args[1])
kfPID.publish = [None] * len(kfPID.orig)
# adjust R based on T and alpha
kfPID.setR(len(kfPID.orig) * len(kfPID.orig) / (0.0 + budget * budget))
# set optional control gains
if len(args) >= 6:
d = args[5]
if d > 1:
d = 1
kfPID.setCp(d)
if len(args) >= 7:
d = args[6]
if d + kfPID.Cp > 1:
d = 1 - kfPID.Cp
kfPID.setCi(d)
else:
kfPID.setCi(1 - kfPID.Cp)
if len(args) >= 8:
d = args[7]
if d + kfPID.Cp + kfPID.Ci > 1:
d = 1 - kfPID.Cp - kfPID.Ci
kfPID.setCd(d)
else:
kfPID.setCd(1 - kfPID.Cp - kfPID.Ci)
# kfPID.adjustParams()
start = time.time()
kfPID.publishCounts()
end = time.time()
Parser.outputData(output, kfPID.publish)
print "Method:\tKalman Filter with Adaptive Sampling"
print "Data Series Length:\t" + str(len(kfPID.orig))
print "Queries Issued:\t" + str(kfPID.query.count(1))
print "Privacy Budget Used:\t" + str(kfPID.query.count(1) * kfPID.epsilon)
print "Average Relative Error:\t" + str(kfPID.getRelError())
print "Time Used (in second):\t" + str(end - start)
def kalmanFilter(self, orig, budget, samplingRate=None):
self.totalBudget = budget
self.orig = orig
if samplingRate is not None:
self.isSampling = True
self.ratioM = samplingRate
else:
self.isSampling = False
# self.adjustParams()
self.publish = [None] * len(self.orig)
# adjust R based on T and alpha
self.setR(len(self.orig) * len(self.orig) / (0.0 + budget * budget))
self.publishCounts()
return self.publish
def getCount(self, value, epsilon):
"""
return true count or noisy count of a node, depending on epsilon.
Note that the noisy count can be negative
"""
if epsilon < 10 ** (-8):
return value
else:
return value + self.differ.getNoise(1, epsilon) # sensitivity is 1
# data publication procedure
def publishCounts(self):
""" generated source for method publish """
self.query = BitArray(len(self.orig))
self.predict = [None] * len(self.orig)
# recalculate individual budget based on M
if (self.isSampling):
M = int(self.ratioM * (len(self.orig))) # 0.25 optimal percentile
else:
M = len(self.orig)
if M <= 0:
M = 1
self.epsilon = (self.totalBudget + 0.0) / M
# error = 0
self.interval = 1
nextQuery = max(1, self.windowPID) + self.interval - 1
for i in range(len(self.orig)):
if i == 0:
# the first time instance
self.publish[i] = self.getCount(self.orig[i], self.epsilon)
self.query[i] = 1
self.correctKF(i, 0)
else:
predct = self.predictKF(i)
self.predict[i] = predct
if self.query.count(1) < self.windowPID and self.query.count(1) < M:
# i is NOT the sampling point
self.publish[i] = self.getCount(self.orig[i], self.epsilon)
self.query[i] = 1
# update count using observation
self.correctKF(i, predct)
elif i == nextQuery and self.query.count(1) < M:
# if i is the sampling point
# query
self.publish[i] = self.getCount(self.orig[i], self.epsilon)
self.query[i] = 1
# update count using observation
self.correctKF(i, predct)
# update freq
if (self.isSampling):
ratio = self.PID(i)
frac = min(20, (ratio - self.xi) / self.xi)
deltaI = self.theta * (1 - math.exp(frac))
deltaI = int(deltaI) + (random.random() < deltaI - int(deltaI))
self.interval += deltaI
else:
self.interval = 1
if self.interval < self.minIntvl:
self.interval = self.minIntvl
nextQuery += self.interval # nextQuery is ns in the paper
else:
# --> predict
self.publish[i] = predct
# del self.orig
# del self.predict
# del self.query
# if self.isPostProcessing:
# self.postProcessing()
# def postProcessing(self):
# print len(self.samples), self.samples
# remainedEps = self.totalBudget - len(self.samples) * self.epsilon
# self.epsilon = self.epsilon + remainedEps/len(self.samples)
#
# # recompute noisy counts
# prev = 0
# for i in self.samples:
# self.publish[i] = self.getCount(self.orig[i], self.epsilon)
# if i > prev + 1:
# self.publish[prev + 1 : i] = [self.publish[prev]] * (i - prev - 1)
# prev = i
def setR(self, r):
""" generated source for method setR """
self.R = r
def setQ(self, q):
""" generated source for method setQ """
self.Q = q
def setCp(self, cp):
""" generated source for method setCp """
self.Cp = cp
def setCi(self, ci):
""" generated source for method setCi """
self.Ci = ci
def setCd(self, cd):
""" generated source for method setCd """
self.Cd = cd
# prediction step
def predictKF(self, curr):
""" generated source for method predictKF """
# predict using Kalman Filter
lastValue = self.getLastQuery(curr)
# project estimation error
self.P += self.Q # Q is gaussian noise
return lastValue
# correction step
def correctKF(self, curr, predict):
""" generated source for method correctKF """
self.K = (self.P + 0.0) / (self.P + self.R)
correct = predict + self.K * (self.publish[curr] - predict)
# publish[curr] = Math.max((int) correct, 0)
if curr > 0:
# only correct from 2nd values
self.publish[curr] = correct
# print correct, "\t", self.publish[curr], self.K, self.P
# update estimation error variance
self.P *= (1 - self.K)
def getLastQuery(self, curr):
""" generated source for method getLastQuery """
for i in reversed(range(curr)):
if self.query[i]:
break
return self.publish[i]
# adaptive sampling - return feedback error
def PID(self, curr):
""" generated source for method PID """
sum = 0
lastValue = 0
change = 0
timeDiff = 0
next = curr
for j in reversed(range(self.windowPID - 1)):
index = next
while index >= 0:
if self.query[index]:
next = index - 1 # the last nextQuery
break
index -= 1
if j == self.windowPID - 1:
lastValue = abs(self.publish[index] - self.predict[index]) / (0.0 + max(self.publish[index], 1))
change = abs(self.publish[index] - self.predict[index]) / (0.0 + max(self.publish[index], 1))
timeDiff = index
if j == self.windowPID - 2:
change -= abs(self.publish[index] - self.predict[index]) / (0.0 + max(self.publish[index], 1))
timeDiff -= index
sum += (abs(self.publish[index] - self.predict[index]) / (0.0 + max(self.publish[index], 1)))
ratio = self.Cp * lastValue + self.Ci * sum + self.Cd * change / (0.0 + timeDiff)
return ratio
from Crypto.Cipher import DES
from bitstring import BitArray
import itertools
import sys
import copy
L = ['0','1','2','3','4','5','6','7','8','9','A','B','C','D','E','F']
odd_parity = dict()
for i in range(pow(2,7)):
k = BitArray(uint=i, length=7)
p = copy.copy(k)
if k.count(1)%2 == 0: p.append('0b1')
else: p.append('0b0')
odd_parity[k.bin] = p.tobytes()
def replace(indices, elements, keylist):
for i in range(len(indices)):
keylist[indices[i]] = elements[i]
def DES_crypt(keybits, plainbytes, mode):
rev_keybytes = ''
for k in range(0, 56, 7):
rev_keybytes += (odd_parity[keybits[k:k+7]])
obj = DES.new(rev_keybytes, DES.MODE_ECB)
if mode == True:
cipherbytes = obj.encrypt(plainbytes)
else:
cipherbytes = obj.decrypt(plainbytes)
class Sidewalk:
"""Models a sidewalk aas a MxM grid with BitArray
Note that for efficiency, the grid is alternatively set to 0 or 1
The filling algorithm is varied based on dot size and binning
"""
def __init__( self, mesh, dot ):
self.mesh_side = mesh
self.mesh_size = mesh*mesh
self.dot_side = dot
self.dot_size = dot*dot
self.safe_side = self.mesh_side - self.dot_side
self.sidewalk = BitArray( length=self.mesh_size )
self.current = False
self.nex = True
self.Header()
def SingleDot( self ):
self.sidewalk[random.randrange(self.mesh_size)] = self.nex
def MultiDot( self ):
iy = random.randrange(self.mesh_side)
ix = random.randrange(self.mesh_side)
for jy in range(self.dot_side):
ky = ((iy+jy) % self.mesh_side)*self.mesh_side
if ix <= self.safe_side:
self.sidewalk.set( self.nex, range( ix+ky, ix+ky+self.dot_side ) )
else:
# split range
for jx in range(self.dot_side):
self.sidewalk[ ky + (iy+jy)%self.mesh_side] = self.nex
def Cover( self ):
"""One pass covering sidewalk, return number needed"""
cover_count = 0
if self.dot_side == 1:
#simpler single dot
while True:
left = self.mesh_size - self.sidewalk.count(self.nex)
if left == 0:
break ;
cover_count += left
for i in range(left):
self.SingleDot()
else:
while True:
left = self.mesh_size - self.sidewalk.count(self.nex)
if left == 0:
break ;
for i in range(0,left,self.dot_size):
cover_count += 1
self.MultiDot()
self.current = self.nex
self.nex = not self.nex
return cover_count
def Coverbin( self, binwidth ):
"""One pass covering sidewalk, return bins needed"""
bin_number = 0
if self.dot_side == 1:
#simpler single dot
while True:
if self.sidewalk.count(self.current) == 0 :
break ;
bin_number += 1
for i in range(binwidth):
self.SingleDot()
else:
while True:
if self.sidewalk.count(self.current) == 0 :
break ;
for i in range(binwidth):
bin_number += 1
self.MultiDot()
self.current = self.nex
self.nex = not self.nex
return bin_number
def Header(self):
if Globals.quiet<2:
print( "Sidewalk coverage\n\tMesh {:d} X {:d} = {:d}\n\tDot {:d} X {:d} = {:d}\n".format(self.mesh_side,self.mesh_side,self.mesh_size,self.dot_side,self.dot_side,self.dot_size))
def split(self, A):
gain_list = []
A_count = float(A.count(True))
d = self.mean(A)
if A_count <= self.count_thresh:
#print "Node has few elements:", A_count
return None,None,d,None,None
mse_A = self.mse([A])
#if mse_A <= 15:
# print "Node has small MSE:", mse_A
# return None,None,d,None,None
print "Elem, MSE:",A_count,mse_A
F = self.X.T
for f in range(len(F)):
#print f
# Sort the relevant column and keep indices
indices = F[f].argsort()
pairs = zip(indices, F[f][indices])
s = len(pairs)*'0'
B = BitArray(bin=s)
C = A.copy()
i = 0
gain_prev = 0.0
# Threshold emulator loop
while i < len(pairs)-1:
if A[pairs[i][0]]:
B[pairs[i][0]] = 1
C[pairs[i][0]] = 0
if pairs[i][1] < pairs[i+1][1]:
t = pairs[i+1][1]
# Calculate MSE for the split
mse_curr = self.mse([B,C])
gain_curr = mse_A - mse_curr
if gain_curr < 0:
print gain_curr
#print mse_curr
if gain_curr < gain_prev:
pass
#break
else:
gain_prev = gain_curr
# Check if entropy for any branches is below thresh
if B.count(True) == 0 or C.count(True) == 0:
pass
else:
gain_list.append((gain_curr,f,t,d,B.copy(),C.copy(),mse_A,mse_curr))
i += 1
if gain_list == []:
print "mse_list empty"
return None,None,d,None,None
else:
max_val = max(mse[0] for mse in gain_list)
gain,f,t,d,B,C,mse_A,mse_curr = [mse for mse in gain_list if mse[0] == max_val][0]
if B.count(True) == 0 or C.count(True) == 0:
print "Count of B or C = 0:",B.count(True), C.count(True)
for elem in gain_list:
print elem[4].count(True), elem[5].count(True)
print
return None,None,d,None,None
else:
return f,t,d,B,C
