class BloomFilter:
def __init__(self, mapsize=160000, max_node_size=10000, random_num=8):
self.m = mapsize
self.n = max_node_size
self.k = random_num
self.bitmap = BitMap(maxnum=self.m)
self.count = 0
pass
def set(self, string):
calcmap = self.calcMap(string)
for x in calcmap:
self.bitmap.set(x)
pass
def test(self, string):
calcmap = self.calcMap(string)
for x in calcmap:
if not self.bitmap.test(x):
return False
return True
def calcMap(self, string):
r = random.Random(string)
lv1random = [r.random() for x in range(self.k)]
return [int(random.Random(x).random() * self.m) for x in lv1random]
class TorrentFile(object):
def __init__(self, file_length, pieces_hash_array, piece_size, fs):
self.logger = logging.getLogger(__name__)
self.pieces_num = file_length / PIECE_SIZE
self.piece_list = []
for piece in range(self.pieces_num):
self.piece_list.append(
Piece(piece, piece_size, fs, pieces_hash_array[piece]))
#if have extra smaller pieces
if file_length % PIECE_SIZE != 0:
last_piece_size = file_length % piece_size
self.piece_list.append(
Piece(self.pieces_num, last_piece_size, fs,
pieces_hash_array[self.pieces_num]))
self.pieces_num += 1
self.bm = BitMap(self.pieces_num)
self.logger.debug("init bm {}".format(self.bm))
def update_bitmap(self):
for i in range(self.pieces_num):
if self.piece_list[i].is_complete():
self.bm.set(i)
else:
self.bm.reset(i)
def is_complete(self):
# self.update_bitmap()
return self.bm.all()
## return the missing pieces
def missing_pieces(self):
self.update_bitmap()
missing_list = []
for i in range(self.pieces_num):
if not self.bm.test(i):
missing_list.append(self.piece_list[i])
return missing_list
def set_piece(self, index, content):
self.piece_list[index].set_block(content)
self.piece_list[index].write_to_file()
def get_info(self):
for i in range(self.pieces_num):
self.piece_list[i].get_info()
DEADMAP = BitMap(5000000)
else:
print("No deadmap")
# API calls per window
max_api_calls = 300
# 5 minute windows
window_seconds = 300
api_calls = 0
last_thing = args.start
window_start_time = datetime.now()
logging.info(f"Starting window at {window_start_time}")
try:
for thing_id in range(last_thing, 5000000):
if DEADMAP.test(thing_id):
logging.info("Skipping dead thing {}".format(thing_id))
continue
logging.info("Getting thing {}".format(thing_id))
dn = dirname_from_id(thing_id)
os.makedirs(f'thing_json/{dn}', exist_ok=True)
thing_fname = f'thing_json/{dn}/{thing_id}.json'
t = None
try:
if os.path.exists(thing_fname):
with open(thing_fname, 'r') as f:
t = json.load(f)
except json.decoder.JSONDecodeError:
t = None
if not t:
while True:
from bitmap import BitMap
import random
array = range(100)
randomArray = []
while (len(array) > 0):
index = random.randint(0, len(array) - 1)
randomArray.append(array.pop(index))
print randomArray
size = 10000
mp = BitMap(size)
for i in randomArray:
mp.set(i)
for i in range(size):
if mp.test(i):
print i
class buddy_malloc:
def __init__(self, size, start, headerSize, align):
self.level = math.ceil(math.log2(size))
self.size = 1 << self.level
print("Total size for buddy: {}".format(self.size))
self.level += 1
self.buckets = [set() for i in range(self.level)]
self.in_use = BitMap(2 ** self.level - 1)
# initialize root node
self.buckets[0].add(start)
# misc stuffs
self.base = start
self.headerSize = headerSize
self.align = align
self.sizemap = {}
def _ptr_to_index(self, ptr, level):
return (1 << level) - 1 + ((ptr - self.base) >> (self.level - 1 - level))
def _index_to_ptr(self, index, level):
return self.base + ((index - (1 << level) + 1) << (self.level - 1 - level))
def _parent(self, n):
return (n - 1) >> 1
def _lchild(self, n):
return (n << 1) + 1
def _rchild(self, n):
return (n + 1) << 1
def addToMap(self, addr, size):
assert(addr not in self.sizemap)
self.sizemap[addr] = size
def malloc(self, size):
if self.align != -1:
left = size % self.align
if left != 0:
diff = self.align - left
else:
diff = 0
# print 'aligning: adding %d to %d' % (diff, size)
size += diff
size += self.headerSize
dest_level = self.level - 1 - math.ceil(math.log2(size))
level = dest_level
while not len(self.buckets[level]) and level >= 0:
level -= 1
has_flipped = False
if level < 0:
return (-1, 0)
else:
# split curr level
while level != dest_level:
blk = self.buckets[level].pop()
idx = self._ptr_to_index(blk, level)
level += 1
self.buckets[level].add(blk)
self.buckets[level].add(blk + (1 << (self.level - 1 - level)))
addr = self.buckets[level].pop()
idx = self._ptr_to_index(addr, level)
while idx >= 0:
self.in_use.set(idx)
idx = self._parent(idx)
self.addToMap(addr, 1 << (self.level - 1 - level))
return (addr, 0)
def free(self, addr):
if addr not in self.sizemap:
return -1
level = self.level - 1 - math.ceil(math.log2(self.sizemap[addr]))
idx = self._ptr_to_index(addr, level)
while idx > 0:
parent = self._parent(idx)
neighbor = self._rchild(parent) if self._lchild(parent) == idx else self._lchild(parent)
self.in_use.reset(idx)
if not self.in_use.test(neighbor):
# can merge
self.buckets[level].remove(self._index_to_ptr(neighbor, level))
else:
break
level -= 1
idx = parent
self.buckets[level].add(self._index_to_ptr(idx, level))
del self.sizemap[addr]
return 0
def dump(self):
print('Alloc List [ Size %d ]: ' % len(self.sizemap), end='')
for item in self.sizemap.items():
print('[ addr:%d sz:%d ]' % item, end='')
print('')
