class BloomFilter:
def __init__(self, mapsize=160000, max_node_size=10000, random_num=8):
self.m = mapsize
self.n = max_node_size
self.k = random_num
self.bitmap = BitMap(maxnum=self.m)
self.count = 0
pass
def set(self, string):
calcmap = self.calcMap(string)
for x in calcmap:
self.bitmap.set(x)
pass
def test(self, string):
calcmap = self.calcMap(string)
for x in calcmap:
if not self.bitmap.test(x):
return False
return True
def calcMap(self, string):
r = random.Random(string)
lv1random = [r.random() for x in range(self.k)]
return [int(random.Random(x).random() * self.m) for x in lv1random]
class BitfieldMessage(PeerMessage):
def __init__(self, msg_len, msg_id, msg):
PeerMessage.__init__(self, msg_id, msg)
bit_string = ''
for i in range(0, len(msg)):
num = ord(msg[i])
bit_string += "{0:b}".format(ord(msg[i]))
self.bitfield = BitMap(len(bit_string))
self.bitfield_str = bit_string
for i in range(0, len(bit_string)):
if bit_string[i] == '1':
self.bitfield.set(i)
def list_of_ids_to_bitmap(ids, maxitems, bitmap=None):
"""Convert list of 'id' values (entry numbers) to bitmap / bitset
Each 'id' value is an entry number in the original dataset, that
is the 'id' column in the database. Those have are natural
numbers, and have values between 1 and number of entries; the
latter is given as parameter to this function.
Returned bitmap has i-th bit set to "1" (has "1" at i-th place) if
and only if there was identifier 'i' on the list.
Parameters
----------
ids | list
[Sorted] list of integers with values between 1 and
`maxitems`, inclusive; those are 'id' fields for given bug
report / bugfix commit.
maxitems : int
Maximum value of ids onn the list, which is number of entries
in the dataset; this means that it is the minimal number of
bits in the bitmap / bitset.
bitmap : BitMap, optional
Bitmap object to set, in current incarnation in needs
.set(i-th) method to set i-th bit to "1" in resulting bitmap.
The bitmap must be empty (all zeros), and have at least
`maxitems` bits.
Returns
-------
bitmap.BitMap
Bitmap with appropriate bits set. Note that values are from 1
to maxitems, while bit positions are numbered from 0 to
maxitems-1.
"""
if not bitmap:
bitmap = BitMap(maxitems)
# there is no built-in initialization from iterable for bitmap.BitMap
for i in ids:
# ids are numbered from 1, bits are numbered from 0
bitmap.set(i - 1)
return bitmap
class TorrentFile(object):
def __init__(self, file_length, pieces_hash_array, piece_size, fs):
self.logger = logging.getLogger(__name__)
self.pieces_num = file_length / PIECE_SIZE
self.piece_list = []
for piece in range(self.pieces_num):
self.piece_list.append(
Piece(piece, piece_size, fs, pieces_hash_array[piece]))
#if have extra smaller pieces
if file_length % PIECE_SIZE != 0:
last_piece_size = file_length % piece_size
self.piece_list.append(
Piece(self.pieces_num, last_piece_size, fs,
pieces_hash_array[self.pieces_num]))
self.pieces_num += 1
self.bm = BitMap(self.pieces_num)
self.logger.debug("init bm {}".format(self.bm))
def update_bitmap(self):
for i in range(self.pieces_num):
if self.piece_list[i].is_complete():
self.bm.set(i)
else:
self.bm.reset(i)
def is_complete(self):
# self.update_bitmap()
return self.bm.all()
## return the missing pieces
def missing_pieces(self):
self.update_bitmap()
missing_list = []
for i in range(self.pieces_num):
if not self.bm.test(i):
missing_list.append(self.piece_list[i])
return missing_list
def set_piece(self, index, content):
self.piece_list[index].set_block(content)
self.piece_list[index].write_to_file()
def get_info(self):
for i in range(self.pieces_num):
self.piece_list[i].get_info()
def test_thread():
global exitFlag
exitFlag = 0
threads = []
threadID = 1
print "Starting Main Thread"
# Fill the queue
for i in range(3):
queue = Queue.Queue(4)
for j in range(4):
queue.put(i*4 + j)
pieces_queue_list.append(queue)
for i in range(3):
queue = Queue.Queue(4)
data_queue_list.append(queue)
for i , tName in enumerate(threadList):
thread = myThread(threadID, tName, pieces_queue_list[i],data_queue_list[i])
thread.start()
threads.append(thread)
threadID += 1
# Wait for all threads to complete
for t in threads:
t.join()
print "Exiting Main Thread"
bm = BitMap(12)
while 1:
if bm.all():
print "receive all the pieces!!"
break
for q in data_queue_list:
if not q.empty():
item = q.get()
data = item['data']
index = item['index']
print "receive pieces:%s data:%s\n" % (data,data)
bm.set(index)
class Piece(object):
def __init__(self, index, piece_length, file_to_write, piece_hash):
self.index = index
self.piece_length = piece_length
self.num_blocks = piece_length/BLOCK_SIZE
self.file_to_write = file_to_write
self.piece_hash = piece_hash
self.block_list = []
for block in range(self.num_blocks):
self.block_list.append(Block(block, BLOCK_SIZE))
if piece_length % BLOCK_SIZE != 0:
last_block_length = piece_length % BLOCK_SIZE
self.block_list.append(Block(self.num_blocks, last_block_length))
self.num_blocks = self.num_blocks + 1
self.bitmap = BitMap(self.num_blocks)
# if self.index == 16:
# print "init bm: ", self.bitmap
def is_piece_full(self):
for block in range(len(self.block_list)):
if not self.block_list[block].is_block_full():
return False
return True
def fill_the_block(self, block_index, block_content):
if self.block_list[block_index].is_block_full():
return
self.block_list[block_index].write(block_content)
self.bitmap.set(block_index)
def write(self):
if self.bitmap.all():
bytes_to_write = bytearray('')
for block in self.block_list:
bytes_to_write = bytes_to_write + block.block_content
self.file_to_write.seek(self.index * self.piece_length)
self.file_to_write.write(bytes_to_write)
class FileWriter(object):
def __init__(self, total_length, file_to_write, piece_hash_array):
self.num_pieces = total_length / PIECE_SIZE
self.piece_list = []
for piece in range(self.num_pieces):
self.piece_list.append(Piece(piece, PIECE_SIZE, file_to_write, piece_hash_array[piece]))
if total_length % PIECE_SIZE != 0:
last_piece_length = total_length % PIECE_SIZE
self.piece_list.append(Piece(self.num_pieces, last_piece_length, file_to_write, piece_hash_array[self.num_pieces]))
self.num_pieces = self.num_pieces + 1
self.bitmap = BitMap(self.num_pieces)
# print "self.num_pieces, init bm: ", self.num_pieces, self.bitmap
def is_file_full(self):
for piece in range(len(self.piece_list)):
# print piece, self.piece_list[piece].bitmap.size()
if not self.piece_list[piece].is_piece_full():
return False
return True
def update_file_bitmap(self, piece_index):
if self.piece_list[piece_index].bitmap.all():
self.bitmap.set(piece_index)
output_dir = get_output_dir(imdb)
print 'Output will be saved to `{:s}`'.format(output_dir)
# some statistic to record
alamount = 0
ssamount = 0
# set bitmap for AL
tableA = BitMap(imdb.num_images)
# choose initiail samples:VOC2007
sample_num = imdb.num_images
train_num = len(imdb[imdb.item_name(0)].roidb)
print 'All VOC2007 images use for initial train, image numbers:%d' % (
train_num)
for i in range(train_num):
tableA.set(i)
train_roidb = [roidb[i] for i in range(train_num)]
pretrained_model_name = args.pretrained_model
# static parameters
tao = 60000
beta = 1000
# updatable hypeparameters
gamma = 0.6
mylambda = np.array([-np.log(0.9)] * imdb.num_classes)
# train record
loopcounter = 0
train_iters = 0
iters_sum = train_iters
discard_num = 0
net = resnetv1(num_layers=50)
elif args.net == 'res101':
net = resnetv1(num_layers=101)
elif args.net == 'res152':
net = resnetv1(num_layers=152)
elif args.net == 'mobile':
net = mobilenetv1()
else:
raise NotImplementedError
# some statistic record
al_num = 0
ss_num = 0
initial_num = len(imdb[imdb.item_name(0)].roidb)
print ('All VOC2007 images use for initial train after flipped, imagenumbers:%d'%(initial_num))
for i in range(initialnum):
bitmapImdb.set(i)
train_roidb = imdb[imdb.item_name(0)].roidb
# pretrained-model
pretrained_model_name = args.weight
# some parameters
tao = args.max_iters
gamma = 0.15; clslambda = np.array([-np.log(0.9)]*imdb.num_classes)
# train record
loopcounter = 0; train_iters = 0;iters_sum = train_iters
# control al proportion
al_proportion_checkpoint = [x*initialnum for x in np.linspace(0.3,2,10)]
# control ss proportion with respect to al proportion
ss_proportion_checkpoint = [x*initialnum for x in np.linspace(0.2,2,10)]
# get solver object
sw = SolverWrapper(net, imdb, train_roidb, valroidb, output_dir, tb_dir,
def main():
# Initialize C2
workspace.GlobalInit(
['caffe2', '--caffe2_log_level=0', '--caffe2_gpu_memory_tracking=1'])
# Set up logging and load config options
logger = setup_logging(__name__)
logging.getLogger('detectron.roi_data.loader').setLevel(logging.INFO)
args = parse_args()
logger.info('Called with args:')
logger.info(args)
if args.cfg_file is not None:
merge_cfg_from_file(args.cfg_file)
if args.opts is not None:
merge_cfg_from_list(args.opts)
assert_and_infer_cfg()
logger.info('Training with config:')
logger.info(pprint.pformat(cfg))
# Note that while we set the numpy random seed network training will not be
# deterministic in general. There are sources of non-determinism that cannot
# be removed with a reasonble execution-speed tradeoff (such as certain
# non-deterministic cudnn functions).
np.random.seed(cfg.RNG_SEED)
# Execute the training run
fs = open('imgnames.pkl', 'rb')
roidbnames = pickle.load(fs)
fs.close()
logger.info('Loading dataset: {}'.format(cfg.TRAIN.DATASETS))
dataset_names = cfg.TRAIN.DATASETS
proposal_files = cfg.TRAIN.PROPOSAL_FILES
roidb = get_training_roidb(dataset_names, proposal_files)
logger.info('{:d} roidb entries'.format(len(roidb)))
total_num = len(roidb)
# bitmap idx indicated for training
bitmapRoidb = BitMap(total_num)
# initial samples
# initial_num = int(total_num*0.2)
# for i in range(initial_num):
# bitmapRoidb.set(i)
#
# train_roidb = [roidb[i] for i in range(initial_num)]
initialidx = []
train_roidb = []
for i, x in enumerate(roidb):
if x['image'].split('/')[-1] in roidbnames:
initialidx.append(i)
train_roidb.append(x)
for i in initialidx:
bitmapRoidb.set(i)
logger.info('{:d} the number initial roidb entries'.format(
len(train_roidb)))
# append flipped images
train_roidb = flipped_roidb_for_training(train_roidb)
logger.info('{:d} the number initial roidb entries'.format(
len(train_roidb)))
alamount = 0
ssamount = 0
gamma = 0.95
# control al proportion
al_proportion_checkpoint = [
int(x * total_num * 0.4) for x in np.linspace(0.2, 1, 10)
]
# control ss proportion
ss_proportion_checkpoint = [
int(x * total_num) for x in np.linspace(0.2, 2, 10)
]
next_iters = 90000
sum_iters = next_iters
'''load the lasted checkpoints'''
checkpoints = detectron.utils.train.train_model(sum_iters, train_roidb,
cfg.TRAIN.WEIGHTS)
while True:
# to do a test on the test dataset
test_model(checkpoints[(sum_iters - 1)], args.multi_gpu_testing,
args.opts)
if sum_iters > cfg.SOLVER.MAX_ITER:
break
# next detect unlabeled samples
unlabeledidx = list(set(range(total_num)) - set(bitmapRoidb.nonzero()))
# labeled samples
labeledidx = list(set(bitmapRoidb.nonzero()))
# detect unlabeled samples
BBoxes, YClass, Scores, al_candidate_idx, ALScore = detect_im(
checkpoints[(sum_iters - 1)],
roidb,
gamma,
idxs=unlabeledidx,
gpu_id=0)
al_avg_idx = np.argsort(np.array(ALScore))
al_candidate_idx = [al_candidate_idx[i] for i in al_avg_idx]
gamma = max(gamma - 0.05, 0.7)
# the ss candidate idx
ss_candidate_idx = [
i for i in unlabeledidx if i not in al_candidate_idx
]
# update roidb for next training
train_roidb = replace_roidb(roidb, BBoxes, YClass, ss_candidate_idx)
# control the proportion
if alamount + len(al_candidate_idx) >= al_proportion_checkpoint[0]:
al_candidate_idx = al_candidate_idx[:int(
al_proportion_checkpoint[0] - alamount)]
tmp = al_proportion_checkpoint.pop(0)
al_proportion_checkpoint.append(al_proportion_checkpoint[-1])
if ssamount + len(ss_candidate_idx) >= ss_proportion_checkpoint[0]:
ss_candidate_idx = ss_candidate_idx[:int(
ss_proportion_checkpoint[0] - ssamount)]
tmp = ss_proportion_checkpoint.pop(0)
ss_proportion_checkpoint.append(ss_proportion_checkpoint[-1])
# record ss and al factor
alamount += len(al_candidate_idx)
ssamount += len(ss_candidate_idx)
logger.info('alfactor:{},ssfactor:{}'.format(alamount / total_num,
ssamount / total_num))
# for idx in al_candidate_idx:
# bitmapRoidb.set(idx)
next_train_idx = bitmapRoidb.nonzero()
next_train_idx.extend(ss_candidate_idx)
train_roidb = blur_image(train_roidb, ss_candidate_idx)
# the next training roidb
train_roidb = [train_roidb[i] for i in next_train_idx]
# flipped the roidb
train_roidb = flipped_roidb_for_training(train_roidb)
# the next training iters
next_iters = 30000
sum_iters += next_iters
checkpoints = detectron.utils.train.train_model(
sum_iters, train_roidb, checkpoints[(sum_iters - next_iters - 1)])
result = []
for item in mysql_dict:
if mysql_dict[item] == 1:
result.append(item)
def testb(b):
length = max(max(b), len(b))
dest_list = [0 for i in xrange(length + 1)]
for i in b:
dest_list[i] += 1
result = []
for index, i in enumerate(dest_list):
if i == 1:
result.append(index)
from bitmap import BitMap
bm = BitMap(32)
print bm.tostring()
bm.set(21)
print bm.tostring()
from array import array
array.fromfile()
from cli import lock
f.write(chunk)
clen += len(chunk)
print("Received {} bytes".format(clen))
except KeyboardInterrupt as e:
raise e
except:
print("Exception downloading, giving up")
else:
print("Error getting zip")
else:
print("File already exists")
else:
print("Error fetching thing")
if DEADMAP is not None:
print("Adding to deadmap")
DEADMAP.set(thing_id)
last_thing += 1
if api_calls >= max_api_calls:
window_left = window_seconds - (datetime.now() -
window_start_time).total_seconds()
if window_left < 0:
window_left = 0
print(
"Max API calls reached for the window, sleeping {} seconds...".
format(window_left))
time.sleep(window_left)
api_calls = 0
window_start_time = datetime.now()
logging.info(f"Starting new window at {window_start_time}")
except KeyboardInterrupt:
print("Keyboard Interrupt!")
class Piece(object):
def __init__(self, piece_index, piece_size, fs, piece_hash):
self.block_num = piece_size / BLOCK_SIZE
self.block_list = []
self.piece_index = piece_index
self.piece_size = piece_size
self.piece_hash = piece_hash
for block in range(self.block_num):
b = Block(piece_index, block, BLOCK_SIZE)
self.block_list.append(b)
#if have extra smaller blocks
if piece_size % BLOCK_SIZE != 0:
last_block_size = piece_size % BLOCK_SIZE
#print self.block_num,last_block_size
self.block_list.append(
Block(self.piece_index, self.block_num, last_block_size))
self.block_num += 1
self.bm = BitMap(self.block_num)
self.file = fs
return
def __hash__(self):
return hash((self.piece_index, self.piece_size, self.piece_hash))
def __eq__(self, other):
return (self.piece_index, self.piece_size,
self.piece_hash) == (other.piece_index, self.piece_size,
self.piece_hash)
def __ne__(self, other):
return not (self == other)
#check block's bitmap in disk and the whole SHA1
def is_complete(self):
#check SHA1
self.file.seek(self.piece_index * PIECE_SIZE)
content = self.file.read(self.piece_size)
piecehash = hashlib.sha1(content).digest()
# print piecehash, self.piece_hash
# print self.piece_size
return piecehash == self.piece_hash
def written_piece_hash(self):
self.file.seek(self.piece_index * PIECE_SIZE)
content = self.file.read(self.piece_size)
piecehash = hashlib.sha1(content).digest()
return piecehash
def update_bitmap(self):
for i in range(len(self.block_list)):
if self.block_list[i].is_complete():
self.bm.set(i)
else:
self.bm.reset(i)
def get_info(self):
print "piece index:", self.piece_index, " piece size:", self.piece_size
#for i in range(self.block_num):
# self.block_list[i].get_info()
## set every block
def set_block(self, content):
size = len(content)
for block in self.block_list:
#print "boffset",block.block_offset
block.set_complete(
content[block.block_offset:min(block.block_offset +
BLOCK_SIZE, size)])
## get the content from blocks and write to file
def write_to_file(self):
self.update_bitmap()
if self.bm.all():
payload = bytearray('')
for block in self.block_list:
payload += block.payload
self.file.seek(self.piece_index * PIECE_SIZE)
self.file.write(payload)
def clear_data(self):
for block in self.block_list:
block.payload = None
def expire(self):
for block in self.block_list:
block.mark_missing()
self.bm = BitMap(self.block_num)
from bitmap import BitMap
import random
array = range(100)
randomArray = []
while (len(array) > 0):
index = random.randint(0, len(array) - 1)
randomArray.append(array.pop(index))
print randomArray
size = 10000
mp = BitMap(size)
for i in randomArray:
mp.set(i)
for i in range(size):
if mp.test(i):
print i
class buddy_malloc:
def __init__(self, size, start, headerSize, align):
self.level = math.ceil(math.log2(size))
self.size = 1 << self.level
print("Total size for buddy: {}".format(self.size))
self.level += 1
self.buckets = [set() for i in range(self.level)]
self.in_use = BitMap(2 ** self.level - 1)
# initialize root node
self.buckets[0].add(start)
# misc stuffs
self.base = start
self.headerSize = headerSize
self.align = align
self.sizemap = {}
def _ptr_to_index(self, ptr, level):
return (1 << level) - 1 + ((ptr - self.base) >> (self.level - 1 - level))
def _index_to_ptr(self, index, level):
return self.base + ((index - (1 << level) + 1) << (self.level - 1 - level))
def _parent(self, n):
return (n - 1) >> 1
def _lchild(self, n):
return (n << 1) + 1
def _rchild(self, n):
return (n + 1) << 1
def addToMap(self, addr, size):
assert(addr not in self.sizemap)
self.sizemap[addr] = size
def malloc(self, size):
if self.align != -1:
left = size % self.align
if left != 0:
diff = self.align - left
else:
diff = 0
# print 'aligning: adding %d to %d' % (diff, size)
size += diff
size += self.headerSize
dest_level = self.level - 1 - math.ceil(math.log2(size))
level = dest_level
while not len(self.buckets[level]) and level >= 0:
level -= 1
has_flipped = False
if level < 0:
return (-1, 0)
else:
# split curr level
while level != dest_level:
blk = self.buckets[level].pop()
idx = self._ptr_to_index(blk, level)
level += 1
self.buckets[level].add(blk)
self.buckets[level].add(blk + (1 << (self.level - 1 - level)))
addr = self.buckets[level].pop()
idx = self._ptr_to_index(addr, level)
while idx >= 0:
self.in_use.set(idx)
idx = self._parent(idx)
self.addToMap(addr, 1 << (self.level - 1 - level))
return (addr, 0)
def free(self, addr):
if addr not in self.sizemap:
return -1
level = self.level - 1 - math.ceil(math.log2(self.sizemap[addr]))
idx = self._ptr_to_index(addr, level)
while idx > 0:
parent = self._parent(idx)
neighbor = self._rchild(parent) if self._lchild(parent) == idx else self._lchild(parent)
self.in_use.reset(idx)
if not self.in_use.test(neighbor):
# can merge
self.buckets[level].remove(self._index_to_ptr(neighbor, level))
else:
break
level -= 1
idx = parent
self.buckets[level].add(self._index_to_ptr(idx, level))
del self.sizemap[addr]
return 0
def dump(self):
print('Alloc List [ Size %d ]: ' % len(self.sizemap), end='')
for item in self.sizemap.items():
print('[ addr:%d sz:%d ]' % item, end='')
print('')
