def bytes_to_bitmap(buf, maxitems, bitmap=None): """Convert string of bytes back into bitmap This is the inverse of bytes_to_bitmap(), and is used to recover the bitmap from its representation as string of bytes. Parameters ---------- buf : bytes (str in Python 2) [Compact] representation of bitmap as bytes. Result of bitmap_to_bytes(). maxitems : int Number of entries; number of bits in bitmap. bitmap : BitMap, optional Bitmap object to set. If provided, the bitmap must be empty (all zeros), and have at least `maxitems` bits. This can be used to avoid commit creation costs. Returns ------- BitMap Bitmap / bitset with given representation. """ if not bitmap: bitmap = BitMap(maxitems) bitmap.bitmap = array.array('B', buf) return bitmap
def make_file(self, filetype, kernel_sz): # apply for inode, block map1 = BitMap(self.buf, self.start[0], self.size[1]) x = map1.apply_bit() inode_off = self.get_area(self.start[2], x, MkImg.NODE_SZ) map2 = BitMap(self.buf, self.start[1], self.size[2]) y = map2.apply_bit() block_off = self.get_area(self.start[3], y, self.bsize) # set block offset, type in inode node = Inode.from_buffer(self.buf, inode_off) # IDE driver node.dev_id = 5 node.index[0] = block_off + kernel_sz node.link_count = 1 node.type = filetype.value return inode_off, block_off
def __init__(self, size, frame_min, frame_max): self.repeats = dict() self.subseqs = dict() self.bitmap = BitMap(size) self.frame_min = frame_min self.frame_max = frame_max return
def __init__(self, mapsize=160000, max_node_size=10000, random_num=8): self.m = mapsize self.n = max_node_size self.k = random_num self.bitmap = BitMap(maxnum=self.m) self.count = 0 pass
def initLangaugeCounters(p, put=True): for lang in sentAnno.getLanguages(): p.languageSentenceIndex[lang] = 0 size = sentAnno.totalSentLang(lang) p.languageProgress[lang] = BitMap(size) if put: p.put()
def __init__(self, partition_size=2048, block_size=32): self.num_blocks = int(partition_size / block_size) self.block_size = block_size self.partition_size = partition_size self.partition = [0] * self.num_blocks self.bit_map = BitMap(block_size) self.root = Directory('root', None) self.actual_dir = self.root self.table = [" "] * self.num_blocks
def addBitMap(id, size): bm = BitMap(size) bmObj = BitMapTest( id=id, bitmap = bm, name = id ) bmObj.put() return bmObj
def loadBitmap(file) : # generate bitmap from lexicon file (one word per line) words = open(file).readlines() words = map(lambda x: x.strip(), words) # no newlines please bmap = BitMap(2**20) for word in words : hashes = makeHashes(word) for hash in hashes : bmap.setBit(hash) return bmap
def __init__(self, total_length, file_to_write, piece_hash_array): self.num_pieces = total_length / PIECE_SIZE self.piece_list = [] for piece in range(self.num_pieces): self.piece_list.append(Piece(piece, PIECE_SIZE, file_to_write, piece_hash_array[piece])) if total_length % PIECE_SIZE != 0: last_piece_length = total_length % PIECE_SIZE self.piece_list.append(Piece(self.num_pieces, last_piece_length, file_to_write, piece_hash_array[self.num_pieces])) self.num_pieces = self.num_pieces + 1 self.bitmap = BitMap(self.num_pieces)
class PidManager: # Minimum pid in range MIN_PID = 300 # Maximum pid in range MAX_PID = 5000 # Class variable keeps track of current pid pid_counter = 0 # Class variable keeps track of index in bit_map bit_index = 0 # BitMap has a size equal to the range of pids bit_map = BitMap(MAX_PID - MIN_PID) def allocate_map(self): # checks if a map exists if self.bit_map is not None: self.pid_counter = self.MIN_PID self.bit_index = 0 return 1 else: return -1 def allocate_pid(self): # check against empty bitmap if self.bit_map is not None: # this case will trigger when the pid_counter goes out of the range of pids if self.pid_counter > self.MAX_PID: # Set to -1, if not overwritten in the loop, method will return -1 pid = -1 # this loop finds the first empty bit in the bitmap for i in range(self.MIN_PID, self.MAX_PID): # checks if the bit at position i-MIN_PID is 'off' # i must be offset by MIN_PID because index of bitmap does not correspond to pid range directly if not self.bit_map.test(i - self.MIN_PID): # turns on the bit at i-MIN_PID self.bit_map.set(i - self.MIN_PID) # the pid to be allocated will be the index i at this point in the loop pid = i break return pid # this case will trigger for the first n<MAX_PID allocations else: # the pid to be allocated will be the value of the pid_counter at this point pid = self.pid_counter # turns on the bit at bit_index self.bit_map.set(self.bit_index) # increments bit_index to write to next index in bit_map on next iteration self.bit_index += 1 # increments pid_counter to allocate next pid on next iteration self.pid_counter += 1 return pid def release_pid(self, pid): # turns 'off' the bit at pid with an offset of MIN_PID # releasing pid '300' will turn off bit 300-300, or 0 self.bit_map.reset(pid - self.MIN_PID)
def __init__(self, size_byte=1 << 6, n_hash=1 << 2): self.size_byte = size_byte # 字节开销 self.size_bit = 8 * size_byte # bit开销 self.n_hash = n_hash # hash函数数量 self.bf = BitMap(self.size_byte) # 存储 # 通过科里化方式生成n_hash个hash函数列表 self.hash_functions = [ partial(self._hash, capacity=self.size_bit, seed=seed) for seed in self._mk_primes(self.n_hash) ] BloomFilter.N_INSTANCE += 1
def __init__(self, msg_len, msg_id, msg): PeerMessage.__init__(self, msg_id, msg) bit_string = '' for i in range(0, len(msg)): num = ord(msg[i]) bit_string += "{0:b}".format(ord(msg[i])) self.bitfield = BitMap(len(bit_string)) self.bitfield_str = bit_string for i in range(0, len(bit_string)): if bit_string[i] == '1': self.bitfield.set(i)
def __init__(self, size, start, headerSize, align): self.level = math.ceil(math.log2(size)) self.size = 1 << self.level print("Total size for buddy: {}".format(self.size)) self.level += 1 self.buckets = [set() for i in range(self.level)] self.in_use = BitMap(2 ** self.level - 1) # initialize root node self.buckets[0].add(start) # misc stuffs self.base = start self.headerSize = headerSize self.align = align self.sizemap = {}
def __init__(self, index, piece_length, file_to_write, piece_hash): self.index = index self.piece_length = piece_length self.num_blocks = piece_length/BLOCK_SIZE self.file_to_write = file_to_write self.piece_hash = piece_hash self.block_list = [] for block in range(self.num_blocks): self.block_list.append(Block(block, BLOCK_SIZE)) if piece_length % BLOCK_SIZE != 0: last_block_length = piece_length % BLOCK_SIZE self.block_list.append(Block(self.num_blocks, last_block_length)) self.num_blocks = self.num_blocks + 1 self.bitmap = BitMap(self.num_blocks)
def __init__(self, file_length, pieces_hash_array, piece_size, fs): self.logger = logging.getLogger(__name__) self.pieces_num = file_length / PIECE_SIZE self.piece_list = [] for piece in range(self.pieces_num): self.piece_list.append( Piece(piece, piece_size, fs, pieces_hash_array[piece])) #if have extra smaller pieces if file_length % PIECE_SIZE != 0: last_piece_size = file_length % piece_size self.piece_list.append( Piece(self.pieces_num, last_piece_size, fs, pieces_hash_array[self.pieces_num])) self.pieces_num += 1 self.bm = BitMap(self.pieces_num) self.logger.debug("init bm {}".format(self.bm))
def list_of_ids_to_bitmap(ids, maxitems, bitmap=None): """Convert list of 'id' values (entry numbers) to bitmap / bitset Each 'id' value is an entry number in the original dataset, that is the 'id' column in the database. Those have are natural numbers, and have values between 1 and number of entries; the latter is given as parameter to this function. Returned bitmap has i-th bit set to "1" (has "1" at i-th place) if and only if there was identifier 'i' on the list. Parameters ---------- ids | list [Sorted] list of integers with values between 1 and `maxitems`, inclusive; those are 'id' fields for given bug report / bugfix commit. maxitems : int Maximum value of ids onn the list, which is number of entries in the dataset; this means that it is the minimal number of bits in the bitmap / bitset. bitmap : BitMap, optional Bitmap object to set, in current incarnation in needs .set(i-th) method to set i-th bit to "1" in resulting bitmap. The bitmap must be empty (all zeros), and have at least `maxitems` bits. Returns ------- bitmap.BitMap Bitmap with appropriate bits set. Note that values are from 1 to maxitems, while bit positions are numbered from 0 to maxitems-1. """ if not bitmap: bitmap = BitMap(maxitems) # there is no built-in initialization from iterable for bitmap.BitMap for i in ids: # ids are numbered from 1, bits are numbered from 0 bitmap.set(i - 1) return bitmap
def __init__(self, piece_index, piece_size, fs, piece_hash): self.block_num = piece_size / BLOCK_SIZE self.block_list = [] self.piece_index = piece_index self.piece_size = piece_size self.piece_hash = piece_hash for block in range(self.block_num): b = Block(piece_index, block, BLOCK_SIZE) self.block_list.append(b) #if have extra smaller blocks if piece_size % BLOCK_SIZE != 0: last_block_size = piece_size % BLOCK_SIZE #print self.block_num,last_block_size self.block_list.append( Block(self.piece_index, self.block_num, last_block_size)) self.block_num += 1 self.bm = BitMap(self.block_num) self.file = fs return
def testLocalBitMap(size, flips): id = 'test' + str(size) bm = BitMap(size) print('Created ' + id + ':') # + bm.tostring() print("Flipping some random bits:") #bm.tostring() for i in range(0,flips): rnd_index = randint(0, size-1) #print("\tflipping index: " + str(rnd_index)) bm.flip(rnd_index) print("Bit sets to 1: " + str(bm.count())) print("Flipping some random bits:") #bm.tostring() for i in range(0, flips): rnd_index = randint(0, size - 1) #print("\tflipping index: " + str(rnd_index)) bm.flip(rnd_index) print("Bit sets to 1: " + str(bm.count())) stringBM = pickle.dumps(bm) print("Size of the pickled string representing the bm: " + str(len(stringBM))) print("Loading bm from pickled string") bm = pickle.loads(stringBM) print("Bit sets to 1: " + str(bm.count()))
def slove(input_data): lines = input_data.split('\n') n = int(lines[0]) trade = [] tem = 0 mem = np.full((n, pow(2, n)), -1, dtype='int') S = BitMap(n) points = [] for i in range(1, n + 1): line = lines[i] parts = line.split() points.append(Point(float(parts[0]), float(parts[1]))) def tsp(i, S): if (S.count() == n): return length(points[i], points[0]) if (mem[i][int(S.tostring(), 2)] != -1): return mem[i][int(S.tostring(), 2)] res = INF for j in range(n): print(j) if (S.test(j)): continue S.set(j) oldres = res res = min(res, length(points[i], points[j]) + tsp(j, S)) trade.append(tem) mem[i][int(S.tostring(), 2)] = res return res print(tsp(0, S)) print(trade)
def test_thread(): global exitFlag exitFlag = 0 threads = [] threadID = 1 print "Starting Main Thread" # Fill the queue for i in range(3): queue = Queue.Queue(4) for j in range(4): queue.put(i*4 + j) pieces_queue_list.append(queue) for i in range(3): queue = Queue.Queue(4) data_queue_list.append(queue) for i , tName in enumerate(threadList): thread = myThread(threadID, tName, pieces_queue_list[i],data_queue_list[i]) thread.start() threads.append(thread) threadID += 1 # Wait for all threads to complete for t in threads: t.join() print "Exiting Main Thread" bm = BitMap(12) while 1: if bm.all(): print "receive all the pieces!!" break for q in data_queue_list: if not q.empty(): item = q.get() data = item['data'] index = item['index'] print "receive pieces:%s data:%s\n" % (data,data) bm.set(index)
def test_reorganize_hash_file(): nbd.init(128) logicService._g = logicService._logic_service() # for i in BitMapIter(iter(bytearray([1, 0, 0, 2, 3, 4, 5]))): # print(i) with open('/tmp/test.bitmap', 'rb') as f: _bit_map = f.read() bit_map = BitMap() bit_map.bitmap = bytearray(_bit_map) snapshots = [{ "path": "/home/mnt/nodes/ffccc08f7d5e40119e0440fd5845259c/images/c152ad459f4c4b39897bf0127c760f66/d3632a55450d4692a5d3021727975edf.qcow", "ident": 'a3a1debbbe6f4c969fb0fe630d111199' }] disk_bytes = 105066964992 hash_name = '/tmp/hash.new' # import os # os.system('echo 3 > /proc/sys/vm/drop_caches;echo 3 > /proc/sys/vm/drop_caches') ReorganizeHashFile(bit_map, snapshots, hash_name, hash_name + '.tmp', disk_bytes).work() MergeHash(disk_bytes).merge(hash_name + '.reg', [hash_name])
def _split_digit_str(self, digit_str, n_digit_per_bitmap): """ split a digit string by @n_digit_per_bitmap digits as a group, to create bitmaps if dynamically @digit_str: each char in string is represented in digit @n_digit_per_bitmap: number of digit chars used to create a bitmap. @return: [(bm, n_width_digits), ...] NOTICE: for performace and easy code: bitmaps has the reverse order as the digit_str. """ ret = [] s, bms_idx = digit_str, -1 while s: n_width_digits = s[-n_digit_per_bitmap:] s, bms_idx = s[:-n_digit_per_bitmap], bms_idx + 1 if bms_idx >= len(self._bitmaps): # not exist self._bitmaps.append(BitMap()) ret.append((self._bitmaps[bms_idx], n_width_digits)) #finally return ret
# set up caffe caffe.set_mode_gpu() caffe.set_device(args.gpu_id) imdb = get_Imdbs(args.imdb_name) roidb = get_training_roidb(imdb) print '{:d} roidb entries'.format(len(roidb)) output_dir = get_output_dir(imdb) print 'Output will be saved to `{:s}`'.format(output_dir) # some statistic to record alamount = 0 ssamount = 0 # set bitmap for AL tableA = BitMap(imdb.num_images) # choose initiail samples:VOC2007 sample_num = imdb.num_images train_num = len(imdb[imdb.item_name(0)].roidb) print 'All VOC2007 images use for initial train, image numbers:%d' % ( train_num) for i in range(train_num): tableA.set(i) train_roidb = [roidb[i] for i in range(train_num)] pretrained_model_name = args.pretrained_model # static parameters tao = 60000 beta = 1000 # updatable hypeparameters
sensorRadius = 6 swarmSize = 180 shapeWidth = 7 tick = 20 # milliseconds velocity = 1 ang_velocity = 10 # robotRadius = 0.7 # sensorRadius = 6 # swarmSize = 90 # shapeWidth = 7 # tick = 20 # milliseconds # velocity = 1 # ang_velocity = 5 file_path = "shapes/tumor100.png" bitmap = BitMap(file_path) origin = bitmap.origin datafile = open(file_path) img = imread(datafile, mode='L') img[np.nonzero(img - 255)] = 0 fieldSizeX1 = 0 fieldSizeX2 = 100 + fieldSizeX1 fieldSizeY1 = 0 fieldSizeY2 = 100 + fieldSizeY1 #shape to assemble shapeX = np.array([0, 20, 20, 10, 10, 0, 0]) shapeY = np.array([0, 0, 10, 10, 20, 20, 0]) shapeOffsetX = 0 shapeOffsetY = 0
def expire(self): for block in self.block_list: block.mark_missing() self.bm = BitMap(self.block_num)
pprint.pprint(cfg) ############################## begin ####################################### # train set # imdb, roidb = combined_roidb(args.imdb_name) imdb = get_Imdbs(args.imdb_name) # total num_images total_num = imdb.num_images # initial num_images initialnum = imdb[imdb.item_name(0)].num_images # unlabeled num_images remainnum = imdb[imdb.item_name(1)].num_images print('total num:{}, initial num:{}'.format(total_num,initialnum)) bitmapImdb = BitMap(total_num) roidb = get_training_roidb(imdb) print('{:d} roidb entries'.format(len(roidb))) # output directory where the models are saved output_dir = get_output_dir(imdb, args.tag) print('Output will be saved to `{:s}`'.format(output_dir)) # tensorboard directory where the summaries are saved during training tb_dir = get_output_tb_dir(imdb, args.tag) print('TensorFlow summaries will be saved to `{:s}`'.format(tb_dir)) # also add the validation set, but with no flipping images orgflip = cfg.TRAIN.USE_FLIPPED
def main(): # Initialize C2 workspace.GlobalInit( ['caffe2', '--caffe2_log_level=0', '--caffe2_gpu_memory_tracking=1']) # Set up logging and load config options logger = setup_logging(__name__) logging.getLogger('detectron.roi_data.loader').setLevel(logging.INFO) args = parse_args() logger.info('Called with args:') logger.info(args) if args.cfg_file is not None: merge_cfg_from_file(args.cfg_file) if args.opts is not None: merge_cfg_from_list(args.opts) assert_and_infer_cfg() logger.info('Training with config:') logger.info(pprint.pformat(cfg)) # Note that while we set the numpy random seed network training will not be # deterministic in general. There are sources of non-determinism that cannot # be removed with a reasonble execution-speed tradeoff (such as certain # non-deterministic cudnn functions). np.random.seed(cfg.RNG_SEED) # Execute the training run fs = open('imgnames.pkl', 'rb') roidbnames = pickle.load(fs) fs.close() logger.info('Loading dataset: {}'.format(cfg.TRAIN.DATASETS)) dataset_names = cfg.TRAIN.DATASETS proposal_files = cfg.TRAIN.PROPOSAL_FILES roidb = get_training_roidb(dataset_names, proposal_files) logger.info('{:d} roidb entries'.format(len(roidb))) total_num = len(roidb) # bitmap idx indicated for training bitmapRoidb = BitMap(total_num) # initial samples # initial_num = int(total_num*0.2) # for i in range(initial_num): # bitmapRoidb.set(i) # # train_roidb = [roidb[i] for i in range(initial_num)] initialidx = [] train_roidb = [] for i, x in enumerate(roidb): if x['image'].split('/')[-1] in roidbnames: initialidx.append(i) train_roidb.append(x) for i in initialidx: bitmapRoidb.set(i) logger.info('{:d} the number initial roidb entries'.format( len(train_roidb))) # append flipped images train_roidb = flipped_roidb_for_training(train_roidb) logger.info('{:d} the number initial roidb entries'.format( len(train_roidb))) alamount = 0 ssamount = 0 gamma = 0.95 # control al proportion al_proportion_checkpoint = [ int(x * total_num * 0.4) for x in np.linspace(0.2, 1, 10) ] # control ss proportion ss_proportion_checkpoint = [ int(x * total_num) for x in np.linspace(0.2, 2, 10) ] next_iters = 90000 sum_iters = next_iters '''load the lasted checkpoints''' checkpoints = detectron.utils.train.train_model(sum_iters, train_roidb, cfg.TRAIN.WEIGHTS) while True: # to do a test on the test dataset test_model(checkpoints[(sum_iters - 1)], args.multi_gpu_testing, args.opts) if sum_iters > cfg.SOLVER.MAX_ITER: break # next detect unlabeled samples unlabeledidx = list(set(range(total_num)) - set(bitmapRoidb.nonzero())) # labeled samples labeledidx = list(set(bitmapRoidb.nonzero())) # detect unlabeled samples BBoxes, YClass, Scores, al_candidate_idx, ALScore = detect_im( checkpoints[(sum_iters - 1)], roidb, gamma, idxs=unlabeledidx, gpu_id=0) al_avg_idx = np.argsort(np.array(ALScore)) al_candidate_idx = [al_candidate_idx[i] for i in al_avg_idx] gamma = max(gamma - 0.05, 0.7) # the ss candidate idx ss_candidate_idx = [ i for i in unlabeledidx if i not in al_candidate_idx ] # update roidb for next training train_roidb = replace_roidb(roidb, BBoxes, YClass, ss_candidate_idx) # control the proportion if alamount + len(al_candidate_idx) >= al_proportion_checkpoint[0]: al_candidate_idx = al_candidate_idx[:int( al_proportion_checkpoint[0] - alamount)] tmp = al_proportion_checkpoint.pop(0) al_proportion_checkpoint.append(al_proportion_checkpoint[-1]) if ssamount + len(ss_candidate_idx) >= ss_proportion_checkpoint[0]: ss_candidate_idx = ss_candidate_idx[:int( ss_proportion_checkpoint[0] - ssamount)] tmp = ss_proportion_checkpoint.pop(0) ss_proportion_checkpoint.append(ss_proportion_checkpoint[-1]) # record ss and al factor alamount += len(al_candidate_idx) ssamount += len(ss_candidate_idx) logger.info('alfactor:{},ssfactor:{}'.format(alamount / total_num, ssamount / total_num)) # for idx in al_candidate_idx: # bitmapRoidb.set(idx) next_train_idx = bitmapRoidb.nonzero() next_train_idx.extend(ss_candidate_idx) train_roidb = blur_image(train_roidb, ss_candidate_idx) # the next training roidb train_roidb = [train_roidb[i] for i in next_train_idx] # flipped the roidb train_roidb = flipped_roidb_for_training(train_roidb) # the next training iters next_iters = 30000 sum_iters += next_iters checkpoints = detectron.utils.train.train_model( sum_iters, train_roidb, checkpoints[(sum_iters - next_iters - 1)])
result = [] for item in mysql_dict: if mysql_dict[item] == 1: result.append(item) def testb(b): length = max(max(b), len(b)) dest_list = [0 for i in xrange(length + 1)] for i in b: dest_list[i] += 1 result = [] for index, i in enumerate(dest_list): if i == 1: result.append(index) from bitmap import BitMap bm = BitMap(32) print bm.tostring() bm.set(21) print bm.tostring() from array import array array.fromfile() from cli import lock
caffe.set_device(args.gpu_id) ######################## begin ############################# imdb = get_Imdbs(args.imdb_name) roidb = get_training_roidb(imdb) print '{:d} roidb entries'.format(len(roidb)) output_dir = get_output_dir(imdb) print 'Output will be saved to `{:s}`'.format(output_dir) # some statistic to record alamount = 0; ssamount = 0 discardamount = 0 # set bitmap for AL bitmapImdb = BitMap(imdb.num_images) # choose initiail samples:VOC2007 initial_num = len(imdb[imdb.item_name(0)].roidb) print 'All VOC2007 images use for initial train, image numbers:%d'%(initial_num) for i in range(initial_num): bitmapImdb.set(i) train_roidb = [roidb[i] for i in range(initial_num)] pretrained_model_name = args.pretrained_model # static parameters tao = args.max_iters # initial hypeparameters gamma = 0.15; clslambda = np.array([-np.log(0.9)]*imdb.num_classes) # train record loopcounter = 0; train_iters = 0; iters_sum = train_iters
print("Debug logging") logging.basicConfig(level=logging.DEBUG) else: print("Normal logging") logging.basicConfig(level=logging.INFO) # Deal with dead lists DEADMAP = None if args.deadmap: try: print("Attempting to open deadmap") with open(args.deadmap, 'rb') as f: DEADMAP = pickle.load(f) except IOError: print("Creating new deadmap") DEADMAP = BitMap(5000000) else: print("No deadmap") # API calls per window max_api_calls = 300 # 5 minute windows window_seconds = 300 api_calls = 0 last_thing = args.start window_start_time = datetime.now() logging.info(f"Starting window at {window_start_time}") try: for thing_id in range(last_thing, 5000000): if DEADMAP.test(thing_id):