def minimum_hamming_distances(bytes, keysize_range): results = [] for keysize in keysize_range: hamming_distance_count = 0 block_count = 0 # iterate through sequential pairs of 'keysize' bytes, calculate # the hamming_distance(), add result to running total # # also pad lists in case bytes doesn't divide perfectly by keysize for values in grouper(list(grouper(bytes, keysize, fillvalue=0)), 2, fillvalue=[0] * keysize): block_count += 1 buf_x = values[0] buf_y = values[1] hamming_distance_count += hamming_distance(buf_x, buf_y) / keysize results.append({ "keysize": keysize, "normalized_hamming_distance": hamming_distance_count / block_count }) return results
def transpose(bytes, blocksize): block_set = list(grouper(bytes, blocksize, fillvalue=0)) transposed_blocks = [] blocks = len(block_set) eof = False for index in range(blocks - 1): collection = [] for block in block_set: try: collection.append(block[index]) except: eof = True break if eof: break transposed_blocks.append(collection) return transposed_blocks
def reverse_mong(letter_code, division, series, piece, level="Item", max_id_range=260000): """ :param letter_code :param division :param series :param piece :param level :param max_id_range: :return: """ base_id = f"{letter_code}:~{division}:{series}:{piece}" items = [ { "id": base_id + ":" + str(i), "level": level, "letter_code": letter_code, "class_no": series, "series": str(series), "division_no": str(division), "item_ref": str(i), "catalogue_ref": f"{letter_code} {series}/{piece}/{i}", "piece_ref": str(piece), } for i in range(1, max_id_range) ] for group in grouper(items, 200, fillvalue=None): yield [g for g in group if g is not None]
def test_traverse(): """To test the traverse implementation we call gc.collect() while instances of all the C objects are still valid.""" acc = iteration_utilities.accumulate([]) app = iteration_utilities.applyfunc(lambda x: x, 1) cha = iteration_utilities.chained(int, float) cla = iteration_utilities.clamp([], 0, 1) com = iteration_utilities.complement(int) con = iteration_utilities.constant(1) dee = iteration_utilities.deepflatten([]) dup = iteration_utilities.duplicates([]) fli = iteration_utilities.flip(int) gro = iteration_utilities.grouper([], 2) ine = iteration_utilities.intersperse([], 1) iik = iteration_utilities.ItemIdxKey(10, 2) ite = iteration_utilities.iter_except(int, TypeError) mer = iteration_utilities.merge([]) nth = iteration_utilities.nth(1) pac = iteration_utilities.packed(int) par = iteration_utilities.partial(int, 10) rep = iteration_utilities.replicate([], 3) rou = iteration_utilities.roundrobin([]) see = iteration_utilities.Seen() sid = iteration_utilities.sideeffects([], lambda x: x) spl = iteration_utilities.split([], lambda x: True) sta = iteration_utilities.starfilter(lambda x: True, []) suc = iteration_utilities.successive([]) tab = iteration_utilities.tabulate(int) une = iteration_utilities.unique_everseen([]) unj = iteration_utilities.unique_justseen([]) gc.collect()
def detect_ecb(collection, block_size): likely_ecb_texts = [] index = 0 # work out how many repeated chunks each entry has. # lots of repeated chunks means its likely ECB for data in collection: index += 1 chunks = {} for chunk in list(grouper(data, block_size)): if tuple(chunk) in chunks.keys(): chunks[tuple(chunk)] += 1 else: chunks[tuple(chunk)] = 0 repeated_chunk_total = sum(chunks.values()) if repeated_chunk_total == 0: continue likely_ecb_texts.append({ "block_size": block_size, "index": index, "cipher_text": data, "repeated_chunk_total": repeated_chunk_total }) return likely_ecb_texts
def part2(text, size=256): lengths = tuple(chain(map(ord, text.strip()), (17, 31, 73, 47, 23))) args = (tuple(range(size)), lengths, 0, 0) array, _, first, _ = nth(63)(applyfunc(process, args)) array = array[first % size:] + array[:first % size] return bytes(reduce(xor, group) for group in grouper(array, 16)).hex()
def test_grouper_lengthhint_failure2(): of_it = _hf.OverflowLengthHint(toT([1, 2, 3]), sys.maxsize + 1) it = grouper(of_it, 2) with pytest.raises(OverflowError): operator.length_hint(it) with pytest.raises(OverflowError): list(it)
def test_grouper_lengthhint_failure1(): f_it = _hf.FailLengthHint(toT([1, 2, 3])) it = grouper(f_it, 2) with pytest.raises(_hf.FailLengthHint.EXC_TYP, match=_hf.FailLengthHint.EXC_MSG): operator.length_hint(it) with pytest.raises(_hf.FailLengthHint.EXC_TYP, match=_hf.FailLengthHint.EXC_MSG): list(it)
def friends_of_friends(names, keys=None, to_df=False, full_search=False): names = [name.strip('@') for name in names] friends = [set(), set()] if full_search: #loop over 2 screen names for i, name in enumerate(names): #use cursoring to loop over full results from friends queries (5,000 ids per page) cursor = -1 while cursor != 0: res = requests.get(friends_ids_endpoint, params={ 'screen_name': name, 'cursor': cursor }, auth=auth).json() #add ids from each page to respective set friends[i].update(set(res['ids'])) #set next cursor cursor = res['next_cursor'] #just grab first page of friends results for each user else: for i, name in enumerate(names): res = requests.get(friends_ids_endpoint, params={ 'screen_name': name }, auth=auth).json() friends[i].update(set(res['ids'])) #get overlapping ids shared_ids = list(friends[0].intersection(friends[1])) #loop handles cases where there are > 100 ids in common (max users returned from users endpoint) shared_friends = [] #use grouper function to loop over groups of 100 user ids for ids in grouper(shared_ids, 100): ids_str = ','.join(str(num) for num in ids) users = requests.get(users_endpoint, params={ 'user_id': ids_str }, auth=auth).json() #add user objects to shared_friends list shared_friends.extend(users) #select specific keys to return if keys: shared_friends = [{key: user[key] for key in keys} for user in shared_friends] #select return format if to_df: return pd.DataFrame(shared_friends) return shared_friends
def diffusion(t, x): # ////// D = [] blk = 5000 for group in grouper(zip(t, x), blk): dx = ((group[-1][1] - group[0][1])**2).sum() dt = group[-1][0] - group[0][0] D.append(dx / (6 * dt)) tmp = np.asarray(D) print(tmp.mean(), tmp.std(ddof=1) / np.sqrt(len(tmp)))
def diffusion(t, x): # ////// D = [] blk = 50000 for group in grouper(zip(t, x), blk): dx = ((group[-1][1] - group[0][1])**2).sum() dt = group[-1][0] - group[0][0] D.append(dx / (6 * dt)) print(D) tmp = np.asarray(D) return tmp.mean()
def decrypt_ecb_cbc(cipher_text, key, iv): block_length = len(key) if len(iv) != block_length: raise Exception("iv must match key length!") blocks = list(grouper(cipher_text, block_length)) cipher = AES.new(key, AES.MODE_ECB) last_block = iv plain_blocks = [] for block in blocks: decrypted_block = cipher.decrypt(bytes(block)) plain_block = xor(last_block, decrypted_block) plain_blocks.append(plain_block) last_block = block plain_text = pad_strip(b"".join(plain_blocks)) return plain_text
def encrypt_ecb_cbc(message, key, iv): block_length = len(key) if len(iv) != block_length: raise Exception("iv must match key length!") padded_message = pad(message, block_length) blocks = list(grouper(padded_message, block_length)) cipher = AES.new(key, AES.MODE_ECB) last_block = iv encrypted_blocks = [] for block in blocks: encrypted_block = cipher.encrypt(xor(last_block, block)) encrypted_blocks.append(encrypted_block) last_block = encrypted_block return b"".join(encrypted_blocks)
def diffusion(t, x): # ////// D = [] if len(x) > 100000: blk = 5000 else: blk = 500 div = len(x[0, :]) // 3 for group in grouper(zip(t, x), blk): dx = ((group[-1][1] - group[0][1])**2).sum() / div dt = group[-1][0] - group[0][0] D.append(dx / (6 * dt)) tmp = np.asarray(D) print(len(x), div, t[-1]) print(tmp.mean(), tmp.std(ddof=1) / np.sqrt(len(tmp)))
def test_set8(self): # FIXME: replace with detect_ecb function # remember input needs to be array of bytes data..(non-b64 encoded) likely_ecb_texts = [] line_number = 0 # work out how many repeated chunks each entry has. # lots of repeated chunks means its likely ECB for line in self.data: line_number += 1 chunks = {} # split line into 16 byte chunks since key is 16 bytes long for chunk in list(grouper(b64decode(line), 16)): if tuple(chunk) in chunks.keys(): chunks[tuple(chunk)] += 1 else: chunks[tuple(chunk)] = 0 repeated_chunk_total = sum(chunks.values()) if repeated_chunk_total == 0: continue likely_ecb_texts.append({ "line_number": line_number, "cipher_text": line, "repeated_chunk_total": repeated_chunk_total }) most_likely_ecb_text = find_max(likely_ecb_texts, "repeated_chunk_total") self.assertEqual(most_likely_ecb_text['cipher_text'], self.target) self.assertEqual(most_likely_ecb_text['line_number'], 133) self.assertEqual(most_likely_ecb_text['repeated_chunk_total'], 3)
def main(): args = parse_args() img = Image.open(args.input) img = img.convert("RGBA") img.load() vertical_cutpoints = [] state = "seeking" filecounter = 0 for y in range(img.size[1]): row_has_pixel = False for x in range(img.size[0]): if img.getpixel((x, y))[3] != 0: row_has_pixel = True break if state == "seeking" and row_has_pixel: vertical_cutpoints.append(y) state = "filling" elif state == "filling" and not row_has_pixel: vertical_cutpoints.append(y - 1) state = "seeking" print(vertical_cutpoints) ### split each section horizontally for y1, y2 in list(grouper(vertical_cutpoints, 2)): state = "seeking" horizontal_cutpoints = [] for x in range(img.size[0]): column_has_pixel = False for y in range(y1, y2 + 1): if img.getpixel((x, y))[3] != 0: column_has_pixel = True break if state == "seeking" and column_has_pixel: horizontal_cutpoints.append(x) state = "filling" elif state == "filling" and not column_has_pixel: horizontal_cutpoints.append(x - 1) state = "seeking" ### trim the square to as small as possible for x1, x2 in list(grouper(horizontal_cutpoints, 2)): min_y1 = None max_y2 = None for y in range(y1, y2 + 1): for x in range(x1, x2 + 1): if not min_y1 and img.getpixel((x, y))[3] != 0: min_y1 = y elif img.getpixel( (x, y))[3] != 0 and (not max_y2 or y > max_y2): max_y2 = y if min_y1 and max_y2: filecounter += 1 cropped = img.crop((x1, min_y1, x2 + 1, max_y2)) cropped.save('{}_{}.png'.format(args.output_prefix, str(filecounter).zfill(4))) print("saved: {}, {}, {}, {}".format(x1, min_y1, x2 + 1, max_y2))
def checksum(data): return "".join("1" if a == b else "0" for a, b in grouper(data, 2))
for dir_id, dir_name in mapping: input_dirs.append( batchai.models.InputDirectory( id=dir_id, path='$AZ_BATCHAI_MOUNT_ROOT/{0}/{1}'.format( os.getenv('CLUSTER_CONTAINER_MNT_PATH'), dir_name))) # create array of content img names inside content img dir content_img_names = fileshare.list_blobs_in_dir( blob_service=fs_client, blob_dir_name=content_images_blob_dir) # create a job per chunk t0 = time.time() for group_i, img_name_group in \ enumerate(grouper( content_img_names, int(job_batch_size) )): img_list_str = ','.join(img_name_group) # set the job name [ex job_01_01_2000_111111] job_name = datetime.utcnow().strftime("{0}{1}_%m_%d_%Y_%H%M%S".format( os.getenv('JOB_NAME_PREFIX'), group_i)) # create params for the job job_params = bai.create_job_params( cluster=cluster, input_dirs=input_dirs, output_dirs=None, container_image="pytorch/pytorch:0.4_cuda9_cudnn7", command_line=("python $AZ_BATCHAI_INPUT_SCRIPT " + \
c.add("Q", 2, 3) c.add("J", 1, 3) deck = Deck(court=c, decksize=52) bg = BeggarGame(deck, gamenum=start) os.environ['BROKER_POOL_LIMIT'] = 'None' app = Celery('celiter', backend='rpc://', broker=os.environ['BROKERURL']) app.conf.broker_heartbeat = 0 longest = 0 handchunk = 1000 taskchunk = 1000 rescollect = 10 b = grouper(bg.dealer(), handchunk) res = [] results = [] collect = 0 try: while True: pbar = tqdm( total=deck.court.permutations, unit=' hands', ncols=140, bar_format= '{l_bar}{bar}| {n_fmt}/{total_fmt} [{elapsed}<{remaining}] {rate_fmt} {postfix[0]} {postfix[1][maxturns]}', postfix=["Max Turns:", dict(maxturns=0)]) for chunk in b: res.append(
def groeperen(prijzen): prijzen = sorted(prijzen, key=float, reverse=True) return list(grouper(prijzen, 4))
def test_grouper_normal4(): assert list(grouper([T(1), T(2), T(3), T(4)], 3)) == [(T(1), T(2), T(3)), (T(4), )]
def test_grouper_truncate3(): assert list(grouper(toT([1, 2, 3]), 3, truncate=True)) == [(T(1), T(2), T(3))]
def test_grouper_normal6(): assert list(grouper(toT([1, 2, 3, 4, 5, 6]), 3)) == [(T(1), T(2), T(3)), (T(4), T(5), T(6))]
def test_grouper_normal3(): assert list(grouper([T(1), T(2), T(3)], 3)) == [(T(1), T(2), T(3))]
def test_grouper_fill1(): # with fillvalue assert list(grouper(toT([1]), 3, fillvalue=T(0))) == [(T(1), T(0), T(0))]
def test_grouper_fill3(): assert list(grouper(toT([1, 2, 3]), 3, fillvalue=T(0))) == [(T(1), T(2), T(3))]
def test_grouper_fill6(): assert list(grouper(toT([1, 2, 3, 4, 5, 6]), 3, fillvalue=T(0))) == [(T(1), T(2), T(3)), (T(4), T(5), T(6))]
def test_grouper_truncate1(): # with truncate assert list(grouper(toT([1]), 3, truncate=True)) == []
def test_grouper_normal7(): # generator assert list(grouper((i for i in toT([1, 2, 3, 4, 5, 6])), 3)) == [(T(1), T(2), T(3)), (T(4), T(5), T(6))]
def test_grouper_truncate2(): assert list(grouper(toT([1, 2]), 3, truncate=True)) == []