def minimum_hamming_distances(bytes, keysize_range):
results = []
for keysize in keysize_range:
hamming_distance_count = 0
block_count = 0
# iterate through sequential pairs of 'keysize' bytes, calculate
# the hamming_distance(), add result to running total
#
# also pad lists in case bytes doesn't divide perfectly by keysize
for values in grouper(list(grouper(bytes, keysize, fillvalue=0)),
2,
fillvalue=[0] * keysize):
block_count += 1
buf_x = values[0]
buf_y = values[1]
hamming_distance_count += hamming_distance(buf_x, buf_y) / keysize
results.append({
"keysize":
keysize,
"normalized_hamming_distance":
hamming_distance_count / block_count
})
return results
def transpose(bytes, blocksize):
block_set = list(grouper(bytes, blocksize, fillvalue=0))
transposed_blocks = []
blocks = len(block_set)
eof = False
for index in range(blocks - 1):
collection = []
for block in block_set:
try:
collection.append(block[index])
except:
eof = True
break
if eof:
break
transposed_blocks.append(collection)
return transposed_blocks
def reverse_mong(letter_code, division, series, piece, level="Item", max_id_range=260000):
"""
:param letter_code
:param division
:param series
:param piece
:param level
:param max_id_range:
:return:
"""
base_id = f"{letter_code}:~{division}:{series}:{piece}"
items = [
{
"id": base_id + ":" + str(i),
"level": level,
"letter_code": letter_code,
"class_no": series,
"series": str(series),
"division_no": str(division),
"item_ref": str(i),
"catalogue_ref": f"{letter_code} {series}/{piece}/{i}",
"piece_ref": str(piece),
}
for i in range(1, max_id_range)
]
for group in grouper(items, 200, fillvalue=None):
yield [g for g in group if g is not None]
def test_traverse():
"""To test the traverse implementation we call gc.collect() while instances
of all the C objects are still valid."""
acc = iteration_utilities.accumulate([])
app = iteration_utilities.applyfunc(lambda x: x, 1)
cha = iteration_utilities.chained(int, float)
cla = iteration_utilities.clamp([], 0, 1)
com = iteration_utilities.complement(int)
con = iteration_utilities.constant(1)
dee = iteration_utilities.deepflatten([])
dup = iteration_utilities.duplicates([])
fli = iteration_utilities.flip(int)
gro = iteration_utilities.grouper([], 2)
ine = iteration_utilities.intersperse([], 1)
iik = iteration_utilities.ItemIdxKey(10, 2)
ite = iteration_utilities.iter_except(int, TypeError)
mer = iteration_utilities.merge([])
nth = iteration_utilities.nth(1)
pac = iteration_utilities.packed(int)
par = iteration_utilities.partial(int, 10)
rep = iteration_utilities.replicate([], 3)
rou = iteration_utilities.roundrobin([])
see = iteration_utilities.Seen()
sid = iteration_utilities.sideeffects([], lambda x: x)
spl = iteration_utilities.split([], lambda x: True)
sta = iteration_utilities.starfilter(lambda x: True, [])
suc = iteration_utilities.successive([])
tab = iteration_utilities.tabulate(int)
une = iteration_utilities.unique_everseen([])
unj = iteration_utilities.unique_justseen([])
gc.collect()
def detect_ecb(collection, block_size):
likely_ecb_texts = []
index = 0
# work out how many repeated chunks each entry has.
# lots of repeated chunks means its likely ECB
for data in collection:
index += 1
chunks = {}
for chunk in list(grouper(data, block_size)):
if tuple(chunk) in chunks.keys():
chunks[tuple(chunk)] += 1
else:
chunks[tuple(chunk)] = 0
repeated_chunk_total = sum(chunks.values())
if repeated_chunk_total == 0:
continue
likely_ecb_texts.append({
"block_size": block_size,
"index": index,
"cipher_text": data,
"repeated_chunk_total": repeated_chunk_total
})
return likely_ecb_texts
def part2(text, size=256):
lengths = tuple(chain(map(ord, text.strip()), (17, 31, 73, 47, 23)))
args = (tuple(range(size)), lengths, 0, 0)
array, _, first, _ = nth(63)(applyfunc(process, args))
array = array[first % size:] + array[:first % size]
return bytes(reduce(xor, group) for group in grouper(array, 16)).hex()
def test_grouper_lengthhint_failure2():
of_it = _hf.OverflowLengthHint(toT([1, 2, 3]), sys.maxsize + 1)
it = grouper(of_it, 2)
with pytest.raises(OverflowError):
operator.length_hint(it)
with pytest.raises(OverflowError):
list(it)
def test_grouper_lengthhint_failure1():
f_it = _hf.FailLengthHint(toT([1, 2, 3]))
it = grouper(f_it, 2)
with pytest.raises(_hf.FailLengthHint.EXC_TYP,
match=_hf.FailLengthHint.EXC_MSG):
operator.length_hint(it)
with pytest.raises(_hf.FailLengthHint.EXC_TYP,
match=_hf.FailLengthHint.EXC_MSG):
list(it)
def friends_of_friends(names, keys=None, to_df=False, full_search=False):
names = [name.strip('@') for name in names]
friends = [set(), set()]
if full_search:
#loop over 2 screen names
for i, name in enumerate(names):
#use cursoring to loop over full results from friends queries (5,000 ids per page)
cursor = -1
while cursor != 0:
res = requests.get(friends_ids_endpoint,
params={
'screen_name': name,
'cursor': cursor
},
auth=auth).json()
#add ids from each page to respective set
friends[i].update(set(res['ids']))
#set next cursor
cursor = res['next_cursor']
#just grab first page of friends results for each user
else:
for i, name in enumerate(names):
res = requests.get(friends_ids_endpoint,
params={
'screen_name': name
},
auth=auth).json()
friends[i].update(set(res['ids']))
#get overlapping ids
shared_ids = list(friends[0].intersection(friends[1]))
#loop handles cases where there are > 100 ids in common (max users returned from users endpoint)
shared_friends = []
#use grouper function to loop over groups of 100 user ids
for ids in grouper(shared_ids, 100):
ids_str = ','.join(str(num) for num in ids)
users = requests.get(users_endpoint,
params={
'user_id': ids_str
},
auth=auth).json()
#add user objects to shared_friends list
shared_friends.extend(users)
#select specific keys to return
if keys:
shared_friends = [{key: user[key]
for key in keys} for user in shared_friends]
#select return format
if to_df:
return pd.DataFrame(shared_friends)
return shared_friends
def diffusion(t, x):
# //////
D = []
blk = 5000
for group in grouper(zip(t, x), blk):
dx = ((group[-1][1] - group[0][1])**2).sum()
dt = group[-1][0] - group[0][0]
D.append(dx / (6 * dt))
tmp = np.asarray(D)
print(tmp.mean(), tmp.std(ddof=1) / np.sqrt(len(tmp)))
def diffusion(t, x):
# //////
D = []
blk = 50000
for group in grouper(zip(t, x), blk):
dx = ((group[-1][1] - group[0][1])**2).sum()
dt = group[-1][0] - group[0][0]
D.append(dx / (6 * dt))
print(D)
tmp = np.asarray(D)
return tmp.mean()
def decrypt_ecb_cbc(cipher_text, key, iv):
block_length = len(key)
if len(iv) != block_length:
raise Exception("iv must match key length!")
blocks = list(grouper(cipher_text, block_length))
cipher = AES.new(key, AES.MODE_ECB)
last_block = iv
plain_blocks = []
for block in blocks:
decrypted_block = cipher.decrypt(bytes(block))
plain_block = xor(last_block, decrypted_block)
plain_blocks.append(plain_block)
last_block = block
plain_text = pad_strip(b"".join(plain_blocks))
return plain_text
def encrypt_ecb_cbc(message, key, iv):
block_length = len(key)
if len(iv) != block_length:
raise Exception("iv must match key length!")
padded_message = pad(message, block_length)
blocks = list(grouper(padded_message, block_length))
cipher = AES.new(key, AES.MODE_ECB)
last_block = iv
encrypted_blocks = []
for block in blocks:
encrypted_block = cipher.encrypt(xor(last_block, block))
encrypted_blocks.append(encrypted_block)
last_block = encrypted_block
return b"".join(encrypted_blocks)
def diffusion(t, x):
# //////
D = []
if len(x) > 100000:
blk = 5000
else:
blk = 500
div = len(x[0, :]) // 3
for group in grouper(zip(t, x), blk):
dx = ((group[-1][1] - group[0][1])**2).sum() / div
dt = group[-1][0] - group[0][0]
D.append(dx / (6 * dt))
tmp = np.asarray(D)
print(len(x), div, t[-1])
print(tmp.mean(), tmp.std(ddof=1) / np.sqrt(len(tmp)))
def test_set8(self):
# FIXME: replace with detect_ecb function
# remember input needs to be array of bytes data..(non-b64 encoded)
likely_ecb_texts = []
line_number = 0
# work out how many repeated chunks each entry has.
# lots of repeated chunks means its likely ECB
for line in self.data:
line_number += 1
chunks = {}
# split line into 16 byte chunks since key is 16 bytes long
for chunk in list(grouper(b64decode(line), 16)):
if tuple(chunk) in chunks.keys():
chunks[tuple(chunk)] += 1
else:
chunks[tuple(chunk)] = 0
repeated_chunk_total = sum(chunks.values())
if repeated_chunk_total == 0:
continue
likely_ecb_texts.append({
"line_number":
line_number,
"cipher_text":
line,
"repeated_chunk_total":
repeated_chunk_total
})
most_likely_ecb_text = find_max(likely_ecb_texts,
"repeated_chunk_total")
self.assertEqual(most_likely_ecb_text['cipher_text'], self.target)
self.assertEqual(most_likely_ecb_text['line_number'], 133)
self.assertEqual(most_likely_ecb_text['repeated_chunk_total'], 3)
def main():
args = parse_args()
img = Image.open(args.input)
img = img.convert("RGBA")
img.load()
vertical_cutpoints = []
state = "seeking"
filecounter = 0
for y in range(img.size[1]):
row_has_pixel = False
for x in range(img.size[0]):
if img.getpixel((x, y))[3] != 0:
row_has_pixel = True
break
if state == "seeking" and row_has_pixel:
vertical_cutpoints.append(y)
state = "filling"
elif state == "filling" and not row_has_pixel:
vertical_cutpoints.append(y - 1)
state = "seeking"
print(vertical_cutpoints)
### split each section horizontally
for y1, y2 in list(grouper(vertical_cutpoints, 2)):
state = "seeking"
horizontal_cutpoints = []
for x in range(img.size[0]):
column_has_pixel = False
for y in range(y1, y2 + 1):
if img.getpixel((x, y))[3] != 0:
column_has_pixel = True
break
if state == "seeking" and column_has_pixel:
horizontal_cutpoints.append(x)
state = "filling"
elif state == "filling" and not column_has_pixel:
horizontal_cutpoints.append(x - 1)
state = "seeking"
### trim the square to as small as possible
for x1, x2 in list(grouper(horizontal_cutpoints, 2)):
min_y1 = None
max_y2 = None
for y in range(y1, y2 + 1):
for x in range(x1, x2 + 1):
if not min_y1 and img.getpixel((x, y))[3] != 0:
min_y1 = y
elif img.getpixel(
(x, y))[3] != 0 and (not max_y2 or y > max_y2):
max_y2 = y
if min_y1 and max_y2:
filecounter += 1
cropped = img.crop((x1, min_y1, x2 + 1, max_y2))
cropped.save('{}_{}.png'.format(args.output_prefix,
str(filecounter).zfill(4)))
print("saved: {}, {}, {}, {}".format(x1, min_y1, x2 + 1,
max_y2))
def checksum(data):
return "".join("1" if a == b else "0" for a, b in grouper(data, 2))
for dir_id, dir_name in mapping:
input_dirs.append(
batchai.models.InputDirectory(
id=dir_id,
path='$AZ_BATCHAI_MOUNT_ROOT/{0}/{1}'.format(
os.getenv('CLUSTER_CONTAINER_MNT_PATH'), dir_name)))
# create array of content img names inside content img dir
content_img_names = fileshare.list_blobs_in_dir(
blob_service=fs_client, blob_dir_name=content_images_blob_dir)
# create a job per chunk
t0 = time.time()
for group_i, img_name_group in \
enumerate(grouper(
content_img_names,
int(job_batch_size)
)):
img_list_str = ','.join(img_name_group)
# set the job name [ex job_01_01_2000_111111]
job_name = datetime.utcnow().strftime("{0}{1}_%m_%d_%Y_%H%M%S".format(
os.getenv('JOB_NAME_PREFIX'), group_i))
# create params for the job
job_params = bai.create_job_params(
cluster=cluster,
input_dirs=input_dirs,
output_dirs=None,
container_image="pytorch/pytorch:0.4_cuda9_cudnn7",
command_line=("python $AZ_BATCHAI_INPUT_SCRIPT " + \
c.add("Q", 2, 3)
c.add("J", 1, 3)
deck = Deck(court=c, decksize=52)
bg = BeggarGame(deck, gamenum=start)
os.environ['BROKER_POOL_LIMIT'] = 'None'
app = Celery('celiter', backend='rpc://', broker=os.environ['BROKERURL'])
app.conf.broker_heartbeat = 0
longest = 0
handchunk = 1000
taskchunk = 1000
rescollect = 10
b = grouper(bg.dealer(), handchunk)
res = []
results = []
collect = 0
try:
while True:
pbar = tqdm(
total=deck.court.permutations,
unit=' hands',
ncols=140,
bar_format=
'{l_bar}{bar}| {n_fmt}/{total_fmt} [{elapsed}<{remaining}] {rate_fmt} {postfix[0]} {postfix[1][maxturns]}',
postfix=["Max Turns:", dict(maxturns=0)])
for chunk in b:
res.append(
def groeperen(prijzen):
prijzen = sorted(prijzen, key=float, reverse=True)
return list(grouper(prijzen, 4))
def test_grouper_normal4():
assert list(grouper([T(1), T(2), T(3), T(4)], 3)) == [(T(1), T(2), T(3)),
(T(4), )]
def test_grouper_truncate3():
assert list(grouper(toT([1, 2, 3]), 3,
truncate=True)) == [(T(1), T(2), T(3))]
def test_grouper_normal6():
assert list(grouper(toT([1, 2, 3, 4, 5, 6]), 3)) == [(T(1), T(2), T(3)),
(T(4), T(5), T(6))]
def test_grouper_normal3():
assert list(grouper([T(1), T(2), T(3)], 3)) == [(T(1), T(2), T(3))]
def test_grouper_fill1():
# with fillvalue
assert list(grouper(toT([1]), 3, fillvalue=T(0))) == [(T(1), T(0), T(0))]
def test_grouper_fill3():
assert list(grouper(toT([1, 2, 3]), 3,
fillvalue=T(0))) == [(T(1), T(2), T(3))]
def test_grouper_fill6():
assert list(grouper(toT([1, 2, 3, 4, 5, 6]), 3,
fillvalue=T(0))) == [(T(1), T(2), T(3)),
(T(4), T(5), T(6))]
def test_grouper_truncate1():
# with truncate
assert list(grouper(toT([1]), 3, truncate=True)) == []
def test_grouper_normal7():
# generator
assert list(grouper((i for i in toT([1, 2, 3, 4, 5, 6])),
3)) == [(T(1), T(2), T(3)), (T(4), T(5), T(6))]
def test_grouper_truncate2():
assert list(grouper(toT([1, 2]), 3, truncate=True)) == []
