def get_groups_data(self, groups_set):
print('Запущен анализ результирующего списка групп:')
list_len = len(groups_set)
result_group_list = []
try:
print_progress_bar(0,
list_len,
prefix='Прогресс:',
suffix='',
length=30)
except ZeroDivisionError:
print('Информация о группах недоступна.')
return result_group_list
for counter, group_id in enumerate(groups_set):
group = Group(group_id)
result_group_list.append(group.__dict__)
time.sleep(0.8)
print_progress_bar(counter + 1,
list_len,
prefix='Прогресс:',
suffix='',
length=30)
return result_group_list
def add_friends(self):
friend_list = []
friend_id_list = self.victim.friend_list
list_len = len(friend_id_list)
print(
f'Запущен процесс получения информации о '
f'друзьях пользователя {self.victim.name} {self.victim.last_name}:'
)
try:
print_progress_bar(0,
list_len,
prefix='Прогресс:',
suffix='',
length=30)
except ZeroDivisionError:
print('Информация о друзьях недоступна.')
return friend_list
for counter, friend_id in enumerate(friend_id_list):
friend = User(friend_id)
friend_list.append(friend)
time.sleep(0.8)
print_progress_bar(counter + 1,
list_len,
prefix='Прогресс:',
suffix='',
length=30)
print('Данные получены!')
return friend_list
async def fetch(queue, tx_queue, host, start_block, end_block, incrementer,
partition):
current_block = start_block
block_hash = None
while current_block < end_block:
async with aiohttp.ClientSession() as session:
async with session.get(host + '/heights/' +
str(current_block + partition),
headers=headers) as response:
if (response.status == 200):
result = await response.json()
block_hash = result['blockHash']
#print("blockhash:",block_hash, "...")
await queue.put(block_hash)
if (block_hash):
#print('retrieving data')
async with aiohttp.ClientSession() as session:
async with session.get(host + '/blocks/' + block_hash +
'?txidsonly=true',
headers=headers) as response:
if (response.status == 200):
data = await response.json()
[
await tx_queue.put(
(tx['transactionId'], block_hash))
for tx in data['transactions']
]
await rnd_sleep(.001)
#TODO: this is not optimal , should only render at 60fps
print_progress_bar(current_block - start_block,
end_block - start_block,
prefix='Progress:',
suffix='Complete')
block_hash = None
current_block = current_block + incrementer
def encode_to_file(table, input_file, output_file, input_size):
print("Writing to file start.")
start_time = time.time()
ints_in_buffer = 256 # 1Kb
real_bits_size = 0
buffer = bytearray(ints_in_buffer * 4)
buffer_len = 0
current_int_len = 0
buffer_int = int() # 1 int = 4 bytes
current_input_byte = 0
with input_file as f:
file_content = f.read(1024)
while file_content:
for ch in file_content:
current_input_byte += 1
bits_to_write = table.get(bytes([ch]))
for bit in bits_to_write:
if buffer_len == ints_in_buffer:
output_file.write(buffer[:ints_in_buffer * 4:])
buffer = bytearray(ints_in_buffer * 4)
real_bits_size += ints_in_buffer * 32
buffer_len = 0
progress.print_progress_bar(current_input_byte,
input_size)
if current_int_len == 32:
buffer[buffer_len * 4:((buffer_len + 1) * 4) -
1:] = buffer_int.to_bytes(4, "big")
buffer_len += 1
buffer_int = 0
current_int_len = 0
buffer_int |= (bit << (31 - current_int_len))
current_int_len += 1
file_content = f.read(1024)
if buffer_len > 0:
output_file.write(buffer[:buffer_len * 4:])
real_bits_size += buffer_len * 32
if current_int_len > 0:
# Todo optimize
output_file.write(buffer_int.to_bytes(4, "big"))
real_bits_size += current_int_len
output_file.write(real_bits_size.to_bytes(8, "big"))
encode_table_info(table, output_file)
progress.clear_progress()
print("Writing completed:", time.time() - start_time, "s")
def count_weights(input_filename):
print("Weights counting start.")
total_bytes = os.stat(input_filename).st_size
current_byte = 0
start_time = time.time()
symbols_weights = Counter()
with open(input_filename, "rb") as f:
weights_buffer = f.read(1024)
while weights_buffer:
for byte in weights_buffer:
symbols_weights[bytes([byte])] += 1
current_byte += 1
progress.print_progress_bar(current_byte, total_bytes)
weights_buffer = f.read(1024)
# print(symbols_weights)
sorted(symbols_weights.items())
progress.clear_progress()
print("Weights counting completed:", time.time() - start_time, "s")
return symbols_weights
def write_images_to_records(output_file, coder):
"""Encode all dataset entries (images and .txt files) into one records file
Args:
output_file: path to where the records file should be saved to
coder: instance of ImageCoder to provide TensorFlow image coding utils.
"""
writer = tf.python_io.TFRecordWriter(output_file)
print('writing tfRecords...')
progress = 0
# go through every image in the directory
for filename in os.listdir(DatasetDirectory):
if filename.endswith(".png"):
filename_image = os.path.join(DatasetDirectory, filename)
filename_label = os.path.join(DatasetDirectory,
filename[0:-4] + ".txt")
elif filename.endswith(".jpg"):
filename_image = os.path.join(DatasetDirectory, filename)
filename_label = os.path.join(DatasetDirectory,
filename[0:-4] + ".txt")
elif filename.endswith(".jpeg"):
filename_image = os.path.join(DatasetDirectory, filename)
filename_label = os.path.join(DatasetDirectory,
filename[0:-5] + ".txt")
else:
continue
# encode the image
image_buffer, image_height, image_width = process_image(
filename_image, coder)
# load valid labels
bboxes, labels = read_bboxes_from_label_txt(filename_label)
xmins = [] # List of left x coordinates in bounding box (1 per box)
xmaxs = [] # List of right x coordinates in bounding box (1 per box)
ymins = [] # List of top y coordinates in bounding box (1 per box)
ymaxs = [] # List of bottom y coordinates in bounding box (1 per box)
classes_text = [
] # List of string class name of bounding box (1 per box)
classes = [] # List of integer class id of bounding box (1 per box)
# go through every bounding box
for i in enumerate(bboxes):
xmins.append(bboxes[i[0]]['left'] / image_width)
xmaxs.append(bboxes[i[0]]['right'] / image_width)
ymins.append(bboxes[i[0]]['top'] / image_height)
ymaxs.append(bboxes[i[0]]['bottom'] / image_height)
classes_text.append(labels[i[0]]['label_name'])
classes.append(ValidLabels.index(classes_text[
i[0]])) # this gets the index of the label as integer
# save the data to the .records file
tf_dataset_entry = create_tf_dataset_entry(filename, image_buffer,
xmins, xmaxs, ymins, ymaxs,
classes_text, classes,
image_height, image_width)
writer.write(tf_dataset_entry.SerializeToString())
progress += 1
print_progress_bar(progress, len(os.listdir(DatasetDirectory)) / 2)
writer.close()