def test_no_double_counting(self):
rep = ["test1"]
obj = [rep, rep]
obj2 = [rep]
expected_sz = objsize.get_deep_size(obj2) - sys.getsizeof(
obj2) + sys.getsizeof(obj)
self.assertEqual(expected_sz, objsize.get_deep_size(obj))
def test_slots(self):
class MySlots1(object):
__slots__ = ["number1"]
def __init__(self, number1):
self.number1 = number1
class MySlots2(object):
__slots__ = ["number1", "number2"]
def __init__(self, number1, number2):
self.number1 = number1
self.number2 = number2
class MySlots3(object):
__slots__ = ["number1", "number2", "number3"]
def __init__(self, number1, number2, number3):
self.number1 = number1
self.number2 = number2
self.number3 = number3
s1 = MySlots1(7)
s2 = MySlots2(3, 4)
s3 = MySlots3(4, 5, 6)
s1_sz = sys.getsizeof(s1) + sys.getsizeof(7)
s2_sz = sys.getsizeof(s2) + sys.getsizeof(3) + sys.getsizeof(4)
s3_sz = sys.getsizeof(s3) + sys.getsizeof(4) + sys.getsizeof(5) + sys.getsizeof(6)
self.assertEqual(s1_sz, objsize.get_deep_size(s1))
self.assertEqual(s2_sz, objsize.get_deep_size(s2))
self.assertEqual(s3_sz, objsize.get_deep_size(s3))
def test_gracefully_handles_self_referential_objects(self):
class MyClass(object):
pass
obj = MyClass()
obj.prop = obj
self.assertEqual(objsize.get_deep_size(obj), objsize.get_deep_size(obj.prop))
def test_multi_obj(self):
class MyClass(object):
def __init__(self, x, y):
self.x = x
self.y = y
strs = 'hello world', 'foo bar', 'something else'
objs = MyClass(strs[0], strs[1]), MyClass(strs[0], strs[2]), MyClass(strs[1], strs[2])
expected_sz = sum(map(sys.getsizeof, strs))
expected_sz += sum(calc_class_obj_sz(o, 'x', 'y') for o in objs)
self.assertEqual(expected_sz, objsize.get_deep_size(*objs))
self.assertEqual(expected_sz, objsize.get_deep_size(*objs, *objs))
def test_namedtuple(self):
Point = namedtuple('Point', ['x', 'y'])
point = Point(3, 4)
expected_size = (sys.getsizeof(point) +
sys.getsizeof(3) +
sys.getsizeof(4))
self.assertEqual(expected_size, objsize.get_deep_size(point))
def train_fn():
"""
Function which is passed to .train(...) call of an estimator object.
Returns
-------
dataset: tf.data.Dataset object with elements ({'f0': v0, ... 'fx': vx}, label).
"""
#Load the dataset
dataset = tf.data.TFRecordDataset(cfg_train_ds['filename'])
# Apply possible preprocessing, batch and prefetch the dataset.
dataset = dataset.map(preprocess, num_parallel_calls=os.cpu_count())
sample = tf.data.experimental.get_single_element(dataset.take(1))
element_size = get_deep_size(sample)
# Shuffle the dataset
buffer_size = tf.constant(
int((virtual_memory().total / 2) / element_size), tf.int64)
dataset = dataset.shuffle(config['shuffle_size'])
dataset = dataset.batch(config['batch'])
dataset = dataset.prefetch(buffer_size=1)
return dataset.repeat()
def test_subclass_of_namedtuple_with_slots(self):
class MyPoint(namedtuple('MyPoint', ['x', 'y'])):
__slots__ = ()
point = MyPoint(3, 4)
expected_size = (sys.getsizeof(point) + sys.getsizeof(3) +
sys.getsizeof(4))
self.assertEqual(expected_size, objsize.get_deep_size(point))
def test_list_of_collections(self):
collection_list = [[], {}, ()]
empty_list_size = sys.getsizeof([])
empty_tuple_size = sys.getsizeof(())
empty_dict_size = sys.getsizeof({})
expected_size = sys.getsizeof(collection_list) + empty_list_size + empty_tuple_size + empty_dict_size
self.assertEqual(expected_size, objsize.get_deep_size(collection_list))
def compute_row_size(r: Tuple) -> int:
"""
Compute the memory size, in bytes, of the given row's components.
:param r: the row to insert.
:return: the size in bytes.
"""
component_sizes = [get_deep_size(k) for k in r]
return sum(component_sizes)
def test_class_with_string(self):
runtime_int = 15
string = '=' * runtime_int
# the string does occupy space
obj = TestClass(string)
self.assertEqual(
TestClass.sizeof(obj) + sys.getsizeof(string),
objsize.get_deep_size(obj))
def test_custom_class(self):
class MyClass(object):
def __init__(self, x, y):
self.x = x
self.y = y
point = MyClass(3, 4)
expected_size = (calc_class_obj_sz(point, 'x', 'y') +
sys.getsizeof(3) + sys.getsizeof(4))
self.assertEqual(expected_size, objsize.get_deep_size(point))
def get_size(object):
"""Gets the size in bytes of an object"""
size_bytes = get_deep_size(object)
if size_bytes == 0:
return "0B"
size_name = ("B", "KB", "MB", "GB", "TB", "PB", "EB", "ZB", "YB")
i = int(math.floor(math.log(size_bytes, 1024)))
p = math.pow(1024, i)
s = round(size_bytes / p, 2)
return "%s %s" % (s, size_name[i])
def shuffle_minimal_times(num_ciphertexts_l,
number_candidates,
m_value=8,
curve_nid=415,
n_repetitions=1):
"""
Only minimal shuffles times. Attention because this shuffle uses the same
reencryption randomizers for the ciphertexts in BallotBundle.
"""
group = EcGroup(curve_nid)
key_pair = elgamal.KeyPair(group)
pub_key = key_pair.pk
measurements = list()
for num_ciphertexts in num_ciphertexts_l:
ctxts = generate_ciphertexts(num_ciphertexts, group, pub_key)
ctxts, n = ShuffleArgument.prepare_ctxts(ctxts, m_value, pub_key)
com_pub = pedersen_commitment.PublicKey(group, n)
mn = m_value * n
permutation = np.random.permutation(mn).tolist()
randomizers, shuffled_ctxts = generate_shuffled_reencryptions(
ctxts, permutation, group, pub_key, values_vector=False)
# Reshape to shape m*n
ctxts = ShuffleArgument.reshape_m_n(ctxts, m_value)
randomizers = ShuffleArgument.reshape_m_n(randomizers, m_value)
shuffled_ctxts = ShuffleArgument.reshape_m_n(shuffled_ctxts, m_value)
permutation = ShuffleArgument.reshape_m_n(permutation, m_value)
for _ in range(n_repetitions):
shuffle_proof_gen = time.process_time()
proof = ShuffleArgument(com_pub, pub_key, ctxts, shuffled_ctxts,
permutation, randomizers)
size = get_deep_size(proof)
shuffle_proof_verif = time.process_time()
proof.verify(com_pub, pub_key, ctxts[:num_ciphertexts],
shuffled_ctxts)
rt_verify = time.process_time() - shuffle_proof_verif
rt_gen = shuffle_proof_verif - shuffle_proof_gen
measures_vector = (num_ciphertexts, rt_gen, rt_verify, size)
measurements.append(measures_vector)
return measurements
def test_exclusive(self):
import uuid
import gc
obj = [str(uuid.uuid4()) for _ in range(5)]
expected_sz = sys.getsizeof(obj) + sum(map(sys.getsizeof, obj))
self.assertEqual(expected_sz, objsize.get_deep_size(obj))
gc.collect()
self.assertEqual(expected_sz, objsize.get_exclusive_deep_size(obj))
fake_holder = [obj[2]]
expected_sz -= sys.getsizeof(fake_holder[0])
gc.collect()
self.assertEqual(expected_sz, objsize.get_exclusive_deep_size(obj))
def test_subclass_of_namedtuple(self):
class MyPoint(namedtuple('MyPoint', ['x', 'y'])):
pass
point = MyPoint(3, 4)
# namedtuple does not contains a dict.
# It is created on request and is not cached for later calls:
# >>> point = MyPoint(3, 4)
# >>> all_obj = {id(o) for o in gc.get_objects()}
# >>> id(point.__dict__) in all_obj
# False
# >>> all_obj = {id(o) for o in gc.get_objects()}
# >>> id(point.__dict__) in all_obj
# False
expected_size = (sys.getsizeof(point) + sys.getsizeof(3) +
sys.getsizeof(4))
self.assertEqual(expected_size, objsize.get_deep_size(point))
def assert_memory_less_equal(obj, size, exclusive=False):
""" Assert that the memory occupied by an object is less than or equal to a
size
Args:
obj (:obj:`object`): object
size (:obj:`int`): size in bytes
exclusive (:obj:`bool`, optional): if :obj:`True`, check the exclusive
memory of the object
Raises:
:obj:`ValueError`: if the memory occupied by the object is greater than
:obj:`size`
"""
if exclusive:
obj_size = objsize.get_exclusive_deep_size(obj)
else:
obj_size = objsize.get_deep_size(obj)
if obj_size > size:
raise ValueError("{} memory is greater than {}".format(
humanfriendly.format_size(obj_size),
humanfriendly.format_size(size)))
]
for mxd in M_RANGE:
m = int(mxd)
search_times_bf = []
search_times_fl = []
initial_size_bf = []
size_after_insertion_bf = []
false_positives = [0] * ITER
disk_access = [0] * ITER
k = get_k(m, small_n)
print(
f'[STARTED AT] [{time.ctime(time.time())}] n: {big_n}, m: {m}, k: {k}'
)
for i in range(ITER):
bloom_filter = BloomFilter(m, k)
initial_size_bf.append(get_deep_size(bloom_filter))
bloom_filter, L_file, universe_file = generate_files_and_insert_to_bloom_filter(
bloom_filter, big_n, small_n)
size_after_insertion_bf.append(get_deep_size(bloom_filter))
current_time_bf = []
current_time_fl = []
with open('experimentos/files/universe_file.txt',
'r') as universe_file:
total_usernames_list = [
w.strip('\n') for w in universe_file.readlines()
]
for username_query in total_usernames_list:
ti_bf = time.time()
username_might_be_in_file = bloom_filter.check(
username_query)
tf_bf = time.time()
from collections import namedtuple
from dataclasses import dataclass
import objsize
@dataclass
class PetData:
species: str
name: str
age: int
@dataclass(frozen=True)
class PetFrozen:
species: str
name: str
age: int
Pet = namedtuple("Pet", "species name age")
frank = Pet(species="pigeon", name="Френк", age=None)
frank_data = PetData(species="pigeon", name="Френк", age=3)
frank_frozen = PetFrozen(species="pigeon", name="Френк", age=3)
print(objsize.get_deep_size(frank_data))
print(objsize.get_deep_size(frank))
print(objsize.get_deep_size(frank_frozen))
start_time = time.time()
functions[func](text)
end_time = time.time()
print("text", path)
print("function", func)
print("time:", end_time - start_time, "\n")
for path in files:
with open(path, 'r') as file:
text = file.read()
dbf_dic = kmr(text)
print("text:", path)
print("file size:", os.stat(path).st_size)
print("dbf dictionary size:", objsize.get_deep_size(dbf_dic), "\n")
path = files[1]
text = open(path, 'r').read()
for pattern in patterns:
start_time = time.time()
kmp_res = kmp_alg(text, pattern)
end_time = time.time()
kmp_time = end_time - start_time
start_time = time.time()
names, entries = kmr(text)
dbf_res = find_dbf(text, pattern, names, entries)
end_time = time.time()
dbf_with_dict = end_time - start_time
def test_class_with_None(self):
# None doesn't occupy extra space because it is a singleton
obj = TestClass(None)
self.assertEqual(
TestClass.sizeof(obj),
objsize.get_deep_size(obj))
def topological_inventory_data(
_: str,
source_id: str,
dest: str,
headers: dict,
thread
) -> int:
"""Generate Tenant data for topological inventory.
Parameters
----------
_ (str)
Skipped
source_id (str)
Job identifier
dest (str)
URL where to pass data
headers (dict)
RH Identity header
thread
Current Thread
"""
# Build the POST data object
data = {
'id': source_id,
'data': {}
}
for entity in APP_CONFIG:
query_spec = QUERIES[entity]
if query_spec.get('sub_collection'):
try:
all_data = _query_sub_collection(
query_spec,
data['data'],
headers=headers
)
except utils.RetryFailedError as exception:
prometheus_metrics.METRICS['get_errors'].inc()
LOGGER.error(
'%s: Unable to fetch source data for "%s": %s',
thread.name, source_id, exception
)
return 0
else:
try:
all_data = _query_main_collection(query_spec, headers=headers)
except utils.RetryFailedError as exception:
prometheus_metrics.METRICS['get_errors'].inc()
LOGGER.error(
'%s: Unable to fetch source data for "%s": %s',
thread.name, source_id, exception
)
return 0
LOGGER.info(
'%s: %s: %s\t%s',
thread.name, source_id, entity, len(all_data)
)
if all_data:
data['data'][entity] = all_data
else:
LOGGER.debug(
'%s: Inadequate Topological Inventory data for this account.',
thread.name
)
return 0
# Pass to next service
prometheus_metrics.METRICS['posts'].inc()
try:
utils.retryable('post', dest, json=data, headers=headers)
prometheus_metrics.METRICS['post_successes'].inc()
except utils.RetryFailedError as exception:
LOGGER.error(
'%s: Failed to pass data for "%s": %s',
thread.name, source_id, exception
)
prometheus_metrics.METRICS['post_errors'].inc()
return get_deep_size(data['data'])
def __sizeof__(self):
"""
Returns the size of the object in bytes (if the module 'objsize' can be used otherwise 0)
"""
try:
from objsize import get_deep_size
print("sizes:")
print("compression_sa:\t\t ", get_deep_size(self.compression_sa))
print("bucket_step_sa:\t\t ", get_deep_size(self.bucket_step_sa))
print("next_chars\t\t\t ", get_deep_size(self.next_chars))
print("SA:\t\t\t\t\t ", get_deep_size(self.sa))
print("bucket_sa:\t\t\t\t ", get_deep_size(self.bucket_sa))
print("F:\t\t\t\t\t ", get_deep_size(self.f))
print("bitvec:\t\t\t\t ", get_deep_size(self.bitvector))
print("bits:\t\t\t\t ", get_deep_size(self.bits))
print("meta:\t\t\t\t ", get_deep_size(self.meta))
print("codes:\t\t\t\t ", get_deep_size(self.codes))
print("n:\t\t\t\t\t ", get_deep_size(self.n))
print("bucket_bits:\t\t\t\t\t ", get_deep_size(self.bucket_bits))
print("bucket_step_bits:\t\t\t ",
get_deep_size(self.bucket_step_bits))
total = get_deep_size(self.compression_sa) + \
get_deep_size(self.next_chars) + get_deep_size(self.sa) + \
get_deep_size(self.f) + get_deep_size(self.bitvector) + get_deep_size(self.bits) + \
get_deep_size(self.meta) + get_deep_size(self.codes) + get_deep_size(self.n) + \
get_deep_size(self.bucket_step_sa) + get_deep_size(self.bucket_sa) + \
get_deep_size(self.bucket_bits) + get_deep_size(self.bucket_step_bits)
print("Total:\t\t\t\t ", total)
return total
except ImportError:
return 0
def test_empty_list(self):
empty_list = []
self.assertEqual(sys.getsizeof(empty_list),
objsize.get_deep_size(empty_list))
def topological_inventory_data(
_: str,
source_id: str,
dest: str,
headers: dict,
thread
) -> int:
"""Generate Tenant data for topological inventory.
Parameters
----------
_ (str)
Skipped
source_id (str)
Job identifier
dest (str)
URL where to pass data
headers (dict)
RH Identity header
thread
Current Thread
"""
# Build the POST data object
data = {
'id': source_id,
'data': {}
}
if not APP_CONFIG:
LOGGER.error('%s: No queries specified', thread.name)
return
for entity in APP_CONFIG:
query_spec = QUERIES[entity]
try:
if query_spec.get('sub_collection'):
all_data = _query_sub_collection(
query_spec,
data['data'],
headers=headers
)
else:
all_data = _query_main_collection(query_spec, headers=headers)
if not all_data:
raise utils.DataMissingError('Insufficient data.')
except (utils.RetryFailedError, utils.DataMissingError) as exception:
prometheus_metrics.METRICS['get_errors'].inc()
LOGGER.error(
'%s: Unable to fetch source data for "%s": %s',
thread.name, source_id, exception
)
return
LOGGER.debug(
'%s: %s: %s\t%s',
thread.name, source_id, entity, len(all_data)
)
data['data'][entity] = all_data
# Pass to next service
prometheus_metrics.METRICS['posts'].inc()
try:
utils.retryable('post', dest, json=data, headers=headers)
prometheus_metrics.METRICS['post_successes'].inc()
except utils.RetryFailedError as exception:
LOGGER.error(
'%s: Failed to pass data for "%s": %s',
thread.name, source_id, exception
)
prometheus_metrics.METRICS['post_errors'].inc()
data_size = get_deep_size(data['data'])
prometheus_metrics.METRICS['data_size'].observe(data_size)
return
def test_string(self):
test_string = "abc"
self.assertEqual(sys.getsizeof(test_string), objsize.get_deep_size(test_string))
def JRpeg_compress(input_filename,
output_filename="JRpeg_encoded_img",
cbcr_downsize_rate=2,
QL_rate=1,
QC_rate=1):
# Read in original image as RGB three channel array and save a resized copy for display later
logging.info("Loading original image file: {} ...".format(input_filename))
original_img = cv2.imread(input_filename)
logging.info("Original image in memory size: {} bytes!".format(
get_deep_size(original_img)))
resized_original_img = cv2.resize(original_img, (1440, 1080))
# Display original input image resized
cv2.imshow('Original Image: ' + input_filename, resized_original_img)
cv2.waitKey(0)
logging.info("... image loaded successfully!")
# Convert image to YCbCr and downsample Cb and Cr channels
logging.info(
"Converting RGB image to YCbCr image, with Cb and Cr downsampled by a factor of: {} ..."
.format(cbcr_downsize_rate))
YCbCr = rgb_to_ycbcr(original_img)
YCbCr_downsampled = down_sample_cbcr(YCbCr, cbcr_downsize_rate)
logging.info("... YCbCr conversion and downsampling successful!")
# Convert YCbCr image into 8x8 blocks and calculate dct on each block, then quantise each block
logging.info(
"Attempting to DCT and quantise YCbCr with QLuminance_rate: {}, and QChrominance_rate {} ..."
.format(QL_rate, QC_rate))
quantised_dct_img = dct_and_quantise_img(YCbCr_downsampled, QL_rate,
QC_rate)
logging.info("... YCbCr DCT and quantisation successful!")
# Encode quantised dct YCbCr image with RLE grouping, and save to a binary file
logging.info("Attempting to encode and save JRpeg image as: {} ...".format(
output_filename + ".jrpg"))
encoded_img = encode_and_save_quantised_dct_img(quantised_dct_img, QL_rate,
QC_rate,
output_filename + ".jrpg")
logging.info("Encoded image in memory size: {} bytes!".format(
get_deep_size(encoded_img)))
logging.info("... JRpeg image saved successfully!")
logging.info(
"############################################################################"
)
logging.info(
"############################################################################"
)
logging.info(
"## THANK YOUR FOR USING JRpeg, brought to you by: github.com/jounaidr !!! ##"
)
logging.info(
"############################################################################"
)
logging.info(
"############################################################################"
)
return [
get_deep_size(original_img),
get_deep_size(encoded_img),
JRpeg_util.get_img_disk_size(input_filename),
JRpeg_util.get_img_disk_size(output_filename + ".jrpg")
]
# TODO: Add debug logging to methods, optimise iterables, add time metrics
