def test_no_double_counting(self): rep = ["test1"] obj = [rep, rep] obj2 = [rep] expected_sz = objsize.get_deep_size(obj2) - sys.getsizeof( obj2) + sys.getsizeof(obj) self.assertEqual(expected_sz, objsize.get_deep_size(obj))
def test_slots(self): class MySlots1(object): __slots__ = ["number1"] def __init__(self, number1): self.number1 = number1 class MySlots2(object): __slots__ = ["number1", "number2"] def __init__(self, number1, number2): self.number1 = number1 self.number2 = number2 class MySlots3(object): __slots__ = ["number1", "number2", "number3"] def __init__(self, number1, number2, number3): self.number1 = number1 self.number2 = number2 self.number3 = number3 s1 = MySlots1(7) s2 = MySlots2(3, 4) s3 = MySlots3(4, 5, 6) s1_sz = sys.getsizeof(s1) + sys.getsizeof(7) s2_sz = sys.getsizeof(s2) + sys.getsizeof(3) + sys.getsizeof(4) s3_sz = sys.getsizeof(s3) + sys.getsizeof(4) + sys.getsizeof(5) + sys.getsizeof(6) self.assertEqual(s1_sz, objsize.get_deep_size(s1)) self.assertEqual(s2_sz, objsize.get_deep_size(s2)) self.assertEqual(s3_sz, objsize.get_deep_size(s3))
def test_gracefully_handles_self_referential_objects(self): class MyClass(object): pass obj = MyClass() obj.prop = obj self.assertEqual(objsize.get_deep_size(obj), objsize.get_deep_size(obj.prop))
def test_multi_obj(self): class MyClass(object): def __init__(self, x, y): self.x = x self.y = y strs = 'hello world', 'foo bar', 'something else' objs = MyClass(strs[0], strs[1]), MyClass(strs[0], strs[2]), MyClass(strs[1], strs[2]) expected_sz = sum(map(sys.getsizeof, strs)) expected_sz += sum(calc_class_obj_sz(o, 'x', 'y') for o in objs) self.assertEqual(expected_sz, objsize.get_deep_size(*objs)) self.assertEqual(expected_sz, objsize.get_deep_size(*objs, *objs))
def test_namedtuple(self): Point = namedtuple('Point', ['x', 'y']) point = Point(3, 4) expected_size = (sys.getsizeof(point) + sys.getsizeof(3) + sys.getsizeof(4)) self.assertEqual(expected_size, objsize.get_deep_size(point))
def train_fn(): """ Function which is passed to .train(...) call of an estimator object. Returns ------- dataset: tf.data.Dataset object with elements ({'f0': v0, ... 'fx': vx}, label). """ #Load the dataset dataset = tf.data.TFRecordDataset(cfg_train_ds['filename']) # Apply possible preprocessing, batch and prefetch the dataset. dataset = dataset.map(preprocess, num_parallel_calls=os.cpu_count()) sample = tf.data.experimental.get_single_element(dataset.take(1)) element_size = get_deep_size(sample) # Shuffle the dataset buffer_size = tf.constant( int((virtual_memory().total / 2) / element_size), tf.int64) dataset = dataset.shuffle(config['shuffle_size']) dataset = dataset.batch(config['batch']) dataset = dataset.prefetch(buffer_size=1) return dataset.repeat()
def test_subclass_of_namedtuple_with_slots(self): class MyPoint(namedtuple('MyPoint', ['x', 'y'])): __slots__ = () point = MyPoint(3, 4) expected_size = (sys.getsizeof(point) + sys.getsizeof(3) + sys.getsizeof(4)) self.assertEqual(expected_size, objsize.get_deep_size(point))
def test_list_of_collections(self): collection_list = [[], {}, ()] empty_list_size = sys.getsizeof([]) empty_tuple_size = sys.getsizeof(()) empty_dict_size = sys.getsizeof({}) expected_size = sys.getsizeof(collection_list) + empty_list_size + empty_tuple_size + empty_dict_size self.assertEqual(expected_size, objsize.get_deep_size(collection_list))
def compute_row_size(r: Tuple) -> int: """ Compute the memory size, in bytes, of the given row's components. :param r: the row to insert. :return: the size in bytes. """ component_sizes = [get_deep_size(k) for k in r] return sum(component_sizes)
def test_class_with_string(self): runtime_int = 15 string = '=' * runtime_int # the string does occupy space obj = TestClass(string) self.assertEqual( TestClass.sizeof(obj) + sys.getsizeof(string), objsize.get_deep_size(obj))
def test_custom_class(self): class MyClass(object): def __init__(self, x, y): self.x = x self.y = y point = MyClass(3, 4) expected_size = (calc_class_obj_sz(point, 'x', 'y') + sys.getsizeof(3) + sys.getsizeof(4)) self.assertEqual(expected_size, objsize.get_deep_size(point))
def get_size(object): """Gets the size in bytes of an object""" size_bytes = get_deep_size(object) if size_bytes == 0: return "0B" size_name = ("B", "KB", "MB", "GB", "TB", "PB", "EB", "ZB", "YB") i = int(math.floor(math.log(size_bytes, 1024))) p = math.pow(1024, i) s = round(size_bytes / p, 2) return "%s %s" % (s, size_name[i])
def shuffle_minimal_times(num_ciphertexts_l, number_candidates, m_value=8, curve_nid=415, n_repetitions=1): """ Only minimal shuffles times. Attention because this shuffle uses the same reencryption randomizers for the ciphertexts in BallotBundle. """ group = EcGroup(curve_nid) key_pair = elgamal.KeyPair(group) pub_key = key_pair.pk measurements = list() for num_ciphertexts in num_ciphertexts_l: ctxts = generate_ciphertexts(num_ciphertexts, group, pub_key) ctxts, n = ShuffleArgument.prepare_ctxts(ctxts, m_value, pub_key) com_pub = pedersen_commitment.PublicKey(group, n) mn = m_value * n permutation = np.random.permutation(mn).tolist() randomizers, shuffled_ctxts = generate_shuffled_reencryptions( ctxts, permutation, group, pub_key, values_vector=False) # Reshape to shape m*n ctxts = ShuffleArgument.reshape_m_n(ctxts, m_value) randomizers = ShuffleArgument.reshape_m_n(randomizers, m_value) shuffled_ctxts = ShuffleArgument.reshape_m_n(shuffled_ctxts, m_value) permutation = ShuffleArgument.reshape_m_n(permutation, m_value) for _ in range(n_repetitions): shuffle_proof_gen = time.process_time() proof = ShuffleArgument(com_pub, pub_key, ctxts, shuffled_ctxts, permutation, randomizers) size = get_deep_size(proof) shuffle_proof_verif = time.process_time() proof.verify(com_pub, pub_key, ctxts[:num_ciphertexts], shuffled_ctxts) rt_verify = time.process_time() - shuffle_proof_verif rt_gen = shuffle_proof_verif - shuffle_proof_gen measures_vector = (num_ciphertexts, rt_gen, rt_verify, size) measurements.append(measures_vector) return measurements
def test_exclusive(self): import uuid import gc obj = [str(uuid.uuid4()) for _ in range(5)] expected_sz = sys.getsizeof(obj) + sum(map(sys.getsizeof, obj)) self.assertEqual(expected_sz, objsize.get_deep_size(obj)) gc.collect() self.assertEqual(expected_sz, objsize.get_exclusive_deep_size(obj)) fake_holder = [obj[2]] expected_sz -= sys.getsizeof(fake_holder[0]) gc.collect() self.assertEqual(expected_sz, objsize.get_exclusive_deep_size(obj))
def test_subclass_of_namedtuple(self): class MyPoint(namedtuple('MyPoint', ['x', 'y'])): pass point = MyPoint(3, 4) # namedtuple does not contains a dict. # It is created on request and is not cached for later calls: # >>> point = MyPoint(3, 4) # >>> all_obj = {id(o) for o in gc.get_objects()} # >>> id(point.__dict__) in all_obj # False # >>> all_obj = {id(o) for o in gc.get_objects()} # >>> id(point.__dict__) in all_obj # False expected_size = (sys.getsizeof(point) + sys.getsizeof(3) + sys.getsizeof(4)) self.assertEqual(expected_size, objsize.get_deep_size(point))
def assert_memory_less_equal(obj, size, exclusive=False): """ Assert that the memory occupied by an object is less than or equal to a size Args: obj (:obj:`object`): object size (:obj:`int`): size in bytes exclusive (:obj:`bool`, optional): if :obj:`True`, check the exclusive memory of the object Raises: :obj:`ValueError`: if the memory occupied by the object is greater than :obj:`size` """ if exclusive: obj_size = objsize.get_exclusive_deep_size(obj) else: obj_size = objsize.get_deep_size(obj) if obj_size > size: raise ValueError("{} memory is greater than {}".format( humanfriendly.format_size(obj_size), humanfriendly.format_size(size)))
] for mxd in M_RANGE: m = int(mxd) search_times_bf = [] search_times_fl = [] initial_size_bf = [] size_after_insertion_bf = [] false_positives = [0] * ITER disk_access = [0] * ITER k = get_k(m, small_n) print( f'[STARTED AT] [{time.ctime(time.time())}] n: {big_n}, m: {m}, k: {k}' ) for i in range(ITER): bloom_filter = BloomFilter(m, k) initial_size_bf.append(get_deep_size(bloom_filter)) bloom_filter, L_file, universe_file = generate_files_and_insert_to_bloom_filter( bloom_filter, big_n, small_n) size_after_insertion_bf.append(get_deep_size(bloom_filter)) current_time_bf = [] current_time_fl = [] with open('experimentos/files/universe_file.txt', 'r') as universe_file: total_usernames_list = [ w.strip('\n') for w in universe_file.readlines() ] for username_query in total_usernames_list: ti_bf = time.time() username_might_be_in_file = bloom_filter.check( username_query) tf_bf = time.time()
from collections import namedtuple from dataclasses import dataclass import objsize @dataclass class PetData: species: str name: str age: int @dataclass(frozen=True) class PetFrozen: species: str name: str age: int Pet = namedtuple("Pet", "species name age") frank = Pet(species="pigeon", name="Френк", age=None) frank_data = PetData(species="pigeon", name="Френк", age=3) frank_frozen = PetFrozen(species="pigeon", name="Френк", age=3) print(objsize.get_deep_size(frank_data)) print(objsize.get_deep_size(frank)) print(objsize.get_deep_size(frank_frozen))
start_time = time.time() functions[func](text) end_time = time.time() print("text", path) print("function", func) print("time:", end_time - start_time, "\n") for path in files: with open(path, 'r') as file: text = file.read() dbf_dic = kmr(text) print("text:", path) print("file size:", os.stat(path).st_size) print("dbf dictionary size:", objsize.get_deep_size(dbf_dic), "\n") path = files[1] text = open(path, 'r').read() for pattern in patterns: start_time = time.time() kmp_res = kmp_alg(text, pattern) end_time = time.time() kmp_time = end_time - start_time start_time = time.time() names, entries = kmr(text) dbf_res = find_dbf(text, pattern, names, entries) end_time = time.time() dbf_with_dict = end_time - start_time
def test_class_with_None(self): # None doesn't occupy extra space because it is a singleton obj = TestClass(None) self.assertEqual( TestClass.sizeof(obj), objsize.get_deep_size(obj))
def topological_inventory_data( _: str, source_id: str, dest: str, headers: dict, thread ) -> int: """Generate Tenant data for topological inventory. Parameters ---------- _ (str) Skipped source_id (str) Job identifier dest (str) URL where to pass data headers (dict) RH Identity header thread Current Thread """ # Build the POST data object data = { 'id': source_id, 'data': {} } for entity in APP_CONFIG: query_spec = QUERIES[entity] if query_spec.get('sub_collection'): try: all_data = _query_sub_collection( query_spec, data['data'], headers=headers ) except utils.RetryFailedError as exception: prometheus_metrics.METRICS['get_errors'].inc() LOGGER.error( '%s: Unable to fetch source data for "%s": %s', thread.name, source_id, exception ) return 0 else: try: all_data = _query_main_collection(query_spec, headers=headers) except utils.RetryFailedError as exception: prometheus_metrics.METRICS['get_errors'].inc() LOGGER.error( '%s: Unable to fetch source data for "%s": %s', thread.name, source_id, exception ) return 0 LOGGER.info( '%s: %s: %s\t%s', thread.name, source_id, entity, len(all_data) ) if all_data: data['data'][entity] = all_data else: LOGGER.debug( '%s: Inadequate Topological Inventory data for this account.', thread.name ) return 0 # Pass to next service prometheus_metrics.METRICS['posts'].inc() try: utils.retryable('post', dest, json=data, headers=headers) prometheus_metrics.METRICS['post_successes'].inc() except utils.RetryFailedError as exception: LOGGER.error( '%s: Failed to pass data for "%s": %s', thread.name, source_id, exception ) prometheus_metrics.METRICS['post_errors'].inc() return get_deep_size(data['data'])
def __sizeof__(self): """ Returns the size of the object in bytes (if the module 'objsize' can be used otherwise 0) """ try: from objsize import get_deep_size print("sizes:") print("compression_sa:\t\t ", get_deep_size(self.compression_sa)) print("bucket_step_sa:\t\t ", get_deep_size(self.bucket_step_sa)) print("next_chars\t\t\t ", get_deep_size(self.next_chars)) print("SA:\t\t\t\t\t ", get_deep_size(self.sa)) print("bucket_sa:\t\t\t\t ", get_deep_size(self.bucket_sa)) print("F:\t\t\t\t\t ", get_deep_size(self.f)) print("bitvec:\t\t\t\t ", get_deep_size(self.bitvector)) print("bits:\t\t\t\t ", get_deep_size(self.bits)) print("meta:\t\t\t\t ", get_deep_size(self.meta)) print("codes:\t\t\t\t ", get_deep_size(self.codes)) print("n:\t\t\t\t\t ", get_deep_size(self.n)) print("bucket_bits:\t\t\t\t\t ", get_deep_size(self.bucket_bits)) print("bucket_step_bits:\t\t\t ", get_deep_size(self.bucket_step_bits)) total = get_deep_size(self.compression_sa) + \ get_deep_size(self.next_chars) + get_deep_size(self.sa) + \ get_deep_size(self.f) + get_deep_size(self.bitvector) + get_deep_size(self.bits) + \ get_deep_size(self.meta) + get_deep_size(self.codes) + get_deep_size(self.n) + \ get_deep_size(self.bucket_step_sa) + get_deep_size(self.bucket_sa) + \ get_deep_size(self.bucket_bits) + get_deep_size(self.bucket_step_bits) print("Total:\t\t\t\t ", total) return total except ImportError: return 0
def test_empty_list(self): empty_list = [] self.assertEqual(sys.getsizeof(empty_list), objsize.get_deep_size(empty_list))
def topological_inventory_data( _: str, source_id: str, dest: str, headers: dict, thread ) -> int: """Generate Tenant data for topological inventory. Parameters ---------- _ (str) Skipped source_id (str) Job identifier dest (str) URL where to pass data headers (dict) RH Identity header thread Current Thread """ # Build the POST data object data = { 'id': source_id, 'data': {} } if not APP_CONFIG: LOGGER.error('%s: No queries specified', thread.name) return for entity in APP_CONFIG: query_spec = QUERIES[entity] try: if query_spec.get('sub_collection'): all_data = _query_sub_collection( query_spec, data['data'], headers=headers ) else: all_data = _query_main_collection(query_spec, headers=headers) if not all_data: raise utils.DataMissingError('Insufficient data.') except (utils.RetryFailedError, utils.DataMissingError) as exception: prometheus_metrics.METRICS['get_errors'].inc() LOGGER.error( '%s: Unable to fetch source data for "%s": %s', thread.name, source_id, exception ) return LOGGER.debug( '%s: %s: %s\t%s', thread.name, source_id, entity, len(all_data) ) data['data'][entity] = all_data # Pass to next service prometheus_metrics.METRICS['posts'].inc() try: utils.retryable('post', dest, json=data, headers=headers) prometheus_metrics.METRICS['post_successes'].inc() except utils.RetryFailedError as exception: LOGGER.error( '%s: Failed to pass data for "%s": %s', thread.name, source_id, exception ) prometheus_metrics.METRICS['post_errors'].inc() data_size = get_deep_size(data['data']) prometheus_metrics.METRICS['data_size'].observe(data_size) return
def test_string(self): test_string = "abc" self.assertEqual(sys.getsizeof(test_string), objsize.get_deep_size(test_string))
def JRpeg_compress(input_filename, output_filename="JRpeg_encoded_img", cbcr_downsize_rate=2, QL_rate=1, QC_rate=1): # Read in original image as RGB three channel array and save a resized copy for display later logging.info("Loading original image file: {} ...".format(input_filename)) original_img = cv2.imread(input_filename) logging.info("Original image in memory size: {} bytes!".format( get_deep_size(original_img))) resized_original_img = cv2.resize(original_img, (1440, 1080)) # Display original input image resized cv2.imshow('Original Image: ' + input_filename, resized_original_img) cv2.waitKey(0) logging.info("... image loaded successfully!") # Convert image to YCbCr and downsample Cb and Cr channels logging.info( "Converting RGB image to YCbCr image, with Cb and Cr downsampled by a factor of: {} ..." .format(cbcr_downsize_rate)) YCbCr = rgb_to_ycbcr(original_img) YCbCr_downsampled = down_sample_cbcr(YCbCr, cbcr_downsize_rate) logging.info("... YCbCr conversion and downsampling successful!") # Convert YCbCr image into 8x8 blocks and calculate dct on each block, then quantise each block logging.info( "Attempting to DCT and quantise YCbCr with QLuminance_rate: {}, and QChrominance_rate {} ..." .format(QL_rate, QC_rate)) quantised_dct_img = dct_and_quantise_img(YCbCr_downsampled, QL_rate, QC_rate) logging.info("... YCbCr DCT and quantisation successful!") # Encode quantised dct YCbCr image with RLE grouping, and save to a binary file logging.info("Attempting to encode and save JRpeg image as: {} ...".format( output_filename + ".jrpg")) encoded_img = encode_and_save_quantised_dct_img(quantised_dct_img, QL_rate, QC_rate, output_filename + ".jrpg") logging.info("Encoded image in memory size: {} bytes!".format( get_deep_size(encoded_img))) logging.info("... JRpeg image saved successfully!") logging.info( "############################################################################" ) logging.info( "############################################################################" ) logging.info( "## THANK YOUR FOR USING JRpeg, brought to you by: github.com/jounaidr !!! ##" ) logging.info( "############################################################################" ) logging.info( "############################################################################" ) return [ get_deep_size(original_img), get_deep_size(encoded_img), JRpeg_util.get_img_disk_size(input_filename), JRpeg_util.get_img_disk_size(output_filename + ".jrpg") ] # TODO: Add debug logging to methods, optimise iterables, add time metrics