def test_add_different_types_equal_bitmath_type(self):
"""Adding two different bitmath types is equal to another type of the same size"""
# One Kibibyte + 1024 Bytes = 2048 bytes = Byte(2048)
kib1 = bitmath.KiB(1)
byte1 = bitmath.Byte(1024)
added_different_types = kib1 + byte1
two_kib_in_bytes = bitmath.Byte(2048)
self.assertEqual(added_different_types, two_kib_in_bytes)
def test_init_value(self):
"""Instantiation works with only 'value' provided"""
bm1 = bitmath.Byte(1)
bm2 = bitmath.Byte(value=1)
# These instances will be equivalent to each other and
# congruent to int(1)
self.assertEqual(bm1, bm2)
self.assertEqual(bm1, int(1))
self.assertEqual(bm2, int(1))
def get_information(self):
try:
list_node = self.api_instance.list_node()
items = list_node.items
total_processor_local = 0
total_memory_local = bitmath.Byte(0)
for item in items:
allocatable = item.status.allocatable
self.architecture = str(item.status.node_info.architecture)
total_processor_local = total_processor_local + float(
allocatable['cpu'])
allocatable_memory_str = allocatable['memory'] + "B"
total_memory_local = total_memory_local + bitmath.parse_string(
allocatable_memory_str)
self.total_processor = str(total_processor_local)
self.total_memory = str(
total_memory_local.best_prefix().KiB.format("{value:.0f}"))
list_pod_for_all_namespaces = self.api_instance.list_pod_for_all_namespaces(
)
allocated_processor_local = 0
allocated_memory_local = bitmath.Byte(0)
for item in list_pod_for_all_namespaces.items:
containers = item.spec.containers
for container in containers:
requests = container.resources.requests
if requests is None:
continue
if 'cpu' in requests:
if 'm' in requests['cpu']:
cpu_str = requests['cpu'].replace("m", "")
container_cpu = float(cpu_str) / 1000
else:
cpu_str = requests['cpu']
container_cpu = float(cpu_str)
allocated_processor_local = allocated_processor_local + container_cpu
if 'memory' in requests:
memory_str = requests['memory'] + "B"
allocated_memory_local = allocated_memory_local + bitmath.parse_string(
memory_str)
allocated_processor_local = float(
"{0:.2f}".format(allocated_processor_local))
self.allocated_processor = str(allocated_processor_local)
self.allocated_memory = allocated_memory_local.KiB.format(
"{value:.0f}")
self.allocatable_processor = str(total_processor_local -
allocated_processor_local)
allocatable_memory_local = total_memory_local - allocated_memory_local
self.allocatable_memory = allocatable_memory_local.KiB.format(
"{value:.0f}")
except ApiException as e:
print(
"Exception when calling CoreV1Api->list_resource_quota_for_all_namespaces: %s\n"
% e)
def get_size(self, obj):
if hasattr(obj, 'arquivo'):
size = bitmath.Byte(
obj.arquivo.size).best_prefix().format("{value:.2f} {unit}")
if hasattr(obj, 'imagem'):
size = bitmath.Byte(
obj.imagem.size).best_prefix().format("{value:.2f} {unit}")
if hasattr(obj, 'audio'):
size = bitmath.Byte(
obj.audio.size).best_prefix().format("{value:.2f} {unit}")
return size
def disk():
disk_usage = psutil.disk_usage('/')
return {
'totalGB':
float(
bitmath.Byte(disk_usage.total, ).to_GB().format('{value:.1f}'), ),
'usedGB':
float(bitmath.Byte(disk_usage.used, ).to_GB().format('{value:.1f}'), ),
'freeGB':
float(bitmath.Byte(disk_usage.free, ).to_GB().format('{value:.1f}'), ),
'percent':
disk_usage.percent,
}
def update(self, pbar):
"""Updates the widget with the current NIST/SI speed.
Basically, this calculates the average rate of update and figures out
how to make a "pretty" prefix unit"""
if pbar.seconds_elapsed < 2e-6 or pbar.currval < 2e-6:
scaled = bitmath.Byte()
else:
speed = pbar.currval / pbar.seconds_elapsed
scaled = bitmath.Byte(speed).best_prefix(system=self.system)
return scaled.format(self.format)
def memory():
memory_usage = psutil.virtual_memory()
return {
'totalGiB':
float(
bitmath.Byte(
memory_usage.total, ).to_GiB().format('{value:.1f}'), ),
'freeGiB':
float(
bitmath.Byte(
memory_usage.available, ).to_GiB().format('{value:.1f}'), ),
'percent':
memory_usage.percent,
}
def test_parse_string_unsafe_request_NIST(self):
"""parse_string_unsafe can convert to NIST on request"""
unsafe_input = "100M"
_parsed = bitmath.parse_string_unsafe(unsafe_input, system=bitmath.NIST)
expected = bitmath.MiB(100)
self.assertEqual(_parsed, expected)
self.assertIs(type(_parsed), type(expected))
unsafe_input2 = "100k"
_parsed2 = bitmath.parse_string_unsafe(unsafe_input2, system=bitmath.NIST)
expected2 = bitmath.KiB(100)
self.assertEqual(_parsed2, expected2)
self.assertIs(type(_parsed2), type(expected2))
unsafe_input3 = "100"
_parsed3 = bitmath.parse_string_unsafe(unsafe_input3, system=bitmath.NIST)
expected3 = bitmath.Byte(100)
self.assertEqual(_parsed3, expected3)
self.assertIs(type(_parsed3), type(expected3))
unsafe_input4 = "100kb"
_parsed4 = bitmath.parse_string_unsafe(unsafe_input4, system=bitmath.NIST)
expected4 = bitmath.KiB(100)
self.assertEqual(_parsed4, expected4)
self.assertIs(type(_parsed4), type(expected4))
def _bytes_to_gig(self, size):
"""Convert size in bytes to GiB.
:param size: The number of bytes.
:returns: The size in gigabytes.
"""
return bitmath.Byte(size).to_GiB().value
def _bytes_to_gib(size):
"""Convert size in bytes to GiB.
:param size: The number of bytes.
:returns: The size in gigabytes.
"""
return int(bitmath.Byte(size).GiB.value)
def test_guest_performance(self):
"""
In-guest performance bandwidth passthrough
"""
if not config.REAL_NICS_ENV:
pytest.skip(msg='Only run on bare metal env')
server_vm = config.VMS_LIST[0]
client_vm = config.VMS_LIST[1]
server_ip = config.VMS.get(server_vm).get('ovs_ip')
with console.Console(vm=server_vm,
distro='fedora',
namespace=config.NETWORK_NS) as server_vm_console:
server_vm_console.sendline(
'iperf3 -sB {server_ip}'.format(server_ip=server_ip))
with console.Console(
vm=client_vm, distro='fedora',
namespace=config.NETWORK_NS) as client_vm_console:
client_vm_console.sendline(
'iperf3 -c {server_ip} -t 5 -u -J'.format(
server_ip=server_ip))
client_vm_console.expect('}\r\r\n}\r\r\n')
iperf_data = client_vm_console.before
server_vm_console.sendline(
chr(3)) # Send ctrl+c to kill iperf3 server
iperf_data += '}\r\r\n}\r\r\n'
iperf_json = json.loads(iperf_data[iperf_data.find('{'):])
sum_sent = iperf_json.get('end').get('sum')
bits_per_second = int(sum_sent.get('bits_per_second'))
assert float(bitmath.Byte(bits_per_second).GiB) >= 2.5
def test_print_byte_singular(self):
"""Byte(1.0) prints out units in singular form"""
expected_result = "1Byte"
fmt_str = "{value:.2g}{unit}"
one_Byte = bitmath.Byte(1.0)
actual_result = one_Byte.format(fmt_str)
self.assertEqual(expected_result, actual_result)
def get_tensors_memory():
tensors_list = []
for obj in gc.get_objects():
try:
if torch.is_tensor(obj) or (hasattr(obj, 'data')
and torch.is_tensor(obj.data)):
tensor_size = reduce(op.mul,
obj.size()) if len(obj.size()) > 0 else 0
mem_size = bitmath.Byte(tensor_size * obj.element_size())
tensor_size_formated = mem_size.best_prefix().format(
"{value:>6.2f} {unit:5}")
tensors_list.append({
'size': mem_size,
'shape': str(list(obj.size())),
'type': str(type(obj)),
'dtype': str(obj.dtype),
'data_size': obj.element_size(),
'device': str(obj.device),
'class': obj.__class__.__name__,
})
except:
pass
for tensor in sorted(tensors_list,
key=lambda d: (d['class'], d['size']),
reverse=True):
tensor_size_formated = tensor['size'].best_prefix().format(
"{value:>6.2f} {unit:5}")
tensor_info_formated = '{p[shape]:20} {p[dtype]:16} {p[device]:8} {p[class]:10}'.format(
p=tensor)
print(tensor_size_formated, tensor_info_formated)
def test_with_format(self):
"""bitmath.format context mgr sets and restores formatting"""
to_print = [
bitmath.Byte(101),
bitmath.KiB(202),
bitmath.MB(303),
bitmath.GiB(404),
bitmath.TB(505),
bitmath.PiB(606),
bitmath.EB(707)
]
str_reps = [
"101.00-Byte", "202.00-KiB", "303.00-MB", "404.00-GiB",
"505.00-TB", "606.00-PiB", "707.00-EB"
]
# Make sure formatting looks right BEFORE the context manager
self.assertEqual(str(bitmath.KiB(1.337)), "1.337 KiB")
with bitmath.format("{value:.2f}-{unit}"):
for (inst, inst_str) in zip(to_print, str_reps):
self.assertEqual(str(inst), inst_str)
# Make sure formatting looks right AFTER the context manager
self.assertEqual(str(bitmath.KiB(1.337)), "1.337 KiB")
def _evict(self) -> None:
with self._memory_lock:
total_size: bitmath.Bitmath = bitmath.Byte(0)
for key, entry in self._index.items():
# heapq.heappush(heap, (self._eval_func(entry), key, entry))
total_size += entry.data_size
while total_size > self._size:
key, entry = self._replacement_policy.evict()
if entry.obj_store:
obj_key = determ_hash(key)
assert (
obj_key in self._obj_store
), "Replacement policy tried to evict something that wasn't there"
del self._obj_store[obj_key]
else:
obj_key = None
total_size -= entry.data_size
if ops_logger.isEnabledFor(logging.DEBUG):
ops_logger.debug(
json.dumps(
{
"event": "evict",
"key": key,
"obj_key": obj_key,
"entry.data_size": entry.data_size.bytes,
"new_total_size": (total_size - entry.data_size).bytes,
}
)
)
del self._index[key]
def _scrape_parallel(args):
results = []
try:
ip, port = args
print(f'Scraping {ip}:{port}')
es = Elasticsearch([{"host": ip, "port": port}])
indices = es.indices.stats('_all')['indices']
if not indices:
print(f'No indexes for IP {ip}')
return []
for idx, data in indices.items():
exclude_index = False
if any(map(lambda rx: re.search(rx, idx),
INDEX_EXCLUSION_LIST_REGEXES)):
continue
uuid = data.get('uuid') or idx
docs_count = data['total']['docs']['count']
docs_deleted = data['total']['docs']['deleted']
store_size_bytes = data['total']['store']['size_in_bytes']
bm = bitmath.Byte(store_size_bytes)
store_size = bm.best_prefix(bitmath.SI).format('{value:.0f} {unit}')
results.append((
ip, port, idx, uuid, docs_count, docs_deleted,
store_size, store_size_bytes,
datetime.now().strftime('%Y-%m-%d %H:%M:%S')))
print(f'Scraped {len(results)} indexes from IP {ip}:{port}')
except ElasticsearchException as e:
print(f'Exception connecting to IP {ip}:{port}: {e}')
return results
def test_listdir_symlinks_nofollow(self):
"""listdir: symbolic links in tree not followed
Similar assumptions as in test_listdir_nosymlinks, except the
directory structure looks like this:
$ tree tests/listdir_symlinks
tests/listdir_symlinks
|-- 10_byte_file_link -> ../listdir/10_byte_file
`-- depth1
`-- depth2
`-- 10_byte_file
2 directories, 2 files
"""
# Call with relpath=True so the paths are easier to verify
contents = list(
bitmath.listdir('./tests/listdir_symlinks/', relpath=True))
# Ensure the returned path matches the expected path
self.assertEqual(contents[0][0],
'tests/listdir_symlinks/depth1/depth2/10_byte_file')
# Ensure the measured size is what we expect
self.assertEqual(contents[0][1], bitmath.Byte(10.0))
def test_cli_script_main_from_and_to_unit(self):
"""CLI script returns correct if given FROM and TO units"""
args = ['-f', 'MiB', '-t', 'Byte', '1']
# Testing FROM 1 MiB TO equivalent Bytes
results = bitmath.cli_script_main(args)
self.assertEqual(results[0], bitmath.Byte(1048576))
self.assertIs(type(results[0]), bitmath.Byte)
def bytes_to_kib(size):
"""Convert size in bytes to KiB.
:param size: The number of bytes.
:returns: The size in Kilobytes.
"""
return bitmath.Byte(size).to_KiB().value
def test_BitmathType_good_two_args(self):
"""Argparse: BitmathType - Works when given two correct parameters"""
args = "--two-args 1337B 0.001GiB"
result = self._parse_two_args(args)
self.assertEqual(len(result.two_args), 2)
self.assertIn(bitmath.Byte(1337), result.two_args)
self.assertIn(bitmath.GiB(0.001), result.two_args)
def test_init_bytes(self):
"""Instantiation works with the 'bytes' kw arg"""
bm = bitmath.Byte(bytes=1024)
# 1024 bytes is 1 KiB, these should be equal
self.assertEqual(bm, bitmath.KiB(1))
self.assertEqual(bm.bytes, 1024)
def test_listdir_nosymlinks(self):
"""listdir: no symbolic links in tree measures right
Assume a directory tree where no sub-directories are symbolic links::
$ tree ./tests/listdir_nosymlinks
./tests/listdir_nosymlinks
`-- depth1
`-- depth2
|-- 1024_byte_file
`-- 10_byte_file
2 directories, 2 files
And the files, ``tests/listdir_nosymlinks/depth1/depth2/10_byte_file``
and ``tests/listdir_nosymlinks/depth1/depth2/1024_byte_file`` are 10
Bytes and 1024 Bytes in size, respectively.
Then:
>>> for f in bitmath.listdir('./tests/listdir_nosymlinks'):
... print f
Would yield 2-tuple's of:
('/path/tests/listdir_nosymlinks/depth1/depth2/10_byte_file', Byte(10.0))
('/path/tests/listdir_nosymlinks/depth1/depth2/1024_byte_file', KiB(1.0))
"""
# Call with relpath=True so the paths are easier to verify
contents = list(
bitmath.listdir('./tests/listdir_nosymlinks/', relpath=True))
# Ensure the returned paths match the expected paths
discovered_paths = [
contents[0][0],
contents[1][0],
]
expected_paths = [
'tests/listdir_nosymlinks/depth1/depth2/10_byte_file',
'tests/listdir_nosymlinks/depth1/depth2/1024_byte_file'
]
self.assertListEqual(discovered_paths, expected_paths)
expected_sizes = [bitmath.Byte(10.0), bitmath.Byte(1024.0)]
discovered_sizes = [contents[0][1], contents[1][1]]
self.assertListEqual(discovered_sizes, expected_sizes)
def __init__(
self,
*,
obj_store: Optional[ObjStore] = None,
replacement_policy: Union[str, ReplacementPolicy] = "gdsize",
size: Union[int, str, bitmath.Bitmath] = bitmath.KiB(100),
pickler: Pickler = pickle,
lock: Optional[RWLock] = None,
fine_grain_persistence: bool = False,
fine_grain_eviction: bool = False,
extra_system_state: Callable[[], Any] = Constant(None),
temporary: bool = False,
) -> None:
"""Construct a memoized group. Use with :py:function:Memoized.
:param obj_store: The object store to use for return values.
:param replacement_policy: See policies submodule for options. You can pass an object conforming to the ReplacementPolicy protocol or one of REPLACEMENT_POLICIES.
:param size: The size as an int (in bytes), as a string (e.g. "3 MiB"), or as a `bitmath.Bitmath`_.
:param pickler: A de/serialization to use on the index, conforming to the Pickler protocol.
:param lock: A ReadersWriterLock to achieve exclusion. If the lock is wrong but the obj_store is atomic, then the memoization is still *correct*, but it may not be able to borrow values that another machine computed. Defaults to a FileRWLock.
:param fine_grain_persistence: De/serialize the index at every access. This is useful if you need to update the cache for multiple simultaneous processes, but it compromises performance in the single-process case.
:param fine_grain_eviction: Maintain the cache's size through eviction at every access (rather than just the de/serialization points). This is useful if the caches size would not otherwise fit in memory, but it compromises performance if not needed.
:param extra_system_state: A callable that returns "extra" system state. If the system state changes, the cache is dumped.
:param temporary: Whether the cache should be cleared at the end of the process; This is useful for tests.
.. _`bitmath.Bitmath`: https://pypi.org/project/bitmath/
"""
self._obj_store = (
obj_store
if obj_store is not None
else DirObjStore(path=DEFAULT_OBJ_STORE_PATH)
)
self._replacement_policy = (
REPLACEMENT_POLICIES[replacement_policy.lower()]()
if isinstance(replacement_policy, str)
else replacement_policy
)
self._size = (
size
if isinstance(size, bitmath.Bitmath)
else bitmath.Byte(size)
if isinstance(size, int)
else bitmath.parse_string(size)
)
self._pickler = pickler
self._index_lock = lock if lock is not None else FileRWLock(DEFAULT_LOCK_PATH)
self._fine_grain_persistence = fine_grain_persistence
self._fine_grain_eviction = fine_grain_eviction
self._extra_system_state = extra_system_state
self._index_key = 0
self.time_cost = DefaultDict[str, datetime.timedelta](datetime.timedelta)
self.time_saved = DefaultDict[str, datetime.timedelta](datetime.timedelta)
self.temporary = temporary
self.__setstate__({})
if self.temporary:
atexit.register(self._obj_store.clear)
# atexit handlers are run in the opposite order they are registered.
atexit.register(self._index_write)
def test_print_GiB_plural_fmt_in_mgr(self):
"""TiB(1/3.0) prints out units in plural form, setting the fmt str in the mgr"""
expected_result = "3Bytes"
with bitmath.format(fmt_str="{value:.1g}{unit}", plural=True):
three_Bytes = bitmath.Byte(3.0)
actual_result = str(three_Bytes)
self.assertEqual(expected_result, actual_result)
def get_capacity_representation(self):
"""
Converts the capacity in bytes to the appropriate value for display
"""
if not self.capacity:
return ''
return (bitmath.Byte(self.capacity).best_prefix(
bitmath.SI).format("{value} {unit}"))
def __init__(self, contents):
self.contents = xmltodict.parse(contents)
self.administrative_metadata = self.__get_all_administrative_metadata()
self.original_files = self.__find_original_files()
self.total_size = sum(
[int(original.size) for original in self.original_files])
self.pretty_total_size = bitmath.Byte(
bytes=int(self.total_size)).best_prefix()
def test_print_byte_plural(self):
"""Byte(3.0) prints out units in plural form"""
expected_result = "3Bytes"
fmt_str = "{value:.1g}{unit}"
three_Bytes = bitmath.Byte(3.0)
with bitmath.format(plural=True):
actual_result = three_Bytes.format(fmt_str)
self.assertEqual(expected_result, actual_result)
def test_best_prefix_negative_less_than_a_byte(self):
"""best_prefix_base: negative values less than a byte stay as bits"""
# assert that a Byte of -4 bits yields Bit(-4)
bm1 = bitmath.Byte(bits=-4)
expected = bitmath.Bit(-4)
res = bitmath.best_prefix(bm1)
# Verify that best prefix math works for negative numbers
self.assertEqual(res, expected)
# Verify that best prefix guessed the correct type
self.assertIs(type(res), bitmath.Bit)
def human_readable_file_size(size_in_bytes):
'''
Returns a nicly human readable file size
:param size_in_bytes: Size in Bytes
:type size_in_bytes: int
:return: str
'''
return bitmath.Byte(
bytes=size_in_bytes).best_prefix().format('{value:.2f} {unit}')
def size(s):
try:
return bitmath.Byte(s).best_prefix()
except NameError:
for unit in ('B', 'KiB', 'MiB', 'GiB', 'TiB', 'PiB', 'EiB', 'ZiB'):
if abs(s) < 1024.0:
return "{0:3.2f} {1}".format(s, unit)
else:
s /= 1024.0
return "{0:.2f} {1}".format(s, 'YiB')