async def update_progress_line(self, line):
if not line:
return
line = line.strip()
try:
(total, ) = self.SIZE_LINE_RE.search(line).groups()
self.progress.total = bitmath.parse_string(total + "b")
except AttributeError:
pass
try:
(self.progress.dest, ) = self.DEST_LINE_RE.search(line).groups()
except AttributeError:
pass
try:
(pct, dled, rate,
eta) = self.PROGRESS_LINE_RE.search(line).groups()
rate = rate.replace("K", "k")
self.progress.pct = float(pct) / 100
self.progress.dled = (self.progress.pct * self.progress.total)
if not "-" in rate:
self.progress.rate = bitmath.parse_string(
rate.split("/")[0]) if rate else None
if eta:
self.progress.eta = eta
except AttributeError:
pass
def Sl_view_quota_disk_changed(self):
raw_quota = self._view.set_quota_disk_widget.dedicated_space.text()
unit = self._view.set_quota_disk_widget.sizes_box.currentText()
new_quota = "%s %s" % (raw_quota, unit)
new_quota_to_byte = bitmath.parse_string(new_quota).to_Byte()
old_quota = "%s %s" % (self._sett_model.get_quota_disco_raw(), "Byte")
old_quota_to_byte = bitmath.parse_string(old_quota).to_Byte()
if old_quota_to_byte != new_quota_to_byte:
self._sett_model.set_quota_disco(new_quota_to_byte)
def set_quota_disco(self, new_quota: bitmath.Byte) -> None:
# TODO: Il controllo non dovremmo farlo nel controller?
folder_size = bitmath.parse_string(self.convert_size(self.get_size()))
free_disk = bitmath.parse_string(
self.convert_size(self.get_free_disk()))
# Controllo che la nuova quota sia minore dello spazio disponibile nell'hdd
# e maggiore dello spazio utilizzato dalla cartella corrente
if folder_size <= new_quota <= free_disk:
settings.update_quota_disco(str(new_quota.value))
self.Sg_model_changed.emit()
async def process_output_line(self, line):
logger.debug(line)
if not line:
return
try:
(dled, elapsed, rate) = self.PROGRESS_RE.search(line).groups()
except AttributeError:
return
self.progress.dled = bitmath.parse_string(dled)
self.progress.rate = bitmath.parse_string(rate.split("/")[0]) if rate else None
def get_information(self):
try:
list_node = self.api_instance.list_node()
items = list_node.items
total_processor_local = 0
total_memory_local = bitmath.Byte(0)
for item in items:
allocatable = item.status.allocatable
self.architecture = str(item.status.node_info.architecture)
total_processor_local = total_processor_local + float(
allocatable['cpu'])
allocatable_memory_str = allocatable['memory'] + "B"
total_memory_local = total_memory_local + bitmath.parse_string(
allocatable_memory_str)
self.total_processor = str(total_processor_local)
self.total_memory = str(
total_memory_local.best_prefix().KiB.format("{value:.0f}"))
list_pod_for_all_namespaces = self.api_instance.list_pod_for_all_namespaces(
)
allocated_processor_local = 0
allocated_memory_local = bitmath.Byte(0)
for item in list_pod_for_all_namespaces.items:
containers = item.spec.containers
for container in containers:
requests = container.resources.requests
if requests is None:
continue
if 'cpu' in requests:
if 'm' in requests['cpu']:
cpu_str = requests['cpu'].replace("m", "")
container_cpu = float(cpu_str) / 1000
else:
cpu_str = requests['cpu']
container_cpu = float(cpu_str)
allocated_processor_local = allocated_processor_local + container_cpu
if 'memory' in requests:
memory_str = requests['memory'] + "B"
allocated_memory_local = allocated_memory_local + bitmath.parse_string(
memory_str)
allocated_processor_local = float(
"{0:.2f}".format(allocated_processor_local))
self.allocated_processor = str(allocated_processor_local)
self.allocated_memory = allocated_memory_local.KiB.format(
"{value:.0f}")
self.allocatable_processor = str(total_processor_local -
allocated_processor_local)
allocatable_memory_local = total_memory_local - allocated_memory_local
self.allocatable_memory = allocatable_memory_local.KiB.format(
"{value:.0f}")
except ApiException as e:
print(
"Exception when calling CoreV1Api->list_resource_quota_for_all_namespaces: %s\n"
% e)
async def process_output_line(self, line):
logger.trace(line)
if not line:
return
if not self.progress.guid:
if not line.startswith("{"):
return
try:
data = json.loads(line)
except json.decoder.JSONDecodeError as e:
logger.warning(e)
return
if "result" not in data:
return
if data["result"] != "success":
logger.warning(data["result"])
return
for status in ["torrent-added", "torrent-duplicate"]:
if status in data["arguments"]:
self.progress.guid = data["arguments"][status]["hashString"]
else:
# FIXME: see below re: workaround for empty status when torrent
# isn't found
self._found = True
if "State:" in line:
self.progress.status = line.split(":")[1].strip().lower()
elif "Location:" in line:
self.progress.dest = line.split(":")[1].strip()
elif "Have:" in line:
try:
verified = re.search("Have: .*\((.*) verified\)", line).groups()[0]
except IndexError:
return
try:
self.progress.dled = bitmath.parse_string(verified)
except ValueError:
return
elif "Total size:" in line:
total = line.split(":")[1].strip().split("(")[0].strip()
self.progress.total = bitmath.parse_string(total)
elif "Percent Done:" in line:
self.progress.pct = line.split(":")[1].strip()
elif "Download Speed:" in line:
rate = line.split(":")[1].strip()
self.progress.rate = bitmath.parse_string(rate.split("/")[0]) if rate else None
elif "ETA:" in line:
self.progress.eta = line.split(":")[1].strip()
else:
return
def Sl_model_changed(self):
"""
Slot collegato ai segnali del model, aggiorna la vista con i nuovi valori
:return: None
"""
# Prendo quota disco con unità e il peso della cartella senza unità (Byte default)
new_max_quota = self._model.get_quota_disco()
_folder_size = self._model.get_size()
# Converto ad oggetto bitmath il peso della cartella e la quota disco
folder_size_parsed = bitmath.parse_string(
self._model.convert_size(_folder_size))
quota_disco_parsed = bitmath.parse_string(new_max_quota)
# Imposto la textbox che mi dice quanto peso ho occupato su quello disponibile
self.disk_quota.setText(
f"{folder_size_parsed} su {new_max_quota} in uso")
free_disk_parsed = bitmath.parse_string(
self._model.convert_size(self._model.get_free_disk()))
# Imposto la textbox che richiede input
if not self.dedicated_space.hasFocus():
self.dedicated_space.setText(str(quota_disco_parsed.value))
# Creo i nuovi valori della combobox
if not self.sizes_box.hasFocus():
self.populate_size_box(folder_size_parsed, free_disk_parsed)
# Imposto l'item in focus della combobox
self.sizes_box.setCurrentText(quota_disco_parsed.unit)
# Prendo dimensione corrente della sync folder e della quota disco
# e metto in proporzione con quotadisco:100=syncfolder:x
_progress_bar_max_value = 100
_tmp = folder_size_parsed.to_Byte().value * _progress_bar_max_value
_progress_bar_current_percentage = _tmp / quota_disco_parsed.to_Byte(
).value
# Inserisco nuovi valori nella progress bar
self.disk_progress.setRange(0, _progress_bar_max_value)
self.disk_progress.setValue(_progress_bar_current_percentage)
# Se la cartella occupa più spazio di quanto voluto allora la porto a quanto occupa
if quota_disco_parsed < folder_size_parsed and not self.dedicated_space.hasFocus(
):
self.dedicated_space.setText(str(folder_size_parsed.value))
self.sizes_box.setCurrentText(folder_size_parsed.unit)
self.Sg_view_changed.emit()
async def update_progress_line(self, line):
if not line:
return
logger.debug(line)
if "[download] Destination:" in line:
self.progress.dest = line.split(":")[1].strip()
return
elif "Invoking downloader" in line:
try:
format = self.FORMATS_RE.search(line).groups()[0]
except AttributeError:
return
video = self.FORMATS[format].video
audio = self.FORMATS[format].audio
if video and audio:
self.progress.status = "downloading 1/1"
elif video:
self.progress.status = "downloading 1/2"
elif audio:
self.progress.status = "downloading 2/2"
elif "Merging formats" in line:
self.progress.status = "muxing"
try:
if isinstance(line, bytes):
logger.debug("mux bytes")
import ipdb
ipdb.set_trace()
self.progress.dest = self.MUXING_RE.search(
line.decode("utf-8")).groups()[0]
else:
logger.debug("mux str")
self.progress.dest = self.MUXING_RE.search(
line).groups()[0]
except AttributeError:
return
else:
try:
(pct, total, rate,
eta) = self.PROGRESS_RE.search(line).groups()
self.progress.pct = float(pct) / 100
self.progress.total = bitmath.parse_string(total)
self.progress.dled = (self.progress.pct * self.progress.total)
self.progress.rate = bitmath.parse_string(
rate.split("/")[0]) if rate else None
self.progress.eta = eta
except AttributeError as e:
return
def test_chunk_size() -> None:
"""Test using --chunk-size."""
def _shell() -> None:
subprocess.check_call(
f"python ./resources/path_collector/path_collector.py {root}"
f" --staging-dir {stage}"
f" --workers 1"
f" --chunk-size {chunk_size}".split(),
cwd=".",
)
def _direct() -> None:
_make_traverse_staging_dir(stage, root)
path_collector.write_all_filepaths_to_files(
stage, root, 1, "", chunk_size, [], None
)
for func in [_direct, _shell]:
logging.warning(f"Using invocation function: {func}")
now = str(time.time())
for kibs in [0] + [10 ** i for i in range(0, 8)]:
print("~ " * 60)
chunk_size = int(bitmath.parse_string(f"{kibs}KiB").to_Byte())
logging.warning(
f"chunk_size => {bitmath.best_prefix(chunk_size).format('{value:.2f} {unit}')} ({chunk_size} bytes)"
)
stage, root = _setup_testfiles(f"{now}-{kibs}KiB")
func()
_assert_out_files(stage, func == _shell)
_assert_out_chunks(stage, chunk_size)
_remove_all(stage, root)
def parse_rate_str(rate_str):
if rate_str.endswith('pps'):
rate_pps = rate_str[:-3]
if not rate_pps:
raise Exception('%s is missing a numeric value' % rate_str)
try:
multiplier = multiplier_map[rate_pps[-1].upper()]
rate_pps = rate_pps[:-1]
except KeyError:
multiplier = 1
rate_pps = int(rate_pps.strip()) * multiplier
if rate_pps <= 0:
raise Exception('%s is out of valid range' % rate_str)
return {'rate_pps': str(rate_pps)}
elif rate_str.endswith('ps'):
rate = rate_str.replace('ps', '').strip()
bit_rate = bitmath.parse_string(rate).bits
if bit_rate <= 0:
raise Exception('%s is out of valid range' % rate_str)
return {'rate_bps': str(int(bit_rate))}
elif rate_str.endswith('%'):
rate_percent = float(rate_str.replace('%', '').strip())
if rate_percent <= 0 or rate_percent > 100.0:
raise Exception('%s is out of valid range (must be 1-100%%)' %
rate_str)
return {'rate_percent': str(rate_percent)}
else:
raise Exception('Unknown rate string format %s' % rate_str)
def test_set_quota_disco(self) -> None:
value = self.sett_model.convert_size(
self.sett_model.get_quota_disco_raw())
value = bitmath.parse_string(value).to_Byte()
self.assertEqual(20971520.0, value)
new_value = self.sett_model.convert_size(self.sett_model.get_size() +
1)
new_value = bitmath.parse_string(new_value)
self.sett_model.set_quota_disco(new_value.to_Byte())
value = self.sett_model.convert_size(
self.sett_model.get_quota_disco_raw())
self.assertEqual(new_value, bitmath.parse_string(value))
value = bitmath.parse_string(self.sett_model.get_quota_disco())
self.assertEqual(new_value, value)
def update_attrs(self, d):
"""Update ConLog instance attributes with new values from
a dictionary.
:param dict d:
Dictionary of user supplied option overrides
"""
if 'level' in d:
self.level = d['level']
if 'log_format' in d:
self.log_format = d['log_format']
if 'debug_format' in d:
self.debug_format = d['debug_format']
if self.level == 'DEBUG':
self.log_format = self.debug_format
if 'log_file' in d:
self.log_file = d['log_file']
if 'max_file_size' in d:
try: # Convert value to bytes
self.max_file_size = int(
bitmath.parse_string(d['max_file_size']).bytes)
except ValueError: # Value is supplied is in bytes
self.max_file_size = int(d['max_file_size'])
if 'max_retention' in d:
self.max_retention = d['max_retention']
def allocate_nodes(self,
nodes: int = 1,
cores: int = 1,
memory_per_node: Union[str, bitmath.Byte] = None,
walltime: Union[str, Walltime] = None,
native_args: Optional[
Dict[str, Optional[
str]]] = None) -> Nodes: # noqa, pylint: disable=bad-whitespace,line-too-long
if memory_per_node is None:
memory_per_node = bitmath.GiB(1)
elif isinstance(memory_per_node, str):
memory_per_node = bitmath.parse_string(memory_per_node)
if walltime is None:
walltime = Walltime(minutes=10)
elif isinstance(walltime, str):
walltime = Walltime.from_string(walltime)
parameters = AllocationParameters(nodes=nodes,
cores=cores,
memory_per_node=memory_per_node,
walltime=walltime,
native_args=native_args)
return allocate_slurm_nodes(parameters=parameters,
config=self._config)
def __init__(
self,
*,
obj_store: Optional[ObjStore] = None,
replacement_policy: Union[str, ReplacementPolicy] = "gdsize",
size: Union[int, str, bitmath.Bitmath] = bitmath.KiB(100),
pickler: Pickler = pickle,
lock: Optional[RWLock] = None,
fine_grain_persistence: bool = False,
fine_grain_eviction: bool = False,
extra_system_state: Callable[[], Any] = Constant(None),
temporary: bool = False,
) -> None:
"""Construct a memoized group. Use with :py:function:Memoized.
:param obj_store: The object store to use for return values.
:param replacement_policy: See policies submodule for options. You can pass an object conforming to the ReplacementPolicy protocol or one of REPLACEMENT_POLICIES.
:param size: The size as an int (in bytes), as a string (e.g. "3 MiB"), or as a `bitmath.Bitmath`_.
:param pickler: A de/serialization to use on the index, conforming to the Pickler protocol.
:param lock: A ReadersWriterLock to achieve exclusion. If the lock is wrong but the obj_store is atomic, then the memoization is still *correct*, but it may not be able to borrow values that another machine computed. Defaults to a FileRWLock.
:param fine_grain_persistence: De/serialize the index at every access. This is useful if you need to update the cache for multiple simultaneous processes, but it compromises performance in the single-process case.
:param fine_grain_eviction: Maintain the cache's size through eviction at every access (rather than just the de/serialization points). This is useful if the caches size would not otherwise fit in memory, but it compromises performance if not needed.
:param extra_system_state: A callable that returns "extra" system state. If the system state changes, the cache is dumped.
:param temporary: Whether the cache should be cleared at the end of the process; This is useful for tests.
.. _`bitmath.Bitmath`: https://pypi.org/project/bitmath/
"""
self._obj_store = (
obj_store
if obj_store is not None
else DirObjStore(path=DEFAULT_OBJ_STORE_PATH)
)
self._replacement_policy = (
REPLACEMENT_POLICIES[replacement_policy.lower()]()
if isinstance(replacement_policy, str)
else replacement_policy
)
self._size = (
size
if isinstance(size, bitmath.Bitmath)
else bitmath.Byte(size)
if isinstance(size, int)
else bitmath.parse_string(size)
)
self._pickler = pickler
self._index_lock = lock if lock is not None else FileRWLock(DEFAULT_LOCK_PATH)
self._fine_grain_persistence = fine_grain_persistence
self._fine_grain_eviction = fine_grain_eviction
self._extra_system_state = extra_system_state
self._index_key = 0
self.time_cost = DefaultDict[str, datetime.timedelta](datetime.timedelta)
self.time_saved = DefaultDict[str, datetime.timedelta](datetime.timedelta)
self.temporary = temporary
self.__setstate__({})
if self.temporary:
atexit.register(self._obj_store.clear)
# atexit handlers are run in the opposite order they are registered.
atexit.register(self._index_write)
def clean(self):
cleaned_data = super(HardDriveForm, self).clean()
capacity = cleaned_data.get('capacity', 0)
if capacity is None:
capacity = 0
unit = cleaned_data.get('unit', '')
obj = bitmath.parse_string('{}{}'.format(capacity, unit))
cleaned_data['capacity'] = obj.to_Byte().value
return cleaned_data
def clean(self):
cleaned_data = super(HardDriveForm, self).clean()
capacity = cleaned_data.get('capacity', 0)
if capacity is None:
capacity = 0
unit = cleaned_data.get('unit', '')
obj = bitmath.parse_string('{}{}'.format(capacity, unit))
cleaned_data['capacity'] = obj.to_Byte().value
return cleaned_data
def __init__(self, config):
"""Create a generator config."""
self.config = config
# name of the generator profile (normally trex or dummy)
# pick the default one if not specified explicitly from cli options
if not config.generator_profile:
config.generator_profile = config.traffic_generator.default_profile
# pick up the profile dict based on the name
gen_config = self.__match_generator_profile(config.traffic_generator,
config.generator_profile)
self.gen_config = gen_config
# copy over fields from the dict
self.tool = gen_config.tool
self.ip = gen_config.ip
self.cores = gen_config.get('cores', 1)
if gen_config.intf_speed:
# interface speed is overriden from config
self.intf_speed = bitmath.parse_string(gen_config.intf_speed.replace('ps', '')).bits
else:
# interface speed is discovered/provided by the traffic generator
self.intf_speed = 0
self.software_mode = gen_config.get('software_mode', False)
self.interfaces = gen_config.interfaces
if self.interfaces[0].port != 0 or self.interfaces[1].port != 1:
raise TrafficClientException('Invalid port order/id in generator_profile.interfaces')
self.service_chain = config.service_chain
self.service_chain_count = config.service_chain_count
self.flow_count = config.flow_count
self.host_name = gen_config.host_name
self.tg_gateway_ip_addrs = gen_config.tg_gateway_ip_addrs
self.ip_addrs = gen_config.ip_addrs
self.ip_addrs_step = gen_config.ip_addrs_step or self.DEFAULT_SRC_DST_IP_STEP
self.tg_gateway_ip_addrs_step = \
gen_config.tg_gateway_ip_addrs_step or self.DEFAULT_IP_STEP
self.gateway_ip_addrs_step = gen_config.gateway_ip_addrs_step or self.DEFAULT_IP_STEP
self.gateway_ips = gen_config.gateway_ip_addrs
self.udp_src_port = gen_config.udp_src_port
self.udp_dst_port = gen_config.udp_dst_port
self.devices = [Device(port, self) for port in [0, 1]]
# This should normally always be [0, 1]
self.ports = [device.port for device in self.devices]
# check that pci is not empty
if not gen_config.interfaces[0].get('pci', None) or \
not gen_config.interfaces[1].get('pci', None):
raise TrafficClientException("configuration interfaces pci fields cannot be empty")
self.pcis = [tgif['pci'] for tgif in gen_config.interfaces]
self.vlan_tagging = config.vlan_tagging
# needed for result/summarizer
config['tg-name'] = gen_config.name
config['tg-tool'] = self.tool
async def process_lines():
async for line in self.get_output():
if not line:
continue
# logger.info(line)
if "[download] Destination:" in line:
self.source.dest = line.split(":")[1].strip()
continue
try:
(pct, total, rate,
eta) = self.PROGRESS_RE.search(line).groups()
self.progress.pct = float(pct) / 100
self.progress.total = bitmath.parse_string(total)
self.progress.dled = (self.progress.pct *
self.progress.total)
self.progress.rate = bitmath.parse_string(
rate.split("/")[0]) if rate else None
self.progress.eta = eta
except AttributeError:
pass
def from_binunit(value, unit='Byte', precision=None, rtype=None):
"""Convert binary unit value to numeric value
:param value: value to convert
:type value: str
:param unit: destination unit ('GiB', 'MiB', 'KiB' or any bitmath unit)
:type unit: str
:param precision: round precision
:type precision: int
:param rtype: return type (default: int if unit is 'Byte' or
precision <= 0, float otherwize)
:type rtype: float, int or any type that can handle float as argument
:returns: converted value
:rtype: float, int or rtype defined
:raises: ValueError, TypeError
:Examples:
>>> from_binunit('1017368Ki')
1041784832
>>> from_binunit('1017368Ki', 'GiB', 2)
0.97
>>> from_binunit('1017368Ki', 'GiB', 0)
1
>>> from_binunit('1017368Ki', 'MiB')
993.5234375
>>> from_binunit('1017368Ki', 'MiB', 0)
994
>>> from_binunit('1017368Ki', 'MiB', rtype=int)
993
>>> from_binunit('1017368Ki', 'MiB', 0, float)
994.0
>>> from_binunit('1017368Ki', 'MiB', rtype=lambda x: float(int(x)))
993.0
"""
if unit.endswith('i'):
unit += 'B'
if isinstance(value, basestring):
if isinstance(value, unicode):
value = str(value)
if value.endswith('i'):
value += 'B'
result = bitmath.parse_string(value)
result = getattr(result, unit).value
else:
result = float(value) / getattr(bitmath, unit)(1).bytes
if precision is not None:
result = round(result, precision)
if rtype is None:
if unit == 'Byte' or (precision is not None and precision <= 0):
rtype = int
else:
rtype = float
return rtype(result)
def parse_att(self, msg, msize, depth=10):
"""Get all attachments in message thread
Args:
msg: gmail msg
msize: A bitmath object with max size
Kwargs:
depth: how deep to look for parts in payload
Returns:
All attachments found in msg, or None
"""
if msize is None:
return [None, None]
parts = msg['payload']
att_fn = None
att_data = None
found = False
try:
i = 0
while not found and i < depth:
i += 1
parts, found = get_next_part(parts,
search='filename',
negation=True,
allowed=u'')
except:
parts = [{'filename': None}]
for part in parts:
if part['filename']:
att_fn = part['filename']
att_id, att_size = part['body'].values()
att_bm = parse_string('{:.9f}B'.format(att_size)).best_prefix()
if att_size < msize.bytes:
att = self.get_att(msg['threadId'], att_id)
body = unicode(att['data']).encode('utf-8')
att_data = body
else:
msg_size = 'NOTE: Att size was %s but limit set to %s'
msize_str = msize.format("{value:.1f} {unit}")
att_str = att_bm.format("{value:.1f} {unit}")
att_data = msg_size % (att_str, msize_str)
att_fn += ' [ATTACHMENT T0O LARGE]'
return [att_fn, att_data]
def deserialize(serialized: dict) -> 'AllocationParameters':
try:
assert serialized['type'] == str(
SerializableTypes.ALLOCATION_PARAMETERS)
return AllocationParameters(
nodes=serialized['nodes'],
cores=serialized['cores'],
memory_per_node=(None if serialized['memory_per_node'] is None
else bitmath.parse_string(
serialized['memory_per_node'])),
walltime=(None if serialized['walltime'] is None else
Walltime.from_string(serialized['walltime'])),
native_args=serialized['native_args'])
except KeyError as e:
raise RuntimeError("Unable to deserialize.") from e
def convert(self, value, param, ctx):
try:
value = bitmath.parse_string(value)
except ValueError:
# no units specified
try:
value = int(value)
value = bitmath.Byte(value)
except ValueError as e:
self.fail(
f"{value} must be specified as bytes or using SI/NIST units. "
+
"For example, the following are all valid and equivalent sizes: '1048576' '1.048576MB', '1 MiB'."
)
return value
def BitmathType(bmstring):
"""An 'argument type' for integrations with the argparse module.
For more information, see
https://docs.python.org/2/library/argparse.html#type Of particular
interest to us is this bit:
``type=`` can take any callable that takes a single string
argument and returns the converted value
I.e., ``type`` can be a function (such as this function) or a class
which implements the ``__call__`` method.
Example usage of the bitmath.BitmathType argparser type:
>>> import bitmath
>>> import argparse
>>> parser = argparse.ArgumentParser()
>>> parser.add_argument("--file-size", type=bitmath.BitmathType)
>>> parser.parse_args("--file-size 1337MiB".split())
Namespace(file_size=MiB(1337.0))
Invalid usage includes any input that the bitmath.parse_string
function already rejects. Additionally, **UNQUOTED** arguments with
spaces in them are rejected (shlex.split used in the following
examples to conserve single quotes in the parse_args call):
>>> parser = argparse.ArgumentParser()
>>> parser.add_argument("--file-size", type=bitmath.BitmathType)
>>> import shlex
>>> # The following is ACCEPTABLE USAGE:
...
>>> parser.parse_args(shlex.split("--file-size '1337 MiB'"))
Namespace(file_size=MiB(1337.0))
>>> # The following is INCORRECT USAGE because the string "1337 MiB" is not quoted!
...
>>> parser.parse_args(shlex.split("--file-size 1337 MiB"))
error: argument --file-size: 1337 can not be parsed into a valid bitmath object
"""
try:
argvalue = bitmath.parse_string(bmstring)
except ValueError:
raise argparse.ArgumentTypeError("'%s' can not be parsed into a valid bitmath object" %
bmstring)
else:
return argvalue
def BitmathType(bmstring):
"""An 'argument type' for integrations with the argparse module.
For more information, see
https://docs.python.org/2/library/argparse.html#type Of particular
interest to us is this bit:
``type=`` can take any callable that takes a single string
argument and returns the converted value
I.e., ``type`` can be a function (such as this function) or a class
which implements the ``__call__`` method.
Example usage of the bitmath.BitmathType argparser type:
>>> import bitmath
>>> import argparse
>>> parser = argparse.ArgumentParser()
>>> parser.add_argument("--file-size", type=bitmath.BitmathType)
>>> parser.parse_args("--file-size 1337MiB".split())
Namespace(file_size=MiB(1337.0))
Invalid usage includes any input that the bitmath.parse_string
function already rejects. Additionally, **UNQUOTED** arguments with
spaces in them are rejected (shlex.split used in the following
examples to conserve single quotes in the parse_args call):
>>> parser = argparse.ArgumentParser()
>>> parser.add_argument("--file-size", type=bitmath.BitmathType)
>>> import shlex
>>> # The following is ACCEPTABLE USAGE:
...
>>> parser.parse_args(shlex.split("--file-size '1337 MiB'"))
Namespace(file_size=MiB(1337.0))
>>> # The following is INCORRECT USAGE because the string "1337 MiB" is not quoted!
...
>>> parser.parse_args(shlex.split("--file-size 1337 MiB"))
error: argument --file-size: 1337 can not be parsed into a valid bitmath object
"""
try:
argvalue = bitmath.parse_string(bmstring)
except ValueError:
raise argparse.ArgumentTypeError(
"'%s' can not be parsed into a valid bitmath object" % bmstring)
else:
return argvalue
def deserialize(config: ClusterConfig, serialized: dict) -> 'NodeImpl':
try:
assert serialized['type'] == str(SerializableTypes.NODE_IMPL)
node = NodeImpl(config=config)
node.make_allocated(
host=serialized['host'],
port=serialized['port'],
cores=serialized['cores'],
memory=(None if serialized['memory'] is None else
bitmath.parse_string(serialized['memory'])),
allocated_until=(None
if serialized['allocated_until'] is None else
utc_from_str(serialized['allocated_until'])))
return node
except KeyError as e:
raise RuntimeError("Unable to deserialize.") from e
def start_traffic_generator(self):
"""Start the traffic generator process (traffic not started yet)."""
self.gen.connect()
# pick up the interface speed if it is not set from config
intf_speeds = self.gen.get_port_speed_gbps()
# convert Gbps unit into bps
tg_if_speed = bitmath.parse_string(str(intf_speeds[0]) + 'Gb').bits
if self.intf_speed:
# interface speed is overriden from config
if self.intf_speed != tg_if_speed:
# Warn the user if the speed in the config is different
LOG.warning('Interface speed provided is different from actual speed (%d Gbps)',
intf_speeds[0])
else:
# interface speed not provisioned by config
self.intf_speed = tg_if_speed
# also update the speed in the tg config
self.generator_config.intf_speed = tg_if_speed
# Save the traffic generator local MAC
for mac, device in zip(self.gen.get_macs(), self.generator_config.devices):
device.set_mac(mac)
def test_parse_Eo(self):
"""parse_string works on exaoctet strings"""
self.assertEqual(
bitmath.parse_string("654 Eo"),
bitmath.EB(654))
def test_parse_Mo(self):
"""parse_string works on megaoctet strings"""
self.assertEqual(
bitmath.parse_string("654 Mo"),
bitmath.MB(654))
def test_parse_Mio(self):
"""parse_string works on mebioctet strings"""
self.assertEqual(
bitmath.parse_string("654 Mio"),
bitmath.MiB(654))
def test_parse_Gb(self):
"""parse_string works on gigabit strings"""
self.assertEqual(
bitmath.parse_string("456Gb"),
bitmath.Gb(456))
def test_parse_bad_unit2(self):
"""parse_string can identify other prefix units"""
with self.assertRaises(ValueError):
bitmath.parse_string("1.23 QB")
def test_parse_string_non_string_input(self):
"""parse_string can identify a non-string input"""
with self.assertRaises(ValueError):
bitmath.parse_string(12345)
def test_parse_no_unit(self):
"""parse_string can identify strings without units at all"""
with self.assertRaises(ValueError):
bitmath.parse_string("12345")
def convert(self, value, param, ctx):
try:
return bitmath.parse_string(value)
except ValueError:
self.fail("'%s' can not be parsed into a valid bitmath object" %
value)
def string_to_byte(stringvar):
encoded = stringvar.encode('ascii', errors='ignore')
for k, v in french_units.iteritems():
encoded = encoded.replace(k, v)
parsed = bitmath.parse_string(encoded)
return int(parsed.to_EB()._byte_value)
def test_parse_bad_float(self):
"""parse_string can identify invalid float values"""
with self.assertRaises(ValueError):
bitmath.parse_string("1.23.45 kb")
def test_parse_bad_unit(self):
"""parse_string can identify invalid prefix units"""
with self.assertRaises(ValueError):
bitmath.parse_string("1.23 GIB")
def test_parse_string_unicode(self):
"""parse_string can handle a unicode string"""
self.assertEqual(
bitmath.parse_string(u"750 GiB"),
bitmath.GiB(750))
def load(self):
bootstrap_timeout = self.env("BOOTSTRAP_TIMEOUT",
default=str(self.BOOTSTRAP_TIMEOUT))
try:
self.BOOTSTRAP_TIMEOUT = int(bootstrap_timeout)
except ValueError:
raise ConfigurationError(
f"Invalid {self._prefix}BOOTSTRAP_TIMEOUT="
f"'{bootstrap_timeout}'. Needs to be a positive integer or 0.")
if self.BOOTSTRAP_TIMEOUT < 0:
raise ConfigurationError(
f"Invalid {self._prefix}BOOTSTRAP_TIMEOUT="
f"'{bootstrap_timeout}'. Needs to be a positive integer or 0.")
if self.BOOTSTRAP_TIMEOUT == 0:
self.BOOTSTRAP_TIMEOUT = None
cloud_provider = self.env("CLOUD_PROVIDER",
default=self.CLOUD_PROVIDER)
if cloud_provider is not None:
try:
self.CLOUD_PROVIDER = CloudProvider(cloud_provider)
except ValueError:
allowed = ", ".join(CloudProvider.__members__.values())
raise ConfigurationError(
f"Invalid {self._prefix}CLOUD_PROVIDER="
f"'{cloud_provider}'. Needs to be of {allowed}.")
self.CLUSTER_BACKUP_IMAGE = self.env("CLUSTER_BACKUP_IMAGE",
default=self.CLUSTER_BACKUP_IMAGE)
debug_volume_size = self.env("DEBUG_VOLUME_SIZE",
default=str(self.DEBUG_VOLUME_SIZE))
try:
self.DEBUG_VOLUME_SIZE = bitmath.parse_string(debug_volume_size)
except ValueError:
raise ConfigurationError(
f"Invalid {self._prefix}DEBUG_VOLUME_SIZE='{debug_volume_size}'."
)
self.DEBUG_VOLUME_STORAGE_CLASS = self.env(
"DEBUG_VOLUME_STORAGE_CLASS",
default=self.DEBUG_VOLUME_STORAGE_CLASS)
secrets = self.env("IMAGE_PULL_SECRETS",
default=self.IMAGE_PULL_SECRETS)
if secrets is not None:
self.IMAGE_PULL_SECRETS = [
s for s in (secret.strip() for secret in secrets.split(","))
if s
]
self.JMX_EXPORTER_VERSION = self.env("JMX_EXPORTER_VERSION")
self.KUBECONFIG = self.env("KUBECONFIG", default=self.KUBECONFIG)
if self.KUBECONFIG is not None:
# When the CRATEDB_OPERATOR_KUBECONFIG env var is set we need to
# ensure that KUBECONFIG env var is set to the same value for
# PyKube login of the Kopf framework to work correctly.
os.environ["KUBECONFIG"] = self.KUBECONFIG
else:
self.KUBECONFIG = os.getenv("KUBECONFIG")
if self.KUBECONFIG is not None:
for path in self.KUBECONFIG.split(ENV_KUBECONFIG_PATH_SEPARATOR):
if not os.path.exists(path):
raise ConfigurationError(
"The KUBECONFIG environment variable contains a path "
f"'{path}' that does not exist.")
self.LOG_LEVEL = self.env("LOG_LEVEL", default=self.LOG_LEVEL)
level = logging.getLevelName(self.LOG_LEVEL)
for logger_name in ("", "crate", "kopf", "kubernetes_asyncio"):
logger = logging.getLogger(logger_name)
logger.setLevel(level)
rolling_restart_timeout = self.env("ROLLING_RESTART_TIMEOUT",
default=str(
self.ROLLING_RESTART_TIMEOUT))
try:
self.ROLLING_RESTART_TIMEOUT = int(rolling_restart_timeout)
except ValueError:
raise ConfigurationError(
f"Invalid {self._prefix}ROLLING_RESTART_TIMEOUT="
f"'{rolling_restart_timeout}'. Needs to be a positive integer or 0."
)
if self.ROLLING_RESTART_TIMEOUT < 0:
raise ConfigurationError(
f"Invalid {self._prefix}ROLLING_RESTART_TIMEOUT="
f"'{rolling_restart_timeout}'. Needs to be a positive integer or 0."
)
scaling_timeout = self.env("SCALING_TIMEOUT",
default=str(self.SCALING_TIMEOUT))
try:
self.SCALING_TIMEOUT = int(scaling_timeout)
except ValueError:
raise ConfigurationError(
f"Invalid {self._prefix}SCALING_TIMEOUT="
f"'{scaling_timeout}'. Needs to be a positive integer or 0.")
if self.SCALING_TIMEOUT < 0:
raise ConfigurationError(
f"Invalid {self._prefix}SCALING_TIMEOUT="
f"'{scaling_timeout}'. Needs to be a positive integer or 0.")
testing = self.env("TESTING", default=str(self.TESTING))
self.TESTING = testing.lower() == "true"
self.WEBHOOK_PASSWORD = self.env("WEBHOOK_PASSWORD",
default=self.WEBHOOK_PASSWORD)
self.WEBHOOK_URL = self.env("WEBHOOK_URL", default=self.WEBHOOK_URL)
self.WEBHOOK_USERNAME = self.env("WEBHOOK_USERNAME",
default=self.WEBHOOK_USERNAME)
self.JOBS_TABLE = self.env("JOBS_TABLE", default=self.JOBS_TABLE)
def test_parse_b(self):
"""parse_string works on bit strings"""
self.assertEqual(
bitmath.parse_string("123b"),
bitmath.Bit(123))
def get_info_compute(self, compute):
name = compute + "-deployment"
api_instance = client.AppsV1Api()
api_response = api_instance.read_namespaced_deployment(
name, namespace="default")
image = api_response.spec.template.spec.containers[0].image
compute_name = api_response.spec.template.spec.containers[0].name
resources = api_response.spec.template.spec.containers[0].resources
if resources.limits is not None:
if 'memory' in resources.limits:
memory_bitmath = bitmath.parse_string(
(resources.limits['memory']) + "B")
memory_reserved = memory_bitmath.KiB.format("{value:.0f}")
else:
memory_reserved = "0"
if 'cpu' in resources.limits:
if 'm' in resources.limits['cpu']:
cpu_str = resources.limits['cpu'].replace("m", "")
cpu_reserved = float(cpu_str) / 1000
else:
cpu_str = resources.limits['cpu']
cpu_reserved = float(cpu_str)
else:
cpu_reserved = "0"
else:
cpu_reserved = "0"
memory_reserved = "0"
node_name = ""
pod_ip = ""
host_ip = ""
status = ""
api_instance = client.CoreV1Api()
pods = api_instance.list_pod_for_all_namespaces()
for pod in pods.items:
if pod.metadata.name.startswith(compute):
status = pod.status.phase
node_name = pod.spec.node_name
pod_ip = pod.status.pod_ip
host_ip = pod.status.host_ip
break
name_service = compute + "-service"
service_response = api_instance.read_namespaced_service(
name_service, namespace="default")
ip_address = service_response.spec.external_i_ps[0]
container_port = service_response.spec.ports[0].target_port
exposed_port = service_response.spec.ports[0].port
service_protocol = service_response.spec.ports[0].protocol
kube_info = KubeGetInfo()
virtual_cpu = VirtualComputeResourceInformationVirtualCPU()
virtual_cpu.cpu_architecture = kube_info.architecture
virtual_cpu.cpu_clock = 0
return_cpu = Decimal(cpu_reserved).quantize(Decimal('1.'),
rounding=ROUND_UP)
virtual_cpu.num_virtual_cpu = str(return_cpu)
virtual_cpu.virtual_cpu_oversubscription_policy = "null"
virtual_cpu.virtual_cpu_pinning_supported = "null"
virtual_memory = VirtualComputeVirtualMemory()
virtual_memory.numa_enabled = False
virtual_memory.virtual_mem_oversubscription_policy = "null"
memory_amount = memory_reserved
if 'i' in memory_amount:
if 'B' in memory_amount:
return_memory_amount = bitmath.parse_string(memory_amount)
else:
return_memory_amount = bitmath.parse_string(memory_amount +
"B")
else:
return_memory_amount = bitmath.parse_string(memory_amount + "KiB")
virtual_memory.virtual_mem_size = return_memory_amount.KiB.format(
"{value:.0f}")
interface = ReservedVirtualComputeVirtualisationContainerReservedVirtualNetworkInterface(
ip_address)
metadata = [{
"key": "container_port",
"value": container_port
}, {
"key": "exposed_port",
"value": exposed_port
}, {
"key": "service_protocol",
"value": service_protocol
}]
interface.metadata = metadata
virtual_compute = VirtualCompute()
virtual_compute.acceleration_capability = []
virtual_compute.compute_id = "1"
virtual_compute.compute_name = compute
virtual_compute.flavour_id = 0
virtual_compute.host_id = node_name
virtual_compute.operational_state = status
virtual_compute.vc_image_id = self.image
virtual_compute.virtual_cpu = virtual_cpu
virtual_compute.virtual_disks = ""
virtual_compute.virtual_memory = virtual_memory
virtual_compute.virtual_network_interface = [interface]
virtual_compute.zone_id = ConfigurationFile(
).config['VIM']['DomainInfo']['Zonelist']['Zone']['zoneId']
return virtual_compute
def test_parse_B(self):
"""parse_string works on byte strings"""
self.assertEqual(
bitmath.parse_string("321B"),
bitmath.Byte(321))
