def generate_adds(self) -> None:
start_ts, days_number = self.start_ts, self.days
with open(os.path.join(self.path, 'additional_contingent.dat'), 'w') as fd:
adds_number = random.randint(13, 27)
for _ in range(adds_number):
adds_type = random.choice(('boost', 'data', 'data', 'data'))
adds_router = random.choice(self.routers)
if adds_router == self.config['test_router']:
self.config['test_router_adds'] = 1 + cast(int, self.config['test_router_adds'])
adds_start_offset = random.randint(int((start_ts - timedelta(days = days_number)).timestamp()),
int((start_ts + timedelta(hours = 13)).timestamp()))
tmp = datetime.fromtimestamp(adds_start_offset)
adds_start = datetime(tmp.year, tmp.month, tmp.day, tmp.hour, tmp.minute)
if adds_type == 'boost':
adds_hours = random.randint(3, 17)
self.adds.append(p.AddsEntry(adds_start, adds_type, adds_router, None, timedelta(hours = adds_hours)))
fd.write('{} boost {} {}h\n'.format(adds_start.strftime('%Y-%m-%d %H:%M'),
adds_router, adds_hours))
elif adds_type == 'data':
adds_host = random.choice(self.hosts)
adds_amount = units2bytes(bytes2units(random.randint(1*MiB, 30*GiB)))
self.adds.append(p.AddsEntry(adds_start, adds_type, adds_router, adds_host, adds_amount))
fd.write('{} data {} {} {}\n'.format(adds_start.strftime('%Y-%m-%d %H:%M'),
adds_router, adds_host,
bytes2units(adds_amount)))
def hosts_usage(self) -> None:
print('<div class="midblock"><h1>HOSTS</h1><table><tr><th> </th>',
file=self._output_stream)
for limit in self._limit_names:
print('<th class="tright">%s</th>' % limit,
end=' ',
file=self._output_stream)
print("</tr>", file=self._output_stream)
for account in sorted(self._accounts, key=lambda x: x.short):
for host in sorted(account.hosts, key=lambda x: x.name):
print('<tr class="tright"><th class="tright user%s" title="rest %s">%s</th>' \
% (account.short, bytes2units(self._rest_adds.get(host.name, 0)),
"%s/%s" % (host.name, account.short)),
end = ' ', file = self._output_stream)
for limit_name in self._limit_names:
if limit_name not in self._host_usage \
or host not in self._host_usage[limit_name]:
usage = p.Usage(account.short)
else:
usage = self._host_usage[limit_name][host]
print('<td class="tright" title="%s / %s / %s">%s</td>' % \
(bytes2units(usage.inp), bytes2units(usage.out), bytes2units(usage.dat),
bytes2units(usage.inp + usage.out, ' ')),
file = self._output_stream)
print("</tr>", file=self._output_stream)
print("</table></div>", file=self._output_stream)
def check_free_space(self):
"""
Checks if the sum of the size of all sparse files is bigger
than the free space on the associated device.
"""
devices = set()
mount_devices = odict()
try:
nodes_conf = self.exaconf.get_nodes_conf()
docker_conf = self.exaconf.get_docker_conf()
except EXAConf.EXAConfError as e:
raise DeviceError("Unable to read EXAConf: %s" % e)
if docker_conf.device_type != "file":
raise DeviceError(
"Space-check is only supported for file-devices!")
# extract all file-devices from the given EXAConf
for node_id in nodes_conf.keys():
my_conf = nodes_conf[node_id]
# add mapped devices (they have absolute paths)
for disk in my_conf.disks.itervalues():
if disk.has_key("mapped_devices"):
for host_path, c in disk.mapped_devices:
devices.add(host_path)
# add "normal" file-devices
for disk in my_conf.disks.itervalues():
for dev, meta in disk.devices:
if not self.is_mapped_device(dev, disk):
devices.add(
os.path.join(
os.path.join(my_conf.docker_volume,
self.exaconf.storage_dir), dev))
devices.add(
os.path.join(
os.path.join(my_conf.docker_volume,
self.exaconf.storage_dir), meta))
# organize devices according to their filesystem
for dev in devices:
mount_point = self.get_mount_point(dev)
if mount_point not in mount_devices:
mount_devices[mount_point] = []
mount_devices[mount_point].append(dev)
# compute free space for each mountpoint
sufficient_free_space = True
for mount_point in mount_devices.keys():
part_free = self.get_free_space(mount_point)
files_size = sum([
os.path.getsize(os.path.realpath(dev))
for dev in mount_devices[mount_point]
])
if part_free < files_size:
print "Free space on '%s' is only %s, but accumulated size of (sparse) file-devices is %s!" % (
mount_point, bytes2units(part_free),
bytes2units(files_size))
sufficient_free_space = False
return sufficient_free_space
def auto_create_file_devices(self, container_internal=False):
"""
Automatically determines the available free space in the root directory of the current
cluster and creates one file device per node with disk name 'default'. 'container_internal'
has to be True if this function is called from within a container (e. g. during the
initialization of a self-contained image).
Throws an exception if the cluster already contains disks and devices.
"""
# Get and check available free space in root directory
root_usable = 0
root_free = self.get_free_space(self.get_mount_point(
self.exaconf.root))
if container_internal == True:
root_usable = min(root_free - self.auto_internal_reserved_size,
self.max_auto_internal_used_space)
else:
if root_free < self.min_auto_free_space:
raise DeviceError(
"Free space on '%s' is only '%s' but '%s' are required for automatic file-device creation!"
% (self.exaconf.root, bytes2units(root_free),
bytes2units(self.min_auto_free_space)))
root_usable = min(root_free - self.auto_reserved_size,
self.max_auto_used_space)
try:
nodes_conf = self.exaconf.get_nodes_conf()
except EXAConf.EXAConfError as e:
raise DeviceError("Failed to read EXAConf: %s" % e)
# check if the nodes already have disks
for node in nodes_conf.values():
if len(node.disks) > 0:
raise DeviceError(
"Devices can't be auto-generated because this cluster alreay has disks!"
)
bytes_per_node = root_usable / len(nodes_conf)
# create the device-file in the local storage directory
if container_internal == True:
self.create_node_file_devices(
"11", "default", 1, bytes_per_node,
os.path.join(self.exaconf.container_root,
self.exaconf.storage_dir), False)
else:
self.create_file_devices("default", 1, bytes_per_node, "", False)
try:
# leave some room for the temporary volume!
self.exaconf.use_disk_for_volumes(
"default",
bytes_per_node * 0.666,
min_vol_size=self.auto_min_vol_size,
vol_resize_step=self.vol_resize_step)
except EXAConf.EXAConfError as e:
raise DeviceError(
"Failed to use new disk for the existing volumes: %s" % e)
def test_61_reports(self) -> None:
if not self.adds_applied:
self.adds.apply_to_storage(self.stor)
self.adds_applied = True
limits_names_set = set()
for acc in self.accounts.values():
for lname in acc.limit.limit_names:
limits_names_set.add((acc.limit.period(lname), lname))
limit_names = tuple(x[1] for x in sorted(limits_names_set, key = lambda x: x[0]))
reports = AccountsReport(limit_names, tuple(self.accounts.values()), self.stor)
print()
for limit in limit_names:
result = reports.account_usage(self.start_ts, self.config['test_router'], limit)
print(limit)
for acc, usage in sorted(result.items(), key = lambda x: x[0].name):
print(acc.name, bytes2units(usage.dat), acc.limit.limit(limit).amount_text)
def limits_usage(self) -> None:
print('<div class="smallblock"><h2>limits</h2><table>',
file=self._output_stream)
sum_usage: Dict[str, p.Usage] = {}
sum_limits: Dict[str, p.Usage] = {}
for limit in self._limit_names:
sum_usage[limit] = p.Usage(limit)
sum_limits[limit] = p.Usage(limit)
for account in sorted(self._accounts, key=lambda x: x.short):
print('<tr class="tright"><th class="tright user%s" title="%s">%s</th>' \
% (account.short, account.name, account.short),
end = ' ', file = self._output_stream)
for limit in self._limit_names:
if limit not in self._account_usage \
or account not in self._account_usage[limit]:
usage = p.Usage(account.short)
else:
usage = self._account_usage[limit][account]
print('<td class="tright" title="%s / %s / %s / %s">%s</td>' \
% (bytes2units(usage.inp), bytes2units(usage.out), bytes2units(usage.dat),
bytes2units(usage.inp + usage.out, ' '),
account.limit(limit).amount_html),
end = ' ', file = self._output_stream)
if not account.ignore:
sum_usage[limit] += usage
sum_limits[limit] += p.Usage('tmp',
dat=account.limit(limit).amount)
print("</tr>", file=self._output_stream)
print('<tr class="tright"><th class="tright"> </th>',
end=' ',
file=self._output_stream)
for limit in self._limit_names:
usage = sum_limits[limit]
real_usage = sum_usage[limit]
print('<th class="tright" title="%s / %s / %s">%s</th>' \
% (bytes2units(real_usage.inp + real_usage.out),
bytes2units((real_usage.inp + real_usage.out) - real_usage.dat),
bytes2units(real_usage.dat),
bytes2units(usage.dat, ' ')),
end = ' ', file = self._output_stream)
print("</tr></table></div>", file=self._output_stream)
def use_disk_for_volumes(self, disk, bytes_per_node, vol_type=None):
"""
Adds the given disk to all volumes of the given type that don't have a disk assigned yet.
The given 'bytes_per_node' space is distributed equally across all suitable volumes.
"""
# we only consider volumes without disks
filters = {"disk": ""}
if vol_type and vol_type != "":
filters["type"] = vol_type
volumes = self.get_storage_volumes(filters=filters)
bytes_per_volume_node = bytes_per_node / len(volumes)
for volume in volumes.iteritems():
vol_sec = self.config["EXAVolume : " + volume[0]]
vol_sec["Disk"] = disk
vol_sec["Size"] = bytes2units(bytes_per_volume_node /
volume[1].redundancy)
self.commit()
def _print_limit_block(self, limit: str) -> None:
print("<table>", file=self._output_stream)
dat_sum, dat_full, dat_limit = 0, 0, 0
for account in sorted(self._accounts, key=lambda x: x.short):
if limit not in self._account_usage or account not in self._account_usage[
limit]:
usage = p.Usage(account.short)
else:
usage = self._account_usage[limit][account]
print('<tr><th class="tright user%s" title="%s">%s</th><td class="tright" title="%s / %s / %s">%s</td>' \
% (account.short, account.name, account.short,
bytes2units(usage.inp), bytes2units(usage.out), bytes2units(usage.dat),
bytes2units(usage.inp + usage.out, ' ')),
file = self._output_stream)
percent = usage.dat / account.limit(limit).amount * 100
if percent > 99. and usage.dat < account.limit(limit).amount:
percent = 99.
percent_real = (usage.inp +
usage.out) / account.limit(limit).amount * 100
red, yellow, weight = 0, 0, 'normal'
if percent > 50.:
red = yellow = min(int(percent / 100 * 150 + 50), 255)
if percent > 80.: red = min(int(percent / 100 * 150 + 105), 255)
if percent > 99.:
red, yellow = min(int(((percent - 40) / 100) * 255), 255), 0
if percent > 120.: weight = 'bold'
print('<td class="tright" style="color: rgb(%d, %d, 0); font-weight: %s;" title="%s">%s</td></tr>' \
% (red, yellow, weight, "%2.0f%%" % percent_real, "%2.0f%%" % percent),
file = self._output_stream)
if not account.ignore:
dat_sum += usage.dat
dat_full += usage.inp + usage.out
dat_limit += account.limit(limit).amount
percent = dat_sum / dat_limit * 100
percent_real = dat_full / dat_limit * 100
print('<tr><th class="tright">%s</th><th class="tright" title="%s">%s</th><th title="%s">%s</th></tr></table>' \
% (" ", bytes2units(dat_full), bytes2units(dat_sum), "%2.0f%%" % percent_real, "%2.0f%%" % percent),
file = self._output_stream)
def amount_html(self) -> str:
return bytes2units(self._amount, ' ')
def amount_text(self) -> str:
return bytes2units(self._amount, ' ')