def fixture_perun_test_group() -> Group:
# these are real objects inside Perun so do not change them, otherwise all perun
# tests will fail
group_id = 11482
group_name = "os_credits_test"
resource_id = 8676
# resource_name = "test"
group = Group(group_name, resource_id)
# set the group_id already
group.id = group_id
return group
async def process_influx_line(
influx_line: str, app: Application, group_locks: Dict[str, Lock]
) -> None:
"""Performs all preliminary task before actually billing a Group/Project.
#. Determine whether the passed item/str/Influx Line is billable/needed
#. Deserialize it into a :class:`~os_credits.models.UsageMeasurement` by calling
:func:`~os_credits.credits.models.measurement_by_name`, see :ref:`Metrics and
Measurements`.
#. Create a :class:`~os_credits.perun.group.Group` object, see :ref:`Perun`.
#. If a project whitelist is set in :ref:`Settings`, see whether the group is part
of it
#. Calls :func:`update_credits` once the correct :ref:`lock <Group Locks>` could be
acquired. Catch every notification, see :ref:`Notifications`, and send it.
:param influx_line: String/Influx Line to process
:param app: Application object holding our helper class instances
:param group_locks: Dictionary with :ref:`Group Locks`
"""
task_logger.debug("Processing Influx Line `%s`", influx_line)
# we want to end this task as quickly as possible if the InfluxDB Point is not
# needed
try:
measurement_class = measurement_by_name(influx_line)
except ValueError:
task_logger.debug("Ignoring since the measurement is not needed/billable")
return
try:
measurement = measurement_class.from_lineprotocol(influx_line)
except (KeyError, ValueError):
task_logger.exception(
"Could not convert influx line %s to UsageMeasurement. Appending "
"stacktrace",
influx_line,
)
return
perun_group = Group(measurement.project_name, measurement.location_id)
if (
config["OS_CREDITS_PROJECT_WHITELIST"] is not None
and perun_group.name not in config["OS_CREDITS_PROJECT_WHITELIST"]
):
task_logger.info(
"Group `%s` is not part of given whitelist (%s). Ignoring measurement",
perun_group.name,
config["OS_CREDITS_PROJECT_WHITELIST"],
)
return
task_logger.info(
"Processing UsageMeasurement `%s` - Group `%s`", measurement, perun_group
)
task_logger.debug("Awaiting async lock for Group %s", perun_group.name)
try:
# since group_locks is a defaultdict a new Lock is automatically created if
# necessary
async with group_locks[perun_group.name]:
task_logger.debug("Acquired async lock for Group %s", perun_group.name)
await update_credits(perun_group, measurement, app)
except EmailNotificationBase as notification:
task_logger.info("Sending notification %s", notification)
await send_notification(notification)
def notification_group():
from os_credits.perun.attributes import (
DenbiCreditsGranted,
DenbiCreditsUsed,
ToEmail,
)
from os_credits.perun.group import Group
from .conftest import TEST_INITIAL_CREDITS_GRANTED
test_group = Group("TestGroup")
test_group.email = ToEmail(value=["admin@project"])
test_group.credits_used = DenbiCreditsUsed(value=str(TEST_INITIAL_CREDITS_GRANTED))
test_group.credits_granted = DenbiCreditsGranted(
value=str(TEST_INITIAL_CREDITS_GRANTED)
)
return test_group
async def test_not_associated_error(perun_test_group: Group):
assert await perun_test_group.is_assigned_resource()
# OPENSTACK_RESOURCE
not_associated_but_existing_resource_id = 8675
perun_test_group.resource_id = not_associated_but_existing_resource_id
assert not await perun_test_group.is_assigned_resource()
async def test_group_resource_not_associated(perun_test_group: Group):
# name of Perun group without association
perun_test_group.name = f"{perun_test_group.name}_no_resource"
# attr should not be set
with pytest.raises(GroupResourceNotAssociatedError):
await perun_test_group.connect()
async def test_credits_granted_missing(perun_test_group: Group):
# name of Perun group without any value
perun_test_group.name = f"{perun_test_group.name}_credits_granted"
# attr should not be set
with pytest.raises(DenbiCreditsGrantedMissing):
await perun_test_group.connect()
async def delete_mb_and_vcpu_since(request: web.Request) -> web.Response:
internal_logger.info(f"Called: {request.rel_url}")
await stop_worker(request.app, 500)
try:
influx_client: InfluxDBClient = request.app["influx_client"]
since_date = request.query["since_date"]
datetime_format = "%Y-%m-%d %H:%M:%S"
since_date = datetime.strptime(since_date, datetime_format)
project_names = request.query["project_names"]
project_names = project_names.split(",")
since_date = int(since_date.timestamp())
except KeyError as e:
internal_logger.exception(
f"Exception when getting request information for "
f"deleting value history:\n"
f"{e}")
await create_worker(request.app)
return web.HTTPException(text="Key Error.")
except ValueError as e:
internal_logger.exception(
f"Exception when getting request information for "
f"deleting value history:\n"
f"{e}")
await create_worker(request.app)
return web.HTTPException(text="Value Error.")
except Exception as e:
internal_logger.exception(
f"Exception when getting request information for "
f"deleting value history:\n"
f"{e}")
await create_worker(request.app)
return web.HTTPException(text="Exception.")
return_list = []
internal_logger.info(
f"Trying to delete usage values for project: {project_names} "
f"since {since_date}.")
for project_name in project_names:
try:
last_timestamps = await influx_client.delete_mb_and_vcpu_measurements(
project_name, since_date)
if "project_name" not in last_timestamps \
or "location_id" not in last_timestamps:
internal_logger.info(f"Could not find group {project_name} in "
f"influxdb!")
return_list.append({
"error":
f"Could not find '{project_name}' "
f"in influxdb."
})
continue
perun_group = Group(last_timestamps["project_name"],
int(last_timestamps["location_id"]))
await perun_group.connect()
last_mb = last_timestamps.get("last_mb", None)
if last_mb:
perun_group.credits_timestamps.value[
"project_mb_usage"] = datetime.fromtimestamp(
last_timestamps["last_mb"]["time"])
else:
perun_group.credits_timestamps.value[
"project_mb_usage"] = datetime.now()
last_vcpu = last_timestamps.get("last_vcpu", None)
if last_vcpu:
perun_group.credits_timestamps.value[
"project_vcpu_usage"] = datetime.fromtimestamp(
last_timestamps["last_vcpu"]["time"])
else:
perun_group.credits_timestamps.value[
"project_vcpu_usage"] = datetime.now()
await perun_group.save()
internal_logger.info(f"Deleted values and set timestamps for "
f"'{project_name}' with {last_timestamps}.")
return_list.append(last_timestamps)
except GroupNotExistsError as e:
internal_logger.warning(
"Could not resolve group with name `%s` against perun. %r",
project_name, e)
return_list.append(
{"error": f"Could not find perun group "
f"'{project_name}'."})
continue
except Exception as e:
internal_logger.exception(
f"Exception when deleting value history:\n"
f"{e}")
return_list.append({
"error":
f"Could not delete values for "
f"'{project_name}'."
})
continue
await create_worker(request.app)
return web.json_response(return_list)