def ingest(self):
"""Ingest jobs."""
messages = [message for message in request.get_json(force=True) if message["data"]["state"] == "exited"]
for message in messages:
data = message["data"]
queue = get_or_create(Queue, name=data["queue"])
owner = get_or_create(Owner, name=data["owner"])
job = get_or_create(Job, job_id=data["jobid"])
job.name = data["jobname"]
job.queue = queue
job.owner = owner
job.start = data["start"]
job.end = data["end"]
total_cores = 0
for hostname, slots in data["exec_host"].items():
host = get_or_create(Host, name=hostname)
get_or_create(Allocation, job=job, host=host, cores=len(slots))
total_cores += len(slots)
job.cores = total_cores
job.cpu_seconds = total_cores * (data["end"] - data["start"])
commit()
return "", 204
def ingest(self):
@lru_cache(maxsize=100000)
def cache(model, **kwargs):
return get_or_create(model, **kwargs)
for message in request.get_json(force=True):
data = message["data"]
snapshot = cache(Snapshot, ts=data["timestamp"])
for key, value in data.items():
# Ugly hack until swift data pushed down into own dict.
valid = [c in string.hexdigits for c in key]
if not reduce(lambda x, y: x and y, valid):
continue
account = cache(Account, openstack_id=key)
add(Usage(bytes=value["bytes"],
containers=value["containers"],
objects=value["objects"],
quota=value["quota"],
account=account,
snapshot=snapshot))
commit()
return "", 204
def ingest(self):
"""Ingest jobs."""
messages = [
message for message in request.get_json(force=True)
if message["data"].get("state") == "exited"
]
for message in messages:
data = message["data"]
queue = get_or_create(Queue, name=data["queue"])
owner = get_or_create(Owner, name=data["owner"])
job = get_or_create(Job, job_id=data["jobid"])
job.name = data["jobname"]
job.queue = queue
job.owner = owner
job.start = data["start"]
job.end = data["end"]
total_cores = 0
for hostname, slots in data["exec_host"].items():
host = get_or_create(Host, name=hostname)
get_or_create(Allocation, job=job, host=host, cores=len(slots))
total_cores += len(slots)
job.cores = total_cores
job.cpu_seconds = total_cores * (data["end"] - data["start"])
commit()
return "", 204
def put(self):
try:
for item in request.get_json(force=True):
self.ingest(item)
commit()
return "", 204
except Exception as e:
rollback()
return str(e), 400
def put(self):
try:
for item in request.get_json(force=True):
self.ingest(item)
commit()
return "", 204
except Exception as e:
rollback()
return str(e), 400
def ingest(self):
"""Ingest usage."""
@lru_cache(maxsize=1000)
def cache(model, **kwargs):
return get_or_create(model, **kwargs)
for message in request.get_json(force=True):
if not message["schema"] == "hnas.filesystems":
continue
data = message["data"]
snapshot = cache(Snapshot, ts=data["timestamp"])
for name, details in data["filesystems"].items():
fs = cache(Filesystem, name=name)
fs_usage = {
"filesystem": fs,
"snapshot": snapshot,
"capacity": details["capacity"],
"free": details["free"],
"live_usage": details["live-fs-used"],
"snapshot_usage": details["snapshot-used"]
}
add(FilesystemUsage(**fs_usage))
if "virtual_volumes" in details:
for vusage in details["virtual_volumes"]:
name = vusage["volume-name"]
if name.startswith("/"):
name = name[1:]
vivol = cache(VirtualVolume,
name=name,
filesystem=fs)
vivol_usage = {
"snapshot": snapshot,
"virtual_volume": vivol,
"files": vusage["file-count"],
"usage": vusage["usage"],
"quota": vusage["usage-limit"]
}
if len(vusage["user-group-account"]) > 0:
owner = cache(Owner,
name=vusage["user-group-account"])
vivol_usage["owner"] = owner
add(VirtualVolumeUsage(**vivol_usage))
commit()
return "", 204
def ingest(self):
"""Ingest data."""
@lru_cache(maxsize=10000)
def cache(model, **kwargs):
return get_or_create(model, **kwargs)
for message in request.get_json(force=True):
data = message["data"]
snapshot = cache(Snapshot, ts=data["timestamp"])
for account_detail in data["users"]:
account = cache(Account, openstack_id=account_detail["id"])
if not account_detail["email"]:
continue
# Fix broken emails containing ";"
email = account_detail["email"].split(";")[0]
domain_name = get_domain(email)
domain = cache(Domain,
name=domain_name) if domain_name else None
reference = cache(AccountReference, value=email, domain=domain)
cache(AccountReferenceMapping,
account=account,
reference=reference,
snapshot=snapshot)
for tenant_detail in data["tenants"]:
tenant = cache(Tenant, openstack_id=tenant_detail["id"])
tenant.name = tenant_detail["name"]
tenant.description = tenant_detail["description"]
if "allocation_id" in tenant_detail:
try:
tenant.allocation = int(tenant_detail["allocation_id"])
except:
pass
if "users" not in tenant_detail:
continue
for member in tenant_detail["users"]:
account = cache(Account, openstack_id=member["id"])
cache(Membership,
account=account,
tenant=tenant,
snapshot=snapshot)
commit()
return "", 204
def ingest(self):
"""Ingest usage."""
@lru_cache(maxsize=10000)
def cache(model, **kwargs):
return get_or_create(model, **kwargs)
# This one is experimentally optimised for performance.
# Internally creates a TSV and copies it straight
# into the database.
# Probably not necessary though.
tsv = io.StringIO()
for ingest_pass in [1, 2]:
for message in request.get_json(force=True):
if message["schema"] != "xfs.quota.report":
continue
data = message["data"]
host = cache(Host, name=data["hostname"])
snapshot = cache(Snapshot,
ts=data["timestamp"],
host=host,
message=message["id"])
for entry in data["filesystems"]:
filesystem = cache(Filesystem,
name=entry["filesystem"],
host=host)
for record in entry["quota"]:
owner = cache(Owner, name=record["username"])
if ingest_pass == 2:
columns = [
uuid.uuid4(), record["soft"], record["hard"],
record["used"], owner.id, snapshot.id,
filesystem.id
]
tsv.write("\t".join([str(c) for c in columns]) +
"\n")
if ingest_pass == 2:
tsv.seek(0)
cursor = db.session.connection().connection.cursor()
cursor.copy_from(tsv, "usage")
commit()
return "", 204
def ingest(self):
"""Ingest data."""
@lru_cache(maxsize=10000)
def cache(model, **kwargs):
return get_or_create(model, **kwargs)
for message in request.get_json(force=True):
data = message["data"]
snapshot = cache(Snapshot, ts=data["timestamp"])
for account_detail in data["users"]:
account = cache(Account, openstack_id=account_detail["id"])
if not account_detail["email"]:
continue
# Fix broken emails containing ";"
email = account_detail["email"].split(";")[0]
domain_name = get_domain(email)
domain = cache(Domain,
name=domain_name) if domain_name else None
reference = cache(AccountReference, value=email, domain=domain)
cache(AccountReferenceMapping,
account=account,
reference=reference,
snapshot=snapshot)
for tenant_detail in data["tenants"]:
tenant = cache(Tenant, openstack_id=tenant_detail["id"])
tenant.name = tenant_detail["name"]
tenant.description = tenant_detail["description"]
if "allocation_id" in tenant_detail:
try:
tenant.allocation = int(tenant_detail["allocation_id"])
except:
pass
if "users" not in tenant_detail:
continue
for member in tenant_detail["users"]:
account = cache(Account, openstack_id=member["id"])
cache(Membership,
account=account,
tenant=tenant,
snapshot=snapshot)
commit()
return "", 204
def ingest(self):
"""Data ingest"""
@lru_cache(maxsize=100000)
def cache(model, **kwargs):
return get_or_create(model, **kwargs)
for message in request.get_json(force=True):
data = message["data"]
snapshot = cache(Snapshot, ts=data["timestamp"])
if "volumes" in data:
for metadata in data["volumes"]:
az = None
if "availability_zone" in metadata:
az = cache(AvailabilityZone,
name=metadata["availability_zone"])
volume = cache(
Volume,
openstack_id=metadata["id"],
availability_zone=az,
owner=metadata["user_id"],
tenant=metadata["os-vol-tenant-attr:tenant_id"])
status = cache(VolumeStatus, name=metadata["status"])
cache(VolumeState,
name=metadata["name"],
size=metadata["size"],
status=status,
snapshot=snapshot,
volume=volume)
for instance in metadata["attachments"]:
cache(VolumeAttachment,
instance=instance["server_id"],
volume=volume,
snapshot=snapshot)
if "volume_snapshots" in data:
for metadata in data["volume_snapshots"]:
cache(VolumeSnapshot,
openstack_id=metadata["id"],
name=metadata["name"],
description=metadata["description"],
size=metadata["size"],
source=metadata["volume_id"])
commit()
return "", 204
def ingest(self):
"""Ingest snapshots."""
for message in request.get_json(force=True):
data = message["data"]
snapshot = Snapshot(ts=data["timestamp"])
add(snapshot)
for entry in data["organisations"]:
organisation = get_or_create(Organisation,
insightly_id=entry["id"])
organisation.name = entry["name"]
for entry in data["contacts"]:
person = get_or_create(Person, insightly_id=entry["id"])
person.first_name = entry["first_name"]
person.last_name = entry["last_name"]
if entry["username"]:
username = get_or_create(Username,
username=entry["username"])
get_or_create(PersonUsername,
snapshot=snapshot,
person=person,
username=username)
if entry["email"]:
for address in entry["email"]:
email = get_or_create(Email, address=address)
get_or_create(PersonEmail,
snapshot=snapshot,
person=person,
email=email)
if entry["organisations"]:
for insightly_id in entry["organisations"]:
organisation = get_or_create(Organisation,
insightly_id=insightly_id)
get_or_create(Membership,
snapshot=snapshot,
organisation=organisation,
person=person)
commit()
return "", 204
def ingest(self):
"""Ingest snapshots."""
for message in request.get_json(force=True):
data = message["data"]
snapshot = Snapshot(ts=data["timestamp"])
add(snapshot)
for entry in data["organisations"]:
organisation = get_or_create(Organisation,
insightly_id=entry["id"])
organisation.name = entry["name"]
for entry in data["contacts"]:
person = get_or_create(Person, insightly_id=entry["id"])
person.first_name = entry["first_name"]
person.last_name = entry["last_name"]
if entry["username"]:
username = get_or_create(Username,
username=entry["username"])
get_or_create(PersonUsername,
snapshot=snapshot,
person=person,
username=username)
if entry["email"]:
for address in entry["email"]:
email = get_or_create(Email, address=address)
get_or_create(PersonEmail,
snapshot=snapshot,
person=person,
email=email)
if entry["organisations"]:
for insightly_id in entry["organisations"]:
organisation = get_or_create(Organisation,
insightly_id=insightly_id)
get_or_create(Membership,
snapshot=snapshot,
organisation=organisation,
person=person)
commit()
return "", 204
def ingest(self):
"""Ingest usage."""
for message in request.get_json(force=True):
inserts = []
data = message["data"]
host = get_or_create(Host, name=data["hostname"])
metadata = data["fs"]
filesystem = get_or_create(Filesystem,
name=metadata["name"],
host=host)
snapshot = get_or_create(Snapshot,
ts=data["timestamp"],
filesystem=filesystem,
bavail=metadata["bavail"],
bfree=metadata["bfree"],
blocks=metadata["blocks"],
bsize=metadata["bsize"],
favail=metadata["favail"],
ffree=metadata["ffree"],
files=metadata["files"],
frsize=metadata["frsize"])
for who, details in data["usage"].items():
who = who.split("/")
owner = get_or_create(Owner, name=who[0])
project = get_or_create(Project, name=who[1])
add(
Usage(owner=owner,
project=project,
snapshot=snapshot,
blocks=details["blocks"],
bytes=details["bytes"],
files=details["files"]))
commit()
return "", 204
def ingest(self):
"""Ingest usage."""
for message in request.get_json(force=True):
inserts = []
data = message["data"]
host = get_or_create(Host, name=data["hostname"])
metadata = data["fs"]
filesystem = get_or_create(Filesystem,
name=metadata["name"],
host=host)
snapshot = get_or_create(Snapshot,
ts=data["timestamp"],
filesystem=filesystem,
bavail=metadata["bavail"],
bfree=metadata["bfree"],
blocks=metadata["blocks"],
bsize=metadata["bsize"],
favail=metadata["favail"],
ffree=metadata["ffree"],
files=metadata["files"],
frsize=metadata["frsize"])
for who, details in data["usage"].items():
who = who.split("/")
owner = get_or_create(Owner, name=who[0])
project = get_or_create(Project, name=who[1])
add(Usage(owner=owner,
project=project,
snapshot=snapshot,
blocks=details["blocks"],
bytes=details["bytes"],
files=details["files"]))
commit()
return "", 204
def ingest(self):
"""Ingest data."""
@lru_cache(maxsize=100000)
def cache(model, **kwargs):
return get_or_create(model, **kwargs)
for message in request.get_json(force=True):
data = message["data"]
snapshot = cache(Snapshot, ts=data["timestamp"])
for flavor_detail in data["flavors"]:
flavor = cache(Flavor, openstack_id=flavor_detail["id"])
flavor.name = flavor_detail["name"]
flavor.vcpus = flavor_detail["vcpus"]
flavor.ram = flavor_detail["ram"]
flavor.disk = flavor_detail["disk"]
flavor.ephemeral = flavor_detail["OS-FLV-EXT-DATA:ephemeral"]
flavor.public = flavor_detail["os-flavor-access:is_public"]
for instance_detail in data["instances"]:
availability_zone_name = instance_detail[
"OS-EXT-AZ:availability_zone"]
if not availability_zone_name:
continue
availability_zone = cache(AvailabilityZone,
name=availability_zone_name)
if not availability_zone_name.startswith('sa'):
logger.debug("Skip non-sa zone: %s" % availability_zone_name)
continue
hypervisor_hostname = instance_detail[
"OS-EXT-SRV-ATTR:hypervisor_hostname"]
if not hypervisor_hostname:
continue
hypervisor = cache(Hypervisor,
name=hypervisor_hostname,
availability_zone=availability_zone)
flavor = cache(Flavor,
openstack_id=instance_detail["flavor"]["id"])
account = cache(Account,
openstack_id=instance_detail["user_id"])
tenant = cache(Tenant,
openstack_id=instance_detail["tenant_id"])
status = cache(InstanceStatus,
name=instance_detail["OS-EXT-STS:vm_state"])
if not isinstance(instance_detail["image"], dict):
continue
image = cache(Image,
openstack_id=instance_detail["image"]["id"])
instance = cache(Instance, openstack_id=instance_detail["id"])
instance.account = account
instance.tenant = tenant
instance.flavor = flavor
instance.availability_zone = availability_zone
add(InstanceState(snapshot=snapshot,
instance=instance,
image=image,
name=instance_detail["name"],
hypervisor=hypervisor,
status=status))
for network in instance_detail["addresses"].values():
for address in network:
mac = cache(MACAddress,
address=address["OS-EXT-IPS-MAC:mac_addr"])
add(MACAddressMapping(snapshot=snapshot,
instance=instance,
address=mac))
ip = cache(IPAddress,
address=address["addr"],
family=address["version"])
add(IPAddressMapping(snapshot=snapshot,
instance=instance,
address=ip))
commit()
return "", 204
def ingest(self):
"""Ingest usage."""
timestamps = set()
@lru_cache(maxsize=10000)
def cache(model, **kwargs):
return get_or_create(model, **kwargs)
for message in request.get_json(force=True):
data = message["data"]
timestamp = data["timestamp"]
if timestamp in timestamps:
continue
else:
timestamps.add(timestamp)
snapshot = cache(Snapshot, ts=timestamp)
for tenant_name, namespaces in data.items():
if not isinstance(namespaces, list):
continue
allocation = None
allocation_id = extract_allocation(tenant_name)
if allocation_id:
allocation = cache(Allocation, allocation=allocation_id)
tenant = cache(Tenant, name=tenant_name, allocation=allocation)
for details in namespaces:
if "namespaceName" in details:
namespace_name = details["namespaceName"]
else:
namespace_name = "__total__"
allocation = None
allocation_id = extract_allocation(namespace_name)
if allocation_id:
allocation = cache(Allocation,
allocation=allocation_id)
namespace = cache(Namespace,
name=namespace_name,
tenant=tenant,
allocation=allocation)
start_time = arrow.get(details["startTime"]).timestamp
end_time = arrow.get(details["endTime"]).timestamp
usage = {
"snapshot": snapshot,
"namespace": namespace,
"start_time": start_time,
"end_time": end_time,
"ingested_bytes": details["ingestedVolume"],
"raw_bytes": details["storageCapacityUsed"],
"reads": details["reads"],
"writes": details["writes"],
"deletes": details["deletes"],
"objects": details["objectCount"],
"bytes_in": details["bytesIn"],
"bytes_out": details["bytesOut"]
}
if "metadataOnlyObjects" in details:
usage["metadata_only_objects"] = details["metadataOnlyObjects"]
else:
usage["metadata_only_objects"] = 0
if "metadataOnlyBytes" in details:
usage["metadata_only_bytes"] = details["metadataOnlyBytes"],
else:
usage["metadata_only_bytes"] = 0
if "tieredObjects" in details:
usage["tiered_objects"] = details["tieredObjects"],
else:
usage["tiered_objects"] = 0
if "tieredBytes" in details:
usage["tiered_bytes"] = details["tieredBytes"]
else:
usage["tiered_bytes"] = 0
add(Usage(**usage))
commit()
return "", 204
def ingest(self):
"""Ingest usage."""
timestamps = set()
@lru_cache(maxsize=10000)
def cache(model, **kwargs):
return get_or_create(model, **kwargs)
for message in request.get_json(force=True):
data = message["data"]
timestamp = data["timestamp"]
if timestamp in timestamps:
continue
else:
timestamps.add(timestamp)
snapshot = cache(Snapshot, ts=timestamp)
for tenant_name, namespaces in data.items():
if not isinstance(namespaces, list):
continue
allocation = None
allocation_id = extract_allocation(tenant_name)
if allocation_id:
allocation = cache(Allocation, allocation=allocation_id)
tenant = cache(Tenant, name=tenant_name, allocation=allocation)
for details in namespaces:
if "namespaceName" in details:
namespace_name = details["namespaceName"]
else:
namespace_name = "__total__"
allocation = None
allocation_id = extract_allocation(namespace_name)
if allocation_id:
allocation = cache(Allocation,
allocation=allocation_id)
namespace = cache(Namespace,
name=namespace_name,
tenant=tenant,
allocation=allocation)
start_time = arrow.get(details["startTime"]).timestamp
end_time = arrow.get(details["endTime"]).timestamp
usage = {
"snapshot": snapshot,
"namespace": namespace,
"start_time": start_time,
"end_time": end_time,
"ingested_bytes": details["ingestedVolume"],
"raw_bytes": details["storageCapacityUsed"],
"reads": details["reads"],
"writes": details["writes"],
"deletes": details["deletes"],
"objects": details["objectCount"],
"bytes_in": details["bytesIn"],
"bytes_out": details["bytesOut"]
}
if "metadataOnlyObjects" in details:
usage["metadata_only_objects"] = details[
"metadataOnlyObjects"]
else:
usage["metadata_only_objects"] = 0
if "metadataOnlyBytes" in details:
usage["metadata_only_bytes"] = details[
"metadataOnlyBytes"],
else:
usage["metadata_only_bytes"] = 0
if "tieredObjects" in details:
usage["tiered_objects"] = details["tieredObjects"],
else:
usage["tiered_objects"] = 0
if "tieredBytes" in details:
usage["tiered_bytes"] = details["tieredBytes"]
else:
usage["tiered_bytes"] = 0
add(Usage(**usage))
commit()
return "", 204
def ingest(self):
"""Ingest data."""
@lru_cache(maxsize=100000)
def cache(model, **kwargs):
return get_or_create(model, **kwargs)
for message in request.get_json(force=True):
data = message["data"]
snapshot = cache(Snapshot, ts=data["timestamp"])
for flavor_detail in data["flavors"]:
flavor = cache(Flavor, openstack_id=flavor_detail["id"])
flavor.name = flavor_detail["name"]
flavor.vcpus = flavor_detail["vcpus"]
flavor.ram = flavor_detail["ram"]
flavor.disk = flavor_detail["disk"]
flavor.ephemeral = flavor_detail["OS-FLV-EXT-DATA:ephemeral"]
flavor.public = flavor_detail["os-flavor-access:is_public"]
for instance_detail in data["instances"]:
availability_zone_name = instance_detail[
"OS-EXT-AZ:availability_zone"]
if not availability_zone_name:
continue
availability_zone = cache(AvailabilityZone,
name=availability_zone_name)
if not availability_zone_name.startswith('sa'):
logger.debug("Skip non-sa zone: %s" %
availability_zone_name)
continue
hypervisor_hostname = instance_detail[
"OS-EXT-SRV-ATTR:hypervisor_hostname"]
if not hypervisor_hostname:
continue
hypervisor = cache(Hypervisor,
name=hypervisor_hostname,
availability_zone=availability_zone)
flavor = cache(Flavor,
openstack_id=instance_detail["flavor"]["id"])
account = cache(Account,
openstack_id=instance_detail["user_id"])
tenant = cache(Tenant,
openstack_id=instance_detail["tenant_id"])
status = cache(InstanceStatus,
name=instance_detail["OS-EXT-STS:vm_state"])
if not isinstance(instance_detail["image"], dict):
continue
image = cache(Image,
openstack_id=instance_detail["image"]["id"])
instance = cache(Instance, openstack_id=instance_detail["id"])
instance.account = account
instance.tenant = tenant
instance.flavor = flavor
instance.availability_zone = availability_zone
add(
InstanceState(snapshot=snapshot,
instance=instance,
image=image,
name=instance_detail["name"],
hypervisor=hypervisor,
status=status))
for network in instance_detail["addresses"].values():
for address in network:
mac = cache(MACAddress,
address=address["OS-EXT-IPS-MAC:mac_addr"])
add(
MACAddressMapping(snapshot=snapshot,
instance=instance,
address=mac))
ip = cache(IPAddress,
address=address["addr"],
family=address["version"])
add(
IPAddressMapping(snapshot=snapshot,
instance=instance,
address=ip))
commit()
return "", 204