def ingest(self):
@lru_cache(maxsize=100000)
def cache(model, **kwargs):
return get_or_create(model, **kwargs)
for message in request.get_json(force=True):
data = message["data"]
snapshot = cache(Snapshot, ts=data["timestamp"])
for key, value in data.items():
# Ugly hack until swift data pushed down into own dict.
valid = [c in string.hexdigits for c in key]
if not reduce(lambda x, y: x and y, valid):
continue
account = cache(Account, openstack_id=key)
add(Usage(bytes=value["bytes"],
containers=value["containers"],
objects=value["objects"],
quota=value["quota"],
account=account,
snapshot=snapshot))
commit()
return "", 204
def ingest(self):
"""Ingest usage."""
@lru_cache(maxsize=1000)
def cache(model, **kwargs):
return get_or_create(model, **kwargs)
for message in request.get_json(force=True):
if not message["schema"] == "hnas.filesystems":
continue
data = message["data"]
snapshot = cache(Snapshot, ts=data["timestamp"])
for name, details in data["filesystems"].items():
fs = cache(Filesystem, name=name)
fs_usage = {
"filesystem": fs,
"snapshot": snapshot,
"capacity": details["capacity"],
"free": details["free"],
"live_usage": details["live-fs-used"],
"snapshot_usage": details["snapshot-used"]
}
add(FilesystemUsage(**fs_usage))
if "virtual_volumes" in details:
for vusage in details["virtual_volumes"]:
name = vusage["volume-name"]
if name.startswith("/"):
name = name[1:]
vivol = cache(VirtualVolume,
name=name,
filesystem=fs)
vivol_usage = {
"snapshot": snapshot,
"virtual_volume": vivol,
"files": vusage["file-count"],
"usage": vusage["usage"],
"quota": vusage["usage-limit"]
}
if len(vusage["user-group-account"]) > 0:
owner = cache(Owner,
name=vusage["user-group-account"])
vivol_usage["owner"] = owner
add(VirtualVolumeUsage(**vivol_usage))
commit()
return "", 204
def put(self):
"""Ingest snapshots."""
record_input()
for message in request.json:
data = message["data"]
snapshot = Snapshot(ts=data["timestamp"])
add(snapshot)
for entry in data["organisations"]:
organisation = get_or_create(Organisation,
insightly_id=entry["id"])
organisation.name = entry["name"]
for entry in data["contacts"]:
person = get_or_create(Person, insightly_id=entry["id"])
person.first_name = entry["first_name"]
person.last_name = entry["last_name"]
if entry["username"]:
username = get_or_create(Username,
username=entry["username"])
get_or_create(PersonUsername,
snapshot=snapshot,
person=person,
username=username)
if entry["email"]:
for address in entry["email"]:
email = get_or_create(Email, address=address)
get_or_create(PersonEmail,
snapshot=snapshot,
person=person,
email=email)
if entry["organisations"]:
for insightly_id in entry["organisations"]:
organisation = get_or_create(Organisation,
insightly_id=insightly_id)
get_or_create(Membership,
snapshot=snapshot,
organisation=organisation,
person=person)
commit()
return "", 204
def ingest(self):
"""Ingest snapshots."""
for message in request.get_json(force=True):
data = message["data"]
snapshot = Snapshot(ts=data["timestamp"])
add(snapshot)
for entry in data["organisations"]:
organisation = get_or_create(Organisation,
insightly_id=entry["id"])
organisation.name = entry["name"]
for entry in data["contacts"]:
person = get_or_create(Person, insightly_id=entry["id"])
person.first_name = entry["first_name"]
person.last_name = entry["last_name"]
if entry["username"]:
username = get_or_create(Username,
username=entry["username"])
get_or_create(PersonUsername,
snapshot=snapshot,
person=person,
username=username)
if entry["email"]:
for address in entry["email"]:
email = get_or_create(Email, address=address)
get_or_create(PersonEmail,
snapshot=snapshot,
person=person,
email=email)
if entry["organisations"]:
for insightly_id in entry["organisations"]:
organisation = get_or_create(Organisation,
insightly_id=insightly_id)
get_or_create(Membership,
snapshot=snapshot,
organisation=organisation,
person=person)
commit()
return "", 204
def ingest(self):
"""Ingest usage."""
for message in request.get_json(force=True):
inserts = []
data = message["data"]
host = get_or_create(Host, name=data["hostname"])
metadata = data["fs"]
filesystem = get_or_create(Filesystem,
name=metadata["name"],
host=host)
snapshot = get_or_create(Snapshot,
ts=data["timestamp"],
filesystem=filesystem,
bavail=metadata["bavail"],
bfree=metadata["bfree"],
blocks=metadata["blocks"],
bsize=metadata["bsize"],
favail=metadata["favail"],
ffree=metadata["ffree"],
files=metadata["files"],
frsize=metadata["frsize"])
for who, details in data["usage"].items():
who = who.split("/")
owner = get_or_create(Owner, name=who[0])
project = get_or_create(Project, name=who[1])
add(
Usage(owner=owner,
project=project,
snapshot=snapshot,
blocks=details["blocks"],
bytes=details["bytes"],
files=details["files"]))
commit()
return "", 204
def ingest(self):
"""Ingest usage."""
for message in request.get_json(force=True):
inserts = []
data = message["data"]
host = get_or_create(Host, name=data["hostname"])
metadata = data["fs"]
filesystem = get_or_create(Filesystem,
name=metadata["name"],
host=host)
snapshot = get_or_create(Snapshot,
ts=data["timestamp"],
filesystem=filesystem,
bavail=metadata["bavail"],
bfree=metadata["bfree"],
blocks=metadata["blocks"],
bsize=metadata["bsize"],
favail=metadata["favail"],
ffree=metadata["ffree"],
files=metadata["files"],
frsize=metadata["frsize"])
for who, details in data["usage"].items():
who = who.split("/")
owner = get_or_create(Owner, name=who[0])
project = get_or_create(Project, name=who[1])
add(Usage(owner=owner,
project=project,
snapshot=snapshot,
blocks=details["blocks"],
bytes=details["bytes"],
files=details["files"]))
commit()
return "", 204
def ingest(self):
"""Ingest data."""
@lru_cache(maxsize=100000)
def cache(model, **kwargs):
return get_or_create(model, **kwargs)
for message in request.get_json(force=True):
data = message["data"]
snapshot = cache(Snapshot, ts=data["timestamp"])
for flavor_detail in data["flavors"]:
flavor = cache(Flavor, openstack_id=flavor_detail["id"])
flavor.name = flavor_detail["name"]
flavor.vcpus = flavor_detail["vcpus"]
flavor.ram = flavor_detail["ram"]
flavor.disk = flavor_detail["disk"]
flavor.ephemeral = flavor_detail["OS-FLV-EXT-DATA:ephemeral"]
flavor.public = flavor_detail["os-flavor-access:is_public"]
for instance_detail in data["instances"]:
availability_zone_name = instance_detail[
"OS-EXT-AZ:availability_zone"]
if not availability_zone_name:
continue
availability_zone = cache(AvailabilityZone,
name=availability_zone_name)
if not availability_zone_name.startswith('sa'):
logger.debug("Skip non-sa zone: %s" % availability_zone_name)
continue
hypervisor_hostname = instance_detail[
"OS-EXT-SRV-ATTR:hypervisor_hostname"]
if not hypervisor_hostname:
continue
hypervisor = cache(Hypervisor,
name=hypervisor_hostname,
availability_zone=availability_zone)
flavor = cache(Flavor,
openstack_id=instance_detail["flavor"]["id"])
account = cache(Account,
openstack_id=instance_detail["user_id"])
tenant = cache(Tenant,
openstack_id=instance_detail["tenant_id"])
status = cache(InstanceStatus,
name=instance_detail["OS-EXT-STS:vm_state"])
if not isinstance(instance_detail["image"], dict):
continue
image = cache(Image,
openstack_id=instance_detail["image"]["id"])
instance = cache(Instance, openstack_id=instance_detail["id"])
instance.account = account
instance.tenant = tenant
instance.flavor = flavor
instance.availability_zone = availability_zone
add(InstanceState(snapshot=snapshot,
instance=instance,
image=image,
name=instance_detail["name"],
hypervisor=hypervisor,
status=status))
for network in instance_detail["addresses"].values():
for address in network:
mac = cache(MACAddress,
address=address["OS-EXT-IPS-MAC:mac_addr"])
add(MACAddressMapping(snapshot=snapshot,
instance=instance,
address=mac))
ip = cache(IPAddress,
address=address["addr"],
family=address["version"])
add(IPAddressMapping(snapshot=snapshot,
instance=instance,
address=ip))
commit()
return "", 204
def ingest(self):
"""Ingest usage."""
timestamps = set()
@lru_cache(maxsize=10000)
def cache(model, **kwargs):
return get_or_create(model, **kwargs)
for message in request.get_json(force=True):
data = message["data"]
timestamp = data["timestamp"]
if timestamp in timestamps:
continue
else:
timestamps.add(timestamp)
snapshot = cache(Snapshot, ts=timestamp)
for tenant_name, namespaces in data.items():
if not isinstance(namespaces, list):
continue
allocation = None
allocation_id = extract_allocation(tenant_name)
if allocation_id:
allocation = cache(Allocation, allocation=allocation_id)
tenant = cache(Tenant, name=tenant_name, allocation=allocation)
for details in namespaces:
if "namespaceName" in details:
namespace_name = details["namespaceName"]
else:
namespace_name = "__total__"
allocation = None
allocation_id = extract_allocation(namespace_name)
if allocation_id:
allocation = cache(Allocation,
allocation=allocation_id)
namespace = cache(Namespace,
name=namespace_name,
tenant=tenant,
allocation=allocation)
start_time = arrow.get(details["startTime"]).timestamp
end_time = arrow.get(details["endTime"]).timestamp
usage = {
"snapshot": snapshot,
"namespace": namespace,
"start_time": start_time,
"end_time": end_time,
"ingested_bytes": details["ingestedVolume"],
"raw_bytes": details["storageCapacityUsed"],
"reads": details["reads"],
"writes": details["writes"],
"deletes": details["deletes"],
"objects": details["objectCount"],
"bytes_in": details["bytesIn"],
"bytes_out": details["bytesOut"]
}
if "metadataOnlyObjects" in details:
usage["metadata_only_objects"] = details["metadataOnlyObjects"]
else:
usage["metadata_only_objects"] = 0
if "metadataOnlyBytes" in details:
usage["metadata_only_bytes"] = details["metadataOnlyBytes"],
else:
usage["metadata_only_bytes"] = 0
if "tieredObjects" in details:
usage["tiered_objects"] = details["tieredObjects"],
else:
usage["tiered_objects"] = 0
if "tieredBytes" in details:
usage["tiered_bytes"] = details["tieredBytes"]
else:
usage["tiered_bytes"] = 0
add(Usage(**usage))
commit()
return "", 204
def ingest(self):
"""Ingest data."""
@lru_cache(maxsize=100000)
def cache(model, **kwargs):
return get_or_create(model, **kwargs)
for message in request.get_json(force=True):
data = message["data"]
snapshot = cache(Snapshot, ts=data["timestamp"])
for flavor_detail in data["flavors"]:
flavor = cache(Flavor, openstack_id=flavor_detail["id"])
flavor.name = flavor_detail["name"]
flavor.vcpus = flavor_detail["vcpus"]
flavor.ram = flavor_detail["ram"]
flavor.disk = flavor_detail["disk"]
flavor.ephemeral = flavor_detail["OS-FLV-EXT-DATA:ephemeral"]
flavor.public = flavor_detail["os-flavor-access:is_public"]
for instance_detail in data["instances"]:
availability_zone_name = instance_detail[
"OS-EXT-AZ:availability_zone"]
if not availability_zone_name:
continue
availability_zone = cache(AvailabilityZone,
name=availability_zone_name)
if not availability_zone_name.startswith('sa'):
logger.debug("Skip non-sa zone: %s" %
availability_zone_name)
continue
hypervisor_hostname = instance_detail[
"OS-EXT-SRV-ATTR:hypervisor_hostname"]
if not hypervisor_hostname:
continue
hypervisor = cache(Hypervisor,
name=hypervisor_hostname,
availability_zone=availability_zone)
flavor = cache(Flavor,
openstack_id=instance_detail["flavor"]["id"])
account = cache(Account,
openstack_id=instance_detail["user_id"])
tenant = cache(Tenant,
openstack_id=instance_detail["tenant_id"])
status = cache(InstanceStatus,
name=instance_detail["OS-EXT-STS:vm_state"])
if not isinstance(instance_detail["image"], dict):
continue
image = cache(Image,
openstack_id=instance_detail["image"]["id"])
instance = cache(Instance, openstack_id=instance_detail["id"])
instance.account = account
instance.tenant = tenant
instance.flavor = flavor
instance.availability_zone = availability_zone
add(
InstanceState(snapshot=snapshot,
instance=instance,
image=image,
name=instance_detail["name"],
hypervisor=hypervisor,
status=status))
for network in instance_detail["addresses"].values():
for address in network:
mac = cache(MACAddress,
address=address["OS-EXT-IPS-MAC:mac_addr"])
add(
MACAddressMapping(snapshot=snapshot,
instance=instance,
address=mac))
ip = cache(IPAddress,
address=address["addr"],
family=address["version"])
add(
IPAddressMapping(snapshot=snapshot,
instance=instance,
address=ip))
commit()
return "", 204
def ingest(self):
"""Ingest usage."""
timestamps = set()
@lru_cache(maxsize=10000)
def cache(model, **kwargs):
return get_or_create(model, **kwargs)
for message in request.get_json(force=True):
data = message["data"]
timestamp = data["timestamp"]
if timestamp in timestamps:
continue
else:
timestamps.add(timestamp)
snapshot = cache(Snapshot, ts=timestamp)
for tenant_name, namespaces in data.items():
if not isinstance(namespaces, list):
continue
allocation = None
allocation_id = extract_allocation(tenant_name)
if allocation_id:
allocation = cache(Allocation, allocation=allocation_id)
tenant = cache(Tenant, name=tenant_name, allocation=allocation)
for details in namespaces:
if "namespaceName" in details:
namespace_name = details["namespaceName"]
else:
namespace_name = "__total__"
allocation = None
allocation_id = extract_allocation(namespace_name)
if allocation_id:
allocation = cache(Allocation,
allocation=allocation_id)
namespace = cache(Namespace,
name=namespace_name,
tenant=tenant,
allocation=allocation)
start_time = arrow.get(details["startTime"]).timestamp
end_time = arrow.get(details["endTime"]).timestamp
usage = {
"snapshot": snapshot,
"namespace": namespace,
"start_time": start_time,
"end_time": end_time,
"ingested_bytes": details["ingestedVolume"],
"raw_bytes": details["storageCapacityUsed"],
"reads": details["reads"],
"writes": details["writes"],
"deletes": details["deletes"],
"objects": details["objectCount"],
"bytes_in": details["bytesIn"],
"bytes_out": details["bytesOut"]
}
if "metadataOnlyObjects" in details:
usage["metadata_only_objects"] = details[
"metadataOnlyObjects"]
else:
usage["metadata_only_objects"] = 0
if "metadataOnlyBytes" in details:
usage["metadata_only_bytes"] = details[
"metadataOnlyBytes"],
else:
usage["metadata_only_bytes"] = 0
if "tieredObjects" in details:
usage["tiered_objects"] = details["tieredObjects"],
else:
usage["tiered_objects"] = 0
if "tieredBytes" in details:
usage["tiered_bytes"] = details["tieredBytes"]
else:
usage["tiered_bytes"] = 0
add(Usage(**usage))
commit()
return "", 204
