def put(self):
"""Ingest jobs."""
record_input()
messages = [message
for message in request.json
if message["data"]["state"] == "exited"]
for message in messages:
data = message["data"]
queue = get_or_create(Queue, name=data["queue"])
owner = get_or_create(Owner, name=data["owner"])
job = get_or_create(Job, job_id=data["jobid"])
job.name = data["jobname"]
job.queue = queue
job.owner = owner
job.start = data["start"]
job.end = data["end"]
total_cores = 0
for hostname, slots in data["exec_host"].items():
host = get_or_create(Host, name=hostname)
get_or_create(Allocation, job=job, host=host, cores=len(slots))
total_cores += len(slots)
job.cores = total_cores
job.cpu_seconds = total_cores * (data["end"] - data["start"])
commit()
return "", 204
def put(self):
"""Ingest data."""
@lru_cache(maxsize=10000)
def cache(model, **kwargs):
return get_or_create(model, **kwargs)
record_input()
for message in request.json:
data = message["data"]
snapshot = cache(Snapshot, ts=data["timestamp"])
for account_detail in data["users"]:
account = cache(Account, openstack_id=account_detail["id"])
if not account_detail["email"]:
continue
# Fix broken emails containing ";"
email = account_detail["email"].split(";")[0]
domain_name = get_domain(email)
domain = cache(Domain, name=domain_name) if domain_name else None
reference = cache(AccountReference, value=email, domain=domain)
cache(AccountReferenceMapping, account=account, reference=reference, snapshot=snapshot)
for tenant_detail in data["tenants"]:
tenant = cache(Tenant, openstack_id=tenant_detail["id"])
tenant.name = tenant_detail["name"]
tenant.description = tenant_detail["description"]
if "allocation_id" in tenant_detail:
try:
tenant.allocation = int(tenant_detail["allocation_id"])
except:
pass
if "users" not in tenant_detail:
continue
for member in tenant_detail["users"]:
account = cache(Account, openstack_id=member["id"])
cache(Membership, account=account, tenant=tenant, snapshot=snapshot)
commit()
return "", 204
def put(self):
"""Ingest snapshots."""
record_input()
for message in request.json:
data = message["data"]
snapshot = Snapshot(ts=data["timestamp"])
add(snapshot)
for entry in data["organisations"]:
organisation = get_or_create(Organisation,
insightly_id=entry["id"])
organisation.name = entry["name"]
for entry in data["contacts"]:
person = get_or_create(Person, insightly_id=entry["id"])
person.first_name = entry["first_name"]
person.last_name = entry["last_name"]
if entry["username"]:
username = get_or_create(Username,
username=entry["username"])
get_or_create(PersonUsername,
snapshot=snapshot,
person=person,
username=username)
if entry["email"]:
for address in entry["email"]:
email = get_or_create(Email, address=address)
get_or_create(PersonEmail,
snapshot=snapshot,
person=person,
email=email)
if entry["organisations"]:
for insightly_id in entry["organisations"]:
organisation = get_or_create(Organisation,
insightly_id=insightly_id)
get_or_create(Membership,
snapshot=snapshot,
organisation=organisation,
person=person)
commit()
return "", 204
def put(self):
"""Ingest usage."""
@lru_cache(maxsize=10000)
def cache(model, **kwargs):
return get_or_create(model, **kwargs)
tsv = io.StringIO()
record_input()
for ingest_pass in [1, 2]:
for message in request.json:
data = message["data"]
host = cache(Host, name=data["hostname"])
snapshot = cache(Snapshot,
ts=data["timestamp"],
host=host,
message=message["id"])
for entry in data["filesystems"]:
filesystem = cache(Filesystem,
name=entry["filesystem"],
host=host)
for record in entry["quota"]:
owner = cache(Owner, name=record["username"])
if ingest_pass == 2:
columns = [
uuid.uuid4(), record["soft"], record["hard"],
record["used"], owner.id, snapshot.id,
filesystem.id
]
tsv.write("\t".join([str(c) for c in columns]) +
"\n")
if ingest_pass == 2:
tsv.seek(0)
cursor = db.session.connection().connection.cursor()
cursor.copy_from(tsv, "usage")
commit()
return "", 204
def put(self):
"""Ingest data."""
@lru_cache(maxsize=100000)
def cache(model, **kwargs):
return get_or_create(model, **kwargs)
record_input()
for message in request.json:
data = message["data"]
snapshot = cache(Snapshot, ts=data["timestamp"])
for flavor_detail in data["flavors"]:
flavor = cache(Flavor, openstack_id=flavor_detail["id"])
flavor.name = flavor_detail["name"]
flavor.vcpus = flavor_detail["vcpus"]
flavor.ram = flavor_detail["ram"]
flavor.disk = flavor_detail["disk"]
flavor.ephemeral = flavor_detail["OS-FLV-EXT-DATA:ephemeral"]
flavor.public = flavor_detail["os-flavor-access:is_public"]
for instance_detail in data["instances"]:
availability_zone_name = instance_detail[
"OS-EXT-AZ:availability_zone"]
if not availability_zone_name:
continue
availability_zone = cache(AvailabilityZone,
name=availability_zone_name)
hypervisor_hostname = instance_detail[
"OS-EXT-SRV-ATTR:hypervisor_hostname"]
if not hypervisor_hostname:
continue
hypervisor = cache(Hypervisor,
name=hypervisor_hostname,
availability_zone=availability_zone)
flavor = cache(Flavor,
openstack_id=instance_detail["flavor"]["id"])
account = cache(Account,
openstack_id=instance_detail["user_id"])
tenant = cache(Tenant,
openstack_id=instance_detail["tenant_id"])
status = cache(InstanceStatus,
name=instance_detail["OS-EXT-STS:vm_state"])
if not isinstance(instance_detail["image"], dict):
continue
image = cache(Image,
openstack_id=instance_detail["image"]["id"])
instance = cache(Instance, openstack_id=instance_detail["id"])
instance.account = account
instance.tenant = tenant
instance.flavor = flavor
instance.availability_zone = availability_zone
add(InstanceState(snapshot=snapshot,
instance=instance,
image=image,
name=instance_detail["name"],
hypervisor=hypervisor,
status=status))
for network in instance_detail["addresses"].values():
for address in network:
mac = cache(MACAddress,
address=address["OS-EXT-IPS-MAC:mac_addr"])
add(MACAddressMapping(snapshot=snapshot,
instance=instance,
address=mac))
ip = cache(IPAddress,
address=address["addr"],
family=address["version"])
add(IPAddressMapping(snapshot=snapshot,
instance=instance,
address=ip))
commit()
return "", 204
def put(self):
"""Ingest usage."""
@lru_cache(maxsize=1000)
def cache(model, **kwargs):
return get_or_create(model, **kwargs)
record_input()
for message in request.json:
if not message["schema"] == "hnas.filesystems":
continue
data = message["data"]
snapshot = cache(Snapshot, ts=data["timestamp"])
for name, details in data["filesystems"].items():
allocation = None
allocation_id = extract_allocation(name)
if allocation_id:
allocation = cache(Allocation, allocation=allocation_id)
fs = cache(Filesystem, name=name, allocation=allocation)
fs_usage = {
"filesystem": fs,
"snapshot": snapshot,
"capacity": details["capacity"],
"free": details["free"],
"live_usage": details["live-fs-used"],
"snapshot_usage": details["snapshot-used"],
}
add(FilesystemUsage(**fs_usage))
if "virtual_volumes" in details:
for vusage in details["virtual_volumes"]:
name = vusage["volume-name"]
if name.startswith("/"):
name = name[1:]
allocation = None
allocation_id = extract_allocation(name)
if allocation_id:
allocation = cache(Allocation, allocation=allocation_id)
vivol = cache(VirtualVolume, name=name, filesystem=fs, allocation=allocation)
vivol_usage = {
"snapshot": snapshot,
"virtual_volume": vivol,
"files": vusage["file-count"],
"usage": vusage["usage"],
"quota": vusage["usage-limit"],
}
if len(vusage["user-group-account"]) > 0:
owner = cache(Owner, name=vusage["user-group-account"])
vivol_usage["owner"] = owner
add(VirtualVolumeUsage(**vivol_usage))
commit()
return "", 204
