def ingest(self): """Ingest jobs.""" messages = [message for message in request.get_json(force=True) if message["data"]["state"] == "exited"] for message in messages: data = message["data"] queue = get_or_create(Queue, name=data["queue"]) owner = get_or_create(Owner, name=data["owner"]) job = get_or_create(Job, job_id=data["jobid"]) job.name = data["jobname"] job.queue = queue job.owner = owner job.start = data["start"] job.end = data["end"] total_cores = 0 for hostname, slots in data["exec_host"].items(): host = get_or_create(Host, name=hostname) get_or_create(Allocation, job=job, host=host, cores=len(slots)) total_cores += len(slots) job.cores = total_cores job.cpu_seconds = total_cores * (data["end"] - data["start"]) commit() return "", 204
def ingest(self): @lru_cache(maxsize=100000) def cache(model, **kwargs): return get_or_create(model, **kwargs) for message in request.get_json(force=True): data = message["data"] snapshot = cache(Snapshot, ts=data["timestamp"]) for key, value in data.items(): # Ugly hack until swift data pushed down into own dict. valid = [c in string.hexdigits for c in key] if not reduce(lambda x, y: x and y, valid): continue account = cache(Account, openstack_id=key) add(Usage(bytes=value["bytes"], containers=value["containers"], objects=value["objects"], quota=value["quota"], account=account, snapshot=snapshot)) commit() return "", 204
def ingest(self): """Ingest jobs.""" messages = [ message for message in request.get_json(force=True) if message["data"].get("state") == "exited" ] for message in messages: data = message["data"] queue = get_or_create(Queue, name=data["queue"]) owner = get_or_create(Owner, name=data["owner"]) job = get_or_create(Job, job_id=data["jobid"]) job.name = data["jobname"] job.queue = queue job.owner = owner job.start = data["start"] job.end = data["end"] total_cores = 0 for hostname, slots in data["exec_host"].items(): host = get_or_create(Host, name=hostname) get_or_create(Allocation, job=job, host=host, cores=len(slots)) total_cores += len(slots) job.cores = total_cores job.cpu_seconds = total_cores * (data["end"] - data["start"]) commit() return "", 204
def put(self): try: for item in request.get_json(force=True): self.ingest(item) commit() return "", 204 except Exception as e: rollback() return str(e), 400
def ingest(self): """Ingest usage.""" @lru_cache(maxsize=1000) def cache(model, **kwargs): return get_or_create(model, **kwargs) for message in request.get_json(force=True): if not message["schema"] == "hnas.filesystems": continue data = message["data"] snapshot = cache(Snapshot, ts=data["timestamp"]) for name, details in data["filesystems"].items(): fs = cache(Filesystem, name=name) fs_usage = { "filesystem": fs, "snapshot": snapshot, "capacity": details["capacity"], "free": details["free"], "live_usage": details["live-fs-used"], "snapshot_usage": details["snapshot-used"] } add(FilesystemUsage(**fs_usage)) if "virtual_volumes" in details: for vusage in details["virtual_volumes"]: name = vusage["volume-name"] if name.startswith("/"): name = name[1:] vivol = cache(VirtualVolume, name=name, filesystem=fs) vivol_usage = { "snapshot": snapshot, "virtual_volume": vivol, "files": vusage["file-count"], "usage": vusage["usage"], "quota": vusage["usage-limit"] } if len(vusage["user-group-account"]) > 0: owner = cache(Owner, name=vusage["user-group-account"]) vivol_usage["owner"] = owner add(VirtualVolumeUsage(**vivol_usage)) commit() return "", 204
def ingest(self): """Ingest data.""" @lru_cache(maxsize=10000) def cache(model, **kwargs): return get_or_create(model, **kwargs) for message in request.get_json(force=True): data = message["data"] snapshot = cache(Snapshot, ts=data["timestamp"]) for account_detail in data["users"]: account = cache(Account, openstack_id=account_detail["id"]) if not account_detail["email"]: continue # Fix broken emails containing ";" email = account_detail["email"].split(";")[0] domain_name = get_domain(email) domain = cache(Domain, name=domain_name) if domain_name else None reference = cache(AccountReference, value=email, domain=domain) cache(AccountReferenceMapping, account=account, reference=reference, snapshot=snapshot) for tenant_detail in data["tenants"]: tenant = cache(Tenant, openstack_id=tenant_detail["id"]) tenant.name = tenant_detail["name"] tenant.description = tenant_detail["description"] if "allocation_id" in tenant_detail: try: tenant.allocation = int(tenant_detail["allocation_id"]) except: pass if "users" not in tenant_detail: continue for member in tenant_detail["users"]: account = cache(Account, openstack_id=member["id"]) cache(Membership, account=account, tenant=tenant, snapshot=snapshot) commit() return "", 204
def ingest(self): """Ingest usage.""" @lru_cache(maxsize=10000) def cache(model, **kwargs): return get_or_create(model, **kwargs) # This one is experimentally optimised for performance. # Internally creates a TSV and copies it straight # into the database. # Probably not necessary though. tsv = io.StringIO() for ingest_pass in [1, 2]: for message in request.get_json(force=True): if message["schema"] != "xfs.quota.report": continue data = message["data"] host = cache(Host, name=data["hostname"]) snapshot = cache(Snapshot, ts=data["timestamp"], host=host, message=message["id"]) for entry in data["filesystems"]: filesystem = cache(Filesystem, name=entry["filesystem"], host=host) for record in entry["quota"]: owner = cache(Owner, name=record["username"]) if ingest_pass == 2: columns = [ uuid.uuid4(), record["soft"], record["hard"], record["used"], owner.id, snapshot.id, filesystem.id ] tsv.write("\t".join([str(c) for c in columns]) + "\n") if ingest_pass == 2: tsv.seek(0) cursor = db.session.connection().connection.cursor() cursor.copy_from(tsv, "usage") commit() return "", 204
def ingest(self): """Data ingest""" @lru_cache(maxsize=100000) def cache(model, **kwargs): return get_or_create(model, **kwargs) for message in request.get_json(force=True): data = message["data"] snapshot = cache(Snapshot, ts=data["timestamp"]) if "volumes" in data: for metadata in data["volumes"]: az = None if "availability_zone" in metadata: az = cache(AvailabilityZone, name=metadata["availability_zone"]) volume = cache( Volume, openstack_id=metadata["id"], availability_zone=az, owner=metadata["user_id"], tenant=metadata["os-vol-tenant-attr:tenant_id"]) status = cache(VolumeStatus, name=metadata["status"]) cache(VolumeState, name=metadata["name"], size=metadata["size"], status=status, snapshot=snapshot, volume=volume) for instance in metadata["attachments"]: cache(VolumeAttachment, instance=instance["server_id"], volume=volume, snapshot=snapshot) if "volume_snapshots" in data: for metadata in data["volume_snapshots"]: cache(VolumeSnapshot, openstack_id=metadata["id"], name=metadata["name"], description=metadata["description"], size=metadata["size"], source=metadata["volume_id"]) commit() return "", 204
def ingest(self): """Ingest snapshots.""" for message in request.get_json(force=True): data = message["data"] snapshot = Snapshot(ts=data["timestamp"]) add(snapshot) for entry in data["organisations"]: organisation = get_or_create(Organisation, insightly_id=entry["id"]) organisation.name = entry["name"] for entry in data["contacts"]: person = get_or_create(Person, insightly_id=entry["id"]) person.first_name = entry["first_name"] person.last_name = entry["last_name"] if entry["username"]: username = get_or_create(Username, username=entry["username"]) get_or_create(PersonUsername, snapshot=snapshot, person=person, username=username) if entry["email"]: for address in entry["email"]: email = get_or_create(Email, address=address) get_or_create(PersonEmail, snapshot=snapshot, person=person, email=email) if entry["organisations"]: for insightly_id in entry["organisations"]: organisation = get_or_create(Organisation, insightly_id=insightly_id) get_or_create(Membership, snapshot=snapshot, organisation=organisation, person=person) commit() return "", 204
def ingest(self): """Ingest usage.""" for message in request.get_json(force=True): inserts = [] data = message["data"] host = get_or_create(Host, name=data["hostname"]) metadata = data["fs"] filesystem = get_or_create(Filesystem, name=metadata["name"], host=host) snapshot = get_or_create(Snapshot, ts=data["timestamp"], filesystem=filesystem, bavail=metadata["bavail"], bfree=metadata["bfree"], blocks=metadata["blocks"], bsize=metadata["bsize"], favail=metadata["favail"], ffree=metadata["ffree"], files=metadata["files"], frsize=metadata["frsize"]) for who, details in data["usage"].items(): who = who.split("/") owner = get_or_create(Owner, name=who[0]) project = get_or_create(Project, name=who[1]) add( Usage(owner=owner, project=project, snapshot=snapshot, blocks=details["blocks"], bytes=details["bytes"], files=details["files"])) commit() return "", 204
def ingest(self): """Ingest usage.""" for message in request.get_json(force=True): inserts = [] data = message["data"] host = get_or_create(Host, name=data["hostname"]) metadata = data["fs"] filesystem = get_or_create(Filesystem, name=metadata["name"], host=host) snapshot = get_or_create(Snapshot, ts=data["timestamp"], filesystem=filesystem, bavail=metadata["bavail"], bfree=metadata["bfree"], blocks=metadata["blocks"], bsize=metadata["bsize"], favail=metadata["favail"], ffree=metadata["ffree"], files=metadata["files"], frsize=metadata["frsize"]) for who, details in data["usage"].items(): who = who.split("/") owner = get_or_create(Owner, name=who[0]) project = get_or_create(Project, name=who[1]) add(Usage(owner=owner, project=project, snapshot=snapshot, blocks=details["blocks"], bytes=details["bytes"], files=details["files"])) commit() return "", 204
def ingest(self): """Ingest data.""" @lru_cache(maxsize=100000) def cache(model, **kwargs): return get_or_create(model, **kwargs) for message in request.get_json(force=True): data = message["data"] snapshot = cache(Snapshot, ts=data["timestamp"]) for flavor_detail in data["flavors"]: flavor = cache(Flavor, openstack_id=flavor_detail["id"]) flavor.name = flavor_detail["name"] flavor.vcpus = flavor_detail["vcpus"] flavor.ram = flavor_detail["ram"] flavor.disk = flavor_detail["disk"] flavor.ephemeral = flavor_detail["OS-FLV-EXT-DATA:ephemeral"] flavor.public = flavor_detail["os-flavor-access:is_public"] for instance_detail in data["instances"]: availability_zone_name = instance_detail[ "OS-EXT-AZ:availability_zone"] if not availability_zone_name: continue availability_zone = cache(AvailabilityZone, name=availability_zone_name) if not availability_zone_name.startswith('sa'): logger.debug("Skip non-sa zone: %s" % availability_zone_name) continue hypervisor_hostname = instance_detail[ "OS-EXT-SRV-ATTR:hypervisor_hostname"] if not hypervisor_hostname: continue hypervisor = cache(Hypervisor, name=hypervisor_hostname, availability_zone=availability_zone) flavor = cache(Flavor, openstack_id=instance_detail["flavor"]["id"]) account = cache(Account, openstack_id=instance_detail["user_id"]) tenant = cache(Tenant, openstack_id=instance_detail["tenant_id"]) status = cache(InstanceStatus, name=instance_detail["OS-EXT-STS:vm_state"]) if not isinstance(instance_detail["image"], dict): continue image = cache(Image, openstack_id=instance_detail["image"]["id"]) instance = cache(Instance, openstack_id=instance_detail["id"]) instance.account = account instance.tenant = tenant instance.flavor = flavor instance.availability_zone = availability_zone add(InstanceState(snapshot=snapshot, instance=instance, image=image, name=instance_detail["name"], hypervisor=hypervisor, status=status)) for network in instance_detail["addresses"].values(): for address in network: mac = cache(MACAddress, address=address["OS-EXT-IPS-MAC:mac_addr"]) add(MACAddressMapping(snapshot=snapshot, instance=instance, address=mac)) ip = cache(IPAddress, address=address["addr"], family=address["version"]) add(IPAddressMapping(snapshot=snapshot, instance=instance, address=ip)) commit() return "", 204
def ingest(self): """Ingest usage.""" timestamps = set() @lru_cache(maxsize=10000) def cache(model, **kwargs): return get_or_create(model, **kwargs) for message in request.get_json(force=True): data = message["data"] timestamp = data["timestamp"] if timestamp in timestamps: continue else: timestamps.add(timestamp) snapshot = cache(Snapshot, ts=timestamp) for tenant_name, namespaces in data.items(): if not isinstance(namespaces, list): continue allocation = None allocation_id = extract_allocation(tenant_name) if allocation_id: allocation = cache(Allocation, allocation=allocation_id) tenant = cache(Tenant, name=tenant_name, allocation=allocation) for details in namespaces: if "namespaceName" in details: namespace_name = details["namespaceName"] else: namespace_name = "__total__" allocation = None allocation_id = extract_allocation(namespace_name) if allocation_id: allocation = cache(Allocation, allocation=allocation_id) namespace = cache(Namespace, name=namespace_name, tenant=tenant, allocation=allocation) start_time = arrow.get(details["startTime"]).timestamp end_time = arrow.get(details["endTime"]).timestamp usage = { "snapshot": snapshot, "namespace": namespace, "start_time": start_time, "end_time": end_time, "ingested_bytes": details["ingestedVolume"], "raw_bytes": details["storageCapacityUsed"], "reads": details["reads"], "writes": details["writes"], "deletes": details["deletes"], "objects": details["objectCount"], "bytes_in": details["bytesIn"], "bytes_out": details["bytesOut"] } if "metadataOnlyObjects" in details: usage["metadata_only_objects"] = details["metadataOnlyObjects"] else: usage["metadata_only_objects"] = 0 if "metadataOnlyBytes" in details: usage["metadata_only_bytes"] = details["metadataOnlyBytes"], else: usage["metadata_only_bytes"] = 0 if "tieredObjects" in details: usage["tiered_objects"] = details["tieredObjects"], else: usage["tiered_objects"] = 0 if "tieredBytes" in details: usage["tiered_bytes"] = details["tieredBytes"] else: usage["tiered_bytes"] = 0 add(Usage(**usage)) commit() return "", 204
def ingest(self): """Ingest usage.""" timestamps = set() @lru_cache(maxsize=10000) def cache(model, **kwargs): return get_or_create(model, **kwargs) for message in request.get_json(force=True): data = message["data"] timestamp = data["timestamp"] if timestamp in timestamps: continue else: timestamps.add(timestamp) snapshot = cache(Snapshot, ts=timestamp) for tenant_name, namespaces in data.items(): if not isinstance(namespaces, list): continue allocation = None allocation_id = extract_allocation(tenant_name) if allocation_id: allocation = cache(Allocation, allocation=allocation_id) tenant = cache(Tenant, name=tenant_name, allocation=allocation) for details in namespaces: if "namespaceName" in details: namespace_name = details["namespaceName"] else: namespace_name = "__total__" allocation = None allocation_id = extract_allocation(namespace_name) if allocation_id: allocation = cache(Allocation, allocation=allocation_id) namespace = cache(Namespace, name=namespace_name, tenant=tenant, allocation=allocation) start_time = arrow.get(details["startTime"]).timestamp end_time = arrow.get(details["endTime"]).timestamp usage = { "snapshot": snapshot, "namespace": namespace, "start_time": start_time, "end_time": end_time, "ingested_bytes": details["ingestedVolume"], "raw_bytes": details["storageCapacityUsed"], "reads": details["reads"], "writes": details["writes"], "deletes": details["deletes"], "objects": details["objectCount"], "bytes_in": details["bytesIn"], "bytes_out": details["bytesOut"] } if "metadataOnlyObjects" in details: usage["metadata_only_objects"] = details[ "metadataOnlyObjects"] else: usage["metadata_only_objects"] = 0 if "metadataOnlyBytes" in details: usage["metadata_only_bytes"] = details[ "metadataOnlyBytes"], else: usage["metadata_only_bytes"] = 0 if "tieredObjects" in details: usage["tiered_objects"] = details["tieredObjects"], else: usage["tiered_objects"] = 0 if "tieredBytes" in details: usage["tiered_bytes"] = details["tieredBytes"] else: usage["tiered_bytes"] = 0 add(Usage(**usage)) commit() return "", 204
def ingest(self): """Ingest data.""" @lru_cache(maxsize=100000) def cache(model, **kwargs): return get_or_create(model, **kwargs) for message in request.get_json(force=True): data = message["data"] snapshot = cache(Snapshot, ts=data["timestamp"]) for flavor_detail in data["flavors"]: flavor = cache(Flavor, openstack_id=flavor_detail["id"]) flavor.name = flavor_detail["name"] flavor.vcpus = flavor_detail["vcpus"] flavor.ram = flavor_detail["ram"] flavor.disk = flavor_detail["disk"] flavor.ephemeral = flavor_detail["OS-FLV-EXT-DATA:ephemeral"] flavor.public = flavor_detail["os-flavor-access:is_public"] for instance_detail in data["instances"]: availability_zone_name = instance_detail[ "OS-EXT-AZ:availability_zone"] if not availability_zone_name: continue availability_zone = cache(AvailabilityZone, name=availability_zone_name) if not availability_zone_name.startswith('sa'): logger.debug("Skip non-sa zone: %s" % availability_zone_name) continue hypervisor_hostname = instance_detail[ "OS-EXT-SRV-ATTR:hypervisor_hostname"] if not hypervisor_hostname: continue hypervisor = cache(Hypervisor, name=hypervisor_hostname, availability_zone=availability_zone) flavor = cache(Flavor, openstack_id=instance_detail["flavor"]["id"]) account = cache(Account, openstack_id=instance_detail["user_id"]) tenant = cache(Tenant, openstack_id=instance_detail["tenant_id"]) status = cache(InstanceStatus, name=instance_detail["OS-EXT-STS:vm_state"]) if not isinstance(instance_detail["image"], dict): continue image = cache(Image, openstack_id=instance_detail["image"]["id"]) instance = cache(Instance, openstack_id=instance_detail["id"]) instance.account = account instance.tenant = tenant instance.flavor = flavor instance.availability_zone = availability_zone add( InstanceState(snapshot=snapshot, instance=instance, image=image, name=instance_detail["name"], hypervisor=hypervisor, status=status)) for network in instance_detail["addresses"].values(): for address in network: mac = cache(MACAddress, address=address["OS-EXT-IPS-MAC:mac_addr"]) add( MACAddressMapping(snapshot=snapshot, instance=instance, address=mac)) ip = cache(IPAddress, address=address["addr"], family=address["version"]) add( IPAddressMapping(snapshot=snapshot, instance=instance, address=ip)) commit() return "", 204