def ingest(self):
"""Ingest usage."""
for message in request.get_json(force=True):
inserts = []
data = message["data"]
host = get_or_create(Host, name=data["hostname"])
metadata = data["fs"]
filesystem = get_or_create(Filesystem,
name=metadata["name"],
host=host)
snapshot = get_or_create(Snapshot,
ts=data["timestamp"],
filesystem=filesystem,
bavail=metadata["bavail"],
bfree=metadata["bfree"],
blocks=metadata["blocks"],
bsize=metadata["bsize"],
favail=metadata["favail"],
ffree=metadata["ffree"],
files=metadata["files"],
frsize=metadata["frsize"])
for who, details in data["usage"].items():
who = who.split("/")
owner = get_or_create(Owner, name=who[0])
project = get_or_create(Project, name=who[1])
add(
Usage(owner=owner,
project=project,
snapshot=snapshot,
blocks=details["blocks"],
bytes=details["bytes"],
files=details["files"]))
commit()
return "", 204
def ingest(self, item):
obj = get_or_create(self.query_class, id=item["id"])
for key, value in item.items():
if key == "id":
continue
key_id = "%s_id" % key
if hasattr(obj, key_id):
setattr(obj, key_id, value)
else:
setattr(obj, key, value)
def ingest(self, item):
obj = get_or_create(self.query_class, id=item["id"])
for key, value in item.items():
if key == "id":
continue
key_id = "%s_id" % key
if hasattr(obj, key_id):
setattr(obj, key_id, value)
else:
setattr(obj, key, value)
def ingest(self):
"""Ingest usage."""
for message in request.get_json(force=True):
inserts = []
data = message["data"]
host = get_or_create(Host, name=data["hostname"])
metadata = data["fs"]
filesystem = get_or_create(Filesystem,
name=metadata["name"],
host=host)
snapshot = get_or_create(Snapshot,
ts=data["timestamp"],
filesystem=filesystem,
bavail=metadata["bavail"],
bfree=metadata["bfree"],
blocks=metadata["blocks"],
bsize=metadata["bsize"],
favail=metadata["favail"],
ffree=metadata["ffree"],
files=metadata["files"],
frsize=metadata["frsize"])
for who, details in data["usage"].items():
who = who.split("/")
owner = get_or_create(Owner, name=who[0])
project = get_or_create(Project, name=who[1])
add(Usage(owner=owner,
project=project,
snapshot=snapshot,
blocks=details["blocks"],
bytes=details["bytes"],
files=details["files"]))
commit()
return "", 204
def put(self):
"""Ingest jobs."""
record_input()
messages = [message
for message in request.json
if message["data"]["state"] == "exited"]
for message in messages:
data = message["data"]
queue = get_or_create(Queue, name=data["queue"])
owner = get_or_create(Owner, name=data["owner"])
job = get_or_create(Job, job_id=data["jobid"])
job.name = data["jobname"]
job.queue = queue
job.owner = owner
job.start = data["start"]
job.end = data["end"]
total_cores = 0
for hostname, slots in data["exec_host"].items():
host = get_or_create(Host, name=hostname)
get_or_create(Allocation, job=job, host=host, cores=len(slots))
total_cores += len(slots)
job.cores = total_cores
job.cpu_seconds = total_cores * (data["end"] - data["start"])
commit()
return "", 204
def ingest(self):
"""Ingest jobs."""
messages = [
message for message in request.get_json(force=True)
if message["data"].get("state") == "exited"
]
for message in messages:
data = message["data"]
queue = get_or_create(Queue, name=data["queue"])
owner = get_or_create(Owner, name=data["owner"])
job = get_or_create(Job, job_id=data["jobid"])
job.name = data["jobname"]
job.queue = queue
job.owner = owner
job.start = data["start"]
job.end = data["end"]
total_cores = 0
for hostname, slots in data["exec_host"].items():
host = get_or_create(Host, name=hostname)
get_or_create(Allocation, job=job, host=host, cores=len(slots))
total_cores += len(slots)
job.cores = total_cores
job.cpu_seconds = total_cores * (data["end"] - data["start"])
commit()
return "", 204
def cache(model, **kwargs):
return get_or_create(model, **kwargs)
def cache(model, **kwargs):
return get_or_create(model, **kwargs)
def ingest(self):
"""Ingest snapshots."""
for message in request.get_json(force=True):
data = message["data"]
snapshot = Snapshot(ts=data["timestamp"])
add(snapshot)
for entry in data["organisations"]:
organisation = get_or_create(Organisation,
insightly_id=entry["id"])
organisation.name = entry["name"]
for entry in data["contacts"]:
person = get_or_create(Person, insightly_id=entry["id"])
person.first_name = entry["first_name"]
person.last_name = entry["last_name"]
if entry["username"]:
username = get_or_create(Username,
username=entry["username"])
get_or_create(PersonUsername,
snapshot=snapshot,
person=person,
username=username)
if entry["email"]:
for address in entry["email"]:
email = get_or_create(Email, address=address)
get_or_create(PersonEmail,
snapshot=snapshot,
person=person,
email=email)
if entry["organisations"]:
for insightly_id in entry["organisations"]:
organisation = get_or_create(Organisation,
insightly_id=insightly_id)
get_or_create(Membership,
snapshot=snapshot,
organisation=organisation,
person=person)
commit()
return "", 204
def put(self):
"""Ingest snapshots."""
record_input()
for message in request.json:
data = message["data"]
snapshot = Snapshot(ts=data["timestamp"])
add(snapshot)
for entry in data["organisations"]:
organisation = get_or_create(Organisation,
insightly_id=entry["id"])
organisation.name = entry["name"]
for entry in data["contacts"]:
person = get_or_create(Person, insightly_id=entry["id"])
person.first_name = entry["first_name"]
person.last_name = entry["last_name"]
if entry["username"]:
username = get_or_create(Username,
username=entry["username"])
get_or_create(PersonUsername,
snapshot=snapshot,
person=person,
username=username)
if entry["email"]:
for address in entry["email"]:
email = get_or_create(Email, address=address)
get_or_create(PersonEmail,
snapshot=snapshot,
person=person,
email=email)
if entry["organisations"]:
for insightly_id in entry["organisations"]:
organisation = get_or_create(Organisation,
insightly_id=insightly_id)
get_or_create(Membership,
snapshot=snapshot,
organisation=organisation,
person=person)
commit()
return "", 204
