def get_pycapnpstuff():
"return capnp adapted message factories for StuffToTest schema"
# loads the StuffToTest schema
import capnp
capnp.remove_import_hook()
schmod = capnp.load(join(PROJECT_ROOT,'idl/StuffToTest.capnp'))
# contruct schema
schema = SchemaContainer('pycapnp', capnp.__version__)
context = {}
for message, fieldnames in FNIDX:
# load capnp Struct class
structClass = getattr(schmod,message)
# contruct Struct proxy object
structProxy = CapNpStructProxy(structClass, fieldnames, context)
# add proxy to schema
setattr(schema, message, structProxy)
# update context for it to hold new structProxy reference
context[message] = structProxy
return schema
def test_remove_import_hook():
capnp.add_import_hook([this_dir])
capnp.remove_import_hook()
if 'addressbook_capnp' in sys.modules:
del sys.modules['addressbook_capnp'] # hack to deal with it being imported already
with pytest.raises(ImportError):
import addressbook_capnp
def make_schemas_module():
"""
small hack to make the schemas available in ``sys.modules``
"""
import os
import sys
capnp.remove_import_hook()
pkg_path = os.path.dirname(os.path.realpath(__file__))
schemas_path = os.path.join(pkg_path, 'schemas.capnp')
sys.modules['schemas'] = capnp.load(
encoding.ensure_native_str(schemas_path))
''' Cap'n Proto FFI tools for Python Cap'n Proto is a fast binary format which is being used to pass messages across the FFI boundary. This allows easy declaration and modification of complex messages that get passed between function boundaries. There are other possible ways of doing the same thing but Cap'n Proto is the best candidate at this time for spporting Python 2, 3 and Rust at the same time. Cap'n Proto has the advantage that it also defines interfaces which may be used later. This feature currently is not being used in this library. ''' import cffi import capnp import functools from types import FunctionType capnp.remove_import_hook() def external(rtype): ''' This decorator declares a function as being external (to python) :param rtype: the string value of the return type from the capnproto schema ''' def _external(func): def callme(self, imessage): msgbytes = imessage.to_bytes() rvalue = getattr(self.lib, func.__name__)(msgbytes, len(msgbytes)) return getattr(self.MessageType, rtype).from_bytes(b''.join( rvalue.values[i] for i in range(rvalue.len) )) callme.__name__ = func.__name__ callme.__doc__ = func.__doc__
from pyspark.mllib.classification import LogisticRegressionModel, LogisticRegressionWithSGD
from pyspark.mllib.tree import RandomForestModel
import argparse
import socket
import random
import capnp
import os
from numpy.random import exponential as expn
from sklearn.externals import joblib
# print(joblib.__version__)
import numpy as np
# from collections import OrderedDict
from sklearn import linear_model as lm
import sklearn.svm as svm
capnp.remove_import_hook()
feature_capnp = capnp.load(
os.path.abspath('../../clipper_server/schema/feature.capnp'))
from sample_feature import TestFeature
EXPN_SCALE_PARAM = 5.0
# import graphlab as gl
# def load_gl_model(local_path):
# return gl.load_model(local_path)
# import caffe
def load_scikit_model(pickle_path):
def action():
import CloudscalerLibcloud
import os
import capnp
redises = {}
now = datetime.utcnow()
month = now.month
hour = now.hour
day = now.day
year = now.year
capnp.remove_import_hook()
schemapath = os.path.join(
os.path.dirname(CloudscalerLibcloud.__file__),
"schemas",
"resourcemonitoring.capnp",
)
cloudspace_capnp = capnp.load(schemapath)
nodecl = j.clients.osis.getCategory(j.core.osis.client, "system", "node")
imagecl = j.clients.osis.getCategory(j.core.osis.client, "cloudbroker", "image")
stackcl = j.clients.osis.getCategory(j.core.osis.client, "cloudbroker", "stack")
vcl = j.clients.osis.getNamespace("vfw")
virtualfirewalls = {
vfw["id"]: vfw
for vfw in vcl.virtualfirewall.search({"gid": j.application.whoAmI.gid})[1:]
}
nodes = {
node["id"]: node
for node in nodecl.search({"gid": j.application.whoAmI.gid})[1:]
}
stacks = {
stack["id"]: stack
for stack in stackcl.search({"gid": j.application.whoAmI.gid})[1:]
}
images_list = imagecl.search({"$fields": ["id", "name"]})[1:]
images_dict = {}
for image in images_list:
images_dict[image["id"]] = image["name"]
gid = j.application.whoAmI.gid
# nid = j.application.whoAmI.nid
cached_accounts = get_cached_accounts()
def get_node_redis(node, port=9999):
redis = redises.get(node["id"])
if not redis:
for nicinfo in node["netaddr"]:
if nicinfo["name"] == "backplane1":
ip = nicinfo["ip"][0]
break
else:
return None
redis = j.clients.redis.getRedisClient(ip, port)
redises[node["id"]] = redis
return redis
for account_id, cloudspaces_dict in cached_accounts.items():
folder_name = "/opt/jumpscale7/var/resourcetracking/active/%s/%s/%s/%s/%s" % (
account_id,
year,
month,
day,
hour,
)
j.do.createDir(folder_name)
for cloudspace_id, cs in cloudspaces_dict.items():
vms = cs["vms"]
cloudspace = cloudspace_capnp.CloudSpace.new_message()
cloudspace.accountId = account_id
cloudspace.cloudSpaceId = cloudspace_id
if cs["status"] == "DEPLOYED" and cs["networkId"] in virtualfirewalls:
networkId = hex(cs["networkId"])
net = virtualfirewalls[cs["networkId"]]
nid = net["nid"]
node = nodes[nid]
redis = get_node_redis(node)
data = dict(gid=gid, nid=nid, id=networkId)
publicTX = get_last_hour_val(
redis,
"stats:{gid}_{nid}:[email protected]{id}".format(
**data
),
)
publicRX = get_last_hour_val(
redis,
"stats:{gid}_{nid}:[email protected]{id}".format(
**data
),
)
spaceRX = get_last_hour_val(
redis,
"stats:{gid}_{nid}:[email protected]{id}".format(
**data
),
)
spaceTX = get_last_hour_val(
redis,
"stats:{gid}_{nid}:[email protected]{id}".format(
**data
),
)
else:
publicTX = publicRX = spaceRX = spaceTX = 0
machines = cloudspace.init("machines", len(vms) + 1)
m = machines[0]
m.type = "routeros"
nics = m.init("networks", 2)
nic1 = nics[0]
nic1.tx = publicTX
nic1.tx = publicRX
nic1.type = "external"
nic2 = nics[1]
nic2.tx = spaceTX
nic2.rx = spaceRX
nic2.type = "space"
for idx, machine_dict in enumerate(vms):
vm_id = machine_dict["id"]
m = machines[idx + 1]
m.type = "vm"
m.id = vm_id
stack_id = machine_dict.get("stackId", None)
# get Image name
image_name = images_dict.get(machine_dict["imageId"], "")
has_stack = machine_dict["status"] != "HALTED" and stack_id
if has_stack:
# get redis for this stack
stack = stacks[stack_id]
nid = int(stack["referenceId"])
redis = get_node_redis(nodes[nid])
# get CPU
cpu_key = "stats:{gid}_{nid}:machine.CPU.utilisation@virt.{vm_id}".format(
gid=gid, nid=nid, vm_id=vm_id
)
cpu_seconds = get_last_hour_val(redis, cpu_key)
m.cpuMinutes = cpu_seconds / 60
else:
redis = None
m.cpuMinutes = 0
disks_capnp = m.init("disks", len(machine_dict["disks"]))
# calculate iops
for index, disk in enumerate(machine_dict["disks"]):
disk_id = disk["id"]
disk_capnp = disks_capnp[index]
disk_capnp.id = disk_id
disk_capnp.size = disk["sizeMax"]
disk_iops_read_key = "stats:{gid}_{nid}:disk.iops.read@virt.{disk_id}".format(
gid=gid, nid=nid, disk_id=disk_id
)
val = get_val(redis, disk_iops_read_key)
disk_capnp.iopsRead = val.get("h_last", 0)
disk_capnp.iopsReadMax = val.get("h_last_max", 0)
disk_iops_write_key = "stats:{gid}_{nid}:disk.iops.write@virt.{disk_id}".format(
gid=gid, nid=nid, disk_id=disk_id
)
val = get_val(redis, disk_iops_write_key)
disk_capnp.iopsWrite = val.get("h_last", 0)
disk_capnp.iopsWriteMax = val.get("h_last_max", 0)
# Calculate Network tx and rx
nics = m.init("networks", len(machine_dict["nics"]))
for index, nic in enumerate(machine_dict["nics"]):
mac = nic["macAddress"]
nic_capnp = nics[index]
nic_capnp.type = "external" if nic["type"] == "PUBLIC" else "space"
tx_key = "stats:{gid}_{nid}:network.packets.tx@virt.{mac}".format(
gid=gid, nid=nid, mac=mac
)
rx_key = "stats:{gid}_{nid}:network.packets.rx@virt.{mac}".format(
gid=gid, nid=nid, mac=mac
)
nic_capnp.tx = get_last_hour_val(redis, tx_key)
nic_capnp.rx = get_last_hour_val(redis, rx_key)
m.imageName = image_name
m.mem = machine_dict["memory"]
m.vcpus = machine_dict["vcpus"]
m.status = machine_dict["status"]
# write files to disk
with open("%s/%s.bin" % (folder_name, cloudspace_id), "w+b") as f:
cloudspace.write(f)
def session_capnp():
"""Load out capnproto schema
"""
import capnp
capnp.remove_import_hook()
return capnp.load(os.path.join(capture.__path__[0], 'session.capnp'))
def load_capnp(filename):
src_dir = os.path.dirname(__file__)
capnp.remove_import_hook()
return capnp.load(os.path.join(src_dir, "../capnp", filename))
def __init__(self, schema):
capnp.remove_import_hook()
self.schema = capnp.load(schema)
def main(options):
now = datetime.utcnow()
hour = now.hour
day = now.day
month = now.month
year = now.year
capnp.remove_import_hook()
schemapath = os.path.join(os.path.dirname(CloudscalerLibcloud.__file__),
'schemas', 'resourcemonitoring.capnp')
resources_capnp = capnp.load(schemapath)
root_path = "/opt/jumpscale7/var/resourcetracking"
accounts = listdir(root_path)
book = xlwt.Workbook(encoding='utf-8')
nosheets = True
for account in accounts:
hour = now.hour
file_path = os.path.join(root_path,
str(account), str(year), str(month), str(day),
str(hour), 'account_capnp.bin')
while not os.path.exists(file_path) and hour != 0:
hour -= 1
file_path = os.path.join(root_path, str(account), str(year),
str(month), str(day), str(hour),
'account_capnp.bin')
if not os.path.exists(file_path):
print('Skipping %s' % file_path)
continue
nosheets = False
sheet = book.add_sheet("account %s" % account)
sheet.write(0, 0, 'Cloud Space ID')
sheet.write(0, 1, 'Machine Count')
sheet.write(0, 2, 'Total Memory')
sheet.write(0, 3, 'Total VCPUs')
sheet.write(0, 4, 'Disk Size')
sheet.write(0, 5, 'Disk IOPS Read')
sheet.write(0, 6, 'Disk IOPS Write')
sheet.write(0, 7, 'NICs TX')
sheet.write(0, 8, 'NICs RX')
try:
with open(file_path, 'rb') as f:
account_obj = resources_capnp.Account.read(f)
if options.debug:
pprint.pprint(account_obj.to_dict())
for idx, cs in enumerate(account_obj.cloudspaces):
cs_id = cs.cloudSpaceId
machines = len(cs.machines)
vcpus = 0
mem = 0
disksize = 0
disk_iops_read = 0
disk_iops_write = 0
nics_tx = 0
nics_rx = 0
for machine in cs.machines:
vcpus += machine.vcpus
mem += machine.mem
for disk in machine.disks:
disk_iops_read += disk.iopsRead
disk_iops_write += disk.iopsWrite
disksize += disk.size
for nic in machine.networks:
nics_tx += nic.tx
nics_rx += nic.rx
sheet.write(idx + 1, 0, cs_id)
sheet.write(idx + 1, 1, machines)
sheet.write(idx + 1, 2, mem)
sheet.write(idx + 1, 3, vcpus)
sheet.write(idx + 1, 4, disksize)
sheet.write(idx + 1, 5, disk_iops_read)
sheet.write(idx + 1, 6, disk_iops_write)
sheet.write(idx + 1, 7, nics_tx)
sheet.write(idx + 1, 8, nics_rx)
except Exception as e:
print(e)
if nosheets is False:
book.save('example.xls')
else:
print('No data found')
def init(self, socket):
capnp.remove_import_hook()
self.service = capnp.load('FeedbackCamera.capnp')
self.client = capnp.TwoPartyClient(socket)
self.camera = self.client.bootstrap()
self.camera = self.camera.cast_as(self.service.FeedbackCamera)
def init(self, socket):
capnp.remove_import_hook()
self.service = capnp.load('RobotService.capnp')
self.client = capnp.TwoPartyClient(socket)
self.robot = self.client.bootstrap()
self.robot = self.robot.cast_as(self.service.RobotService)
def main(options):
now = datetime.utcnow()
hour = now.hour
day = now.day
month = now.month
year = now.year
capnp.remove_import_hook()
resources_capnp = capnp.load("resourcemonitoring.capnp")
root_path = environ["DESTINATION_FOLDER"]
accounts = listdir(root_path)
book = xlwt.Workbook(encoding="utf-8")
nosheets = True
for account in accounts:
hour = now.hour
file_path = os.path.join(
root_path,
str(account),
str(year),
str(month),
str(day),
str(hour),
"account_capnp.bin",
)
while not os.path.exists(file_path) and hour != 0:
hour -= 1
file_path = os.path.join(
root_path,
str(account),
str(year),
str(month),
str(day),
str(hour),
"account_capnp.bin",
)
if not os.path.exists(file_path):
print("Skipping %s" % file_path)
continue
nosheets = False
sheet = book.add_sheet("account %s" % account)
sheet.write(0, 0, "Cloud Space ID")
sheet.write(0, 1, "Machine Count")
sheet.write(0, 2, "Total Memory")
sheet.write(0, 3, "Total VCPUs")
sheet.write(0, 4, "Disk Size")
sheet.write(0, 5, "Disk IOPS Read")
sheet.write(0, 6, "Disk IOPS Write")
sheet.write(0, 7, "NICs TX")
sheet.write(0, 8, "NICs RX")
try:
with open(file_path, "rb") as f:
account_obj = resources_capnp.Account.read(f)
if options.debug:
pprint.pprint(account_obj.to_dict())
for idx, cs in enumerate(account_obj.cloudspaces):
cs_id = cs.cloudSpaceId
machines = len(cs.machines)
vcpus = 0
mem = 0
disksize = 0
disk_iops_read = 0
disk_iops_write = 0
nics_tx = 0
nics_rx = 0
for machine in cs.machines:
vcpus += machine.vcpus
mem += machine.mem
for disk in machine.disks:
disk_iops_read += disk.iopsRead
disk_iops_write += disk.iopsWrite
disksize += disk.size
for nic in machine.networks:
nics_tx += nic.tx
nics_rx += nic.rx
sheet.write(idx + 1, 0, cs_id)
sheet.write(idx + 1, 1, machines)
sheet.write(idx + 1, 2, mem)
sheet.write(idx + 1, 3, vcpus)
sheet.write(idx + 1, 4, disksize)
sheet.write(idx + 1, 5, disk_iops_read)
sheet.write(idx + 1, 6, disk_iops_write)
sheet.write(idx + 1, 7, nics_tx)
sheet.write(idx + 1, 8, nics_rx)
except Exception as e:
print(e)
if nosheets is False:
book.save("example.xls")
else:
print("No data found")
def action(gid=None):
"""
Send tar of account data on each enviroment
"""
import CloudscalerLibcloud
import capnp
agentcontroller = j.clients.agentcontroller.get()
cbcl = j.clients.osis.getNamespace("cloudbroker")
jobs = list()
capnp.remove_import_hook()
schemapath = os.path.join(os.path.dirname(CloudscalerLibcloud.__file__),
'schemas')
resources_capnp = capnp.load(
os.path.join(schemapath, 'resourcemonitoring.capnp'))
# schedule command
for location in cbcl.location.search({})[1:]:
jobs.append(
agentcontroller.scheduleCmd(cmdcategory="greenitglobe",
cmdname="collect_account_data",
nid=None,
timeout=60,
roles=['controller'],
gid=location["gid"],
wait=True))
# get return from each job.
accounts = dict()
for job in jobs:
result = agentcontroller.waitJumpscript(job=job)
# read the tar.
c = io.BytesIO()
if result['state'] != 'OK':
raise RuntimeError("Failed to collect account data from grid %s" %
(job['gid']))
result_decoded = base64.decodestring(result['result'])
c.write(result_decoded)
c.seek(0)
tar = tarfile.open(mode="r", fileobj=c)
members = tar.getmembers()
for member in members:
if member.name.endswith(".bin"):
accountid, year, month, day, hour = re.findall(
"opt/jumpscale7/var/resourcetracking/active/([\d]+)/([\d]+)/([\d]+)/([\d]+)/([\d]+)/",
member.name)[0]
datekey = (year, month, day, hour)
accounts.setdefault(accountid, {
datekey: []
}).setdefault(datekey, []).append(member)
for account_id, dates in accounts.iteritems():
for date, members in dates.iteritems():
account = resources_capnp.Account.new_message()
year, month, day, hour = date
account.accountId = int(account_id)
cloudspaces = account.init("cloudspaces", len(members))
for i, member in enumerate(members):
# read the capnp file obj.
binary_content = tar.extractfile(member).read()
cloudspace_obj = resources_capnp.CloudSpace.from_bytes(
binary_content)
cloudspaces[i] = cloudspace_obj
filepath = '/opt/jumpscale7/var/resourcetracking/%s/' % os.path.join(
account_id, year, month, day, hour)
try:
os.makedirs(filepath)
except OSError as err:
if err.errno != 17:
raise
with open(os.path.join(filepath, "account_capnp.bin"), 'w+b') as f:
account.write(f)
