def auth_account(data):
"""Auth account.
data = {'name':, 'pass'}
"""
b_ret = False
s_rsc = '{}/account/{}'.format(etcdc.prefix, data['name'])
try:
r = etcdc.read(s_rsc)
except etcd.EtcdKeyNotFound as e:
log.error(e)
return b_ret
d = ast.literal_eval(r.value)
# check state.
if d.get('state') and d.get('state') == 'Disabled':
return (False, 'Disabled')
elif d.get('state') and d.get('state') == 'Locked':
return (False, 'Locked')
elif 'state' not in d or d.get('state') != 'Enabled':
return (False, 'UnknownError')
# check expireAt
if datetime.utcnow().isoformat() > d.get('expireAt'):
return (False, 'Expired')
s_pass = bcrypt.hashpw(data['pass'].encode(),
d['salt'].encode()).decode()
if s_pass != d['pass']:
return (False, 'PasswordMismatch')
s_role = d['role'] if 'role' in d else 'user'
return (True, s_role)
def admin_auth_account(data):
"""Authenticate admin account.
data = {'name':, 'pass':}
"""
b_ret = False
s_rsc = '{}/account/{}'.format(etcdc.prefix, data['name'])
try:
r = etcdc.read(s_rsc)
except etcd.EtcdKeyNotFound as e:
log.debug(e)
# auth against config file.
s_pass = bcrypt.hashpw(data['pass'].encode(),
ADMIN_SALT.encode()).decode()
log.debug(data['name'] + ':' + s_pass)
if data['name'] == ADMIN_ID and s_pass == ADMIN_PW:
# Add admin entry in etcd and return True.
d = dict()
d['name'] = ADMIN_ID
d['pass'] = data['pass']
d['cn'] = 'Admin'
d['role'] = 'admin'
d['state'] = 'Enabled'
s_expire = (datetime.now() + timedelta(days=90)).isoformat() + 'Z'
d['expireAt'] = s_expire
(b_ret, s_msg) = create_account(d)
return b_ret
else:
return False
else:
d = ast.literal_eval(r.value)
s_pass = bcrypt.hashpw(data['pass'].encode(),
d['salt'].encode()).decode()
return data['name'] == ADMIN_ID and s_pass == d['pass']
def update_account(name, data):
"""Update account.
"""
data = dict((k, v) for k, v in data.items() if v)
t_ret = (False, '')
if not name: # <name> should be specified.
return t_ret
s_rsc = '{}/account/{}'.format(etcdc.prefix, name)
try:
r = etcdc.read(s_rsc)
except etcd.EtcdKeyNotFound as e:
log.error(e)
t_ret = (False, e)
return t_ret
d = ast.literal_eval(r.value)
# Get iso 8601 format datetime for modified timestamp
s_modified = datetime.utcnow().isoformat() + 'Z'
# Put s_modified into data dict.
data['modifiedAt'] = s_modified
d.update(data.items())
try:
etcdc.write(s_rsc, d, prevExist=True)
except etcd.EtcdKeyNotFound as e:
log.error(e)
t_ret = (False, e)
else:
t_ret = (True, 'user {} is updated.'.format(name))
finally:
return t_ret
def update_catalog(name, data):
"""Update catalog
catalogPatchSerializer
"""
data = dict((k, v) for k, v in data.items() if v)
t_ret = (False, '')
if not name:
return t_ret
s_rsc = '{}/catalog/{}'.format(etcdc.prefix, name)
try:
r = etcdc.read(s_rsc)
except etcd.EtcdKeyNotFound as e:
log.error(e)
t_ret = (False, e)
return t_ret
d = ast.literal_eval(r.value)
data['modifiedAt'] = datetime.utcnow().isoformat() + 'Z'
d.update(data.items())
try:
etcdc.write(s_rsc, d, prevExist=True)
except etcd.EtcdKeyAlreadyExist as e:
log.error(e)
t_ret = (False, e)
else:
t_ret = (True, 'catalog {} is updated.'.format(name))
finally:
return t_ret
def get_tenant(edge_name=None, name=None):
"""Get tenant list.
"""
l_tenant = list()
if name is None:
s_rsc = '{}/tenant'.format(etcdc.prefix)
else:
s_rsc = '{}/tenant/{}'.format(etcdc.prefix, name)
try:
r = etcdc.read(s_rsc, recursive=True, sorted=True)
except etcd.EtcdKeyNotFound as e:
log.error(e)
else:
for child in r.children:
if child.value is not None:
d = ast.literal_eval(child.value)
if edge_name is None:
l_tenant.append(d)
else:
if d['edge'] == edge_name:
# Get app list for the tenant
app_rsc = '{}/app'.format(etcdc.prefix)
try:
app_r = etcdc.read(app_rsc, recursive=True)
except etcd.EtcdKeyNotFound as e:
log.error(e)
else:
d['app'] = list()
for child in app_r.children:
if child.value is not None:
app = ast.literal_eval(child.value)
if d['name'] == app['tenant']:
d_tmp = {
'name': app['name'],
'logo': app['cat_logo'],
'type': app['cat_type'],
'user': app['user'],
'status': app['status'],
'desc': app['desc']
}
d['app'].append(d_tmp)
d['app'] = sorted(d['app'],
key=lambda k: k['name'])
l_tenant.append(d)
finally:
return l_tenant
def user_password_account(data):
"""Modify user account password.
etcd_key: <ETCD_PREFIX>/account/<name>
data: {'name': , 'pass': , 'pass1': , 'pass2': }
"""
t_ret = (False, '')
s_rsc = '{}/account/{}'.format(etcdc.prefix, data['name'])
try:
r = etcdc.read(s_rsc)
except etcd.EtcdKeyNotFound as e:
log.error(e)
return (False, 'EtcdKeyNotFound')
d = ast.literal_eval(r.value)
# check if data['pass'] == d['pass']
s_pass = bcrypt.hashpw(data['pass'].encode(),
d['salt'].encode()).decode()
if s_pass != d['pass']: # current password mismatch
return (False, 'Current password is not matched.')
# since data['pass'] is matched, overwrite data['pass1'] to data['pass']
# to validate new password.
data['pass'] = data['pass1']
(b_ret, s_msg) = _pass_validate(data)
if not b_ret:
log.debug((b_ret, s_msg))
return (b_ret, s_msg)
# password is okay. go head.
new_data = dict()
s_modified = datetime.utcnow().isoformat() + 'Z'
data['modifiedAt'] = s_modified
# Put d['pass'] to oldpass entry.
if 'oldpass' in d:
d['oldpass'].append(d['pass'])
else:
d['oldpass'] = [d['pass']]
# Create new hashed password.
bytes_salt = bytes(d['salt'], 'utf-8')
d['pass'] = bcrypt.hashpw(str.encode(data['pass']),
bytes_salt).decode()
s_rsc = '{}/account/{}'.format(etcdc.prefix, data['name'])
try:
etcdc.write(s_rsc, d, prevExist=True)
except etcd.EtcdKeyNotFound as e:
log.error(e)
t_ret = (False, e)
else:
t_ret = (True, 'user {} password is modified.'.format(data['name']))
finally:
return t_ret
def operate_app(name, action):
"""Run app."""
if not (name and action):
return (False, 'App name and action should be specified.')
t_ret = (False, '')
# Get edge from app
s_rsc = '{}/app/{}'.format(etcdc.prefix, name)
try:
r = etcdc.read(s_rsc)
except etcd.EtcdKeyNotFound as e:
log.error(e)
t_ret = (False, e)
return t_ret
else:
d = ast.literal_eval(r.value)
s_edge = d['edge']
# Get endpoint from s_edge.
s_rsc = '{}/edge/{}'.format(etcdc.prefix, s_edge)
try:
r = etcdc.read(s_rsc)
except etcd.EtcdKeyNotFound as e:
log.error(e)
t_ret = (False, e)
return t_ret
else:
d = ast.literal_eval(r.value)
s_endpoint = d['endpoint']
# Run an app.
CMD = "/usr/local/bin/kubectl-virt --kubeconfig " + PROJECT_ROOT + '/' + d['endpoint'] \
+ ' {} '.format(action) + name
t_ret = cmd(CMD, 5, False)
log.debug(t_ret)
return t_ret
def get_tenant_info(name):
"""Get tenant info
"""
d_tenant = dict()
if name is None:
return (False, 'tenant name should be specified.')
s_rsc = '{}/tenant/{}'.format(etcdc.prefix, name)
try:
r = etcdc.read(s_rsc)
except etcd.EtcdKeyNotFound as e:
log.error(e)
else:
d_tenant = ast.literal_eval(r.value)
finally:
return (True, d_tenant)
def get_dashboard(name=None):
"""Get dashboard information
"""
l_edge = list()
if name is None:
s_rsc = '{}/edge'.format(etcdc.prefix)
else:
s_rsc = '{}/edge/{}'.format(etcdc.prefix, name)
try:
r = etcdc.read(s_rsc, recursive=True, sorted=True)
except etcd.EtcdKeyNotFound as e:
log.error(e)
else:
for child in r.children:
d = ast.literal_eval(child.value)
l_edge.append(d)
finally:
return l_edge
def get_catalog(name=None):
"""Get catalog list.
"""
l_catalog = list()
if name is None:
s_rsc = '{}/catalog'.format(etcdc.prefix)
else:
s_rsc = '{}/catalog/{}'.format(etcdc.prefix, name)
try:
r = etcdc.read(s_rsc, recursive=True, sorted=True)
except etcd.EtcdKeyNotFound as e:
log.error(e)
else:
for child in r.children:
if child.value is not None:
d = ast.literal_eval(child.value)
l_catalog.append(d)
finally:
return l_catalog
def get_account(name=None):
"""Get account list.
"""
l_account = list()
if name is None: # get all account.
s_rsc = '{}/account'.format(etcdc.prefix)
else:
s_rsc = '{}/account/{}'.format(etcdc.prefix, name)
try:
r = etcdc.read(s_rsc, recursive=True, sorted=True)
except etcd.EtcdKeyNotFound as e:
log.error(e)
else:
for child in r.children:
if child.value is not None:
# need to use ast to convert str to dict.
d = ast.literal_eval(child.value)
l_account.append(d)
finally:
return l_account
def _pass_validate(data):
"""password validation
* check if pass is not in old pass entry.
"""
s_rsc = '{}/account/{}'.format(etcdc.prefix, data['name'])
try:
r = etcdc.read(s_rsc)
except etcd.EtcdKeyNotFound as e:
log.error(e)
return (False, 'EtcdKeyNotFound')
d = ast.literal_eval(r.value)
s_pass = bcrypt.hashpw(data['pass'].encode(),
d['salt'].encode()).decode()
if s_pass == d['pass']:
return (False, 'NewPasswordSameAsCurrentPassword')
if 'oldpass' in d:
for s_oldpass in d['oldpass']:
if s_oldpass == s_pass:
return (False, 'PasswordPreviouslyUsed')
return (True, 'PasswordMatched')
def get_app(name=None, user=None):
"""Get app list.
"""
l_app = list()
if name is None:
s_rsc = '{}/app'.format(etcdc.prefix)
else:
s_rsc = '{}/app/{}'.format(etcdc.prefix, name)
try:
r = etcdc.read(s_rsc, recursive=True, sorted=True)
except etcd.EtcdKeyNotFound as e:
log.error(e)
else:
for child in r.children:
if child.value is not None:
d = ast.literal_eval(child.value)
if user is None:
l_app.append(d)
else:
if user == d['user']:
l_app.append(d)
finally:
return l_app
def create_app(data):
"""Create app
appPostSerializer
"""
if not data.get('name'):
return (False, 'App name should be specified.')
t_ret = (False, '')
s_uuid = str(uuid.uuid4())
s_created = datetime.utcnow().isoformat() + 'Z'
data['createdAt'] = s_created
data['uid'] = s_uuid
# Get catalog information from data['catalog']
s_catalog_rsc = '{}/catalog/{}'.format(etcdc.prefix, data['catalog'])
try:
r = etcdc.read(s_catalog_rsc)
except etcd.EtcdKeyNotFound as e:
log.error(e)
return t_ret
else:
if r.value is not None:
d = ast.literal_eval(r.value)
log.debug(d)
data['cpu_spec'] = d['cpu_spec']
data['mem_spec'] = d['mem_spec']
data['disk_spec'] = d['disk_spec'] + 1
data['image_url'] = d['image_url']
# Get type(vm or container) from data['catalog']
s_cat_rsc = '{}/catalog/{}'.format(etcdc.prefix, data['catalog'])
try:
r = etcdc.read(s_cat_rsc)
except etcd.EtcdKeyNotFound as e:
log.error(e)
return t_ret
else:
if r.value is not None:
d = ast.literal_eval(r.value)
log.debug(d)
data['cat_type'] = d['type']
data['cat_name'] = d['name']
else:
return t_ret
# Get tenant from data['user']
s_user_rsc = '{}/account/{}'.format(etcdc.prefix, data['user'])
try:
r = etcdc.read(s_user_rsc)
except etcd.EtcdKeyNotFound as e:
log.error(e)
return t_ret
else:
if r.value is not None:
d = ast.literal_eval(r.value)
data['tenant'] = d['tenant']
# Get edge from tenant
s_tenant_rsc = '{}/tenant/{}'.format(etcdc.prefix, d['tenant'])
try:
r = etcdc.read(s_tenant_rsc)
except etcd.EtcdKeyNotFound as e:
log.error(e)
return t_ret
else:
if r.value is not None:
d = ast.literal_eval(r.value)
log.debug(d)
data['edge'] = d['edge']
# Get edge endpoint
s_edge_rsc = '{}/edge/{}'.format(etcdc.prefix, data['edge'])
try:
r = etcdc.read(s_edge_rsc)
except etcd.EtcdKeyNotFound as e:
log.error(e)
return t_ret
else:
if r.value is not None:
d = ast.literal_eval(r.value)
log.debug(d)
data['endpoint'] = d['endpoint']
s_kubeconfig = PROJECT_ROOT + '/' + data['endpoint']
if data['cat_type'] == 'vm':
# Create VM image == app name <username>-<catalog>-<4_random_strings>
# create vm manifest
s_out = render_template('vm.j2', d=data)
s_tpl = '{}/{}_vm.yaml'.format(KUBEMANIFEST, data['name'])
try:
with open(s_tpl, 'w') as f:
f.write(s_out)
except Exception as e:
log.error(e)
return (False, 'Cannot create a manifest file - {}.'.format(s_tpl))
else:
# Run kubectl apply
s_cmd = '{} --kubeconfig={} '.format(KUBECTL, s_kubeconfig) + \
'apply -f {}'.format(s_tpl)
log.debug(s_cmd)
Popen(s_cmd, shell=True)
# create vm service manifest
s_out = render_template('svc.j2', d=data)
s_tpl = '{}/{}_svc.yaml'.format(KUBEMANIFEST, data['name'])
try:
with open(s_tpl, 'w') as f:
f.write(s_out)
except Exception as e:
log.error(e)
return (False, 'Cannot create a manifest file - {}.'.format(s_tpl))
else:
# Run kubectl apply
s_cmd = '{} --kubeconfig={} '.format(KUBECTL, s_kubeconfig) + \
'apply -f {}'.format(s_tpl)
Popen(s_cmd, shell=True)
elif data['cat_type'] == 'container':
# Install using helm
s_cmd = '{}/{}.sh install {} {}'.\
format(BIN_DIR, data['cat_name'], s_kubeconfig, data['name'])
log.debug(s_cmd)
Popen(s_cmd, shell=True)
s_rsc = '{}/app/{}'.format(etcdc.prefix, data['name'])
try:
etcdc.write(s_rsc, data, prevExist=False)
except etcd.EtcdKeyAlreadyExist as e:
log.error(e)
t_ret = (False, e)
else:
t_ret = (True, 'App {} is created.'.format(data['name']))
finally:
return t_ret
def connect_app(name):
"""Return app connection URL."""
if not name:
return (False, 'App name should be specified.')
t_ret = (False, '')
# Get user from app
s_rsc = '{}/app/{}'.format(etcdc.prefix, name)
try:
r = etcdc.read(s_rsc)
except etcd.EtcdKeyNotFound as e:
log.error(e)
t_ret = (False, e)
return t_ret
else:
d = ast.literal_eval(r.value)
s_user = d['user']
if d['cat_type'] == 'vm':
# Get tenant from user
s_rsc = '{}/account/{}'.format(etcdc.prefix, s_user)
try:
r = etcdc.read(s_rsc)
except etcd.EtcdKeyNotFound as e:
log.error(e)
t_ret = (False, e)
return t_ret
else:
d = ast.literal_eval(r.value)
s_tenant = d['tenant']
# Get edge from tenant
s_rsc = '{}/tenant/{}'.format(etcdc.prefix, s_tenant)
try:
r = etcdc.read(s_rsc)
except etcd.EtcdKeyNotFound as e:
log.error(e)
t_ret = (False, e)
return t_ret
else:
d = ast.literal_eval(r.value)
s_edge = d['edge']
# Get broker from edge
s_rsc = '{}/edge/{}'.format(etcdc.prefix, s_edge)
try:
r = etcdc.read(s_rsc)
except etcd.EtcdKeyNotFound as e:
log.error(e)
t_ret = (False, e)
return t_ret
else:
d = ast.literal_eval(r.value)
#s_broker = d['broker'] + '/guacamole/#/client/' + name
s_broker = d['broker'] + '/ktedge/#/client/' + name
elif d['cat_type'] == 'container':
# Get .status.loadBalancer.ingress[0].ip
# for now just return harden.iorchard.co.kr:40080
s_broker = 'harden.iorchard.co.kr:40080'
t_ret = (True, s_broker)
return t_ret
def cli():
"""Management script for collector."""
while True:
try:
# Get the whole information on each edge.
l_edge = list()
s_rsc = '{}/edge'.format(etcdc.prefix)
try:
r = etcdc.read(s_rsc, recursive=True)
except etcd.EtcdKeyNotFound as e:
log.error(e)
else:
for child in r.children:
l_app = list()
d = ast.literal_eval(child.value)
# get hosts
print(PROJECT_ROOT + '/' + d['endpoint'])
l_hosts = kube_list_node(PROJECT_ROOT + '/' +
d['endpoint'])
d['hosts'] = len(l_hosts)
d_nodes = dict() # {'name': 'ip', ...}
for item in l_hosts:
d_nodes[item.metadata.
name] = item.status.addresses[0].address
# log.debug(d_nodes)
# get # of tenants and apps
l_tenants = get_tenant(d['name'])
d['tenants'] = len(l_tenants)
d['apps'] = 0
for e in l_tenants:
if 'app' in e:
d['apps'] += len(e['app'])
d['cpu'] = 0
d['memory'] = 0
i_total_cores = 0
i_total_memory = 0
i_total_storage = 0
for h in l_hosts:
i_total_cores += int(h.status.capacity['cpu'])
i_total_memory += int(
h.status.capacity['memory'].replace('Ki', ''))
d['tot_cpu'] = i_total_cores
d['tot_mem'] = int(i_total_memory / (1024 * 1024))
# Get loadavg and free mem
if d['name'] == 'edge1':
ssh_server = 'harden.iorchard.co.kr'
elif d['name'] == 'edge2':
ssh_server = 'durant.iorchard.co.kr'
RSC = 'ssh -p42544 {} get_rsc.sh'.format(ssh_server)
(b_res, s_out) = cmd(RSC, 3, False)
l = s_out.split("\n")
d['used_cpu'] = (float(l[0]) + float(l[1]) + float(l[2]))
avail_mem = (int(l[3]) + int(l[4]) + int(l[5])) / (1024 *
1024)
d['used_mem'] = d['tot_mem'] - avail_mem
d['cpu'] = int(d['used_cpu'] / d['tot_cpu'] * 100)
d['memory'] = int(d['used_mem'] / d['tot_mem'] * 100)
# ceph storage
CEPH = "kubectl --kubeconfig " + PROJECT_ROOT + '/' \
+ d['endpoint'] + " -n rook-ceph exec -it " \
+ "$(kubectl --kubeconfig " + PROJECT_ROOT + '/' \
+ d['endpoint'] + " -n rook-ceph get po " \
+ "-l app=rook-ceph-tools " \
+ "-o jsonpath='{.items[0].metadata.name}') -- " \
+ "ceph df --format json"
(b_res, s_out) = cmd(CEPH, 3, False)
print(s_out)
d['status'] = 'Healthy' if b_res else 'Unhealthy'
d_stor = ast.literal_eval(s_out)
d['tot_stor'] = int(d_stor['stats']['total_bytes'] /
pow(1024, 3))
d['used_stor'] = int(d_stor['stats']['total_used_bytes'] /
pow(1024, 3))
d['storage'] = int(d['used_stor'] / d['tot_stor'] * 100)
# Update etcd status
try:
s = '{}/edge/{}'.format(etcdc.prefix, d['name'])
# log.debug(d)
etcdc.write(s, d, prevExist=True)
except etcd.EtcdKeyNotFound as e:
log.error(e)
# Update app status
s_app = '{}/app'.format(etcdc.prefix)
try:
r_app = etcdc.read(s_app, recursive=True)
except etcd.EtcdKeyNotFound as e:
log.error(e)
else:
for app_child in r_app.children:
if app_child.value is not None:
d_app = dict()
app = ast.literal_eval(app_child.value)
if app['edge'] == d['name']:
d_app['name'] = app['name']
d_app['username'] = GUAC_USER
d_app['password'] = GUAC_PASS
# Get catalog info.
s_cat = '{}/catalog/{}'.format(
etcdc.prefix, app['catalog'])
try:
r_cat = etcdc.read(s_cat)
except etcd.EtcdKeyNotFound as e:
log.error(e)
else:
cat = ast.literal_eval(r_cat.value)
app['cat_type'] = cat['type']
app['cat_name'] = cat['name']
app['cat_logo'] = cat['logo']
# Get app status
if app['cat_type'] == 'vm':
# first, look at DataVolume status of app.
CMD = "kubectl --kubeconfig " + PROJECT_ROOT + '/' \
+ d['endpoint'] + ' get dv ' \
+ app['name'] \
+ " -o jsonpath='{range .status}{.phase},{.progress}{end}'"
(b_res, s_out) = cmd(CMD, 5, False)
l_out = s_out.split(',')
if l_out[0] == 'Succeeded':
# Get vm status of app
CMD = "kubectl --kubeconfig " + PROJECT_ROOT \
+ '/' \
+ d['endpoint'] + ' get vm ' \
+ app['name'] \
+ " -o jsonpath='{.status.ready}'"
(b_res, s_out) = cmd(CMD, 5, False)
if b_res and s_out == 'true':
# update app status 'running'.
app.update({'status': 'running'})
if app['edge'] == d['name']:
# Get where app is running.
CMD = "kubectl --kubeconfig " \
+ PROJECT_ROOT + '/' \
+ d['endpoint'] + ' get vmi ' \
+ app['name'] \
+ " -o jsonpath='{.status.nodeName}'"
(b_res,
s_out) = cmd(CMD, 5, False)
if b_res:
d_app[
'hostname'] = d_nodes[
s_out]
# Get nodeport for app.
CMD = "kubectl --kubeconfig " \
+ PROJECT_ROOT + '/' \
+ d['endpoint'] + ' get svc ' \
+ app['name'] \
+ " -o jsonpath='{.spec.ports[0].nodePort}'"
(b_res,
s_out) = cmd(CMD, 5, False)
if b_res:
d_app['port'] = s_out
else:
# update app status 'stopped'
app.update({'status': 'stopped'})
elif l_out[0] == 'ImportInProgress':
# update app status 'building' and
app.update({
'status':
'building ({})'.format(l_out[1])
})
elif app['cat_type'] == 'container':
app.update({'status': 'running'})
try:
s = '{}/app/{}'.format(
etcdc.prefix, app['name'])
# log.debug(app)
etcdc.write(s, app, prevExist=True)
except etcd.EtcdKeyNotFound as e:
log.error(e)
if 'port' in d_app:
l_app.append(d_app)
# render guac-config.j2 and copy it to guac broker server
log.debug(l_app)
template = env.get_template('broker.j2')
s_out = template.render(l_app=l_app)
s_tmp = '/tmp/{}.broker'.format(d['name'])
try:
with open(s_tmp, 'w') as f:
f.write(s_out)
except Exception as e:
log.error(e)
else:
CMD = "scp " \
+ "-P42544 {} {}".format(s_tmp, d['broker_ip']) \
+ ":/etc/guacamole/noauth-config.xml"
log.debug(CMD)
(b_res, s_out) = cmd(CMD, 5, False)
if b_res:
d_app['port'] = s_out
l_edge.append(d)
# log.debug(l_edge)
log.debug(l_app)
time.sleep(1)
except:
log.error('unknown error')