def send_beans_email(to_address, username, beans):
email_from_address = settings.get('EMAIL_FROM_ADDRESS')
if (email_from_address in ['\'\'', '\"\"', '']):
return
#text = 'Dear '+ username + ':\n' + ' Your beans in docklet are less than' + beans + '.'
text = '<html><h4>Dear '+ username + ':</h4>'
text += '''<p> Your beans in <a href='%s'>docklet</a> are %d now. </p>
<p> If your beans are less than or equal to 0, all your worksapces will be stopped.</p>
<p> Please apply for more beans to keep your workspaces running by following link:</p>
<p> <a href='%s/beans/application/'>%s/beans/application/</p>
<br>
<p> Note: DO NOT reply to this email!</p>
<br><br>
<p> <a href='http://docklet.unias.org'>Docklet Team</a>, SEI, PKU</p>
''' % (env.getenv("PORTAL_URL"), beans, env.getenv("PORTAL_URL"), env.getenv("PORTAL_URL"))
text += '<p>'+ str(datetime.datetime.now()) + '</p>'
text += '</html>'
subject = 'Docklet beans alert'
msg = MIMEMultipart()
textmsg = MIMEText(text,'html','utf-8')
msg['Subject'] = Header(subject, 'utf-8')
msg['From'] = email_from_address
msg['To'] = to_address
msg.attach(textmsg)
s = smtplib.SMTP()
s.connect()
s.sendmail(email_from_address, to_address, msg.as_string())
s.close()
def send_remind_activating_email(username):
nulladdr = ['\'\'', '\"\"', '']
if (email_from_address in nulladdr or admin_email_address in nulladdr):
return
#text = 'Dear '+ username + ':\n' + ' Your account in docklet has been activated'
text = '<html><h4>Dear ' + 'admin' + ':</h4>'
text += '''<p> An activating request for %s in <a href='%s'>%s</a> has been sent</p>
<p> Please check it !</p>
<br/><br/>
<p> Docklet Team, SEI, PKU</p>
''' % (username, env.getenv("PORTAL_URL"),
env.getenv("PORTAL_URL"))
text += '<p>' + str(datetime.utcnow()) + '</p>'
text += '</html>'
subject = 'An activating request in Docklet has been sent'
msg = MIMEMultipart()
textmsg = MIMEText(text, 'html', 'utf-8')
msg['Subject'] = Header(subject, 'utf-8')
msg['From'] = email_from_address
msg['To'] = admin_email_address
msg.attach(textmsg)
s = smtplib.SMTP()
s.connect()
s.sendmail(email_from_address, admin_email_address, msg.as_string())
s.close()
def send_activated_email(to_address, username):
if (email_from_address == '\'\''):
return
#text = 'Dear '+ username + ':\n' + ' Your account in docklet has been activated'
text = '<html><h4>Dear ' + username + ':</h4>'
text += '''<p> Your account in <a href='%s'>%s</a> has been activated</p>
<p> Please enjoy !</p>
<br/>
<p> Please No Reply !</p>
<br/><br/>
<p> Docklet Team, SEI, PKU</p>
''' % (env.getenv("PORTAL_URL"), env.getenv("PORTAL_URL"))
text += '<p>' + str(datetime.utcnow()) + '</p>'
text += '</html>'
subject = 'Docklet account activated'
msg = MIMEMultipart()
textmsg = MIMEText(text, 'html', 'utf-8')
msg['Subject'] = Header(subject, 'utf-8')
msg['From'] = email_from_address
msg['To'] = to_address
msg.attach(textmsg)
s = smtplib.SMTP()
s.connect()
s.sendmail(email_from_address, to_address, msg.as_string())
s.close()
def send_activated_email(to_address, username):
if (email_from_address in ['\'\'', '\"\"', '']):
return
#text = 'Dear '+ username + ':\n' + ' Your account in docklet has been activated'
text = '<html><h4>Dear ' + username + ':</h4>'
text += '''<p> Your account in <a href='%s'>%s</a> has been activated</p>
<p> Enjoy your personal workspace in the cloud !</p>
<br>
<p> Note: DO NOT reply to this email!</p>
<br><br>
<p> <a href='http://docklet.unias.org'>Docklet Team</a>, SEI, PKU</p>
''' % (env.getenv("PORTAL_URL"), env.getenv("PORTAL_URL"))
text += '<p>' + str(datetime.utcnow()) + '</p>'
text += '</html>'
subject = 'Docklet account activated'
msg = MIMEMultipart()
textmsg = MIMEText(text, 'html', 'utf-8')
msg['Subject'] = Header(subject, 'utf-8')
msg['From'] = email_from_address
msg['To'] = to_address
msg.attach(textmsg)
s = smtplib.SMTP()
s.connect()
s.sendmail(email_from_address, to_address, msg.as_string())
s.close()
def __init__(self, nodemgr, networkmgr, etcdclient, addr, mode):
self.mode = mode
self.nodemgr = nodemgr
self.imgmgr = imagemgr.ImageMgr()
self.networkmgr = networkmgr
self.addr = addr
self.etcd = etcdclient
self.defaultsize = env.getenv("CLUSTER_SIZE")
self.fspath = env.getenv("FS_PREFIX")
logger.info("vcluster start on %s" % (self.addr))
if self.mode == 'new':
logger.info("starting in new mode on %s" % (self.addr))
# check if all clusters data are deleted in httprest.py
clean = True
usersdir = self.fspath + "/global/users/"
for user in os.listdir(usersdir):
if len(os.listdir(usersdir + user + "/clusters")) > 0 or len(
os.listdir(usersdir + user + "/hosts")) > 0:
clean = False
if not clean:
logger.error("clusters files not clean, start failed")
sys.exit(1)
elif self.mode == "recovery":
logger.info("starting in recovery mode on %s" % (self.addr))
self.recover_allclusters()
else:
logger.error("not supported mode:%s" % self.mode)
sys.exit(1)
def send_remind_activating_email(username):
if (email_from_address == '\'\'' or admin_email_address == "\'\'"):
return
#text = 'Dear '+ username + ':\n' + ' Your account in docklet has been activated'
text = '<html><h4>Dear '+ 'admin' + ':</h4>'
text += '''<p> An activating request for %s in <a href='%s'>%s</a> has been sent</p>
<p> Please check it !</p>
<br/>
<p> Please No Reply !</p>
<br/><br/>
<p> Docklet Team, SEI, PKU</p>
''' % (username, env.getenv("PORTAL_URL"), env.getenv("PORTAL_URL"))
text += '<p>'+ str(datetime.utcnow()) + '</p>'
text += '</html>'
subject = 'An activating request in Docklet has been sent'
msg = MIMEMultipart()
textmsg = MIMEText(text,'html','utf-8')
msg['Subject'] = Header(subject, 'utf-8')
msg['From'] = email_from_address
msg['To'] = admin_email_address
msg.attach(textmsg)
s = smtplib.SMTP()
s.connect()
s.sendmail(email_from_address, admin_email_address, msg.as_string())
s.close()
def send_beans_email(to_address, username, beans):
global email_from_address
if (email_from_address in ['\'\'', '\"\"', '']):
return
#text = 'Dear '+ username + ':\n' + ' Your beans in docklet are less than' + beans + '.'
text = '<html><h4>Dear ' + username + ':</h4>'
text += '''<p> Your beans in <a href='%s'>docklet</a> are %d now. </p>
<p> If your beans are less than or equal to 0, all your worksapces will be stopped.</p>
<p> Please apply for more beans to keep your workspaces running by following link:</p>
<p> <a href='%s/beans/application/'>%s/beans/application/</p>
<br>
<p> Note: DO NOT reply to this email!</p>
<br><br>
<p> <a href='http://docklet.unias.org'>Docklet Team</a>, SEI, PKU</p>
''' % (env.getenv("PORTAL_URL"), beans, env.getenv("PORTAL_URL"),
env.getenv("PORTAL_URL"))
text += '<p>' + str(datetime.datetime.now()) + '</p>'
text += '</html>'
subject = 'Docklet beans alert'
msg = MIMEMultipart()
textmsg = MIMEText(text, 'html', 'utf-8')
msg['Subject'] = Header(subject, 'utf-8')
msg['From'] = email_from_address
msg['To'] = to_address
msg.attach(textmsg)
s = smtplib.SMTP()
s.connect()
s.sendmail(email_from_address, to_address, msg.as_string())
s.close()
def send_activated_email(to_address, username):
if (email_from_address in ['\'\'', '\"\"', '']):
return
#text = 'Dear '+ username + ':\n' + ' Your account in docklet has been activated'
text = '<html><h4>Dear '+ username + ':</h4>'
text += '''<p> Your account in <a href='%s'>%s</a> has been activated</p>
<p> Enjoy your personal workspace in the cloud !</p>
<br>
<p> Note: DO NOT reply to this email!</p>
<br><br>
<p> <a href='http://docklet.unias.org'>Docklet Team</a>, SEI, PKU</p>
''' % (env.getenv("PORTAL_URL"), env.getenv("PORTAL_URL"))
text += '<p>'+ str(datetime.utcnow()) + '</p>'
text += '</html>'
subject = 'Docklet account activated'
msg = MIMEMultipart()
textmsg = MIMEText(text,'html','utf-8')
msg['Subject'] = Header(subject, 'utf-8')
msg['From'] = email_from_address
msg['To'] = to_address
msg.attach(textmsg)
s = smtplib.SMTP()
s.connect()
s.sendmail(email_from_address, to_address, msg.as_string())
s.close()
def __init__(self, nodemgr, networkmgr, etcdclient, addr, mode):
self.mode = mode
self.nodemgr = nodemgr
self.imgmgr = imagemgr.ImageMgr()
self.networkmgr = networkmgr
self.addr = addr
self.etcd = etcdclient
self.defaultsize = env.getenv("CLUSTER_SIZE")
self.fspath = env.getenv("FS_PREFIX")
logger.info ("vcluster start on %s" % (self.addr))
if self.mode == 'new':
logger.info ("starting in new mode on %s" % (self.addr))
# check if all clusters data are deleted in httprest.py
clean = True
usersdir = self.fspath+"/global/users/"
for user in os.listdir(usersdir):
if len(os.listdir(usersdir+user+"/clusters")) > 0 or len(os.listdir(usersdir+user+"/hosts")) > 0:
clean = False
if not clean:
logger.error ("clusters files not clean, start failed")
sys.exit(1)
elif self.mode == "recovery":
logger.info ("starting in recovery mode on %s" % (self.addr))
self.recover_allclusters()
else:
logger.error ("not supported mode:%s" % self.mode)
sys.exit(1)
def new_group(group_name, size = "5000", file_path = "/opt/docklet/local/docklet-storage"):
storage = env.getenv("STORAGE")
logger.info("begin initialize lvm group:%s with size %sM" % (group_name,size))
if storage == "file":
#check vg
Ret = sys_run("vgdisplay " + group_name)
if Ret.returncode == 0:
logger.info("lvm group: " + group_name + " already exists, delete it")
Ret = sys_run("vgremove -f " + group_name)
if Ret.returncode != 0:
logger.error("delete VG failed")
#check pv
Ret = sys_run("pvdisplay /dev/loop0")
if Ret.returncode == 0:
Ret = sys_run("pvremove -ff /dev/loop0")
if Ret.returncode != 0:
logger.error("remove pv failed")
#check mountpoint
Ret = sys_run("losetup /dev/loop0")
if Ret.returncode == 0:
logger.info("/dev/loop0 already exists, detach it")
Ret = sys_run("losetup -d /dev/loop0")
if Ret.returncode != 0:
logger.error("losetup -d failed")
#check file_path
if os.path.exists(file_path):
logger.info(file_path + " for lvm group already exists, delete it")
os.remove(file_path)
if not os.path.isdir(file_path[:file_path.rindex("/")]):
os.makedirs(file_path[:file_path.rindex("/")])
sys_run("dd if=/dev/zero of=%s bs=1M seek=%s count=0" % (file_path,size))
sys_run("losetup /dev/loop0 " + file_path)
sys_run("vgcreate %s /dev/loop0" % group_name)
logger.info("initialize lvm group:%s with size %sM success" % (group_name,size))
return True
elif storage == "disk":
disk = env.getenv("DISK")
if disk is None:
logger.error("use disk for story without a physical disk")
return False
#check vg
Ret = sys_run("vgdisplay " + group_name)
if Ret.returncode == 0:
logger.info("lvm group: " + group_name + " already exists, delete it")
Ret = sys_run("vgremove -f " + group_name)
if Ret.returncode != 0:
logger.error("delete VG failed")
sys_run("vgcreate %s %s" % (group_name,disk))
logger.info("initialize lvm group:%s with size %sM success" % (group_name,size))
return True
else:
logger.info("unknown storage type:" + storage)
return False
def __init__(self, addr, etcdclient):
self.addr = addr
self.etcd = etcdclient
self.libpath = env.getenv('DOCKLET_LIB')
self.confpath = env.getenv('DOCKLET_CONF')
self.fspath = env.getenv('FS_PREFIX')
# set jupyter running dir in container
self.rundir = "/home/jupyter"
# set root running dir in container
self.nodehome = "/root"
self.lxcpath = "/var/lib/lxc"
self.imgmgr = imagemgr.ImageMgr()
def __init__(self, addr, etcdclient):
self.addr = addr
self.etcd=etcdclient
self.libpath = env.getenv('DOCKLET_LIB')
self.confpath = env.getenv('DOCKLET_CONF')
self.fspath = env.getenv('FS_PREFIX')
# set jupyter running dir in container
self.rundir = "/home/jupyter"
# set root running dir in container
self.nodehome = "/root"
self.lxcpath = "/var/lib/lxc"
self.imgmgr = imagemgr.ImageMgr()
def __init__(self, username = '******', password = None):
'''
Try to create the database when there is none
initialize 'root' user and 'root' & 'primary' group
'''
try:
User.query.all()
except:
db.create_all()
if password == None:
#set a random password
password = os.urandom(16)
password = b64encode(password).decode('utf-8')
fsdir = env.getenv('FS_PREFIX')
f = open(fsdir + '/local/generated_password.txt', 'w')
f.write("User=%s\nPass=%s\n"%(username, password))
f.close()
sys_admin = User(username, hashlib.sha512(password.encode('utf-8')).hexdigest())
sys_admin.status = 'normal'
sys_admin.nickname = 'root'
sys_admin.description = 'Root_User'
sys_admin.user_group = 'root'
sys_admin.auth_method = 'local'
db.session.add(sys_admin)
path = env.getenv('DOCKLET_LIB')
subprocess.call([path+"/userinit.sh", username])
db.session.commit()
if not os.path.exists(fspath+"/global/sys/quota"):
groupfile = open(fspath+"/global/sys/quota",'w')
groups = []
groups.append({'name':'root', 'quotas':{ 'cpu':'4', 'disk':'2000', 'data':'100', 'memory':'2000', 'image':'10', 'idletime':'24', 'vnode':'8' }})
groups.append({'name':'admin', 'quotas':{'cpu':'4', 'disk':'2000', 'data':'100', 'memory':'2000', 'image':'10', 'idletime':'24', 'vnode':'8'}})
groups.append({'name':'primary', 'quotas':{'cpu':'4', 'disk':'2000', 'data':'100', 'memory':'2000', 'image':'10', 'idletime':'24', 'vnode':'8'}})
groups.append({'name':'foundation', 'quotas':{'cpu':'4', 'disk':'2000', 'data':'100', 'memory':'2000', 'image':'10', 'idletime':'24', 'vnode':'8'}})
groupfile.write(json.dumps(groups))
groupfile.close()
if not os.path.exists(fspath+"/global/sys/quotainfo"):
quotafile = open(fspath+"/global/sys/quotainfo",'w')
quotas = {}
quotas['default'] = 'foundation'
quotas['quotainfo'] = []
quotas['quotainfo'].append({'name':'cpu', 'hint':'the cpu quota, number of cores, e.g. 4'})
quotas['quotainfo'].append({'name':'memory', 'hint':'the memory quota, number of MB , e.g. 4000'})
quotas['quotainfo'].append({'name':'disk', 'hint':'the disk quota, number of MB, e.g. 4000'})
quotas['quotainfo'].append({'name':'data', 'hint':'the quota of data space, number of GB, e.g. 100'})
quotas['quotainfo'].append({'name':'image', 'hint':'how many images the user can save, e.g. 10'})
quotas['quotainfo'].append({'name':'idletime', 'hint':'will stop cluster after idletime, number of hours, e.g. 24'})
quotas['quotainfo'].append({'name':'vnode', 'hint':'how many containers the user can have, e.g. 8'})
quotafile.write(json.dumps(quotas))
quotafile.close()
def mail_notification(self, notify_id):
email_from_address = env.getenv('EMAIL_FROM_ADDRESS')
if (email_from_address in ['\'\'', '\"\"', '']):
return {'success': 'true'}
notify = Notification.query.filter_by(id=notify_id).first()
notify_groups = NotificationGroups.query.filter_by(
notification_id=notify_id).all()
to_addr = []
groups = []
for group in notify_groups:
groups.append(group.group_name)
if 'all' in groups:
users = User.query.all()
for user in users:
to_addr.append(user.e_mail)
else:
for group in notify_groups:
users = User.query.filter_by(user_group=group.group_name).all()
for user in users:
to_addr.append(user.e_mail)
content = notify.content
text = '<html><h4>Dear ' + 'user' + ':</h4>' #user.username + ':</h4>'
text += '''<p> Your account in <a href='%s'>%s</a> has been recieved a notification:</p>
<p>%s</p>
<br>
<p> Note: DO NOT reply to this email!</p>
<br><br>
<p> <a href='http://docklet.unias.org'>Docklet Team</a>, SEI, PKU</p>
''' % (env.getenv("PORTAL_URL"), env.getenv("PORTAL_URL"),
content)
text += '<p>' + str(datetime.utcnow()) + '</p>'
text += '</html>'
subject = 'Docklet Notification: ' + notify.title
msg = MIMEMultipart()
textmsg = MIMEText(text, 'html', 'utf-8')
msg['Subject'] = Header(subject, 'utf-8')
msg['From'] = email_from_address
msg.attach(textmsg)
s = smtplib.SMTP()
s.connect()
for address in to_addr:
try:
msg['To'] = address
s.sendmail(email_from_address, address, msg.as_string())
except Exception as e:
logger.error(e)
s.close()
return {"success": 'true'}
def stop_cluster(self, clustername, username):
[status, info] = self.get_clusterinfo(clustername, username)
if not status:
return [False, "cluster not found"]
if info['status'] == 'stopped':
return [False, 'cluster is already stopped']
if self.distributedgw == 'True':
worker = self.nodemgr.ip_to_rpc(info['proxy_server_ip'])
worker.delete_route("/" + info['proxy_public_ip'] + '/go/' +
username + '/' + clustername)
else:
proxytool.delete_route("/" + info['proxy_public_ip'] + '/go/' +
username + '/' + clustername)
for container in info['containers']:
self.delete_all_port_mapping(username, clustername,
container['containername'])
worker = xmlrpc.client.ServerProxy(
"http://%s:%s" %
(container['host'], env.getenv("WORKER_PORT")))
if worker is None:
return [
False, "The worker can't be found or has been stopped."
]
worker.stop_container(container['containername'])
[status, info] = self.get_clusterinfo(clustername, username)
info['status'] = 'stopped'
info['start_time'] = "------"
infofile = open(
self.fspath + "/global/users/" + username + "/clusters/" +
clustername, 'w')
infofile.write(json.dumps(info))
infofile.close()
return [True, "stop cluster"]
def getalldesc(self):
masterips = self.post_to_all()
res={}
for masterip in masterips:
mastername = getname(masterip)
res[mastername]=env.getenv(mastername+"_desc")
return res
def release_port_mapping(container_name, container_ip, container_port):
global free_ports
global allocated_ports
global ports_lock
if container_name not in allocated_ports.keys():
return [False, "This container does not have a port mapping."]
free_port = allocated_ports[container_name][container_port]
public_ip = env.getenv("PUBLIC_IP")
try:
subprocess.run([
'iptables', '-t', 'nat', '-D', 'PREROUTING', '-p', 'tcp',
'--dport',
str(free_port), "-j", "DNAT", '--to-destination',
'%s:%s' % (container_ip, container_port)
],
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
shell=False,
check=True)
except subprocess.CalledProcessError as suberror:
return [
False,
"release port mapping failed : %s" %
suberror.stdout.decode('utf-8')
]
ports_lock.acquire()
free_ports[free_port] = True
allocated_ports[container_name].pop(container_port)
ports_lock.release()
return [True, ""]
def register(*args, **kwargs):
'''
Usage: register(user = modified_from_newuser())
'''
if (kwargs['user'].username == None or kwargs['user'].username == ''):
return {"success": 'false', "reason": "Empty username"}
user_check = User.query.filter_by(
username=kwargs['user'].username).first()
if (user_check != None and user_check.status != "init"):
#for the activating form
return {"success": 'false', "reason": "Unauthorized action"}
if (user_check != None and (user_check.status == "init")):
db.session.delete(user_check)
db.session.commit()
newuser = kwargs['user']
newuser.password = hashlib.sha512(
newuser.password.encode('utf-8')).hexdigest()
db.session.add(newuser)
db.session.commit()
# if newuser status is normal, init some data for this user
# now initialize for all kind of users
#if newuser.status == 'normal':
path = env.getenv('DOCKLET_LIB')
subprocess.call([path + "/userinit.sh", newuser.username])
return {"success": 'true'}
def acquire_port_mapping(container_name, container_ip, container_port, host_port=None):
global free_ports
global allocated_ports
# if container_name in allocated_ports.keys():
# return [False, "This container already has a port mapping."]
if container_name not in allocated_ports.keys():
allocated_ports[container_name] = {}
elif container_port in allocated_ports[container_name].keys():
return [False, "This container already has a port mapping."]
if container_name == "" or container_ip == "" or container_port == "":
return [False, "Node Name or Node IP or Node Port can't be null."]
#print("acquire_port_mapping1")
free_port = 1
if host_port is not None:
# recover from host_port
free_port = int(host_port)
else:
# acquire new free port
while free_port <= 65535:
if free_ports[free_port]:
break
free_port += 1
if free_port == 65536:
return [False, "No free ports."]
free_ports[free_port] = False
allocated_ports[container_name][container_port] = free_port
public_ip = env.getenv("PUBLIC_IP")
try:
subprocess.run(['iptables','-t','nat','-A','PREROUTING','-p','tcp','--dport',str(free_port),"-j","DNAT",'--to-destination','%s:%s'%(container_ip,container_port)], stdout=subprocess.PIPE, stderr=subprocess.STDOUT, shell=False, check=True)
return [True, str(free_port)]
except subprocess.CalledProcessError as suberror:
return [False, "set port mapping failed : %s" % suberror.stdout.decode('utf-8')]
def _watchnewnode(self):
workerport = env.getenv('WORKER_PORT')
while (True):
time.sleep(0.1)
[status, runlist] = self.etcd.listdir("machines/runnodes")
if not status:
logger.warning("get runnodes list failed from etcd ")
continue
for node in runlist:
nodeip = node['key'].rsplit('/', 1)[1]
if node['value'] == 'waiting':
logger.info("%s want to joins, call it to init first" %
nodeip)
# 'docklet-br' of worker do not need IP Addr. Not need to allocate an IP to it
#if nodeip != self.addr:
# [status, result] = self.networkmgr.acquire_sysips_cidr()
# self.networkmgr.printpools()
# if not status:
# logger.error("no IP for worker bridge, please check network system pool")
# continue
# bridgeip = result[0]
# self.etcd.setkey("network/workbridge", bridgeip)
if nodeip in self.allnodes:
######## HERE MAYBE NEED TO FIX ###############
# here we must use "machines/runnodes/nodeip"
# we cannot use node['key'], node['key'] is absolute
# path, etcd client will append the path to prefix,
# which is wrong
###############################################
self.etcd.setkey("machines/runnodes/" + nodeip,
"init-" + self.mode)
else:
self.etcd.setkey('machines/runnodes/' + nodeip,
"init-new")
elif node['value'] == 'work':
logger.info("new node %s joins" % nodeip)
# setup GRE tunnels for new nodes
if self.addr == nodeip:
logger.debug(
"worker start on master node. not need to setup GRE"
)
else:
logger.debug("setup GRE for %s" % nodeip)
if netcontrol.gre_exists('docklet-br', nodeip):
logger.debug(
"GRE for %s already exists, reuse it" % nodeip)
else:
netcontrol.setup_gre('docklet-br', nodeip)
self.runnodes.append(nodeip)
self.etcd.setkey("machines/runnodes/" + nodeip, "ok")
if nodeip not in self.allnodes:
self.allnodes.append(nodeip)
self.etcd.setkey("machines/allnodes/" + nodeip, "ok")
logger.debug("all nodes are: %s" % self.allnodes)
logger.debug("run nodes are: %s" % self.runnodes)
self.rpcs.append(
xmlrpc.client.ServerProxy("http://%s:%s" %
(nodeip, workerport)))
logger.info("add %s:%s in rpc client list" %
(nodeip, workerport))
def delete_cluster(self, clustername, username, user_info):
[status, info] = self.get_clusterinfo(clustername, username)
if not status:
return [False, "cluster not found"]
if info['status']=='running':
return [False, "cluster is still running, you need to stop it and then delete"]
ips = []
for container in info['containers']:
worker = xmlrpc.client.ServerProxy("http://%s:%s" % (container['host'], env.getenv("WORKER_PORT")))
if worker is None:
return [False, "The worker can't be found or has been stopped."]
worker.delete_container(container['containername'])
ips.append(container['ip'])
logger.info("delete vcluster and release vcluster ips")
self.networkmgr.release_userips(username, ips)
self.networkmgr.printpools()
os.remove(self.fspath+"/global/users/"+username+"/clusters/"+clustername)
os.remove(self.fspath+"/global/users/"+username+"/hosts/"+str(info['clusterid'])+".hosts")
groupname = json.loads(user_info)["data"]["group"]
uid = json.loads(user_info)["data"]["id"]
[status, clusters] = self.list_clusters(username)
if len(clusters) == 0:
self.networkmgr.del_user(username)
self.networkmgr.del_usrgwbr(username, uid, self.nodemgr)
#logger.info("vlanid release triggered")
return [True, "cluster delete"]
def copyImage(self,user,image,token,target):
path = "/opt/docklet/global/images/private/"+user+"/"
'''image_info_file = open(path+"."+image+".info", 'r')
[createtime, isshare] = image_info_file.readlines()
recordshare = isshare
isshare = "unshared"
image_info_file.close()
image_info_file = open(path+"."+image+".info", 'w')
image_info_file.writelines([createtime, isshare])
image_info_file.close()'''
try:
sys_run('ssh root@%s "mkdir -p %s"' % (target,path))
sys_run('scp %s%s.tz root@%s:%s' % (path,image,target,path))
#sys_run('scp %s.%s.description root@%s:%s' % (path,image,target,path))
#sys_run('scp %s.%s.info root@%s:%s' % (path,image,target,path))
resimage = Image.query.filter_by(ownername=user,imagename=image).first()
auth_key = env.getenv('AUTH_KEY')
url = "http://" + target + ":" + master_port + "/image/copytarget/"
data = {"token":token,"auth_key":auth_key,"user":user,"imagename":image,"description":resimage.description}
result = requests.post(url, data=data).json()
logger.info("Response from target master: " + str(result))
except Exception as e:
logger.error(e)
'''image_info_file = open(path+"."+image+".info", 'w')
image_info_file.writelines([createtime, recordshare])
image_info_file.close()'''
return {'success':'false', 'message':str(e)}
'''image_info_file = open(path+"."+image+".info", 'w')
image_info_file.writelines([createtime, recordshare])
image_info_file.close()'''
logger.info("copy image %s of %s to %s success" % (image,user,target))
return {'success':'true', 'action':'copy image'}
def create_image(self,
username,
clustername,
containername,
imagename,
description,
imagenum=10):
[status, info] = self.get_clusterinfo(clustername, username)
if not status:
return [False, "cluster not found"]
containers = info['containers']
for container in containers:
if container['containername'] == containername:
logger.info("container: %s found" % containername)
worker = xmlrpc.client.ServerProxy(
"http://%s:%s" %
(container['host'], env.getenv("WORKER_PORT")))
if worker is None:
return [
False, "The worker can't be found or has been stopped."
]
res = worker.create_image(username, imagename, containername,
description, imagenum)
container['lastsave'] = datetime.datetime.now().strftime(
"%Y-%m-%d %H:%M:%S")
container['image'] = imagename
break
else:
res = [False, "container not found"]
logger.error("container: %s not found" % containername)
clusterpath = self.fspath + "/global/users/" + username + "/clusters/" + clustername
infofile = open(clusterpath, 'w')
infofile.write(json.dumps(info))
infofile.close()
return res
def recover_allclusters(self):
logger.info("recovering all vclusters for all users...")
usersdir = self.fspath + "/global/users/"
auth_key = env.getenv('AUTH_KEY')
res = post_to_user("/master/user/groupinfo/", {'auth_key': auth_key})
#logger.info(res)
groups = json.loads(res['groups'])
quotas = {}
for group in groups:
#logger.info(group)
quotas[group['name']] = group['quotas']
for user in os.listdir(usersdir):
for cluster in self.list_clusters(user)[1]:
logger.info("recovering cluster:%s for user:%s ..." %
(cluster, user))
#res = post_to_user('/user/uid/',{'username':user,'auth_key':auth_key})
recover_info = post_to_user("/master/user/recoverinfo/", {
'username': user,
'auth_key': auth_key
})
uid = recover_info['uid']
groupname = recover_info['groupname']
input_rate_limit = quotas[groupname]['input_rate_limit']
output_rate_limit = quotas[groupname]['output_rate_limit']
self.recover_cluster(cluster, user, uid, input_rate_limit,
output_rate_limit)
logger.info("recovered all vclusters for all users")
def register(self, *args, **kwargs):
'''
Usage: register(user = modified_from_newuser())
'''
if (kwargs['user'].username == None or kwargs['user'].username == ''):
return {"success":'false', "reason": "Empty username"}
user_check = User.query.filter_by(username = kwargs['user'].username).first()
if (user_check != None and user_check.status != "init"):
#for the activating form
return {"success":'false', "reason": "Unauthorized action"}
if (user_check != None and (user_check.status == "init")):
db.session.delete(user_check)
db.session.commit()
newuser = kwargs['user']
newuser.password = hashlib.sha512(newuser.password.encode('utf-8')).hexdigest()
db.session.add(newuser)
db.session.commit()
# if newuser status is normal, init some data for this user
# now initialize for all kind of users
#if newuser.status == 'normal':
path = env.getenv('DOCKLET_LIB')
subprocess.call([path+"/userinit.sh", newuser.username])
res = self.groupQuery(name=newuser.user_group)
if res['success']:
self.set_nfs_quota(newuser.username,res['data']['data'])
return {"success":'true'}
def run(self):
global monitor_hosts
global monitor_vnodes
while not self.thread_stop:
for worker in monitor_hosts.keys():
monitor_hosts[worker]['running'] = False
workers = self.nodemgr.get_nodeips()
for worker in workers:
try:
ip = worker
workerrpc = xmlrpc.client.ServerProxy("http://%s:%s" % (worker, env.getenv("WORKER_PORT")))
# fetch data
info = list(eval(workerrpc.workerFetchInfo(self.master_ip)))
#logger.info(info[0])
# store data in monitor_hosts and monitor_vnodes
monitor_hosts[ip] = info[0]
for container in info[1].keys():
owner = get_owner(container)
if not owner in monitor_vnodes.keys():
monitor_vnodes[owner] = {}
monitor_vnodes[owner][container] = info[1][container]
except Exception as err:
logger.warning(traceback.format_exc())
logger.warning(err)
time.sleep(2)
#logger.info(History.query.all())
#logger.info(VNode.query.all())
return
def net_billings(self, username, now_bytes_total):
global monitor_vnodes
if not username in self.net_lastbillings.keys():
self.net_lastbillings[username] = 0
elif int(now_bytes_total /
self.bytes_per_beans) < self.net_lastbillings[username]:
self.net_lastbillings[username] = 0
diff = int(now_bytes_total /
self.bytes_per_beans) - self.net_lastbillings[username]
if diff > 0:
auth_key = env.getenv('AUTH_KEY')
data = {
"owner_name": username,
"billing": diff,
"auth_key": auth_key
}
header = {'Content-Type': 'application/x-www-form-urlencoded'}
http = Http()
[resp, content
] = http.request("http://" + self.master_ip + "/billing/beans/",
"POST",
urlencode(data),
headers=header)
logger.info("response from master:" + content.decode('utf-8'))
self.net_lastbillings[username] += diff
monitor_vnodes[username]['net_stats'][
'net_billings'] = self.net_lastbillings[username]
def config_prepare(self, content, configuration):
content = content.replace("%ROOTFS%", configuration['rootfs'])
content = content.replace("%HOSTNAME%", configuration['hostname'])
content = content.replace("%IP%", configuration['ip'])
content = content.replace("%GATEWAY%", configuration['gateway'])
content = content.replace("%CONTAINER_MEMORY%",
str(configuration['memory']))
content = content.replace("%CONTAINER_MEMORY_SW%",
str(configuration['memory_sw']))
content = content.replace("%CONTAINER_MEMORY_SOFT%",
str(configuration['memory_soft']))
content = content.replace("%CONTAINER_CPU%", str(configuration['cpu']))
content = content.replace("%FS_PREFIX%", self.fspath)
content = content.replace("%USERNAME%", configuration['username'])
content = content.replace("%CLUSTERID%",
str(configuration['clusterid']))
content = content.replace("%LXCSCRIPT%", env.getenv("LXC_SCRIPT"))
content = content.replace("%LXCNAME%", configuration['lxc_name'])
content = content.replace("%VLANID%", str(configuration['vlanid']))
content = content.replace("%CLUSTERNAME%",
configuration['clustername'])
content = content.replace(
"%VETHPAIR%",
str(configuration['clusterid']) + '-' +
str(configuration['containerid']))
return content
def mail_notification(self, notify_id):
email_from_address = settings.get('EMAIL_FROM_ADDRESS')
if (email_from_address in ['\'\'', '\"\"', '']):
return {'success' : 'true'}
notify = Notification.query.filter_by(id=notify_id).first()
notify_groups = NotificationGroups.query.filter_by(notification_id=notify_id).all()
to_addr = []
groups = []
for group in notify_groups:
groups.append(group.group_name)
if 'all' in groups:
users = User.query.all()
for user in users:
to_addr.append(user.e_mail)
else:
for group in notify_groups:
users = User.query.filter_by(user_group=group.group_name).all()
for user in users:
to_addr.append(user.e_mail)
content = notify.content
text = '<html><h4>Dear '+ 'user' + ':</h4>' #user.username + ':</h4>'
text += '''<p> Your account in <a href='%s'>%s</a> has been recieved a notification:</p>
<p>%s</p>
<br>
<p> Note: DO NOT reply to this email!</p>
<br><br>
<p> <a href='http://docklet.unias.org'>Docklet Team</a>, SEI, PKU</p>
''' % (env.getenv("PORTAL_URL"), env.getenv("PORTAL_URL"), content)
text += '<p>'+ str(datetime.utcnow()) + '</p>'
text += '</html>'
subject = 'Docklet Notification: ' + notify.title
msg = MIMEMultipart()
textmsg = MIMEText(text,'html','utf-8')
msg['Subject'] = Header(subject, 'utf-8')
msg['From'] = email_from_address
msg.attach(textmsg)
s = smtplib.SMTP()
s.connect()
for address in to_addr:
try:
msg['To'] = address
s.sendmail(email_from_address, address, msg.as_string())
except Exception as e:
logger.error(e)
s.close()
return {"success": 'true'}
def ip_to_rpc(self,ip):
if ip in self.runnodes:
return xmlrpc.client.ServerProxy("http://%s:%s" % (ip, env.getenv("WORKER_PORT")))
else:
logger.info('Worker %s is not connected, create rpc client failed, push task into queue')
if not ip in self.tasks:
self.tasks[ip] = []
return self.tasks[ip]
def wrapper(*args, **kwargs):
key_1 = env.getenv('AUTH_KEY')
key_2 = request.form.get("auth_key",None)
#logger.info(str(ip) + " " + str(G_userip))
if key_2 is not None and key_1 == key_2:
return func(*args, **kwargs)
else:
return json.dumps({'success':'false','message': 'auth_key is required!'})
def __init__(self, networkmgr, etcdclient, addr, mode):
self.addr = addr
logger.info ("begin initialize on %s" % self.addr)
self.networkmgr = networkmgr
self.etcd = etcdclient
self.mode = mode
self.workerport = env.getenv('WORKER_PORT')
self.tasks = {}
# delete the existing network
logger.info ("delete the existing network")
[success, bridges] = ovscontrol.list_bridges()
if success:
for bridge in bridges:
if bridge.startswith("docklet-br"):
ovscontrol.del_bridge(bridge)
else:
logger.error(bridges)
'''if self.mode == 'new':
if netcontrol.bridge_exists('docklet-br'):
netcontrol.del_bridge('docklet-br')
netcontrol.new_bridge('docklet-br')
else:
if not netcontrol.bridge_exists('docklet-br'):
logger.error("docklet-br not found")
sys.exit(1)'''
# get allnodes
self.allnodes = self._nodelist_etcd("allnodes")
self.runnodes = []
[status, runlist] = self.etcd.listdir("machines/runnodes")
for node in runlist:
nodeip = node['key'].rsplit('/',1)[1]
if node['value'] == 'ok':
logger.info ("running node %s" % nodeip)
self.runnodes.append(nodeip)
logger.info ("all nodes are: %s" % self.allnodes)
logger.info ("run nodes are: %s" % self.runnodes)
# start new thread to watch whether a new node joins
logger.info ("start thread to watch new nodes ...")
self.thread_watchnewnode = threading.Thread(target=self._watchnewnode)
self.thread_watchnewnode.start()
# wait for all nodes joins
# while(True):
for i in range(10):
allin = True
for node in self.allnodes:
if node not in self.runnodes:
allin = False
break
if allin:
logger.info("all nodes necessary joins ...")
break
time.sleep(1)
logger.info ("run nodes are: %s" % self.runnodes)
def recover_cluster(self, clustername, username, uid, input_rate_limit, output_rate_limit):
[status, info] = self.get_clusterinfo(clustername, username)
if not status:
return [False, "cluster not found"]
if not "proxy_server_ip" in info.keys():
info['proxy_server_ip'] = self.addr
self.write_clusterinfo(info,clustername,username)
[status, info] = self.get_clusterinfo(clustername, username)
if not "proxy_public_ip" in info.keys():
self.update_proxy_ipAndurl(clustername,username,info['proxy_server_ip'])
[status, info] = self.get_clusterinfo(clustername, username)
self.update_cluster_baseurl(clustername,username,info['proxy_server_ip'],info['proxy_public_ip'])
if not 'port_mapping' in info.keys():
info['port_mapping'] = []
self.write_clusterinfo(info,clustername,username)
if info['status'] == 'stopped':
return [True, "cluster no need to start"]
# recover proxy of cluster
try:
target = 'http://'+info['containers'][0]['ip'].split('/')[0]+":10000"
if self.distributedgw == 'True':
worker = self.nodemgr.ip_to_rpc(info['proxy_server_ip'])
# check public ip
if not self.check_public_ip(clustername,username):
[status, info] = self.get_clusterinfo(clustername, username)
worker.set_route("/" + info['proxy_public_ip'] + '/go/'+username+'/'+clustername, target)
else:
if not info['proxy_server_ip'] == self.addr:
logger.info("%s %s proxy_server_ip has been changed, base_url need to be modified."%(username,clustername))
oldpublicIP= info['proxy_public_ip']
self.update_proxy_ipAndurl(clustername,username,self.addr)
[status, info] = self.get_clusterinfo(clustername, username)
self.update_cluster_baseurl(clustername,username,oldpublicIP,info['proxy_public_ip'])
# check public ip
if not self.check_public_ip(clustername,username):
[status, info] = self.get_clusterinfo(clustername, username)
proxytool.set_route("/" + info['proxy_public_ip'] + '/go/'+username+'/'+clustername, target)
except:
return [False, "start cluster failed with setting proxy failed"]
# need to check and recover gateway of this user
self.networkmgr.check_usergw(input_rate_limit, output_rate_limit, username, uid, self.nodemgr,self.distributedgw=='True')
# recover containers of this cluster
for container in info['containers']:
# set up gre from user's gateway host to container's host.
self.networkmgr.check_usergre(username, uid, container['host'], self.nodemgr, self.distributedgw=='True')
worker = xmlrpc.client.ServerProxy("http://%s:%s" % (container['host'], env.getenv("WORKER_PORT")))
if worker is None:
return [False, "The worker can't be found or has been stopped."]
worker.recover_container(container['containername'])
namesplit = container['containername'].split('-')
portname = namesplit[1] + '-' + namesplit[2]
worker.recover_usernet(portname, uid, info['proxy_server_ip'], container['host']==info['proxy_server_ip'])
# recover ports mapping
[success, msg] = self.recover_port_mapping(username,clustername)
if not success:
return [False, msg]
return [True, "start cluster"]
def flush_cluster(self, username, clustername, containername):
begintime = datetime.datetime.now()
[status, info] = self.get_clusterinfo(clustername, username)
if not status:
return [False, "cluster not found"]
containers = info['containers']
imagetmp = username + "_tmp_docklet"
for container in containers:
if container['containername'] == containername:
logger.info("container: %s found" % containername)
worker = xmlrpc.client.ServerProxy(
"http://%s:%s" %
(container['host'], env.getenv("WORKER_PORT")))
worker.create_image(username, imagetmp, containername)
fimage = container['image']
logger.info("image: %s created" % imagetmp)
break
else:
logger.error("container: %s not found" % containername)
for container in containers:
if container['containername'] != containername:
logger.info("container: %s now flush" %
container['containername'])
worker = xmlrpc.client.ServerProxy(
"http://%s:%s" %
(container['host'], env.getenv("WORKER_PORT")))
#t = threading.Thread(target=onework.flush_container,args=(username,imagetmp,container['containername']))
#threads.append(t)
worker.flush_container(username, imagetmp,
container['containername'])
container['lastsave'] = datetime.datetime.now().strftime(
"%Y-%m-%d %H:%M:%S")
container['image'] = fimage
logger.info("thread for container: %s has been prepared" %
container['containername'])
clusterpath = self.fspath + "/global/users/" + username + "/clusters/" + clustername
infofile = open(clusterpath, 'w')
infofile.write(json.dumps(info))
infofile.close()
self.imgmgr.removeImage(username, imagetmp)
endtime = datetime.datetime.now()
dtime = (endtime - begintime).seconds
logger.info("flush spend %s seconds" % dtime)
logger.info("flush success")
def initlogging(name='docklet'):
# Deafults
global logger
homepath = env.getenv('FS_PREFIX')
LOG_FILENAME = homepath + '/local/log/' + name + '.log'
LOG_LIFE = env.getenv('LOG_LIFE')
LOG_LEVEL = env.getenv('LOG_LEVEL')
if LOG_LEVEL == "DEBUG":
LOG_LEVEL = logging.DEBUG
elif LOG_LEVEL == "INFO":
LOG_LEVEL = logging.INFO
elif LOG_LEVEL == "WARNING":
LOG_LEVEL = logging.WARNING
elif LOG_LEVEL == "ERROR":
LOG_LEVEL = logging.ERROR
elif LOG_LEVEL == "CRITICAL":
LOG_LEVEL = logging.CRITIAL
else:
LOG_LEVEL = logging.DEBUG
logger = logging.getLogger(name)
# Configure logging to log to a file, making a new file at midnight and keeping the last 3 day's data
# Give the logger a unique name (good practice)
# Set the log level to LOG_LEVEL
logger.setLevel(LOG_LEVEL)
# Make a handler that writes to a file, making a new file at midnight and keeping 3 backups
handler = logging.handlers.TimedRotatingFileHandler(LOG_FILENAME,
when="midnight",
backupCount=LOG_LIFE,
encoding='utf-8')
# Format each log message like this
formatter = logging.Formatter(
'%(asctime)s %(levelname)-8s %(module)s[%(lineno)d] %(message)s')
# Attach the formatter to the handler
handler.setFormatter(formatter)
# Attach the handler to the logger
logger.addHandler(handler)
# Replace stdout with logging to file at INFO level
sys.stdout = RedirectLogger(logger, logging.INFO)
# Replace stderr with logging to file at ERROR level
sys.stderr = RedirectLogger(logger, logging.ERROR)
def billing_beans():
logger.info("handle request: /billing/beans/")
form = request.form
owner_name = form.get("owner_name", None)
billing = int(form.get("billing", None))
if owner_name is None or billing is None:
return json.dumps({
'success':
'false',
'message':
'owner_name and beans fields are required.'
})
G_lockmgr.acquire('__beans_' + str(owner_name))
# update users' tables in database
owner = User.query.filter_by(username=owner_name).first()
if owner is None:
logger.warning("Error!!! Billing User %s doesn't exist!" %
(owner_name))
else:
#logger.info("Billing User:"******" " + str(owner.beans))
if oldbeans > 0 and owner.beans <= 0 or oldbeans >= 100 and owner.beans < 100 or oldbeans >= 500 and owner.beans < 500 or oldbeans >= 1000 and owner.beans < 1000:
# send mail to remind users of their beans if their beans decrease to 0,100,500 and 1000
data = {
"to_address": owner.e_mail,
"username": owner.username,
"beans": owner.beans
}
# request_master("/beans/mail/",data)
beansapplicationmgr.send_beans_email(owner.e_mail, owner.username,
int(owner.beans))
try:
db.session.commit()
except Exception as err:
db.session.rollback()
logger.warning(traceback.format_exc())
logger.warning(err)
G_lockmgr.release('__beans_' + str(owner_name))
return json.dumps({
'success': 'false',
'message': 'Fail to wirte to databases.'
})
#logger.info("Billing User:"******"The beans of User(" + str(owner) +
") are less than or equal to zero, all his or her vclusters will be stopped."
)
auth_key = env.getenv('AUTH_KEY')
form = {'username': owner.username, 'auth_key': auth_key}
request_master("/cluster/stopall/", form)
G_lockmgr.release('__beans_' + str(owner_name))
return json.dumps({'success': 'true'})
def __init__(self, networkmgr, etcdclient, addr, mode):
self.addr = addr
logger.info("begin initialize on %s" % self.addr)
self.networkmgr = networkmgr
self.etcd = etcdclient
self.mode = mode
self.workerport = env.getenv('WORKER_PORT')
# delete the existing network
logger.info("delete the existing network")
[success, bridges] = ovscontrol.list_bridges()
if success:
for bridge in bridges:
if bridge.startswith("docklet-br"):
ovscontrol.del_bridge(bridge)
else:
logger.error(bridges)
'''if self.mode == 'new':
if netcontrol.bridge_exists('docklet-br'):
netcontrol.del_bridge('docklet-br')
netcontrol.new_bridge('docklet-br')
else:
if not netcontrol.bridge_exists('docklet-br'):
logger.error("docklet-br not found")
sys.exit(1)'''
# get allnodes
self.allnodes = self._nodelist_etcd("allnodes")
self.runnodes = []
[status, runlist] = self.etcd.listdir("machines/runnodes")
for node in runlist:
nodeip = node['key'].rsplit('/', 1)[1]
if node['value'] == 'ok':
logger.info("running node %s" % nodeip)
self.runnodes.append(nodeip)
logger.info("all nodes are: %s" % self.allnodes)
logger.info("run nodes are: %s" % self.runnodes)
# start new thread to watch whether a new node joins
logger.info("start thread to watch new nodes ...")
self.thread_watchnewnode = threading.Thread(target=self._watchnewnode)
self.thread_watchnewnode.start()
# wait for all nodes joins
# while(True):
for i in range(60):
allin = True
for node in self.allnodes:
if node not in self.runnodes:
allin = False
break
if allin:
logger.info("all nodes necessary joins ...")
break
time.sleep(1)
logger.info("run nodes are: %s" % self.runnodes)
def recover_allclusters(self):
logger.info("recovering all vclusters for all users...")
usersdir = self.fspath+"/global/users/"
auth_key = env.getenv('AUTH_KEY')
for user in os.listdir(usersdir):
for cluster in self.list_clusters(user)[1]:
logger.info ("recovering cluster:%s for user:%s ..." % (cluster, user))
res = post_to_user('/user/uid/',{'username':user,'auth_key':auth_key})
self.recover_cluster(cluster, user, res['uid'])
logger.info("recovered all vclusters for all users")
def mount_cluster(self, clustername, username):
[status, info] = self.get_clusterinfo(clustername, username)
if not status:
return [False, "cluster not found"]
for container in info['containers']:
worker = xmlrpc.client.ServerProxy("http://%s:%s" % (container['host'], env.getenv("WORKER_PORT")))
if worker is None:
return [False, "The worker can't be found or has been stopped."]
worker.mount_container(container['containername'])
return [True, "mount cluster"]
def __init__(self, nodemgr, networkmgr, etcdclient, addr, mode, distributedgw='False'):
self.mode = mode
self.distributedgw = distributedgw
self.nodemgr = nodemgr
self.imgmgr = imagemgr.ImageMgr()
self.networkmgr = networkmgr
self.addr = addr
self.etcd = etcdclient
self.defaultsize = env.getenv("CLUSTER_SIZE")
self.fspath = env.getenv("FS_PREFIX")
self.clusterid_locks = threading.Lock()
# check database
try:
Container.query.all()
PortMapping.query.all()
VCluster.query.all()
except:
# create database
db.create_all()
logger.info ("vcluster start on %s" % (self.addr))
if self.mode == 'new':
logger.info ("starting in new mode on %s" % (self.addr))
# check if all clusters data are deleted in httprest.py
clean = True
usersdir = self.fspath+"/global/users/"
vclusters = VCluster.query.all()
if len(vclusters) != 0:
clean = False
for user in os.listdir(usersdir):
if len(os.listdir(usersdir+user+"/hosts")) > 0:
clean = False
if not clean:
logger.error ("clusters files not clean, start failed")
sys.exit(1)
elif self.mode == "recovery":
logger.info ("starting in recovery mode on %s" % (self.addr))
self.recover_allclusters()
else:
logger.error ("not supported mode:%s" % self.mode)
sys.exit(1)
def get(self):
if is_authenticated():
refreshInfo()
return redirect(request.args.get('next',None) or '/dashboard/')
if (env.getenv('EXTERNAL_LOGIN') == 'True'):
url = external_generate.external_login_url
link = external_generate.external_login_link
else:
link = ''
url = ''
return render_template(self.template_path, link = link, url = url)
def update_cluster_baseurl(self, clustername, username, oldip, newip):
[status, info] = self.get_clusterinfo(clustername, username)
if not status:
return [False, "cluster not found"]
logger.info("%s %s:base_url need to be modified(%s %s)."%(username,clustername,oldip,newip))
for container in info['containers']:
worker = xmlrpc.client.ServerProxy("http://%s:%s" % (container['host'], env.getenv("WORKER_PORT")))
if worker is None:
return [False, "The worker can't be found or has been stopped."]
worker.update_baseurl(container['containername'],oldip,newip)
worker.stop_container(container['containername'])
def get(self):
if is_authenticated():
#refreshInfo()
return redirect(request.args.get('next', None) or '/dashboard/')
if (env.getenv('EXTERNAL_LOGIN') == 'True'):
url = external_generate.external_login_url
link = external_generate.external_login_link
else:
link = ''
url = ''
return render_template(self.template_path, link=link, url=url)
def wrapper(*args, **kwargs):
key_1 = env.getenv('AUTH_KEY')
key_2 = request.form.get("auth_key", None)
#logger.info(str(ip) + " " + str(G_userip))
if key_2 is not None and key_1 == key_2:
return func(*args, **kwargs)
else:
return json.dumps({
'success': 'false',
'message': 'auth_key is required!'
})
def acquire_port_mapping(container_name,
container_ip,
container_port,
host_port=None):
global free_ports
global allocated_ports
global ports_lock
ports_lock.acquire()
# if container_name in allocated_ports.keys():
# return [False, "This container already has a port mapping."]
if container_name not in allocated_ports.keys():
allocated_ports[container_name] = {}
elif container_port in allocated_ports[container_name].keys():
ports_lock.release()
return [False, "This container port already has a port mapping."]
if container_name == "" or container_ip == "" or container_port == "":
ports_lock.release()
return [False, "Node Name or Node IP or Node Port can't be null."]
#print("acquire_port_mapping1")
free_port = 1
if host_port is not None:
# recover from host_port
free_port = int(host_port)
else:
# acquire new free port
while free_port <= 65535:
if free_ports[free_port]:
break
free_port += 1
if free_port == 65536:
ports_lock.release()
return [False, "No free ports."]
free_ports[free_port] = False
allocated_ports[container_name][container_port] = free_port
public_ip = env.getenv("PUBLIC_IP")
ports_lock.release()
try:
subprocess.run([
'iptables', '-t', 'nat', '-A', 'PREROUTING', '-p', 'tcp',
'--dport',
str(free_port), "-j", "DNAT", '--to-destination',
'%s:%s' % (container_ip, container_port)
],
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
shell=False,
check=True)
return [True, str(free_port)]
except subprocess.CalledProcessError as suberror:
return [
False,
"set port mapping failed : %s" %
suberror.stdout.decode('utf-8')
]
class external_loginView(normalView):
if (env.getenv('EXTERNAL_LOGIN') == 'True'):
template_path = external_generate.html_path
@classmethod
def post(self):
return render_template(self.template_path)
@classmethod
def get(self):
return self.post()
def start_cluster(self, clustername, username, user_info):
uid = user_info['data']['id']
input_rate_limit = user_info['data']['groupinfo']['input_rate_limit']
output_rate_limit = user_info['data']['groupinfo']['output_rate_limit']
[status, info] = self.get_clusterinfo(clustername, username)
if not status:
return [False, "cluster not found"]
if info['status'] == 'running':
return [False, "cluster is already running"]
# set proxy
if not "proxy_server_ip" in info.keys():
info['proxy_server_ip'] = self.addr
try:
target = 'http://'+info['containers'][0]['ip'].split('/')[0]+":10000"
if self.distributedgw == 'True':
worker = self.nodemgr.ip_to_rpc(info['proxy_server_ip'])
# check public ip
if not self.check_public_ip(clustername,username):
[status, info] = self.get_clusterinfo(clustername, username)
worker.set_route("/" + info['proxy_public_ip'] + '/go/'+username+'/'+clustername, target)
else:
if not info['proxy_server_ip'] == self.addr:
logger.info("%s %s proxy_server_ip has been changed, base_url need to be modified."%(username,clustername))
oldpublicIP= info['proxy_public_ip']
self.update_proxy_ipAndurl(clustername,username,self.addr)
[status, info] = self.get_clusterinfo(clustername, username)
self.update_cluster_baseurl(clustername,username,oldpublicIP,info['proxy_public_ip'])
# check public ip
if not self.check_public_ip(clustername,username):
[status, info] = self.get_clusterinfo(clustername, username)
proxytool.set_route("/" + info['proxy_public_ip'] + '/go/'+username+'/'+clustername, target)
except:
logger.info(traceback.format_exc())
return [False, "start cluster failed with setting proxy failed"]
# check gateway for user
# after reboot, user gateway goes down and lose its configuration
# so, check is necessary
self.networkmgr.check_usergw(input_rate_limit, output_rate_limit, username, uid, self.nodemgr,self.distributedgw=='True')
# start containers
for container in info['containers']:
# set up gre from user's gateway host to container's host.
self.networkmgr.check_usergre(username, uid, container['host'], self.nodemgr, self.distributedgw=='True')
worker = xmlrpc.client.ServerProxy("http://%s:%s" % (container['host'], env.getenv("WORKER_PORT")))
if worker is None:
return [False, "The worker can't be found or has been stopped."]
worker.start_container(container['containername'])
worker.start_services(container['containername'])
namesplit = container['containername'].split('-')
portname = namesplit[1] + '-' + namesplit[2]
worker.recover_usernet(portname, uid, info['proxy_server_ip'], container['host']==info['proxy_server_ip'])
info['status']='running'
info['start_time']=datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
self.write_clusterinfo(info,clustername,username)
return [True, "start cluster"]
def initlogging(name='docklet'):
# Deafults
global logger
homepath = env.getenv('FS_PREFIX')
LOG_FILENAME = homepath + '/local/log/' + name + '.log'
LOG_LIFE = env.getenv('LOG_LIFE')
LOG_LEVEL = env.getenv('LOG_LEVEL')
if LOG_LEVEL == "DEBUG":
LOG_LEVEL = logging.DEBUG
elif LOG_LEVEL == "INFO":
LOG_LEVEL = logging.INFO
elif LOG_LEVEL == "WARNING":
LOG_LEVEL = logging.WARNING
elif LOG_LEVEL == "ERROR":
LOG_LEVEL = logging.ERROR
elif LOG_LEVEL == "CRITICAL":
LOG_LEVEL = logging.CRITIAL
else:
LOG_LEVEL = logging.DEBUG
logger = logging.getLogger(name)
# Configure logging to log to a file, making a new file at midnight and keeping the last 3 day's data
# Give the logger a unique name (good practice)
# Set the log level to LOG_LEVEL
logger.setLevel(LOG_LEVEL)
# Make a handler that writes to a file, making a new file at midnight and keeping 3 backups
handler = logging.handlers.TimedRotatingFileHandler(LOG_FILENAME,
when="midnight", backupCount=LOG_LIFE, encoding='utf-8')
# Format each log message like this
formatter = logging.Formatter('%(asctime)s %(levelname)-8s %(module)s[%(lineno)d] %(message)s')
# Attach the formatter to the handler
handler.setFormatter(formatter)
# Attach the handler to the logger
logger.addHandler(handler)
# Replace stdout with logging to file at INFO level
sys.stdout = RedirectLogger(logger, logging.INFO)
# Replace stderr with logging to file at ERROR level
sys.stderr = RedirectLogger(logger, logging.ERROR)
def __init__(self):
settingPath = env.getenv('FS_PREFIX') + '/local/settings.conf'
if not os.path.exists(settingPath):
settingFile = open(settingPath, 'w')
setting = {}
settingFile.write(json.dumps(setting))
settingFile.close()
else:
settingFile = open(settingPath, 'r')
settingText = settingFile.read()
settingFile.close()
self.setting = json.loads(settingText)
def create_cluster(self, clustername, username, image, user_info):
if self.is_cluster(clustername, username):
return [False, "cluster:%s already exists" % clustername]
clustersize = int(self.defaultsize)
logger.info("starting cluster %s with %d containers for %s" % (clustername, int(clustersize), username))
workers = self.nodemgr.get_rpcs()
image_json = json.dumps(image)
if len(workers) == 0:
logger.warning("no workers to start containers, start cluster failed")
return [False, "no workers are running"]
# check user IP pool status, should be moved to user init later
if not self.networkmgr.has_user(username):
[status, result] = self.networkmgr.add_user(username, cidr=29)
if not status:
logger.info("add user: %s to networkmgr failed" % (username))
return [False, result]
gateway = self.networkmgr.get_usergw(username)
logger.info("create cluster with gateway : %s" % gateway)
netid = self.networkmgr.get_usernetid(username)
[status, result] = self.networkmgr.acquire_userips_cidr(username, clustersize)
if not status:
logger.info("create cluster failed: %s" % result)
return [False, result]
self.networkmgr.printpools()
ips = result
clusterid = self._acquire_id()
clusterpath = self.fspath+"/global/users/"+username+"/clusters/"+clustername
hostpath = self.fspath+"/global/users/"+username+"/hosts/"+str(clusterid)+".hosts"
hosts = "127.0.0.1\tlocalhost\n"
containers = []
for i in range(0, clustersize):
onework = workers[random.randint(0, len(workers)-1)]
lxc_name = username + "-" + str(clusterid) + "-" + str(i)
hostname = "host-"+str(i)
logger.info("create container with : name-%s, username-%s, clustername-%s, clusterid-%s, hostname-%s, ip-%s, gateway-%s, image-%s" % (lxc_name, username, clustername, str(clusterid), hostname, ips[i], gateway, image_json))
[switchid, _] = netid_decode(netid)
self.networkmgr.vsmgr.connect_switch(switchid, onework)
[success, message] = onework.create_container(lxc_name, username, user_info, clustername, str(clusterid), str(i), hostname, ips[i], gateway, str(netid), image_json)
if not success:
logger.info("container create failed, so vcluster create failed")
return [False, message]
logger.info("container create success")
hosts = hosts + ips[i].split("/")[0] + "\t" + hostname + "\t" + hostname + "."+clustername + "\n"
containers.append({'containername':lxc_name, 'hostname':hostname, 'ip':ips[i], 'host':self.nodemgr.rpc_to_ip(onework), 'image':image['name'], 'lastsave':datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")})
hostfile = open(hostpath, 'w')
hostfile.write(hosts)
hostfile.close()
clusterfile = open(clusterpath, 'w')
proxy_url = env.getenv("PORTAL_URL") + "/_web/" + username + "/" + clustername
info = {'clusterid':clusterid, 'status':'stopped', 'size':clustersize, 'containers':containers, 'nextcid': clustersize, 'create_time':datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"), 'start_time':"------", 'proxy_url':proxy_url}
clusterfile.write(json.dumps(info))
clusterfile.close()
return [True, info]
def detach_cluster(self, clustername, username):
[status, info] = self.get_clusterinfo(clustername, username)
if not status:
return [False, "cluster not found"]
if info['status'] == 'running':
return [False, 'cluster is running, please stop it first']
for container in info['containers']:
worker = xmlrpc.client.ServerProxy("http://%s:%s" % (container['host'], env.getenv("WORKER_PORT")))
if worker is None:
return [False, "The worker can't be found or has been stopped."]
worker.detach_container(container['containername'])
return [True, "detach cluster"]
def __init__(self):
settingPath = env.getenv('FS_PREFIX') + '/local/settings.conf'
if not os.path.exists(settingPath):
settingFile = open(settingPath,'w')
setting = {}
settingFile.write(json.dumps(setting))
settingFile.close()
else:
settingFile = open(settingPath, 'r')
settingText = settingFile.read()
settingFile.close()
self.setting = json.loads(settingText)
def _watchnewnode(self):
workerport = env.getenv('WORKER_PORT')
while(True):
time.sleep(0.1)
[status, runlist] = self.etcd.listdir("machines/runnodes")
if not status:
logger.warning ("get runnodes list failed from etcd ")
continue
for node in runlist:
nodeip = node['key'].rsplit('/',1)[1]
if node['value']=='waiting':
logger.info ("%s want to joins, call it to init first" % nodeip)
# 'docklet-br' of worker do not need IP Addr. Not need to allocate an IP to it
#if nodeip != self.addr:
# [status, result] = self.networkmgr.acquire_sysips_cidr()
# self.networkmgr.printpools()
# if not status:
# logger.error("no IP for worker bridge, please check network system pool")
# continue
# bridgeip = result[0]
# self.etcd.setkey("network/workbridge", bridgeip)
if nodeip in self.allnodes:
######## HERE MAYBE NEED TO FIX ###############
# here we must use "machines/runnodes/nodeip"
# we cannot use node['key'], node['key'] is absolute
# path, etcd client will append the path to prefix,
# which is wrong
###############################################
self.etcd.setkey("machines/runnodes/"+nodeip, "init-"+self.mode)
else:
self.etcd.setkey('machines/runnodes/'+nodeip, "init-new")
elif node['value']=='work':
logger.info ("new node %s joins" % nodeip)
# setup GRE tunnels for new nodes
if self.addr == nodeip:
logger.debug ("worker start on master node. not need to setup GRE")
else:
logger.debug ("setup GRE for %s" % nodeip)
if netcontrol.gre_exists('docklet-br', nodeip):
logger.debug("GRE for %s already exists, reuse it" % nodeip)
else:
netcontrol.setup_gre('docklet-br', nodeip)
self.runnodes.append(nodeip)
self.etcd.setkey("machines/runnodes/"+nodeip, "ok")
if nodeip not in self.allnodes:
self.allnodes.append(nodeip)
self.etcd.setkey("machines/allnodes/"+nodeip, "ok")
logger.debug ("all nodes are: %s" % self.allnodes)
logger.debug ("run nodes are: %s" % self.runnodes)
self.rpcs.append(xmlrpc.client.ServerProxy("http://%s:%s"
% (nodeip, workerport)))
logger.info ("add %s:%s in rpc client list" %
(nodeip, workerport))
def netsetup(action, ip):
path = env.getenv("DOCKLET_LIB")
if action == 'init':
logger.info ("initialize bridges")
else:
logger.info ("setup GRE tunnel")
# subprocess.getstatusoutput will get status and output
# and arg must be a string
#[status, output] = subprocess.getstatusoutput(path+"/netsetup.sh "+action+" "+ip)
# subprocess.call will print the result to console
# and args must be a list of string
logger.debug ("calling netsetup.sh %s %s" % (action, ip))
subprocess.call([path+"/netsetup.sh", action, ip])
def send_remind_activating_email(username):
#admin_email_address = env.getenv('ADMIN_EMAIL_ADDRESS')
nulladdr = ['\'\'', '\"\"', '']
email_from_address = settings.get('EMAIL_FROM_ADDRESS')
admin_email_address = settings.get('ADMIN_EMAIL_ADDRESS')
if (email_from_address in nulladdr or admin_email_address in nulladdr):
return
#text = 'Dear '+ username + ':\n' + ' Your account in docklet has been activated'
text = '<html><h4>Dear ' + 'admin' + ':</h4>'
text += '''<p> An activating request for %s in <a href='%s'>%s</a> has been sent</p>
<p> Please check it !</p>
<br/><br/>
<p> Docklet Team, SEI, PKU</p>
''' % (username, env.getenv("PORTAL_URL"),
env.getenv("PORTAL_URL"))
text += '<p>' + str(datetime.utcnow()) + '</p>'
text += '</html>'
subject = 'An activating request in Docklet has been sent'
if admin_email_address[0] == '"':
admins_addr = admin_email_address[1:-1].split(" ")
else:
admins_addr = admin_email_address.split(" ")
alladdr = ""
for addr in admins_addr:
alladdr = alladdr + addr + ", "
alladdr = alladdr[:-2]
msg = MIMEMultipart()
textmsg = MIMEText(text, 'html', 'utf-8')
msg['Subject'] = Header(subject, 'utf-8')
msg['From'] = email_from_address
msg['To'] = alladdr
msg.attach(textmsg)
s = smtplib.SMTP()
s.connect()
try:
s.sendmail(email_from_address, admins_addr, msg.as_string())
except Exception as e:
logger.error(e)
s.close()
def recover_group(group_name, file_path="/opt/docklet/local/docklet-storage"):
storage = env.getenv("STORAGE")
if storage == "file":
if not os.path.exists(file_path):
logger.error("%s not found, unable to recover VG" % file_path)
return False
#recover mountpoint
Ret = sys_run("losetup /dev/loop0")
if Ret.returncode != 0:
Ret = sys_run("losetup /dev/loop0 " + file_path)
if Ret.returncode != 0:
logger.error("losetup failed:%s" % Ret.stdout.decode('utf-8'))
return False
time.sleep(1)
#recover vg
Ret = sys_run("vgdisplay " + group_name)
if Ret.returncode != 0:
Ret = sys_run("vgcreate %s /dev/loop0" % group_name)
if Ret.returncode != 0:
logger.error("create VG %s failed:%s" %
(group_name, Ret.stdout.decode('utf-8')))
return False
logger.info("recover VG %s success" % group_name)
elif storage == "disk":
disk = env.getenv("DISK")
if disk is None:
logger.error("use disk for story without a physical disk")
return False
#recover vg
Ret = sys_run("vgdisplay " + group_name)
if Ret.returncode != 0:
Ret = sys_run("vgcreate %s %s" % (group_name, disk))
if Ret.returncode != 0:
logger.error("create VG %s failed:%s" %
(group_name, Ret.stdout.decode('utf-8')))
return False
logger.info("recover VG %s success" % group_name)
def __init__(self, username='******', password=None):
'''
Try to create the database when there is none
initialize 'root' user and 'root' & 'primary' group
'''
try:
User.query.all()
UserGroup.query.all()
except:
db.create_all()
root = UserGroup('root')
db.session.add(root)
db.session.commit()
if password == None:
#set a random password
password = os.urandom(16)
password = b64encode(password).decode('utf-8')
fsdir = env.getenv('FS_PREFIX')
f = open(fsdir + '/local/generated_password.txt', 'w')
f.write("User=%s\nPass=%s\n" % (username, password))
f.close()
sys_admin = User(
username,
hashlib.sha512(password.encode('utf-8')).hexdigest())
sys_admin.status = 'normal'
sys_admin.nickname = 'root'
sys_admin.description = 'Root_User'
sys_admin.user_group = 'root'
sys_admin.auth_method = 'local'
db.session.add(sys_admin)
path = env.getenv('DOCKLET_LIB')
subprocess.call([path + "/userinit.sh", username])
db.session.commit()
admin = UserGroup('admin')
primary = UserGroup('primary')
db.session.add(admin)
db.session.add(primary)
db.session.commit()
def start(self):
# start collector
self.con_collector.start()
self.hosts_collector.start()
logger.info("Monitor Collector has been started.")
# worker change it state itself. Independedntly from master.
self.etcd.setkey("machines/runnodes/" + self.addr, "work")
publicIP = env.getenv("PUBLIC_IP")
self.etcd.setkey("machines/publicIP/" + self.addr, publicIP)
self.thread_sendheartbeat = threading.Thread(target=self.sendheartbeat)
self.thread_sendheartbeat.start()
# start serving for rpc
logger.info("begins to work")
self.rpcserver.serve_forever()
