def main():
db = DatabaseManager()
user_info_list = db.get_all_user_info() + db.get_cs280_user_info()
count = 0
for user_info in user_info_list:
username = user_info['username']
chs_name = user_info['chinese_name']
receiver = user_info['email']
container_port = user_info['container_port']
open_port_range = user_info['open_port_range']
if receiver in ['test', 'test2', 'NA']:
continue
node_name_list = [p['name'] for p in user_info['permission']]
success = send_email(chs_name, receiver, username, 'plus',
container_port, open_port_range, node_name_list)
while not success:
# count = 0
time.sleep(10)
success = send_email(chs_name, receiver, username, 'plus',
container_port, open_port_range,
node_name_list)
def main():
db = DatabaseManager()
user_info_list = db.get_all_user_info()
for user_info in user_info_list:
username = user_info['username']
cname = username
container_port = user_info['container_port']
open_port_range = user_info['open_port_range']
if username not in ['zhangsy']:
continue
for permission_detail in user_info['permission']:
node_name = permission_detail['name']
docker_type = 'docker' if node_name == 'admin' else 'nvidia-docker'
container_name = '%s-%s' % (username, node_name)
add_open_port_str = "-p %s:%s" % (open_port_range, open_port_range
) if node_name == 'admin' else ''
memory_size = os.popen(
'''ssh %s free -h | head -n 2 | tail -n 1 | awk -F' ' '{print $2}' '''
% node_name).read().strip()
memory_unit = memory_size[-1]
memory_size = int(memory_size[:-1])
shm_size = memory_size // 2
shm_size = str(shm_size) + memory_unit
rm_container_on_remote(node_name, container_name)
create_container_on_remote(node_name, docker_type, container_name,
cname, shm_size, container_port,
add_open_port_str)
print("create container %s successfully." % container_name)
def main():
db = DatabaseManager()
cursor = db.get_cursor()
# time_condition = " AND query_time >= '%s' AND query_time <= '%s'"
# DELETE FROM gpu_old WHERE DATE_FORMAT(query_time, '%s') <> 0 ORDER BY query_time LIMIT 1;
for i in tqdm(range(16000)):
cursor.execute(
"DELETE FROM gpu_old WHERE DATE_FORMAT(query_time, '%s') <> 0 LIMIT 10000"
)
db.commit()
def main():
db = DatabaseManager()
user_info_list = db.get_all_user_info()
for user_info in user_info_list:
username = user_info['username']
advisor = user_info['advisor']
cname = username
container_port = user_info['container_port']
open_port_range = user_info['open_port_range']
if advisor != '何旭明':
continue
if username not in ['yanshp']:
continue
# if username in ['zhangsy', 'lishl', 'zhoujl', 'maoym']:
# continue
# if username!='huangshy':
# continue
for permission_detail in user_info['permission']:
node_name = permission_detail['name']
# if '30' not in node_name and '25' not in node_name:
# continue
docker_type = 'docker' if node_name == 'admin' else 'nvidia-docker'
container_name = '%s-%s' % (username, node_name)
add_open_port_str = "-p %s:%s" % (open_port_range, open_port_range
) if node_name == 'admin' else ''
try:
memory_size = subprocess.check_output(
'''ssh %s free -h | head -n 2 | tail -n 1 | awk -F' ' '{print $2}' '''
% node_name,
shell=True).decode('utf-8').strip()
except:
print(f"{node_name} fails.")
continue
memory_unit = memory_size[-1]
memory_size = int(memory_size[:-1])
shm_size = memory_size // 2
shm_size = str(shm_size) + memory_unit
# rm_container_on_remote(node_name, container_name)
create_container_on_remote(node_name, docker_type, container_name,
cname, shm_size, container_port,
add_open_port_str)
print("create container %s successfully." % container_name)
def main():
db = DatabaseManager()
user_info_list = db.get_all_user_info()
permission_list = ['node47', 'node48']
# permission_list = ['node48']
"""
gaossh permission
"""
for user_info in tqdm(user_info_list):
username = user_info['username']
cname = username
container_port = user_info['container_port']
advisor = user_info['advisor']
if cname not in ['liuwen']:
continue
for node_name in permission_list:
docker_type = 'docker' if node_name == 'admin' else 'nvidia-docker'
rm_container_on_remote(node_name, cname)
create_container_on_remote(node_name, docker_type, cname,
container_port, advisor)
print("create %s-%s successfully." % (cname, node_name))
"""
AI Cluster permission
"""
for user_info in tqdm(user_info_list):
username = user_info['username']
cname = username
container_port = user_info['container_port']
advisor = user_info['advisor']
if cname not in ['liuwen']:
continue
for permission_detail in user_info['permission']:
node_name = permission_detail['name']
docker_type = 'docker' if node_name == 'admin' else 'nvidia-docker'
if node_name != 'node01':
continue
BaseHandler.rm_container_on_remote(node_name, cname)
BaseHandler.create_container_on_remote(node_name, docker_type,
cname, container_port,
advisor)
print("create %s-%s successfully." % (cname, node_name))
def __init__(self):
handlers = [
(r"/", IndexHandler),
(r"/login", LoginHandler),
(r"/system", SystemHandler),
(r"/create", CreateHandler),
(r"/remove", RemoveHandler),
(r"/delete", DeleteHandler),
(r"/permission", PermissionHandler),
# (r"/cs280_permission", CS280PermissionHandler),
(r"/user", UserHandler),
(r"/gpu", GpuHandler),
(r"/p40_gpu", P40GpuHandler),
(r"/ppppp40_gpu", PPPPP40GpuHandler),
(r"/plus_gpu", PLUSGpuHandler),
(r"/plus_plus_gpu", PLUSPLUSGpuHandler),
(r"/svip_gpu", SVIPGpuHandler),
(r"/svip_svip_gpu", SVIPSVIPGpuHandler),
(r"/real_gpu", RealGpuHandler)
]
settings = dict(
cookie_secret="7CA71A57B571B5AEAC5E64C6042415DE",
# TemplateData_path = os.path.join(os.path.dirname(__file__).decode('gbk'), u'TemplateData'),
static_path='static',
login_url="/login",
debug=True)
self.db = DatabaseManager()
tornado.web.Application.__init__(self, handlers, **settings)
def main():
db = DatabaseManager()
user_info_list = db.get_all_user_info()
"""
start without admin
"""
for user_info in tqdm(user_info_list):
username = user_info['username']
cname = username
container_port = user_info['container_port']
advisor = user_info['advisor']
for permission_detail in user_info['permission']:
node_name = permission_detail['name']
docker_type = 'docker' if node_name == 'admin' else 'nvidia-docker'
if node_name == 'admin':
continue
BaseHandler.rm_container_on_remote(node_name, cname)
BaseHandler.create_container_on_remote(node_name, docker_type,
cname, container_port,
advisor)
print("create %s-%s successfully." % (cname, node_name))
"""
start with admin
"""
for user_info in tqdm(user_info_list):
username = user_info['username']
cname = username
container_port = user_info['container_port']
advisor = user_info['advisor']
for permission_detail in user_info['permission']:
node_name = permission_detail['name']
docker_type = 'docker' if node_name == 'admin' else 'nvidia-docker'
if node_name != 'admin':
continue
BaseHandler.rm_container_on_remote(node_name, cname)
BaseHandler.create_container_on_remote(node_name, docker_type,
cname, container_port,
advisor)
print("create %s-%s successfully." % (cname, node_name))
def main():
db = DatabaseManager()
start_time = time.time()
stat_node_id = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 27, 28, 29, 30, 31, 32, 33, 34, 35]
# stat_node_id = [1]
date_list = get_date_list()
wb = xlsxwriter.Workbook('gpu_stat.xlsx')
gpu_utils_sheet = wb.add_worksheet('gpu_utils')
user_num_sheet = wb.add_worksheet('user_num')
"""
sheet title
"""
gpu_utils_sheet.write(0, 0, 'node')
gpu_utils_sheet.write(0, 1, 'gpu_id')
for i in range(len(date_list) - 1):
gpu_utils_sheet.write(0, 2 + i, date_list[i])
user_num_sheet.write(0, 0, 'node')
for i in range(len(date_list) - 1):
user_num_sheet.write(0, 1 + i, date_list[i])
"""
sheet content
"""
for i in range(len(date_list) - 1):
cluster_gpu_stat_list = []
row_cursor = 1
for node_idx, node_id in enumerate(stat_node_id):
print('-' * 30, node_id, '-', date_list[i], '-', date_list[i + 1], '-' * 30)
gpu_stat_list = get_one_node_gpu_usage_info(db, node_id, date_list[i], date_list[i + 1])
cluster_gpu_stat_list.append(gpu_stat_list)
print(gpu_stat_list)
node_user_dict = {}
for gpu_id, gpu_stat in enumerate(gpu_stat_list):
# gpu utils sheet
gpu_utils_sheet.write(row_cursor + gpu_id, 0, node_id)
gpu_utils_sheet.write(row_cursor + gpu_id, 1, gpu_id)
gpu_utils_sheet.write(row_cursor + gpu_id, 2 + i, gpu_stat['avg_gpu_utils'])
# user num sheet
gpu_user_dict = gpu_stat['user_dict']
for user_name, user_count in gpu_user_dict.items():
if user_name not in node_user_dict:
node_user_dict[user_name] = user_count
else:
node_user_dict[user_name] += user_count
row_cursor += len(gpu_stat_list)
user_num_sheet.write(1 + node_idx, 0, node_id)
user_num_sheet.write(1 + node_idx, 1 + i, len(node_user_dict))
wb.close()
def main():
db = DatabaseManager()
user_info_list = db.get_all_user_info()
new_node_list = [1, 2, 3, 4, 5, 19, 20, 21]
for user_info in user_info_list:
uid = user_info['uid']
username = user_info['username']
cname = username
container_port = user_info['container_port']
open_port_range = user_info['open_port_range']
advisor = user_info['advisor']
if advisor != '高盛华':
continue
db.add_user_permission(uid, new_node_list, 'yes', '', '', '')
for node_id in new_node_list:
node_name = 'node%.2d' % node_id
docker_type = 'docker' if node_name == 'admin' else 'nvidia-docker'
container_name = '%s_%s' % (username, node_name)
add_open_port_str = "-p %s:%s" % (open_port_range, open_port_range
) if node_name == 'admin' else ''
memory_size = os.popen(
'''ssh %s free -h | head -n 2 | tail -n 1 | awk -F' ' '{print $2}' '''
% node_name).read().strip()
memory_unit = memory_size[-1]
memory_size = int(memory_size[:-1])
shm_size = memory_size // 2
shm_size = str(shm_size) + memory_unit
# rm_container_on_remote(node_name, container_name, username)
create_container_on_remote(node_name, docker_type, container_name,
cname, shm_size, container_port,
add_open_port_str)
print("create container %s successfully." % container_name)
def main():
db = DatabaseManager()
user_info_list = db.get_all_user_info()
for user_info in user_info_list:
username = user_info['username']
cname = username
container_port = user_info['container_port']
open_port_range = user_info['open_port_range']
# if username not in ['piaozx']:
# continue
if os.path.exists('/public/docker/%s/root/.ssh/config' % username):
os.system("rm /public/docker/%s/root/.ssh/config" % username)
os.system(
"cp /public/docker/baseline-1/root/.ssh/config /public/docker/%s/root/.ssh/config"
% username)
os.system(
'sed -i "s/user_port/%d/g" /public/docker/%s/root/.ssh/config'
% (container_port, cname))
print("update %s successfully." % username)
def main():
db = DatabaseManager()
user_info_list = db.get_all_user_info()
svip_node_list = [1, 2, 5, 17, 18, 19, 20, 24, 29, 30]
public_node_list = [13, 14, 26, 23, 32, 33, 15, 16]
new_node_list = svip_node_list + public_node_list
for user_info in user_info_list:
uid = user_info['uid']
username = user_info['username']
cname = username
container_port = user_info['container_port']
open_port_range = user_info['open_port_range']
advisor = user_info['advisor']
"""
SVIP Lab
"""
if advisor != '高盛华':
continue
if username not in ['xuyy']:
continue
exist_compute_node_list = []
for node_info in user_info['permission']:
node_name = node_info['name']
if node_name == 'admin':
continue
else:
node_id = int(node_name[4:])
exist_compute_node_list.append(node_id)
need_remove_node_list = list(
set(exist_compute_node_list) - set(new_node_list))
remove_container_on_remote_by_node_list(need_remove_node_list, db, uid,
username)
need_add_node_list = list(
set(new_node_list) - set(exist_compute_node_list))
for node_id in need_add_node_list:
node_name = 'node%.2d' % node_id
docker_type = 'docker' if node_name == 'admin' else 'nvidia-docker'
container_name = '%s_%s' % (username, node_name)
add_open_port_str = "-p %s:%s" % (open_port_range, open_port_range
) if node_name == 'admin' else ''
memory_size = os.popen(
'''ssh %s free -h | head -n 2 | tail -n 1 | awk -F' ' '{print $2}' '''
% node_name).read().strip()
memory_unit = memory_size[-1]
memory_size = int(memory_size[:-1])
shm_size = memory_size // 2
shm_size = str(shm_size) + memory_unit
addition_str = utils.ContainerAdditionStr(
node_name, advisor, cname).get_additional_str()
create_container_on_remote(node_name, docker_type, container_name,
cname, shm_size, container_port,
addition_str)
print("create container %s successfully." % container_name)
db.add_user_permission(uid, need_add_node_list, 'yes', '', '', '')
def main():
db = DatabaseManager()
user_info_list = db.get_all_user_info()
permission_list = [
'node40', 'node41', 'node42', 'node43', 'node44', 'node46', 'node47',
'node48', 'node49'
]
# permission_list = ['node40', 'node41', 'node42', 'node43', 'node44', 'node46']
# permission_list = ['node45']
"""
gaossh permission
"""
for user_info in tqdm(user_info_list):
username = user_info['username']
cname = username
container_port = user_info['container_port']
advisor = user_info['advisor']
if advisor not in ['高盛华']:
continue
if cname not in ['piaozx']:
continue
for node_name in permission_list:
docker_type = 'docker' if node_name == 'admin' else 'nvidia-docker'
rm_container_on_remote(node_name, cname)
create_container_on_remote(node_name, docker_type, cname,
container_port, advisor)
print("create %s-%s successfully." % (cname, node_name))
os.system("cp gaoshh/gaossh_node.txt gaoshh/tmp.txt")
os.system('sed -i "s/user_port/%d/g" gaoshh/tmp.txt' %
(container_port))
os.system('cat gaoshh/tmp.txt > /public/docker/%s/root/.ssh/config' %
(cname))
os.system("rm gaoshh/tmp.txt")
return
"""
AI Cluster permission
"""
for user_info in tqdm(user_info_list):
username = user_info['username']
cname = username
container_port = user_info['container_port']
advisor = user_info['advisor']
if cname not in ['piaozx']:
continue
for permission_detail in user_info['permission']:
node_name = permission_detail['name']
docker_type = 'docker' if node_name == 'admin' else 'nvidia-docker'
if node_name != 'node01':
continue
BaseHandler.rm_container_on_remote(node_name, cname)
BaseHandler.create_container_on_remote(node_name, docker_type,
cname, container_port,
advisor)
print("create %s-%s successfully." % (cname, node_name))
from db.db_manager import DatabaseManager
from datetime import datetime
import sys
import requests
db = DatabaseManager("bookmarks.db")
class CreateBookmarksTableCommand:
"""Class that creates bookmarks table"""
def execute(self):
db.create_table(
"bookmarks",
{
"id": "integer primary key autoincrement",
"title": "text not null",
"url": "text not null",
"notes": "text",
"date_added": "text not null",
"date_updated": "text not null",
},
)
class AddBookMarkCommand:
"""Command for adding bookmark and returning success string"""
def execute(self, data, timestamp=None):
data["date_added"] = timestamp or datetime.utcnow().isoformat()
data["date_updated"] = data["date_added"]
db.add("bookmarks", data)
def main():
soft_max_card_num = 8
hard_max_card_num = 20
soft_time = 3600
db = DatabaseManager()
middle_user_dict = {}
history_p_info_dict = get_history_p_info_dict(db)
while True:
current_user_process_info_dict = get_current_user_process_info_dict(db)
user_info_list = db.get_all_user_info()
user_info_dict = convert_user_info_list_to_dict(user_info_list)
for user, process_info in current_user_process_info_dict.items():
if user not in user_info_dict:
continue
permission_num = len(user_info_dict[user]['permission'])
if permission_num <= 10:
continue
card_num = len(process_info)
"""
normal
"""
if card_num <= soft_max_card_num:
middle_user_dict[user] = {'count': 0, 'card_num': card_num}
continue
"""
middle
"""
if card_num <= hard_max_card_num:
if user not in middle_user_dict:
middle_user_dict[user] = {'count': 1, 'card_num': card_num}
else:
middle_user_dict[user]['count'] += 1
middle_user_dict[user]['card_num'] = card_num
sorted_card_info_list = get_sorted_card_info_list(process_info)
for card_info in sorted_card_info_list[soft_max_card_num:]:
for p_info in card_info:
if p_info['runtime'] > soft_time:
kill_process_by_p_info(p_info, history_p_info_dict)
continue
"""
hard
"""
if user not in middle_user_dict:
middle_user_dict[user] = {'count': 1, 'card_num': card_num}
else:
middle_user_dict[user]['count'] += 1
middle_user_dict[user]['card_num'] = card_num
sorted_card_info_list = get_sorted_card_info_list(process_info)
for card_info in sorted_card_info_list[hard_max_card_num:]:
for p_info in card_info:
kill_process_by_p_info(p_info, history_p_info_dict)
"""
send email
"""
for user_info in user_info_list:
uname = user_info['username']
if uname in middle_user_dict and middle_user_dict[uname][
'count'] == 1:
send_email(user_info, middle_user_dict[uname]['card_num'],
soft_max_card_num, hard_max_card_num, soft_time)