def upstream_check():
groupModel = GroupModel()
groups = groupModel.rows()
for group in groups:
#这里返回ip,port去重后的结果
all_upstreams = groupModel.allUpstreams(group['id'])
#协程批量curl,填充请求结果
all_upstreams = batch_check(all_upstreams)
for upstream in all_upstreams:
key = cache.gen_upstream_key(upstream['host'], upstream['port'])
status = cache.get(key)
if status == '':
if upstream['status'] == 'failed':
do_offline(group, upstream)
log.info('set upstream %s:%s status to %s' %
(upstream['host'], upstream['port'],
upstream['status']))
else:
if status == upstream['status']:
continue
if upstream['status'] == 'failed':
do_offline(group, upstream)
else:
do_online(group, upstream)
log.info('set upstream %s:%s status from %s to %s' %
(upstream['host'], upstream['port'], status,
upstream['status']))
def __init__(self, number):
self.number = number
self.db = db_test
self.user = UserModel(self.db)
self.group = GroupModel(self.db)
self.article = ArticleModel(self.db)
self.admin = {
'email': '*****@*****.**',
'penname': 'wutong',
'password': '******',
}
self.gen_test_data()
def DELETE(self, group_name, host, port):
group = GroupModel().load(group_name)
if group == None:
return self.resp(errno=1, errmsg='no this group')
group_id = group['id']
OpenrestyModel().delete(group_id, host, port)
return self.resp()
def GET(self, group_name):
group = GroupModel().load(group_name)
if group == None:
return self.resp(errno=1, errmsg='no this group')
group_id = group['id']
rows = ProjectModel().rows(group_id)
return self.resp(data=rows)
def DELETE(self, group_name, project_name):
group = GroupModel().load(group_name)
if group == None:
return self.resp(errno=1, errmsg='no this group')
group_id = group['id']
ProjectModel().delete(group_id, project_name)
return self.resp()
def POST(self, group_name, proj_name):
group = GroupModel().load(group_name)
if group == None:
return self.resp(errno=1, errmsg='no this group')
group_id = group['id']
project = ProjectModel().load(group_id, proj_name)
if project == None:
return self.resp(errno=2, errmsg='no this project')
project_id = project['id']
all_envs = EnvironmentModel().rows(project_id)
all_env_names = [ env['name'] for env in all_envs ]
if len(all_envs) == 0:
return self.resp(errno=3, errmsg='no environment in this project')
param_keys = web.input().keys()
if param_keys == None or len(param_keys) == 0:
return self.resp(errno=4, errmsg='no envoronment and uid_rate param in request body')
for param_key in param_keys:
if param_key not in all_env_names:
return self.resp(errno=5, errmsg='%s is not in project environment, please check, all env names is:%s' % (
param_key, str(all_env_names) ))
if len(param_keys) != len(all_envs):
return self.resp(errno=6, errmsg="param count is not equal to this project's environment name count, all env names is:%s" % (
str(all_env_names) ))
param_val_sum = 0
param_vals = web.input().values()
for param_val in param_vals:
if '.' in param_val:
return self.resp(errno=7, errmsg='uid_rate should be interger not float')
try:
param_val = int(param_val)
except Exception, ex:
return self.resp(errno=8, errmsg='uid_rate should be interger not float')
if param_val < 0:
return self.resp(errno=9, errmsg='uid_rate should be more than or equal 0')
param_val_sum = param_val_sum + param_val
def PUT(self, group_name, project_name):
group = GroupModel().load(group_name)
if group == None:
return self.resp(errno=1, errmsg='no this group')
group_id = group['id']
try:
ProjectModel().add(group_id, project_name)
except Exception, ex:
return self.resp(errno=2, errmsg=str(ex))
def DELETE(self, group_name, proj_name, env_name):
group = GroupModel().load(group_name)
if group == None:
return self.resp(errno=1, errmsg='no this group')
group_id = group['id']
project = ProjectModel().load(group_id, proj_name)
if project == None:
return self.resp(errno=2, errmsg='no this project')
project_id =project['id']
EnvironmentModel().delete(project_id, env_name)
return self.resp()
def GET(self, group_name, proj_name):
group = GroupModel().load(group_name)
if group == None:
return self.resp(errno=1, errmsg='no this group')
group_id = group['id']
project = ProjectModel().load(group_id, proj_name)
if project == None:
return self.resp(errno=2, errmsg='no this project')
project_id =project['id']
rows = EnvironmentModel().rows(project_id)
return self.resp(data=rows)
def PUT(self, group_name, proj_name, env_name):
group = GroupModel().load(group_name)
if group == None:
return self.resp(errno=1, errmsg='no this group')
group_id = group['id']
project = ProjectModel().load(group_id, proj_name)
if project == None:
return self.resp(errno=2, errmsg='no this project')
project_id =project['id']
try:
EnvironmentModel().add(project_id, env_name)
except Exception, ex:
return self.resp(errno=3, errmsg=str(ex))
def PUT(self, group_name, host, port):
group = GroupModel().load(group_name)
if group == None:
return self.resp(errno=1, errmsg='no this group')
group_id = group['id']
if self.isValidIp(host) == False:
return self.resp(errno=2, errmsg='%s is invalid ip' % host)
if self.isValidPort(port) == False:
return self.resp(errno=3, errmsg='%s is invalid port' % port)
try:
OpenrestyModel().add(group_id, host, port)
except Exception, ex:
return self.resp(errno=4, errmsg=str(ex))
def DELETE(self, group_name, proj_name, env_name, host, port):
group = GroupModel().load(group_name)
if group == None:
return self.resp(errno=1, errmsg='no this group')
group_id = group['id']
project = ProjectModel().load(group_id, proj_name)
if project == None:
return self.resp(errno=2, errmsg='no this project')
project_id =project['id']
environment = EnvironmentModel().load(project_id, env_name)
if environment == None:
return self.resp(errno=3, errmsg='no this environment')
env_id = environment['id']
UpstreamModel().delete(env_id, host, port)
return self.resp()
def PUT(self, group_name, proj_name, no_uid_env_name):
group = GroupModel().load(group_name)
if group == None:
return self.resp(errno=1, errmsg='no this group')
group_id = group['id']
projModel = ProjectModel()
project = projModel.load(group_id, proj_name)
if project == None:
return self.resp(errno=2, errmsg='no this project')
project_id = project['id']
all_envs = EnvironmentModel().rows(project_id)
all_env_names = [env['name'] for env in all_envs]
if no_uid_env_name not in all_env_names:
if no_uid_env_name != '':
return self.resp(errno=3, errmsg='no this env, please call api set')
projModel.setNoUidEnv(project_id, no_uid_env_name)
return self.resp()
def batch_set_resty_upstream(group, upstream, oper):
if group == None or upstream == None: return
if oper not in ['up', 'down']: return
all_openresty = GroupModel().allOpenrestys(group['id'])
if len(all_openresty) == 0:
log.error('group:%s has no openresty', group['name'])
return
from gevent import monkey
monkey.patch_socket()
import gevent
greenlets = [
gevent.spawn(set_resty_upstream, openresty, upstream, oper)
for openresty in all_openresty
]
gevent.joinall(greenlets)
key = cache.gen_upstream_key(upstream['host'], upstream['port'])
cache.set(key, upstream['status'])
log.info('set cache finish')
def PUT(self, group_name, proj_name, spec_uid_str):
group = GroupModel().load(group_name)
if group == None:
return self.resp(errno=1, errmsg='no this group')
group_id = group['id']
projModel = ProjectModel()
project = projModel.load(group_id, proj_name)
if project == None:
return self.resp(errno=2, errmsg='no this priject')
project_id = project['id']
spec_uid_list = list(
set([spec_uid.strip() for spec_uid in spec_uid_str.split(',')]))
try:
spec_uid_list.remove('')
except:
pass
spec_uid_str = ','.join(spec_uid_list)
ProjectModel().setSpecUid(project_id, spec_uid_str)
return self.resp()
def setUpClass(cls):
cls.db = util.db_test
cls.tearDownClass()
cls.uid = cls.db.getfirstfield("""
insert into "user"
(email, penname, password, is_activated)
values ('*****@*****.**', 'test', 'test', true)
returning uid
""")
cls.model = GroupModel(util.db_test)
cls.name = 'test_group'
sql = """insert into "group"
(uid, name)
values (%s, %s)
returning gid"""
cls.gid = cls.db.getfirstfield(sql, cls.uid, cls.name)
def PUT(self, group_name, proj_name, env_name, host, port):
group = GroupModel().load(group_name)
if group == None:
return self.resp(errno=1, errmsg='no this group')
group_id = group['id']
project = ProjectModel().load(group_id, proj_name)
if project == None:
return self.resp(errno=2, errmsg='no this project')
project_id =project['id']
environment = EnvironmentModel().load(project_id, env_name)
if environment == None:
return self.resp(errno=3, errmsg='no this environment')
env_id = environment['id']
if self.isValidIp(host) == False:
return self.resp(errno=4, errmsg = '%s is invalid ip' % host)
if self.isValidPort(port) == False:
return self.resp(errno=5, errmsg='%s is invalid port' % port)
try:
UpstreamModel().add(env_id, host, port)
except Exception, ex:
return self.resp(errno=6, errmsg=str(ex))
def PUT(self, group_name, proj_name, spec_env_name):
group = GroupModel().load(group_name)
if group == None:
return self.resp(errno=1, errmsg='no this group')
group_id = group['id']
projModel = ProjectModel()
project = projModel.load(group_id, proj_name)
if project == None:
return self.resp(errno=2, errmsg='no this priject')
project_id = project['id']
all_envs = EnvironmentModel().rows(project_id)
if len(all_envs) == 0:
return self.resp(
errno=3,
errmsg='now this project has no environment, please add')
all_env_names = [env['name'] for env in all_envs]
if spec_env_name not in all_env_names and spec_env_name != '':
return self.resp(
errno=4,
errmsg="special env name is not in this project's environment")
projModel.setSpecialEnv(project_id, spec_env_name)
return self.resp()
def PUT(self, group_name):
try:
GroupModel().add(group_name)
except Exception, ex:
return self.resp(errno=1, errmsg=str(ex))
class TestDataGenerator(object):
def __init__(self, number):
self.number = number
self.db = db_test
self.user = UserModel(self.db)
self.group = GroupModel(self.db)
self.article = ArticleModel(self.db)
self.admin = {
'email': '*****@*****.**',
'penname': 'wutong',
'password': '******',
}
self.gen_test_data()
def initdb(self):
sql_dir = wutong_path("model/dbschema/")
paths = ["schema.sql", "function.sql"]
paths = [(sql_dir + p) for p in paths]
for path in paths:
sql = open(path, "r").read()
assert (self.db.execute(sql))
def gen_admin_user(self):
hashuid = self.user.do_register(self.admin['email'],
self.admin['penname'],
self.admin['password'])
self.admin["uid"] = self.user.do_activate(hashuid)
# gen avatar
avatar_dir = os.path.join(os.path.dirname(__file__),
'../static/avatar/')
random_path = os.path.join(avatar_dir, 'random')
avatar_path = os.path.join(random_path,
random.choice(os.listdir(random_path)))
with open(avatar_path, 'r') as avatar_fp:
avatar = genavatar(avatar_fp, avatar_dir, self.admin['penname'])
self.user.update_user_avatar_by_penname(self.admin['penname'],
avatar)
return self.admin
def gen_users(self):
"""
Generate random user from randomuser.me
"""
url = 'http://api.randomuser.me/?results=%s'
users = []
results = []
for i in xrange(self.number / 5):
response = requests.get(url % 5)
results += response.json()['results']
response = requests.get(url % (self.number % 5))
results += response.json()['results'] or []
for res in results:
user = res["user"]
user["penname"] = user["name"]["first"] + user["name"]["last"]
user["password"] = '******'
user["sex"] = user["gender"]
user["address"] = user["location"]["city"] + user["location"][
"state"]
user["avatar"] = user["picture"]
user["intro"] = ''.join(
[str(v) for v in user["location"].values()])
user["motto"] = ''.join([str(v) for v in user.values()])
user["age"] = user["location"]["zip"][-2:]
users.append(user)
# insert into db
for user in users:
hashuid = self.user.do_register(user["email"], user["penname"],
user["password"])
user["uid"] = self.user.do_activate(hashuid)
self.user.update_user_avatar_by_penname(user['penname'],
user['avatar'])
return users
def gen_groups(self, users):
gid = self.group.do_create(self.admin['uid'], 'admin_group')
gids = [gid]
for i in xrange(random.randint(1, self.number)):
user = random.choice(users)
gid = self.group.do_create(
user['uid'],
'%s group by %s' % (random.random(), user['uid']),
is_public=random.randint(0, 2),
)
self.group.do_join_group(gid, self.admin["uid"])
gids.append(gid)
return gids
def gen_random_group_data(self, users, gids):
tids = []
cids = []
bids = []
def random_reply_id():
if random_bool():
try:
return random.choice(tids)
except:
pass
return None
for gid in gids:
uid = random.choice(users)['uid']
self.group.do_join_group(gid, uid)
tid = self.group.do_create_topic(
gid, uid, 'random topic in group %s by %s' % (gid, uid),
'%s' % random.random(), random_reply_id())
tids.append(tid)
cid = self.group.do_create_chat(gid, uid,
'random chat in group %s' % gid,
random_reply_id())
cids.append(cid)
bid = self.group.do_create_bulletin(
gid, uid, 'random bulletin in group %s' % gid,
'%s' % random.random())
bids.append(bid)
return tids, cids, bids
def gen_articles(self, users):
aids = []
for i in xrange(random.randint(self.number, self.number * 10)):
uid = random.choice(users)['uid']
aid = self.article.do_create(
uid,
'%s article title by %s' % (random.random(), uid),
'<p>random mainbody %s</p>' % random.random(),
tags=['技术', 'linux', 'python'])
aids.append(aid)
return aids
def gen_random_article_data(self, users, aids):
for aid in aids:
for i in xrange(random.randint(0, self.number)):
uid = random.choice(users)['uid']
self.article.create_bottom_comment(
aid, uid, '%s comment' % random.random())
paragraph_id = str(random.randint(0, 1))
self.article.create_side_comment(
aid, uid, '%s comment' % random.random(), paragraph_id)
def gen_test_data(self):
self.initdb()
admin = self.gen_admin_user()
users = self.gen_users()
gids = self.gen_groups(users)
tids, cids, bids = self.gen_random_group_data(users, gids)
aids = self.gen_articles(users)
self.gen_random_article_data(users, aids)
def GET(self):
return self.resp(data=GroupModel().rows())
def DELETE(self, group_name):
try:
GroupModel().delete(group_name)
except Exception, ex:
return self.resp(errno=1, errmsg=str(ex))