def execute_action(self):
# 流程的起点,可能会有多个
action_tuple = self.parser.get_streams(self.action_name)
origin = self.parser.get_origin_streamlet(action_tuple)
action_type = self.parser.get_action_type(action_tuple)
self.log.debug('action type : %s' % action_type)
r = redis_tool.get_it()
try:
self.log.debug(len(origin))
for node_id in origin:
yield self.execute_multi_streamlet(node_id, action_tuple)
if action_type == 'deploy':
self.update_status('normal')
except Exception as e:
self.log.error('stream execute error :\n %s' % generals.trace())
self.interrupt = True
if action_type == 'deploy':
self.update_status('failure')
"""
使用redis来保证一致性。由于监控的线程是并行在多条的进程中的,
而且它只会把状态报告给本进程。
所以需要保证所以进程中的监控线程都能够拿到信息……
"""
r.set("manage_error_$_%s" % self.serial,
json.dumps({
'action': self.action_name,
'msg': str(e)
}))
raise e
def __get_messages(self):
r=redis_tool.get_it()
rs=[]
messages=r.keys('execute_script_$_%s_$_*'%self.serial)
rs=rs+[_.split('_$_')[2] for _ in messages if
_.split('_$_')[3] in self.command_ids]
return rs
def message_watcher():
log = logging.getLogger('manor')
r = redis_tool.get_it()
_keys = r.keys('script_info*')
for k in _keys:
date = k.split('_$_')[2]
tp = time.strptime(date, "%Y-%m-%d %H:%M:%S")
now = time.localtime()
tpd = datetime.datetime(*tp[:6])
nowd = datetime.datetime(*now[:6])
poor = (nowd - tpd).days
if poor >= 1:
log.debug('delete %s' % k)
r.delete(k)
_keys = r.keys('execute_script*')
for k in _keys:
date = k.split('_$_')[4]
tp = time.strptime(date, "%Y-%m-%d %H:%M:%S")
tpd = datetime.datetime(*tp[:6])
now = time.localtime()
nowd = datetime.datetime(*now[:6])
poor = (nowd - tpd).days
if poor >= 1:
log.debug('delete %s' % k)
r.delete(k)
def message_sending(self):
while True:
time.sleep(0.5)
if self.msg != self.current_msg:
self.msg = self.current_msg
send_msg = json.loads(self.msg)
info = list_app_resources(self.serial)
while not info.done():
pass
send_msg['info'] = info.result()
send_msg = json.dumps(send_msg)
clients = []
for c in _clients.values():
if self.serial in set(c['serial']):
c['client'].write_message(send_msg)
clients.append(1)
if len([
_ for _ in _clients.values() if self.serial in _['serial']
]) == 0:
self.status = 'interrupt'
# 这段代码的位置很重要。
if self.status == 'interrupt':
del _monitors[self.serial]
break
if self.status == 'building':
def do_result(rows):
if len(rows) > 0:
self.status = rows[0]['state']
self.log.debug('check state from db : %s' %
self.status)
execute_query(("SELECT * FROM manor.manor_app_instance "
"where app_serial='%s'") % self.serial,
do_result)
if self.status == 'failure':
try:
r = redis_tool.get_it()
error = r.get('manage_error_$_%s' % self.serial)
if error is None:
error = '""'
error = json.loads(error)
self.current_msg = '{"serial":"%s","status":"failure","msg":[],"error":%s}' % (
self.serial, json.dumps(error))
thread.start_new(self.delete_error_msg, (r, ))
except:
self.log.error(generals.trace())
self.status = 'interrupt'
if self.status == 'thread_error':
self.current_msg = '{"serial":"%s","status":"error","msg":[]}' % self.serial
self.status = 'interrupt'
def check_mapup():
log = logging.getLogger('manor')
log.debug('clean mapup .......start')
rs = yield list_server(with_task=False)
ips = [_['network_info'][0]['ip'] for _ in rs]
log.debug(ips)
r = redis_tool.get_it()
_keys = r.keys('mapup_$_*')
for k in _keys:
ip = k.split('_$_')[3]
log.debug(ip)
if ip not in ips:
log.debug('delete %s' % k)
r.delete(k)
log.debug('clean mapup .......finish')
def info_watcher(self, msg):
while True:
self.log.debug(msg)
r = redis_tool.get_it()
_keys = r.keys('script_info_$_%s_$_*' % msg['message_token'])
for k in _keys:
token = k.split('_$_')[1]
date = k.split('_$_')[2]
if token == msg['message_token']:
self.write_message(
json.dumps({
'msg': r.get(k),
'date': date
}))
self.close()
break
time.sleep(1)
def send_message(logger, msg):
try:
target_ip = msg['target']['ip']
r = redis_tool.get_it()
farm_ips = r.keys('%s_$_%s_$_*_$_%s' % ('mapup', '*', target_ip))
logger.debug('farm_ips : %s', farm_ips)
for farm_ip in farm_ips:
logger.debug('forward msg to ip: %s' % r.get(farm_ip))
request = HTTPRequest('http://%s/msg' % r.get(farm_ip))
request.method = 'POST'
request.body = json.dumps(msg)
response = yield AsyncHTTPClient().fetch(request)
logger.debug('response is %s' % response.body)
rs = json.loads(response.body)
if not rs['success']:
logger.error(rs['msg'])
raise Exception('manor.error.send.command')
except:
logger.error(generals.trace())
def get(self, chunk):
try:
keys = self.get_key_params(self.request.path)
if keys[0] == 'get_stack':
from manor.screwdriver.vendor_ecloud import get_stack
rs = yield get_stack(keys[1])
self.write(json.dumps(rs.to_dict()))
if keys[0] == 'get_stack_resources':
from manor.streamlet import get_stack_resources
rs = yield get_stack_resources(keys[1])
self.write(json.dumps(rs))
if keys[0] == 'get_roles':
from manor.streamlet import get_roles
rs = yield get_roles(keys[1])
self.write(json.dumps(rs))
if keys[0] == 'get_resources_info':
from manor.streamlet import get_resources_info
rs = yield get_resources_info(keys[1])
self.write(json.dumps(rs))
if keys[0] == 'redis':
r = redis_tool.get_it()
if keys[1] == '':
_keys = r.keys('*_$_*')
else:
_keys = r.keys('%s_$_*' % keys[1])
rs = []
for k in _keys:
rs.append({"key": k, 'value': r.get(k)})
self.write(json.dumps(rs))
if keys[0] == 'download_path':
from manor.streamlet import download_path
rs = yield download_path(keys[1])
self.write(json.dumps(rs))
if keys[0] == 'list_instances':
from manor.services import app_service
rs = yield app_service.do_list_instance()
self.write(json.dumps(rs))
if keys[0] == 'list_app':
from manor.screwdriver.vendor_ecloud import list_app_resources
rs = yield list_app_resources(keys[1])
self.write(json.dumps(rs))
if keys[0] == 'clear_redis_msg':
r = redis_tool.get_it()
_keys = r.keys('%s_*' % keys[1])
for k in _keys:
r.delete(k)
self.write(json.dumps({"result": 'ok'}))
if keys[0] == 'rename':
from manor.screwdriver.vendor_ecloud import update_vm_display_name
yield update_vm_display_name(keys[1], keys[2])
self.write(json.dumps({"result": 'ok'}))
if keys[0] == 'app_count':
from manor.app_state_counter import get_state_count
rs = yield get_state_count()
self.write(json.dumps(rs))
except:
self.log.error(generals.trace())
self.write('error')
def set_rs(rows):
resources = []
for r in rows:
stack_id = r['stack_id']
group_name = r['group_name']
res = get_resources_info(stack_id)
while True:
if res.done():
break
time.sleep(0.1)
for info in res.result():
info['group_id'] = group_name
resources = resources + res.result()
for r in resources:
redis = redis_tool.get_it()
road_map = redis.keys('mapup*')
ip = r['addresses'].values()[0][0]['addr']
ips = [_.split('_$_')[3] for _ in road_map]
if ip in ips:
r['agent_state'] = 'online'
else:
r['agent_state'] = 'offline'
if len(resources) == len(
[_ for _ in resources if _['status'] == 'ACTIVE']):
self.current_msg = json.dumps({
'serial':
self.serial,
'status':
'normal',
'msg':
self.return_msg(resources)
})
if 'offline' in [
_['agent_state'] for _ in resources
if _['status'] == 'ACTIVE'
]:
self.current_msg = json.dumps({
'serial':
self.serial,
'status':
'offline',
'msg':
self.return_msg(resources)
})
elif len([
_ for _ in resources if _['status'] == 'ACTIVE'
]) != 0:
group_ids = [_['group_id'] for _ in resources]
down = False
for g in group_ids:
if len([
_ for _ in resources
if _['status'] == 'ACTIVE'
and _['group_id'] == g
]) == 0:
down = True
if not down:
self.current_msg = json.dumps({
'serial':
self.serial,
'status':
'part',
'msg':
self.return_msg(resources)
})
else:
self.current_msg = json.dumps({
'serial':
self.serial,
'status':
'down',
'msg':
self.return_msg(resources)
})
else:
self.current_msg = json.dumps({
'serial':
self.serial,
'status':
'stop',
'msg':
self.return_msg(resources)
})
r = redis_tool.get_it()
error = r.get('manage_error_$_%s' % self.serial)
if error:
self.log.debug(error)
current_msg = json.loads(self.current_msg)
current_msg['error'] = json.loads(error)
self.current_msg = json.dumps(current_msg)
thread.start_new(self.delete_error_msg, (r, ))
def __get_road_map(self):
r=redis_tool.get_it()
road_map=r.keys('mapup*')
return [_.split('_$_')[3] for _ in road_map]