def do_delete_app(serial):
log = logging.getLogger('manor')
def do_delete(rows):
for r in rows:
log.debug('delete stack %s' % r['stack_id'])
stack_util.delete_stack(r['stack_id'])
execute_query(
"SELECT * FROM manor.manor_stacks where app_serial='%s'" % serial,
do_delete)
execute("DELETE FROM manor.manor_app_instance WHERE app_serial=%s", serial)
execute("DELETE FROM manor.manor_stacks WHERE app_serial=%s", serial)
execute("DELETE FROM manor.manor_app_group_seq WHERE app_serial=%s",
serial)
log_file = '%s/stream_%s.log' % (cfgutils.getval('log', 'path'), serial)
if os.path.isfile(log_file):
os.remove(log_file)
if os.path.exists('%s/%s' % (cfgutils.getval('log', 'path'), serial)):
shutil.rmtree('%s/%s' % (cfgutils.getval('log', 'path'), serial))
path = cfgutils.getval('app', 'instance_path')
if os.path.isfile('%s/%s.yaml' % (path, serial)):
os.remove('%s/%s.yaml' % (path, serial))
def get_connection():
user = cfgutils.getval('db', 'user')
password = cfgutils.getval('db', 'password')
host = cfgutils.getval('db', 'host')
con = db.connect(host, user, password, 'manor')
con.set_character_set('utf8')
return con
def get_it():
global __pool
if __pool is None:
__pool=redis.ConnectionPool(
host=cfgutils.getval('redis','host'),
port=cfgutils.getval('redis','port'),db=0)
return redis.Redis(connection_pool=__pool)
def init():
log_formatter = ('%(asctime)s %(filename)s(line:%(lineno)d) - '
'[%(levelname)s] - %(message)s')
log_date_format = '%Y-%m-%d %H:%M:%S'
log_handler = logging.FileHandler(cfgutils.getval('log', 'manor'))
log_handler.setFormatter(
logging.Formatter(log_formatter, datefmt=log_date_format))
logger_manor = logging.getLogger('manor')
logger_manor.setLevel(cfgutils.getval('log', 'level'))
logger_manor.addHandler(log_handler)
def get_client():
from manor.util import cfgutils
auth = v2.Password(auth_url=cfgutils.getval('heat', 'auth_url'),
username=cfgutils.getval('heat', 'username'),
password=cfgutils.getval('heat', 'password'),
tenant_name=cfgutils.getval('heat', 'tenant_name'))
sess = session.Session(auth=auth)
region = cfgutils.getval('heat', 'region')
if region == 'None':
region = None
return client.Client('2', session=sess, region_name=region)
def create_client():
auth=v2.Password(
auth_url=cfgutils.getval('heat','auth_url'),
username=cfgutils.getval('heat','username'),
password=cfgutils.getval('heat','password'),
tenant_name=cfgutils.getval('heat','tenant_name')
)
sess=session.Session(auth=auth)
return Client(
'1',
endpoint=cfgutils.getval('heat','end_point')+'/'+sess.get_project_id(),
token=sess.get_token()
)
def __init__(self, template, stream_name, serial):
"""
:param template:模板,python对象
:param stream_name: 流程的名称.因为一个模板中有很多流程,但是只有一个安装流程.
其他的都是管理流程.
:param serial: 任务的序列号,此序列号会返回给前台,前台用此序列号来追踪任务.
"""
# 避免静态引入时导致的配置错误
self._thread_pool = generals.get_thread_pool()
self.action_name = stream_name
self.serial = str(serial)
# 保存所有的步骤,防止重复执行。
self._streamlet_instance = {}
logger = logging.getLogger('stream')
formatter = logging.Formatter('%(asctime)s - %(lineno)d - %(message)s')
ch = logging.FileHandler('%s/stream_%s.log' %
(cfgutils.getval('log', 'path'), self.serial))
logger.setLevel(logging.DEBUG)
ch.setLevel(logging.DEBUG)
ch.setFormatter(formatter)
logger.addHandler(ch)
self.log = logger
self.parser = TmpParser(template)
# 中断线程的执行
self.interrupt = False
self.log.debug('execute action: %s' % stream_name)
def streamlet_list(self, step_type, *resgs):
app_path = cfgutils.getval('app', 'streamlet_path')
files = [f.replace('.yaml', '') for f in os.listdir(app_path)]
if step_type == 'deploy':
del files[files.index('delete_node')]
del files[files.index('start_node')]
del files[files.index('stop_node')]
self.response(generals.gen_response(files))
def delete_template(self, name):
template_path = cfgutils.getval('app', 'template_path')
with open('%s/%s.yaml' % (template_path, name)) as f:
tmp = f.read()
t_content = yaml.safe_load(tmp)
template = '%s/%s.yaml' % (template_path, name)
shutil.rmtree('%s/_%s' % (template_path, name))
os.remove(template)
optLog.write(self.request, optLog.Type.APP_TEMPLATE, name,
Operator.DELETE, '%s' % t_content['label'])
self.response(generals.gen_response('ok'))
def list_app_template_detail(self):
rs = []
try:
app_path = cfgutils.getval('app', 'template_path')
self.log.debug('get templates path: %s' % app_path)
for name in os.listdir(app_path):
if not os.path.isdir(app_path + '/' +
name) and name != '.yaml':
rs.append(load_template(app_path, name))
self.log.debug('get templates: %s' % json.dumps(rs))
rs = generals.gen_response(rs)
except:
self.log.error(generals.trace())
self.response(rs)
def __init__(self, node_id, params, serial):
self.stack_id = None
self.node_id = node_id
self.serial = serial
logger = logging.getLogger('streamlet_' + serial + '_' + node_id)
path = '%s/%s' % (cfgutils.getval('log', 'path'), serial)
if not os.path.exists(path):
os.mkdir(path)
formatter = logging.Formatter(
'%(asctime)s - %(filename)s(line:%(lineno)d)'
' - [%(levelname)s] - %(message)s')
ch = logging.FileHandler(
('%s/%s/%s.log' %
(cfgutils.getval('log', 'path'), serial, node_id)))
logger.setLevel(logging.DEBUG)
ch.setLevel(logging.DEBUG)
ch.setFormatter(formatter)
logger.addHandler(ch)
logger.info('app serial: %s' % self.serial)
logger.info('origin params:\n %s' % pyaml.dumps(params))
self.log = logger
self.params = self.merge_params(params)
# 所有的步骤,都不应该被执行两次
self.executed = False
def change_status(self, name, body):
t_path = cfgutils.getval('app', 'template_path')
with open('%s/%s.yaml' % (t_path, name)) as f:
tmp = f.read()
t_content = yaml.safe_load(tmp)
value = json.loads(self.request.body)['value']
t_content['status'] = value
tmp_f = str(uuid.uuid1())
with codecs.open('%s/%s' % (t_path, tmp_f), 'w', 'utf-8') as f:
f.write(yaml.safe_dump(t_content))
os.rename('%s/%s' % (t_path, tmp_f), '%s/%s.yaml' % (t_path, name))
optLog.write(self.request, optLog.Type.APP_TEMPLATE, name,
Operator.ONLINE if value == 1 else Operator.OFFLINE,
'%s' % t_content['label'])
self.response(generals.gen_response('ok'))
def get_app_template(self, serial, *args):
instance_path = cfgutils.getval('app', 'instance_path')
with open('%s/%s.yaml' %
(instance_path, serial.replace(' ', ''))) as f:
rs = f.read()
content = yaml.safe_load(rs)
for a in content['action']:
pk = a['streamlet'].keys()
for k in pk:
c_a_p = a['streamlet'][k]['params']
s = [json.dumps(_) for _ in c_a_p]
ss = set(s)
a['streamlet'][k]['params'] = [json.loads(_) for _ in list(ss)]
self.response(generals.gen_response(content))
pars.add_argument('-serial', required=True, help='app serial ...')
pars.add_argument('-tmp_name', required=True, help='templates name')
pars.add_argument('-action_name', required=True)
args = pars.parse_args()
try:
os.chdir(sys.path[0])
sys.path.append('../common')
from manor.util import generals
from manor.util import cfgutils
cfgutils.init('../etc/manor.conf')
instance_path = cfgutils.getval('app', 'instance_path')
template_path = cfgutils.getval('app', 'template_path')
serial = args.serial
template = args.tmp_name
action_name = args.action_name
content = load_template(template_path, template + '.yaml')
with open('%s/%s' % (instance_path, serial + '.yaml')) as f:
i_c = f.read()
i_c = yaml.safe_load(i_c)
pp = {}
for a in content['action']:
class C(object):
SUCCESS_FLAG = cfgutils.getval('stream', 'stack_success_flag')
CREATING_FLAG = cfgutils.getval('stream', 'stack_creating_flag')
DOWNLOAD_PATH = cfgutils.getval('stream', 'client_download_path')
TIME_OUT = cfgutils.getval('stream', 'streamlet_time_out')
def create_instance(request, name, action_name, seq):
log = logging.getLogger('manor')
try:
log.debug(request.body)
instance_path = cfgutils.getval('app', 'instance_path')
template_path = cfgutils.getval('app', 'template_path')
params = json.loads(request.body)
app_name = params['app_name']
if 'type' in params:
action_type = params['type']
else:
action_type = 'deploy'
if action_type == 'deploy':
if not os.path.isfile('%s/%s.yaml' % (template_path, name)):
raise Exception('error.manor.templates.not.exist')
content = load_template(template_path,
name.replace(' ', '') + '.yaml')
else:
with open(
'%s/%s.yaml' %
(instance_path, params['app_name'].replace(' ', ''))) as f:
rs = f.read()
content = yaml.safe_load(rs)
if action_name not in [_['name'] for _ in content['action']]:
raise Exception('error.manor.action.not.exist')
if 'app_description' not in content:
content['app_description'] = ''
if params['app_description']:
content['app_description'] = params['app_description']
# 合并参数
merge_params(action_name, content, log, params)
check_template(content)
info_token_list = []
if action_type == 'deploy':
serial = uuid.uuid1()
execute((
"INSERT INTO manor.manor_app_instance"
"(template_name,app_name,app_serial,state,app_description,app_id)"
"VALUES(%s,%s,%s,%s,%s,%s)"),
(name, app_name, serial, 'building',
content['app_description'], seq))
# save origin templates.
log.debug('save app templates : %s' % str(serial))
with codecs.open(
'%s/%s.yaml' %
(instance_path, str(serial).replace(' ', '')), 'w+',
'utf-8') as f:
f.write(yaml.safe_dump(content))
else:
# 执行管理流程
serial = params['app_name']
streamlet_key = params['params'].keys()
for k in streamlet_key:
s_ps = params['params'][k]['params']
for s_p in s_ps:
if 'info_return' in s_p and s_p['info_return']:
token = str(uuid.uuid1())
s_ps.append({'info_token': token})
info_token_list.append(token)
# 为了token
merge_params(action_name, content, log, params)
logging.getLogger('manor').debug(pyaml.dumps(content))
stream.execute(content, action_name, str(serial))
if action_type == 'deploy':
optLog.write(request, optLog.Type.APP_INSTANCE, seq,
Operator.CREATE, '%s' % app_name)
else:
action_label = [
_ for _ in content['action'] if _['name'] == action_name
][0]['label']
def get_result(rows):
app_name = rows[0]['app_name']
optLog.write(request, optLog.Type.APP_INSTANCE, seq,
Operator.EXECUTE_ACTION,
'%s %s' % (app_name, action_label))
execute_query(
"select * from manor.manor_app_instance where app_serial='%s'"
% serial, get_result)
return info_token_list, serial
except Exception as e:
log.error(generals.trace())
raise e
def get_thread_pool():
global __Executor
if __Executor is None:
__Executor = ThreadPoolExecutor(
max_workers=cfgutils.getval('app', 'thread_pool'))
return __Executor
def save_template(self, action):
app_path = cfgutils.getval('app', 'template_path')
body = json.loads(self.request.body)
try:
check_template(body)
# 为流程生成ID
for a in body['action']:
if 'name' not in a or a['name'] == '':
seq = yield sequence.number_seq('SA', '')
a['name'] = 'SA-%s' % seq
logging.getLogger('manor').debug(
'generate action name: %s' % a['name'])
else:
logging.getLogger('manor').debug('action name: %s' %
a['name'])
if action == 'create':
operation_log_type = Operator.CREATE
seq = yield sequence.number_seq('TM', '')
body['name'] = 'TM-%s' % seq
body['status'] = 0
else:
operation_log_type = Operator.UPDATE
if 'name' not in body:
raise Exception('error.manor.templates.no.name')
if body['name'] == '':
raise Exception('error.manor.templates.name.is.empty')
if not os.path.isfile('%s/%s.yaml' % (app_path, body['name'])):
raise Exception('error.manor.templates.update.not.exist')
for a in body['action']:
if 'deploy' == a['type']:
a['label'] = 'install_stream'
if 'label' not in a:
raise Exception(
'error.manor.templates.action.no.label')
if a['label'] == '':
raise Exception(
'error.manor.templates.action.label.is.empty')
script_path = '%s/_%s' % (app_path, body['name'])
if not os.path.isdir(script_path):
os.mkdir(script_path)
for a in body['action']:
for k in a['streamlet'].keys():
if k.split('$')[0] == 'execute_script':
node_params = a['streamlet'][k]['params']
for p in node_params:
if 'execute_script_content' in p:
content = p['execute_script_content']
self.log.debug('get script content:\n%s' %
content)
e_name = get_script_ex_name(p)
with codecs.open(
'%s/%s%s' % (script_path, k, e_name),
'w+', 'utf-8') as f:
f.write(content)
del p['execute_script_content']
tmp_f = str(uuid.uuid1())
with codecs.open('%s/%s' % (app_path, tmp_f), "w+",
'utf-8') as text_file:
text_file.write(yaml.safe_dump(body))
os.rename('%s/%s' % (app_path, tmp_f),
'%s/%s.yaml' % (app_path, body['name']))
optLog.write(self.request, optLog.Type.APP_TEMPLATE, body['name'],
operation_log_type, '%s' % body['label'])
self.response(generals.gen_response("ok"))
except Exception as e:
logging.getLogger('manor').error(generals.trace())
raise e
