def file(dest,
mode='644',
owner='root:root',
src=None,
src_file=None,
src_str=None,
override=False):
if src_str is not None:
src_file = __create_src_file(dest, src_str)
is_updated = False
with api.warn_only():
if exists(dest) and not override:
log.info('file "{0}" exists'.format(dest))
else:
if not src_file:
src_file = __get_src_file(dest, src_dirname='files', src=src)
if env.is_local:
sudo('cp {0} {1}'.format(src_file, dest))
else:
scp(src_file, dest)
is_updated = True
sudo('chmod -R {0} {1}'.format(mode, dest) +
' && chown -R {0} {1}'.format(owner, dest))
return is_updated
def local(cmd,
retry_ttl=0,
retry_interval=3,
capture=True,
timeout=60,
**kwargs):
def retry(*args, **kwargs):
log.info('failed cmd: {0}, ttl: {1}, sleep: {2}'.format(
cmd, retry_ttl, retry_interval))
if retry_ttl > 0:
time.sleep(retry_interval)
local(cmd, retry_ttl - 1, retry_interval, capture)
log_cmd = 'local> ' + cmd
api.env.cmd_history.append(log_cmd)
log.info(log_cmd)
print_for_test(log_cmd)
if api.env.is_test:
result = test_cmd(cmd)
else:
signal.signal(signal.SIGALRM, retry)
signal.alarm(timeout)
try:
result = api.local(cmd, capture=capture, **kwargs)
finally:
signal.alarm(0)
result_msg = 'return> {0}'.format(result.return_code)
log.info(result_msg)
print_for_test(result_msg)
return result
def local(cmd, retry_ttl=0, retry_interval=3, capture=True, timeout=60, **kwargs):
def retry(*args, **kwargs):
log.info('failed cmd: {0}, ttl: {1}, sleep: {2}'.format(
cmd, retry_ttl, retry_interval))
if retry_ttl > 0:
time.sleep(retry_interval)
local(cmd, retry_ttl - 1, retry_interval, capture)
log_cmd = 'local> ' + cmd
api.env.cmd_history.append(log_cmd)
log.info(log_cmd)
print_for_test(log_cmd)
if api.env.is_test:
result = test_cmd(cmd)
else:
signal.signal(signal.SIGALRM, retry)
signal.alarm(timeout)
try:
result = api.local(cmd, capture=capture, **kwargs)
finally:
signal.alarm(0)
result_msg = 'return> {0}'.format(result.return_code)
log.info(result_msg)
print_for_test(result_msg)
return result
def exec_command(cluster, run, command, **kwargs):
log.info('exec_command: {0}'.format(command))
action = command.split(' ', 1)
result = 0
if action[0] == 'sh':
result = cmd(action[1])[0]
elif action[0] == 'create':
env.is_local = True
env.host = 'localhost'
container.create(cluster[action[1]])
env.is_local = False
elif action[0] == 'delete':
env.is_local = True
env.host = 'localhost'
container.delete(cluster[action[1]])
env.is_local = False
elif action[0] == 'setup':
env.runs = [run]
env.user = CONF.job_user
env.password = CONF.job_password
CONF.user = CONF.job_user
CONF.password = CONF.job_password
setup(**kwargs)
elif action[0] == 'manage':
env.runs = [run]
env.user = CONF.job_user
env.password = CONF.job_password
CONF.user = CONF.job_user
CONF.password = CONF.job_password
manage(action[1], **kwargs)
log.info('result_command: {0}({1})'.format(command, result))
return result
def check_basic():
if env.is_test:
ip = {
'default': {
'ip': '192.168.1.1',
'dev': 'eth0',
},
'eth0': {
'ip': '127.0.0.1',
'dev': 'eth0',
},
'default_dev': {
'ip': '127.0.0.1',
'dev': 'eth0',
},
}
env.node['ip'] = ip
env.node['os'] = 'CentOS 7'
env.node['package_manager'] = 'yum'
env.node['service_manager'] = 'systemd'
return {
'msg': status.SUCCESS_CHECK_MSG,
'task_status': status.SUCCESS,
}
# Check ping
result = cmd(cmd_ping.format(env.host), retry_ttl=2, retry_interval=1)
if result[0] != 0:
log.warning(status.FAILED_CHECK_PING_MSG)
return {
'msg': status.FAILED_CHECK_PING_MSG,
'task_status': status.FAILED_CHECK_PING,
}
if not set_os():
log.warning(status.FAILED_CHECK_OS_MSG)
return {
'msg': status.FAILED_CHECK_OS_MSG,
'task_status': status.FAILED_CHECK_OS,
}
# Set IP
if not set_ip():
log.warning(status.FAILED_CHECK_SSH_MSG)
return {
'msg': status.FAILED_CHECK_SSH_MSG,
'task_status': status.FAILED_CHECK_SSH,
}
log.info(status.SUCCESS_CHECK_MSG)
return {
'msg': status.SUCCESS_CHECK_MSG,
'task_status': status.SUCCESS,
}
def check_basic():
if env.is_test:
ip = {
'default': {
'ip': '192.168.1.1',
'dev': 'eth0',
},
'eth0': {
'ip': '127.0.0.1',
'dev': 'eth0',
},
'default_dev': {
'ip': '127.0.0.1',
'dev': 'eth0',
},
}
env.node['ip'] = ip
env.node['os'] = 'CentOS 7'
env.node['package_manager'] = 'yum'
env.node['service_manager'] = 'systemd'
return {
'msg': status.SUCCESS_CHECK_MSG,
'task_status': status.SUCCESS,
}
# Check ping
result = cmd(cmd_ping.format(env.host), retry_ttl=2, retry_interval=1)
if result[0] != 0:
log.warning(status.FAILED_CHECK_PING_MSG)
return {
'msg': status.FAILED_CHECK_PING_MSG,
'task_status': status.FAILED_CHECK_PING,
}
if not set_os():
log.warning(status.FAILED_CHECK_OS_MSG)
return {
'msg': status.FAILED_CHECK_OS_MSG,
'task_status': status.FAILED_CHECK_OS,
}
# Set IP
if not set_ip():
log.warning(status.FAILED_CHECK_SSH_MSG)
return {
'msg': status.FAILED_CHECK_SSH_MSG,
'task_status': status.FAILED_CHECK_SSH,
}
log.info(status.SUCCESS_CHECK_MSG)
return {
'msg': status.SUCCESS_CHECK_MSG,
'task_status': status.SUCCESS,
}
def reboot(wait=60):
log_cmd = 'reboot> wait={0}'.format(wait)
api.env.cmd_history.append(log_cmd)
log.info(log_cmd)
print_for_test(log_cmd)
if api.env.is_test:
result = test_cmd('uptime')
else:
api.reboot(wait=wait)
result = api.run('uptime')
result_msg = 'return> {0} out>\n{1}'.format(result.return_code, result)
log.info(result_msg)
def reboot(wait=60):
log_cmd = 'reboot> wait={0}'.format(wait)
api.env.cmd_history.append(log_cmd)
log.info(log_cmd)
print_for_test(log_cmd)
if api.env.is_test:
result = test_cmd('uptime')
else:
api.reboot(wait=wait)
result = api.run('uptime')
result_msg = 'return> {0} out>\n{1}'.format(result.return_code, result)
log.info(result_msg)
def local(cmd, retry_ttl=0, retry_interval=3, capture=True, **kwargs):
log_cmd = 'local> ' + cmd
api.env.cmd_history.append(log_cmd)
log.info(log_cmd)
print_for_test(log_cmd)
if api.env.is_test:
result = test_cmd(cmd)
else:
result = api.local(cmd, capture=capture, **kwargs)
result_msg = 'return> {0}'.format(result.return_code)
log.info(result_msg)
print_for_test(result_msg)
return result
def file(dest, mode='644', owner='root:root', src=None, src_file=None, src_str=None,
override=False):
if src_str is not None:
src_file = __create_src_file(dest, src_str)
is_updated = False
with api.warn_only():
if exists(dest) and not override:
log.info('file "{0}" exists'.format(dest))
else:
if not src_file:
src_file = __get_src_file(dest, src_dirname='files', src=src)
if env.is_local:
sudo('cp {0} {1}'.format(src_file, dest))
else:
scp(src_file, dest)
is_updated = True
sudo('chmod -R {0} {1}'.format(mode, dest)
+ ' && chown -R {0} {1}'.format(owner, dest))
return is_updated
def cmd(cmd_str, retry_ttl=0, retry_interval=3):
log_cmd = 'cmd> ' + cmd_str
log.info(log_cmd)
print_for_test(log_cmd)
if api.env.is_test:
api.env.cmd_history.append(log_cmd)
result = (0, cmd_str)
else:
result = commands.getstatusoutput(cmd_str)
if result[0] != 0:
log.info('failed cmd: {0}, ttl: {1}, sleep: {2}'.format(
cmd_str, retry_ttl, retry_interval))
if retry_ttl > 0:
time.sleep(retry_interval)
cmd(cmd_str, retry_ttl - 1, retry_interval)
result_msg = 'return> {0[0]} out>\n{0[1]}'.format(result)
log.info(result_msg)
print_for_test(result_msg)
return result
def cmd(cmd_str, retry_ttl=0, retry_interval=3):
log_cmd = 'cmd> ' + cmd_str
log.info(log_cmd)
print_for_test(log_cmd)
if api.env.is_test:
api.env.cmd_history.append(log_cmd)
result = (0, cmd_str)
else:
result = commands.getstatusoutput(cmd_str)
if result[0] != 0:
log.info('failed cmd: {0}, ttl: {1}, sleep: {2}'.format(
cmd_str, retry_ttl, retry_interval))
if retry_ttl > 0:
time.sleep(retry_interval)
cmd(cmd_str, retry_ttl - 1, retry_interval)
result_msg = 'return> {0[0]} out>\n{0[1]}'.format(result)
log.info(result_msg)
print_for_test(result_msg)
return result
def template(dest, mode='644', owner='root:root', data={},
src=None, src_file=None, src_str=None, insert_eol_crlf=True):
template_data = {}
template_data.update(data)
template_data['node'] = env.node
template_data['cluster'] = env.cluster
is_updated = False
if src_str:
src_file = __create_src_file(dest, src_str)
if not src_file:
src_file = __get_src_file(dest, src_dirname='templates', src=src)
timestamp = int(time.time())
tmp_path = 'templates/{0}_{1}'.format(dest, timestamp)
tmp_path = os.path.join(CONF._remote_storage_dir, tmp_path)
# local_tmp_file = os.path.join(conf.TMP_DIR, env.host, tmp_path)
local_tmp_file = CONF._tmp_dir + '/' + env.host + '/' + tmp_path
local_tmp_dir = local_tmp_file.rsplit('/', 1)[0]
mkdir(local_tmp_dir, is_local=True, use_sudo=False)
template = j2_env.get_template(src_file)
if not env.is_test:
with open(local_tmp_file, 'w') as exf:
exf.write(template.render(**template_data).encode('utf-8'))
if insert_eol_crlf:
exf.write('\n')
if env.is_local:
with api.warn_only():
if exists(dest):
result = sudo('diff {0} {1}'.format(dest, local_tmp_file))
if result.return_code != 0:
sudo('mv {0} {1}_old'.format(dest, local_tmp_file))
sudo('cp -f {0} {1}'.format(local_tmp_file, dest))
is_updated = True
else:
log.info('No change')
else:
sudo('diff /dev/null {1}'.format(dest, local_tmp_file))
sudo('cp -f {0} {1}'.format(local_tmp_file, dest))
is_updated = True
else:
tmp_dir = tmp_path.rsplit('/', 1)[0]
mkdir(tmp_dir, mode='770', owner='{0}:root'.format(env.user))
scp(local_tmp_file, tmp_path)
with api.warn_only():
if exists(dest):
result = sudo('diff {0} {1}'.format(dest, tmp_path))
if result.return_code != 0:
sudo('mv {0} {1}_old'.format(dest, tmp_path))
sudo('cp -f {0} {1}'.format(tmp_path, dest))
is_updated = True
else:
log.info('No change')
else:
sudo('diff /dev/null {1}'.format(dest, tmp_path))
sudo('cp -f {0} {1}'.format(tmp_path, dest))
is_updated = True
sudo('sh -c "chmod {0} {1}'.format(mode, dest)
+ ' && chown {0} {1}"'.format(owner, dest))
return is_updated
def template(dest,
mode='644',
owner='root:root',
data={},
src=None,
src_file=None,
src_str=None,
insert_eol_crlf=True):
template_data = {}
template_data.update(data)
template_data['node'] = env.node
template_data['cluster'] = env.cluster
is_updated = False
if src_str:
src_file = __create_src_file(dest, src_str)
if not src_file:
src_file = __get_src_file(dest, src_dirname='templates', src=src)
timestamp = int(time.time())
tmp_path = 'templates/{0}_{1}'.format(dest, timestamp)
tmp_path = os.path.join(CONF._remote_storage_dir, tmp_path)
# local_tmp_file = os.path.join(conf.TMP_DIR, env.host, tmp_path)
local_tmp_file = CONF._tmp_dir + '/' + env.host + '/' + tmp_path
local_tmp_dir = local_tmp_file.rsplit('/', 1)[0]
mkdir(local_tmp_dir, is_local=True, use_sudo=False)
template = j2_env.get_template(src_file)
if not env.is_test:
with open(local_tmp_file, 'w') as exf:
exf.write(template.render(**template_data).encode('utf-8'))
if insert_eol_crlf:
exf.write('\n')
if env.is_local:
with api.warn_only():
if exists(dest):
result = sudo('diff {0} {1}'.format(dest, local_tmp_file))
if result.return_code != 0:
sudo('mv {0} {1}_old'.format(dest, local_tmp_file))
sudo('cp -f {0} {1}'.format(local_tmp_file, dest))
is_updated = True
else:
log.info('No change')
else:
sudo('diff /dev/null {1}'.format(dest, local_tmp_file))
sudo('cp -f {0} {1}'.format(local_tmp_file, dest))
is_updated = True
else:
tmp_dir = tmp_path.rsplit('/', 1)[0]
mkdir(tmp_dir, mode='770', owner='{0}:root'.format(env.user))
scp(local_tmp_file, tmp_path)
with api.warn_only():
if exists(dest):
result = sudo('diff {0} {1}'.format(dest, tmp_path))
if result.return_code != 0:
sudo('mv {0} {1}_old'.format(dest, tmp_path))
sudo('cp -f {0} {1}'.format(tmp_path, dest))
is_updated = True
else:
log.info('No change')
else:
sudo('diff /dev/null {1}'.format(dest, tmp_path))
sudo('cp -f {0} {1}'.format(tmp_path, dest))
is_updated = True
sudo('sh -c "chmod {0} {1}'.format(mode, dest) +
' && chown {0} {1}"'.format(owner, dest))
return is_updated
def retry(*args, **kwargs):
log.info('failed cmd: {0}, ttl: {1}, sleep: {2}'.format(
cmd, retry_ttl, retry_interval))
if retry_ttl > 0:
time.sleep(retry_interval)
local(cmd, retry_ttl - 1, retry_interval, capture)
def retry(*args, **kwargs):
log.info('failed cmd: {0}, ttl: {1}, sleep: {2}'.format(
cmd, retry_ttl, retry_interval))
if retry_ttl > 0:
time.sleep(retry_interval)
local(cmd, retry_ttl - 1, retry_interval, capture)
def run_func(func_names=[], *args, **kwargs):
fabrun_filter = []
is_filter = False
env.is_test = False
env.is_help = False
if len(args) > 0:
if args[0] == 'test':
env.is_test = True
args = args[1:]
elif args[0] == 'help':
env.is_help = True
args = args[1:]
if 'f' in kwargs:
fabrun_filter = kwargs['f'].split('+')
fabrun_filter = [re.compile(f) for f in fabrun_filter]
is_filter = True
func_patterns = [re.compile(name) for name in func_names]
host_filter = {}
for run in env.runs:
cluster_name = run['cluster']
cluster = env.cluster_map[run['cluster']]
env.cluster = cluster
log.init_logger(cluster_name)
for cluster_run in run['runs']:
run_results = []
script_name = cluster_run['fabscript']
if is_filter:
for f in fabrun_filter:
if f.search(script_name):
break
else:
continue
if env.is_check or env.is_manage:
# 二重実行を防ぐ
hosts = host_filter.get(script_name, [])
tmp_hosts = list(set(cluster_run['hosts']) - set(hosts))
cluster_run['hosts'] = tmp_hosts
hosts.extend(tmp_hosts)
host_filter[script_name] = hosts
if len(cluster_run['hosts']) == 0:
continue
hosts = []
for host in cluster_run['hosts']:
env.node_map[host] = env.node_map.get(host, {
'host': host,
'bootstrap_status': -1,
})
if env.node_map[host]['bootstrap_status'] != status.FAILED_CHECK:
hosts.append(host)
env.script_name = script_name
env.hosts = hosts
# override env
override_env = env.cluster.get('env', {})
override_env.update(cluster_run.get('env', {}))
default_env = {}
for key, value in override_env.items():
default_env[key] = getattr(env, key, None)
setattr(env, key, value)
env.cluster_status = env.cluster['__status']
env.node_status_map = env.cluster_status['node_map']
env.fabscript_status_map = env.cluster_status['fabscript_map']
env.fabscript = env.fabscript_status_map[script_name]
log.info('hosts: {0}'.format(env.hosts))
log.info('run: {0}: {1}'.format(script_name, env.fabscript))
log.debug('node_status_map: {0}'.format(env.node_status_map))
# check require
require = env.cluster['fabscript_map'][script_name]['require']
if env.is_setup:
is_require = True
for script, status_code in require.items():
required_status = env.fabscript_status_map[script]['status']
if required_status != status_code:
log.error('Require Error\n'
+ '{0} is require {1}:{2}.\nbut {1} status is {3}.'.format(
script_name, script, status_code, required_status))
is_require = False
break
if not is_require:
break
script = '.'.join([CONF.fabscript_module, script_name.replace('/', '.')])
# importlibは、2.7以上じゃないと使えない
# module = importlib.import_module(script)
module = __import__(script, globals(), locals(), ['*'], -1)
module_funcs = []
for member in inspect.getmembers(module):
if inspect.isfunction(member[1]):
module_funcs.append(member[0])
if env.is_help and len(func_patterns) == 0:
for candidate in module_funcs:
func = getattr(module, candidate)
print 'Task: {0}'.format(func.__name__)
print func.__doc__
continue
# func_patterns にマッチしたタスク関数をすべて実行する
is_expected = False
is_contain_unexpected = False
for func_pattern in func_patterns:
for candidate in module_funcs:
if not func_pattern.match(candidate):
continue
func = getattr(module, candidate)
# taskデコレータの付いているものだけ実行する
if not hasattr(func, 'is_task') or not func.is_task:
continue
if env.is_help:
print 'Task: {0}'.format(func.__name__)
print func.__doc__
continue
results = api.execute(func, *args, **kwargs)
# check results
data_map = {}
tmp_status = None
is_contain_failed = False
for host, result in results.items():
env.node_map[host].update(result['node'])
if not result or type(result) is not DictType:
result = {}
node_result = env.node_status_map[host]['fabscript_map'][script_name]
result_status = result.get('status')
task_status = result.get('task_status', status.SUCCESS)
msg = result.get('msg')
if msg is None:
if task_status is status.SUCCESS:
msg = status.FABSCRIPT_SUCCESS_MSG
else:
msg = status.FABSCRIPT_FAILED_MSG
if func.is_bootstrap:
if task_status == status.FAILED_CHECK or \
task_status == status.FAILED_CHECK_PING:
env.node_map[host]['bootstrap_status'] = status.FAILED_CHECK
else:
env.node_map[host]['bootstrap_status'] = status.SUCCESS
node_result['task_status'] = task_status
tmp_data_map = result.get('data_map')
if tmp_data_map is not None:
for map_name, tmp_map_data in tmp_data_map.items():
if tmp_map_data['type'] == 'table':
map_data = data_map.get(map_name, {
'name': map_name,
'type': 'table',
'data': [],
})
tmp_data = {'!!host': host}
tmp_data.update(tmp_map_data['data'])
map_data['data'].append(tmp_data)
data_map[map_name] = map_data
elif tmp_map_data['type'] == 'multi-table':
map_data = data_map.get(map_name, {
'name': map_name,
'type': 'multi-table',
'data': [],
})
tmp_data = {'!!host': host}
tmp_data.update(tmp_map_data['data'])
map_data['data'].append(tmp_data)
data_map[map_name] = map_data
elif tmp_map_data['type'] == 'line-chart':
map_data = data_map.get(map_name, {
'name': map_name,
'type': 'line-chart',
'data': [],
})
map_data['ex_data'] = tmp_map_data['ex_data']
map_data['layout'] = tmp_map_data['layout']
tmp_data = {'!!host': host}
tmp_data.update(tmp_map_data['data'])
map_data['data'].append(tmp_data)
data_map[map_name] = map_data
if env.is_setup:
node_result['msg'] = msg
if result_status is not None:
tmp_status = result_status
node_result['status'] = result_status
expected = cluster_run['expected_status']
if expected == result_status:
is_expected = True
log.info('{0}: {1} is expected status.'.format(
host, msg, result_status))
else:
is_contain_unexpected = True
log.error('expected status is {0}, bad status is {1}.'.format( # noqa
expected, result_status), host)
elif env.is_check:
node_result['check_msg'] = msg
if result_status is None:
result_status = status.SUCCESS
node_result.update({
'check_status': result_status,
})
if result_status != status.SUCCESS:
log.error('Failed check {0}.{1} [{2}]. {3}'.format( # noqa
script_name, candidate, result_status, msg), host)
if task_status == status.SUCCESS:
log.info('Success task {0}.{1} [{2}]. {3}'.format(
script_name, candidate, task_status, msg), host)
else:
log.error('Failed task {0}.{1} [{2}]. {3}'.format(
script_name, candidate, task_status, msg), host)
is_contain_failed = True
# end for host, result in results.items():
if len(data_map) > 0:
util.dump_datamap(data_map)
if is_contain_failed:
log.error('Failed task {0}.{1}. Exit setup.'.format(
script_name, candidate))
util.dump_status()
exit()
if tmp_status is not None:
env.fabscript['tmp_status'] = tmp_status
# for candidate in module_funcs:
# for func_pattern in func_patterns:
if env.is_setup:
if is_expected and not is_contain_unexpected or cluster_run['expected_status'] == 0:
env.cluster['__status']['fabscript_map'][script_name] = {
'status': cluster_run['expected_status'],
'task_status': status.SUCCESS,
}
util.dump_status()
else:
log.error('bad status.')
exit()
elif env.is_check:
env.cluster['__status']['fabscript_map'][script_name]['task_status'] = status.SUCCESS # noqa
util.dump_status()
elif env.is_manage:
env.cluster['__status']['fabscript_map'][script_name]['task_status'] = status.SUCCESS # noqa
util.dump_status()
# reset env
for key, value in default_env.items():
if value is not None:
setattr(env, key, value)