def run(cmd, retry_ttl=0, retry_interval=3, **kwargs):
log_cmd = 'run> ' + cmd
api.env.cmd_history.append(log_cmd)
log.debug(log_cmd)
print_for_test(log_cmd)
if api.env.is_test:
result = test_cmd(cmd)
else:
if env.is_local:
result = local(cmd)
else:
result = api.run(cmd, **kwargs)
result_msg = 'return> {0} out>\n{1}'.format(result.return_code, result)
log.debug(result_msg)
print_for_test(result_msg)
return result
def run(cmd, retry_ttl=0, retry_interval=3, **kwargs):
log_cmd = 'run> ' + cmd
api.env.cmd_history.append(log_cmd)
log.debug(log_cmd)
print_for_test(log_cmd)
if api.env.is_test:
result = test_cmd(cmd)
else:
if env.is_local:
result = local(cmd)
else:
result = api.run(cmd, **kwargs)
result_msg = 'return> {0} out>\n{1}'.format(result.return_code, result)
log.debug(result_msg)
print_for_test(result_msg)
return result
def load_runs(query,
find_depth=1,
use_tmp_node=False,
fubruns=None,
cluster_data=None):
""" queryに基づいて、nodeを読み込む
env.runs にクラスタごとの実行タスクリスト
env.cluster_mapにクラスタごとのデータ情報を格納
"""
env.host_script_map = {}
# query rule
# <cluster_name>/<node_search_pattern>
is_contain_candidates = True
splited_query = query.rsplit('/', 1)
cluster_name = splited_query[0]
if len(splited_query) > 1 and splited_query[1]:
host_pattern = splited_query[1]
if host_pattern.find('!') == 0:
is_contain_candidates = False
host_pattern = host_pattern[1:]
candidates = [
re.compile(c.replace('*', '.*'))
for c in get_expanded_hosts(host_pattern)
]
else:
host_pattern = ''
candidates = None
if use_tmp_node:
cluster_dir = os.path.join(CONF._tmp_node_dir, cluster_name)
_create_tmp_cluster_dir(cluster_dir, cluster_name, fubruns,
cluster_data)
else:
cluster_dir = os.path.join(CONF._node_dir, cluster_name)
# load cluster data from node dir
runs = []
depth = 1
for root, dirs, files in os.walk(CONF._node_dir):
if root.find(cluster_dir) == -1:
continue
# load cluster
cluster_name = root.split(CONF._node_dir)[1]
if cluster_name.find('/') == 0:
cluster_name = cluster_name[1:]
# default cluster dict
cluster = {
'name': cluster_name,
'host_pattern': host_pattern,
'__status': {
'node_map': {},
'fabscript_map': {},
},
}
cluster.update(include_cluster(cluster_name))
# node_map is definition of nodes in cluster
cluster_node_map = cluster.get('node_map', {})
# status is setup status of cluster
cluster_status = cluster['__status']
# status of nodes
node_status_map = cluster_status['node_map']
# status of fabscripts
fabscript_status_map = cluster_status['fabscript_map']
# expand hosts of node_map, and load fabscript of node
# tmp_fabscript_mapに実行候補のfabscriptをすべて格納する
tmp_fabscript_map = {}
for role, cluster_node in cluster_node_map.items():
node_hosts = cluster_node['hosts']
tmp_hosts = []
for node_host in node_hosts:
node_host = decode_data(node_host)
if type(node_host) is ListType:
for tmp_node_host in node_host:
tmp_hosts.extend(get_expanded_hosts(tmp_node_host))
else:
tmp_hosts.extend(get_expanded_hosts(node_host))
cluster_node['hosts'] = tmp_hosts
for tmp_fabscript in cluster_node['fabruns']:
# fabruns = [<cluster>/<python_mod_path>, <cluster>/<python_mod_path>]
fabscript = tmp_fabscript
fabscript_kwargs = {}
if isinstance(tmp_fabscript, dict):
for f, kwargs in tmp_fabscript.items():
fabscript = f
fabscript_kwargs = kwargs
break
update_host_script_map(tmp_hosts, fabscript,
fabscript_kwargs)
# fabscriptは、クラスタ単位で管理する
fabscript_cluster = fabscript.rsplit('/', 1)[0]
fabscript_mod = fabscript.rsplit('/', 1)[1]
if fabscript not in tmp_fabscript_map:
if fabscript not in fabscript_status_map:
# このクラスタにおいて実行履歴のないfabscriptはデフォルト値を登録する
fabscript_status_map[fabscript] = {
'status': 0,
'task_status': status.REGISTERED,
}
# デフォルト値
fabscript_data = {
'fabscript': fabscript,
'fabscript_kwargs': fabscript_kwargs,
'hosts': [],
'status_flow': [0],
'require': {},
'required': [],
}
fabscript_file = os.path.join(CONF._fabscript_module_dir,
fabscript_cluster,
CONF._fabscript_yaml)
if os.path.exists(fabscript_file):
with open(fabscript_file, 'r') as f:
tmp_data = yaml.load(f)
fabscript_data.update(
tmp_data.get(fabscript_mod, {}))
tmp_fabscript_map[fabscript] = fabscript_data
tmp_fabscript_map[fabscript]['hosts'].extend(tmp_hosts)
tmp_fabscript_map[fabscript]['env'] = cluster_node.get(
'env', {})
# tmp_fabscript_map を解析して、スクリプトの実行順序を組み立てる
# 実行順序はcluster_runsに配列として格納する
cluster_runs = []
# スクリプトの依存関係を再帰的に求める
def resolve_require(fabscript, data):
for require_script, require_status in data['require'].items():
require = tmp_fabscript_map.get(require_script)
if require:
require['required'].append(fabscript)
if (require['require']) > 0:
resolve_require(fabscript, require)
else:
require = fabscript_status_map.get(require_script)
if require:
if require['status'] == require_status and require[
'task_status'] == 0:
return
log.error(
'Require Error\nCluster: {0}\nFabscript: {1} require {2}:{3}'
.format(cluster_name, fabscript, require_script,
require_status))
exit()
for fabscript, data in tmp_fabscript_map.items():
resolve_require(fabscript, data)
# スクリプト同士の依存関係が求められたので、これを使って正しい実行順序を組立てる
for fabscript, data in tmp_fabscript_map.items():
index = len(cluster_runs)
for required in data['required']:
for i, run in enumerate(cluster_runs):
if run['fabscript'] == required:
if i < index:
index = i
# セットアップの対象となるノードの初期化
tmp_hosts = []
for host in data['hosts']:
if candidates:
is_match = False
for candidate in candidates:
if candidate.search(host):
is_match = True
break
if is_match and not is_contain_candidates:
continue
elif not is_match and is_contain_candidates:
continue
tmp_hosts.append(host)
node = node_status_map.get(host, {'fabscript_map': {}})
if fabscript not in node['fabscript_map']:
node['fabscript_map'][fabscript] = {
'status': 0,
'check_status': -1,
'msg': '',
'check_msg': '',
}
if env.is_setup:
node['fabscript_map'][fabscript].update({
'msg':
status.REGISTERED_MSG,
'task_status':
status.REGISTERED,
})
elif env.is_check:
node['fabscript_map'][fabscript].update({
'check_msg':
status.REGISTERED_MSG,
'task_status':
status.REGISTERED,
})
node_status_map[host] = node
if len(tmp_hosts) > 0:
fabscript_status_map[fabscript][
'task_status'] = status.REGISTERED
data['hosts'] = tmp_hosts
cluster_status['node_map'] = node_status_map
fabscript_status = fabscript_status_map[fabscript]['status']
for flow in data['status_flow']:
if fabscript_status < flow or flow == data['status_flow'][-1]:
tmp_data = data.copy()
tmp_data['expected_status'] = flow
cluster_runs.insert(index, tmp_data)
index += 1
runs.append({
'cluster': cluster_name,
'runs': cluster_runs,
})
cluster_status['fabscript_map'] = fabscript_status_map
cluster['fabscript_map'] = tmp_fabscript_map
env.cluster_map[cluster_name] = cluster
depth += 1
if depth > find_depth:
break
# end load_cluster
env.runs = runs
log.debug('env.runs = {0}\n'.format(env.runs))
log.debug('env.cluster_map = {0}\n'.format(env.cluster_map))
def load_runs(query, find_depth=1):
""" queryに基づいて、nodeを読み込む
env.runs にクラスタごとの実行タスクリスト
env.cluster_mapにクラスタごとのデータ情報を格納
"""
env.host_script_map = {}
# query rule
# <cluster_name>/<node_search_pattern>
is_contain_candidates = True
splited_query = query.rsplit('/', 1)
cluster_name = splited_query[0]
if len(splited_query) > 1 and splited_query[1]:
host_pattern = splited_query[1]
if host_pattern.find('!') == 0:
is_contain_candidates = False
host_pattern = host_pattern[1:]
candidates = [re.compile(c.replace('*', '.*')) for c in get_expanded_hosts(host_pattern)]
else:
host_pattern = ''
candidates = None
cluster_dir = os.path.join(CONF._node_dir, cluster_name)
# load cluster data from node dir
runs = []
depth = 1
for root, dirs, files in os.walk(CONF._node_dir):
if root.find(cluster_dir) == -1:
continue
# load cluster
cluster_name = root.split(CONF._node_dir)[1]
if cluster_name.find('/') == 0:
cluster_name = cluster_name[1:]
# default cluster dict
cluster = {
'name': cluster_name,
'host_pattern': host_pattern,
'__status': {
'node_map': {},
'fabscript_map': {},
},
}
cluster.update(include_cluster(cluster_name))
# node_map is definition of nodes in cluster
cluster_node_map = cluster.get('node_map', {})
# status is setup status of cluster
cluster_status = cluster['__status']
# status of nodes
node_status_map = cluster_status['node_map']
# status of fabscripts
fabscript_status_map = cluster_status['fabscript_map']
# expand hosts of node_map, and load fabscript of node
# tmp_fabscript_mapに実行候補のfabscriptをすべて格納する
tmp_fabscript_map = {}
for role, cluster_node in cluster_node_map.items():
node_hosts = cluster_node['hosts']
tmp_hosts = []
for node_host in node_hosts:
node_host = decode_data(node_host)
if type(node_host) is ListType:
for tmp_node_host in node_host:
tmp_hosts.extend(get_expanded_hosts(tmp_node_host))
else:
tmp_hosts.extend(get_expanded_hosts(node_host))
cluster_node['hosts'] = tmp_hosts
for tmp_fabscript in cluster_node['fabruns']:
# fabruns = [<cluster>/<python_mod_path>, <cluster>/<python_mod_path>]
fabscript = tmp_fabscript
fabscript_kwargs = {}
if isinstance(tmp_fabscript, dict):
for f, kwargs in tmp_fabscript.items():
fabscript = f
fabscript_kwargs = kwargs
break
update_host_script_map(tmp_hosts, fabscript, fabscript_kwargs)
# fabscriptは、クラスタ単位で管理する
fabscript_cluster = fabscript.rsplit('/', 1)[0]
fabscript_mod = fabscript.rsplit('/', 1)[1]
if fabscript not in tmp_fabscript_map:
if fabscript not in fabscript_status_map:
# このクラスタにおいて実行履歴のないfabscriptはデフォルト値を登録する
fabscript_status_map[fabscript] = {
'status': 0,
'task_status': status.REGISTERED,
}
# デフォルト値
fabscript_data = {
'fabscript': fabscript,
'fabscript_kwargs': fabscript_kwargs,
'hosts': [],
'status_flow': [0],
'require': {},
'required': [],
}
fabscript_file = os.path.join(CONF._fabscript_module_dir,
fabscript_cluster, CONF._fabscript_yaml)
if os.path.exists(fabscript_file):
with open(fabscript_file, 'r') as f:
tmp_data = yaml.load(f)
fabscript_data.update(tmp_data.get(fabscript_mod, {}))
tmp_fabscript_map[fabscript] = fabscript_data
tmp_fabscript_map[fabscript]['hosts'].extend(tmp_hosts)
tmp_fabscript_map[fabscript]['env'] = cluster_node.get('env', {})
# tmp_fabscript_map を解析して、スクリプトの実行順序を組み立てる
# 実行順序はcluster_runsに配列として格納する
cluster_runs = []
# スクリプトの依存関係を再帰的に求める
def resolve_require(fabscript, data):
for require_script, require_status in data['require'].items():
require = tmp_fabscript_map.get(require_script)
if require:
require['required'].append(fabscript)
if (require['require']) > 0:
resolve_require(fabscript, require)
else:
require = fabscript_status_map.get(require_script)
if require:
if require['status'] == require_status and require['task_status'] == 0:
return
log.error('Require Error\nCluster: {0}\nFabscript: {1} require {2}:{3}'.format(
cluster_name, fabscript, require_script, require_status))
exit()
for fabscript, data in tmp_fabscript_map.items():
resolve_require(fabscript, data)
# スクリプト同士の依存関係が求められたので、これを使って正しい実行順序を組立てる
for fabscript, data in tmp_fabscript_map.items():
index = len(cluster_runs)
for required in data['required']:
for i, run in enumerate(cluster_runs):
if run['fabscript'] == required:
if i < index:
index = i
# セットアップの対象となるノードの初期化
tmp_hosts = []
for host in data['hosts']:
if candidates:
is_match = False
for candidate in candidates:
if candidate.search(host):
is_match = True
break
if is_match and not is_contain_candidates:
continue
elif not is_match and is_contain_candidates:
continue
tmp_hosts.append(host)
node = node_status_map.get(host, {'fabscript_map': {}})
if fabscript not in node['fabscript_map']:
node['fabscript_map'][fabscript] = {
'status': 0,
'check_status': -1,
'msg': '',
'check_msg': '',
}
if env.is_setup:
node['fabscript_map'][fabscript].update({
'msg': status.REGISTERED_MSG,
'task_status': status.REGISTERED,
})
elif env.is_check:
node['fabscript_map'][fabscript].update({
'check_msg': status.REGISTERED_MSG,
'task_status': status.REGISTERED,
})
node_status_map[host] = node
if len(tmp_hosts) > 0:
fabscript_status_map[fabscript]['task_status'] = status.REGISTERED
data['hosts'] = tmp_hosts
cluster_status['node_map'] = node_status_map
fabscript_status = fabscript_status_map[fabscript]['status']
for flow in data['status_flow']:
if fabscript_status < flow or flow == data['status_flow'][-1]:
tmp_data = data.copy()
tmp_data['expected_status'] = flow
cluster_runs.insert(index, tmp_data)
index += 1
runs.append({
'cluster': cluster_name,
'runs': cluster_runs,
})
cluster_status['fabscript_map'] = fabscript_status_map
cluster['fabscript_map'] = tmp_fabscript_map
env.cluster_map[cluster_name] = cluster
depth += 1
if depth > find_depth:
break
# end load_cluster
env.runs = runs
log.debug('env.runs = {0}\n'.format(env.runs))
log.debug('env.cluster_map = {0}\n'.format(env.cluster_map))
def run_func(func_names=[], *args, **kwargs):
fabrun_filter = []
is_filter = False
env.is_test = False
env.is_help = False
if len(args) > 0:
if args[0] == 'test':
env.is_test = True
args = args[1:]
elif args[0] == 'help':
env.is_help = True
args = args[1:]
if 'f' in kwargs:
fabrun_filter = kwargs['f'].split('+')
fabrun_filter = [re.compile(f) for f in fabrun_filter]
is_filter = True
func_patterns = [re.compile(name) for name in func_names]
host_filter = {}
for run in env.runs:
cluster_name = run['cluster']
cluster = env.cluster_map[run['cluster']]
env.cluster = cluster
log.init_logger(cluster_name)
for cluster_run in run['runs']:
run_results = []
script_name = cluster_run['fabscript']
if is_filter:
for f in fabrun_filter:
if f.search(script_name):
break
else:
continue
if env.is_check or env.is_manage:
# 二重実行を防ぐ
hosts = host_filter.get(script_name, [])
tmp_hosts = list(set(cluster_run['hosts']) - set(hosts))
cluster_run['hosts'] = tmp_hosts
hosts.extend(tmp_hosts)
host_filter[script_name] = hosts
if len(cluster_run['hosts']) == 0:
continue
hosts = []
for host in cluster_run['hosts']:
env.node_map[host] = env.node_map.get(host, {
'host': host,
'bootstrap_status': -1,
})
if env.node_map[host]['bootstrap_status'] != status.FAILED_CHECK:
hosts.append(host)
env.script_name = script_name
env.hosts = hosts
# override env
override_env = env.cluster.get('env', {})
override_env.update(cluster_run.get('env', {}))
default_env = {}
for key, value in override_env.items():
default_env[key] = getattr(env, key, None)
setattr(env, key, value)
env.cluster_status = env.cluster['__status']
env.node_status_map = env.cluster_status['node_map']
env.fabscript_status_map = env.cluster_status['fabscript_map']
env.fabscript = env.fabscript_status_map[script_name]
log.info('hosts: {0}'.format(env.hosts))
log.info('run: {0}: {1}'.format(script_name, env.fabscript))
log.debug('node_status_map: {0}'.format(env.node_status_map))
# check require
require = env.cluster['fabscript_map'][script_name]['require']
if env.is_setup:
is_require = True
for script, status_code in require.items():
required_status = env.fabscript_status_map[script]['status']
if required_status != status_code:
log.error('Require Error\n'
+ '{0} is require {1}:{2}.\nbut {1} status is {3}.'.format(
script_name, script, status_code, required_status))
is_require = False
break
if not is_require:
break
script = '.'.join([CONF.fabscript_module, script_name.replace('/', '.')])
# importlibは、2.7以上じゃないと使えない
# module = importlib.import_module(script)
module = __import__(script, globals(), locals(), ['*'], -1)
module_funcs = []
for member in inspect.getmembers(module):
if inspect.isfunction(member[1]):
module_funcs.append(member[0])
if env.is_help and len(func_patterns) == 0:
for candidate in module_funcs:
func = getattr(module, candidate)
print 'Task: {0}'.format(func.__name__)
print func.__doc__
continue
# func_patterns にマッチしたタスク関数をすべて実行する
is_expected = False
is_contain_unexpected = False
for func_pattern in func_patterns:
for candidate in module_funcs:
if not func_pattern.match(candidate):
continue
func = getattr(module, candidate)
# taskデコレータの付いているものだけ実行する
if not hasattr(func, 'is_task') or not func.is_task:
continue
if env.is_help:
print 'Task: {0}'.format(func.__name__)
print func.__doc__
continue
results = api.execute(func, *args, **kwargs)
# check results
data_map = {}
tmp_status = None
is_contain_failed = False
for host, result in results.items():
env.node_map[host].update(result['node'])
if not result or type(result) is not DictType:
result = {}
node_result = env.node_status_map[host]['fabscript_map'][script_name]
result_status = result.get('status')
task_status = result.get('task_status', status.SUCCESS)
msg = result.get('msg')
if msg is None:
if task_status is status.SUCCESS:
msg = status.FABSCRIPT_SUCCESS_MSG
else:
msg = status.FABSCRIPT_FAILED_MSG
if func.is_bootstrap:
if task_status == status.FAILED_CHECK or \
task_status == status.FAILED_CHECK_PING:
env.node_map[host]['bootstrap_status'] = status.FAILED_CHECK
else:
env.node_map[host]['bootstrap_status'] = status.SUCCESS
node_result['task_status'] = task_status
tmp_data_map = result.get('data_map')
if tmp_data_map is not None:
for map_name, tmp_map_data in tmp_data_map.items():
if tmp_map_data['type'] == 'table':
map_data = data_map.get(map_name, {
'name': map_name,
'type': 'table',
'data': [],
})
tmp_data = {'!!host': host}
tmp_data.update(tmp_map_data['data'])
map_data['data'].append(tmp_data)
data_map[map_name] = map_data
elif tmp_map_data['type'] == 'multi-table':
map_data = data_map.get(map_name, {
'name': map_name,
'type': 'multi-table',
'data': [],
})
tmp_data = {'!!host': host}
tmp_data.update(tmp_map_data['data'])
map_data['data'].append(tmp_data)
data_map[map_name] = map_data
elif tmp_map_data['type'] == 'line-chart':
map_data = data_map.get(map_name, {
'name': map_name,
'type': 'line-chart',
'data': [],
})
map_data['ex_data'] = tmp_map_data['ex_data']
map_data['layout'] = tmp_map_data['layout']
tmp_data = {'!!host': host}
tmp_data.update(tmp_map_data['data'])
map_data['data'].append(tmp_data)
data_map[map_name] = map_data
if env.is_setup:
node_result['msg'] = msg
if result_status is not None:
tmp_status = result_status
node_result['status'] = result_status
expected = cluster_run['expected_status']
if expected == result_status:
is_expected = True
log.info('{0}: {1} is expected status.'.format(
host, msg, result_status))
else:
is_contain_unexpected = True
log.error('expected status is {0}, bad status is {1}.'.format( # noqa
expected, result_status), host)
elif env.is_check:
node_result['check_msg'] = msg
if result_status is None:
result_status = status.SUCCESS
node_result.update({
'check_status': result_status,
})
if result_status != status.SUCCESS:
log.error('Failed check {0}.{1} [{2}]. {3}'.format( # noqa
script_name, candidate, result_status, msg), host)
if task_status == status.SUCCESS:
log.info('Success task {0}.{1} [{2}]. {3}'.format(
script_name, candidate, task_status, msg), host)
else:
log.error('Failed task {0}.{1} [{2}]. {3}'.format(
script_name, candidate, task_status, msg), host)
is_contain_failed = True
# end for host, result in results.items():
if len(data_map) > 0:
util.dump_datamap(data_map)
if is_contain_failed:
log.error('Failed task {0}.{1}. Exit setup.'.format(
script_name, candidate))
util.dump_status()
exit()
if tmp_status is not None:
env.fabscript['tmp_status'] = tmp_status
# for candidate in module_funcs:
# for func_pattern in func_patterns:
if env.is_setup:
if is_expected and not is_contain_unexpected or cluster_run['expected_status'] == 0:
env.cluster['__status']['fabscript_map'][script_name] = {
'status': cluster_run['expected_status'],
'task_status': status.SUCCESS,
}
util.dump_status()
else:
log.error('bad status.')
exit()
elif env.is_check:
env.cluster['__status']['fabscript_map'][script_name]['task_status'] = status.SUCCESS # noqa
util.dump_status()
elif env.is_manage:
env.cluster['__status']['fabscript_map'][script_name]['task_status'] = status.SUCCESS # noqa
util.dump_status()
# reset env
for key, value in default_env.items():
if value is not None:
setattr(env, key, value)