def module_configure(deploy_info, conf_module_mod, local_path):
cfg_modify_info = parse_json(conf_module_mod)
get_conf(deploy_info['online_info']['module_machine'], deploy_info['online_info']['module_conf'], local_path,
cfg_modify_info)
is_ok, conf_name = gen_conf(cfg_modify_info, local_path)
exe_sys_cmd('cp -f ' + local_path + '/conf/* ' + local_path + '/' + deploy_info['service_info']['service_exe_path'] + '/')
return is_ok, conf_name
def gen_job_config_module_data_mod(mysql_info, job_id):
is_ok = False
mysql_client = MYSQLClient(mysql_info['mysql_user'],
mysql_info['mysql_password'],
mysql_info['mysql_ip'],
mysql_info['mysql_port'],
mysql_info['mysql_database'])
status, title, data = mysql_client.execute_sql(
'select * from offline_mission_info where mission_id = ' + str(job_id))
if status and len(title) > 0 and len(data) == 1:
module_type = data[0][title.index('module')]
else:
return is_ok
template_file = sys.path[
0] + '/conf/data_modify_template_' + module_type + '.json'
config_dict = parse_json(template_file)
if len(config_dict) == 0:
return is_ok
config_dict['dev_data'][0]['data_path'].clear()
status, title, data = mysql_client.execute_sql(
'select * from offline_data_info where mission_id = ' + job_id)
if status and len(title) > 0 and len(data) > 0:
for it in data:
if it[title.index('key')] == 'data_path':
config_dict['dev_data'][0]['data_path'].append(
it[title.index('content')])
else:
try:
config_dict[it[title.index('key')]] = json.loads(
it[title.index('content')])
except:
config_dict[it[title.index('key')]] = {}
config_dict['need_data_modify'] = True
is_ok = True
elif status:
config_dict['need_data_modify'] = False
deploy_server_info = dict()
status, title, data = mysql_client.execute_sql(
'select * from offline_machine_info where mission_id = ' + job_id)
if status and len(title) > 0 and len(data) > 0:
for it in data:
deploy_server_info[it[title.index('ip')]] = it[title.index('path')]
mysql_client.close()
for addr in deploy_server_info.keys():
config_dict['service_info'][
'service_deploy_path'] = deploy_server_info[addr]
config_dict[
'data_modify_path'] = deploy_server_info[addr] + '/test_data/'
if not write_json(
config_dict, sys.path[0] + '/conf/data_modify_' + module_type +
'_' + job_id + '_' + addr + '.json'):
is_ok = False
return is_ok
def processor(build_tag='', step=''):
# gen job_config in control server
mysql_info = {'mysql_user': '******',
'mysql_password': '******',
'mysql_ip': '10.134.104.40',
'mysql_port': 3306,
'mysql_database': 'main_project'}
if step == 'gen_job_config':
if build_tag != '':
if not (gen_job_config_main(mysql_info, build_tag) and
gen_job_config_module_conf_mod(mysql_info, build_tag) and
gen_job_config_module_data_mod(mysql_info, build_tag) and
gen_job_config_module_start_script(mysql_info, build_tag)):
sys.exit(1)
sys.exit(0)
# build step by step in test server
conf_main, conf_module_mod, conf_data_mod = get_job_config_file_name(os.path.join(sys.path[0], 'conf/'), build_tag)
deploy_info = parse_json(conf_main)
if step not in deploy_info['deploy_info']['deploy_step'].keys() or \
deploy_info['deploy_info']['deploy_step'][step] == '0':
write_log('step ' + step + ' is illegal, skip')
sys.exit(0)
local_path = ''
for server_info in deploy_info['offline_server_info']:
if check_is_local_server(server_info['addr']):
# local_path = server_info['deploy_path'] + '/' + deploy_info['service_info']['service_name'] + '/'
local_path = server_info['deploy_path'] + '/'
break
if local_path == '':
sys.exit(1)
if not exe_sys_cmd_v3('mkdir -p ' + local_path):
sys.exit(1)
env_prepare()
if step == 'is_establish':
if not module_install(deploy_info, conf_module_mod, local_path):
sys.exit(2)
elif step == 'is_config':
if not module_configure(deploy_info, conf_module_mod, local_path)[0]:
sys.exit(3)
elif step == 'is_data':
if not module_data_modify(deploy_info, conf_data_mod):
sys.exit(4)
elif step == 'is_start':
if not module_start(deploy_info, local_path, build_tag):
sys.exit(5)
elif step == 'is_dailybuild':
from proc.package_deploy_dailybuild import gen_dailybuild
is_ok, conf_name, dailybuild_file = gen_dailybuild(deploy_info, local_path)
if not is_ok:
sys.exit(5)
else:
print(dailybuild_file)
def module_install(deploy_info, conf_module_mod, local_path):
if not get_job_ini(deploy_info['online_info']['module_machine'], deploy_info['online_info']['module_path'],
local_path):
return False
cfg_modify_info = parse_json(conf_module_mod)
if not modify_job_ini(cfg_modify_info, local_path):
return False
if svn_get_package(local_path + '/job.ini', local_path) == '':
return False
if not compile_cplus(local_path):
return False
exe_sys_cmd('rm -rf ' + local_path + '/' + deploy_info['service_info']['service_exe_path'] + '/data')
exe_sys_cmd('ln -s ' + deploy_info['offline_data_info']['data_base_path'] + ' ' +
local_path + '/' + deploy_info['service_info']['service_exe_path'] + '/data')
return True
def gen_dailybuild(deploy_info, local_path):
is_ok = False
conf_name = ''
dailybuild_file = ''
get_job_ini(deploy_info['online_info']['module_machine'],
deploy_info['online_info']['module_path'], local_path)
cfg_modify_info = parse_json(
os.path.join(sys.path[0], deploy_info['config_modify_file']))
modify_job_ini(cfg_modify_info, local_path)
if svn_get_package(local_path + '/job.ini', local_path) != '':
get_conf(deploy_info['online_info']['module_machine'],
deploy_info['online_info']['module_conf'], local_path,
cfg_modify_info)
is_ok, conf_name = gen_conf(cfg_modify_info, local_path)
is_ok, dailybuild_file = compile_dailybuild(local_path + '/job.ini',
local_path)
return is_ok, conf_name, dailybuild_file
def module_start(deploy_info, local_path, job_id):
is_ok = False
module_type = deploy_info['service_info']['service_type']
start_shell = 'start_shell_' + module_type + '.sh'
exe_sys_cmd_v3('cp -f ' + os.path.join(sys.path[0], start_shell) + ' ' +
local_path + '/' + deploy_info['service_info']['service_exe_path'])
start_shell_param_list = parse_json(
os.path.join(sys.path[0], 'start_shell_param' + module_type + '_' + job_id + '.json'))
os.chdir(local_path + '/' + deploy_info['service_info']['service_exe_path'])
if len(start_shell_param_list) == 0:
return is_ok
is_ok = True
for start_shell_param in start_shell_param_list:
start_script = local_path + '/' + deploy_info['service_info']['service_exe_path'] + '/' + \
start_shell + ' ' + start_shell_param
if not start_service(module_type,
local_path + '/' + deploy_info['service_info']['service_exe_path'],
start_script, '0', deploy_info['service_info']['startup_time'])[0]:
is_ok = False
return is_ok
def processor(deploy_conf, build_tag=''):
deploy_info = parse_json(deploy_conf)
env_prepare()
if build_tag == '':
build_tag = str(int(time.time()))
local_path = deploy_info['deploy_info'][
'deploy_local_path'] + '/' + build_tag + '/'
exe_sys_cmd('mkdir -p ' + local_path)
is_ok, conf_name, dailybuild_file = gen_dailybuild(deploy_info, local_path)
if not is_ok:
sys.exit(2)
if not distribute_dailybuild(
deploy_info['offline_server_info'], dailybuild_file, local_path,
'', deploy_info['offline_data_info']['data_base_path'], build_tag):
sys.exit(3)
if not distribute_customized_data(deploy_info['offline_server_info'],
deploy_info['offline_data_info'], '',
build_tag):
sys.exit(4)
if not start_module_service(deploy_info['offline_server_info'], '',
deploy_info['service_info']['service_name'],
conf_name, build_tag):
sys.exit(5)
dir_list_all += vr_file_get(offline_data_conf)
for index in range(len(dir_list_all) - 1, -1, -1):
if dir_list_all[index] in dir_list_all[:index]:
dir_list_all.pop(index)
for loop_id_1 in range(len(dir_list_all)):
for loop_id_2 in range(loop_id_1 + 1, len(dir_list_all)):
if len(dir_list_all[loop_id_1].split('/')) < len(dir_list_all[loop_id_2].split('/')):
dir_list_all[loop_id_1], dir_list_all[loop_id_2] = dir_list_all[loop_id_2], dir_list_all[loop_id_1]
for it in dir_list_all:
if (re.search('base/vr$', it) is not None) or (re.search('base/vr/$', it) is not None):
file_prefix_list = split_sys_cmd_echo(exe_sys_cmd_get_echo('ls | grep -o ^... | sort -u'), 0)
for file_prefix in file_prefix_list:
exe_sys_cmd(' ln -s ' + offline_data_conf['data_base_path'] + '/' + it + '/' + file_prefix + '* ' +
offline_data_conf['data_modify_path'] + '/' + it + '/' + file_prefix + '*')
else:
exe_sys_cmd(' ln -s ' + offline_data_conf['data_base_path'] + '/' + it + '/* ' +
offline_data_conf['data_modify_path'] + '/' + it + '/')
if __name__ == "__main__":
if len(sys.argv) == 2:
data_modify_file = sys.argv[1]
if os.path.exists(data_modify_file):
offline_data_conf = parse_json(data_modify_file)
if os.path.exists(offline_data_conf['data_base_path']):
os.chdir(offline_data_conf['data_base_path'])
exe_sys_cmd_get_echo('sh Down.sh')
else:
sys.exit(1)
mk_data_file_link(offline_data_conf)
def gen_job_config_main(mysql_info, job_id=''):
is_ok = False
mysql_client = MYSQLClient(mysql_info['mysql_user'],
mysql_info['mysql_password'],
mysql_info['mysql_ip'],
mysql_info['mysql_port'],
mysql_info['mysql_database'])
status, title, data = mysql_client.execute_sql(
'select * from offline_mission_info where mission_id = ' + job_id)
deploy_step = {}
if status and len(title) > 0 and len(data) == 1:
module_type = data[0][title.index('module')]
for column_name in title:
if re.search('^is_', column_name) is not None:
deploy_step[column_name] = str(
data[0][title.index(column_name)])
else:
return is_ok
if len(deploy_step) == 0:
return is_ok
template_file = sys.path[0] + '/conf/conf_template_' + module_type + '.json'
config_dict = parse_json(template_file)
if len(config_dict) == 0:
return is_ok
config_dict['service_info']['service_name'] = module_type
config_dict['service_info']['service_type'] = module_type
config_dict['deploy_info']['deploy_step'] = deploy_step
config_dict['deploy_info']['deploy_local_path'] = config_dict[
'deploy_info']['deploy_local_path'] + '/' + job_id + '/'
status, title, data = mysql_client.execute_sql(
'select * from offline_module_config_info where module = "' +
module_type + '"')
if status and len(title) > 0 and len(data) == 1:
config_dict['service_info']['startup_time'] = int(
data[0][title.index('max_start_time')])
config_dict['offline_server_info'].clear()
status, title, data = mysql_client.execute_sql(
'select * from offline_machine_info where mission_id = ' + job_id)
if status and len(title) > 0 and len(data) > 0:
for it in data:
addr = it[title.index('ip')]
deploy_path = it[title.index('path')]
it_status, it_title, it_data = mysql_client.execute_sql(
'select * from offline_machine_config where ip = "' + addr +
'"')
if it_status and len(it_title) > 0 and len(it_data) == 1:
username = it_data[0][it_title.index('user')]
password = it_data[0][it_title.index('passwd')]
config_dict['offline_server_info'].append({
"addr":
addr,
"username":
username,
"password":
password,
"deploy_path":
deploy_path
})
mysql_client.close()
if len(config_dict['offline_server_info']) > 0:
is_ok = True
config_dict['offline_data_info'][
'data_conf'] = './conf/data_modify_' + module_type + '_' + job_id + '.json'
config_dict[
'config_modify_file'] = './conf/config_modify_' + module_type + '_' + job_id + '.json'
if not write_json(
config_dict, sys.path[0] + '/conf/conf_' + module_type + '_' +
job_id + '.json'):
is_ok = False
return is_ok