def vr_file_get(offline_data_conf):
if (offline_data_conf.get('pre_server_info') is None):
return []
if (offline_data_conf.get('vr_files') is None) or \
(offline_data_conf['vr_files'].get('class_ids') is None) or \
(len(offline_data_conf['vr_files']['class_ids']) == 0):
return []
class_ids = offline_data_conf['vr_files']['class_ids']
target_path = offline_data_conf['data_modify_path']
pre_server_info = offline_data_conf['pre_server_info']
sftp_client = SFTPClient(pre_server_info['addr'], "22", pre_server_info['username'], pre_server_info['password'])
dir_list = ['base/vr/bin']
exe_sys_cmd_get_echo('mkdir -p {}/base/vr/bin'.format(target_path))
for class_id in class_ids:
if class_id[0] == '7':
file_re = class_id[:6]
else:
file_re = class_id
ssh_echo, err = sftp_client.exe_ssh_cmd_get_echo('ls /search/odin/daemon/vrqo/data/base/vr/*{}*')
if err == '':
files = ssh_echo.format(file_re).strip('\n').split('\n')
for file in files:
sftp_client.sftp_get('/search/odin/daemon/vrqo/data/base/vr/*{}*'.format(file),
'{}/base/vr/bin/'.format(target_path))
sftp_client.close()
return dir_list
def start_module_service(server_info_list,
remote_path='',
module_name='',
conf_name='',
build_tag=''):
from start_shell.gen_start_shell import gen_start_shell_vrqo_dailybuild
result_list = [False] * len(server_info_list)
result_index = 0
for server_info in server_info_list:
if remote_path == '':
distribute_path = server_info['deploy_path']
else:
distribute_path = remote_path
distribute_path = os.path.join(distribute_path, build_tag)
start_shell = 'start_shell_' + module_name + '.sh'
if not eval('gen_start_shell_' + module_name +
'_dailybuild')(conf_name):
result_index += 1
continue
sftp_client = SFTPClient(server_info['addr'], "22",
server_info['username'],
server_info['password'])
sftp_client.sftp_put(os.path.join(sys.path[0], start_shell),
distribute_path + '/dailybuild/opt/sogou/')
if '' == sftp_client.exe_ssh_cmd_get_echo('cd ' + distribute_path +
'/dailybuild/opt/sogou/' +
' && ' + 'sh ' +
start_shell)[1]:
result_list[result_index] = True
sftp_client.close()
result_index += 1
if False not in result_list:
return True
return False
def vr_info_adjust(offline_data_conf):
if (offline_data_conf.get('pre_server_info') is None):
return []
if (offline_data_conf.get('vrqo_VRInfo') is None) or \
(offline_data_conf['vrqo_VRInfo'].get('class_ids') is None) or \
(len(offline_data_conf['vrqo_VRInfo']['class_ids']) == 0):
return []
class_ids = offline_data_conf['vrqo_VRInfo']['class_ids']
base_path = offline_data_conf['data_base_path']
target_path = offline_data_conf['data_modify_path']
pre_server_info = offline_data_conf['pre_server_info']
dir_list = ['base/norm']
vr_info_base = '{}/base/norm/vrqo_VRInfo'.format(base_path)
exe_sys_cmd_get_echo('mkdir -p {}/base/norm'.format(target_path))
exe_sys_cmd_get_echo('cp -f {} {}/base/norm/'.format(vr_info_base, target_path))
vr_info_pre = '{}/base/norm/vrqo_VRInfo.pre'.format(target_path)
sftp_client = SFTPClient(pre_server_info['addr'], "22", pre_server_info['username'], pre_server_info['password'])
sftp_client.sftp_get('/search/odin/daemon/vrqo/data/base/norm/vrqo_VRInfo', vr_info_pre)
sftp_client.close()
info_dict = {}
modify_list = []
if not os.path.exists(vr_info_pre):
return []
with open(vr_info_pre) as f:
for line in f:
infos = line.split('\t')
info_dict.update({infos[1]: line.strip('\n')})
for class_id in class_ids:
modify_list.append({
"anchor": ['^{}'.format(class_id)],
"action": "replace",
"content": info_dict[class_id]
})
is_effect = modify_text(modify_list, '{}/base/norm/vrqo_VRInfo'.format(target_path))
second_modify_list = []
for i in range(len(is_effect)):
if is_effect[i] == 0:
second_modify_list.append({
"anchor": ['$'],
"action": "append",
"content": info_dict[class_ids[i]]
})
modify_text(second_modify_list, '{}/base/norm/vrqo_VRInfo'.format(target_path))
return dir_list
def regular_file_get(offline_data_conf):
dir_list = []
if offline_data_conf.get('dev_data') is None:
return []
for dev_data_info in offline_data_conf['dev_data']:
if (dev_data_info.get('data_path') is None) or (len(dev_data_info['data_path']) == 0):
continue
re_path = re.compile('(?P<dir>base.*)/(?P<name>[^/]+)/*$')
sftp_client = SFTPClient(dev_data_info['addr'], "22", dev_data_info['username'], dev_data_info['password'])
for path in dev_data_info['data_path']:
path_suffix = re.search(re_path, path)
if path_suffix:
file_dict = path_suffix.groupdict()
exe_sys_cmd_get_echo('mkdir -p {}/{}'.format(offline_data_conf['data_modify_path'], file_dict['dir']))
dir_list.append(file_dict['dir'])
sftp_client.sftp_get(path, '{}/{}/{}'.format(offline_data_conf['data_modify_path'], file_dict['dir'],
file_dict['name']))
sftp_client.close()
return dir_list
def qo_config_adjust(offline_data_conf):
if (offline_data_conf.get('pre_server_info') is None):
return []
if (offline_data_conf.get('vrqo_QOConfig') is None) or \
(offline_data_conf['vrqo_QOConfig'].get('class_ids') is None) or \
(len(offline_data_conf['vrqo_QOConfig']['class_ids']) == 0):
return []
class_ids = offline_data_conf['vrqo_QOConfig']['class_ids']
base_path = offline_data_conf['data_base_path']
target_path = offline_data_conf['data_modify_path']
pre_server_info = offline_data_conf['pre_server_info']
dir_list = ['base/norm']
qo_config_base = '{}/base/norm/vrqo_QOConfig'.format(base_path)
qo_config_target = '{}/base/norm/vrqo_QOConfig'.format(target_path)
qo_config_pre = '{}/base/norm/vrqo_QOConfig.pre'.format(target_path)
exe_sys_cmd_get_echo('mkdir -p {}/base/norm'.format(target_path))
exe_sys_cmd_get_echo('cp -f {} {}/base/norm/'.format(qo_config_base, target_path))
sftp_client = SFTPClient(pre_server_info['addr'], "22", pre_server_info['username'], pre_server_info['password'])
sftp_client.sftp_get('/search/odin/daemon/vrqo/data/base/norm/vrqo_QOConfig', qo_config_pre)
sftp_client.close()
pattern_names = []
rule_names = []
for class_id in class_ids:
cmd_echo = exe_sys_cmd_get_echo('fgrep {} {}|fgrep "PATTERN_NAME"'.format(class_id, qo_config_target))
if cmd_echo != '':
pattern_names += cmd_echo.split('\n')
cmd_echo = exe_sys_cmd_get_echo('fgrep {} {}|fgrep "RULE_NAME"'.format(class_id, qo_config_target))
if cmd_echo != '':
rule_names += cmd_echo.split('\n')
merge_QO_config(qo_config_pre, qo_config_target,
[{"start": "<PATTERN NEW>", "end": "<PATTERN END>",
"key": "PATTERN_NAME"},
{"start": "<RULE NEW>", "end": "<RULE END>",
"key": "RULE_NAME"}],
{'<PATTERN NEW>': pattern_names,
'<RULE NEW>': rule_names}
)
return dir_list
def distribute_customized_data(server_info_list,
offline_data_info,
remote_path='',
build_tag=''):
result_list = [False] * len(server_info_list)
result_index = 0
for server_info in server_info_list:
if remote_path == '':
distribute_path = server_info['deploy_path']
else:
distribute_path = remote_path
distribute_path = os.path.join(distribute_path, build_tag)
data_script = os.path.join(sys.path[0],
offline_data_info['data_script'])
data_conf = os.path.join(sys.path[0], offline_data_info['data_conf'])
if os.path.exists(data_script) and os.path.exists(data_conf):
sftp_client = SFTPClient(server_info['addr'], "22",
server_info['username'],
server_info['password'])
sftp_client.sftp_mkdir(distribute_path)
sftp_client.sftp_put(data_script, distribute_path)
sftp_client.sftp_put(data_conf, distribute_path)
sftp_client.sftp_put(os.path.join(sys.path[0], 'lib'),
distribute_path + '/lib/')
if '' == sftp_client.exe_ssh_cmd_get_echo(
'python3 ' + os.path.join(distribute_path,
os.path.basename(data_script)) +
' ' +
os.path.join(distribute_path, os.path.basename(data_conf))
)[1]:
result_list[result_index] = True
sftp_client.close()
result_index += 1
if False not in result_list:
return True
return False
def distribute_dailybuild(server_info_list,
dailybuild_file='',
local_path='',
remote_path='',
remote_data_path='',
build_tag=''):
result_list = [False] * len(server_info_list)
result_index = 0
if os.path.exists(dailybuild_file):
for server_info in server_info_list:
if remote_path == '':
distribute_path = server_info['deploy_path']
else:
distribute_path = remote_path
distribute_path = os.path.join(distribute_path, build_tag)
sftp_client = SFTPClient(server_info['addr'], "22",
server_info['username'],
server_info['password'])
if not yum_install_referto_jobini(local_path + '/job.ini',
'remote', sftp_client):
continue
sftp_client.sftp_rmdir(distribute_path)
sftp_client.sftp_mkdir(distribute_path)
sftp_client.sftp_put(dailybuild_file, distribute_path)
if '' == sftp_client.exe_ssh_cmd_get_echo(
'tar -xf ' + distribute_path + '/dailybuild.tar.gz -C ' +
distribute_path + '/')[1]:
sftp_client.exe_ssh_cmd_get_echo(
'rm -f ' +
os.path.join(distribute_path, 'dailybuild/opt/sogou/data'))
sftp_client.exe_ssh_cmd_get_echo(
'ln -s ' + remote_data_path + ' ' +
os.path.join(distribute_path, 'dailybuild/opt/sogou/data'))
result_list[result_index] = True
sftp_client.close()
result_index += 1
if False not in result_list:
return True
return False