def vr_file_get(offline_data_conf): if (offline_data_conf.get('pre_server_info') is None): return [] if (offline_data_conf.get('vr_files') is None) or \ (offline_data_conf['vr_files'].get('class_ids') is None) or \ (len(offline_data_conf['vr_files']['class_ids']) == 0): return [] class_ids = offline_data_conf['vr_files']['class_ids'] target_path = offline_data_conf['data_modify_path'] pre_server_info = offline_data_conf['pre_server_info'] sftp_client = SFTPClient(pre_server_info['addr'], "22", pre_server_info['username'], pre_server_info['password']) dir_list = ['base/vr/bin'] exe_sys_cmd_get_echo('mkdir -p {}/base/vr/bin'.format(target_path)) for class_id in class_ids: if class_id[0] == '7': file_re = class_id[:6] else: file_re = class_id ssh_echo, err = sftp_client.exe_ssh_cmd_get_echo('ls /search/odin/daemon/vrqo/data/base/vr/*{}*') if err == '': files = ssh_echo.format(file_re).strip('\n').split('\n') for file in files: sftp_client.sftp_get('/search/odin/daemon/vrqo/data/base/vr/*{}*'.format(file), '{}/base/vr/bin/'.format(target_path)) sftp_client.close() return dir_list
def modify_xml_rude(modify_list=dict(), xml_file='./file.xml'): result_list = [False] * len(modify_list) xml_list = parse_xml_rude(xml_file) if len(xml_list) == 0: return False result_index = 0 for modify_point in modify_list: for index in range(len(xml_list) - 1, -1, -1): if xml_list[index].find('<' + modify_point['anchor'][0] + '>') == -1: continue if modify_point['action'] == 'replace': xml_list[index] = '<' + modify_point['anchor'][0] + '>' + modify_point['content'] + \ '</' + modify_point['anchor'][0] + '>' + '\n' result_list[result_index] = True if modify_point['action'] == 'add': xml_list[ index] = modify_point['content'] + '\n' + xml_list[index] result_list[result_index] = True if modify_point['action'] == 'delete': xml_list.pop(index) result_list[result_index] = True result_index += 1 exe_sys_cmd_get_echo('cp -f ' + xml_file + ' ' + xml_file + '_bak_' + str(int(time.time()))) write_xml_rude(xml_list, xml_file) return result_list
def get_job_ini(server='', path='', local_path='.'): exe_sys_cmd_get_echo('rsync -vazP ' + server + '::' + path + '/info/job.ini ' + local_path + '/') if os.path.exists(local_path + '/job.ini'): return True else: return False
def get_conf(server = '', conf_online = '', local_path = '', config_modify_list = list()): exe_sys_cmd('mkdir -p ' + local_path + '/conf') exe_sys_cmd_get_echo('rsync -vazP ' + server + '::' + conf_online + ' ' + local_path + '/conf/conf_online') if os.path.exists(local_path + '/conf/conf_online'): for config_modify in config_modify_list: if config_modify['config_type'] == 'cfg': exe_sys_cmd_get_echo('cp -f ' + local_path + '/conf/conf_online ' + local_path + '/conf/' + config_modify['config_file']) return True else: return False
def merge_QO_config(ABMF_file_s, ABMF_file_d, split_point, merge_key_list): QO_config_s_main, QO_config_s_fragment = parse_QO_config( ABMF_file_s, split_point) QO_config_d_main, QO_config_d_fragment = parse_QO_config( ABMF_file_d, split_point) merge_QO_config_fragment(QO_config_s_fragment, QO_config_d_fragment, merge_key_list) exe_sys_cmd_get_echo('cp -f ' + ABMF_file_d + ' ' + ABMF_file_d + '_bak_' + str(int(time.time()))) write_QO_config(QO_config_d_main, QO_config_d_fragment, ABMF_file_d)
def vr_type_adjust(offline_data_conf): if (offline_data_conf.get('vrqo_TransVrType') is None) or \ (len(offline_data_conf['vrqo_TransVrType']) == 0): return [] vrqo_TransVrType_base = '{}/base/vrqo_TransVrType'.format(offline_data_conf['data_base_path']) vrqo_TransVrType_target = '{}/base/vrqo_TransVrType'.format(offline_data_conf['data_modify_path']) exe_sys_cmd_get_echo('mkdir -p {}/base/'.format(offline_data_conf['data_modify_path'])) exe_sys_cmd_get_echo('cp -f ' + vrqo_TransVrType_base + ' ' + vrqo_TransVrType_target) modify_list = offline_data_conf['vrqo_TransVrType'] modify_text(modify_list, vrqo_TransVrType_target) return ['base']
def vr_info_adjust(offline_data_conf): if (offline_data_conf.get('pre_server_info') is None): return [] if (offline_data_conf.get('vrqo_VRInfo') is None) or \ (offline_data_conf['vrqo_VRInfo'].get('class_ids') is None) or \ (len(offline_data_conf['vrqo_VRInfo']['class_ids']) == 0): return [] class_ids = offline_data_conf['vrqo_VRInfo']['class_ids'] base_path = offline_data_conf['data_base_path'] target_path = offline_data_conf['data_modify_path'] pre_server_info = offline_data_conf['pre_server_info'] dir_list = ['base/norm'] vr_info_base = '{}/base/norm/vrqo_VRInfo'.format(base_path) exe_sys_cmd_get_echo('mkdir -p {}/base/norm'.format(target_path)) exe_sys_cmd_get_echo('cp -f {} {}/base/norm/'.format(vr_info_base, target_path)) vr_info_pre = '{}/base/norm/vrqo_VRInfo.pre'.format(target_path) sftp_client = SFTPClient(pre_server_info['addr'], "22", pre_server_info['username'], pre_server_info['password']) sftp_client.sftp_get('/search/odin/daemon/vrqo/data/base/norm/vrqo_VRInfo', vr_info_pre) sftp_client.close() info_dict = {} modify_list = [] if not os.path.exists(vr_info_pre): return [] with open(vr_info_pre) as f: for line in f: infos = line.split('\t') info_dict.update({infos[1]: line.strip('\n')}) for class_id in class_ids: modify_list.append({ "anchor": ['^{}'.format(class_id)], "action": "replace", "content": info_dict[class_id] }) is_effect = modify_text(modify_list, '{}/base/norm/vrqo_VRInfo'.format(target_path)) second_modify_list = [] for i in range(len(is_effect)): if is_effect[i] == 0: second_modify_list.append({ "anchor": ['$'], "action": "append", "content": info_dict[class_ids[i]] }) modify_text(second_modify_list, '{}/base/norm/vrqo_VRInfo'.format(target_path)) return dir_list
def regular_file_get(offline_data_conf): dir_list = [] if offline_data_conf.get('dev_data') is None: return [] for dev_data_info in offline_data_conf['dev_data']: if (dev_data_info.get('data_path') is None) or (len(dev_data_info['data_path']) == 0): continue re_path = re.compile('(?P<dir>base.*)/(?P<name>[^/]+)/*$') sftp_client = SFTPClient(dev_data_info['addr'], "22", dev_data_info['username'], dev_data_info['password']) for path in dev_data_info['data_path']: path_suffix = re.search(re_path, path) if path_suffix: file_dict = path_suffix.groupdict() exe_sys_cmd_get_echo('mkdir -p {}/{}'.format(offline_data_conf['data_modify_path'], file_dict['dir'])) dir_list.append(file_dict['dir']) sftp_client.sftp_get(path, '{}/{}/{}'.format(offline_data_conf['data_modify_path'], file_dict['dir'], file_dict['name'])) sftp_client.close() return dir_list
def modify_text(modify_list = dict(), text_file = './file.text', encoding = 'utf8'): result_list = [False] * len(modify_list) text_list = parse_text(text_file, encoding) if len(text_list) == 0: return result_list result_index = 0 for modify_point in modify_list: if modify_point['action'] == 'tail': text_list.append(modify_point['content'] + '\n') result_list[result_index] = True result_index += 1 continue if modify_point['action'] == 'head': text_list[0] = modify_point['content'] + '\n' + text_list[0] result_list[result_index] = True result_index += 1 continue for index in range(len(text_list) - 1, -1, -1): if re.search(modify_point['anchor'][0], text_list[index]) is None: continue if modify_point['action'] == 'replace': text_list[index] = modify_point['content'] + '\n' result_list[result_index] = True if modify_point['action'] == 'append': text_list[index] = text_list[index] + modify_point['content'] + '\n' result_list[result_index] = True if modify_point['action'] == 'insert': text_list[index] = modify_point['content'] + '\n' + text_list[index] result_list[result_index] = True if modify_point['action'] == 'delete': text_list.pop(index) result_list[result_index] = True result_index += 1 exe_sys_cmd_get_echo('cp -f ' + text_file + ' ' + text_file + '_bak_' + str(int(time.time()))) write_text(text_list, text_file, encoding) return result_list
def modify_ini(modify_list, ini_file = './file.ini'): result_list = [False] * len(modify_list) ini_dict = parse_ini(ini_file) if len(ini_dict) == 0: return result_list result_index = 0 for modify_point in modify_list: if len(modify_point['anchor']) == 2: section = modify_point['anchor'][0] param = modify_point['anchor'][1] value = modify_point['content'] if modify_point['action'] == 'replace': if ini_dict.get(section) is not None and ini_dict[section].get(param) is not None: ini_dict[section][param] = value result_list[result_index] = True if modify_point['action'] == 'add': if ini_dict.get(section) is not None: if ini_dict[section].get(param) is not None: ini_dict[section][param] = ini_dict[section][param] + ' ' + value else: ini_dict[section][param] = value else: ini_dict[section] = {param:value} result_list[result_index] = True if modify_point['action'] == 'delete': if ini_dict.get(section) is not None: if param == '': ini_dict.pop(section) result_list[result_index] = True elif ini_dict[section].get(param) is not None: ini_dict[section].pop(param) result_list[result_index] = True result_index += 1 exe_sys_cmd_get_echo('cp -f ' + ini_file + ' ' + ini_file + '_bak_' + str(int(time.time()))) write_ini(ini_dict, ini_file) return result_list
def mk_data_file_link(offline_data_conf): if not offline_data_conf['need_data_modify']: return if offline_data_conf['data_modify_path'] == '': offline_data_conf['data_modify_path'] = sys.path[0] + '/test_data/' module_path = os.path.join(sys.path[0], 'dailybuild/opt/sogou/') else: module_path = os.path.join(offline_data_conf['service_info']['service_deploy_path'], offline_data_conf['service_info']['service_name'], offline_data_conf['service_info']['service_exe_path']) exe_sys_cmd('rm -rf ' + module_path + '/data') exe_sys_cmd('rm -rf ' + offline_data_conf['data_modify_path']) exe_sys_cmd('mkdir -p ' + offline_data_conf['data_modify_path']) exe_sys_cmd('ln -s ' + offline_data_conf['data_modify_path'] + ' ' + module_path + '/data') dir_list_all = list() dir_list_all += regular_file_get(offline_data_conf) dir_list_all += vr_info_adjust(offline_data_conf) dir_list_all += qo_config_adjust(offline_data_conf) dir_list_all += vr_type_adjust(offline_data_conf) dir_list_all += vr_file_get(offline_data_conf) for index in range(len(dir_list_all) - 1, -1, -1): if dir_list_all[index] in dir_list_all[:index]: dir_list_all.pop(index) for loop_id_1 in range(len(dir_list_all)): for loop_id_2 in range(loop_id_1 + 1, len(dir_list_all)): if len(dir_list_all[loop_id_1].split('/')) < len(dir_list_all[loop_id_2].split('/')): dir_list_all[loop_id_1], dir_list_all[loop_id_2] = dir_list_all[loop_id_2], dir_list_all[loop_id_1] for it in dir_list_all: if (re.search('base/vr$', it) is not None) or (re.search('base/vr/$', it) is not None): file_prefix_list = split_sys_cmd_echo(exe_sys_cmd_get_echo('ls | grep -o ^... | sort -u'), 0) for file_prefix in file_prefix_list: exe_sys_cmd(' ln -s ' + offline_data_conf['data_base_path'] + '/' + it + '/' + file_prefix + '* ' + offline_data_conf['data_modify_path'] + '/' + it + '/' + file_prefix + '*') else: exe_sys_cmd(' ln -s ' + offline_data_conf['data_base_path'] + '/' + it + '/* ' + offline_data_conf['data_modify_path'] + '/' + it + '/')
def qo_config_adjust(offline_data_conf): if (offline_data_conf.get('pre_server_info') is None): return [] if (offline_data_conf.get('vrqo_QOConfig') is None) or \ (offline_data_conf['vrqo_QOConfig'].get('class_ids') is None) or \ (len(offline_data_conf['vrqo_QOConfig']['class_ids']) == 0): return [] class_ids = offline_data_conf['vrqo_QOConfig']['class_ids'] base_path = offline_data_conf['data_base_path'] target_path = offline_data_conf['data_modify_path'] pre_server_info = offline_data_conf['pre_server_info'] dir_list = ['base/norm'] qo_config_base = '{}/base/norm/vrqo_QOConfig'.format(base_path) qo_config_target = '{}/base/norm/vrqo_QOConfig'.format(target_path) qo_config_pre = '{}/base/norm/vrqo_QOConfig.pre'.format(target_path) exe_sys_cmd_get_echo('mkdir -p {}/base/norm'.format(target_path)) exe_sys_cmd_get_echo('cp -f {} {}/base/norm/'.format(qo_config_base, target_path)) sftp_client = SFTPClient(pre_server_info['addr'], "22", pre_server_info['username'], pre_server_info['password']) sftp_client.sftp_get('/search/odin/daemon/vrqo/data/base/norm/vrqo_QOConfig', qo_config_pre) sftp_client.close() pattern_names = [] rule_names = [] for class_id in class_ids: cmd_echo = exe_sys_cmd_get_echo('fgrep {} {}|fgrep "PATTERN_NAME"'.format(class_id, qo_config_target)) if cmd_echo != '': pattern_names += cmd_echo.split('\n') cmd_echo = exe_sys_cmd_get_echo('fgrep {} {}|fgrep "RULE_NAME"'.format(class_id, qo_config_target)) if cmd_echo != '': rule_names += cmd_echo.split('\n') merge_QO_config(qo_config_pre, qo_config_target, [{"start": "<PATTERN NEW>", "end": "<PATTERN END>", "key": "PATTERN_NAME"}, {"start": "<RULE NEW>", "end": "<RULE END>", "key": "RULE_NAME"}], {'<PATTERN NEW>': pattern_names, '<RULE NEW>': rule_names} ) return dir_list
dir_list_all += vr_file_get(offline_data_conf) for index in range(len(dir_list_all) - 1, -1, -1): if dir_list_all[index] in dir_list_all[:index]: dir_list_all.pop(index) for loop_id_1 in range(len(dir_list_all)): for loop_id_2 in range(loop_id_1 + 1, len(dir_list_all)): if len(dir_list_all[loop_id_1].split('/')) < len(dir_list_all[loop_id_2].split('/')): dir_list_all[loop_id_1], dir_list_all[loop_id_2] = dir_list_all[loop_id_2], dir_list_all[loop_id_1] for it in dir_list_all: if (re.search('base/vr$', it) is not None) or (re.search('base/vr/$', it) is not None): file_prefix_list = split_sys_cmd_echo(exe_sys_cmd_get_echo('ls | grep -o ^... | sort -u'), 0) for file_prefix in file_prefix_list: exe_sys_cmd(' ln -s ' + offline_data_conf['data_base_path'] + '/' + it + '/' + file_prefix + '* ' + offline_data_conf['data_modify_path'] + '/' + it + '/' + file_prefix + '*') else: exe_sys_cmd(' ln -s ' + offline_data_conf['data_base_path'] + '/' + it + '/* ' + offline_data_conf['data_modify_path'] + '/' + it + '/') if __name__ == "__main__": if len(sys.argv) == 2: data_modify_file = sys.argv[1] if os.path.exists(data_modify_file): offline_data_conf = parse_json(data_modify_file) if os.path.exists(offline_data_conf['data_base_path']): os.chdir(offline_data_conf['data_base_path']) exe_sys_cmd_get_echo('sh Down.sh') else: sys.exit(1) mk_data_file_link(offline_data_conf)
def svn_checkout(self, svn_url, local_path): if not os.path.exists(local_path): os.makedirs(local_path) exe_sys_cmd_get_echo("svn --username '" + self.username + "' --password '" + self.password + "' checkout " + svn_url + " " + local_path)
def compile_dailybuild(job_ini='', local_path=''): dailybuild_file = '' exe_sys_cmd_get_echo('rm -f ' + local_path + '/_result_/result.tar.gz') if not yum_install_referto_jobini(job_ini, 'local'): return False, dailybuild_file exe_sys_cmd_get_echo( 'sgbuild -i docker-reg.sogou-inc.com/sgbuild/centos7.4:6 -c ' + job_ini + ' -d ' + local_path) if os.path.exists(local_path + '/_result_/result.tar.gz'): if os.path.exists(local_path + '/_result_/result.tar.gz'): exe_sys_cmd_get_echo('rm -rf ' + local_path + '/dailybuild/') exe_sys_cmd_get_echo('mkdir -p ' + local_path + '/dailybuild/') exe_sys_cmd_get_echo('tar -xf ' + local_path + '/_result_/result.tar.gz -C ' + local_path + '/dailybuild/') exe_sys_cmd_get_echo('mv ' + local_path + '/dailybuild/opt/sogou/lib64 ' + local_path + '/dailybuild/opt/sogou/lib') # add modified conf files in daily build exe_sys_cmd_get_echo('cp -rf ' + local_path + '/conf ' + local_path + '/dailybuild/opt/sogou/') exe_sys_cmd_get_echo('tar -czf ' + local_path + '/dailybuild/dailybuild.tar.gz -C ' + local_path + '/ dailybuild/opt') if os.path.exists(local_path + '/dailybuild/dailybuild.tar.gz'): dailybuild_file = local_path + '/dailybuild/dailybuild.tar.gz' return True, dailybuild_file return False, dailybuild_file