def execute_sql(self, command='', database=''):
is_ok = True
table_head_list = list()
data_list = list()
try:
cursor = self.mysql_conn.cursor()
if database != '':
cursor.execute('use ' + database)
rowcount = cursor.execute(command)
if rowcount > 0:
if cursor.description is not None:
table_head_list = [
tuple[0] for tuple in cursor.description
]
for data in cursor.fetchall():
if len(table_head_list) == len(data):
data_list.append(list(data))
else:
is_ok = False
table_head_list.clear()
data_list.clear()
break
self.mysql_conn.commit()
cursor.close()
except Exception as err:
is_ok = False
table_head_list.clear()
data_list.clear()
write_log(command)
write_log(err)
return is_ok, table_head_list, data_list
def sftp_rmdir(self, dir=''):
stdin, stdout, stderr = self.ssh_conn.exec_command('rm -rf ' + dir)
if len(stderr.read()) == 0:
write_log("ftp remove dir " + dir + " succ")
return True
else:
write_log("ftp remove dir " + dir + " fail")
return False
def __init__(self, ip, port, user, password):
self.sftp_is_alive = False
try:
super().__init__(ip, port, user, password)
self.sftp_conn = self.ssh_conn.open_sftp()
self.sftp_is_alive = True
except Exception as err:
write_log(err)
def exe_sys_cmd_get_echo_v3(os_cmd=''):
# need python3
cmd_echo = ''
echo_error_code = -1
if os_cmd != '':
write_log(os_cmd)
echo_error_code, cmd_echo = subprocess.getstatusoutput(os_cmd)
write_log(cmd_echo)
return cmd_echo, echo_error_code
def exe_sys_cmd_v3(os_cmd=''):
# need python3
is_ok = False
if os_cmd != '':
write_log(os_cmd)
echo_error_code = subprocess.call(os_cmd, shell=True)
if echo_error_code == 0:
is_ok = True
return is_ok
def __init__(self, ip, port, user, password):
self.ssh_conn = paramiko.SSHClient()
self.ssh_conn.set_missing_host_key_policy(paramiko.AutoAddPolicy())
self.ssh_is_alive = False
try:
self.ssh_conn.connect(ip, port, user, password)
self.ssh_is_alive = True
except Exception as err:
write_log(err)
def parse_text(text_file = './file.text', encoding = 'utf8'):
text_list = list()
if os.path.exists(text_file):
f_hd = open(text_file, 'r', encoding = encoding, errors='ignore')
text_list = f_hd.readlines()
f_hd.close()
else:
write_log("failed. non-existent, please check " + text_file)
return text_list
def parse_xml_rude(xml_file='./file.xml'):
xml_list = list()
if os.path.exists(xml_file):
f_hd = open(xml_file, 'r')
xml_list = f_hd.readlines()
f_hd.close()
else:
write_log("failed. non-existent, please check " + xml_file)
return xml_list
def exe_sys_cmd_get_echo(os_cmd=''):
cmd_echo = ''
if os_cmd != '':
write_log(os_cmd)
cmd_pipe = os.popen(os_cmd)
cmd_echo = cmd_pipe.read()
cmd_pipe.close()
write_log(cmd_echo)
return cmd_echo
def __init__(self, user, password, ip, port, database):
self.mysql_is_alive = False
try:
self.mysql_conn = pymysql.connect(user=user,
password=password,
host=ip,
port=int(port),
database=database)
self.mysql_is_alive = True
except Exception as err:
write_log(err)
def modify_job_ini(config_modify_list=list(), local_path=''):
is_ok = False
if not os.path.exists(local_path + '/job.ini'):
return is_ok
for config_modify_item in config_modify_list:
if config_modify_item['config_type'] == 'ini':
is_ok = modify_ini(config_modify_item['modContent'],
local_path + '/job.ini')
if not is_ok:
write_log("modify ini file failed. " + local_path +
'/job.ini')
return is_ok
def processor(build_tag='', step=''):
# gen job_config in control server
mysql_info = {'mysql_user': '******',
'mysql_password': '******',
'mysql_ip': '10.134.104.40',
'mysql_port': 3306,
'mysql_database': 'main_project'}
if step == 'gen_job_config':
if build_tag != '':
if not (gen_job_config_main(mysql_info, build_tag) and
gen_job_config_module_conf_mod(mysql_info, build_tag) and
gen_job_config_module_data_mod(mysql_info, build_tag) and
gen_job_config_module_start_script(mysql_info, build_tag)):
sys.exit(1)
sys.exit(0)
# build step by step in test server
conf_main, conf_module_mod, conf_data_mod = get_job_config_file_name(os.path.join(sys.path[0], 'conf/'), build_tag)
deploy_info = parse_json(conf_main)
if step not in deploy_info['deploy_info']['deploy_step'].keys() or \
deploy_info['deploy_info']['deploy_step'][step] == '0':
write_log('step ' + step + ' is illegal, skip')
sys.exit(0)
local_path = ''
for server_info in deploy_info['offline_server_info']:
if check_is_local_server(server_info['addr']):
# local_path = server_info['deploy_path'] + '/' + deploy_info['service_info']['service_name'] + '/'
local_path = server_info['deploy_path'] + '/'
break
if local_path == '':
sys.exit(1)
if not exe_sys_cmd_v3('mkdir -p ' + local_path):
sys.exit(1)
env_prepare()
if step == 'is_establish':
if not module_install(deploy_info, conf_module_mod, local_path):
sys.exit(2)
elif step == 'is_config':
if not module_configure(deploy_info, conf_module_mod, local_path)[0]:
sys.exit(3)
elif step == 'is_data':
if not module_data_modify(deploy_info, conf_data_mod):
sys.exit(4)
elif step == 'is_start':
if not module_start(deploy_info, local_path, build_tag):
sys.exit(5)
elif step == 'is_dailybuild':
from proc.package_deploy_dailybuild import gen_dailybuild
is_ok, conf_name, dailybuild_file = gen_dailybuild(deploy_info, local_path)
if not is_ok:
sys.exit(5)
else:
print(dailybuild_file)
def parse_QO_config(ABMF_file, split_point):
ABMF_list = list()
ABMF_partial_fragment = _tree()
if not os.path.exists(ABMF_file):
write_log("failed. non-existent, please check " + ABMF_file)
return ABMF_list, ABMF_partial_fragment
f_hd = open(ABMF_file, 'r')
if len(split_point) == 0:
ABMF_list = f_hd.readlines()
f_hd.close()
return ABMF_list, ABMF_partial_fragment
is_find = False
sub_content = ''
sub_key = ''
split_point_index = 0
for line in f_hd:
# hard code for vrqo_QOConfig modification
if line.find('# rule : map.index blk') != -1:
ABMF_list.append('#---divide_line---\n')
if not is_find:
for check_point in split_point:
if line.find(check_point["start"]) != -1:
sub_content = line
is_find = True
break
split_point_index += 1
if is_find:
continue
else:
split_point_index = 0
if is_find:
if line.find(split_point[split_point_index]["end"]) != -1:
sub_content = sub_content + line
ABMF_partial_fragment[split_point[split_point_index]
["start"]][sub_key] = sub_content
ABMF_list.append('PlaceHolder__' + '--'.join(
[split_point[split_point_index]["start"], sub_key]))
split_point_index = 0
is_find = False
if not is_find:
continue
if is_find:
sub_content = sub_content + line
if line.find(split_point[split_point_index]["key"]) != -1:
sub_key = trim_head_tail(line.split('=')[1])
continue
ABMF_list.append(line)
f_hd.close()
return ABMF_list, ABMF_partial_fragment
def parse_json(json_file='./file.json'):
json_dict = dict()
if os.path.exists(json_file) and os.path.splitext(json_file)[1] == '.json':
f_hd = open(json_file, 'r')
try:
json_dict = json.load(f_hd)
except:
json_dict.clear()
write_log("failed. parse json file error, please check " +
json_file)
f_hd.close()
else:
write_log("failed. non-existent or non-json-format , please check " +
json_file)
#sys.exit(1)
return json_dict
def parse_ini(ini_file = './file.ini'):
ini_dict = _tree()
if os.path.exists(ini_file):
parser = ConfigParser()
try:
parser.read(ini_file)
except:
write_log("failed. parse ini file error, please check " + ini_file)
return ini_dict
section_list = parser.sections()
for section in section_list:
param_list = parser.options(section)
for param in param_list:
ini_dict[section][param] = parser[section][param]
else:
write_log("failed. non-existent or non-ini-format , please check " + ini_file)
return ini_dict
def write_ini(data = dict(), ini_file = './file.ini'):
parser = ConfigParser()
try:
parser.read_dict(data)
except:
write_log("failed. parse ini data error, please check " + str(data))
return False
try:
f_hd = open(ini_file, 'w')
except:
return False
try:
parser.write(f_hd)
except:
f_hd.close()
return False
f_hd.close()
return True
def exe_ssh_cmd_get_echo(self, os_cmd=''):
cmd_echo = ''
cmd_echo_err = ''
if os_cmd != '':
write_log(os_cmd)
stdin, stdout, stderr = self.ssh_conn.exec_command(os_cmd)
if len(stderr.read()) == 0:
cmd_echo = bytes.decode(stdout.read())
write_log(cmd_echo)
else:
cmd_echo_err = bytes.decode(stderr.read())
write_log(bytes.decode(stderr.read()))
return cmd_echo, cmd_echo_err
def gen_cnv_report(cnv_data_file, cu, cl, genelist, outfile, source):
"""
If required, run an Oncomine CNV report to get that data from the VCF file
as well.
"""
cnv_data = []
parsed_cnv_results = []
if source == 'oca':
cmd = (os.path.join(scripts_dir, 'get_cnvs.pl'), '--cu', cu, '--cl',
cl, cnv_data_file)
parsed_cnv_results = run(cmd,
"Generating Oncomine CNV results data",
True,
silent=not verbose)
elif source == 'tso500':
cmd = (os.path.join(scripts_dir, 'tso500_cnvs.pl'), cnv_data_file)
parsed_cnv_results = run(cmd,
"Generating TSO500 CNV results data",
True,
silent=not verbose)
elif source == 'wes':
logger.write_log(
'error', 'CNV data acquisition is not yet implemented '
'for WES panels yet. Skipping this step.')
return None
header = parsed_cnv_results.pop(0).rstrip('\n').split(',')
# If we have results, then generate a report. Else, bail out.
if parsed_cnv_results:
for r in parsed_cnv_results:
data = dict(zip(header, r.rstrip('\n').split(',')))
# Fix measurements to accomodate multiple assays.
if source == 'oca':
cn = float(data['Raw_CN'])
lower_reading = float(data['CI_05'])
upper_reading = float(data['CI_95'])
elif source == 'tso500':
cn = lower_reading = upper_reading = float(data['FoldChange'])
# cnv_str = '{},{}'.format(data['Gene'], data['Raw_CN'])
cnv_str = '{},{}'.format(data['Gene'], cn)
if lower_reading >= float(cu):
data.update({'var_type': 'Amplification'})
elif upper_reading <= float(cl):
data.update({'var_type': 'Deletion'})
# Remove any genes not in our filter list if there is one.
if genelist and data['Gene'] not in genelist:
log.write_log(
'debug', 'Filtering out {} because it is not '
'in the requested list.'.format(cnv_str))
continue
# Filter events outside of our thresholds.
if not __cnv_filter(lower_reading, upper_reading, cu, cl):
log.write_log(
'debug', 'Filtering out {} because it is not '
'an copy number event within the threshold (copy loss={}; '
'copy_gain={}).'.format(cnv_str, cl, cu))
continue
cnv_data.append(data)
if not cnv_data:
log.write_log(
'info', 'No copy number amplifications or deletions '
'found in this specimen.')
else:
log.write_log(
'info', 'Found {} copy number events in the '
'specimen.'.format(len(cnv_data)))
log.write_log('info', 'Annotating with OncoKB lookup.')
__oncokb_cnv_and_fusion_annotate(cnv_data, oncokb_cnv_file, 'cnv')
# Set up column headers for different assays.
assay_elems = {
'oca': ('Chr', 'Start', 'End', 'Gene', 'CN', 'CI_05', 'CI_95', 'MAPD',
'Oncogenicity', 'Effect'),
'tso500':
('Chr', 'Start', 'End', 'Gene', 'FoldChange', 'Oncogenicity', 'Effect')
}
with open(outfile, 'w') as outfh:
csv_writer = csv.writer(outfh, lineterminator="\n", delimiter=",")
wanted = assay_elems[source]
csv_writer.writerow(wanted)
if cnv_data:
for elem in cnv_data:
data = [elem[x] for x in wanted]
csv_writer.writerow(data)
else:
outfh.write("No CNVs found.\n")
return cnv_data
self.ssh_conn.close()
@ssh_alive
def exe_ssh_cmd_get_echo(self, os_cmd=''):
cmd_echo = ''
cmd_echo_err = ''
if os_cmd != '':
write_log(os_cmd)
stdin, stdout, stderr = self.ssh_conn.exec_command(os_cmd)
if len(stderr.read()) == 0:
cmd_echo = bytes.decode(stdout.read())
write_log(cmd_echo)
else:
cmd_echo_err = bytes.decode(stderr.read())
write_log(bytes.decode(stderr.read()))
return cmd_echo, cmd_echo_err
# for test
if __name__ == "__main__":
test_ssh_client = SSHClient('10.134.91.94', "22", "root",
"noSafeNoWork@2014")
echo, err = test_ssh_client.exe_ssh_cmd_get_echo('ls -rtl ')
#test_ssh_client.exe_ssh_cmd_get_echo('cd /search/odin/xujiang/vrqo_test/QueryOptimizer; nohup ./vrqo vrqo.cfg.ex >std.ex.diff 2>err.ex.diff &')
test_ssh_client.close()
from lib.sys_inter_act import split_sys_cmd_echo
echo_split_to_list = split_sys_cmd_echo(echo)
write_log(echo_split_to_list)
echo_split_to_list_partial = split_sys_cmd_echo(echo, 5)
write_log(echo_split_to_list_partial)
def exe_sys_cmd(os_cmd=''):
if os_cmd != '':
write_log(os_cmd)
os.system(os_cmd)
return
