def install_some(self): tempDir = [] for i in range(len(self.args)): tree = self.installDir + "tools/" + self.args[i] tempDir.append(tree) for i in range(len(self.args)): target = self.args[i] if target in ["crips", "arachni", "brutex", "revsh", "nmap"]: self.build_tools(target, tempDir[i]) else: # target is package if target in pkg: print(color.LOGGING + "[+] - Installing " + color.NOTICE + "{}".format(target) + color.LOGGING + "...") shell("git clone -q {0} {1}".format( pkg[target], tempDir[i])) else: print( color.RED + "[*] - Unkown package. Please retry or use the custom package installer" + color.END) self.add_words(self.args)
def perform_centrifuge_cycle(self, name, cycle): # Dont start if door is open #is_door_closed doesn't return "yes" or "no" string, it returns one bit, 1 or 0, corresponding to True or False #this should be if self.is_door_closed() == 1: if self.is_door_closed() == "no": return "door not closed" self._cycle_running = True for step in cycle.split("\n"): s = int(step.split(" for ")[0][:-3]) t = int(step.split(" for ")[1][:-8]) if s > self._speed_cap: continue self.speed(s) # Wait for it to get to our desired speed self.target_speed = s #instead of not self.got_speed > self.target_speed, the not can be taken out and the inequality can be flipped #this can become while self.got_speed < self.target_speed: while not self.got_speed > self.target_speed: self.getSpeed() # Run at our desired speed for the given start_wait = datetime.now() while (datetime.now() - start_wait).total_seconds() < t: pass self._cycle_running = False os.shell("net send localhost \"Done cycle " + name + '"')
def updateIp(self, recur=3): if not self.alive:self.exit() socks.socket.setdefaulttimeout(5) socks.setdefaultproxy(socks.PROXY_TYPE_SOCKS5, '127.0.0.1', 9050, True) socket.socket = socks.socksocket try: ip = self.getIp() if all([not ip, recur]): print('Error: Network unreachable') reset_counts = 2 for _ in range(30): if not self.alive:return ip = self.getIp() if ip:break else: if reset_counts: reset_counts -= 1 shell('service network-manager restart') sleep(1) if not ip:self.restartTor(recur-1) if all([not ip, not recur]):self.connection() if ip in self.recentIPs.queue:self.restartTor() else: self.ip = ip self.recentIPs.put(ip) except:pass
def __init__(self, installDir, toolDir): self.installDir, self.toolDir = installDir, toolDir link = input("Git repository of the tool (full link): ") name = input("Tool name: ") ver = input("Python version: ") exe = input("Name of the file to launch (w/o extension): ") cmds = input("Custom command (leave blank if unsure): ") issudo = input( "Does the package needs root permissions? [y/N]: ").lower() #Add question if script has a different name #g.e: main.py insttead of <projectname>.py temp = 0 if not cmds: if issudo != "y": cmds = "python{0} {1}{2}/{3}.py".format( ver, self.toolDir, name, exe) else: cmds = "sudo python{0} {1}{2}/{3}.py".format( ver, self.toolDir, name, exe) try: shell("git clone %s %s%s" % (link, self.toolDir, name)) temp = 1 except: temp = -1 ErrorHandler(err(), False) if temp: dictmgr.addWords(self.installDir, [name]) dictmgr.addCustomWords(self.installDir, name) dictmgr.updateConfig(self.installDir, name, cmds) print( "[*] - You may need to restart onifw in order to use the custom tool." )
def remove_all(installDir, root=1): toolDir = installDir + 'tools/' # To avoid errors if root == 1: shell("sudo rm -rf {}*".format(toolDir)) else: shell("rm -rf {}*".format(toolDir))
def display(self): # Affichage de la map avec/sans les statistiques shell('clear' if(system() == "Linux") else 'cls') if(bool(self.stat)): # Initialisation & Mise à jours des statistiques active = 0 for cells in self.__map: for cell in cells: active += cell stats = ( f"Name : {self.mapName}", f"Dimensions : {self.__dims[0]}x{self.__dims[1]}", f"Actives : {Colors.green if(active < int(self.__cells/2)) else Colors.red}{active}{Colors.end}" ) for i, line in enumerate(self.__map): row = "" for value in line: row += f'{f"{Colors.green}O" if(value) else f"{Colors.cyan}."}{Colors.end} ' if(bool(self.stat) and (i < len(stats))): # Affichage des statistiques row += f" {stats[i]}" print(row) return(True)
def vpn_connexion_off(): shell("anonsurf stop") ip3 = shell("curl -X GET https://api.ipify.org > output.txt && clear") with open("output.txt", "r") as r: my_ip3 = r.readlines() r.close() ip_label3 = Label(root, text=my_ip3, fg="black").place(x=110, y=50)
def vpn_connexion_on(): shell("anonsurf start") ip2 = shell("curl -X GET https://api.ipify.org > output.txt && clear") with open("output.txt", "r") as p: my_ip2 = p.readlines() p.close() ip_label2 = Label(root, text=my_ip2, fg="black").place(x=110, y=50)
def parse_args(args): "parse arguments" package_manager_name = args[0] def get_settings_path(current_platform): if current_platform.startswith("linux"): return f'{getenv("HOME")}/.config/PMM/settings.json' if current_platform.startswith("darwin"): return ( f'/Users/{getenv("USER")}/Library/Application Support/PMM/settings.json' ) if current_platform.startswith("win32"): return f'{getenv("APPDATA")}/\\PMM\\settings.json' return "" url = "https://raw.githubusercontent.com/Charlie-Sumorok/PMM/main/package-managers/data.json" package_manager_data = requests.get(url).json() package_manager = package_manager_data[package_manager_name] with open(get_settings_path(platform), "r+t", encoding="utf-8") as settings: old_settings_data = json.loads(settings.read()) shell(package_manager["commands"]["install"][ old_settings_data["package_manager"]]) new_settings_data = { **old_settings_data, args[0]: { "enabled": package_manager["enabled"], "installed": True }, } settings.seek(0) # move to start settings.write(json.dumps(new_settings_data, indent=4))
def deploy_button(channel): template = DRY_RUN_DEPLOY_TEMPLATE if DRY_RUN_MODE else DEPLOY_TEMPLATE if SAFE_MODE: sleep(0.5) if (not GPIO.input(channel)): print( 'Safe mode is on. Press the deploy button for at least half a second. Button cooldown is 3 seconds' ) return print(figlet_format('DEPLOYING')) commands = [GIT_LATEST] # Make sure video-k8s is up to date environments_to_deploy = [[pin, env] for pin, env in SWITCHES.items() if GPIO.input(pin)] if not environments_to_deploy: shell('bash /home/pi/skedify-deploy-button/roll.sh' ) # Hit em with the man return for pin, env in environments_to_deploy: commands.append( template.format(cluster_name=CLUSTER_NAMES[env], environment=env, k8s_folder=VIDEO_K8S_FOLDER)) shell('&& '.join(commands)) print(figlet_format('DONE'))
def installTor(self): self.connection() if not self.alive: return print('Installing Tor ...') shell( 'echo "deb http://http.kali.org/kali kali-rolling main contrib non-free" > /etc/apt/sources.list \ && apt-get update && apt-get install tor -y && apt autoremove -y' )
def bot_deactivate_vk_posts(pk: int): """ Функция деактивации бота :param pk: ID бота :type pk: int """ shell('rm /home/$(whoami)/virtual-person/website/bot/cron/{pk}.cron'.format(pk=pk))
def handle_option(option: int): if option in handled: return option = OPTIONS[option - 1] if 'depends' in option: handle_option(tag2idx[option['tag']] - 1) if 'shell' in option: shell('bash -c "{}"'.format(option['shell'])) handled.append(option)
def install_all(self): wordlist = [] for i in pkg: temp_install_dir = self.toolDir + i if i in ["crips", "arachni", "brutex", "revsh", "nmap"]: self.build_tools(i, temp_install_dir) else: print(color.LOGGING + "[+] - Installing " + color.NOTICE + "{}".format(i) + "...") shell("git clone -q {0} {1}".format(pkg[i], temp_install_dir)) wordlist.append(i) self.add_words(wordlist)
def bot_activate_vk_posts(time: str, pk: int): """ Функция активации бота :param time: Время запуска бота :type time: str :param pk: ID бота :type pk: int """ activate_venv = '/home/$(whoami)/virtual-person/venv/bin/python' command = 'echo "{time} $(whoami) {activate_venv} /home/$(whoami)/virtual-person/website/bot/posts_to_vk.py {id}" ' \ '> /home/$(whoami)/virtual-person/website/bot/cron/{id}.cron'.format(time=time, id=pk, activate_venv=activate_venv) shell(command)
def ping_host(host, pings): # Ping parameters as function of OS parameters = "-n " if system_name().lower() == "windows" else "-A -w15 -c " # Pinging result = shell("ping " + parameters + str(pings) + " " + host + " >" + devnull + " 2>&1") == 0 return result
def __init__(self, installDir, toolDir): self.installDir, self.toolDir = installDir, toolDir link = input("Git repository of the tool (full link): ") name = input("Tool name: ") nb_cmd = int(input("How many commands to build the tool?: ")) try: shell("git clone %s %s %s" % (link, self.toolDir, name)) for i in range(nb_cmd): print("[*] - Current directory: %s" % shell("pwd")) cmd = input("Custom command: ") shell(cmd) cmds = input("Launch command: ") dictmgr.addWords(self.installDir, name) dictmgr.addCustomWords(self.installDir, name) dictmgr.updateConfig(self.installDir, name, cmds) except: ErrorHandler(err(), False)
def main(system=system()): if (len(argv) > 1): if (argv[1] in ("-l", "--load")): try: repo = Rep(argv[2]) except Exception: print(f"{Icons.warn}Insérer le nom du fichier en argument") return (False) else: shell("clear" if (system == "Linux") else "cls") path = str(input(f"Entrer le nom du répertoire: {Colors.green}")) print(Colors.end) repo = Rep(path) while (True): try: entry = str(input(f"\n> {Colors.cyan}"))[0] print(Colors.end) commands = ( ("Cc", lambda: repo.insert()), ("Dd", lambda: repo.rem()), ("Aa", lambda: repo.sortOnceByName()), ("Gg", lambda: repo.sortAll()), ("?", lambda: repo.menu()), ) for command in commands: if (entry in command[0]): command[1]() if (entry in "Qq"): print(f"{Icons.info}Fermeture du répertoire") break except Exception: print(f"{Icons.warn}Céléstin arrête tes carabistouilles !") print("Au revoir kheyou ;)") return (True)
def uninstall(installDir, cmd, root=0): print("[*] - Removing folder") if root == 0: for i in cmd: shell("rm -rf {0}tools/{1}".format(installDir, i)) else: for i in cmd: shell("sudo rm -rf {0}tools/{1}".format(installDir, i)) print(color.LOGGING + "[*] - Cleaning dictionnary..." + color.END) f = open("{}data/dict.txt".format(installDir)) out = [] for line in f: for i in cmd: if not i in line: out.append(line) f.close() f = open("{}data/dict.txt".format(installDir), 'w') f.writelines(out) f.close()
def __init__(self, installDir, toolDir): lang_dict = { "perl": "perl", "ruby": "ruby", "go": "go", "java-jar": "jar", "java": "java", } self.installDir, self.toolDir = installDir, toolDir print(color.OKBLUE + "Available languages:") for i in lang_dict.keys(): print(i) print(color.END) lang = input("Select lang: ") link = input("Git repository of the tool (full link): ") name = input("Tool name: ") name_exe = input("Name of the main file (w/ entension): ") nb_cmd = int(input("How many commands to build the tool?: ")) try: shell("git clone %s %s%s" % (link, self.toolDir, name)) for i in range(nb_cmd): print("[*] - Current directory: %s" % shell("pwd")) cmd = input("Custom command: ") shell(cmd) if lang == "java": cmds = "{0} = cd {1}{2} && {3}{4}".format( name, toolDir, name, lang_dict[lang], name_exe) else: cmds = "{0} = {1} {2}{3}{4}".format(name, lang_dict[lang], toolDir, name, name_exe) with open("{}onirc".format(self.installDir), "a") as f: f.write("{0} = {1}\n".format(name, cmds)) f.close() with open("{}data/dict.txt".format(self.installDir), "a") as f: f.write(name + '\n') f.close() with open("{}data/ctools.txt".format(self.installDir), "a") as f: f.write(name + '\n') f.close() except: ErrorHandler(err(), False)
def jump(now, next_jump): distance_x = abs(now[0] - next_jump[0]) distance_y = abs(now[1] - next_jump[1]) distance = sqrt(pow(distance_x, 2) + pow(distance_y, 2)) # 要跳跃的距离 millisecond = distance * k millisecond = round(millisecond) adb_command = "adb shell input swipe {} {} {} {} {}".format( now[0], now[1], now[0] + 10, now[1] + 10, millisecond) if shell(adb_command) != 0: print("adb swipe执行失败") exit(0)
def target_pileup_from_mut(mut_file, base_file, bam, chrom): ''' piles up the mutation list in the tumor bam ''' # bed file can contain all chromosomes because chrom restriction comes with the -r parameter bed_file = f"{base_file}.bed" # create the bed file for mpileup shell(f"{csv2bed} < {mut_file} > {bed_file}") # # if I want to restrict chromosome in file: # mut_chr_file = f"{base_file}.csv" # mut_df.to_csv(mut_chr_file, sep='\t', index=False) # # create the bed file for mpileup from the mutation file # shell(f"{csv2bed} < {mut_chr_file} > {bed_file}") # do the pileup into the matrix file matrix_file = f"{base_file}.matrix" pileup_cmd = f"samtools mpileup -B -q {EBparams['MAPQ']} -Q {EBparams['Q']}" pileup_cmd += f" -l {bed_file} -r {chrom} {tumor_bam}" pipe_cmd = f"{pileup_cmd} | cut -f 1,2,5 | {cleanpileup} | {pile2count} > {matrix_file}" show_output(f"Piling up tumor bam {tumor_bam}", color='normal') # do the pileup to matrix_file show_command(pipe_cmd, multi=False) shell(pipe_cmd) # cleanup shell(f"rm -f {bed_file}") show_output( f"Pileup matrix for chrom {chrom} of {tumor_bam} completed. Merging with cache file...", color='normal' ) return matrix_file
def __init__(self, installDir): self.installDir = installDir if not self.check_branch(): try: with open("{}data/version.txt".format(installDir)) as f: local_version = version.parse( f.readlines()[0].rstrip("\n\r")) f.close() latest_version = check_output( "curl -s https://raw.githubusercontent.com/w0bos/onifw/master/src/data/version.txt", shell=True).decode("utf-8").strip('\r\n') late = version.parse(latest_version) if late > local_version: ans = input( color.NOTICE + "[*] - A new version is available\nDo you wish to install the new update? [y/N] :" + color.END) if ans.lower() in ["yes", "y"]: # Won't wipe old install shell("cd {} && git pull".format(installDir)) else: print("[*] - Update aborted") elif late == local_version: print( color.OKGREEN + "[*] - You're already running the latest version of onifw" + color.END) elif late < local_version: print(color.BOLD + color.IMPORTANT + "[+] - You are running an alpha version of onifw" + color.END) else: print(color.WARNING + "[!] - Unknown error" + color.END) shell("rm -rf {}/temp".format(installDir)) except: ErrorHandler(err(), False, True)
def perform_centrifuge_cycle(self, name, cycle): # Dont start if door is open if self.is_door_closed() == "no": return "door not closed" self._cycle_running = True for step in cycle.split("\n"): s = int(step.split(" for ")[0][:-3]) t = int(step.split(" for ")[1][:-8]) if s > self._speed_cap: continue self.speed(s) # Wait for it to get to our desired speed self.target_speed = s while not self.got_speed > self.target_speed: self.getSpeed() # Run at our desired speed for the given t start_wait = datetime.now() while (datetime.now() - start_wait).total_seconds() < t: pass self._cycle_running = False os.shell("net send localhost \"Done cycle " + name + '"')
def compile(file: str) -> None: """ Compile a file using latexmk and xelatex. The output PDF file is placed in the same directory as the source file. Also removes auxiliary files produced by the latex compiler when finished. Parameters ---------- file : str The path to the file to compile Raises ------ RuntimeError: If xelatex or latexmk cannot be count. """ log.info('Current path variable: {}'.format(os.getenv('PATH'))) if which('latexmk') is None or which('xelatex') is None: raise RuntimeError( 'xelatex or latexmk could not be found. Leaving Latex source files as-is.' ) output_dir = path.split(file)[0] command = r'/usr/local/texlive/2020/bin/x86_64-linux/latexmk -xelatex -output-directory={} {}'.format( output_dir.replace(' ', '\\ '), file.replace(' ', '\\ ')) log.info('Running shell command {}'.format(command)) code = shell(command).read() if code != '0': log.error('latexmk failed. Check the log file for errors') else: log.info('PDF file successfully generated. ') log.info('Cleaning up') # Delete remaining files to_delete = list() basename = path.splitext(path.basename(file))[0] to_delete.append(path.join(output_dir, '{}.log'.format(basename))) to_delete.append(path.join(output_dir, '{}.aux'.format(basename))) for file in to_delete: log.debug('Deleting file {}'.format(path.basename(file))) remove(file)
def runJDFTx(self, inputfile): """ Runs a JDFTx calculation """ # Make a temp directory directory = 'temp.%s' % (int(round(100000 * random.random()))) shell('mkdir %s' % (directory)) shell('cd %s && echo \'%s\' | %s -o temp.out' % (directory, inputfile, self.executable)) self.E = self.__readEnergy('%s/temp.Ecomponents' % (directory)) self.Forces = self.__readForces('%s/temp.force' % (directory)) # Delete the temp directory shell('rm -rf %s' % (directory))
def main(s): input = s.input output = s.output threads = s.threads extension = os.path.splitext(input[0])[1] if extension == '.fastq': # compress fastq as fastq.gz into workdir shell(f"pigz -5 -p {threads} {input} > {output}") elif extension == '.gz': show_output(f"Creating symlink for file {input}") # create shortcut to fastq.gz in workdir/fastq shell(f"ln -s {input} {output}") elif extension == '.bz2': show_output(f"file extension {extension} --> unzipping with bzcat") # uncompress fastq.b2 and recompress to fastq.gz in workdir/fastq shell(f"bzcat {input} | pigz -5 -p {threads} > {output}")
def setUp(self): shell("mkdir _test_dir_") shell("echo * > _test_dir_/_test_file_")
def stopTor(self): shell('service tor stop')
def restartTor(self, num=3): shell('service tor restart') sleep(1.5) self.updateIp(num)
def main(paths, paths_l, nw_path, lesses, files, Lakefile, config, ignore, quite, force, *args, **kwargs): if not quite: shell('clear') print bcolors.HEADER + __doc__.split('\n')[1] + bcolors.ENDC print "\n".join(__doc__.split('\n')[2:]) print "I'm presuming the files are at %s " % "".join( [(path + "/*.less") for path in paths_l]) print "and files at sub directory e.g. %s" % paths_l[0] + "/imports/*.less" print "are to be ignored." if Lakefile: print bcolors.OKBLUE + "Ha!Ha! that's a Lakefile I see " +\ "well those files shell be ignored!" + bcolors.ENDC print print print if not force: print bcolors.OKGREEN + "final list to compile is:" + bcolors.ENDC for old_file, nw_file in files: print " - \033[94m%s\033[0m -> \033[92m%s\033[0m" % (old_file, nw_file) confirm() s = signal.signal(signal.SIGINT, signal.SIG_IGN) if not quite: print print print i = 0 c = 0 compiled = [] ignored = [] total_ops = float(len(files) + 1) n = 0 for f in files: if f not in ignore: n += 1 if not quite: center("compiling %r \n" % f[1] + get_loader(n / total_ops)) less(*f) compiled.append(f) c += 1 else: n += 1 if not quite: center("ignoring %s \n" % f[0] + get_loader(n / total_ops)) ignored.append(f) i += 1 p = ["'" + path + "'" for path in paths_l] msg = "\n" msg += " \033[92m compiled %r files\033[0m from %s" % ( c, "".join(p) + (":" if c > 0 else "")) msg += "\n" for f in compiled: msg += " - \033[94m%s\033[0m -> \033[92m%s\033[0m" % f msg += "\n" msg += "\n" msg += " \033[94mignored %r files\033[0m from %s" %\ (i, "".join(p) + (":" if i > 0 else "")) for f in ignored: msg += " - \033[94m%s\033[0m" % f[0] msg += "\n" i += 1 if not quite: center("DONE!\n" + get_loader(1) + msg) signal.signal(signal.SIGINT, s) if not force: confirm(msg=False) raise KeyboardInterrupt
def run_eb(table, tumor_bam, output, pon_list, chrom, log, threads, EBparams, full_output, cleanpileup, csv2bed, pon2cols, pile2count, matrix2EBinput, makeponlist ): ''' master function to start eb_computation ''' # ############## LOAD DATA ############################### show_output(f"Computing EBscore for chrom", color='normal') # get the sceleton mutation file for that chromosome mut_df = pd.read_csv(table, sep='\t', index_col=False, header=None, names=['Chr', 'Start', 'End', 'Ref', 'Alt', 'somatic_status', 'TR1', 'TR1+', 'TR2', 'TR2+', 'NR1', 'NR1+', 'NR2', 'NR2+', 'somaticP', 'variantP']).query('Chr == @chrom').iloc[:, :5] mut_cols = list(mut_df.columns) # set base_name for intermediate files base_file = output[0].replace(".EB", "") # ############## PILEUP --> MATRIX FILE ################## # bed file can contain all chromosomes because chrom restriction comes with the -r parameter bed_file = f"{base_file}.bed" # create the bed file for mpileup shell(f"{csv2bed} < {table} > {bed_file}") # # if I want to restrict chromosome in file: # mut_chr_file = f"{base_file}.csv" # mut_df.to_csv(mut_chr_file, sep='\t', index=False) # # create the bed file for mpileup from the mutation file # shell(f"{csv2bed} < {mut_chr_file} > {bed_file}") # create the pon_list containing the tumor-bam as first file sample_list = f"{base_file}.pon" # makeponlist removes the sample itself from list if it is part of PoN shell(f"{makeponlist} {tumor_bam} {pon_list} {sample_list}") show_output(f"Piling up {chrom} of {tumor_bam} with Pon List.", color='normal') shell(f"cat {sample_list}") # do the pileup into the matrix file matrix_file = f"{base_file}.matrix" pileup_cmd = f"samtools mpileup -B -q {EBparams['MAPQ']} -Q {EBparams['Q']}" pileup_cmd += f" -l {bed_file} -r {chrom} -b {sample_list}" # cut -f $({pon2cols}< {sample_list}) creates a cut command only including the desired pipe_cmd = f"{pileup_cmd} | cut -f $({pon2cols} < {sample_list}) | {cleanpileup} | {pile2count} > {matrix_file}" # do the pileup to matrix_file show_command(pipe_cmd, multi=False) shell(pipe_cmd) # cleanup shell(f"rm {bed_file} {sample_list}") # check if matrix_file has input if not os.path.getsize(matrix_file): # create empty file open(output[0], 'a').close() show_output(f"Pileup for {chrom} of {tumor_bam} was empty! Created empty file {output[0]}", color='warning') else: show_output(f"Pileup matrix for chrom {chrom} of {tumor_bam} completed.", color='normal') # ################ MERGE INTO MUTFILE ###################### # change mutation positions for deletions in mutation file mut_df.loc[mut_df['Alt'] == "-", 'Start'] = mut_df['Start'] - 1 # read matrix file into df matrix_df = pd.read_csv(matrix_file, sep='\t', index_col=False) # merge mut_matrix = mut_df.merge(matrix_df, on=['Chr', 'Start'], how='inner') # reset deletion positions mut_matrix.loc[mut_matrix['Alt'] == "-", 'Start'] = mut_matrix['Start'] + 1 # ####### if using matrix2EBinput.mawk ####################### # write to file mutmatrix_file = f"{base_file}.mutmatrix" mut_matrix.to_csv(mutmatrix_file, sep='\t', index=False) # convert mutmatrix to direct EBinput EB_matrix_input_file = f"{base_file}.EB.matrix" shell(f"cat {mutmatrix_file} | {matrix2EBinput} > {EB_matrix_input_file}") # load in the EB.matrix file eb_matrix = pd.read_csv(EB_matrix_input_file, sep='\t') # multithreaded computation EB_df = compute_matrix2EB_multi(eb_matrix, EBparams['fitting_penalty'], threads) # add EBscore to columns mut_cols.append('EBscore') # get the pon_matrix containing the Pon coverages in Alt and Ref pon_matrix = get_pon_bases(eb_matrix) # transfer PoN-Ref and PoN-Alt to EB_df EB_df[['PoN-Ref', 'PoN-Alt']] = pon_matrix[['PoN-Ref', 'PoN-Alt']] mut_cols += ['PoN-Ref', 'PoN-Alt'] # ###### add the full output ########## if full_output: # condense base info print('full_output') base_cols = list("AaGgCcTtIiDd") col_name = "|".join(base_cols) # convert base coverage to str for ch in base_cols: # take the letter info from the mut_matrix which is not yet condensated # str.replace removes the tumor bases EB_df[ch] = mut_matrix[ch].map(str).str.replace(r'^[0-9]+\|', "") # condense base info into col "A|a|G|g|C|c|T|t|I|i|D|d" EB_df[col_name] = EB_df[base_cols].apply(lambda row: "-".join(row), axis=1) # add "A|a|G|g|C|c|T|t|I|i|D|d" to columns mut_cols.append(col_name) # rm unnecessary columns EB_df = EB_df[mut_cols] # ######### WRITE TO FILE ############################################## EB_file = output[0] EB_df.to_csv(EB_file, sep='\t', index=False) # cleanup shell(f"rm {matrix_file} {EB_matrix_input_file}") # {mutmatrix_file} show_output(f"Created EBscore for chrom {chrom} of {tumor_bam} and written to {output[0]}", color='success')
def __call__(self): args = ["%s=%s" for kv in self.items()] args = str.join(' ', args) cmd = "%s %s" % (self.Command, args) os.shell(cmd)
def run_eb_from_cache(table, tumor_bam, output, pon_list, chrom, log, threads, EBparams, full_output, cleanpileup, csv2bed, pile2count, matrix2EBinput, reorder_matrix ): def target_pileup_from_mut(mut_file, base_file, bam, chrom): ''' piles up the mutation list in the tumor bam ''' # bed file can contain all chromosomes because chrom restriction comes with the -r parameter bed_file = f"{base_file}.bed" # create the bed file for mpileup shell(f"{csv2bed} < {mut_file} > {bed_file}") # # if I want to restrict chromosome in file: # mut_chr_file = f"{base_file}.csv" # mut_df.to_csv(mut_chr_file, sep='\t', index=False) # # create the bed file for mpileup from the mutation file # shell(f"{csv2bed} < {mut_chr_file} > {bed_file}") # do the pileup into the matrix file matrix_file = f"{base_file}.matrix" pileup_cmd = f"samtools mpileup -B -q {EBparams['MAPQ']} -Q {EBparams['Q']}" pileup_cmd += f" -l {bed_file} -r {chrom} {tumor_bam}" pipe_cmd = f"{pileup_cmd} | cut -f 1,2,5 | {cleanpileup} | {pile2count} > {matrix_file}" show_output(f"Piling up tumor bam {tumor_bam}", color='normal') # do the pileup to matrix_file show_command(pipe_cmd, multi=False) shell(pipe_cmd) # cleanup shell(f"rm -f {bed_file}") show_output( f"Pileup matrix for chrom {chrom} of {tumor_bam} completed. Merging with cache file...", color='normal' ) return matrix_file # ############## LOAD DATA ############################### show_output(f"Computing EBscore for chrom {chrom} of {tumor_bam} using EBcache {AB_cache_file}", color='normal') # get the mutation file for the chromosome mut_df = pd.read_csv(mut_file, sep='\t', index_col=False, header=None, names=['Chr', 'Start', 'End', 'Ref', 'Alt', 'somatic_status', 'TR1', 'TR1+', 'TR2', 'TR2+', 'NR1', 'NR1+', 'NR2', 'NR2+', 'somaticP', 'variantP']).query('Chr == @chrom').iloc[:, :5] mut_cols = list(mut_df.columns) # check for empty df if mut_df.empty: EB_df = pd.DataFrame(columns=mut_cols) EB_df.to_csv(output[0], sep='\t', index=False) show_output(f"No mutations for {chrom} in mutation list! Writing empty file to {output[0]}", color='warning') else: # set base_name for intermediate files base_file = output[0].replace(".cachedEB", "") # ############## LOAD PILEUP MATRIX CACHE AND MERGE INTO MUT_DF ##### # change mutation positions for deletions in mutation file mut_df.loc[mut_df['Alt'] == "-", 'Start'] = mut_df['Start'] - 1 show_output(f"Loading compressed matrix cache file {matrix_cache_file}", color='normal') # load in the target matrix file as df cache_matrix_df = pd.read_csv(matrix_cache_file, sep='\t', index_col=False, compression='gzip') # merge mut_matrix = mut_df.merge(cache_matrix_df, on=['Chr', 'Start'], how='inner') # reset deletion positions mut_matrix.loc[mut_matrix['Alt'] == "-", 'Start'] = mut_matrix['Start'] + 1 show_output(f"Loaded and merged into mutation list", color='normal') # ############### CHECK IF SAMPLE IN PON #################### # if sample_inpon == 0, then sample is not in PoN # else, pon matrix has to be acquired from cache and used in EBscore sample_in_pon = get_sample_pos(pon_list, tumor_bam) # ########################################### CACHE FROM MATRIX ##################################### if sample_in_pon: show_output( f"Corresponding normal sample for {tumor_bam} has been found in PoNs! EBcache cannot be used!", color='warning' ) show_output(f"Falling back to cached matrix file..", color='normal') # EBcache cannot be used directly # ######### REMOVE SAMPLE BASES FROM MATRIX FILE # get the cached matrix file and reorder sample bases to first position to create valid mutmatrix # reorder_matrix takes position of sample in pon_list as argument # if position of tumor bam in pon == 1, everything is already fine mutmatrix_file = f"{base_file}.mutmatrix" if sample_in_pon > 1: prematrix_file = f"{base_file}.prematrix" mut_matrix.to_csv(prematrix_file, sep='\t', index=False) # row is 0-based --> sample_in_pon + 1 reduce_matrix_cmd = f"cat {prematrix_file} | {reorder_matrix} {sample_in_pon - 1} > {mutmatrix_file}" show_command(reduce_matrix_cmd, multi=False) shell(reduce_matrix_cmd) # cleanup shell(f"rm {prematrix_file}") else: # tumor sample already in the right position mut_matrix.to_csv(mutmatrix_file, sep='\t', index=False) show_output(f"Retrieving target data from cached matrix", color='normal') # # CONTINUE LIKE UNCACHED EBscore # convert mutmatrix to direct EBinput EB_matrix_input_file = f"{base_file}.EB.matrix" EBinput_cmd = f"cat {mutmatrix_file} | {matrix2EBinput} > {EB_matrix_input_file}" show_command(EBinput_cmd, multi=False) shell(EBinput_cmd) # load in the EB.matrix file eb_matrix = pd.read_csv(EB_matrix_input_file, sep='\t') print('Start computation file') # multithreaded computation # passing attempts to threads EB_df = compute_matrix2EB_multi(eb_matrix, EBparams['fitting_penalty'], threads) print('Computation finished') # get the pon_matrix containing the Pon coverages in Alt and Ref pon_matrix = get_pon_bases(eb_matrix) # ########################################### CACHE FROM ABcache ########################### else: # ############## TARGET PILEUP --> MATRIX FILE ################## tumor_matrix_file = target_pileup_from_mut(mut_file, base_file, tumor_bam, chrom) # check if matrix_file has input # if not os.path.getsize(tumor_matrix_file): # # create empty file # EB_df = mut_df # EB_df['EBscore'] = 0 # has_pileup = False # else: # has input # has_pileup = True # reloading the target pileup into pileup_df # use dtype to ensure str encoding of chromosome columns pileup_df = pd.read_csv(tumor_matrix_file, sep='\t', dtype={'Chr': str, 'Start': int}, index_col=False) show_output(f"Loading compressed AB cache file {AB_cache_file}", color='normal') cache_df = pd.read_csv(AB_cache_file, compression='gzip', sep='\t') pileAB_df = pileup_df.merge(cache_df, on=['Chr', 'Start']) # change coords for merge with start and merge into mut_df for Ref mut_df.loc[mut_df['Alt'] == "-", 'Start'] = mut_df['Start'] - 1 pileAB_df = mut_df.merge(pileAB_df, on=['Chr', 'Start']) pileAB_df.loc[pileAB_df['Alt'] == "-", 'Start'] = pileAB_df['Start'] + 1 # save for debugging # pileAB_file = f"{base_file}.pileAB" # pileAB_df.to_csv(pileAB_file, sep='\t', index=False) show_output( f"Pileup matrix for for chrom {chrom} of {tumor_bam} merged with AB matrix." + " Going on with EB computation...", color='normal' ) # ############## EBSCORE COMPUTATION ######## # multithreaded computation EB_df = compute_AB2EB_multi(pileAB_df, threads) # convert matrix file to EB_input for getting PoN-Ref and Pon-Alt mutmatrix_file = f"{base_file}.mutmatrix" mut_matrix.to_csv(mutmatrix_file, sep='\t', index=False) # do the conversion EB_matrix_input_file = f"{base_file}.EB.matrix" convert_cmd = (f"cat {mutmatrix_file} | {matrix2EBinput} > {EB_matrix_input_file}") show_command(convert_cmd) shell(convert_cmd) # load in the EB.matrix file eb_matrix = pd.read_csv(EB_matrix_input_file, sep='\t') # get the pon_matrix containing the Pon coverages in Alt and Ref # tumor sample is not in PoN --> no removal neccessary pon_matrix = get_pon_bases(eb_matrix, remove_sample=False) # cleanup shell(f"rm -f {tumor_matrix_file}") # add EBscore to columns mut_cols.append('EBscore') # transfer PoN-Ref and PoN-Alt from pon_matrix to EB_df EB_df[['PoN-Ref', 'PoN-Alt']] = pon_matrix[['PoN-Ref', 'PoN-Alt']] mut_cols += ['PoN-Ref', 'PoN-Alt'] # ###### add the full output ########## if config['EBFilter']['full_pon_output']: # condense base info base_cols = list("AaGgCcTtIiDd") col_name = "|".join(base_cols) # convert base coverage to str for ch in base_cols: # take the letter info from the mut_matrix which is not yet condensated # str.replace removes the tumor bases EB_df[ch] = mut_matrix[ch].map(str).str.replace(r'^[0-9]+\|', "") # condense base info into col "A|a|G|g|C|c|T|t|I|i|D|d" EB_df[col_name] = EB_df[base_cols].apply(lambda row: "-".join(row), axis=1) # add "A|a|G|g|C|c|T|t|I|i|D|d" to columns mut_cols.append(col_name) # rm unnecessary columns EB_df = EB_df[mut_cols] # ######### WRITE TO FILE ############################################## EB_df.to_csv(output[0], sep='\t', index=False) # cleanup shell(f"rm -f {EB_matrix_input_file}") show_output( f"Created EBscore for chrom {chrom} of {tumor_bam} using EBcache and written to {output[0]}", color='success' )
from os import system as shell from select_py import SQLServer q = "select top 10 userid, birthday from zooskods..odsusers" r = "biods3" o = "_sql_server_temp_file_.tsv" job = SQLServer(q,r,o) job.execute() data = open(o) for line in data: print line shell("rm {0}".format(o))
def clear(): shell("clear") # print "\n"*getTerminalSize()[1]
def tearDown(self): shell("rm -rd _test_dir_")
def less(*args, **kwargs): c = "lessc %s %s" % args if (not 'compress' in kwargs) or kwargs['compress']: c += " --yui-compress" shell(c)