def align(command, output_dir, fastq_name): """ This method contains the actual alignment execution, to prevent duplicated code :param command: The alignment command that will be run :param fastq_name: The name of the file, without its extension, this name is used to create a log file of the alignment :param output_dir: The directory where to the output will be stored """ # Make a list out of the command, so it's safe to use with subprocess: input_command = command.split() # Set a directory for the error log: log_file = f'{output_dir}/Results/alignment/{fastq_name.replace("_trimmed", "")}.log' # Execute command with subprocess: with open(log_file, 'a+') as err: alignment = sub_run(input_command, capture_output=True, text=True, check=False) # Print the error to the log file print(alignment.stderr, file=err) # Use samtools to create bam file: # Write samtools command: samtools_command = f'samtools view -Sbo {output_dir}/Preprocessing/aligned/' \ f'{fastq_name.replace("_trimmed", ".bam")} -' # Run samtools command using the the output of the alignment as input: sam = sub_run(samtools_command.split(), capture_output=True, text=True, input=alignment.stdout, check=False)
def get_gaff_types(self, fftype='gaff', file=None): """ Convert the pdb file into a mol2 antechamber file and get the gaff atom types and bonds if we need them. """ # TODO Instead of file argument, just look for a mol2 file? # call Antechamber to convert if we don't have the mol2 file if file is None: cwd = os.getcwd() pdb_path = os.path.abspath(self.molecule.filename) mol2_path = os.path.abspath(f'{self.molecule.name}.mol2') # Do this in a temp directory as it produces a lot of files with TemporaryDirectory() as temp: os.chdir(temp) copy(pdb_path, 'in.pdb') # Call Antechamber cmd = f'antechamber -i in.pdb -fi pdb -o out.mol2 -fo mol2 -s 2 -at {fftype} -c bcc' if self.molecule.charge != 0: cmd += f' -nc {self.molecule.charge}' with open('ante_log.txt', 'w+') as log: sub_run(cmd, shell=True, stdout=log, stderr=log) # Ensure command worked try: # Copy the gaff mol2 and antechamber file back copy('out.mol2', mol2_path) copy('ante_log.txt', cwd) except FileNotFoundError: # If the molecule contains boron we expect this so use RDKit print('using OpenBabel') mol2_file = f'{self.molecule.name}.mol2' Babel.convert('in.pdb', mol2_file) copy(mol2_file, mol2_path) os.chdir(cwd) else: mol2_path = file # Check if the pdb file had connections if not we should remake the file remake = True if self.molecule.bond_lengths is None else False # Get the gaff atom types and bonds in case we don't have this info self.molecule.read_mol2(mol2_path) # Check if the molecule has bond lengths if not call the update method if remake: self.molecule.update() # Now we need to rewrite the pdb file to have the conect terms # Back up the old pdb file os.rename(self.molecule.filename, 'backup.pdb') # Rewrite the pdb file with the conect terms self.molecule.write_pdb(input_type='input')
def perform_multiqc(output_dir): """ Function that performs the multiqc step """ print(colored('Running MultiQC...', 'blue')) sub_run( ["multiqc", output_dir, "-o", f"{output_dir}/Results/multiQC/", '-f'], check=False) return 0
def fixed_point_timer(project, project_file): analysis_filename = '../pyt/reaching_definitions_taint.py' list_of_functions = ['arrow', 'join', 'fixpointmethod'] insert_profile(analysis_filename, list_of_functions) if project: sub_run([KERNPROF, '-l', '-v', PYT_PATH, '-pr', project, project_file]) else: sub_run([KERNPROF, '-l', '-v', PYT_PATH, project_file]) remove_profile(analysis_filename)
def visualise(): try: sub_run([SNAKEVIZ, STATS_FILENAME], stdout=PIPE) except KeyboardInterrupt: pass except: print('It seems that snakeviz is not installed.') print( 'To install snakeviz see: https://jiffyclub.github.io/snakeviz/ .') exit(0)
def _exec(self, *args: List[str]) -> CompletedProcess: """ Call the underlying CLI. :param args: List arguments :return: CompletedProcess """ if self.pipe: return sub_run(args, stdout=PIPE, stderr=PIPE, check=True) return sub_run(args)
def run(project, project_file, number_of_results): if project: sub_run([ PYTHON, '-m', 'cProfile', '-o', STATS_FILENAME, PYT_PATH, '-pr', project, project_file ], stdout=PIPE) else: sub_run([ PYTHON, '-m', 'cProfile', '-o', STATS_FILENAME, PYT_PATH, project_file ], stdout=PIPE) prepare_results(number_of_results)
def fetch_assignment(log, assignment_name): '''Fetch the most up-to-date version of the assignment specified''' url_format = 'https://rpal.cs.cornell.edu/teaching/foundations/assignments/{}.sig' # Check if assignment has already been fetched and is up to date if os.path.isdir(assignment_name): log.info('Found directory for {}'.format(assignment_name)) log.info('Checking version signature for {}'.format(assignment_name)) current_sig = sub_run( 'wget -qO- {}'.format(url_format.format(assignment_name)).split(), stdout=PIPE, universal_newlines=True, check=True ).stdout with open('.sigs/{}'.format(assignment_name, assignment_name), 'r') as sigfile: old_sig = sigfile.readline() if old_sig != current_sig: log.info('Version of {} appears to be outdated. Downloading current version.' .format(assignment_name)) remove_assignment(log, assignment_name) os.mkdir(assignment_name) download_assignment(log, assignment_name) else: log.info('Version of {} is up to date; proceeding'.format(assignment_name)) else: log.info('Found no existing files for {}. Downloading current version.' .format(assignment_name)) os.mkdir(assignment_name) if not os.path.isdir('.sigs'): os.mkdir('.sigs') download_assignment(log, assignment_name)
def create_count_matrix(feature_counts, gtf_file, output_dir): """ The last step is the creation of the count matrix file. This is done with the tool feature counts. :param feature_counts: The path to the feature counts tool :param gtf_file: Gene Transfer Format is a file format used to hold information about gene structure :param output_dir: the output_directory """ for bam_file in glob(f'{output_dir}/Preprocessing/markDuplicates/*_sorted.bam'): command = f'{feature_counts} ' \ f'-a {gtf_file} ' \ f'-o {output_dir}/RawData/counts/geneCounts.txt' \ f' {bam_file}' print(command) command = command.split() sub_run(command, check=True)
def run(cmd: Union[str, List[str]]) -> str: from subprocess import run as sub_run, STDOUT, PIPE if type(cmd) == str: p = sub_run(cmd, stdout=PIPE, stderr=STDOUT, universal_newlines=True, shell=True) elif type(cmd) == list: p = sub_run(cmd, stdout=PIPE, stderr=STDOUT, universal_newlines=True) output = p.stdout if p.returncode != 0: print(output) return output
def download_assignment(log, assignment_name): '''Downloads and extracts the provided files for the given assignment''' url = 'https://rpal.cs.cornell.edu/teaching/foundations/assignments/' + assignment_name base_cmd = 'wget ' + url log.info('Downloading and extracting archive for {}'.format(assignment_name)) def unzip(f): if not f.exception(): sub_run('tar xzvf {}.tar.gz -C {}'.format(assignment_name, assignment_name).split()) else: raise f.exception() with ThreadPoolExecutor(max_workers=1) as e: download_future = e.submit(lambda: sub_run(('wget ' + url + '.tar.gz').split(), check=True)) download_future.add_done_callback(unzip) log.info('Downloading version signature for {}'.format(assignment_name)) sub_run(('wget -O .sigs/{} ' + url + '.sig').format(assignment_name).split(), check=True) wait([download_future]) log.info('Done downloading and extracting materials for {}'.format(assignment_name))
def install_gazebo(log): '''Installs the latest version of Gazebo. Note: Only Ubuntu is officially supported.''' log.info('Installing Gazebo') # These commands are taken from the official Ubuntu install sequence # at http://gazebosim.org/tutorials?tut=install_ubuntu&cat=install install_commands = [ 'sudo sh -c \'echo "deb http://packages.osrfoundation.org/gazebo/ubuntu-stable\ `lsb_release -cs` main" > /etc/apt/sources.list.d/gazebo-stable.list\'', 'wget http://packages.osrfoundation.org/gazebo.key -O - | sudo apt-key add -', 'sudo apt-get update', 'sudo apt-get install gazebo7 libgazebo7-dev' ] for command in install_commands: # Yes, running in a shell has security concerns. It should be OK for this instance, # though - no non-hardcoded commands running, and it's easier than setting up the # necessary pipes and such sub_run(command, shell=True, check=True) log.info('Gazebo has been successfully installed!')
def build_assignment(log, assignment_name): # TODO: This seems unnecessary def ensure_dir(dirname): if not os.path.isdir(dirname): os.mkdir(dirname) curr_dir = os.getcwd() log.info('Moving into assignment directory') os.chdir(assignment_name) log.info('Ensuring existence of build directory') ensure_dir('build') os.chdir('build') if not os.path.isfile('Makefile'): log.info('No Makefile; running CMake') sub_run('cmake ..'.split(), check=True) log.info('Running make') sub_run('make', check=True) os.chdir(curr_dir) log.info('Building done. {} is ready to run'.format(assignment_name))
def __init__(self, output_dir): """ This sets global variables within the class :param output_dir: This is the base directory were all new directories will be made. """ directory = str(output_dir) if directory.endswith('/'): directory = directory.rstrip('/') if not len(os.listdir(output_dir)) == 0: print( "Output directory is not empty, do you want to delete all files and continue?" "(Y/N)") yes_no = input().lower()[0] if yes_no == 'y': sub_run(["rm", "-rf", f"{directory}/*"]) print(f"directory {colored(directory, 'green')} emptied\n") else: sys.exit("output directory is not empty, please make sure" "that the directory is empty, exiting now...") self.output_dir = directory
def run(self): check_dependencies() if sys.platform == 'win32': path_to_bat = os.path.join(here, 'src') bat_name = "make.bat" try: batch_process = Popen(bat_name, cwd=path_to_bat, shell=True) stdout, stderr = batch_process.communicate() if stderr: raise Exception( 'An error occur during Shadow compilation. Message: {}' .format(stderr)) except OSError as err: raise OSError( '{} should be located on src. Current path: {}'.format( bat_name, path_to_bat)) super().run() else: sub_run(['make', '-C', os.path.join(here, 'src'), 'clean']) sub_run(['make', '-C', os.path.join(here, 'src'), 'lib']) super().run()
def trimmer(self, file): """ Wrapper for trim_galore. :param file: File to trim. :return: Nothing. """ if self.trim is None: print("Performing trimming using trimGalore.") sub_run([ self.trim_galore, file, "-o", f"{self.output_dir}/Preprocessing/trimmed" ], check=False) else: print("Performing trimming using fastx trimmer.") sep_trim = self.trim.split("-") if len(sep_trim) == 1: sub_run([ self.trim_galore, file, "--clip_R1", sep_trim[0], "-o", f"{self.output_dir}/Preprocessing/trimmed" ], check=False) else: sub_run([ self.trim_galore, file, "--three_prime_clip_R1", sep_trim[1], "--clip_R1", sep_trim[0], "-o", f"{self.output_dir}/Preprocessing/trimmed" ], check=False)
def mtr_report(ip_address): try: mtr_result = sub_run([MTR_LOCATION, '--show-ips', '--json', '-c', '5', ip_address], capture_output=True) except: return False if mtr_result.returncode != 0: return False mtr_json = loads(mtr_result.stdout.strip()) last_hop = mtr_json['report']['hubs'][-1] return f"Hops: {last_hop['count']} - Loss: {last_hop['Loss%']}% - Last: {last_hop['Last']}ms - Avg: {last_hop['Avg']}ms - Best: {last_hop['Best']}ms - Worst: {last_hop['Wrst']}ms - StdDev: {last_hop['StDev']}ms"
def prmsl_download(year): # ダウンロードするファイル if year < 2014: _f = ['anl_surf125.002_prmsl.{0}010100_{0}123118'.format(year)] elif ((year % 4 == 0) & (year % 100 != 0)) | (year % 400 == 0): _f = [ 'anl_surf125.002_prmsl.{0}020100_{0}022918'.format(year), 'anl_surf125.002_prmsl.{0}030100_{0}033118'.format(year), 'anl_surf125.002_prmsl.{0}040100_{0}043018'.format(year), 'anl_surf125.002_prmsl.{0}050100_{0}053118'.format(year), 'anl_surf125.002_prmsl.{0}060100_{0}063018'.format(year) ] else: _f = [ 'anl_surf125.002_prmsl.{0}020100_{0}022818'.format(year), 'anl_surf125.002_prmsl.{0}030100_{0}033118'.format(year), 'anl_surf125.002_prmsl.{0}040100_{0}043018'.format(year), 'anl_surf125.002_prmsl.{0}050100_{0}053118'.format(year), 'anl_surf125.002_prmsl.{0}060100_{0}063018'.format(year) ] # 年単位でダウンロード for i in _f: if path.exists("original/" + i): continue else: URL = 'http://rda.ucar.edu/data/ds628.0/anl_surf125/{0}/{1}'.format( year, i) stdout.write('Downloading {0} ...\n'.format(i)) stdout.flush() infile = opener.open(URL) outfile = open(i, 'wb') outfile.write(infile.read()) outfile.close() sub_run(['mv', i, "original/" + i]) del _f
def run(command, input_env, user, region): """ Runs the provided command in the cluster using IAM if necessary. Use 'psql' to start an interactive session. Use 'pg_dump' to get a dump of the database.\n Examples.:\n biomage rds run psql\n biomage rds run pg_dump > dump.sql """ password = None internal_port = 5432 if input_env == "development": password = "******" internal_port = 5431 else: rds_client = boto3.client("rds") remote_endpoint = get_rds_endpoint(input_env, rds_client, ENDPOINT_TYPE) print(f"Generating temporary token for {input_env}", file=sys.stderr) password = rds_client.generate_db_auth_token(remote_endpoint, internal_port, user, region) print("Token generated", file=sys.stderr) result = sub_run( f'PGPASSWORD="******" {command} \ --host=localhost \ --port={internal_port} \ --username={user} \ --dbname=aurora_db', shell=True, ) if result.returncode != 0: print( "\n" "There was an error connecting to the db. " 'You may need to install psql, run "brew install postgresql"' "\n\n" 'Or try running "biomage rds tunnel" before this command if connecting' "to staging/production")
def antechamber_cmd(self): """Method to run Antechamber, parmchk2 and tleap.""" # file paths when moving in and out of temp locations cwd = os.getcwd() mol2 = os.path.abspath(f'{self.molecule.name}.mol2') frcmod_file = os.path.abspath(f'{self.molecule.name}.frcmod') prmtop_file = os.path.abspath(f'{self.molecule.name}.prmtop') inpcrd_file = os.path.abspath(f'{self.molecule.name}.inpcrd') ant_log = os.path.abspath('Antechamber.log') # Call Antechamber self.get_gaff_types(fftype=self.fftype) # Work in temp directory due to the amount of files made by antechamber with TemporaryDirectory() as temp: os.chdir(temp) copy(mol2, 'out.mol2') # Run parmchk with open('Antechamber.log', 'a') as log: sub_run( f"parmchk2 -i out.mol2 -f mol2 -o out.frcmod -s {self.fftype}", shell=True, stdout=log, stderr=log) # Ensure command worked if not os.path.exists('out.frcmod'): raise FileNotFoundError( 'out.frcmod not found parmchk2 failed!') # Now get the files back from the temp folder copy('out.mol2', mol2) copy('out.frcmod', frcmod_file) copy('Antechamber.log', ant_log) # Now we need to run tleap to get the prmtop and inpcrd files with TemporaryDirectory() as temp: os.chdir(temp) copy(mol2, 'in.mol2') copy(frcmod_file, 'in.frcmod') copy(ant_log, 'Antechamber.log') # make tleap command file with open('tleap_commands', 'w+') as tleap: tleap.write("""source oldff/leaprc.ff99SB source leaprc.gaff LIG = loadmol2 in.mol2 check LIG loadamberparams in.frcmod saveamberparm LIG out.prmtop out.inpcrd quit""") # Now run tleap with open('Antechamber.log', 'a') as log: sub_run('tleap -f tleap_commands', shell=True, stdout=log, stderr=log) # Check results present if not os.path.exists('out.prmtop') or not os.path.exists( 'out.inpcrd'): raise FileNotFoundError( 'Neither out.prmtop nor out.inpcrd found; tleap failed!') copy('Antechamber.log', ant_log) copy('out.prmtop', prmtop_file) copy('out.inpcrd', inpcrd_file) os.chdir(cwd) # Now give the file names to parametrisation method self.prmtop = f'{self.molecule.name}.prmtop' self.inpcrd = f'{self.molecule.name}.inpcrd'
def run(self): check_dependencies() sub_run(['autoreconf', '-i']) sub_run(['sh', '{}/configure'.format(here)]) sub_run(['make', '-C', here]) super().run()
# Attempts to login to currently running keycloak instance def login(self) -> None: print('Logging into KeyCloak...') cli_args = f'config credentials --server http://localhost:8080/auth --realm master --user {self._kc_user} --password {self._kc_pass}' self.kcadm_cli_raise_error(cli_args) print('...Successfully logged into KeyCloak!') # Force kills a keycloak instance that's running anywhere on localhost def kill(self) -> None: print('Attempting to kill Keycloak...') _, msg = self.jboss_cli('shutdown', 'connect\nshutdown') print(msg) print('...Done attempting to kill Keycloak!') def __del__(self) -> None: if self._running: print(f'Keycloak Destructor: Premature destruction of running keycloak handle! Calling stop.') self.stop() print(f'Keycloak Destructor: Keycloak has (hopefully) been shutdown gracefully. Bye now!') singleton = KeycloakHandle(KCBASE, KEYCLOAK_MODE, KEYCLOAK_USER, KEYCLOAK_PASSWORD, KC_EXECUTION_STRATEGY) # When invoked as a script, simply start and login to keycloak! if __name__ == '__main__': singleton.kill() singleton.start() singleton.login() sub_run(['sleep', 'infinity'])
def file_process(self, aligned_file): """ Runs all commands to create files. :param aligned_file: File to use with picard. :return: Nothing. """ picard = "lib/Picard-2.21.6/picard.jar" aligned_files_sep = aligned_file.split("/") current_file = aligned_files_sep[-1].replace(".bam", "") # The follow code runs command line arguments to create the count files # SortSam: sort_sam = sub_run([ "java", "-jar", picard, "SortSam", f"I={self.output_dir}/Preprocessing/aligned/{current_file}.bam", f"O={self.output_dir}/Preprocessing/sortedBam/{current_file}.bam", "SO=queryname" ], capture_output=True, text=True, check=False) # Write stdout to file: write_preprocess_to_file( sort_sam.stdout, f'{self.output_dir}/Preprocessing/{current_file}-SortSam.txt') # AddOrReplaceReadGroups: add_or_replace = sub_run([ "java", "-jar", picard, "AddOrReplaceReadGroups", f"INPUT={self.output_dir}/Preprocessing/sortedBam/" f"{current_file}.bam", f"OUTPUT={self.output_dir}/Preprocessing/" f"addOrReplace/{current_file}.bam", f"LB={current_file}", f"PU={current_file}", f"SM={current_file}", "PL=illumina", "CREATE_INDEX=true" ], capture_output=True, text=True, check=False) # Write stdout to file: write_preprocess_to_file( add_or_replace.stdout, f'{self.output_dir}/Preprocessing/{current_file}-' f'AddOrReplaceReadGroups.txt') # FixMateInformation: fix_mate_information = sub_run([ "java", "-jar", picard, "FixMateInformation", f"INPUT={self.output_dir}/Preprocessing/" f"addOrReplace/{current_file}.bam" ], capture_output=True, text=True, check=False) # Write stdout to file: write_preprocess_to_file( fix_mate_information.stdout, f'{self.output_dir}/Preprocessing/{current_file}-' f'FixMateInformation.txt') # MergeSamFiles: merge_sam_files = sub_run([ "java", "-jar", picard, "MergeSamFiles", f"INPUT={self.output_dir}/Preprocessing/" f"addOrReplace/{current_file}.bam", f"OUTPUT={self.output_dir}/Preprocessing/" f"mergeSam/{current_file}.bam", "CREATE_INDEX=true", "USE_THREADING=true" ], capture_output=True, text=True, check=False) # Write stdout to file: write_preprocess_to_file( merge_sam_files.stdout, f'{self.output_dir}/Preprocessing/{current_file}-' f'MergeSamFiles.txt') # MarkDuplicates mark_duplicates = sub_run([ "java", "-jar", picard, "MarkDuplicates", f"INPUT={self.output_dir}/Preprocessing/" f"mergeSam/{current_file}.bam", f"OUTPUT={self.output_dir}/Preprocessing/" f"markDuplicates/{current_file}.bam", "CREATE_INDEX=true", f"METRICS_FILE={self.output_dir}/Preprocessing/" f"markDuplicates/{current_file}.metrics.log" ], capture_output=True, text=True, check=False) # Write stdout to file: write_preprocess_to_file( mark_duplicates.stdout, f'{self.output_dir}/Preprocessing/{current_file}-' f'MarkDuplicates.txt') # SamtoolsSort: samtools_sort = sub_run([ "samtools", "sort", "-n", f"{self.output_dir}/Preprocessing/" f"markDuplicates/{current_file}.bam", "-o", f"{self.output_dir}/Preprocessing/" f"markDuplicates/{current_file}_sorted.bam" ], capture_output=True, text=True, check=False) # Write stdout to file: write_preprocess_to_file( samtools_sort.stdout, f'{self.output_dir}/Preprocessing/{current_file}-SamtoolsSort.txt')
def unzip(f): if not f.exception(): sub_run('tar xzvf {}.tar.gz -C {}'.format(assignment_name, assignment_name).split()) else: raise f.exception()
#!/usr/bin/python3 from subprocess import PIPE, CalledProcessError, Popen, TimeoutExpired from subprocess import run as sub_run import logging import os import shutil import shlex import signal from concurrent.futures import ThreadPoolExecutor, wait try: import click except ImportError as e: if 'No module named \'click\'' in e.msg: sub_run('sudo pip3 install click'.split(), check=True) import click else: raise ImportError('Something other than a missing module is wrong: {}'.format(e.msg)) logging.basicConfig(format='[%(asctime)s]\t%(levelname)s: %(message)s') def install_gazebo(log): '''Installs the latest version of Gazebo. Note: Only Ubuntu is officially supported.''' log.info('Installing Gazebo') # These commands are taken from the official Ubuntu install sequence # at http://gazebosim.org/tutorials?tut=install_ubuntu&cat=install install_commands = [ 'sudo sh -c \'echo "deb http://packages.osrfoundation.org/gazebo/ubuntu-stable\ `lsb_release -cs` main" > /etc/apt/sources.list.d/gazebo-stable.list\'',