def align(command, output_dir, fastq_name):
"""
This method contains the actual alignment execution, to prevent duplicated code
:param command: The alignment command that will be run
:param fastq_name: The name of the file, without its extension,
this name is used to create a log file of the alignment
:param output_dir: The directory where to the output will be stored
"""
# Make a list out of the command, so it's safe to use with subprocess:
input_command = command.split()
# Set a directory for the error log:
log_file = f'{output_dir}/Results/alignment/{fastq_name.replace("_trimmed", "")}.log'
# Execute command with subprocess:
with open(log_file, 'a+') as err:
alignment = sub_run(input_command, capture_output=True, text=True, check=False)
# Print the error to the log file
print(alignment.stderr, file=err)
# Use samtools to create bam file:
# Write samtools command:
samtools_command = f'samtools view -Sbo {output_dir}/Preprocessing/aligned/' \
f'{fastq_name.replace("_trimmed", ".bam")} -'
# Run samtools command using the the output of the alignment as input:
sam = sub_run(samtools_command.split(), capture_output=True, text=True, input=alignment.stdout,
check=False)
def get_gaff_types(self, fftype='gaff', file=None):
"""
Convert the pdb file into a mol2 antechamber file and get the gaff atom types and bonds if we need them.
"""
# TODO Instead of file argument, just look for a mol2 file?
# call Antechamber to convert if we don't have the mol2 file
if file is None:
cwd = os.getcwd()
pdb_path = os.path.abspath(self.molecule.filename)
mol2_path = os.path.abspath(f'{self.molecule.name}.mol2')
# Do this in a temp directory as it produces a lot of files
with TemporaryDirectory() as temp:
os.chdir(temp)
copy(pdb_path, 'in.pdb')
# Call Antechamber
cmd = f'antechamber -i in.pdb -fi pdb -o out.mol2 -fo mol2 -s 2 -at {fftype} -c bcc'
if self.molecule.charge != 0:
cmd += f' -nc {self.molecule.charge}'
with open('ante_log.txt', 'w+') as log:
sub_run(cmd, shell=True, stdout=log, stderr=log)
# Ensure command worked
try:
# Copy the gaff mol2 and antechamber file back
copy('out.mol2', mol2_path)
copy('ante_log.txt', cwd)
except FileNotFoundError:
# If the molecule contains boron we expect this so use RDKit
print('using OpenBabel')
mol2_file = f'{self.molecule.name}.mol2'
Babel.convert('in.pdb', mol2_file)
copy(mol2_file, mol2_path)
os.chdir(cwd)
else:
mol2_path = file
# Check if the pdb file had connections if not we should remake the file
remake = True if self.molecule.bond_lengths is None else False
# Get the gaff atom types and bonds in case we don't have this info
self.molecule.read_mol2(mol2_path)
# Check if the molecule has bond lengths if not call the update method
if remake:
self.molecule.update()
# Now we need to rewrite the pdb file to have the conect terms
# Back up the old pdb file
os.rename(self.molecule.filename, 'backup.pdb')
# Rewrite the pdb file with the conect terms
self.molecule.write_pdb(input_type='input')
def perform_multiqc(output_dir):
"""
Function that performs the multiqc step
"""
print(colored('Running MultiQC...', 'blue'))
sub_run(
["multiqc", output_dir, "-o", f"{output_dir}/Results/multiQC/", '-f'],
check=False)
return 0
def fixed_point_timer(project, project_file):
analysis_filename = '../pyt/reaching_definitions_taint.py'
list_of_functions = ['arrow', 'join', 'fixpointmethod']
insert_profile(analysis_filename, list_of_functions)
if project:
sub_run([KERNPROF, '-l', '-v', PYT_PATH, '-pr', project, project_file])
else:
sub_run([KERNPROF, '-l', '-v', PYT_PATH, project_file])
remove_profile(analysis_filename)
def visualise():
try:
sub_run([SNAKEVIZ, STATS_FILENAME], stdout=PIPE)
except KeyboardInterrupt:
pass
except:
print('It seems that snakeviz is not installed.')
print(
'To install snakeviz see: https://jiffyclub.github.io/snakeviz/ .')
exit(0)
def _exec(self, *args: List[str]) -> CompletedProcess:
"""
Call the underlying CLI.
:param args: List arguments
:return: CompletedProcess
"""
if self.pipe:
return sub_run(args, stdout=PIPE, stderr=PIPE, check=True)
return sub_run(args)
def run(project, project_file, number_of_results):
if project:
sub_run([
PYTHON, '-m', 'cProfile', '-o', STATS_FILENAME, PYT_PATH, '-pr',
project, project_file
],
stdout=PIPE)
else:
sub_run([
PYTHON, '-m', 'cProfile', '-o', STATS_FILENAME, PYT_PATH,
project_file
],
stdout=PIPE)
prepare_results(number_of_results)
def fetch_assignment(log, assignment_name):
'''Fetch the most up-to-date version of the assignment specified'''
url_format = 'https://rpal.cs.cornell.edu/teaching/foundations/assignments/{}.sig'
# Check if assignment has already been fetched and is up to date
if os.path.isdir(assignment_name):
log.info('Found directory for {}'.format(assignment_name))
log.info('Checking version signature for {}'.format(assignment_name))
current_sig = sub_run(
'wget -qO- {}'.format(url_format.format(assignment_name)).split(),
stdout=PIPE,
universal_newlines=True,
check=True
).stdout
with open('.sigs/{}'.format(assignment_name, assignment_name), 'r') as sigfile:
old_sig = sigfile.readline()
if old_sig != current_sig:
log.info('Version of {} appears to be outdated. Downloading current version.'
.format(assignment_name))
remove_assignment(log, assignment_name)
os.mkdir(assignment_name)
download_assignment(log, assignment_name)
else:
log.info('Version of {} is up to date; proceeding'.format(assignment_name))
else:
log.info('Found no existing files for {}. Downloading current version.'
.format(assignment_name))
os.mkdir(assignment_name)
if not os.path.isdir('.sigs'):
os.mkdir('.sigs')
download_assignment(log, assignment_name)
def create_count_matrix(feature_counts, gtf_file, output_dir):
"""
The last step is the creation of the count matrix file.
This is done with the tool feature counts.
:param feature_counts: The path to the feature counts tool
:param gtf_file: Gene Transfer Format is a file format used to
hold information about gene structure
:param output_dir: the output_directory
"""
for bam_file in glob(f'{output_dir}/Preprocessing/markDuplicates/*_sorted.bam'):
command = f'{feature_counts} ' \
f'-a {gtf_file} ' \
f'-o {output_dir}/RawData/counts/geneCounts.txt' \
f' {bam_file}'
print(command)
command = command.split()
sub_run(command, check=True)
def run(cmd: Union[str, List[str]]) -> str:
from subprocess import run as sub_run, STDOUT, PIPE
if type(cmd) == str:
p = sub_run(cmd,
stdout=PIPE,
stderr=STDOUT,
universal_newlines=True,
shell=True)
elif type(cmd) == list:
p = sub_run(cmd, stdout=PIPE, stderr=STDOUT, universal_newlines=True)
output = p.stdout
if p.returncode != 0:
print(output)
return output
def download_assignment(log, assignment_name):
'''Downloads and extracts the provided files for the given assignment'''
url = 'https://rpal.cs.cornell.edu/teaching/foundations/assignments/' + assignment_name
base_cmd = 'wget ' + url
log.info('Downloading and extracting archive for {}'.format(assignment_name))
def unzip(f):
if not f.exception():
sub_run('tar xzvf {}.tar.gz -C {}'.format(assignment_name, assignment_name).split())
else:
raise f.exception()
with ThreadPoolExecutor(max_workers=1) as e:
download_future = e.submit(lambda: sub_run(('wget ' + url + '.tar.gz').split(), check=True))
download_future.add_done_callback(unzip)
log.info('Downloading version signature for {}'.format(assignment_name))
sub_run(('wget -O .sigs/{} ' + url + '.sig').format(assignment_name).split(), check=True)
wait([download_future])
log.info('Done downloading and extracting materials for {}'.format(assignment_name))
def install_gazebo(log):
'''Installs the latest version of Gazebo. Note: Only Ubuntu is officially supported.'''
log.info('Installing Gazebo')
# These commands are taken from the official Ubuntu install sequence
# at http://gazebosim.org/tutorials?tut=install_ubuntu&cat=install
install_commands = [
'sudo sh -c \'echo "deb http://packages.osrfoundation.org/gazebo/ubuntu-stable\
`lsb_release -cs` main" > /etc/apt/sources.list.d/gazebo-stable.list\'',
'wget http://packages.osrfoundation.org/gazebo.key -O - | sudo apt-key add -',
'sudo apt-get update',
'sudo apt-get install gazebo7 libgazebo7-dev'
]
for command in install_commands:
# Yes, running in a shell has security concerns. It should be OK for this instance,
# though - no non-hardcoded commands running, and it's easier than setting up the
# necessary pipes and such
sub_run(command, shell=True, check=True)
log.info('Gazebo has been successfully installed!')
def build_assignment(log, assignment_name):
# TODO: This seems unnecessary
def ensure_dir(dirname):
if not os.path.isdir(dirname):
os.mkdir(dirname)
curr_dir = os.getcwd()
log.info('Moving into assignment directory')
os.chdir(assignment_name)
log.info('Ensuring existence of build directory')
ensure_dir('build')
os.chdir('build')
if not os.path.isfile('Makefile'):
log.info('No Makefile; running CMake')
sub_run('cmake ..'.split(), check=True)
log.info('Running make')
sub_run('make', check=True)
os.chdir(curr_dir)
log.info('Building done. {} is ready to run'.format(assignment_name))
def __init__(self, output_dir):
"""
This sets global variables within the class
:param output_dir: This is the base directory were all
new directories will be made.
"""
directory = str(output_dir)
if directory.endswith('/'):
directory = directory.rstrip('/')
if not len(os.listdir(output_dir)) == 0:
print(
"Output directory is not empty, do you want to delete all files and continue?"
"(Y/N)")
yes_no = input().lower()[0]
if yes_no == 'y':
sub_run(["rm", "-rf", f"{directory}/*"])
print(f"directory {colored(directory, 'green')} emptied\n")
else:
sys.exit("output directory is not empty, please make sure"
"that the directory is empty, exiting now...")
self.output_dir = directory
def run(self):
check_dependencies()
if sys.platform == 'win32':
path_to_bat = os.path.join(here, 'src')
bat_name = "make.bat"
try:
batch_process = Popen(bat_name, cwd=path_to_bat, shell=True)
stdout, stderr = batch_process.communicate()
if stderr:
raise Exception(
'An error occur during Shadow compilation. Message: {}'
.format(stderr))
except OSError as err:
raise OSError(
'{} should be located on src. Current path: {}'.format(
bat_name, path_to_bat))
super().run()
else:
sub_run(['make', '-C', os.path.join(here, 'src'), 'clean'])
sub_run(['make', '-C', os.path.join(here, 'src'), 'lib'])
super().run()
def trimmer(self, file):
"""
Wrapper for trim_galore.
:param file: File to trim.
:return: Nothing.
"""
if self.trim is None:
print("Performing trimming using trimGalore.")
sub_run([
self.trim_galore, file, "-o",
f"{self.output_dir}/Preprocessing/trimmed"
],
check=False)
else:
print("Performing trimming using fastx trimmer.")
sep_trim = self.trim.split("-")
if len(sep_trim) == 1:
sub_run([
self.trim_galore, file, "--clip_R1", sep_trim[0], "-o",
f"{self.output_dir}/Preprocessing/trimmed"
],
check=False)
else:
sub_run([
self.trim_galore, file, "--three_prime_clip_R1",
sep_trim[1], "--clip_R1", sep_trim[0], "-o",
f"{self.output_dir}/Preprocessing/trimmed"
],
check=False)
def mtr_report(ip_address):
try:
mtr_result = sub_run([MTR_LOCATION, '--show-ips', '--json', '-c', '5', ip_address], capture_output=True)
except:
return False
if mtr_result.returncode != 0:
return False
mtr_json = loads(mtr_result.stdout.strip())
last_hop = mtr_json['report']['hubs'][-1]
return f"Hops: {last_hop['count']} - Loss: {last_hop['Loss%']}% - Last: {last_hop['Last']}ms - Avg: {last_hop['Avg']}ms - Best: {last_hop['Best']}ms - Worst: {last_hop['Wrst']}ms - StdDev: {last_hop['StDev']}ms"
def prmsl_download(year):
# ダウンロードするファイル
if year < 2014:
_f = ['anl_surf125.002_prmsl.{0}010100_{0}123118'.format(year)]
elif ((year % 4 == 0) & (year % 100 != 0)) | (year % 400 == 0):
_f = [
'anl_surf125.002_prmsl.{0}020100_{0}022918'.format(year),
'anl_surf125.002_prmsl.{0}030100_{0}033118'.format(year),
'anl_surf125.002_prmsl.{0}040100_{0}043018'.format(year),
'anl_surf125.002_prmsl.{0}050100_{0}053118'.format(year),
'anl_surf125.002_prmsl.{0}060100_{0}063018'.format(year)
]
else:
_f = [
'anl_surf125.002_prmsl.{0}020100_{0}022818'.format(year),
'anl_surf125.002_prmsl.{0}030100_{0}033118'.format(year),
'anl_surf125.002_prmsl.{0}040100_{0}043018'.format(year),
'anl_surf125.002_prmsl.{0}050100_{0}053118'.format(year),
'anl_surf125.002_prmsl.{0}060100_{0}063018'.format(year)
]
# 年単位でダウンロード
for i in _f:
if path.exists("original/" + i):
continue
else:
URL = 'http://rda.ucar.edu/data/ds628.0/anl_surf125/{0}/{1}'.format(
year, i)
stdout.write('Downloading {0} ...\n'.format(i))
stdout.flush()
infile = opener.open(URL)
outfile = open(i, 'wb')
outfile.write(infile.read())
outfile.close()
sub_run(['mv', i, "original/" + i])
del _f
def run(command, input_env, user, region):
"""
Runs the provided command in the cluster using IAM if necessary.
Use 'psql' to start an interactive session.
Use 'pg_dump' to get a dump of the database.\n
Examples.:\n
biomage rds run psql\n
biomage rds run pg_dump > dump.sql
"""
password = None
internal_port = 5432
if input_env == "development":
password = "******"
internal_port = 5431
else:
rds_client = boto3.client("rds")
remote_endpoint = get_rds_endpoint(input_env, rds_client,
ENDPOINT_TYPE)
print(f"Generating temporary token for {input_env}", file=sys.stderr)
password = rds_client.generate_db_auth_token(remote_endpoint,
internal_port, user,
region)
print("Token generated", file=sys.stderr)
result = sub_run(
f'PGPASSWORD="******" {command} \
--host=localhost \
--port={internal_port} \
--username={user} \
--dbname=aurora_db',
shell=True,
)
if result.returncode != 0:
print(
"\n"
"There was an error connecting to the db. "
'You may need to install psql, run "brew install postgresql"'
"\n\n"
'Or try running "biomage rds tunnel" before this command if connecting'
"to staging/production")
def antechamber_cmd(self):
"""Method to run Antechamber, parmchk2 and tleap."""
# file paths when moving in and out of temp locations
cwd = os.getcwd()
mol2 = os.path.abspath(f'{self.molecule.name}.mol2')
frcmod_file = os.path.abspath(f'{self.molecule.name}.frcmod')
prmtop_file = os.path.abspath(f'{self.molecule.name}.prmtop')
inpcrd_file = os.path.abspath(f'{self.molecule.name}.inpcrd')
ant_log = os.path.abspath('Antechamber.log')
# Call Antechamber
self.get_gaff_types(fftype=self.fftype)
# Work in temp directory due to the amount of files made by antechamber
with TemporaryDirectory() as temp:
os.chdir(temp)
copy(mol2, 'out.mol2')
# Run parmchk
with open('Antechamber.log', 'a') as log:
sub_run(
f"parmchk2 -i out.mol2 -f mol2 -o out.frcmod -s {self.fftype}",
shell=True,
stdout=log,
stderr=log)
# Ensure command worked
if not os.path.exists('out.frcmod'):
raise FileNotFoundError(
'out.frcmod not found parmchk2 failed!')
# Now get the files back from the temp folder
copy('out.mol2', mol2)
copy('out.frcmod', frcmod_file)
copy('Antechamber.log', ant_log)
# Now we need to run tleap to get the prmtop and inpcrd files
with TemporaryDirectory() as temp:
os.chdir(temp)
copy(mol2, 'in.mol2')
copy(frcmod_file, 'in.frcmod')
copy(ant_log, 'Antechamber.log')
# make tleap command file
with open('tleap_commands', 'w+') as tleap:
tleap.write("""source oldff/leaprc.ff99SB
source leaprc.gaff
LIG = loadmol2 in.mol2
check LIG
loadamberparams in.frcmod
saveamberparm LIG out.prmtop out.inpcrd
quit""")
# Now run tleap
with open('Antechamber.log', 'a') as log:
sub_run('tleap -f tleap_commands',
shell=True,
stdout=log,
stderr=log)
# Check results present
if not os.path.exists('out.prmtop') or not os.path.exists(
'out.inpcrd'):
raise FileNotFoundError(
'Neither out.prmtop nor out.inpcrd found; tleap failed!')
copy('Antechamber.log', ant_log)
copy('out.prmtop', prmtop_file)
copy('out.inpcrd', inpcrd_file)
os.chdir(cwd)
# Now give the file names to parametrisation method
self.prmtop = f'{self.molecule.name}.prmtop'
self.inpcrd = f'{self.molecule.name}.inpcrd'
def run(self):
check_dependencies()
sub_run(['autoreconf', '-i'])
sub_run(['sh', '{}/configure'.format(here)])
sub_run(['make', '-C', here])
super().run()
# Attempts to login to currently running keycloak instance
def login(self) -> None:
print('Logging into KeyCloak...')
cli_args = f'config credentials --server http://localhost:8080/auth --realm master --user {self._kc_user} --password {self._kc_pass}'
self.kcadm_cli_raise_error(cli_args)
print('...Successfully logged into KeyCloak!')
# Force kills a keycloak instance that's running anywhere on localhost
def kill(self) -> None:
print('Attempting to kill Keycloak...')
_, msg = self.jboss_cli('shutdown', 'connect\nshutdown')
print(msg)
print('...Done attempting to kill Keycloak!')
def __del__(self) -> None:
if self._running:
print(f'Keycloak Destructor: Premature destruction of running keycloak handle! Calling stop.')
self.stop()
print(f'Keycloak Destructor: Keycloak has (hopefully) been shutdown gracefully. Bye now!')
singleton = KeycloakHandle(KCBASE, KEYCLOAK_MODE, KEYCLOAK_USER, KEYCLOAK_PASSWORD, KC_EXECUTION_STRATEGY)
# When invoked as a script, simply start and login to keycloak!
if __name__ == '__main__':
singleton.kill()
singleton.start()
singleton.login()
sub_run(['sleep', 'infinity'])
def file_process(self, aligned_file):
"""
Runs all commands to create files.
:param aligned_file: File to use with picard.
:return: Nothing.
"""
picard = "lib/Picard-2.21.6/picard.jar"
aligned_files_sep = aligned_file.split("/")
current_file = aligned_files_sep[-1].replace(".bam", "")
# The follow code runs command line arguments to create the count files
# SortSam:
sort_sam = sub_run([
"java", "-jar", picard, "SortSam",
f"I={self.output_dir}/Preprocessing/aligned/{current_file}.bam",
f"O={self.output_dir}/Preprocessing/sortedBam/{current_file}.bam",
"SO=queryname"
],
capture_output=True,
text=True,
check=False)
# Write stdout to file:
write_preprocess_to_file(
sort_sam.stdout,
f'{self.output_dir}/Preprocessing/{current_file}-SortSam.txt')
# AddOrReplaceReadGroups:
add_or_replace = sub_run([
"java", "-jar", picard, "AddOrReplaceReadGroups",
f"INPUT={self.output_dir}/Preprocessing/sortedBam/"
f"{current_file}.bam", f"OUTPUT={self.output_dir}/Preprocessing/"
f"addOrReplace/{current_file}.bam", f"LB={current_file}",
f"PU={current_file}", f"SM={current_file}", "PL=illumina",
"CREATE_INDEX=true"
],
capture_output=True,
text=True,
check=False)
# Write stdout to file:
write_preprocess_to_file(
add_or_replace.stdout,
f'{self.output_dir}/Preprocessing/{current_file}-'
f'AddOrReplaceReadGroups.txt')
# FixMateInformation:
fix_mate_information = sub_run([
"java", "-jar", picard, "FixMateInformation",
f"INPUT={self.output_dir}/Preprocessing/"
f"addOrReplace/{current_file}.bam"
],
capture_output=True,
text=True,
check=False)
# Write stdout to file:
write_preprocess_to_file(
fix_mate_information.stdout,
f'{self.output_dir}/Preprocessing/{current_file}-'
f'FixMateInformation.txt')
# MergeSamFiles:
merge_sam_files = sub_run([
"java", "-jar", picard, "MergeSamFiles",
f"INPUT={self.output_dir}/Preprocessing/"
f"addOrReplace/{current_file}.bam",
f"OUTPUT={self.output_dir}/Preprocessing/"
f"mergeSam/{current_file}.bam", "CREATE_INDEX=true",
"USE_THREADING=true"
],
capture_output=True,
text=True,
check=False)
# Write stdout to file:
write_preprocess_to_file(
merge_sam_files.stdout,
f'{self.output_dir}/Preprocessing/{current_file}-'
f'MergeSamFiles.txt')
# MarkDuplicates
mark_duplicates = sub_run([
"java", "-jar", picard, "MarkDuplicates",
f"INPUT={self.output_dir}/Preprocessing/"
f"mergeSam/{current_file}.bam",
f"OUTPUT={self.output_dir}/Preprocessing/"
f"markDuplicates/{current_file}.bam", "CREATE_INDEX=true",
f"METRICS_FILE={self.output_dir}/Preprocessing/"
f"markDuplicates/{current_file}.metrics.log"
],
capture_output=True,
text=True,
check=False)
# Write stdout to file:
write_preprocess_to_file(
mark_duplicates.stdout,
f'{self.output_dir}/Preprocessing/{current_file}-'
f'MarkDuplicates.txt')
# SamtoolsSort:
samtools_sort = sub_run([
"samtools", "sort", "-n", f"{self.output_dir}/Preprocessing/"
f"markDuplicates/{current_file}.bam", "-o",
f"{self.output_dir}/Preprocessing/"
f"markDuplicates/{current_file}_sorted.bam"
],
capture_output=True,
text=True,
check=False)
# Write stdout to file:
write_preprocess_to_file(
samtools_sort.stdout,
f'{self.output_dir}/Preprocessing/{current_file}-SamtoolsSort.txt')
def unzip(f):
if not f.exception():
sub_run('tar xzvf {}.tar.gz -C {}'.format(assignment_name, assignment_name).split())
else:
raise f.exception()
#!/usr/bin/python3
from subprocess import PIPE, CalledProcessError, Popen, TimeoutExpired
from subprocess import run as sub_run
import logging
import os
import shutil
import shlex
import signal
from concurrent.futures import ThreadPoolExecutor, wait
try:
import click
except ImportError as e:
if 'No module named \'click\'' in e.msg:
sub_run('sudo pip3 install click'.split(), check=True)
import click
else:
raise ImportError('Something other than a missing module is wrong: {}'.format(e.msg))
logging.basicConfig(format='[%(asctime)s]\t%(levelname)s: %(message)s')
def install_gazebo(log):
'''Installs the latest version of Gazebo. Note: Only Ubuntu is officially supported.'''
log.info('Installing Gazebo')
# These commands are taken from the official Ubuntu install sequence
# at http://gazebosim.org/tutorials?tut=install_ubuntu&cat=install
install_commands = [
'sudo sh -c \'echo "deb http://packages.osrfoundation.org/gazebo/ubuntu-stable\
`lsb_release -cs` main" > /etc/apt/sources.list.d/gazebo-stable.list\'',
