def test_symlinksymlink(self):
f = 'somefile.txt'
s1 = join('sym1', f)
s2 = join('sym2', f)
os.mkdir('sym1')
os.mkdir('sym2')
touch(f)
# s1 link to f
os.symlink(join('..', f), s1)
# s2 link to s1
os.symlink(join('..', 'sym1', f), s2)
ok_(exists(s1))
ok_(exists(s2))
ok_(exists(f))
print compat.check_output('find ' + self.tempdir +
' -exec ls -ld {} \;',
shell=True)
# Try to rename a symlink -> symlink -> file
self._C(s2, 'somefile', 'some1')
print compat.check_output('find ' + self.tempdir +
' -exec ls -ld {} \;',
shell=True)
ok_(exists(join('sym2', 'some1.txt')), 'Did not rename sym2 file')
ok_(exists(join('sym1', 'some1.txt')), 'Did not rename sym1 file')
ok_(exists('some1.txt'), 'Did not rename actual file')
def make_project_repo( projpath ):
'''
Turn a project into a git repository. Basically just git init a project path
'''
gitdir = os.path.join( projpath, '.git' )
cmd = ['git', '--work-tree', projpath, '--git-dir', gitdir, 'init']
output = compat.check_output( cmd, stderr=subprocess.STDOUT )
logger.debug( output )
def make_project_repo(projpath):
'''
Turn a project into a git repository. Basically just git init a project path
'''
gitdir = os.path.join(projpath, '.git')
cmd = ['git', '--work-tree', projpath, '--git-dir', gitdir, 'init']
output = compat.check_output(cmd, stderr=subprocess.STDOUT)
logger.debug(output)
def check_git_repo( self, path ):
gitdir = join( path, '.git' )
cmd = 'git status status'.format( path, gitdir )
try:
output = compat.check_output( cmd, cwd=path, stderr=subprocess.STDOUT, shell=True )
return 'Not a git repository' not in output
except subprocess.CalledProcessError as e:
print e.output
return False
def rsync_run( rundir, ngsdata ):
# No trailing /
src = normpath( rundir )
# Dst will end in MiSeq since we removed the trailing / in src
dst = join( ngsdata, 'RawData', 'MiSeq' )
logger.info( 'The fastq read data is synced. Syncing the rest of the data' )
cmd = 'rsync -av --progress --size-only {0} {1}'.format( src, dst )
cmd = shlex.split( cmd )
logger.debug( compat.check_output( cmd ) )
def check_git_repo( self, path ):
gitdir = join( path, '.git' )
cmd = 'git status status'.format( path, gitdir )
try:
output = compat.check_output( cmd, cwd=path, stderr=subprocess.STDOUT, shell=True )
return 'Not a git repository' not in output
except subprocess.CalledProcessError as e:
print e.output
return False
def run_vcf_diff(self, vcffile, *args, **kwargs):
import subprocess
script = 'vcf_diff'
cmd = [script, vcffile]
print "Running {0}".format(' '.join(cmd))
try:
return compat.check_output(cmd, stderr=subprocess.STDOUT)
except subprocess.CalledProcessError as e:
return e.output
def run_vcf_diff( self, vcffile, *args, **kwargs ):
import subprocess
script = 'vcf_diff'
cmd = [script,vcffile]
print "Running {0}".format(' '.join(cmd))
try:
return compat.check_output( cmd, stderr=subprocess.STDOUT )
except subprocess.CalledProcessError as e:
return e.output
def get_numreads(self, bam):
try:
out = compat.check_output(
'samtools flagstat {0} | head -1 | cut -d\' \' -f 1'.format(
bam),
shell=True)
except subprocess.CalledProcessError as e:
print e.output
print e
raise e
nreads = int(out)
return nreads
def _rungraphsample( self, bamfile, **kwargs ):
script_path = 'graphsample'
args = ' '.join( ['-{0} {1}'.format(aname,aval) for aname, aval in kwargs.items()] )
cmd = script_path + ' {0} '.format(bamfile) + args
print "Running: {0}".format(cmd)
cmd = shlex.split( cmd )
try:
sout = check_output( cmd, stderr=STDOUT )
except CalledProcessError as e:
print e.output
assert False
return sout
def run_script(self, script):
print "Running {0}".format(script)
fh = open('/tmp/test_run.txt', 'w')
fh.write('Running {0}'.format(script))
try:
out = compat.check_output(script,
stderr=subprocess.STDOUT,
shell=True)
fh.write(out)
fh.close()
return (0, out)
except subprocess.CalledProcessError as e:
return (e.returncode, e.output)
def _rungraphsample(self, bamfile, **kwargs):
script_path = 'graphsample'
args = ' '.join(
['-{0} {1}'.format(aname, aval) for aname, aval in kwargs.items()])
cmd = script_path + ' {0} '.format(bamfile) + args
print "Running: {0}".format(cmd)
cmd = shlex.split(cmd)
try:
sout = check_output(cmd, stderr=STDOUT)
except CalledProcessError as e:
print e.output
assert False
return sout
def _run_cmd(self, jsons, outpath=None):
script_path = 'graph_mapunmap'
args = ' '.join(jsons)
if outpath is not None:
args += ' -o {0}'.format(outpath)
cmd = script_path + ' {0} '.format(args)
print "Running: {0}".format(cmd)
cmd = shlex.split(cmd)
try:
sout = check_output(cmd, stderr=STDOUT)
except CalledProcessError as e:
print e.output
assert False
return sout
def _run_cmd( self, jsons, outpath=None ):
script_path = 'graph_mapunmap'
args = ' '.join( jsons )
if outpath is not None:
args += ' -o {0}'.format(outpath)
cmd = script_path + ' {0} '.format(args)
print "Running: {0}".format(cmd)
cmd = shlex.split( cmd )
try:
sout = check_output( cmd, stderr=STDOUT )
except CalledProcessError as e:
print e.output
assert False
return sout
def test_symlinksymlink( self ):
f = 'somefile.txt'
s1 = join( 'sym1', f )
s2 = join( 'sym2', f )
os.mkdir( 'sym1' )
os.mkdir( 'sym2' )
touch(f)
# s1 link to f
os.symlink( join('..',f), s1 )
# s2 link to s1
os.symlink( join('..','sym1',f), s2 )
ok_(exists(s1))
ok_(exists(s2))
ok_(exists(f))
print compat.check_output('find '+self.tempdir+' -exec ls -ld {} \;',shell=True)
# Try to rename a symlink -> symlink -> file
self._C( s2, 'somefile', 'some1' )
print compat.check_output('find '+self.tempdir+' -exec ls -ld {} \;',shell=True)
ok_( exists( join('sym2','some1.txt') ), 'Did not rename sym2 file' )
ok_( exists( join('sym1','some1.txt') ), 'Did not rename sym1 file' )
ok_( exists( 'some1.txt' ), 'Did not rename actual file' )
def _run_runsample( self, readdir, reference, fileprefix, od=None, configfile=None,qsubargs=[] ):
script_path = 'runsample'
cmd = script_path + ' {0} {1} {2}'.format(readdir, reference, fileprefix)
if od is not None:
cmd += ' -od {0}'.format(od)
if configfile:
cmd += ' -c {0}'.format(configfile)
if qsubargs:
cmd += ' {0}'.format(' '.join(qsubargs))
print "Running: {0}".format(cmd)
cmd = shlex.split( cmd )
try:
sout = compat.check_output( cmd, stderr=subprocess.STDOUT )
except subprocess.CalledProcessError as e:
return (e.output,-1)
return sout,0
def _run_runsample( self, readdir, reference, fileprefix, od=None, configfile=None,qsubargs=[] ):
script_path = 'runsample'
cmd = script_path + ' {0} {1} {2}'.format(readdir, reference, fileprefix)
if od is not None:
cmd += ' -od {0}'.format(od)
if configfile:
cmd += ' -c {0}'.format(configfile)
if qsubargs:
cmd += ' {0}'.format(' '.join(qsubargs))
print "Running: {0}".format(cmd)
cmd = shlex.split( cmd )
try:
sout = compat.check_output( cmd, stderr=subprocess.STDOUT )
except subprocess.CalledProcessError as e:
return (e.output,-1)
return sout,0
def run_trimmomatic( *args, **kwargs ):
'''
Runs trimmomatic
@param arg0 - SE|PE -- Only SE supported at this time
@param arg1-arg6 - Input/Ouput files
@param arg7-argN - Tuples of (Trimmomatic Step,Options)
@param kwargs are any --options
run_trimmomatic( 'SE', 'input.fq', 'output.fq', ('LEADING','20), ('TRAILING','20'), trim_log='out.log' )
would result in
java -jar trimmomatic.jar input.fq output.fq LEADING:20 TRAILING:20 --trim_log out.log
'''
if args[0] == 'SE':
inputs = [args[1]]
outputs = [args[2]]
# Trimmomatic doesn't seem to be able to detect Sanger quality encoding
# so we will try to force it here to phred33
if data.is_sanger_readfile(args[1]):
kwargs['phred33'] = ''
steps = args[3:]
elif args[0] == 'PE':
inputs = list(args[1:3])
outputs = list(args[3:7])
steps = args[7:]
else:
raise ValueError( 'SE or PE need to be supplied' )
# Change all steps to strings of STEPNAME:VALUE
steps = [':'.join([str(x) for x in s]) for s in steps]
# Set all options
primer_info = kwargs.pop('primer_info', None)
options = shlex.split( ' '.join( ['-{0} {1}'.format(k,v) for k,v in kwargs.items()] ) )
cmd = ['trimmomatic', args[0]] + options + inputs + outputs + steps
if primer_info and primer_info[0]: # primer file was passed
cmd += [':'.join(['ILLUMINACLIP'] + primer_info)]
# Write stdout to output argument(should be fastq)
# Allow us to read stderr which should be stats from cutadapt
logger.debug( "Running {0}".format(' '.join(cmd)) )
try:
output = compat.check_output( cmd, stderr=subprocess.STDOUT )
return output
except subprocess.CalledProcessError as e:
logger.critical( "Trimmomatic error: {0}".format(e.output) )
raise e
def test_handles_sff( self ):
sff = glob( join( fixtures.THIS, 'fixtures', 'reads', '*.sff' ) )[0]
shutil.copy( sff, 'expected/reads' )
sff = join( 'expected', 'reads', basename( sff ) )
print "All files in expected directory:" + str( glob( 'expected/*' ) )
ff = self.fixture_files
argv = ['expected/reads', ff['REF']]
self._CM( argv )
r = 'bwa_mem.bam'
assert os.stat(r)
import subprocess
out = compat.check_output( ['samtools', 'view', '{0}'.format(r)] )
rochecount = out.count( 'IA52U1' )
print rochecount
eq_( 100, rochecount, 'Sff file reads did not make it into bam file' )
os.unlink( sff )
def test_handles_sff(self):
sff = glob(join(fixtures.THIS, 'fixtures', 'reads', '*.sff'))[0]
shutil.copy(sff, 'expected/reads')
sff = join('expected', 'reads', basename(sff))
print "All files in expected directory:" + str(glob('expected/*'))
ff = self.fixture_files
argv = ['expected/reads', ff['REF']]
self._CM(argv)
r = 'bwa_mem.bam'
assert os.stat(r)
import subprocess
out = compat.check_output(['samtools', 'view', '{0}'.format(r)])
rochecount = out.count('IA52U1')
print rochecount
eq_(100, rochecount, 'Sff file reads did not make it into bam file')
os.unlink(sff)
def print_ngs(self):
print compat.check_output('find . -ls', shell=True)
def get_readgroup(self, bam, rg):
cmd = ['samtools', 'view', '-r', rg, bam]
return compat.check_output(cmd)
def print_tempdir(self):
print compat.check_output('find ' + self.tempdir +
' -exec ls -ld {} \;',
shell=True)
def print_tempdir( self ):
print compat.check_output('find '+self.tempdir+' -exec ls -ld {} \;',shell=True)
