def align():
'''
Fastq read alignment with bowtie2
equivalent to: bowtie2 --threads "$THREADS" --local -x "$tmpGenome" -q -U "$INPUTFILE" | samtools view -@ "$THREADS" -Sb1 - | samtools sort -m 10G -@ "$THREADS" - "$OUTFILE"
'''
import json
import sh
env = samtools_bowtie_sh_env()
sh_align = sh(_env=env)
from sh_align import bowtie2
from sh_align import samtools
input_fastq = "input/test2.fastq"
output_bam = "output/test2.bam"
threads = 1
# genome = "/Users/steve/ref/iGenomes/Homo_sapiens/UCSC/hg19/Sequence/Bowtie2Index/genome"
# "/local/data/iGenomes/Homo_sapiens/UCSC/hg19/Sequence/WholeGenomeFasta/genome.fa"
genome = "/local/data/iGenomes/Homo_sapiens/UCSC/hg19/Sequence/Bowtie2Index/genome"
memory = "2G"
samtools_threads = "-@ {0}".format(threads)
mkdirs("output")
# print(bowtie2("--version"))
print(
samtools.sort(samtools.view(
bowtie2("-q",
"--local",
threads=threads,
x=genome,
U=input_fastq,
_env=env), "-Sb1", samtools_threads),
samtools_threads,
m=memory,
_out=output_bam))
def __init__(self):
super().__init__()
self._name = 'LCW'
self._problems = []
self._filename = ''
self._title = ''
self._lang = 'python3'
self._testcase = ''
_sh = sh(_out=sys.stdout, _err=sys.stderr)
self._cmd = _sh.leetcode
def test_shared_secial_args(self):
import sh
if IS_PY3:
from io import StringIO
from io import BytesIO as cStringIO
else:
from StringIO import StringIO
from cStringIO import StringIO as cStringIO
out1 = sh.ls(".")
out2 = StringIO()
sh_new = sh(_out=out2)
sh_new.ls(".")
self.assertEqual(out1, out2.getvalue())
out2.close()
def test_shared_secial_args(self):
import sh
if IS_PY3:
from io import StringIO
from io import BytesIO as cStringIO
else:
from StringIO import StringIO
from cStringIO import StringIO as cStringIO
out1 = sh.ls('.')
out2 = StringIO()
sh_new = sh(_out=out2)
sh_new.ls('.')
self.assertEqual(out1, out2.getvalue())
out2.close()
print 'Missing required Python 2.7 module: sh'
print 'URL: https://pypi.python.org/pypi/sh'
print 'To install run:'
print 'sudo pip2 install sh'
sys.exit(1)
HELP_MESSAGE='''
This script is intended for mass unpacking of directories.
'''.strip()
ARCH_CMDS = {
r'.*\.(tar\.gz|tgz)': lambda f,d: sh.tar.bake('xvzf')(f, C=d),
r'.*\.(tar.bz2|tbz2|tbz)': lambda f,d: sh.tar.bake('xvjf')(f, C=d),
r'.*\.(zip)': lambda f,d: sh.unzip.bake()(o=f, d=d),
r'.*\.(rar)': lambda f,d: sh.unrar.bake('x -o- -ad')(f),
r'.*\.(7z)': lambda f,d: sh('7z').bake('x -y')(f, o=d),
r'.*\.(zsh)': lambda f,d: sh.lha.bake('e')(f, w=d),
}
def unpack(filename, overwrite=False):
for regex, cmd in ARCH_CMDS.iteritems():
basename, ext = os.path.splitext(filename)
matches = re.findall(regex, filename)
if len(matches) == 0:
continue
if os.path.isdir(basename) and not overwrite:
LOG.info('SKIP - Exists: %s', basename)
return None
def on_any_event(self, event):
if event.src_path.endswith('json') or event.src_path.endswith('md'):
sh('./build-all.sh')
#!/usr/bin/env python
""" The sh module.
sh (previously pbs) is a full-fledged subprocess interface for Python 2.6 - 3.2
that allows you to call any program as if it were a function::
from sh import ifconfig
print ifconfig("eth0")
sh is not a collection of system commands implemented in Python.
Pythonic. Magical.
"""
print "Commands"
print "--------"
import sh
print sh.echo("hai there")
print sh.ls(a=True, l=True, h=True)
print "The 'from' syntax"
print "-----------------"
from sh import ls
import pathlib
import sys
import threading
import traceback
from datetime import datetime, timedelta
from queue import SimpleQueue
from signal import signal, SIGHUP, SIGINT, SIGTERM, setitimer, SIGALRM, ITIMER_REAL, SIGUSR1, SIGUSR2, strsignal
import click
import sh
from sh import podman
SERVICES_BASE_PATH = "/docker/services/"
# noinspection PyCallingNonCallable
shlog = sh(_out=sys.stdout, _err=sys.stderr)
sdnotify = sh.Command("systemd-notify")
class PodKeeper:
def __init__(self, network, log_driver, log_level, replace, remove, identifier):
self.podnet_args = ()
self.podnet_args += ("--network", network) if network else ()
self.podnet_args += ("--log-driver", log_driver) if log_driver else ()
self.podnet_args += ("--log-level", log_level) if log_level else ()
self.replace = replace
self.remove = remove
identifier_path = pathlib.PurePath(identifier)
if len(identifier_path.parts) != 1:
raise ValueError(f"identifier has path parts: {identifier_path}")
self.podhome = pathlib.Path(SERVICES_BASE_PATH) / identifier_path
#!/usr/bin/env python3
# INSTALL: pip3 install --user pytz sh
# CLI shell: lftp -u $FTP_LOGIN:$FTP_PASSWORD $FTP_HOST
import argparse, hashlib, os, sh, sys, tempfile
from datetime import datetime
from pytz import timezone
from ftplib import FTP
_ = sh(_err=sys.stderr.buffer, _out=sys.stdout.buffer)
def main(args):
timestamp = timezone('CET').localize(datetime.utcnow()).strftime('%Y%m%d')
if args.check_archives:
tmp_dir = tempfile.mkdtemp()
print(
'It is left to you to remove the temporary directory created: {}'.
format(tmp_dir))
with FTP(os.environ['FTP_HOST']) as ftp:
if args.debug:
ftp.set_debuglevel(2)
ftp.login(user=os.environ['FTP_LOGIN'],
passwd=os.environ['FTP_PASSWORD'])
md5_lines = []
ftp.retrlines('RETR chezsoi-{}.md5'.format(timestamp),
md5_lines.append)
for md5_line in md5_lines:
md5, filename = md5_line.split()
print(filename, md5)
self.original_dir = getcwd()
chdir(self.cwd)
return self
def __exit__(self, type, value, tb):
chdir(self.original_dir)
def pushd(dirname):
return PushdContext(dirname)
def add_env(*envs):
new_env = os.environ.copy()
for env in envs:
new_env.update(env)
return new_env
# @contextlib.contextmanager
# def pushd(path):
# starting_directory = os.getcwd()
# try:
# os.chdir(path)
# yield
# finally:
# os.chdir(starting_directory)
# Navio shell overriden call
nsh = sh(_out=sys.stdout, _err_to_out=True)
def test_short_option(self):
from sh import sh
s1 = sh(c="echo test").strip()
s2 = "test"
self.assertEqual(s1, s2)
def start(self):
"""starts a release runner instance"""
# it's a blocking operation.
log.info('starting release runner')
startup_path = self.configuration.get('release-runner', 'startup_path')
sh(startup_path)
def commands(*names):
argvs = []
for name in names:
script = os.path.join(TEST_BIN_DIR, '{0}.py'.format(name))
argvs.append([sys.executable, script])
return sh(*argvs)
#!/usr/bin/python
import os
import pathlib
import click
import sh
shfg = sh(_fg=True)
@click.command()
@click.option('--spec', help='spec file path')
@click.option('--sourcedir', help='sources directory', default='.') # noqa
@click.option('--define', 'definitions', help='define a macro',
multiple=True) # noqa
@click.option('--with',
'withs',
help='enable configuration option',
multiple=True) # noqa
@click.option('--without',
'withouts',
help='disable configuration option',
multiple=True) # noqa
@click.option('--repo', 'repos', help='add repo', multiple=True) # noqa
@click.option('--workspace',
'workspace',
help='working directory',
default='.') # noqa
def main(spec, sourcedir, definitions, withs, withouts, repos, workspace):
# INSTALL: pip3 install --user pytz sh
import argparse, hashlib, os, sh, sys, tempfile
from datetime import datetime
from pytz import timezone
from ftplib import FTP
_ = sh(_err=sys.stderr.buffer, _out=sys.stdout.buffer)
def main(args):
timestamp = timezone('CET').localize(datetime.utcnow()).strftime('%Y%m%d')
if args.check_archives:
tmp_dir = tempfile.mkdtemp()
print('It is left to you to remove the temporary directory created: {}'.format(tmp_dir))
with FTP(os.environ['FTP_HOST']) as ftp:
if args.debug:
ftp.set_debuglevel(2)
ftp.login(user=os.environ['FTP_LOGIN'], passwd=os.environ['FTP_PASSWORD'])
md5_lines = []
ftp.retrlines('RETR chezsoi-{}.md5'.format(timestamp), md5_lines.append)
for md5_line in md5_lines:
md5, filename = md5_line.split()
print(filename, md5, sizeof_fmt(ftp.size(filename)))
if not args.check_archives:
continue
file_full_path = os.path.join(tmp_dir, filename)
with open(file_full_path, 'wb') as new_file:
ftp.retrbinary('RETR {}'.format(filename), new_file.write)
with open(file_full_path, 'rb') as fetched_file:
computed_md5 = hashlib.md5(fetched_file.read()).hexdigest()
# Caveat: does not work under Windows
# Alt (but less pythonic/simple IMHO): gawel/chut, plumbum, sarge
# Special keyword args: https://amoffat.github.io/sh/special_arguments.html#special-arguments
- all commands are checked at 'from sh import' time so they are guaranteed to exist
- Always use `_err=sys.stderr` ou `_err_to_out=True` because default is to discard commands stderr
- `print()` is NEEDED to display command output (or you need to use `_out=sys.stdout`)
- `_piped='direct'` is useful to connect processes without consuming any memory
- `_iter` : creates a line generator => you can chain lazy functions taking a 'input_iterator' as input & output
- a command invocation return a `RunningCommand` object, on which you can wait for the text output (by calling `str()` on it)
or get a list of output lines (by calling `list()` on it)
import pip
pip.main(['install', '--proxy=' + PROXY, 'requests==2.7.0', 'retrying==1.3.3', 'sh==1.11'])
import sh, sys
sh = sh(_err=sys.stderr, _out=None) # setting default commands redirections
sh.bzcat(...)
if len(argv) > 1:
pipe = cat(argv[1], _iter=True, _err=stderr) # `pipe` is an input_lines_iterator
else:
pipe = cat(_in=stdin, _iter=True, _err=stderr)
(import [sh [cat grep wc]]) # in Hy, aka Python with Lisp syntax
(-> (cat "/usr/share/dict/words") (grep "-E" "^hy") (wc "-l"))
""""""""""""""""""""""""""
"" Libs & tools for DEVS !
""""""""""""""""""""""""""
pew > virtualenv # sandbox. To move an existing environment: virtualenv --relocatable $env
run("import __main__")
run("import " + ",".join(
set(
map(lambda s: s.split(".")[0],
filter(lambda s: not s.startswith("_"), sys.modules)))))
run("del __builtins__['input']")
run("del __builtins__['exit']")
run("del __builtins__['quit']")
run("def ___(s): ")
run(" l = shlex.split(s) ")
run(" c, a = l[0], l[1:] ")
run(" try: ")
run(" cmd = getattr(sh, c) ")
run(" args = map(sh.glob, a) ")
run(" return cmd(*args, _tty_out=False, _timeout=3.3) ")
run(" except sh.CommandNotFound: ")
run(" print('not found') ")
run(" except sh.TimeoutException: ")
run(" print('timeout') ")
run("")
return run
if __name__ == "__main__":
import doctest
doctest.testmod()
sh = instance()
print(sh("print(dir())"))
def inner(working_dir, *args, **kwargs):
sh2 = sh(_cwd=working_dir)
kwargs["sh"] = sh2
return f(working_dir, *args, **kwargs)
from datetime import datetime
import smtplib
import hashlib
import json
from email.mime.multipart import MIMEMultipart
from email.mime.text import MIMEText
from pathlib import Path
from zc.lockfile import LockError, LockFile
import pytz
import sh
from configs import START_HOUR, STOP_HOUR, DEBUG
from secrets import GMAIL_PASSWORD, GMAIL_USERNAME
my_sh = sh(_env={"PATH": "/usr/local/bin:/bin:/usr/bin"})
@contextmanager
def lock_instance():
try:
lock = LockFile('file_syncer.lock')
except LockError:
print('Can not run multiple instance of this script. Try it later! :)')
exit()
yield
lock.close()
def is_valid_interval():
if DEBUG:
def test_short_option(self):
from sh import sh
s1 = sh(c="echo test").strip()
s2 = "test"
self.assertEqual(s1, s2)
import csv
import sh
h = open('output.sql','a')
with open('all_emails1.csv') as csvfile:
reader = csv.reader(csvfile,quotechar='\t')
for row in reader:
sh('create_mail_user_SQL.sh', row[0], row[1], _out=h)
print(row[0] row[1])
import re
import sys
import time
from functools import wraps
from kobo import rpmlib
import json
import koji
import pytest
import requests
import requests_kerberos
import sh
import tempfile
import yaml
our_sh = sh(_out=sys.stdout, _err=sys.stderr, _tee=True)
from our_sh import Command, git, pushd # noqa
def retry(exception_to_check, tries=4, delay=3, backoff=2):
"""Retry calling the decorated function using an exponential backoff.
from: http://wiki.python.org/moin/PythonDecoratorLibrary#Retry
:param exception_to_check: the exception to check. may be a tuple of
exceptions to check
:type exception_to_check: Exception or tuple
:param tries: number of times to try (not retry) before giving up
:type tries: int
:param delay: initial delay between retries in seconds
:type delay: int
@contextlib.contextmanager
def safe_cd(path):
starting_directory = os.getcwd()
try:
os.chdir(path)
yield
finally:
os.chdir(starting_directory)
# Exr shell overriden methods
def print_out(line):
sys.stdout.write(line)
sys.stdout.write("\n")
sys.stdout.flush()
def print_err(line):
sys.stderr.write(line)
sys.stderr.write("\n")
sys.stderr.flush()
nsh = None
if os.environ.get('TRAVIS', 'false') == 'true':
nsh = sh(_out=sys.stdout, _err_to_out=True)
else:
nsh = sh(_out=sys.stdout, _err_to_out=True, _tty_in=True)
import click
import sh
import os
from subprocess import run, PIPE, STDOUT, Popen
import configbag
from executors.executor import ExecutorInterface
sh2 = sh(_iter=True, _err_to_out=True, _env=os.environ.copy())
class LocalExecutor(ExecutorInterface):
"""
Execute the low level operations on local host.
"""
def remove_microk8s_directory(self):
cmd = "rm -rf microk8s"
self._run_cmd(cmd)
def clone_microk8s_repo(self):
cmd = "git clone https://github.com/ubuntu/microk8s"
self._run_cmd(cmd)
def has_tests_for_track(self, track):
cmd = ("git ls-remote --exit-code "
"--heads https://github.com/ubuntu/microk8s.git refs/heads/{}".
format(track).split())
run(cmd, check=True, stdout=PIPE, stderr=STDOUT)
def checkout_branch(self, branch):
wd = os.getcwd()
return output.getvalue()
run("import __main__")
run("import " + ",".join(set(map(
lambda s: s.split(".")[0], filter(
lambda s: not s.startswith("_"), sys.modules)))))
run("del __builtins__['input']")
run("del __builtins__['exit']")
run("del __builtins__['quit']")
run("def ___(s): ")
run(" l = shlex.split(s) ")
run(" c, a = l[0], l[1:] ")
run(" try: ")
run(" cmd = getattr(sh, c) ")
run(" args = map(sh.glob, a) ")
run(" return cmd(*args, _tty_out=False, _timeout=3.3) ")
run(" except sh.CommandNotFound: ")
run(" print('not found') ")
run(" except sh.TimeoutException: ")
run(" print('timeout') ")
run("")
return run
if __name__ == "__main__":
import doctest
doctest.testmod()
sh = instance()
print(sh("print(dir())"))
