def test_git_push(git_dir):
with tempfile.TemporaryDirectory() as tempdir:
bare_dir = Path(tempdir)
run('git --bare init', cwd=bare_dir.as_posix())
run('git remote add origin {}'.format(bare_dir), cwd=git_dir.as_posix())
run('git push --set-upstream origin master', cwd=git_dir.as_posix())
# create tag, add file and do commit
run('git tag v1', cwd=git_dir.as_posix())
new_file = git_dir / 'new_file.txt'
with new_file.open('w') as f:
f.write('Some text')
run('git add {}'.format(new_file), cwd=git_dir.as_posix())
run('git commit -m "message"', cwd=git_dir.as_posix())
git.git_push(Repo(git_dir.as_posix()), 'v1')
p = capture_stdout('git ls-tree -r HEAD --name-only', cwd=str(bare_dir.as_posix()))
files = p.stdout.text.splitlines()
assert 'new_file.txt' in files
p = capture_stdout('git describe --abbrev=0 --tags', cwd=bare_dir.as_posix())
assert p.stdout.text == 'v1\n'
def gather_local_git_info():
remote_names = sarge.capture_stdout('git remote show').stdout.text.splitlines()
remote_urls = [
sarge.capture_stdout('git ls-remote --get-url {}'.format(name)).stdout.text.strip()
for name in remote_names
]
return dict(
repos=remote_urls,
ref=sarge.capture_stdout('git rev-parse --symbolic-full-name HEAD').stdout.text.strip(),
sha=sarge.capture_stdout('git rev-parse HEAD').stdout.text.strip()
)
def gather_local_git_info():
remote_names = sarge.capture_stdout(
'git remote show').stdout.text.splitlines()
remote_urls = [
sarge.capture_stdout(
'git ls-remote --get-url {}'.format(name)).stdout.text.strip()
for name in remote_names
]
return dict(
repos=remote_urls,
ref=sarge.capture_stdout(
'git rev-parse --symbolic-full-name HEAD').stdout.text.strip(),
sha=sarge.capture_stdout('git rev-parse HEAD').stdout.text.strip())
def test_init_repo(dir_with_file):
git.init_repo(dir_with_file)
p = capture_stdout('git ls-files', cwd=str(dir_with_file))
assert p.stdout.text == 'readme.txt\n'
p = capture_stdout('git config --get user.name', cwd=str(dir_with_file))
assert p.stdout.text == '\n'
p = capture_stdout('git config --get user.email', cwd=str(dir_with_file))
assert p.stdout.text == '\n'
p = capture_stdout('git rev-list --all --count', cwd=str(dir_with_file))
assert p.returncode != 0
assert p.stdout.text == ''
def test_init_repo_with_user(dir_with_file):
git.init_repo(dir_with_file, initial_commit=True,
user_name='John', user_email='*****@*****.**')
p = capture_stdout('git ls-files', cwd=str(dir_with_file))
assert p.stdout.text == 'readme.txt\n'
p = capture_stdout('git config --get user.name', cwd=str(dir_with_file))
assert p.stdout.text == 'John\n'
p = capture_stdout('git config --get user.email', cwd=str(dir_with_file))
assert p.stdout.text == '[email protected]\n'
p = capture_stdout('git rev-list --all --count', cwd=str(dir_with_file))
assert p.stdout.text == '1\n'
def crop(filename):
""" Call PDFCrop on a file, and replace it with its cropped version """
r = sarge.capture_stdout("pdfcrop %s" % (filename))
if r.returncode > 0:
print "PDFCrop error, return code %d\n\t%s" % (r.returncode, r.stdout.text)
else:
shutil.move(filename.replace(".pdf","-crop.pdf"), filename)
def test_feeder(self):
feeder = Feeder()
p = capture_stdout([sys.executable, 'echoer.py'], input=feeder,
async=True)
try:
lines = ('hello', 'goodbye')
gen = iter(lines)
# p.commands may not be set yet (separate thread)
while not p.commands or p.commands[0].returncode is None:
logger.debug('commands: %s', p.commands)
try:
data = next(gen)
except StopIteration:
break
feeder.feed(data + '\n')
if p.commands:
p.commands[0].poll()
time.sleep(0.05) # wait for child to return echo
finally:
# p.commands may not be set yet (separate thread)
if p.commands:
p.commands[0].terminate()
feeder.close()
self.assertEqual(p.stdout.text.splitlines(),
['hello hello', 'goodbye goodbye'])
def get_docker_version(self):
docker_version = None
docker_command = str(self.location) + " version --format '{{.Server.Version}}'"
output = capture_stdout(docker_command)
for line in io.TextIOWrapper(output.stdout):
docker_version = line
return docker_version
def test_feeder(self):
feeder = Feeder()
p = capture_stdout([sys.executable, 'echoer.py'],
input=feeder,
async=True)
try:
lines = ('hello', 'goodbye')
gen = iter(lines)
# p.commands may not be set yet (separate thread)
while not p.commands or p.commands[0].returncode is None:
logger.debug('commands: %s', p.commands)
try:
data = next(gen)
except StopIteration:
break
feeder.feed(data + '\n')
if p.commands:
p.commands[0].poll()
time.sleep(0.05) # wait for child to return echo
finally:
# p.commands may not be set yet (separate thread)
if p.commands:
p.commands[0].terminate()
feeder.close()
self.assertEqual(p.stdout.text.splitlines(),
['hello hello', 'goodbye goodbye'])
def list_windows():
"""Returns a list of all windows.
Returns:
List of Window instances.
"""
stdout = capture_stdout('wmctrl -lG').stdout.text
stdout_lines = stdout.split('\n')
result = []
for line in stdout_lines:
if line:
columns = line.split()
result.append(
Window(id=columns[0],
desktop=int(columns[1]),
geometry=Geometry(x=int(columns[2]),
y=int(columns[3]),
w=int(columns[4]),
h=int(columns[5])),
name=' '.join(columns[7:])))
return result
def async_parse_tracert(target, on_new_node):
p = capture_stdout('tracert {}'.format(target), async_=True)
num_timeouts = 0
while p.returncodes[0] is None:
line = p.stdout.readline()
line = line.decode('utf-8')
elems = line.split()
elems = [value for value in elems if value != 'ms']
if (len(elems) == 5 or len(elems) == 6) and elems[0].isdigit():
new_node = {
'delays':
(to_int(elems[1]), to_int(elems[2]), to_int(elems[3]))
}
if len(elems) == 6:
new_node['name'] = elems[4]
new_node['addr'] = elems[5].strip('[').strip(']')
else:
new_node['addr'] = elems[4]
on_new_node(new_node)
num_timeouts = 0
elif len(elems) > 0 and elems[-1] == 'out.':
num_timeouts += 1
# on_new_node(None)
if num_timeouts >= 1:
return -1
p.commands[0].poll()
time.sleep(0.05)
return p.returncodes[0]
def get_docker_version(self):
docker_version = None
docker_command = str(
self.location) + " version --format '{{.Server.Version}}'"
output = capture_stdout(docker_command)
for line in io.TextIOWrapper(output.stdout):
docker_version = line
return docker_version
def test_run_found_command(self):
with open('hello.py', 'w') as f:
f.write('print("Hello, world!")')
cmd = find_command('hello')
if not cmd:
raise unittest.SkipTest('.py not in PATHEXT or not registered')
p = capture_stdout('hello')
self.assertEqual(p.stdout.text.rstrip(), 'Hello, world!')
def test_run_found_command(self):
with open('hello.py', 'w') as f:
f.write('print("Hello, world!")')
cmd = find_command('hello')
if not cmd:
raise unittest.SkipTest('.py not in PATHEXT or not registered')
p = capture_stdout('hello')
self.assertEqual(p.stdout.text.rstrip(), 'Hello, world!')
def fummy(threshold, branches, path):
cwd(path)
all_branches = capture_stdout('git branch')
# remove spaces and any blank spaces
temp = all_branches.stdout.text.replace('*', '').replace(' ',
'').split('\n')
for branch in temp:
if branch and branch not in branches:
click.echo('Processing branch: {}'.format(branch))
p = capture_stdout(
'git show {} --format="%cI" --no-patch'.format(branch))
diff_days = get_time_difference(p.stdout.text)
if diff_days > threshold:
click.echo('Deleting {}'.format(branch))
p = capture_stdout('git branch -D {}'.format(branch))
click.echo(p.stdout.text)
def test_run_found_command(self):
with open("hello.py", "w") as f:
f.write('print("Hello, world!")')
cmd = find_command("hello")
if not cmd:
raise unittest.SkipTest(".py not in PATHEXT or not registered")
p = capture_stdout("hello")
self.assertEqual(p.stdout.text.rstrip(), "Hello, world!")
def extract_text(self,
input_path,
output_path='',
password=None,
encoding=None,
html=False,
sort=False,
ignore_beads=False,
start_page=1,
end_page=None):
"""
Extract all text from PDF file.
Parameters
----------
input_path : str
Input PDF file.
output_path : str
Output text file. If not specified, the extracted text is returned.
password : str
PDF password.
encoding : str
Text file encoding.
html : bool
If True, extract as HTML.
sort : bool
If True, sort text before returning it.
ignore_beads : bool
If True, ignore separation by beads.
start_page : int
First page to extract (starting with 1).
end_page : int
Last page to extract (starting with 1).
Returns
-------
text : str
Extracted text. If `output_path` is not specified, nothing is returned.
"""
options = (' -password {password}'.format(password=password) if password else '') +\
(' -encoding {encoding}'.format(encoding=encoding) if encoding else '') +\
(' -html' if html else '') +\
(' -sort' if sort else '') +\
(' -ignoreBeads' if ignore_beads else '') +\
(' -startPage {start_page}'.format(start_page=start_page) if start_page else '') +\
(' -endPage {end_page}'.format(end_page=end_page) if end_page else '')
if not output_path:
options += ' -console'
cmd = '"{java_path}" -jar "{pdfbox_path}" ExtractText {options} "{input_path}" "{output_path}"'.format(
java_path=self.java_path,
pdfbox_path=self.pdfbox_path,
options=options,
input_path=input_path,
output_path=output_path)
p = sarge.capture_stdout(cmd)
if not output_path:
return p.stdout.text
def fummy(threshold, branches, path):
cwd(path)
all_branches = capture_stdout('git branch')
# remove spaces and any blank spaces
temp = all_branches.stdout.text.replace(
'*', '').replace(' ', '').split('\n')
for branch in temp:
if branch and branch not in branches:
click.echo('Processing branch: {}'.format(branch))
p = capture_stdout(
'git show {} --format="%cI" --no-patch'.format(branch))
diff_days = get_time_difference(p.stdout.text)
if diff_days > threshold:
click.echo('Deleting {}'.format(branch))
p = capture_stdout(
'git branch -D {}'.format(branch))
click.echo(p.stdout.text)
def git_toplevel():
"""
Grab absolute path of repo using git command.
:returns: git.stdout.text.rstrip()
:rtype: str.
"""
git = capture_stdout("git rev-parse --show-toplevel")
return git.stdout.text.rstrip()
def get_image_info(self, imageID):
docker_command = str(self.location) + ' images'
output = capture_stdout(docker_command)
for line in TextIOWrapper(output.stdout):
if imageID in repr(line):
repository = line.split()[0]
tag = line.split()[1]
if imageID is None:
raise DockerImageError
return repository, tag
def get_imageID(self, image):
# default_container = "phusion/baseimage"
imageID = None
docker_command = str(self.location) + ' images'
output = capture_stdout(docker_command)
for line in io.TextIOWrapper(output.stdout):
if image in repr(line):
imageID = line.split()[2]
if imageID is None:
raise DockerImageError('No Image')
return imageID
def get_containerImage(self, containerID):
imageName = None
docker_command = str(self.location) + ' ps -a'
#print docker_command
output = capture_stdout(docker_command)
for line in io.TextIOWrapper(output.stdout):
if containerID in repr(line):
imageName = line.split()[1]
if imageName is None:
raise DockerImageError('No Image for Container')
return imageName
def get_containerImage(self, containerID):
imageName = None
docker_command = str(self.location) + ' ps -a'
#print docker_command
output = capture_stdout(docker_command)
for line in io.TextIOWrapper(output.stdout):
if containerID in repr(line):
imageName = line.split()[1]
if imageName is None:
raise DockerImageError('No Image for Container')
return imageName
def get_containerID(self, image):
containerID = None
docker_command = str(self.location) + ' ps -a'
#print docker_command
output = capture_stdout(docker_command)
for line in io.TextIOWrapper(output.stdout):
if image in repr(line):
containerID = line.split()[0]
if containerID is None:
raise DockerImageError('No Container')
return containerID
def extract_data(database: str, query_type: str,
phenotype: int=1) -> pd.DataFrame:
command = GEMINI_COMMAND.format(QUERIES[query_type] + " in_1kg=0 ",
phenotype, database)
print("Running", command)
command = sarge.capture_stdout(command)
series = pd.read_table(command.stdout, squeeze=True)
return set(series.values)
def get_image_info(self):
docker_command = str(self.location) + ' images'
output = capture_stdout(docker_command)
if self.imageID is not None:
for line in io.TextIOWrapper(output.stdout):
if self.imageID in repr(line):
repository = line.split()[0]
tag = line.split()[1]
return repository, tag
else:
raise DockerImageError
def get_imageID(self):
default_container = "phusion/baseimage"
imageID = None
docker_command = str(self.location) + ' images'
output = capture_stdout(docker_command)
for line in TextIOWrapper(output.stdout):
if default_container in repr(line):
imageID = line.split()[2]
if imageID is None:
raise DockerImageError
return imageID
def get_containerID(self, image):
containerID = None
docker_command = str(self.location) + ' ps -a'
#print docker_command
output = capture_stdout(docker_command)
for line in io.TextIOWrapper(output.stdout):
if image in repr(line):
containerID = line.split()[0]
if containerID is None:
raise DockerImageError('No Container')
return containerID
def _auth_token():
"""Call the heroku auth:token api.
This function is inside settings, not Heroku, as it's part of the
settings process, and is only called once, on startup.
"""
r = sarge.capture_stdout('heroku auth:token')
if r.returncode > 0:
# git doesn't play nicely so r.stderr is None even though it failed
raise Exception(u"Error getting heroku auth_token")
return r.stdout.text.strip()
def extract_data(database: str,
query_type: str,
phenotype: int = 1) -> pd.DataFrame:
command = GEMINI_COMMAND.format(QUERIES[query_type] + " in_1kg=0 ",
phenotype, database)
print("Running", command)
command = sarge.capture_stdout(command)
series = pd.read_table(command.stdout, squeeze=True)
return set(series.values)
def main():
parser = optparse.OptionParser()
parser.add_option('-n', '--no-dots', dest='dots', default=True,
action='store_false', help='Show dots for progress')
options, args = parser.parse_args()
p = capture_stdout('python /home/abhishek/Downloads/sarge/lister.py -d 0.1 -c 100', async=True)
t = threading.Thread(target=progress, args=(p.stdout, options))
t.start()
while(p.returncodes[0] is None):
# We could do other useful work here. If we have no useful
# work to do here, we can call readline() and process it
# directly in this loop, instead of creating a thread to do it in.
p.commands[0].poll()
time.sleep(0.05)
t.join()
def pants_list(args):
"""
Non-interactive output of pants list parsed to only show bare targets without paths.
:param args: arguments to pass to sarge.
:type args: str
:returns: _pants
"""
os.chdir(git_toplevel())
_pants_list = capture_stdout("./pants %s" % args)
for target in _pants_list.stdout.text.split("\n"):
if ":" in target:
bare_target = target.split(":", 1)[-1]
print(":%s" % bare_target)
def find_active_window(windows):
"""Returns the currently active (focused) window.
Args:
windows: A list containing all Windows instances.
Returns:
An instance of Window representing the currently active window or None if it can't found.
"""
stdout = capture_stdout('xdotool getactivewindow').stdout.text
windows_id = '0x' + hex(int(stdout))[2:].rjust(8, '0')
for w in windows:
if w.id == windows_id:
return w
def get_auth_token():
"""Use the heroku auth:token command to fetch the user's API token.
By using the heroku CLI itself we remove the requirement to set up
an environment variable for the token.
"""
r = sarge.capture_stdout('heroku auth:token')
if r.returncode == 0:
return r.stdout.text
else:
raise HerokuError(
"Unable to retrieve user auth token from Heroku, "
"please ensure that you are logged in using `heroku login`."
)
def ping(host, count=5, deadline=5):
"""
ICMP ECHO ping a host (with ``ping``)
Args:
host (str): hostname or IP
count (int): number of packets to send (``-c``)
deadline (int): timeout in seconds (``-w``)
Returns:
:py:class:`PingSummary`: parsed ping summary
"""
cmd = sarge.shell_format("""ping -q -c {0} -w {1} -i 0.2 {2}""", count,
deadline, host)
p = sarge.capture_stdout(cmd)
return _parse_ping_output(TextIOWrapper(p.stdout))
def ping(host, count=5, deadline=5):
"""
ICMP ECHO ping a host (with ``ping``)
Args:
host (str): hostname or IP
count (int): number of packets to send (``-c``)
deadline (int): timeout in seconds (``-w``)
Returns:
:py:class:`PingSummary`: parsed ping summary
"""
cmd = sarge.shell_format(
"""ping -q -c {0} -w {1} -i 0.2 {2}""",
count,
deadline,
host)
p = sarge.capture_stdout(cmd)
return _parse_ping_output(TextIOWrapper(p.stdout))
def calculate_coverage(sample, bed_data):
sample_name = extract_samplename(str(sample))
coverage = sarge.shell_format(COMMAND, sample, bed_data)
coverage = sarge.capture_stdout(coverage)
coverage_raw = pd.read_table(
coverage.stdout,
header=None,
names=["chr", "start", "end", "name", "coverage"])
# FIXME: No idea why this happens (misplaced index) but we must fix it
if pd.Series(["+", "-"]).isin(coverage_raw["name"]).any():
coverage_raw.reset_index(inplace=True)
coverage_raw.columns = [
"chr", "start", "end", "name", "score", "strand", "coverage"
]
# clean up bed results (Split at "_", get first result)
# FIXME: Be more flexible
coverage_raw["name"] = coverage_raw["name"].apply(
lambda x: x.split("_")[0].strip())
coverage_table = coverage_raw[["coverage", "name"
]].groupby("name").agg(["mean", "std"])
mean_coverage = coverage_table["coverage", "mean"]
mean_std = coverage_table["coverage", "std"]
coverage_table.columns.set_names(["", ""], inplace=True)
coverage_table.index.set_names(["Gene"], inplace=True)
coverage_table["coverage", "mean"] = mean_coverage.round(3)
coverage_table["coverage", "std"] = mean_std.round(3)
global_coverage_mean = coverage_raw["coverage"].mean()
global_coverage_std = coverage_raw["coverage"].std()
return dict(name=sample_name,
cov=coverage_table,
mean=global_coverage_mean,
std=global_coverage_std)
def get_screen_size():
"""Returns the size of the screen.
With multiple monitors it returns the size of the combined surfaces of the monitors.
Returns:
An instance of Gemoetry with x=0, y=0, w=Width of screen, h=Height of screen
"""
stdout = capture_stdout('wmctrl -d').stdout.text
stdout_lines = stdout.split('\n')
# Example string: '0 - DG: 1366x768 VP: N/A WA: 0,31 1366x737 1'
size = stdout_lines[0].split()[3]
# Example string: '1366x768'
sizes = size.split('x')
width = int(sizes[0])
height = int(sizes[1])
return Geometry(0, 0, width, height)
def list_desktops():
"""Returns a list of all desktops.
Returns:
List of Dekstop instances.
"""
stdout = capture_stdout('wmctrl -d').stdout.text
stdout_lines = stdout.split('\n')
result = []
for line in stdout_lines:
if line:
columns = line.split()
result.append(
Desktop(id=int(columns[0]),
active=(columns[1] == '*'),
dimensions=columns[3]))
return result
def run_git_cmd(command):
"""Run specified git command.
This function is used to ensure that all git commands are run against
the correct repository by including the the --git-dir and --work-tree
options. This is required as we are running the commands outside of
the target repo directory.
Args:
command: the command to run - without the 'git ' prefix, e.g.
"status", "log --oneline", etc.
Returns the output of the command as a string.
"""
cmd = GIT_CMD_PREFIX + command
r = sarge.capture_stdout(cmd)
if r.returncode > 0:
# git doesn't play nicely so r.stderr is None even though it failed
raise Exception(u"Error running git command '%s'" % cmd)
return r.stdout.text
def run(command, get_output=False, cwd=None):
"""By default, run all commands at GITPATH directory.
If command fails, stop program execution.
"""
if cwd is None:
cwd = GITPATH
cprint('===')
cprint('=== Command: ', command)
cprint('=== CWD: ', cwd)
cprint('===')
if get_output:
proc = capture_stdout(command, cwd=cwd)
out = proc.stdout.read().decode()
print(out, end='')
check_exit_code(proc.returncode)
return out
else:
proc = sarge_run(command, cwd=cwd)
check_exit_code(proc.returncode)
def main():
parser = optparse.OptionParser()
parser.add_option('-n',
'--no-dots',
dest='dots',
default=True,
action='store_false',
help='Show dots for progress')
options, args = parser.parse_args()
# p = capture_stdout('ncat -k -l -p 42421', async_=True)
p = capture_stdout('python lister.py -d 0.1 -c 100', async_=True)
time.sleep(0.01)
t = threading.Thread(target=progress, args=(p.stdout, options))
t.start()
while (p.returncodes[0] is None):
# We could do other useful work here. If we have no useful
# work to do here, we can call readline() and process it
# directly in this loop, instead of creating a thread to do it in.
p.commands[0].poll()
time.sleep(0.05)
t.join()
def test_partial_line(self):
p = capture_stdout('echo foobarbaz')
lines = [p.stdout.readline(6), p.stdout.readline().strip()]
self.assertEqual(lines, [b'foobar', b'baz'])
def test_returncodes(self):
p = capture_stdout('echo foo; echo bar; echo baz; false')
self.assertEqual(p.returncodes, [0, 0, 0, 1])
self.assertEqual(p.returncode, 1)
def test_processes(self):
p = capture_stdout('echo foo; echo bar; echo baz; false')
plist = p.processes
for p in plist:
self.assertTrue(isinstance(p, Popen))
def test_returncodes(self):
p = capture_stdout('echo foo; echo bar; echo baz; false')
self.assertEqual(p.returncodes, [0, 0, 0, 1])
self.assertEqual(p.returncode, 1)
def test_partial_line(self):
p = capture_stdout('echo foobarbaz')
lines = [p.stdout.readline(6), p.stdout.readline().strip()]
self.assertEqual(lines, [b'foobar', b'baz'])
def run_remote(self, cmd, input=None):
host = self.args['--host']
return capture_stdout('ssh %s -- %s' % (host, cmd), input=input)
#!/usr/bin/env python3 sh1 = 'cat input.txt | ./sollution.py' sh2 = 'cat input.txt | ./sollution.py | cmp -lb output.txt | head -n 20' import sarge print(sarge.capture_stdout(sh1).stdout.text) print(sarge.capture_stdout(sh2).stdout.text)
def stdout(command):
return sarge.capture_stdout(command).stdout.text.strip()
def run_remote(self, cmd, input=None):
host = self.args['--host']
return capture_stdout('ssh %s -- %s' % (host, cmd), input=input)
def _pipe(command, data_in, encoding='utf8'):
p = sarge.capture_stdout(command, input=data_in.encode(encoding))
return p.stdout.read().decode(encoding)
def test_capture_stdout(self):
p = capture_stdout('echo foo')
self.assertEqual(p.stdout.text.strip(), 'foo')
def test_byte_iterator(self):
p = capture_stdout('echo foo; echo bar')
lines = []
for line in p.stdout:
lines.append(line.strip())
self.assertEqual(lines, [b'foo', b'bar'])
def test_text_iterator(self):
p = capture_stdout('echo foo; echo bar')
lines = []
for line in TextIOWrapper(p.stdout):
lines.append(line)
self.assertEqual(lines, ['foo\n', 'bar\n'])
def test_text_iterator(self):
p = capture_stdout('echo foo; echo bar')
lines = []
for line in TextIOWrapper(p.stdout):
lines.append(line)
self.assertEqual(lines, ['foo\n', 'bar\n'])
def test_processes(self):
p = capture_stdout('echo foo; echo bar; echo baz; false')
plist = p.processes
for p in plist:
self.assertTrue(isinstance(p, Popen))
def scan(device, settings):
command = build_commandline(device, settings)
c = sarge.capture_stdout(command)
return TIFF.imread(io.BytesIO(c.stdout.read()))