def test_group_reuse():
"""We can re-use a group."""
grp = Group([
Docker('py35', image='python:3.5'),
Docker('py36', image='python:3.6')
])
with grp:
grp.call(time.time)
grp.call(time.time)
def test_group_repr():
"""Groups have a usable repr."""
grp = Group([
Docker('py35', image='python:3.5'),
Docker('py36', image='python:3.6')
])
assert repr(grp) == "Group([Docker('py35'), Docker('py36')])"
else:
fmt = '[{host}] {l}'
for l in result.splitlines():
print(fmt.format(host=host, color=color, l=l))
if __name__ == '__main__':
from chopsticks.tunnel import Docker
from chopsticks.group import Group
import chopsticks.ioloop
chopsticks.tunnel.PICKLE_LEVEL = 2
class Py2Docker(Docker):
python3 = 'python2'
group = Group([
Py2Docker('python2.7', image='python:2.7'),
Docker('python3.3', image='python:3.3'),
Docker('python3.4', image='python:3.4'),
Docker('python3.5', image='python:3.5'),
Docker('python3.6', image='python:3.6'),
])
try:
while True:
dorepl(group)
finally:
del group
def _as_group(self):
"""Tunnels behave like groups of one tunnel."""
from chopsticks.group import Group
return Group([self])
from chopsticks.group import Group
from chopsticks.facts import ip
from to_compute import uptime
NODES_FILE_PATH = 'nodes.txt'
addrs = open(NODES_FILE_PATH).read().strip().splitlines()
group = Group(addrs)
def run_on_nodes(func):
for host, result in group.call(func).iteritems():
print('%s result:' % host, result)
run_on_nodes(uptime)
import time
import os.path
from chopsticks.queue import Queue, AsyncResult
from chopsticks.tunnel import Docker, Local
from chopsticks.group import Group, GroupResult
def char_range(start, stop):
start = ord(start)
stop = ord(stop) + 1
return ''.join(chr(c) for c in range(start, stop))
tunnel = None
group = Group([
Docker('unittest-%d' % random.randint(0, 1e9)),
Docker('unittest-%d' % random.randint(0, 1e9))
])
def setup_module():
global tunnel
tunnel = Docker('unittest-%d' % random.randint(0, 1e9))
def teardown_module():
global tunnel
tunnel.close()
group.close()
tunnel = None
from __future__ import print_function
from chopsticks.tunnel import Tunnel, Docker
from chopsticks.group import Group
t = Tunnel('byzantium')
res = t.fetch('/etc/passwd', 'byzantium-passwd')
print(res)
print(open(res['local_path']).read())
del t
g = Group(['byzantium', 'office', Docker('docker-1')])
for host, res in g.fetch('/etc/passwd', local_path='fetches/passwd-{host}').successful():
print(host, res)
print(open(res['local_path']).read())
"""Exercise the chopsticks parallel API."""
from __future__ import print_function
from chopsticks.tunnel import Local, Docker
from chopsticks.group import Group
import chopsticks.facts
import time
group = Group([
Local('worker-1'),
'byzantium',
'office',
Docker('docker-1')
])
for host, t in group.call(time.time).successful():
print('Time on %s:' % host, t)
print()
for host, addr in group.call(chopsticks.facts.ip).successful():
print('%s ip:' % host, addr)
print()
for host, ver in group.call(chopsticks.facts.python_version).successful():
print('%s Python version:' % host, tuple(ver))
"""Exercise the chopsticks Docker connectivity."""
import sys
from chopsticks.facts import python_version
from funcs import print_is_function
def hello():
print >> sys.stderr, "hello"
return "hello chopsticks"
if __name__ == '__main__':
from chopsticks.tunnel import Docker
from chopsticks.group import Group
group = Group([
Docker('worker-1'),
# Docker('worker-1', image='python:3.4'),
# Docker('worker-2', image='python:3.5'),
# Docker('worker-3', image='python:3.6'),
])
for host, result in group.call(print_is_function).items():
print(host, result)
# for host, python_version in group.call(python_version).items():
# print('%s Python version:' % host, python_version)
"""Exercise the chopsticks Docker connectivity."""
import sys
from chopsticks.facts import python_version
if __name__ == '__main__':
from chopsticks.tunnel import Docker
from chopsticks.group import Group
group = Group([
Docker('worker-1', image='python:3.4'),
Docker('worker-2', image='python:3.5'),
Docker('worker-3', image='python:3.6'),
])
for host, python_version in group.call(python_version).items():
print('%s Python version:' % host, python_version)
from chopsticks.group import Group
from chopsticks.tunnel import Docker
from chopsticks.facts import python_version
class PyDocker(Docker):
"""Docker tunnel for Python images."""
# These Docker containers put their Python into /usr/local/bin/python,
# regardless of 2/3. Because we just want to connect to whatever is in the
# container here, we can override the 2/3 selection logic.
python2 = python3 = 'python'
grp = Group([
PyDocker('unittest-27', image='python:2.7'),
PyDocker('unittest-35', image='python:3.5'),
PyDocker('unittest-36', image='python:3.6'),
])
def teardown_module():
"""Disconnect the group."""
grp.close()
def is_py3():
"""Return True if the host is Python 3."""
return tuple(python_version(short=True)) >= (3, 0)
def test_ping():
"""Exercise the chopsticks parallel API."""
from __future__ import print_function
from chopsticks.tunnel import Local, Docker
from chopsticks.group import Group
import chopsticks.facts
import time
group = Group([Local('worker-1'), 'byzantium', 'office', Docker('docker-1')])
for host, t in group.call(time.time).successful():
print('Time on %s:' % host, t)
print()
for host, addr in group.call(chopsticks.facts.ip).successful():
print('%s ip:' % host, addr)
print()
for host, ver in group.call(chopsticks.facts.python_version).successful():
print('%s Python version:' % host, tuple(ver))
from chopsticks.group import Group
from chopsticks.tunnel import Docker
dck1 = Docker('docker1')
dck2 = Docker('docker2')
grp_a = Group(['host1', 'host2'])
grp_b = Group(['host1', 'host3'])
def test_union():
"""We can calculate the union of two groups."""
union = grp_a + grp_b
hostnames = sorted(t.host for t in union.tunnels)
assert hostnames == ['host1', 'host2', 'host3']
def test_intersection():
"""We can calculate the intersection of two groups."""
union = grp_a & grp_b
hostnames = sorted(t.host for t in union.tunnels)
assert hostnames == ['host1']
def test_difference():
"""We can calculate the difference of two groups."""
union = grp_a - grp_b
hostnames = sorted(t.host for t in union.tunnels)
assert hostnames == ['host2']
def test_symdifference():
from __future__ import print_function
from chopsticks.tunnel import Tunnel, Docker, Local
from chopsticks.group import Group
from chopsticks.helpers import check_output
from chopsticks.facts import python_version
t = Docker('test-1')
res = t.put('/usr/share/common-licenses/GPL', 'blah', mode=0o755)
print(res)
print(t.call(python_version))
print(t.call(check_output, ['ls', '-l', res['remote_path']]))
g = Group([t, Docker('test-2')])
for host, res in g.put('/usr/share/common-licenses/GPL').iteritems():
print(host, res)
for host, res in g.call(check_output, ['ls', '-l', '/tmp']).iteritems():
print(host)
print(res)
from __future__ import print_function
from chopsticks.tunnel import Tunnel, Docker
from chopsticks.group import Group
t = Tunnel('byzantium')
res = t.fetch('/etc/passwd', 'byzantium-passwd')
print(res)
print(open(res['local_path']).read())
del t
g = Group(['byzantium', 'office', Docker('docker-1')])
for host, res in g.fetch('/etc/passwd',
local_path='fetches/passwd-{host}').successful():
print(host, res)
print(open(res['local_path']).read())
