def test_tar_with_inaccessible_file(self):
base = tempfile.mkdtemp()
full_path = os.path.join(base, 'foo')
self.addCleanup(shutil.rmtree, base)
with open(full_path, 'w') as f:
f.write('content')
os.chmod(full_path, 0o222)
with pytest.raises(IOError) as ei:
tar(base)
assert f'Can not read file in context: {full_path}' in (ei.exconly())
def test_tar_with_inaccessible_file(self):
base = tempfile.mkdtemp()
full_path = os.path.join(base, 'foo')
self.addCleanup(shutil.rmtree, base)
with open(full_path, 'w') as f:
f.write('content')
os.chmod(full_path, 0o222)
with pytest.raises(IOError) as ei:
tar(base)
assert 'Can not read file in context: {}'.format(full_path) in (
ei.exconly()
)
def test_tar_with_excludes(self):
dirs = ["foo", "foo/bar", "bar"]
files = [
"Dockerfile",
"Dockerfile.alt",
".dockerignore",
"a.py",
"a.go",
"b.py",
"cde.py",
"foo/a.py",
"foo/b.py",
"foo/bar/a.py",
"bar/a.py",
]
exclude = ["*.py", "!b.py", "!a.go", "foo", "Dockerfile*", ".dockerignore"]
expected_names = set(["Dockerfile", ".dockerignore", "a.go", "b.py", "bar", "bar/a.py"])
base = make_tree(dirs, files)
self.addCleanup(shutil.rmtree, base)
with tar(base, exclude=exclude) as archive:
tar_data = tarfile.open(fileobj=archive)
assert sorted(tar_data.getnames()) == sorted(expected_names)
def build(self, path=None, tag=None, quiet=False, fileobj=None, nocache=False, rm=False, stream=False):
remote = context = headers = None
if path is None and fileobj is None:
raise Exception("Either path or fileobj needs to be provided.")
if fileobj is not None:
context = utils.mkbuildcontext(fileobj)
elif path.startswith(("http://", "https://", "git://", "github.com/")):
remote = path
else:
context = utils.tar(path)
u = self._url("/build")
params = {"t": tag, "remote": remote, "q": quiet, "nocache": nocache, "rm": rm}
if context is not None:
headers = {"Content-Type": "application/tar"}
response = self._post(u, data=context, params=params, headers=headers, stream=stream)
if context is not None:
context.close()
if stream:
return self._stream_result(response)
else:
output = self._result(response)
srch = r"Successfully built ([0-9a-f]+)"
match = re.search(srch, output)
if not match:
return None, output
return match.group(1), output
def build(self, path=None, tag=None, quiet=False, fileobj=None,
nocache=False, rm=False):
remote = context = headers = None
if path is None and fileobj is None:
raise Exception("Either path or fileobj needs to be provided.")
if fileobj is not None:
context = utils.mkbuildcontext(fileobj)
elif path.startswith(('http://', 'https://', 'git://', 'github.com/')):
remote = path
else:
context = utils.tar(path)
u = self._url('/build')
params = {
't': tag,
'remote': remote,
'q': quiet,
'nocache': nocache,
'rm': rm
}
if context is not None:
headers = {'Content-Type': 'application/tar'}
res = self._result(self.post(u, context, params=params,
headers=headers, stream=True))
if context is not None:
context.close()
srch = r'Successfully built ([0-9a-f]+)'
match = re.search(srch, res)
if not match:
return None, res
return match.group(1), res
def build(self, path=None, tag=None, quiet=False, fileobj=None,
nocache=False, rm=False):
remote = context = headers = None
if path is None and fileobj is None:
raise Exception("Either path or fileobj needs to be provided.")
if fileobj is not None:
context = utils.mkbuildcontext(fileobj)
elif path.startswith(('http://', 'https://', 'git://', 'github.com/')):
remote = path
else:
context = utils.tar(path)
u = self._url('/build')
params = {
't': tag,
'remote': remote,
'q': quiet,
'nocache': nocache,
'rm': rm
}
if context is not None:
headers = {'Content-Type': 'application/tar'}
res = self._result(self.post(u, context, params=params,
headers=headers, stream=True))
if context is not None:
context.close()
srch = r'Successfully built ([0-9a-f]+)'
match = re.search(srch, res)
if not match:
return None, res
return match.group(1), res
def test_tar_with_empty_directory(self):
base = tempfile.mkdtemp()
self.addCleanup(shutil.rmtree, base)
for d in ['foo', 'bar']:
os.makedirs(os.path.join(base, d))
with tar(base) as archive:
tar_data = tarfile.open(fileobj=archive)
self.assertEqual(sorted(tar_data.getnames()), ['bar', 'foo'])
def test_tar_with_empty_directory(self):
base = tempfile.mkdtemp()
self.addCleanup(shutil.rmtree, base)
for d in ['foo', 'bar']:
os.makedirs(os.path.join(base, d))
with tar(base) as archive:
tar_data = tarfile.open(fileobj=archive)
self.assertEqual(sorted(tar_data.getnames()), ['bar', 'foo'])
def test_tar_with_directory_symlinks(self):
base = tempfile.mkdtemp()
self.addCleanup(shutil.rmtree, base)
for d in ["foo", "bar"]:
os.makedirs(os.path.join(base, d))
os.symlink("../foo", os.path.join(base, "bar/foo"))
with tar(base) as archive:
tar_data = tarfile.open(fileobj=archive)
self.assertEqual(sorted(tar_data.getnames()), ["bar", "bar/foo", "foo"])
def test_tar_with_directory_symlinks(self):
base = tempfile.mkdtemp()
self.addCleanup(shutil.rmtree, base)
for d in ['foo', 'bar']:
os.makedirs(os.path.join(base, d))
os.symlink('../foo', os.path.join(base, 'bar/foo'))
with tar(base) as archive:
tar_data = tarfile.open(fileobj=archive)
assert sorted(tar_data.getnames()) == ['bar', 'bar/foo', 'foo']
def test_tar_with_directory_symlinks(self):
base = tempfile.mkdtemp()
self.addCleanup(shutil.rmtree, base)
for d in ['foo', 'bar']:
os.makedirs(os.path.join(base, d))
os.symlink('../foo', os.path.join(base, 'bar/foo'))
with tar(base) as archive:
tar_data = tarfile.open(fileobj=archive)
assert sorted(tar_data.getnames()) == ['bar', 'bar/foo', 'foo']
def test_tar_with_file_symlinks(self):
base = tempfile.mkdtemp()
self.addCleanup(shutil.rmtree, base)
with open(os.path.join(base, 'foo'), 'w') as f:
f.write("content")
os.makedirs(os.path.join(base, 'bar'))
os.symlink('../foo', os.path.join(base, 'bar/foo'))
with tar(base) as archive:
tar_data = tarfile.open(fileobj=archive)
assert sorted(tar_data.getnames()) == ['bar', 'bar/foo', 'foo']
def test_tar_with_file_symlinks(self):
base = tempfile.mkdtemp()
self.addCleanup(shutil.rmtree, base)
with open(os.path.join(base, 'foo'), 'w') as f:
f.write("content")
os.makedirs(os.path.join(base, 'bar'))
os.symlink('../foo', os.path.join(base, 'bar/foo'))
with tar(base) as archive:
tar_data = tarfile.open(fileobj=archive)
assert sorted(tar_data.getnames()) == ['bar', 'bar/foo', 'foo']
def test_tar_with_file_symlinks(self):
base = tempfile.mkdtemp()
self.addCleanup(shutil.rmtree, base)
with open(os.path.join(base, "foo"), "w") as f:
f.write("content")
os.makedirs(os.path.join(base, "bar"))
os.symlink("../foo", os.path.join(base, "bar/foo"))
with tar(base) as archive:
tar_data = tarfile.open(fileobj=archive)
self.assertEqual(sorted(tar_data.getnames()), ["bar", "bar/foo", "foo"])
def test_tar_socket_file(self):
base = tempfile.mkdtemp()
self.addCleanup(shutil.rmtree, base)
for d in ['foo', 'bar']:
os.makedirs(os.path.join(base, d))
sock = socket.socket(socket.AF_UNIX)
self.addCleanup(sock.close)
sock.bind(os.path.join(base, 'test.sock'))
with tar(base) as archive:
tar_data = tarfile.open(fileobj=archive)
self.assertEqual(sorted(tar_data.getnames()), ['bar', 'foo'])
def tar_test_negative_mtime_bug(self):
base = tempfile.mkdtemp()
filename = os.path.join(base, 'th.txt')
self.addCleanup(shutil.rmtree, base)
with open(filename, 'w') as f:
f.write('Invisible Full Moon')
os.utime(filename, (12345, -3600.0))
with tar(base) as archive:
tar_data = tarfile.open(fileobj=archive)
assert tar_data.getnames() == ['th.txt']
assert tar_data.getmember('th.txt').mtime == -3600
def tar_test_negative_mtime_bug(self):
base = tempfile.mkdtemp()
filename = os.path.join(base, 'th.txt')
self.addCleanup(shutil.rmtree, base)
with open(filename, 'w') as f:
f.write('Invisible Full Moon')
os.utime(filename, (12345, -3600.0))
with tar(base) as archive:
tar_data = tarfile.open(fileobj=archive)
assert tar_data.getnames() == ['th.txt']
assert tar_data.getmember('th.txt').mtime == -3600
def test_tar_socket_file(self):
base = tempfile.mkdtemp()
self.addCleanup(shutil.rmtree, base)
for d in ['foo', 'bar']:
os.makedirs(os.path.join(base, d))
sock = socket.socket(socket.AF_UNIX)
self.addCleanup(sock.close)
sock.bind(os.path.join(base, 'test.sock'))
with tar(base) as archive:
tar_data = tarfile.open(fileobj=archive)
assert sorted(tar_data.getnames()) == ['bar', 'foo']
def bundle_docker_dir(tag, docker_path):
"""
Tars up a directory using the normal docker.util.tar method but
adds tag if it's not None.
"""
# tar up the directory minus the Dockerfile,
path = os.path.dirname(docker_path)
try:
with open(join(path, '.dockerignore')) as f:
stripped = (p.strip() for p in f.readlines())
ignore = [x for x in stripped if x]
except IOError:
ignore = []
dockerfile_name = os.path.basename(docker_path)
if tag is None:
return utils.tar(path, ignore, dockerfile=dockerfile_name)
# docker-py 1.6+ won't ignore the dockerfile
# passing dockerfile='' works around the issuen
# and lets us add the modified file when we're done.
ignore.append(dockerfile_name)
fileobj = utils.tar(path, ignore, dockerfile='')
t = tarfile.open(fileobj=fileobj, mode='a')
dfinfo = tarfile.TarInfo(dockerfile_name)
contents = tag_parent(tag, open(join(path, dockerfile_name)).read())
if not isinstance(contents, bytes):
contents = contents.encode('utf8')
dockerfile = io.BytesIO(contents)
dfinfo.size = len(dockerfile.getvalue())
t.addfile(dfinfo, dockerfile)
t.close()
fileobj.seek(0)
return fileobj
def bundle_docker_dir(tag, docker_path):
"""
Tars up a directory using the normal docker.util.tar method but
adds tag if it's not None.
"""
# tar up the directory minus the Dockerfile,
path = os.path.dirname(docker_path)
try:
with open(join(path, '.dockerignore')) as f:
stripped = (p.strip() for p in f.readlines())
ignore = [x for x in stripped if x]
except IOError:
ignore = []
dockerfile_name = os.path.basename(docker_path)
if tag is None:
return utils.tar(path, ignore, dockerfile=dockerfile_name)
# docker-py 1.6+ won't ignore the dockerfile
# passing dockerfile='' works around the issuen
# and lets us add the modified file when we're done.
ignore.append(dockerfile_name)
fileobj = utils.tar(path, ignore, dockerfile='')
t = tarfile.open(fileobj=fileobj, mode='a')
dfinfo = tarfile.TarInfo(dockerfile_name)
contents = tag_parent(tag, open(join(path, dockerfile_name)).read())
if not isinstance(contents, bytes):
contents = contents.encode('utf8')
dockerfile = io.BytesIO(contents)
dfinfo.size = len(dockerfile.getvalue())
t.addfile(dfinfo, dockerfile)
t.close()
fileobj.seek(0)
return fileobj
def test_tar_directory_link(self):
dirs = ['a', 'b', 'a/c']
files = ['a/hello.py', 'b/utils.py', 'a/c/descend.py']
base = make_tree(dirs, files)
self.addCleanup(shutil.rmtree, base)
os.symlink(os.path.join(base, 'b'), os.path.join(base, 'a/c/b'))
with tar(base) as archive:
tar_data = tarfile.open(fileobj=archive)
names = tar_data.getnames()
for member in dirs + files:
assert member in names
assert 'a/c/b' in names
assert 'a/c/b/utils.py' not in names
def test_tar_directory_link(self):
dirs = ['a', 'b', 'a/c']
files = ['a/hello.py', 'b/utils.py', 'a/c/descend.py']
base = make_tree(dirs, files)
self.addCleanup(shutil.rmtree, base)
os.symlink(os.path.join(base, 'b'), os.path.join(base, 'a/c/b'))
with tar(base) as archive:
tar_data = tarfile.open(fileobj=archive)
names = tar_data.getnames()
for member in dirs + files:
assert member in names
assert 'a/c/b' in names
assert 'a/c/b/utils.py' not in names
def test_tar_with_excludes(self):
dirs = [
'foo',
'foo/bar',
'bar',
]
files = [
'Dockerfile',
'Dockerfile.alt',
'.dockerignore',
'a.py',
'a.go',
'b.py',
'cde.py',
'foo/a.py',
'foo/b.py',
'foo/bar/a.py',
'bar/a.py',
]
exclude = [
'*.py',
'!b.py',
'!a.go',
'foo',
'Dockerfile*',
'.dockerignore',
]
expected_names = set([
'Dockerfile',
'.dockerignore',
'a.go',
'b.py',
'bar',
'bar/a.py',
])
base = make_tree(dirs, files)
self.addCleanup(shutil.rmtree, base)
with tar(base, exclude=exclude) as archive:
tar_data = tarfile.open(fileobj=archive)
assert sorted(tar_data.getnames()) == sorted(expected_names)
def test_tar_with_excludes(self):
dirs = [
'foo',
'foo/bar',
'bar',
]
files = [
'Dockerfile',
'Dockerfile.alt',
'.dockerignore',
'a.py',
'a.go',
'b.py',
'cde.py',
'foo/a.py',
'foo/b.py',
'foo/bar/a.py',
'bar/a.py',
]
exclude = [
'*.py',
'!b.py',
'!a.go',
'foo',
'Dockerfile*',
'.dockerignore',
]
expected_names = set([
'Dockerfile',
'.dockerignore',
'a.go',
'b.py',
'bar',
'bar/a.py',
])
base = make_tree(dirs, files)
self.addCleanup(shutil.rmtree, base)
with tar(base, exclude=exclude) as archive:
tar_data = tarfile.open(fileobj=archive)
assert sorted(tar_data.getnames()) == sorted(expected_names)
def bundle_docker_dir(modify_docker_func, path):
"""
Tars up a directory using the normal docker.util.tar method but
first relpaces the contents of the Dockerfile found at path with
the result of calling modify_docker_func with a string containing
the complete contents of the docker file.
For example to prepend the phrase "bogus header" to a dockerfile
we first create a function that takes the contents of the current
dockerfile as it's contents.
>>> def append_bogus(docker_content):
... return "bogus header " + docker_content
Then we can tar it up.. but first we'll need some test content.
These routines create a temporary directory containing the following
3 files.
./Dockerfile
./bogus1
./bogus2
>>> from shipwright import tar
>>> path = join(tar.TEST_ROOT, 'Dockerfile')
>>> _ = open(path, 'w').write(u'blah')
>>> _ = open(join(tar.TEST_ROOT, 'bogus1'),'w').write('hi mom')
>>> _ = open(join(tar.TEST_ROOT, 'bogus2'), 'w').write('hello world')
Now we can call bundle_docker_dir passing it our append_bogus function to
mutate the docker contents. We'll receive a file like object which
is stream of the contents encoded as a tar file (the format Docker
build expects)
>>> fileobj = bundle_docker_dir(append_bogus, tar.TEST_ROOT)
Normally we'd just pass this directly to the docker build command
but for the purpose of this test, we'll use tarfile to decode the string
and ensure that our mutation happened as planned.
First lets ensure that our tarfile contains our test files
>>> t = tarfile.open(fileobj=fileobj)
>>> t.getnames() # doctest: +SKIP
['bogus1', 'bogus2', 'Dockerfile']
And if we exctart the Dockerfile it starts with 'bogus header'
>>> ti = t.extractfile('Dockerfile')
>>> ti.read().startswith(b'bogus header')
True
Obviously a real mutation would ensure that the the contents
of the Dockerfile are valid docker commands and not some
bogus content.
"""
# tar up the directory minus the Dockerfile,
# TODO: respect .dockerignore
try:
ignore = filter(None, [
p.strip() for p in open(join(path, '.dockerignore')).readlines()
])
except IOError:
ignore = []
# docker-py 1.6+ won't ignore the dockerfile
# passing dockerfile='' works around the issuen
# and lets us add the modified file when we're done.
ignore.append('Dockerfile')
fileobj = utils.tar(path, ignore, dockerfile='')
# append a dockerfile after running it through a mutation
# function first
t = tarfile.open(fileobj=fileobj, mode='a')
dfinfo = tarfile.TarInfo('Dockerfile')
contents = modify_docker_func(open(join(path, 'Dockerfile')).read())
if not isinstance(contents, bytes):
contents = contents.encode('utf8')
dockerfile = io.BytesIO(contents)
dfinfo.size = len(dockerfile.getvalue())
t.addfile(dfinfo, dockerfile)
t.close()
fileobj.seek(0)
return fileobj
def bundle_docker_dir(modify_docker_func, path):
"""
Tars up a directory using the normal docker.util.tar method but
first relpaces the contents of the Dockerfile found at path with
the result of calling modify_docker_func with a string containing
the complete contents of the docker file.
For example to prepend the phrase "bogus header" to a dockerfile
we first create a function that takes the contents of the current
dockerfile as it's contents.
>>> def append_bogus(docker_content):
... return "bogus header " + docker_content
Then we can tar it up.. but first we'll need some test content.
These routines create a temporary directory containing the following
3 files.
./Dockerfile
./bogus1
./bogus2
>>> from shipwright import tar
>>> path = join(tar.TEST_ROOT, 'Dockerfile')
>>> open(path, 'w').write('blah')
>>> open(join(tar.TEST_ROOT, 'bogus1'),'w').write('hi mom')
>>> open(join(tar.TEST_ROOT, 'bogus2'), 'w').write('hello world')
Now we can call bundle_docker_dir passing it our append_bogus function to
mutate the docker contents. We'll receive a file like object which
is stream of the contents encoded as a tar file (the format Docker
build expects)
>>> fileobj = bundle_docker_dir(append_bogus, tar.TEST_ROOT)
Normally we'd just pass this directly to the docker build command
but for the purpose of this test, we'll use trfile to decode the string
and ensure that our mutation happened as planned.
First lets ensure that our tarfile contains our test files
>>> t = tarfile.open(fileobj=fileobj)
>>> t.getnames()
['bogus1', 'bogus2', 'Dockerfile']
And if we exctart the Dockerfile it starts with 'bogus header'
>>> ti = t.extractfile('Dockerfile')
>>> ti.read().startswith('bogus header')
True
Obviously a real mutation would ensure that the the contents
of the Dockerfile are valid docker commands and not some
bogus content.
"""
# tar up the directory minus the Dockerfile,
# TODO: respect .dockerignore
try:
ignore = filter(
None,
[p.strip() for p in open(join(path, '.dockerignore')).readlines()])
except IOError:
ignore = []
ignore.append('Dockerfile')
fileobj = utils.tar(path, ignore)
# append a dockerfile after running it through a mutation
# function first
t = tarfile.open(fileobj=fileobj, mode='a')
dfinfo = tarfile.TarInfo('Dockerfile')
dockerfile = io.BytesIO(
modify_docker_func(open(join(path, 'Dockerfile')).read()))
dfinfo.size = len(dockerfile.getvalue())
t.addfile(dfinfo, dockerfile)
t.close()
fileobj.seek(0)
return fileobj