Example #1
0
    def test_tar_with_inaccessible_file(self):
        base = tempfile.mkdtemp()
        full_path = os.path.join(base, 'foo')
        self.addCleanup(shutil.rmtree, base)
        with open(full_path, 'w') as f:
            f.write('content')
        os.chmod(full_path, 0o222)
        with pytest.raises(IOError) as ei:
            tar(base)

        assert f'Can not read file in context: {full_path}' in (ei.exconly())
Example #2
0
    def test_tar_with_inaccessible_file(self):
        base = tempfile.mkdtemp()
        full_path = os.path.join(base, 'foo')
        self.addCleanup(shutil.rmtree, base)
        with open(full_path, 'w') as f:
            f.write('content')
        os.chmod(full_path, 0o222)
        with pytest.raises(IOError) as ei:
            tar(base)

        assert 'Can not read file in context: {}'.format(full_path) in (
            ei.exconly()
        )
Example #3
0
    def test_tar_with_excludes(self):
        dirs = ["foo", "foo/bar", "bar"]

        files = [
            "Dockerfile",
            "Dockerfile.alt",
            ".dockerignore",
            "a.py",
            "a.go",
            "b.py",
            "cde.py",
            "foo/a.py",
            "foo/b.py",
            "foo/bar/a.py",
            "bar/a.py",
        ]

        exclude = ["*.py", "!b.py", "!a.go", "foo", "Dockerfile*", ".dockerignore"]

        expected_names = set(["Dockerfile", ".dockerignore", "a.go", "b.py", "bar", "bar/a.py"])

        base = make_tree(dirs, files)
        self.addCleanup(shutil.rmtree, base)

        with tar(base, exclude=exclude) as archive:
            tar_data = tarfile.open(fileobj=archive)
            assert sorted(tar_data.getnames()) == sorted(expected_names)
Example #4
0
    def build(self, path=None, tag=None, quiet=False, fileobj=None, nocache=False, rm=False, stream=False):
        remote = context = headers = None
        if path is None and fileobj is None:
            raise Exception("Either path or fileobj needs to be provided.")

        if fileobj is not None:
            context = utils.mkbuildcontext(fileobj)
        elif path.startswith(("http://", "https://", "git://", "github.com/")):
            remote = path
        else:
            context = utils.tar(path)

        u = self._url("/build")
        params = {"t": tag, "remote": remote, "q": quiet, "nocache": nocache, "rm": rm}
        if context is not None:
            headers = {"Content-Type": "application/tar"}

        response = self._post(u, data=context, params=params, headers=headers, stream=stream)

        if context is not None:
            context.close()
        if stream:
            return self._stream_result(response)
        else:
            output = self._result(response)
            srch = r"Successfully built ([0-9a-f]+)"
            match = re.search(srch, output)
            if not match:
                return None, output
            return match.group(1), output
Example #5
0
    def build(self, path=None, tag=None, quiet=False, fileobj=None,
              nocache=False, rm=False):
        remote = context = headers = None
        if path is None and fileobj is None:
            raise Exception("Either path or fileobj needs to be provided.")

        if fileobj is not None:
            context = utils.mkbuildcontext(fileobj)
        elif path.startswith(('http://', 'https://', 'git://', 'github.com/')):
            remote = path
        else:
            context = utils.tar(path)

        u = self._url('/build')
        params = {
            't': tag,
            'remote': remote,
            'q': quiet,
            'nocache': nocache,
            'rm': rm
        }
        if context is not None:
            headers = {'Content-Type': 'application/tar'}
        res = self._result(self.post(u, context, params=params,
                           headers=headers, stream=True))
        if context is not None:
            context.close()
        srch = r'Successfully built ([0-9a-f]+)'
        match = re.search(srch, res)
        if not match:
            return None, res
        return match.group(1), res
Example #6
0
    def build(self, path=None, tag=None, quiet=False, fileobj=None,
              nocache=False, rm=False):
        remote = context = headers = None
        if path is None and fileobj is None:
            raise Exception("Either path or fileobj needs to be provided.")

        if fileobj is not None:
            context = utils.mkbuildcontext(fileobj)
        elif path.startswith(('http://', 'https://', 'git://', 'github.com/')):
            remote = path
        else:
            context = utils.tar(path)

        u = self._url('/build')
        params = {
            't': tag,
            'remote': remote,
            'q': quiet,
            'nocache': nocache,
            'rm': rm
        }
        if context is not None:
            headers = {'Content-Type': 'application/tar'}
        res = self._result(self.post(u, context, params=params,
                           headers=headers, stream=True))
        if context is not None:
            context.close()
        srch = r'Successfully built ([0-9a-f]+)'
        match = re.search(srch, res)
        if not match:
            return None, res
        return match.group(1), res
Example #7
0
 def test_tar_with_empty_directory(self):
     base = tempfile.mkdtemp()
     self.addCleanup(shutil.rmtree, base)
     for d in ['foo', 'bar']:
         os.makedirs(os.path.join(base, d))
     with tar(base) as archive:
         tar_data = tarfile.open(fileobj=archive)
         self.assertEqual(sorted(tar_data.getnames()), ['bar', 'foo'])
Example #8
0
 def test_tar_with_empty_directory(self):
     base = tempfile.mkdtemp()
     self.addCleanup(shutil.rmtree, base)
     for d in ['foo', 'bar']:
         os.makedirs(os.path.join(base, d))
     with tar(base) as archive:
         tar_data = tarfile.open(fileobj=archive)
         self.assertEqual(sorted(tar_data.getnames()), ['bar', 'foo'])
Example #9
0
 def test_tar_with_directory_symlinks(self):
     base = tempfile.mkdtemp()
     self.addCleanup(shutil.rmtree, base)
     for d in ["foo", "bar"]:
         os.makedirs(os.path.join(base, d))
     os.symlink("../foo", os.path.join(base, "bar/foo"))
     with tar(base) as archive:
         tar_data = tarfile.open(fileobj=archive)
         self.assertEqual(sorted(tar_data.getnames()), ["bar", "bar/foo", "foo"])
Example #10
0
 def test_tar_with_directory_symlinks(self):
     base = tempfile.mkdtemp()
     self.addCleanup(shutil.rmtree, base)
     for d in ['foo', 'bar']:
         os.makedirs(os.path.join(base, d))
     os.symlink('../foo', os.path.join(base, 'bar/foo'))
     with tar(base) as archive:
         tar_data = tarfile.open(fileobj=archive)
         assert sorted(tar_data.getnames()) == ['bar', 'bar/foo', 'foo']
Example #11
0
 def test_tar_with_directory_symlinks(self):
     base = tempfile.mkdtemp()
     self.addCleanup(shutil.rmtree, base)
     for d in ['foo', 'bar']:
         os.makedirs(os.path.join(base, d))
     os.symlink('../foo', os.path.join(base, 'bar/foo'))
     with tar(base) as archive:
         tar_data = tarfile.open(fileobj=archive)
         assert sorted(tar_data.getnames()) == ['bar', 'bar/foo', 'foo']
Example #12
0
 def test_tar_with_file_symlinks(self):
     base = tempfile.mkdtemp()
     self.addCleanup(shutil.rmtree, base)
     with open(os.path.join(base, 'foo'), 'w') as f:
         f.write("content")
     os.makedirs(os.path.join(base, 'bar'))
     os.symlink('../foo', os.path.join(base, 'bar/foo'))
     with tar(base) as archive:
         tar_data = tarfile.open(fileobj=archive)
         assert sorted(tar_data.getnames()) == ['bar', 'bar/foo', 'foo']
Example #13
0
 def test_tar_with_file_symlinks(self):
     base = tempfile.mkdtemp()
     self.addCleanup(shutil.rmtree, base)
     with open(os.path.join(base, 'foo'), 'w') as f:
         f.write("content")
     os.makedirs(os.path.join(base, 'bar'))
     os.symlink('../foo', os.path.join(base, 'bar/foo'))
     with tar(base) as archive:
         tar_data = tarfile.open(fileobj=archive)
         assert sorted(tar_data.getnames()) == ['bar', 'bar/foo', 'foo']
Example #14
0
 def test_tar_with_file_symlinks(self):
     base = tempfile.mkdtemp()
     self.addCleanup(shutil.rmtree, base)
     with open(os.path.join(base, "foo"), "w") as f:
         f.write("content")
     os.makedirs(os.path.join(base, "bar"))
     os.symlink("../foo", os.path.join(base, "bar/foo"))
     with tar(base) as archive:
         tar_data = tarfile.open(fileobj=archive)
         self.assertEqual(sorted(tar_data.getnames()), ["bar", "bar/foo", "foo"])
Example #15
0
 def test_tar_socket_file(self):
     base = tempfile.mkdtemp()
     self.addCleanup(shutil.rmtree, base)
     for d in ['foo', 'bar']:
         os.makedirs(os.path.join(base, d))
     sock = socket.socket(socket.AF_UNIX)
     self.addCleanup(sock.close)
     sock.bind(os.path.join(base, 'test.sock'))
     with tar(base) as archive:
         tar_data = tarfile.open(fileobj=archive)
         self.assertEqual(sorted(tar_data.getnames()), ['bar', 'foo'])
Example #16
0
 def tar_test_negative_mtime_bug(self):
     base = tempfile.mkdtemp()
     filename = os.path.join(base, 'th.txt')
     self.addCleanup(shutil.rmtree, base)
     with open(filename, 'w') as f:
         f.write('Invisible Full Moon')
     os.utime(filename, (12345, -3600.0))
     with tar(base) as archive:
         tar_data = tarfile.open(fileobj=archive)
         assert tar_data.getnames() == ['th.txt']
         assert tar_data.getmember('th.txt').mtime == -3600
Example #17
0
 def tar_test_negative_mtime_bug(self):
     base = tempfile.mkdtemp()
     filename = os.path.join(base, 'th.txt')
     self.addCleanup(shutil.rmtree, base)
     with open(filename, 'w') as f:
         f.write('Invisible Full Moon')
     os.utime(filename, (12345, -3600.0))
     with tar(base) as archive:
         tar_data = tarfile.open(fileobj=archive)
         assert tar_data.getnames() == ['th.txt']
         assert tar_data.getmember('th.txt').mtime == -3600
Example #18
0
 def test_tar_socket_file(self):
     base = tempfile.mkdtemp()
     self.addCleanup(shutil.rmtree, base)
     for d in ['foo', 'bar']:
         os.makedirs(os.path.join(base, d))
     sock = socket.socket(socket.AF_UNIX)
     self.addCleanup(sock.close)
     sock.bind(os.path.join(base, 'test.sock'))
     with tar(base) as archive:
         tar_data = tarfile.open(fileobj=archive)
         assert sorted(tar_data.getnames()) == ['bar', 'foo']
Example #19
0
def bundle_docker_dir(tag, docker_path):
    """
    Tars up a directory using the normal docker.util.tar method but
    adds tag if it's not None.
    """

    # tar up the directory minus the Dockerfile,

    path = os.path.dirname(docker_path)
    try:
        with open(join(path, '.dockerignore')) as f:
            stripped = (p.strip() for p in f.readlines())
            ignore = [x for x in stripped if x]
    except IOError:
        ignore = []

    dockerfile_name = os.path.basename(docker_path)

    if tag is None:
        return utils.tar(path, ignore, dockerfile=dockerfile_name)

    # docker-py 1.6+ won't ignore the dockerfile
    # passing dockerfile='' works around the issuen
    # and lets us add the modified file when we're done.
    ignore.append(dockerfile_name)
    fileobj = utils.tar(path, ignore, dockerfile='')

    t = tarfile.open(fileobj=fileobj, mode='a')
    dfinfo = tarfile.TarInfo(dockerfile_name)

    contents = tag_parent(tag, open(join(path, dockerfile_name)).read())
    if not isinstance(contents, bytes):
        contents = contents.encode('utf8')
    dockerfile = io.BytesIO(contents)

    dfinfo.size = len(dockerfile.getvalue())
    t.addfile(dfinfo, dockerfile)
    t.close()
    fileobj.seek(0)
    return fileobj
Example #20
0
def bundle_docker_dir(tag, docker_path):
    """
    Tars up a directory using the normal docker.util.tar method but
    adds tag if it's not None.
    """

    # tar up the directory minus the Dockerfile,

    path = os.path.dirname(docker_path)
    try:
        with open(join(path, '.dockerignore')) as f:
            stripped = (p.strip() for p in f.readlines())
            ignore = [x for x in stripped if x]
    except IOError:
        ignore = []

    dockerfile_name = os.path.basename(docker_path)

    if tag is None:
        return utils.tar(path, ignore, dockerfile=dockerfile_name)

    # docker-py 1.6+ won't ignore the dockerfile
    # passing dockerfile='' works around the issuen
    # and lets us add the modified file when we're done.
    ignore.append(dockerfile_name)
    fileobj = utils.tar(path, ignore, dockerfile='')

    t = tarfile.open(fileobj=fileobj, mode='a')
    dfinfo = tarfile.TarInfo(dockerfile_name)

    contents = tag_parent(tag, open(join(path, dockerfile_name)).read())
    if not isinstance(contents, bytes):
        contents = contents.encode('utf8')
    dockerfile = io.BytesIO(contents)

    dfinfo.size = len(dockerfile.getvalue())
    t.addfile(dfinfo, dockerfile)
    t.close()
    fileobj.seek(0)
    return fileobj
Example #21
0
 def test_tar_directory_link(self):
     dirs = ['a', 'b', 'a/c']
     files = ['a/hello.py', 'b/utils.py', 'a/c/descend.py']
     base = make_tree(dirs, files)
     self.addCleanup(shutil.rmtree, base)
     os.symlink(os.path.join(base, 'b'), os.path.join(base, 'a/c/b'))
     with tar(base) as archive:
         tar_data = tarfile.open(fileobj=archive)
         names = tar_data.getnames()
         for member in dirs + files:
             assert member in names
         assert 'a/c/b' in names
         assert 'a/c/b/utils.py' not in names
 def test_tar_directory_link(self):
     dirs = ['a', 'b', 'a/c']
     files = ['a/hello.py', 'b/utils.py', 'a/c/descend.py']
     base = make_tree(dirs, files)
     self.addCleanup(shutil.rmtree, base)
     os.symlink(os.path.join(base, 'b'), os.path.join(base, 'a/c/b'))
     with tar(base) as archive:
         tar_data = tarfile.open(fileobj=archive)
         names = tar_data.getnames()
         for member in dirs + files:
             assert member in names
         assert 'a/c/b' in names
         assert 'a/c/b/utils.py' not in names
Example #23
0
    def test_tar_with_excludes(self):
        dirs = [
            'foo',
            'foo/bar',
            'bar',
        ]

        files = [
            'Dockerfile',
            'Dockerfile.alt',
            '.dockerignore',
            'a.py',
            'a.go',
            'b.py',
            'cde.py',
            'foo/a.py',
            'foo/b.py',
            'foo/bar/a.py',
            'bar/a.py',
        ]

        exclude = [
            '*.py',
            '!b.py',
            '!a.go',
            'foo',
            'Dockerfile*',
            '.dockerignore',
        ]

        expected_names = set([
            'Dockerfile',
            '.dockerignore',
            'a.go',
            'b.py',
            'bar',
            'bar/a.py',
        ])

        base = make_tree(dirs, files)
        self.addCleanup(shutil.rmtree, base)

        with tar(base, exclude=exclude) as archive:
            tar_data = tarfile.open(fileobj=archive)
            assert sorted(tar_data.getnames()) == sorted(expected_names)
Example #24
0
    def test_tar_with_excludes(self):
        dirs = [
            'foo',
            'foo/bar',
            'bar',
        ]

        files = [
            'Dockerfile',
            'Dockerfile.alt',
            '.dockerignore',
            'a.py',
            'a.go',
            'b.py',
            'cde.py',
            'foo/a.py',
            'foo/b.py',
            'foo/bar/a.py',
            'bar/a.py',
        ]

        exclude = [
            '*.py',
            '!b.py',
            '!a.go',
            'foo',
            'Dockerfile*',
            '.dockerignore',
        ]

        expected_names = set([
            'Dockerfile',
            '.dockerignore',
            'a.go',
            'b.py',
            'bar',
            'bar/a.py',
        ])

        base = make_tree(dirs, files)
        self.addCleanup(shutil.rmtree, base)

        with tar(base, exclude=exclude) as archive:
            tar_data = tarfile.open(fileobj=archive)
            assert sorted(tar_data.getnames()) == sorted(expected_names)
Example #25
0
def bundle_docker_dir(modify_docker_func, path):
    """
    Tars up a directory using the normal docker.util.tar method but
    first relpaces the contents of the Dockerfile found at path with
    the result of calling modify_docker_func with a string containing
    the complete contents of the docker file.


    For example to prepend the phrase "bogus header"  to a dockerfile
    we first create a function that takes the contents of the current
    dockerfile as it's contents.

    >>> def append_bogus(docker_content):
    ...   return "bogus header " + docker_content

    Then we can tar it up.. but first we'll need some test content.
    These routines create a  temporary directory containing the following
    3 files.

    ./Dockerfile
    ./bogus1
    ./bogus2

    >>> from shipwright import tar
    >>> path = join(tar.TEST_ROOT, 'Dockerfile')
    >>> _ = open(path, 'w').write(u'blah')

    >>> _ = open(join(tar.TEST_ROOT, 'bogus1'),'w').write('hi mom')
    >>> _ = open(join(tar.TEST_ROOT, 'bogus2'), 'w').write('hello world')


    Now we can call bundle_docker_dir passing it our append_bogus function to
    mutate the docker contents. We'll receive a file like object which
    is stream of the contents encoded as a  tar file (the format Docker
    build expects)

    >>> fileobj = bundle_docker_dir(append_bogus, tar.TEST_ROOT)

    Normally we'd just pass this directly to the docker build command
    but for the purpose of this test, we'll use tarfile to decode the string
    and ensure that our mutation happened as planned.


    First lets ensure that our tarfile contains our test files

    >>> t = tarfile.open(fileobj=fileobj)
    >>> t.getnames()  # doctest: +SKIP
    ['bogus1', 'bogus2', 'Dockerfile']

    And if we exctart the Dockerfile it starts with 'bogus header'

    >>> ti = t.extractfile('Dockerfile')
    >>> ti.read().startswith(b'bogus header')
    True

    Obviously a real mutation would ensure that the the contents
    of the Dockerfile are valid docker commands and not some
    bogus content.
    """

    # tar up the directory minus the Dockerfile,
    # TODO: respect .dockerignore

    try:
        ignore = filter(None, [
            p.strip() for p in open(join(path, '.dockerignore')).readlines()
        ])
    except IOError:
        ignore = []

    # docker-py 1.6+ won't ignore the dockerfile
    # passing dockerfile='' works around the issuen
    # and lets us add the modified file when we're done.
    ignore.append('Dockerfile')
    fileobj = utils.tar(path, ignore, dockerfile='')

    # append a dockerfile after running it through a mutation
    # function first
    t = tarfile.open(fileobj=fileobj, mode='a')
    dfinfo = tarfile.TarInfo('Dockerfile')

    contents = modify_docker_func(open(join(path, 'Dockerfile')).read())
    if not isinstance(contents, bytes):
        contents = contents.encode('utf8')
    dockerfile = io.BytesIO(contents)

    dfinfo.size = len(dockerfile.getvalue())
    t.addfile(dfinfo, dockerfile)
    t.close()
    fileobj.seek(0)
    return fileobj
Example #26
0
def bundle_docker_dir(modify_docker_func, path):
    """
  Tars up a directory using the normal docker.util.tar method but
  first relpaces the contents of the Dockerfile found at path with
  the result of calling modify_docker_func with a string containing
  the complete contents of the docker file.


  For example to prepend the phrase "bogus header"  to a dockerfile
  we first create a function that takes the contents of the current
  dockerfile as it's contents. 

  >>> def append_bogus(docker_content):
  ...   return "bogus header " + docker_content

  Then we can tar it up.. but first we'll need some test content.
  These routines create a  temporary directory containing the following
  3 files.

  ./Dockerfile
  ./bogus1
  ./bogus2

  >>> from shipwright import tar 
  >>> path = join(tar.TEST_ROOT, 'Dockerfile')
  >>> open(path, 'w').write('blah')
  
  >>> open(join(tar.TEST_ROOT, 'bogus1'),'w').write('hi mom')
  >>> open(join(tar.TEST_ROOT, 'bogus2'), 'w').write('hello world')


  Now we can call bundle_docker_dir passing it our append_bogus function to
  mutate the docker contents. We'll receive a file like object which
  is stream of the contents encoded as a  tar file (the format Docker 
  build expects)

  >>> fileobj = bundle_docker_dir(append_bogus, tar.TEST_ROOT)

  Normally we'd just pass this directly to the docker build command
  but for the purpose of this test, we'll use trfile to decode the string
  and ensure that our mutation happened as planned.


  First lets ensure that our tarfile contains our test files

  >>> t = tarfile.open(fileobj=fileobj)
  >>> t.getnames()
  ['bogus1', 'bogus2', 'Dockerfile']

  And if we exctart the Dockerfile it starts with 'bogus header'

  >>> ti = t.extractfile('Dockerfile')
  >>> ti.read().startswith('bogus header')
  True
 
  Obviously a real mutation would ensure that the the contents 
  of the Dockerfile are valid docker commands and not some
  bogus content.
  
  """

    # tar up the directory minus the Dockerfile,
    # TODO: respect .dockerignore

    try:
        ignore = filter(
            None,
            [p.strip() for p in open(join(path, '.dockerignore')).readlines()])
    except IOError:
        ignore = []

    ignore.append('Dockerfile')

    fileobj = utils.tar(path, ignore)

    # append a dockerfile after running it through a mutation
    # function first
    t = tarfile.open(fileobj=fileobj, mode='a')
    dfinfo = tarfile.TarInfo('Dockerfile')

    dockerfile = io.BytesIO(
        modify_docker_func(open(join(path, 'Dockerfile')).read()))

    dfinfo.size = len(dockerfile.getvalue())
    t.addfile(dfinfo, dockerfile)
    t.close()
    fileobj.seek(0)
    return fileobj