def checkout(self, url, version='', verbose=False,
shallow=False, timeout=None):
"""
untars tar at url to self.path.
If version was given, only the subdirectory 'version' of the
tar will end up in self.path. Also creates a file next to the
checkout named *.tar which is a yaml file listing origin url
and version arguments.
"""
if not ensure_dir_notexists(self.get_path()):
self.logger.error("Can't remove %s" % self.get_path())
return False
tempdir = None
result = False
try:
tempdir = tempfile.mkdtemp()
if os.path.isfile(url):
filename = url
else:
(filename, _) = urlretrieve_netrc(url)
# print "filename", filename
temp_tarfile = tarfile.open(filename, 'r:*')
members = None # means all members in extractall
if version == '' or version is None:
subdir = tempdir
self.logger.warn("No tar subdirectory chosen via the 'version' argument for url: %s" % url)
else:
# getmembers lists all files contained in tar with
# relative path
subdirs = []
members = []
for m in temp_tarfile.getmembers():
if m.name.startswith(version + '/'):
members.append(m)
if m.name.split('/')[0] not in subdirs:
subdirs.append(m.name.split('/')[0])
if not members:
raise VcsError("%s is not a subdirectory with contents in members %s" % (version, subdirs))
subdir = os.path.join(tempdir, version)
temp_tarfile.extractall(path=tempdir, members=members)
if not os.path.isdir(subdir):
raise VcsError("%s is not a subdirectory\n" % subdir)
try:
# os.makedirs(os.path.dirname(self._path))
shutil.move(subdir, self._path)
except Exception as ex:
raise VcsError("%s failed to move %s to %s" % (ex, subdir, self._path))
metadata = yaml.dump({'url': url, 'version': version})
with open(self.metadata_path, 'w') as mdat:
mdat.write(metadata)
result = True
except Exception as exc:
self.logger.error("Tarball download unpack failed: %s" % str(exc))
finally:
if tempdir is not None and os.path.exists(tempdir):
rmtree(tempdir)
return result
def test_urlretrieve_netrc(self):
root_directory = tempfile.mkdtemp()
examplename = os.path.join(root_directory, "foo")
outname = os.path.join(root_directory, "fooout")
with open(examplename, "w") as fhand:
fhand.write('content')
mockget = Mock()
mockopen = Mock()
mock_fhand = Mock()
backopen = vcstools.common.urlopen
backget = vcstools.common._netrc_open
try:
# vcstools.common.urlopen = mockopen
# vcstools.common.urlopen.return_value = mock_fhand
# mock_fhand.read.return_value = 'content'
mockopen.open.return_value
vcstools.common._netrc_open = Mock()
vcstools.common._netrc_open.return_value = mockget
(fname, headers) = urlretrieve_netrc(_get_file_uri(examplename))
self.assertTrue(fname)
self.assertFalse(os.path.exists(outname))
(fname, headers) = urlretrieve_netrc(_get_file_uri(examplename),
outname)
self.assertEqual(outname, fname)
self.assertTrue(os.path.isfile(outname))
finally:
vcstools.common.urlopen = backopen
vcstools.common._netrc_open = backget
rmtree(root_directory)
def test_netrc_open(self):
root_directory = tempfile.mkdtemp()
machine = 'foo.org'
uri = 'https://%s/bim/bam' % machine
netrcname = os.path.join(root_directory, "netrc")
mock_build_opener = Mock()
mock_build_opener_fun = Mock()
mock_build_opener_fun.return_value = mock_build_opener
back_build_opener = vcstools.common.build_opener
try:
vcstools.common.build_opener = mock_build_opener_fun
filelike = _netrc_open(uri, netrcname)
self.assertFalse(filelike)
with open(netrcname, 'w') as fhand:
fhand.write('machine %s login fooname password foopass' %
machine)
filelike = _netrc_open(uri, netrcname)
self.assertTrue(filelike)
filelike = _netrc_open('other', netrcname)
self.assertFalse(filelike)
filelike = _netrc_open(None, netrcname)
self.assertFalse(filelike)
finally:
rmtree(root_directory)
vcstools.common.build_opener = back_build_opener
def export_repository(self, version, basepath):
# Run the svn export cmd
cmd = 'svn export {0} {1}'.format(os.path.join(self._path, version),
basepath)
result, _, _ = run_shell_command(cmd, shell=True)
if result:
return False
try:
# tar gzip the exported repo
targzip_file = tarfile.open(basepath + '.tar.gz', 'w:gz')
try:
targzip_file.add(basepath, '')
finally:
targzip_file.close()
finally:
# clean up
rmtree(basepath)
return True
def export_repository(self, version, basepath):
if not self.detect_presence():
return False
# Export should work regardless of the state of the local dir (might have modified files),
# so we clone to a temporary folder and checkout the specified version there.
# If the repo has submodules with relative URLs, cloning to a temp dir doesn't work.
# To support this case, first check if current version is already checked out and there are no
# modifications and try to export from current local dir.
# see https://github.com/vcstools/vcstools/pull/130
current_sha = self.get_version()
export_sha = self.get_version(version)
if current_sha == export_sha and self.get_diff() == '':
archiver = GitArchiver(main_repo_abspath=self.get_path(),
force_sub=True)
filepath = '{0}.tar.gz'.format(basepath)
archiver.create(filepath)
return filepath
# export a different version than currently checked out or there are local changes
try:
# since version may relate to remote branch / tag we do not
# know about yet, do fetch if not already done
self._do_fetch()
# export should work regardless of the state of the local dir (might have modified files)
# so we clone to a temporary folder and checkout the specified version there
tmpd_path = tempfile.mkdtemp()
try:
tmpgit = GitClient(tmpd_path)
if tmpgit.checkout(self._path, version=version, shallow=True):
archiver = GitArchiver(main_repo_abspath=tmpgit.get_path(),
force_sub=True)
filepath = '{0}.tar.gz'.format(basepath)
archiver.create(filepath)
return filepath
else:
return False
finally:
rmtree(tmpd_path)
except GitError:
return False
def tarfile_contents():
'''
:returns: binary string of ROS package-release-like tarfile to serve
'''
tar_directory = tempfile.mkdtemp()
filename = os.path.join(tar_directory, 'sample.tar.gz')
pkgname = "sample-1.0"
pkg_directory = os.path.join(tar_directory, pkgname)
os.mkdir(pkg_directory)
packagexml = os.path.join(pkg_directory, "package.xml")
with open(packagexml, 'w+') as f:
f.write('<?xml version="1.0"?><package>' +
('<name>sample</name>' * 1000) + '</package>')
with tarfile.open(filename, "w:gz") as tar_handle:
tar_handle.addfile(
tarfile.TarInfo(os.path.join(pkgname, "package.xml")), packagexml)
tar_handle.close()
with open(filename, mode='rb') as file: # b is important -> binary
result = file.read()
rmtree(tar_directory)
return result
def tearDown(self):
if os.path.exists(self.local_path):
rmtree(self.local_path)
if os.path.exists(self.export_path):
rmtree(self.export_path)
def tearDownClass(self):
for d in self.directories:
rmtree(self.directories[d])
def tearDown(self):
for d in self.directories:
self.assertTrue(os.path.exists(self.directories[d]))
rmtree(self.directories[d])
self.assertFalse(os.path.exists(self.directories[d]))
