def test_tar_stream(self):
"""Test the stream mode of fs.tar
"""
# Create directories and files to test tarring
# /self.root/apps/testapp/tardir
# /self.root/apps/testapp/tardir/file
# /self.root/apps/testapp/tardir/subdir
# /self.root/apps/testapp/tardir2
testapp_dir = os.path.join(self.root, 'testapp')
tardir = os.path.join(testapp_dir, 'tardir')
tardir2 = os.path.join(testapp_dir, 'tardir2')
os.makedirs(testapp_dir)
os.mkdir(tardir)
os.mkdir(os.path.join(tardir, 'subdir'))
os.mkdir(tardir2)
with open(os.path.join(tardir, 'file'), 'w+'):
pass
with open(os.path.join(tardir2, 'file2'), 'w+'):
pass
fileobj = tempfile.TemporaryFile()
fs.tar(target=fileobj, sources=[tardir, tardir2], compression='gzip')
# seek fileobj and check content
fileobj.seek(0)
tarfileobj = tarfile.open(mode='r:gz', fileobj=fileobj)
names = tarfileobj.getnames()
self.assertTrue('subdir' in names and 'file' in names
and 'file2' in names)
def test_tar_basic(self):
"""Tests the fs.tar function.
"""
# Create directories and files to test tarring
# /self.root/apps/testapp/tardir
# /self.root/apps/testapp/tardir/file
# /self.root/apps/testapp/tardir/subdir
# /self.root/apps/testapp/tardir2
# Archive:
# /self.root/apps/testapp/tar.tar
testapp_dir = os.path.join(self.root, 'testapp')
tardir = os.path.join(testapp_dir, 'tardir')
tardir2 = os.path.join(testapp_dir, 'tardir2')
archive = os.path.join(testapp_dir, 'foo.tar')
os.makedirs(testapp_dir)
os.mkdir(tardir)
os.mkdir(os.path.join(tardir, 'subdir'))
os.mkdir(tardir2)
with open(os.path.join(tardir, 'file'), 'w+'):
pass
self.assertEquals(
fs.tar(archive, tardir).name, archive, 'fs.tar runs successfully')
self.assertTrue(os.path.isfile(archive), 'fs.tar creates a tarfile')
self.assertEquals(
fs.tar(archive, tardir2).name, archive,
'fs.tar will succeed if tarfile already exists')
self.assertEquals(
fs.tar(archive, tardir, compression='gzip').name,
"%s.gz" % archive, 'fs.tar with gzip runs successfully')
self.assertTrue(os.path.isfile('%s.gz' % archive),
'fs.tar creates a tar gzip file')
def test_tar_files(self):
"""Tests the glob/transform of files in fs.tar.
"""
# Create directories and files to test tarring
# /self.root/apps/testapp/tardir
# /self.root/apps/testapp/tardir/file
# /self.root/apps/testapp/tardir/subdir
# /self.root/apps/testapp/tardir2
# Archive:
# /self.root/apps/testapp/tar.tar
testapp_dir = os.path.join(self.root, 'testapp')
tardir = os.path.join(testapp_dir, 'tardir')
tardir2 = os.path.join(testapp_dir, 'tardir2')
archive = os.path.join(testapp_dir, 'foo.tar')
os.makedirs(testapp_dir)
os.mkdir(tardir)
os.mkdir(os.path.join(tardir, 'subdir'))
os.mkdir(tardir2)
with open(os.path.join(tardir, 'file'), 'w+'):
pass
self.assertEquals(
fs.tar(archive, [tardir, tardir2], compression='gzip').name,
'%s.gz' % archive, 'fs.tar runs successfully')
tarfile.TarFile.add.assert_any_call(tardir, '/')
tarfile.TarFile.add.assert_any_call(tardir2, '/')
def collect(approot, archive_filename):
"""Collect node information in case of blackout.
:param approot:
treadmill root, usually /var/tmp/treadmill
:type approot:
``str``
:param archive_filename:
archive path file
:type archive_filename:
``str``
"""
destroot = tempfile.mkdtemp()
_LOGGER.info('save node info in %s', destroot)
collect_init_services(approot, destroot)
collect_running_app(approot, destroot)
if os.name == 'posix':
collect_sysctl(destroot)
collect_cgroup(approot, destroot)
collect_localdisk(approot, destroot)
collect_network(approot, destroot)
collect_message(destroot)
try:
archive_filename = fs.tar(sources=destroot,
target=archive_filename,
compression='gzip').name
_LOGGER.info('node info archive file: %s', archive_filename)
shutil.rmtree(destroot)
return archive_filename
except Exception: # pylint: disable=W0703
# if tar bar is not generated successfully, we keep destroot
# we can find destroot path in log to check the files
_LOGGER.exception('Failed to generate node info archive')
return None
def _cleanup(tm_env, zkclient, container_dir, app):
"""Cleanup a container that actually ran.
"""
# Too many branches.
#
# pylint: disable=R0912
rootdir = os.path.join(container_dir, 'root')
# Generate a unique name for the app
unique_name = appmgr.app_unique_name(app)
# Create service clients
cgroup_client = tm_env.svc_cgroup.make_client(
os.path.join(container_dir, 'cgroups'))
localdisk_client = tm_env.svc_localdisk.make_client(
os.path.join(container_dir, 'localdisk'))
network_client = tm_env.svc_network.make_client(
os.path.join(container_dir, 'network'))
# Make sure all processes are killed
# FIXME(boysson): Should we use `kill_apps_in_cgroup` instead?
_kill_apps_by_root(rootdir)
# Setup the archive filename that will hold this container's data
filetime = utils.datetime_utcnow().strftime('%Y%m%d_%H%M%S%f')
archive_filename = os.path.join(
container_dir, '{instance_name}_{hostname}_{timestamp}.tar'.format(
instance_name=appmgr.appname_task_id(app.name),
hostname=sysinfo.hostname(),
timestamp=filetime))
# Tar up container root filesystem if archive list is in manifest
try:
localdisk = localdisk_client.get(unique_name)
fs.archive_filesystem(localdisk['block_dev'], rootdir,
archive_filename, app.archive)
except services.ResourceServiceError:
_LOGGER.warning('localdisk never allocated')
except subprocess.CalledProcessError:
_LOGGER.exception('Unable to archive root device of %r', unique_name)
except: # pylint: disable=W0702
_LOGGER.exception('Unknown exception while archiving %r', unique_name)
# Destroy the volume
try:
localdisk = localdisk_client.delete(unique_name)
except (IOError, OSError) as err:
if err.errno == errno.ENOENT:
pass
else:
raise
if not app.shared_network:
_cleanup_network(tm_env, app, network_client)
# Add metrics to archive
rrd_file = os.path.join(
tm_env.metrics_dir, 'apps',
'{name}-{instanceid}-{uniqueid}.rrd'.format(
name=app.app,
instanceid=app.task,
uniqueid=app.uniqueid,
))
rrdutils.flush_noexc(rrd_file)
_copy_metrics(rrd_file, container_dir)
# Cleanup our cgroup resources
try:
cgroup_client.delete(unique_name)
except (IOError, OSError) as err:
if err.errno == errno.ENOENT:
pass
else:
raise
try:
_archive_logs(tm_env, container_dir)
except Exception: # pylint: disable=W0703
_LOGGER.exception('Unexpected exception storing local logs.')
# Append or create the tarball with folders outside of container
# Compress and send the tarball to HCP
try:
archive_filename = fs.tar(sources=container_dir,
target=archive_filename,
compression='gzip').name
_send_container_archive(zkclient, app, archive_filename)
except: # pylint: disable=W0702
_LOGGER.exception("Failed to update archive")