def deploy_ve(self, target, repository, app_version):
"""Prepare the deploy virtualenv.
Unpack a virtualenv for the deploy hooks, and return its location on
the FS.
"""
deploy_req_fn = os.path.join(self.appdir, 'versions', app_version,
'deploy', 'requirements.txt')
python_version = virtualenv.get_python_version(self.compat)
platform = self.settings.artifacts.platform
ve_id = virtualenv.get_id(deploy_req_fn, python_version, platform)
ves_dir = os.path.join(self.settings.paths.apps, 'deploy',
'virtualenvs')
ve_dir = os.path.join(ves_dir, ve_id)
if os.path.exists(ve_dir):
return ve_dir
ve_working = os.path.join(ves_dir, 'unpack')
if not os.path.exists(ve_working):
os.makedirs(ve_working)
ve_unpack_root = os.path.join(ve_working, 'virtualenv')
tarball = os.path.join(ve_working, 'virtualenv.tar.gz')
with SpinLockFile(os.path.join(ves_dir, 'deploy.lock'), timeout=30):
log.debug('Deploying hook virtualenv %s', ve_id)
virtualenv.download_ve(
repository, 'deploy', ve_id, target, tarball)
extract_tar(tarball, ve_unpack_root)
if os.path.exists(ve_dir):
shutil.rmtree(ve_dir)
os.rename(ve_unpack_root, ve_dir)
return ve_dir
def unpack(self, target, repository, version):
"""First stage of deployment"""
assert self.lock.held
log.debug('Unpacking %s/%s', self.app, version)
if self.live_version == version:
log.warn('%s/%s is the currently live version'
% (self.app, version))
return
unpack_dir = os.path.join(self.appdir, 'versions', 'unpack')
if not os.path.isdir(unpack_dir):
os.makedirs(unpack_dir)
tarball = os.path.join(unpack_dir, '%s.tar.gz' % self.app)
with repository.get(self.app, version, target) as f1:
self.compat = int(f1.metadata.get('deploy_compat', 1))
if self.compat not in (4, 5):
raise Exception('Unsupported artifact: compat level %s'
% self.compat)
with open(tarball, 'wb') as f2:
shutil.copyfileobj(f1, f2)
extract_tar(tarball, os.path.join(unpack_dir, version))
os.unlink(tarball)
staging = os.path.join(self.appdir, 'versions', version)
if os.path.isdir(staging):
shutil.rmtree(staging)
os.rename(os.path.join(self.appdir, 'versions', 'unpack', version),
staging)
def deploy_ve(self, target, repository, version):
'''Unpack a virtualenv for the deploy hooks, and return its location
on the FS
'''
deploy_req_fn = os.path.join(self.appdir, 'versions', version,
'deploy', 'requirements.txt')
ve_hash = yodeploy.virtualenv.sha224sum(deploy_req_fn)
ve_hash = yodeploy.virtualenv.ve_version(ve_hash)
ves_dir = os.path.join(self.settings.paths.apps, 'deploy',
'virtualenvs')
ve_dir = os.path.join(ves_dir, ve_hash)
if os.path.exists(ve_dir):
return ve_dir
ve_working = os.path.join(ves_dir, 'unpack')
if not os.path.exists(ve_working):
os.makedirs(ve_working)
ve_unpack_root = os.path.join(ve_working, 'virtualenv')
tarball = os.path.join(ve_working, 'virtualenv.tar.gz')
with SpinLockFile(os.path.join(ves_dir, 'deploy.lock'), timeout=30):
log.debug('Deploying hook virtualenv %s', ve_hash)
yodeploy.virtualenv.download_ve(repository, 'deploy', ve_hash,
target, tarball)
extract_tar(tarball, ve_unpack_root)
if os.path.exists(ve_dir):
shutil.rmtree(ve_dir)
os.rename(ve_unpack_root, ve_dir)
return ve_dir
def deploy_ve(self):
log = logging.getLogger(__name__)
ve_hash = ve_version(sha224sum(self.deploy_path('requirements.txt')))
ve_working = os.path.join(self.root, 'virtualenvs', 'unpack')
ve_dir = os.path.join(self.root, 'virtualenvs', ve_hash)
tarball = os.path.join(ve_working, 'virtualenv.tar.gz')
ve_unpack_root = os.path.join(ve_working, 'virtualenv')
if not os.path.exists(ve_dir):
log.debug('Deploying virtualenv %s', ve_hash)
if not os.path.exists(ve_working):
os.makedirs(ve_working)
download_ve(self.repository, self.app, ve_hash, self.target,
dest=tarball)
extract_tar(tarball, ve_unpack_root)
os.rename(ve_unpack_root, ve_dir)
ve_symlink = self.deploy_path('virtualenv')
if not os.path.exists(ve_symlink):
os.symlink(os.path.join('..', '..', 'virtualenvs', ve_hash),
ve_symlink)
def deploy_ve(self):
log = logging.getLogger(__name__)
ve_id = virtualenv.get_id(self.deploy_path('requirements.txt'),
sysconfig.get_python_version(),
self.settings.artifacts.platform)
ve_working = os.path.join(self.root, 'virtualenvs', 'unpack')
ve_dir = os.path.join(self.root, 'virtualenvs', ve_id)
tarball = os.path.join(ve_working, 'virtualenv.tar.gz')
ve_unpack_root = os.path.join(ve_working, 'virtualenv')
if not os.path.exists(ve_dir):
log.debug('Deploying virtualenv %s', ve_id)
if not os.path.exists(ve_working):
os.makedirs(ve_working)
virtualenv.download_ve(
self.repository, self.app, ve_id, self.target, dest=tarball)
extract_tar(tarball, ve_unpack_root)
os.rename(ve_unpack_root, ve_dir)
ve_symlink = self.deploy_path('virtualenv')
if not os.path.exists(ve_symlink):
os.symlink(os.path.join('..', '..', 'virtualenvs', ve_id),
ve_symlink)
def write_config(self):
conf_root = os.path.join(self.settings.paths.apps, 'configs')
if not os.path.exists(conf_root):
os.mkdir(conf_root)
conf_tarball = os.path.join(conf_root, 'configs.tar.gz')
with SpinLockFile(os.path.join(conf_root, 'deploy.lock'), timeout=30):
try:
with self.repository.get('configs', target='master') as f1:
with open(conf_tarball, 'w') as f2:
shutil.copyfileobj(f1, f2)
except KeyError:
raise Exception("No configs in artifacts repository")
configs = os.path.join(conf_root, 'configs')
if os.path.exists(configs):
shutil.rmtree(configs)
extract_tar(conf_tarball, configs)
os.unlink(conf_tarball)
configs_dirs = [configs] + self.settings.deployconfigs.overrides
app_conf_dir = self.deploy_path('deploy', 'configuration')
sources = config_sources(self.app,
self.settings.artifacts.environment,
self.settings.artifacts.cluster,
configs_dirs, app_conf_dir)
config = smush_config(
sources, initial={'yoconfigurator': {'app': self.app}})
public_filter_pn = os.path.join(app_conf_dir, 'public-data.py')
public_config = filter_config(config, public_filter_pn)
write_config(config, self.deploy_dir)
if public_config:
write_config(
public_config, self.deploy_dir, 'configuration_public.json')
def test_simple(self):
self.create_tar('test.tar.gz', 'foo/bar', 'foo/baz')
extract_tar(self.tmppath('test.tar.gz'), self.tmppath('extracted'))
self.assertTMPPExists('extracted')
self.assertTMPPExists('extracted/bar')
self.assertTMPPExists('extracted/baz')
import os import sys from yodeploy.util import extract_tar script, extract_target, dest, stat_path = sys.argv extract_tar(extract_target, dest) s = os.stat(stat_path) assert s.st_uid == 0 assert s.st_gid == 0
