def deploy_ve(self, target, repository, app_version): """Prepare the deploy virtualenv. Unpack a virtualenv for the deploy hooks, and return its location on the FS. """ deploy_req_fn = os.path.join(self.appdir, 'versions', app_version, 'deploy', 'requirements.txt') python_version = virtualenv.get_python_version(self.compat) platform = self.settings.artifacts.platform ve_id = virtualenv.get_id(deploy_req_fn, python_version, platform) ves_dir = os.path.join(self.settings.paths.apps, 'deploy', 'virtualenvs') ve_dir = os.path.join(ves_dir, ve_id) if os.path.exists(ve_dir): return ve_dir ve_working = os.path.join(ves_dir, 'unpack') if not os.path.exists(ve_working): os.makedirs(ve_working) ve_unpack_root = os.path.join(ve_working, 'virtualenv') tarball = os.path.join(ve_working, 'virtualenv.tar.gz') with SpinLockFile(os.path.join(ves_dir, 'deploy.lock'), timeout=30): log.debug('Deploying hook virtualenv %s', ve_id) virtualenv.download_ve( repository, 'deploy', ve_id, target, tarball) extract_tar(tarball, ve_unpack_root) if os.path.exists(ve_dir): shutil.rmtree(ve_dir) os.rename(ve_unpack_root, ve_dir) return ve_dir
def unpack(self, target, repository, version): """First stage of deployment""" assert self.lock.held log.debug('Unpacking %s/%s', self.app, version) if self.live_version == version: log.warn('%s/%s is the currently live version' % (self.app, version)) return unpack_dir = os.path.join(self.appdir, 'versions', 'unpack') if not os.path.isdir(unpack_dir): os.makedirs(unpack_dir) tarball = os.path.join(unpack_dir, '%s.tar.gz' % self.app) with repository.get(self.app, version, target) as f1: self.compat = int(f1.metadata.get('deploy_compat', 1)) if self.compat not in (4, 5): raise Exception('Unsupported artifact: compat level %s' % self.compat) with open(tarball, 'wb') as f2: shutil.copyfileobj(f1, f2) extract_tar(tarball, os.path.join(unpack_dir, version)) os.unlink(tarball) staging = os.path.join(self.appdir, 'versions', version) if os.path.isdir(staging): shutil.rmtree(staging) os.rename(os.path.join(self.appdir, 'versions', 'unpack', version), staging)
def deploy_ve(self, target, repository, version): '''Unpack a virtualenv for the deploy hooks, and return its location on the FS ''' deploy_req_fn = os.path.join(self.appdir, 'versions', version, 'deploy', 'requirements.txt') ve_hash = yodeploy.virtualenv.sha224sum(deploy_req_fn) ve_hash = yodeploy.virtualenv.ve_version(ve_hash) ves_dir = os.path.join(self.settings.paths.apps, 'deploy', 'virtualenvs') ve_dir = os.path.join(ves_dir, ve_hash) if os.path.exists(ve_dir): return ve_dir ve_working = os.path.join(ves_dir, 'unpack') if not os.path.exists(ve_working): os.makedirs(ve_working) ve_unpack_root = os.path.join(ve_working, 'virtualenv') tarball = os.path.join(ve_working, 'virtualenv.tar.gz') with SpinLockFile(os.path.join(ves_dir, 'deploy.lock'), timeout=30): log.debug('Deploying hook virtualenv %s', ve_hash) yodeploy.virtualenv.download_ve(repository, 'deploy', ve_hash, target, tarball) extract_tar(tarball, ve_unpack_root) if os.path.exists(ve_dir): shutil.rmtree(ve_dir) os.rename(ve_unpack_root, ve_dir) return ve_dir
def deploy_ve(self): log = logging.getLogger(__name__) ve_hash = ve_version(sha224sum(self.deploy_path('requirements.txt'))) ve_working = os.path.join(self.root, 'virtualenvs', 'unpack') ve_dir = os.path.join(self.root, 'virtualenvs', ve_hash) tarball = os.path.join(ve_working, 'virtualenv.tar.gz') ve_unpack_root = os.path.join(ve_working, 'virtualenv') if not os.path.exists(ve_dir): log.debug('Deploying virtualenv %s', ve_hash) if not os.path.exists(ve_working): os.makedirs(ve_working) download_ve(self.repository, self.app, ve_hash, self.target, dest=tarball) extract_tar(tarball, ve_unpack_root) os.rename(ve_unpack_root, ve_dir) ve_symlink = self.deploy_path('virtualenv') if not os.path.exists(ve_symlink): os.symlink(os.path.join('..', '..', 'virtualenvs', ve_hash), ve_symlink)
def deploy_ve(self): log = logging.getLogger(__name__) ve_id = virtualenv.get_id(self.deploy_path('requirements.txt'), sysconfig.get_python_version(), self.settings.artifacts.platform) ve_working = os.path.join(self.root, 'virtualenvs', 'unpack') ve_dir = os.path.join(self.root, 'virtualenvs', ve_id) tarball = os.path.join(ve_working, 'virtualenv.tar.gz') ve_unpack_root = os.path.join(ve_working, 'virtualenv') if not os.path.exists(ve_dir): log.debug('Deploying virtualenv %s', ve_id) if not os.path.exists(ve_working): os.makedirs(ve_working) virtualenv.download_ve( self.repository, self.app, ve_id, self.target, dest=tarball) extract_tar(tarball, ve_unpack_root) os.rename(ve_unpack_root, ve_dir) ve_symlink = self.deploy_path('virtualenv') if not os.path.exists(ve_symlink): os.symlink(os.path.join('..', '..', 'virtualenvs', ve_id), ve_symlink)
def write_config(self): conf_root = os.path.join(self.settings.paths.apps, 'configs') if not os.path.exists(conf_root): os.mkdir(conf_root) conf_tarball = os.path.join(conf_root, 'configs.tar.gz') with SpinLockFile(os.path.join(conf_root, 'deploy.lock'), timeout=30): try: with self.repository.get('configs', target='master') as f1: with open(conf_tarball, 'w') as f2: shutil.copyfileobj(f1, f2) except KeyError: raise Exception("No configs in artifacts repository") configs = os.path.join(conf_root, 'configs') if os.path.exists(configs): shutil.rmtree(configs) extract_tar(conf_tarball, configs) os.unlink(conf_tarball) configs_dirs = [configs] + self.settings.deployconfigs.overrides app_conf_dir = self.deploy_path('deploy', 'configuration') sources = config_sources(self.app, self.settings.artifacts.environment, self.settings.artifacts.cluster, configs_dirs, app_conf_dir) config = smush_config( sources, initial={'yoconfigurator': {'app': self.app}}) public_filter_pn = os.path.join(app_conf_dir, 'public-data.py') public_config = filter_config(config, public_filter_pn) write_config(config, self.deploy_dir) if public_config: write_config( public_config, self.deploy_dir, 'configuration_public.json')
def test_simple(self): self.create_tar('test.tar.gz', 'foo/bar', 'foo/baz') extract_tar(self.tmppath('test.tar.gz'), self.tmppath('extracted')) self.assertTMPPExists('extracted') self.assertTMPPExists('extracted/bar') self.assertTMPPExists('extracted/baz')
import os import sys from yodeploy.util import extract_tar script, extract_target, dest, stat_path = sys.argv extract_tar(extract_target, dest) s = os.stat(stat_path) assert s.st_uid == 0 assert s.st_gid == 0