def copySSHKeys(self, config, **kwargs):
if 'ssh' not in config['needs']:
return
key_file = configuration.getSettings('dockerKeyFile')
authorized_keys_file = configuration.getSettings('dockerAuthorizedKeyFile')
known_hosts_file = configuration.getSettings('dockerKnownHostsFile')
with cd(config['rootFolder']), hide('commands', 'output'), lcd(configuration.getBaseDir()):
run('mkdir -p /root/.ssh')
if key_file:
put(key_file, '/root/.ssh/id_rsa')
put(key_file+'.pub', '/root/.ssh/id_rsa.pub')
run('chmod 600 /root/.ssh/id_rsa')
run('chmod 644 /root/.ssh/id_rsa.pub')
put(key_file+'.pub', '/tmp')
run('cat /tmp/'+os.path.basename(key_file)+'.pub >> /root/.ssh/authorized_keys')
run('rm /tmp/'+os.path.basename(key_file)+'.pub')
print green('Copied keyfile to docker.')
if authorized_keys_file:
put(authorized_keys_file, '/root/.ssh/authorized_keys')
print green('Copied authorized keys to docker.')
if known_hosts_file:
put(known_hosts_file, '/root/.ssh/known_hosts')
print green('Copied known hosts to docker.')
run('chmod 700 /root/.ssh')
def copySSHKeys(self, config, **kwargs):
if 'ssh' not in config['needs']:
return
key_file = configuration.getSettings('dockerKeyFile')
authorized_keys_file = configuration.getSettings('dockerAuthorizedKeyFile')
known_hosts_file = configuration.getSettings('dockerKnownHostsFile')
with cd(config['rootFolder']), hide('commands', 'output'), lcd(configuration.getBaseDir()):
run('mkdir -p /root/.ssh')
if key_file:
put(key_file, '/root/.ssh/id_rsa')
put(key_file+'.pub', '/root/.ssh/id_rsa.pub')
run('chmod 600 /root/.ssh/id_rsa')
run('chmod 644 /root/.ssh/id_rsa.pub')
put(key_file+'.pub', '/tmp')
run('cat /tmp/'+os.path.basename(key_file)+'.pub >> /root/.ssh/authorized_keys')
run('rm /tmp/'+os.path.basename(key_file)+'.pub')
print green('Copied keyfile to docker.')
if authorized_keys_file:
put(authorized_keys_file, '/root/.ssh/authorized_keys')
print green('Copied authorized keys to docker.')
if known_hosts_file:
put(known_hosts_file, '/root/.ssh/known_hosts')
print green('Copied known hosts to docker.')
run('chmod 700 /root/.ssh')
def install(self, config, ask='True', distribution='minimal', **kwargs):
self.setRunLocally(config)
if 'database' not in config:
print red('Missing database configuration!')
exit(1)
configuration.validate_dict(['user', 'pass', 'name', 'host'],
config['database'],
'Missing database configuration: ')
print green('Installing fresh database for "%s"' %
config['config_name'])
o = config['database']
with self.cd(config['siteFolder']):
self.run_quietly('mkdir -p %s' % config['siteFolder'])
mysql_cmd = 'CREATE DATABASE IF NOT EXISTS {name}; GRANT ALL PRIVILEGES ON {name}.* TO \'{user}\'@\'%\' IDENTIFIED BY \'{pass}\'; FLUSH PRIVILEGES;'.format(
**o)
self.run_quietly(
'mysql -h {host} -u {user} --password={pass} -e "{mysql_command}"'
.format(mysql_command=mysql_cmd, **o), 'Creating database')
with warn_only():
self.run_quietly('chmod u+w {siteFolder}'.format(**config))
self.run_quietly(
'chmod u+w {siteFolder}/settings.php'.format(**config))
self.run_quietly(
'rm -f {siteFolder}/settings.php.old'.format(**config))
self.run_quietly(
'mv {siteFolder}/settings.php {siteFolder}/settings.php.old 2>/dev/null'
.format(**config))
sites_folder = os.path.basename(config['siteFolder'])
options = ''
if ask.lower() == 'false' or ask.lower() == '0':
options = ' -y'
options += ' --sites-subdir=' + sites_folder
options += ' --account-name=%s' % configuration.getSettings(
'adminUser', 'admin')
options += ' --account-pass=admin'
if 'prefix' in o:
options += " --db-prefix='%s'" % o['prefix']
options += ' --db-url=mysql://' + o['user'] + ':' + o[
'pass'] + '@' + o['host'] + '/' + o['name']
self.run_drush('site-install ' + distribution + ' ' + options)
if self.methodName == 'drush7':
self.run_drush('en features -y')
deploymentModule = configuration.getSettings(
'deploymentModule')
if deploymentModule:
self.run_drush('en -y %s' % deploymentModule)
if self.methodName == 'drush8':
self.setupConfigurationManagement(config)
def reset(self, config, **kwargs):
self.setRunLocally(config)
if 'withPasswordReset' not in kwargs:
kwargs['withPasswordReset'] = True
if self.methodName == 'drush8':
uuid = config['uuid'] if 'uuid' in config else False
if not uuid:
uuid = configuration.getSettings('uuid')
if not uuid:
print red(
'No uuid found in fabfile.yaml. config-import may fail!')
with self.cd(config['siteFolder']):
if config['type'] == 'dev':
admin_user = configuration.getSettings('adminUser', 'admin')
if 'withPasswordReset' in kwargs and kwargs[
'withPasswordReset'] in [True, 'True', '1']:
self.run_drush('user-password %s --password="******"' %
admin_user)
with warn_only():
self.run_quietly('chmod -R 777 ' + config['filesFolder'])
with warn_only():
if configuration.getSettings('deploymentModule'):
self.run_drush(
'en -y ' +
configuration.getSettings('deploymentModule'))
if self.methodName == 'drush8':
self.run_drush('updb --entity-updates -y')
else:
self.run_drush('updb -y')
script_fn = self.factory.get('script', 'runScript')
with warn_only():
if self.methodName == 'drush8':
if uuid:
self.run_drush('cset system.site uuid %s -y' % uuid)
if 'configurationManagement' in config:
for key, cmds in config[
'configurationManagement'].iteritems():
script_fn(config,
script=cmds,
rootFolder=config['siteFolder'])
else:
self.run_drush('fra -y')
fn = self.factory.get('script', 'runTaskSpecificScript')
fn('reset', config, **kwargs)
if self.methodName == 'drush8':
self.run_drush('cr')
else:
self.run_drush(' cc all')
def reset(self, config, **kwargs):
self.setRunLocally(config)
if 'withPasswordReset' not in kwargs:
kwargs['withPasswordReset'] = True
if self.methodName == 'drush8':
uuid = config['uuid'] if 'uuid' in config else False
if not uuid:
uuid = configuration.getSettings('uuid')
if not uuid:
log.error('No uuid found in fabfile.yaml. config-import may fail!')
with self.cd(config['siteFolder']):
if config['type'] == 'dev':
admin_user = config['adminUser']
if 'withPasswordReset' in kwargs and kwargs['withPasswordReset'] in [True, 'True', '1']:
self.run_drush('user-password %s --password="******"' % admin_user)
with warn_only():
self.run_quietly('chmod -R 777 ' + config['filesFolder'])
with warn_only():
if configuration.getSettings('deploymentModule'):
self.run_drush('en -y ' + configuration.getSettings('deploymentModule'))
self.handle_modules(config, 'modules_enabled.txt', True)
self.handle_modules(config, 'modules_disabled.txt', False)
if self.methodName == 'drush8':
self.run_drush('cr -y')
self.run_drush('updb --entity-updates -y')
else:
self.run_drush('updb -y')
script_fn = self.factory.get('script', 'runScript')
with warn_only():
if self.methodName == 'drush8':
if uuid:
self.run_drush('cset system.site uuid %s -y' % uuid)
if 'configurationManagement' in config:
# Clear the cache, so all classes get found.
self.run_drush('cr')
for key, cmds in config['configurationManagement'].iteritems():
script_fn(config, script=cmds, rootFolder=config['siteFolder'])
else:
if config['revertFeatures']:
self.run_drush('fra -y')
fn = self.factory.get('script', 'runTaskSpecificScript')
fn('reset', config, **kwargs)
if self.methodName == 'drush8':
self.run_drush('cr')
else:
self.run_drush(' cc all')
def createApp(**kwargs):
configuration.check(['docker'])
if not configuration.getSettings('repository'):
log.error('Missing repository in fabfile, can\'t continue')
exit(1)
log.info('Create app from source at %s' %
configuration.getSettings('repository'))
stages = [{
'stage': 'checkExistingInstallation',
'connection': 'docker',
'context': {
'installationExists': False
}
}]
createDestroyHelper(stages, 'createApp')
if stages[0]['context']['installationExists']:
log.info('Found an existing installation, running deploy instead!')
# Spin up the container.
stages = [
{
'stage': 'spinUp',
'connection': 'docker'
},
]
createDestroyHelper(stages, 'createApp', **kwargs)
deploy()
return
# Install the app.
stages = configuration.getSettings('createAppStages', [
{
'stage': 'installCode',
'connection': 'docker'
},
{
'stage': 'spinUp',
'connection': 'docker'
},
{
'stage': 'installDependencies',
'connection': 'ssh'
},
{
'stage': 'install',
'connection': 'ssh',
'withReset': 'copyFrom' not in kwargs
},
])
createDestroyHelper(stages, 'createApp', **kwargs)
if 'copyFrom' in kwargs:
copyFrom(kwargs['copyFrom'])
def backupSql(self, config, backup_file_name):
with cd(config['siteFolder']):
with warn_only():
dump_options = ''
if configuration.getSettings('sqlSkipTables'):
dump_options = '--structure-tables-list=' + ','.join(configuration.getSettings('sqlSkipTables'))
self.run_quietly('mkdir -p ' + config['backupFolder'])
self.run_quietly('rm -f '+backup_file_name)
if config['supportsZippedBackups']:
self.run_quietly('rm -f '+backup_file_name+'.gz')
dump_options += ' --gzip'
self.run_drush('sql-dump ' + dump_options + ' --result-file=' + backup_file_name)
def apply(identifier, template):
project_name = configuration.getSettings('name', 'unknown')
project_key = configuration.getSettings('key', project_name[:3])
replacements = {}
replacements['%identifier%'] = identifier
replacements['%slug%'] = slugify(identifier)
replacements['%slug.with-hyphens%'] = slugify(identifier, replacement='-')
replacements['%slug.without-feature%'] = slugify(identifier.replace('feature/', ''), replacement='')
replacements['%slug.with-hyphens.without-feature%'] = slugify(identifier.replace('feature/', ''), replacement='-')
replacements['%project-identifier%'] = project_name
replacements['%project-slug%'] = slugify(project_name)
replacements['%project-slug.with-hypens%'] = slugify(project_name, replacement='-')
replacements['%project-key%'] = slugify(project_key, replacement='')
result = apply_helper(template, replacements)
return result
def rsync(self, source_config, target_config, folder='filesFolder'):
if not target_config['supportsCopyFrom']:
print red(
'The configuration "{c} does not support copyFrom'.format(
c=source_config['config_name']))
return
print green('Copying files from {f} to {t}'.format(
f=source_config['config_name'], t=target_config['config_name']))
with cd(env.config['rootFolder']):
exclude_settings = configuration.getSettings('excludeFiles')
exclude_files_setting = exclude_settings['copyFrom']
rsync_args = ''
if exclude_files_setting:
rsync_args = ' --exclude "' + '" --exclude "'.join(
exclude_files_setting) + '"'
rsync = 'rsync -rav --no-o --no-g -e "ssh -T -o Compression=no {ssh_args} -p {port}" {rsync_args} {user}@{host}:{source_dir}/* {target_dir}'.format(
ssh_args=utils.ssh_no_strict_key_host_checking_params,
source_dir=source_config[folder],
target_dir=target_config[folder],
rsync_args=rsync_args,
**source_config)
with warn_only():
run(rsync)
def createApp(self, config, stage, dockerConfig, **kwargs):
targetPath = dockerConfig['rootFolder'] + '/' + config['docker']['projectFolder']
if (stage == 'checkExistingInstallation'):
kwargs['context']['installationExists'] = False
if (self.exists(targetPath + '/.projectCreated')):
kwargs['context']['installationExists'] = True
return
if (stage == 'installCode'):
repository = False
if 'docker' in config and 'repository' in config['docker']:
repository = config['docker']['repository']
elif 'repository' in config:
repository = config['repository']
else:
repository = configuration.getSettings('repository')
if not repository:
print red('Could not find \'repository\' in host configuration nor in settings')
exit(1)
if (self.exists(targetPath + '/.projectCreated')):
print green('Application already installed!');
with self.cd(targetPath):
self.run('git checkout %s' % config['branch'])
self.run('git pull -q origin %s' % config['branch'])
else:
self.run('git clone -b %s %s %s' % (config['branch'], repository, targetPath))
with self.cd(targetPath):
self.run('git submodule update --init')
self.run('touch .projectCreated')
def handle_modules(self, config, file, enable):
file = config['rootFolder'] + '/' + file
if not self.exists(file):
return
content = ''
if config['runLocally']:
with open(file) as fd:
fd.seek(0)
content=fd.read()
else:
with tempfile.TemporaryFile() as fd:
get(file, fd)
fd.seek(0)
content=fd.read()
if content:
content = content.splitlines()
map(str.strip, content)
ignore_key = 'modulesEnabledIgnore' if enable else 'modulesDisabledIgnore'
ignores = config[ignore_key] if ignore_key in config else configuration.getSettings(ignore_key, [])
if ignores:
for ignore in ignores:
if ignore in content:
content.remove(ignore)
log.warning('Ignoring %s while %s modules from %s' % (' '.join(ignores), 'enabling' if enable else 'disabling', file))
modules = ' '.join(content)
if enable:
self.run_drush('en -y ' + modules)
else:
self.run_drush('dis -y ' + modules)
def rsync(self, source_config, target_config, folder = 'filesFolder'):
if not target_config['supportsCopyFrom']:
log.error('The configuration "{c} does not support copyFrom'.format(c=source_config['config_name']))
return
log.info('Copying files from {f} to {t}'.format(f=source_config['config_name'], t=target_config['config_name']))
with cd(env.config['rootFolder']):
exclude_settings = configuration.getSettings('excludeFiles')
exclude_files_setting = exclude_settings['copyFrom']
rsync_args = ''
if exclude_files_setting:
rsync_args = ' --exclude "' + '" --exclude "'.join(exclude_files_setting) + '"'
rsync = '#!rsync -rav --no-o --no-g -e "ssh -T -o Compression=no {ssh_args} -p {port}" {rsync_args} {user}@{host}:{source_dir}/* {target_dir}'.format(
ssh_args=utils.ssh_no_strict_key_host_checking_params,
source_dir=source_config[folder],
target_dir=target_config[folder],
rsync_args=rsync_args,
**source_config
)
with warn_only():
self.run(rsync)
def runScript(self, config, **kwargs):
script = kwargs["script"]
callbacks = kwargs["callbacks"] if "callbacks" in kwargs else {}
variables = kwargs["variables"] if "variables" in kwargs else {}
environment = kwargs["environment"] if "environment" in kwargs else {}
root_folder = kwargs["rootFolder"] if "rootFolder" in kwargs else config["siteFolder"]
if "environment" in config:
environment = configuration.data_merge(config["environment"], environment)
variables["host"] = config
settings = copy.deepcopy(configuration.getSettings())
map(lambda x: settings.pop(x, None), ["hosts", "dockerHosts"])
variables["settings"] = settings
callbacks["execute"] = self.executeCallback
callbacks["run_task"] = self.runTaskCallback
callbacks["fail_on_error"] = self.failOnErrorCallback
callbacks["fail_on_missing_directory"] = self.failOnMissingDirectory
replacements = self.expandVariables(variables)
commands = self.expandCommands(script, replacements)
# Do it again to support replacements which needs to be replaced again.
commands = self.expandCommands(commands, replacements)
environment = self.expandEnvironment(environment, replacements)
for need in config["needs"]:
environment[need.upper() + "_AVAILABLE"] = "1"
return_code = self.runScriptImpl(root_folder, commands, config, callbacks, environment, replacements)
if return_code:
print red("Due to earlier errors quitting now.")
exit(return_code)
def createApp(**kwargs):
configuration.check(['docker'])
stages = [
{
'stage': 'checkExistingInstallation',
'connection': 'docker',
'context': {
'installationExists': False
}
}
]
createDestroyHelper(stages, 'createApp')
if stages[0]['context']['installationExists']:
print green('Found an existing installaion, running deploy instead!')
deploy()
return
# Install the app.
stages = configuration.getSettings('createAppStages', [
{ 'stage': 'installCode','connection': 'docker' },
{ 'stage': 'installDependencies','connection': 'docker' },
{ 'stage': 'spinUp','connection': 'docker' },
{ 'stage': 'install','connection': 'ssh' },
])
createDestroyHelper(stages, 'createApp', **kwargs)
if 'copyFrom' in kwargs:
copyFrom(kwargs['copyFrom'])
def reset(self, config, **kwargs):
if 'withPasswordReset' not in kwargs:
kwargs['withPasswordReset'] = True
if self.methodName == 'drush8':
uuid = config['uuid'] if 'uuid' in config else False
if not uuid:
uuid = configuration.getSettings('uuid')
if not uuid:
print red('No uuid found in fabfile.yaml. config-import may fail!')
with cd(config['siteFolder']):
if config['type'] == 'dev':
admin_user = configuration.getSettings('adminUser', 'admin')
if 'withPasswordReset' in kwargs and kwargs['withPasswordReset'] in [True, 'True', '1']:
self.run_drush('user-password %s --password="******"' % admin_user)
with warn_only():
self.run_quietly('chmod -R 777 ' + config['filesFolder'])
with warn_only():
if configuration.getSettings('deploymentModule'):
self.run_drush('en -y ' + configuration.getSettings('deploymentModule'))
if self.methodName == 'drush8':
self.run_drush('updb --entity-updates -y')
else:
self.run_drush('updb -y')
with warn_only():
if self.methodName == 'drush8':
if uuid:
self.run_drush('cset system.site uuid %s -y' % uuid)
if 'configurationManagement' in config:
for key in config['configurationManagement']:
self.run_drush('config-import %s -y' % key)
else:
self.run_drush('fra -y')
fn = self.factory.get('script', 'runTaskSpecificScript')
fn('reset', config, **kwargs)
if self.methodName == 'drush8':
self.run_drush('cr')
else:
self.run_drush(' cc all')
def install(self, config, ask='True', distribution='minimal', **kwargs):
if 'database' not in config:
print red('Missing database configuration!')
exit(1)
configuration.validate_dict(['user', 'pass', 'name', 'host'], config['database'], 'Missing database configuration: ')
print green('Installing fresh database for "%s"' % config['config_name'])
o = config['database']
with cd(config['siteFolder']):
self.run_quietly('mkdir -p %s' % config['siteFolder'])
mysql_cmd = 'CREATE DATABASE IF NOT EXISTS {name}; GRANT ALL PRIVILEGES ON {name}.* TO \'{user}\'@\'%\' IDENTIFIED BY \'{pass}\'; FLUSH PRIVILEGES;'.format(**o)
self.run_quietly('mysql -h {host} -u {user} --password={pass} -e "{mysql_command}"'.format(mysql_command=mysql_cmd, **o), 'Creating database')
with warn_only():
self.run_quietly('chmod u+w {siteFolder}'.format(**config))
self.run_quietly('chmod u+w {siteFolder}/settings.php'.format(**config))
self.run_quietly('rm -f {siteFolder}/settings.php.old'.format(**config))
self.run_quietly('mv {siteFolder}/settings.php {siteFolder}/settings.php.old 2>/dev/null'.format(**config))
sites_folder = os.path.basename(config['siteFolder'])
options = ''
if ask.lower() == 'false' or ask.lower() == '0':
options = ' -y'
options += ' --sites-subdir='+sites_folder
options += ' --account-name=%s' % configuration.getSettings('adminUser', 'admin')
options += ' --account-pass=admin'
if 'prefix' in o:
options += " --db-prefix='%s'" % o['prefix']
options += ' --db-url=mysql://' + o['user'] + ':' + o['pass'] + '@' + o['host'] + '/' +o ['name']
self.run_drush('site-install ' + distribution + ' ' + options)
if self.methodName == 'drush7':
self.run_drush('en features -y')
deploymentModule = configuration.getSettings('deploymentModule')
if deploymentModule:
self.run_drush('en -y %s' % deploymentModule)
if self.methodName == 'drush8':
self.setupConfigurationManagement(config)
def destroyApp(**kwargs):
configuration.check(['docker'])
stages = configuration.getSettings('destroyAppStages', [
{ 'stage': 'spinDown','connection': 'docker' },
{ 'stage': 'deleteContainer','connection': 'docker' },
{ 'stage': 'deleteCode','connection': 'docker' },
])
createDestroyHelper(stages, 'destroyApp', **kwargs)
def backupSql(self, config, backup_file_name):
self.setRunLocally(config)
with self.cd(config['siteFolder']):
with warn_only():
dump_options = ''
if configuration.getSettings('sqlSkipTables'):
dump_options = '--structure-tables-list=' + ','.join(
configuration.getSettings('sqlSkipTables'))
self.run_quietly('mkdir -p ' + config['backupFolder'])
self.run_quietly('rm -f ' + backup_file_name)
if config['supportsZippedBackups']:
self.run_quietly('rm -f ' + backup_file_name + '.gz')
dump_options += ' --gzip'
self.run_drush('sql-dump ' + dump_options + ' --result-file=' +
backup_file_name)
def destroyApp(**kwargs):
configuration.check(['docker'])
stages = configuration.getSettings('destroyAppStages', [
{ 'stage': 'spinDown','connection': 'docker' },
{ 'stage': 'deleteContainer','connection': 'docker' },
{ 'stage': 'deleteCode','connection': 'docker' },
])
createDestroyHelper(stages, 'destroyApp', **kwargs)
def tarFiles(self, config, filename, source_folders, type):
excludeFiles = configuration.getSettings('excludeFiles')
excludeFiles = excludeFiles[type] if type in excludeFiles else False
cmd = '#!tar'
if excludeFiles:
cmd += ' --exclude="' + '" --exclude="'.join(excludeFiles) + '"'
cmd += ' -czPf ' + filename
cmd += ' ' + ' '.join(source_folders)
self.run_quietly(cmd)
def script(scriptKey=False, *args, **kwargs):
configuration.check()
scripts = configuration.current('scripts')
scriptData = scripts[scriptKey] if scriptKey in scripts else False
if not scriptData:
scripts = configuration.getSettings('scripts')
scriptData = scripts[scriptKey] if scriptKey in scripts else False
if not scriptData:
log.error('Could not find any script named "%s" in fabfile.yaml' %
scriptKey)
if configuration.current('scripts'):
print 'Available scripts in %s:\n - ' % configuration.current(
'config_name') + '\n - '.join(
configuration.current('scripts').keys())
if configuration.getSettings('scripts'):
print 'Available scripts: \n - ' + '\n - '.join(
configuration.getSettings('scripts').keys())
exit(1)
if isinstance(scriptData, dict):
if 'defaults' in scriptData:
kwargs = configuration.data_merge(scriptData['defaults'], kwargs)
scriptData = scriptData['script']
# compute arguments:
arguments = ' '.join(args)
for key in kwargs.keys():
arguments += ' ' + key + '="' + kwargs[key] + '"'
variables = {
'arguments': kwargs,
}
variables['arguments']['combined'] = arguments
if scriptData:
methods.call('script',
'runScript',
configuration.current(),
script=scriptData,
variables=variables)
def backup(withFiles=True):
configuration.check()
print green(
'backing up files and database of "%s" @ "%s"'
% (configuration.getSettings("name"), configuration.current("config_name"))
)
i = datetime.datetime.now()
basename = [configuration.current("config_name"), i.strftime("%Y-%m-%d--%H-%M-%S")]
methods.runTask(configuration.current(), "backup", withFiles=withFiles, baseName=basename)
def backup(withFiles = True):
configuration.check()
log.info('backing up files and database of "%s" @ "%s"' % (configuration.getSettings('name'), configuration.current('config_name')))
i = datetime.datetime.now()
basename = [
configuration.current('config_name'),
i.strftime('%Y-%m-%d--%H-%M-%S')
]
methods.runTask(configuration.current(), 'backup', withFiles = withFiles, baseName = basename)
def backup(withFiles = True):
configuration.check()
print green('backing up files and database of "%s" @ "%s"' % (configuration.getSettings('name'), configuration.current('config_name')))
i = datetime.datetime.now()
basename = [
configuration.current('config_name'),
i.strftime('%Y-%m-%d--%H-%M-%S')
]
methods.runTask(configuration.current(), 'backup', withFiles = withFiles, baseName = basename)
def tarFiles(self, config, filename, source_folders, type):
excludeFiles = configuration.getSettings('excludeFiles')
excludeFiles = excludeFiles[type] if type in excludeFiles else False
cmd = 'tar'
if excludeFiles:
print excludeFiles
cmd += ' --exclude="' + '" --exclude="'.join(excludeFiles) + '"'
cmd += ' -czPf ' + filename
cmd += ' ' + ' '.join(source_folders)
self.run_quietly(cmd)
def apply(identifier, template):
project_name = configuration.getSettings('name', 'unknown')
project_key = configuration.getSettings('key', project_name[:3])
replacements = {}
replacements['%identifier%'] = identifier
replacements['%slug%'] = slugify(identifier)
replacements['%slug.with-hyphens%'] = slugify(identifier, replacement='-')
replacements['%slug.without-feature%'] = slugify(identifier.replace(
'feature/', ''),
replacement='')
replacements['%slug.with-hyphens.without-feature%'] = slugify(
identifier.replace('feature/', ''), replacement='-')
replacements['%project-identifier%'] = project_name
replacements['%project-slug%'] = slugify(project_name)
replacements['%project-slug.with-hypens%'] = slugify(project_name,
replacement='-')
replacements['%project-key%'] = slugify(project_key, replacement='')
result = apply_helper(template, replacements)
return result
def createApp(**kwargs):
configuration.check(['docker'])
if not configuration.getSettings('repository'):
log.error('Missing repository in fabfile, can\'t continue')
exit(1)
log.info('Create app from source at %s' % configuration.getSettings('repository'))
stages = [
{
'stage': 'checkExistingInstallation',
'connection': 'docker',
'context': {
'installationExists': False
}
}
]
createDestroyHelper(stages, 'createApp')
if stages[0]['context']['installationExists']:
log.info('Found an existing installation, running deploy instead!')
# Spin up the container.
stages = [
{ 'stage': 'spinUp','connection': 'docker' },
]
createDestroyHelper(stages, 'createApp', **kwargs)
deploy()
return
# Install the app.
stages = configuration.getSettings('createAppStages', [
{ 'stage': 'installCode','connection': 'docker' },
{ 'stage': 'spinUp','connection': 'docker' },
{ 'stage': 'installDependencies','connection': 'ssh' },
{ 'stage': 'install','connection': 'ssh', 'withReset': 'copyFrom' not in kwargs },
])
createDestroyHelper(stages, 'createApp', **kwargs)
if 'copyFrom' in kwargs:
copyFrom(kwargs['copyFrom'])
def script(scriptKey = False, *args, **kwargs):
configuration.check()
scripts = configuration.current('scripts')
scriptData = scripts[scriptKey] if scriptKey in scripts else False
if not scriptData:
scripts = configuration.getSettings('scripts')
scriptData = scripts[scriptKey] if scriptKey in scripts else False
if not scriptData:
log.error('Could not find any script named "%s" in fabfile.yaml' % scriptKey)
if configuration.current('scripts'):
print 'Available scripts in %s:\n - ' % configuration.current('config_name') + '\n - '.join(configuration.current('scripts').keys())
if configuration.getSettings('scripts'):
print 'Available scripts: \n - ' + '\n - '.join(configuration.getSettings('scripts').keys())
exit(1)
if isinstance(scriptData, dict):
if 'defaults' in scriptData:
kwargs = configuration.data_merge(scriptData['defaults'], kwargs)
scriptData = scriptData['script']
# compute arguments:
arguments = ' '.join(args)
for key in kwargs.keys():
arguments += ' ' + key + '="' + kwargs[key]+'"'
variables = {
'arguments': kwargs,
}
variables['arguments']['combined'] = arguments
if scriptData:
methods.call('script', 'runScript', configuration.current(), script=scriptData, variables=variables)
def script(scriptKey=False, *args, **kwargs):
configuration.check()
scripts = configuration.current("scripts")
scriptData = scripts[scriptKey] if scriptKey in scripts else False
if not scriptData:
scripts = configuration.getSettings("scripts")
scriptData = scripts[scriptKey] if scriptKey in scripts else False
if not scriptData:
print red('Could not find any script named "%s" in fabfile.yaml' % scriptKey)
if configuration.current("scripts"):
print "Available scripts in %s:\n - " % configuration.current("config_name") + "\n - ".join(
configuration.current("scripts").keys()
)
if configuration.getSettings("scripts"):
print "Available scripts: \n - " + "\n - ".join(configuration.getSettings("scripts").keys())
exit(1)
if isinstance(scriptData, dict):
if "defaults" in scriptData:
kwargs = configuration.data_merge(scriptData["defaults"], kwargs)
scriptData = scriptData["script"]
# compute arguments:
arguments = " ".join(args)
for key in kwargs.keys():
arguments += " " + key + '="' + kwargs[key] + '"'
variables = {"arguments": kwargs}
variables["arguments"]["combined"] = arguments
if scriptData:
methods.call("script", "runScript", configuration.current(), script=scriptData, variables=variables)
def runTaskSpecificScript(self, taskName, config, **kwargs):
common_scripts = configuration.getSettings('common')
type = config['type']
if type in common_scripts and isinstance(common_scripts[type], list):
log.error("Found old-style common-scripts. Please regroup by common > taskName > type > commands.")
if taskName in common_scripts:
if type in common_scripts[taskName]:
script = common_scripts[taskName][type]
log.info('Running common script for task %s and type %s' % (taskName, type))
self.runScript(config, script=script)
if taskName in config:
script = config[taskName]
log.info('Running host-script for task %s and type %s' % (taskName, type))
self.runScript(config, script=script)
def runTaskSpecificScript(self, taskName, config, **kwargs):
common_scripts = configuration.getSettings("common")
type = config["type"]
if type in common_scripts and isinstance(common_scripts[type], list):
print red("Found old-style common-scripts. Please regroup by common > taskName > type > commands.")
if taskName in common_scripts:
if type in common_scripts[taskName]:
script = common_scripts[taskName][type]
print yellow("Running common script for task %s and type %s" % (taskName, type))
self.runScript(config, script=script)
if taskName in config:
script = config[taskName]
print yellow("Running host-script for task %s and type %s" % (taskName, type))
self.runScript(config, script=script)
def getDockerConfig(self, config):
if not 'docker' in config or 'configuration' not in config['docker']:
return False
docker_config_name = config['docker']['configuration']
dockerHosts = configuration.getSettings('dockerHosts')
if not dockerHosts or docker_config_name not in dockerHosts:
return False
docker_config = dockerHosts[docker_config_name]
if 'password' in docker_config:
self.addPasswordToFabricCache(**docker_config)
return docker_config
def runTaskSpecificScript(self, taskName, config, **kwargs):
common_scripts = configuration.getSettings('common')
type = config['type']
if type in common_scripts and isinstance(common_scripts[type], list):
print red(
"Found old-style common-scripts. Please regroup by common > taskName > type > commands."
)
if taskName in common_scripts:
if type in common_scripts[taskName]:
script = common_scripts[taskName][type]
print yellow('Running common script for task %s and type %s' %
(taskName, type))
self.runScript(config, script=script)
if taskName in config:
script = config[taskName]
print yellow('Running host-script for task %s and type %s' %
(taskName, type))
self.runScript(config, script=script)
def runScript(self, config, **kwargs):
self.setRunLocally(config)
script = kwargs['script']
callbacks = kwargs['callbacks'] if 'callbacks' in kwargs else {}
variables = kwargs['variables'] if 'variables' in kwargs else {}
environment = kwargs['environment'] if 'environment' in kwargs else {}
root_folder = kwargs[
'rootFolder'] if 'rootFolder' in kwargs else config['siteFolder']
runLocally = kwargs[
'runLocally'] if 'runLocally' in kwargs else self.run_locally
if 'environment' in config:
environment = configuration.data_merge(config['environment'],
environment)
variables['host'] = config
settings = copy.deepcopy(configuration.getSettings())
map(lambda x: settings.pop(x, None), ['hosts', 'dockerHosts'])
variables['settings'] = settings
callbacks['execute'] = self.executeCallback
callbacks['run_task'] = self.runTaskCallback
callbacks['fail_on_error'] = self.failOnErrorCallback
callbacks['fail_on_missing_directory'] = self.failOnMissingDirectory
replacements = self.expandVariables(variables)
commands = self.expandCommands(script, replacements)
# Do it again to support replacements which needs to be replaced again.
commands = self.expandCommands(commands, replacements)
environment = self.expandEnvironment(environment, replacements)
for need in config['needs']:
environment[need.upper() + '_AVAILABLE'] = "1"
return_code = self.runScriptImpl(root_folder, commands, config,
runLocally, callbacks, environment,
replacements)
if return_code:
print red('Due to earlier errors quitting now.')
exit(return_code)
def handle_modules(self, config, file, enable):
file = config['rootFolder'] + '/' + file
if not self.exists(file):
return
content = ''
if config['runLocally']:
with open(file) as fd:
fd.seek(0)
content = fd.read()
else:
with tempfile.TemporaryFile() as fd:
get(file, fd)
fd.seek(0)
content = fd.read()
if content:
content = content.splitlines()
map(str.strip, content)
ignore_key = 'modulesEnabledIgnore' if enable else 'modulesDisabledIgnore'
ignores = config[
ignore_key] if ignore_key in config else configuration.getSettings(
ignore_key, [])
if ignores:
for ignore in ignores:
if ignore in content:
content.remove(ignore)
log.warning('Ignoring %s while %s modules from %s' %
(' '.join(ignores),
'enabling' if enable else 'disabling', file))
modules = ' '.join(content)
if enable:
self.run_drush('en -y ' + modules)
else:
self.run_drush('dis -y ' + modules)
def createApp(self, config, stage, dockerConfig, **kwargs):
targetPath = dockerConfig['rootFolder'] + '/' + config['docker']['projectFolder']
self.setExecutables(config)
if (stage == 'checkExistingInstallation'):
kwargs['context']['installationExists'] = False
if (self.exists(targetPath + '/.projectCreated')):
kwargs['context']['installationExists'] = True
return
if (stage == 'installCode'):
repository = False
if 'docker' in config and 'repository' in config['docker']:
repository = config['docker']['repository']
elif 'repository' in config:
repository = config['repository']
else:
repository = configuration.getSettings('repository')
if not repository:
log.error('Could not find \'repository\' in host configuration nor in settings')
exit(1)
if (self.exists(targetPath + '/.projectCreated')):
log.info('Application already installed!');
with self.cd(targetPath):
self.run('#!git checkout %s' % config['branch'])
self.run('#!git pull -q origin %s' % config['branch'])
else:
self.run('#!git clone -b %s %s %s' % (config['branch'], repository, targetPath))
with self.cd(targetPath):
self.run('#!git submodule update --init')
self.run('touch .projectCreated')
def runScript(self, config, **kwargs):
self.setRunLocally(config)
script = kwargs['script']
callbacks = kwargs['callbacks'] if 'callbacks' in kwargs else {}
variables = kwargs['variables'] if 'variables' in kwargs else {}
environment = kwargs['environment'] if 'environment' in kwargs else {}
root_folder = kwargs['rootFolder'] if 'rootFolder' in kwargs else config['siteFolder'] if 'siteFolder' in config else '.'
runLocally = kwargs['runLocally'] if 'runLocally' in kwargs else self.run_locally
if 'environment' in config:
environment = configuration.data_merge(config['environment'], environment)
variables['host'] = config
settings = copy.deepcopy(configuration.getSettings())
map(lambda x: settings.pop(x,None), ['hosts', 'dockerHosts'])
variables['settings'] = settings
callbacks['execute'] = self.executeCallback
callbacks['run_task'] = self.runTaskCallback
callbacks['fail_on_error'] = self.failOnErrorCallback
callbacks['fail_on_missing_directory'] = self.failOnMissingDirectory
replacements = self.expandVariables(variables);
commands = self.expandCommands(script, replacements)
# Do it again to support replacements which needs to be replaced again.
commands = self.expandCommands(commands, replacements)
environment = self.expandEnvironment(environment, replacements)
for need in config['needs']:
environment[need.upper() + '_AVAILABLE'] = "1"
return_code = self.runScriptImpl(root_folder, commands, config, runLocally, callbacks, environment, replacements)
if return_code:
log.error('Due to earlier errors quitting now.')
exit(return_code)
def install(self, config, ask='True', distribution=False, locale=False, options=False, **kwargs):
self.setRunLocally(config)
if not distribution:
distribution = config['installOptions']['distribution']
if not locale:
locale = config['installOptions']['locale']
if not options:
options = config['installOptions']['options']
if 'database' not in config:
log.error('Missing database configuration!')
exit(1)
configuration.validate_dict(['user', 'pass', 'name', 'host'], config['database'], 'Missing database configuration: ')
log.info('Installing fresh database for "%s"' % config['config_name'])
o = config['database']
with self.cd(config['rootFolder']):
self.run_quietly('mkdir -p %s' % config['siteFolder'])
with self.cd(config['siteFolder']):
if not o["skipCreateDatabase"]:
mysql_cmd = 'CREATE DATABASE IF NOT EXISTS {name}; GRANT ALL PRIVILEGES ON {name}.* TO \'{user}\'@\'%\' IDENTIFIED BY \'{pass}\'; FLUSH PRIVILEGES;'.format(**o)
self.run_quietly('#!mysql -h {host} -u {user} --password={pass} -e "{mysql_command}"'.format(mysql_command=mysql_cmd, **o), 'Creating database')
with warn_only():
self.run_quietly('chmod u+w {siteFolder}'.format(**config))
self.run_quietly('chmod u+w {siteFolder}/settings.php'.format(**config))
self.run_quietly('rm -f {siteFolder}/settings.php.old'.format(**config))
self.run_quietly('mv {siteFolder}/settings.php {siteFolder}/settings.php.old 2>/dev/null'.format(**config))
sites_folder = os.path.basename(config['siteFolder'])
cmd_options = ''
if ask.lower() == 'false' or ask.lower() == '0':
cmd_options = ' -y'
cmd_options += ' --sites-subdir='+sites_folder
cmd_options += ' --account-name=%s' % configuration.getSettings('adminUser', 'admin')
cmd_options += ' --account-pass=admin'
cmd_options += ' --locale=%s' % locale
if 'prefix' in o:
cmd_options += " --db-prefix='%s'" % o['prefix']
cmd_options += ' --db-url=mysql://' + o['user'] + ':' + o['pass'] + '@' + o['host'] + '/' +o ['name']
cmd_options += ' %s' % options
self.run_drush('site-install ' + distribution + ' ' + cmd_options)
if config['revertFeatures'] and self.methodName == 'drush7':
self.run_drush('en features -y')
deploymentModule = configuration.getSettings('deploymentModule')
if deploymentModule:
self.run_drush('en -y %s' % deploymentModule)
if self.methodName == 'drush8':
self.setupConfigurationManagement(config)
def version():
configuration.check('git')
version = methods.call('git', 'getVersion', configuration.current())
print green('%s @ %s tagged with: %s' % (configuration.getSettings('name'), configuration.current('config_name'), version))
def version():
configuration.check('git')
version = methods.call('git', 'getVersion', configuration.current())
log.info('%s @ %s tagged with: %s' % (configuration.getSettings('name'), configuration.current('config_name'), version))
def version():
configuration.check("git")
version = methods.call("git", "getVersion", configuration.current())
print green(
"%s @ %s tagged with: %s" % (configuration.getSettings("name"), configuration.current("config_name"), version)
)
def copySSHKeys(self, config, **kwargs):
if 'ssh' not in config['needs']:
return
tempfiles = []
files = {
'key': configuration.getSettings('dockerKeyFile'),
'authorized_keys': configuration.getSettings('dockerAuthorizedKeyFile'),
'known_hosts': configuration.getSettings('dockerKnownHostsFile')
}
if files['key']:
files['public_key'] = files['key'] + '.pub'
# Check existance of source files
count = 0;
preflight_succeeded = True
for key, value in files.iteritems():
if not value:
count += 1
else:
full_file_name = value
if value[0:7] == 'http://' or value[0:8] == 'https://':
data = configuration.get_configuration_via_http(value, as_yaml=False)
if not data:
preflight_succeeded = False
continue;
else:
fd, path = tempfile.mkstemp()
with os.fdopen(fd, 'w') as tmp:
tmp.write(data)
tempfiles.append(path)
full_file_name = path
else:
full_file_name = configuration.getBaseDir() + '/' + value
if not os.path.exists(full_file_name):
log.error('Could not copy file to container, missing file: %s' % full_file_name)
preflight_succeeded = False
else:
files[key] = full_file_name
if count >= 3:
return
if not preflight_succeeded:
exit(1)
with cd(config['rootFolder']), hide('commands', 'output'), lcd(configuration.getBaseDir()):
run('mkdir -p /root/.ssh')
if files['key']:
put(files['key'], '/root/.ssh/id_rsa')
put(files['public_key'], '/root/.ssh/id_rsa.pub')
run('chmod 600 /root/.ssh/id_rsa')
run('chmod 644 /root/.ssh/id_rsa.pub')
put(files['public_key'], '/tmp')
run('cat /tmp/'+os.path.basename(files['public_key'])+' >> /root/.ssh/authorized_keys')
run('rm /tmp/'+os.path.basename(files['public_key']))
log.info('Copied keyfile to docker.')
if files['authorized_keys']:
put(files['authorized_keys'], '/root/.ssh/authorized_keys')
log.info('Copied authorized keys to docker.')
if files['known_hosts']:
put(files['known_hosts'], '/root/.ssh/known_hosts')
log.info('Copied known hosts to docker.')
run('chmod 700 /root/.ssh')
for file in tempfiles:
os.remove(file)