def __run_command(self, command_type, manifest_type):
config = getattr(self, manifest_type)
if config.has(command_type):
command = config.get(command_type)
self.logger.debug("Running %s..." % command)
shell = config.has('shell') and config.is_affirmative('shell')
lib.call(command, shell=shell)
def test_call_error(self):
""" Test an exception is thrown for a non-existent command """
try:
lib.call("eahxanea0e0")
except CommandMissingException:
return
raise("Bogus command without proper shell doesn't return proper exception!")
def test_call_error(self):
""" Test an exception is thrown for a non-existent command """
try:
lib.call("eahxanea0e0")
except CommandMissingException:
return
raise("Bogus command without proper shell doesn't return proper exception!")
def _install_packages(self, fresh=False):
""" Install and resolve the required npm packages """
if not self.source or (self.source.get('packages', '') !=
self.target.get('packages', '')):
remove_packages = []
install_packages = []
if self.source and self.source.has('packages') and not fresh:
for package in self.source.get('packages').strip().split('\n'):
remove_packages.append(package.strip())
if self.target and self.target.has('packages'):
for package in self.target.get('packages').strip().split('\n'):
package = package.strip()
if package in remove_packages:
remove_packages.remove(package)
install_packages.append(package)
cwd = self.directory.install_directory(self.feature_name)
for package in remove_packages:
lib.call("bin/npm uninstall --verbose -g %s --prefix %s" %
(package, cwd),
cwd=cwd)
for package in install_packages:
lib.call(
"bin/npm install --verbose -sg %s --prefix %s --production"
% (package, cwd),
cwd=cwd)
def _configure_npmrc(self):
arguments = ""
if self.target.has('global_npm_root'):
global_npm_root = os.path.abspath(os.path.expanduser(self.target.get('global_npm_root')))
arguments += " --prefix={0}".format(global_npm_root)
self.logger.debug("configuring npm with {0}_".format(arguments))
lib.call("lib/node_modules/npm/configure {0}".format(arguments),
cwd=self.directory.install_directory(self.feature_name))
def test_lib_errorcode(self):
""" Test a proper error code is returned """
assert lib.call("sh")[0] == 0, "cd call returns a non-zero exit!"
assert lib.call("cd",
shell=True)[0] == 0, "cd call returns a non-zero exit!"
tools.eq_(
lib.call("exit 1", shell=True)[0], 1,
"exit 1 call returns a zero exit!")
def __call_command(self, command, ssh_path):
ssh_path += ".pub" # make this the public key
ssh_contents = open(ssh_path, 'r').read().rstrip('\n')
command = command.replace('{{ssh}}', ssh_contents)
lib.call(command,
shell=True,
output_log_level=logging.DEBUG,
sensitive_info=True)
def __call_command(self, command, ssh_path):
ssh_path += ".pub" # make this the public key
ssh_contents = open(ssh_path, 'r').read().rstrip('\n')
command = command.replace('{{ssh}}', ssh_contents)
lib.call(command,
shell=True,
output_log_level=logging.DEBUG,
sensitive_info=True)
def _configure_npmrc(self):
arguments = ""
if self.target.has('global_npm_root'):
global_npm_root = os.path.abspath(
os.path.expanduser(self.target.get('global_npm_root')))
arguments += " --prefix={0}".format(global_npm_root)
self.logger.debug("configuring npm with {0}_".format(arguments))
lib.call("lib/node_modules/npm/configure {0}".format(arguments),
cwd=self.directory.install_directory(self.feature_name))
def __checkout_branch(self, target_directory, branch):
self.logger.debug("Checking out branch %s..." % branch)
os.chdir(target_directory)
error, output = lib.call("git fetch origin %s" % branch,
output_log_level=logging.DEBUG)
if not error:
error, output = lib.call("git checkout %s" % branch,
output_log_level=logging.DEBUG)
if error:
raise GitException("An error occurred when checking out a branch!")
def __checkout_branch(self, target_directory, branch):
self.logger.debug("Checking out branch %s..." % branch)
os.chdir(target_directory)
error, output = lib.call("git fetch origin %s" % branch,
output_log_level=logging.DEBUG)
if not error:
error, output = lib.call("git checkout %s" % branch,
output_log_level=logging.DEBUG)
if error:
self.logger.info(output)
raise GitException("An error occurred when checking out a branch!")
def __install_package(self, config):
if self.package_manager and config.has(self.package_manager):
package = config.get(self.package_manager)
self.logger.info("Installing %s..." % package)
call_command = "%s%s install %s" % (self.package_manager, self.args, package)
if self.sudo_required:
call_command = "sudo " + call_command
self.logger.debug("Calling command: %s" % call_command)
# it's not possible to retain remember sudo privileges across shells unless they pipe
# to STDOUT. Nothing we can do about that for now.
lib.call(call_command, output_log_level=logging.DEBUG, stdout=None)
def __install_package(self, config):
if self.package_manager and config.has(self.package_manager):
package = config.get(self.package_manager)
self.logger.info("Installing %s..." % package)
call_command = "%s%s %s" % (self.package_manager, self.args,
package)
if self.sudo_required:
call_command = "sudo " + call_command
self.logger.debug("Calling command: %s" % call_command)
# it's not possible to retain remember sudo privileges across shells unless they pipe
# to STDOUT. Nothing we can do about that for now.
lib.call(call_command, output_log_level=logging.DEBUG, stdout=None)
def install_brew(target_path):
""" Install brew to the target path """
if not os.path.exists(target_path):
try:
os.makedirs(target_path)
except OSError:
LOGGER.warn("Unable to create directory %s for brew." % target_path +
" Trying sudo...")
lib.call("sudo mkdir -p %s" % target_path, stdout=None,
output_log_level=logging.DEBUG)
lib.call("sudo chown %s %s" % (getpass.getuser(), target_path),
output_log_level=logging.DEBUG, stdout=None)
extract_targz(HOMEBREW_URL, target_path, remove_common_prefix=True)
def __install_eggs(self, config):
""" Install eggs for a particular configuration """
eggs = []
if config.has('egg'):
eggs += [config.get('egg')]
if config.has('eggs'):
eggs += [egg.strip() for egg in re.split(',|\n', config.get('eggs'))]
self.logger.debug("Installing eggs %s..." % eggs)
with open(os.path.join(self.directory.install_directory(self.feature_name), 'requirements.txt'),
'w+') as fh:
fh.write('\n'.join(eggs))
lib.call("bin/pip install -r requirements.txt --upgrade",
cwd=self.directory.install_directory(self.feature_name))
def __install_eggs(self, config):
""" Install eggs for a particular configuration """
eggs = []
if config.has('egg'):
eggs += [config.get('egg')]
if config.has('eggs'):
eggs += [egg.strip() for egg in re.split(',|\n', config.get('eggs'))]
self.logger.debug("Installing eggs %s..." % eggs)
with open(os.path.join(self.directory.install_directory(self.feature_name), 'requirements.txt'),
'w+') as fh:
fh.write('\n'.join(eggs))
lib.call("bin/pip install -r requirements.txt --upgrade",
cwd=self.directory.install_directory(self.feature_name),
output_log_level=logging.DEBUG)
def __generate_key(self, config):
"""
Generate the ssh key, and return the ssh config location
"""
cwd = config.get('ssh_path', self.directory.install_directory(self.feature_name))
if not config.has('create') or config.is_affirmative('create'):
if not os.path.exists(cwd):
os.makedirs(cwd)
if not os.path.exists(os.path.join(cwd, config.get('keyname'))):
command = "ssh-keygen -t %(type)s -f %(keyname)s -N " % config.to_dict()
lib.call(command, cwd=cwd, output_log_level=logging.DEBUG)
if not config.has('ssh_path'):
config.set('ssh_path', cwd)
config.set('ssh_key_path', os.path.join(config.get('ssh_path'), config.get('keyname')))
def update(self):
if not lib.which('git'):
self.logger.warn("git is not installed! Please install git to install this feature.")
return
target_directory = self.directory.install_directory(self.feature_name)
source_branch = self.source.get('branch', 'master')
target_branch = self.target.get('branch', 'master')
if self.target.get('url') != self.source.get('url') or \
not os.path.exists(target_directory):
if os.path.exists(target_directory):
self.logger.debug("Old git repository Found. Deleting...")
self.directory.remove_feature(self.feature_name)
self.__clone_repo(self.target.get('url'),
target_directory,
branch=self.target.get('branch', 'master'))
elif source_branch != target_branch:
self.__checkout_branch(target_directory, target_branch)
else:
if not os.path.exists(target_directory):
self.logger.debug("No repository cloned. Re-cloning...")
self.__clone_repo(self.target.get('url'),
target_directory,
branch=target_branch)
os.chdir(target_directory)
error, output = lib.call("git pull origin %s" % target_branch,
output_log_level=logging.DEBUG)
if error:
self.logger.info(output)
raise GitException("An error occurred when pulling!")
FormulaBase.update(self)
def __generate_key(self, config):
"""
Generate the ssh key, and return the ssh config location
"""
cwd = config.get('ssh_path', self._install_directory())
if config.is_affirmative('create', default="yes"):
if not os.path.exists(cwd):
os.makedirs(cwd)
if not os.path.exists(os.path.join(cwd, config.get('keyname'))):
command = "ssh-keygen -t %(type)s -f %(keyname)s -N " % config.to_dict(
)
lib.call(command, cwd=cwd, output_log_level=logging.DEBUG)
if not config.has('ssh_path'):
config.set('ssh_path', cwd)
config.set('ssh_key_path',
os.path.join(config.get('ssh_path'), config.get('keyname')))
def install(self):
"""
Install is called when a feature does not previously exist.
In the case of a feature changing formulas, the old feature/formula is
removed, then the new feature/formula is installed.
Installs are only guaranteed to have the 'target' config set.
errors should either be reported via self._log_error(), or raise an exception
"""
install_directory = self.directory.install_directory(self.feature_name)
cwd = install_directory if os.path.exists(install_directory) else None
if self.target.has('rc'):
self.directory.add_to_rc(self.target.get('rc'))
if self.target.has('command'):
lib.call(self.target.get('command'), shell=True, cwd=cwd)
def __clone_repo(self, repo_url, target_directory, branch):
self.logger.debug("Cloning repository %s into %s..." % (repo_url, target_directory))
error, output = lib.call("git clone %s %s" % (repo_url, target_directory),
output_log_level=logging.DEBUG)
if error:
self.logger.info(output)
raise GitException("An error occurred when cloning!")
self.__checkout_branch(target_directory, branch)
def install_sandboxes(self):
if self.target:
if system.is_osx():
if not self.target.is_affirmative('config', 'use_global_packagemanagers'):
self._install_sandbox('brew', brew.install_brew)
elif lib.which('brew') is None:
install_brew = lib.prompt(
"Looks like you don't have brew, " +
"which is sprinter's package manager of choice for OSX.\n"
"Would you like sprinter to install brew for you?",
default="yes", boolean=True)
if install_brew:
lib.call("sudo mkdir -p /usr/local/", stdout=None,
output_log_level=logging.DEBUG)
lib.call("sudo chown -R %s /usr/local/" % getpass.getuser(),
output_log_level=logging.DEBUG, stdout=None)
brew.install_brew('/usr/local')
def __git(self, command, git_opts):
cmd = command.format(**git_opts)
error, output = lib.call(cmd, output_log_level=logging.DEBUG)
self.logger.info(output)
if error:
self.logger.warning(output)
else:
self.logger.info(output)
return (error, output.strip('\n \t'))
def __run_command(self, command_type, manifest_type):
config = getattr(self, manifest_type)
if config.has(command_type):
command = config.get(command_type)
self.logger.debug("Running %s..." % command)
shell = config.has('shell') and config.is_affirmative('shell')
return_code, output = lib.call(command, shell=shell)
if config.is_affirmative('fail_on_error', True) and return_code != 0:
raise CommandFormulaException("Command returned a return code of {0}!".format(return_code))
def __clone_repo(self, repo_url, target_directory, branch):
self.logger.debug("Cloning repository %s into %s..." %
(repo_url, target_directory))
error, output = lib.call("git clone %s %s" %
(repo_url, target_directory),
output_log_level=logging.DEBUG)
if error:
self.logger.info(output)
raise GitException("An error occurred when cloning!")
self.__checkout_branch(target_directory, branch)
def __run_command(self, command_type, manifest_type):
config = getattr(self, manifest_type)
if config.has(command_type):
command = config.get(command_type)
self.logger.debug("Running %s..." % command)
shell = config.has('shell') and config.is_affirmative('shell')
stdout = subprocess.PIPE if config.is_affirmative('redirect_stdout_to_log', 'true') else None
return_code, output = lib.call(command, shell=shell, stdout=stdout)
if config.is_affirmative('fail_on_error', True) and return_code != 0:
raise CommandFormulaException("Command returned a return code of {0}!".format(return_code))
def install(self):
"""
Install is called when a feature does not previously exist.
In the case of a feature changing formulas, the old feature/formula is
removed, then the new feature/formula is installed.
Installs are only guaranteed to have the 'target' config set.
errors should either be reported via self._log_error(), or raise an exception
"""
install_directory = self.directory.install_directory(self.feature_name)
cwd = install_directory if os.path.exists(install_directory) else None
if self.target.has('env'):
self.directory.add_to_env(self.target.get('env'))
if self.target.has('rc'):
self.directory.add_to_rc(self.target.get('rc'))
if self.target.has('command'):
lib.call(self.target.get('command'), shell=True, cwd=cwd)
def __fetch_merge_repo(self, target_directory, target_branch):
self.logger.debug("Fetching branch %s..." % target_branch)
os.chdir(target_directory)
error, output = lib.call("git fetch origin %s" % target_branch,
output_log_level=logging.DEBUG)
if error:
self.logger.info(output)
raise GitException("An error occurred while fetching!")
self.logger.info(output)
self.logger.debug("Merging branch %s..." % target_branch)
error, output = lib.call("git merge --ff-only origin/%s" % target_branch,
output_log_level=logging.DEBUG)
if error:
#do not want to raise exception on merge failures/conflicts
self.logger.warning(output)
else:
self.logger.info(output)
def __fetch_merge_repo(self, target_directory, target_branch):
self.logger.debug("Fetching branch %s..." % target_branch)
os.chdir(target_directory)
error, output = lib.call("git fetch origin %s" % target_branch,
output_log_level=logging.DEBUG)
if error:
self.logger.info(output)
raise GitException("An error occurred while fetching!")
self.logger.info(output)
self.logger.debug("Merging branch %s..." % target_branch)
error, output = lib.call("git merge --ff-only origin/%s" %
target_branch,
output_log_level=logging.DEBUG)
if error:
#do not want to raise exception on merge failures/conflicts
self.logger.warning(output)
else:
self.logger.info(output)
def __checkout_branch(self, target_directory, branch):
self.logger.debug("Checking out branch %s..." % branch)
for command in ("git fetch origin {0}".format(branch),
"git checkout {0}".format(branch)):
error, output = lib.call(command,
output_log_level=logging.DEBUG,
cwd=target_directory)
if error:
self.logger.info(output)
raise GitException(
"An error occurred when checking out a branch!")
def _install_packages(self, fresh=False):
""" Install and resolve the required npm packages """
if not self.source or (self.source.get('packages', '') != self.target.get('packages', '')):
remove_packages = []
install_packages = []
if self.source and self.source.has('packages') and not fresh:
for package in self.source.get('packages').strip().split('\n'):
remove_packages.append(package.strip())
if self.target and self.target.has('packages'):
for package in self.target.get('packages').strip().split('\n'):
package = package.strip()
if package in remove_packages:
remove_packages.remove(package)
install_packages.append(package)
for package in remove_packages:
lib.call("bin/npm uninstall --verbose %s" % package,
cwd=self.directory.install_directory(self.feature_name))
for package in install_packages:
lib.call("bin/npm install --verbose -sg %s" % package,
cwd=self.directory.install_directory(self.feature_name))
def __run_command(self, command_type, manifest_type):
config = getattr(self, manifest_type)
if config.has(command_type):
command = config.get(command_type)
self.logger.debug("Running %s..." % command)
shell = config.has('shell') and config.is_affirmative('shell')
stdout = subprocess.PIPE if config.is_affirmative('redirect_stdout_to_log', 'true') else None
return_code, output = lib.call(command, shell=shell, stdout=stdout)
if config.is_affirmative('fail_on_error', True) and return_code != 0:
raise CommandFormulaException("Command returned a return code of {0}!".format(return_code))
return True
def __checkout_branch(self, target_directory, branch):
self.logger.debug("Checking out branch %s..." % branch)
for command in ("git fetch origin {0}".format(branch),
"git checkout {0}".format(branch)):
error, output = lib.call(
command,
output_log_level=logging.DEBUG,
cwd=target_directory
)
if error:
self.logger.info(output)
raise GitException("An error occurred when checking out a branch!")
def install_sandboxes(self):
if self.target:
if system.is_osx():
if not self.target.is_affirmative(
'config', 'use_global_packagemanagers'):
self._install_sandbox('brew', brew.install_brew)
elif lib.which('brew') is None:
install_brew = lib.prompt(
"Looks like you don't have brew, " +
"which is sprinter's package manager of choice for OSX.\n"
"Would you like sprinter to install brew for you?",
default="yes",
boolean=True)
if install_brew:
lib.call("sudo mkdir -p /usr/local/",
stdout=None,
output_log_level=logging.DEBUG)
lib.call("sudo chown -R %s /usr/local/" %
getpass.getuser(),
output_log_level=logging.DEBUG,
stdout=None)
brew.install_brew('/usr/local')
def __configure_client(self, config):
""" write the perforce client """
self.logger.info("Configuring p4 client...")
client_dict = config.to_dict()
client_dict['root_path'] = os.path.expanduser(config.get('root_path'))
os.chdir(client_dict['root_path'])
client_dict['hostname'] = system.NODE
client_dict['p4view'] = config['p4view'] % self.environment.target.get_context_dict()
client = re.sub('//depot', ' //depot', p4client_template % client_dict)
self.logger.info(lib.call("%s client -i" % self.p4_command,
stdin=client,
env=self.p4environ,
cwd=client_dict['root_path']))
def update(self):
if not lib.which('git'):
self.logger.warn(
"git is not installed! Please install git to install this feature."
)
return
target_directory = self.directory.install_directory(self.feature_name)
source_branch = self.source.get('branch', 'master')
target_branch = self.target.get('branch', 'master')
if not os.path.exists(target_directory):
# directory doesn't exist, so we just clone again
self.logger.debug("No repository cloned. Re-cloning...")
self.__clone_repo(self.target.get('url'),
target_directory,
branch=target_branch)
if self.target.get('url') != self.source.get('url'):
self.logger.debug("Old git repository Found. Deleting...")
self.directory.remove_feature(self.feature_name)
self.__clone_repo(self.target.get('url'),
target_directory,
branch=self.target.get('branch', 'master'))
elif source_branch != target_branch:
self.__checkout_branch(target_directory, target_branch)
else:
if not os.path.exists(target_directory):
self.logger.debug("No repository cloned. Re-cloning...")
self.__clone_repo(self.target.get('url'),
target_directory,
branch=target_branch)
os.chdir(target_directory)
error, output = lib.call("git pull origin %s" % target_branch,
output_log_level=logging.DEBUG)
if error:
self.logger.info(output)
raise GitException("An error occurred when pulling!")
FormulaBase.update(self)
def __prepare_eggs(self, egg_carton, config):
stdout = None
if config.is_affirmative('redirect_stdout_to_log', 'true'):
stdout = subprocess.PIPE
egg_recipe = "PYTHONPATH='' bin/pip install -r {filename} --upgrade".format(filename=egg_carton[1])
return_code, output = lib.call(egg_recipe,
cwd=egg_carton[0],
output_log_level=logging.DEBUG,
shell=True,
stdout=stdout)
if return_code != 0:
if config.is_affirmative('fail_on_error', True):
raise EggscriptFormulaException("""
Egg script {name} returned a return code of {code}!
pip install output ==================================================
{output}
end pip install output ==============================================
""".format(name=self.feature_name, code=return_code, output=output))
return return_code
def __prepare_eggs(self, egg_carton, config):
stdout = None
if config.is_affirmative('redirect_stdout_to_log', 'true'):
stdout = subprocess.PIPE
egg_recipe = "PYTHONPATH='' bin/pip install -r {filename} --upgrade".format(
filename=egg_carton[1])
return_code, output = lib.call(egg_recipe,
cwd=egg_carton[0],
output_log_level=logging.DEBUG,
shell=True,
stdout=stdout)
if return_code != 0:
if config.is_affirmative('fail_on_error', True):
raise EggscriptFormulaException("""
Egg script {name} returned a return code of {code}!
pip install output ==================================================
{output}
end pip install output ==============================================
""".format(name=self.feature_name, code=return_code, output=output))
return return_code
def test_call_unicode_output(self):
""" A process that returns unicode output should not raise an exception """
lib.call("echo -e \"\\xE2\\x98\\xA0\"")
def test_call_no_output(self):
""" A process that returns nothing back should not raise an exception """
lib.call("echo")
def test_lib_errorcode(self):
""" Test a proper error code is returned """
assert lib.call("sh")[0] == 0, "cd call returns a non-zero exit!"
assert lib.call("cd", shell=True)[0] == 0, "cd call returns a non-zero exit!"
tools.eq_(lib.call("exit 1", shell=True)[0], 1, "exit 1 call returns a zero exit!")
def test_call_unicode_output(self):
""" A process that returns unicode output should not raise an exception """
lib.call("echo -e \"\\xE2\\x98\\xA0\"")
def test_call_no_output(self):
""" A process that returns nothing back should not raise an exception """
lib.call("echo")
def run_buildout(self):
""" Run bootstrap.py and bin/buildout """
lib.call("python -S bootstrap.py", cwd=self.root_path)
lib.call("bin/buildout", cwd=self.root_path)
def test_lib_errorcode(self):
""" Test a proper error code is returned """
assert lib.call("sh") == 0, "cd call returns a non-zero exit!"
assert lib.call("cd", bash=True) == 0, "cd call returns a non-zero exit!"
assert lib.call("exit 1", bash=True) == 1, "gibberish call returns a zero exit!"
