def __init__(self, filename, style=None):
style = style or 'yaml'
check.check_string(filename)
self._style = style
self._filename = path.abspath(filename)
if not path.isfile(self._filename):
file_util.save(self._filename, content='')
def test_env_vars(self):
content = '''\
ape
activity: ${_CONFIG_ACTIVITY}
snack: leaves
bonobo extends ape
activity: loving
snack: ${_CONFIG_SNACK}
chimp extends ape
activity: fighting
snack: eggs
'''
tmp_dir = self.make_temp_dir()
tmp_file = path.join(tmp_dir, 'apes.config')
file_util.save(tmp_file, content=content)
with env_override(env={
'_CONFIG_ACTIVITY': 'resting',
'_CONFIG_SNACK': 'kiwi'
}) as tmp_env:
s = SCL(tmp_dir, '*.config')
s.load()
self.assertEqual('fighting',
s.section('chimp').find_by_key('activity'))
self.assertEqual('loving',
s.section('bonobo').find_by_key('activity'))
self.assertEqual('eggs', s.section('chimp').find_by_key('snack'))
self.assertEqual('kiwi', s.section('bonobo').find_by_key('snack'))
def load(clazz, filename, address, branch, output_filename, working_dir):
db = git_repo_document_db(working_dir, address, branch)
if not output_filename:
output_filename = path.join(os.getcwd(), filename)
content = db.load_document(filename)
file_util.save(output_filename, content)
return 0
def test_search_path_expanduser(self):
content = '''\
ape
activity: resting
snack: leaves
bonobo extends ape
activity: loving
snack: kiwi
chimp extends ape
activity: fighting
snack: eggs
'''
with env_override.temp_home() as tmp_env:
file_util.save(path.join(os.environ['HOME'], '.config',
'apes.config'),
content=content)
s = SCL('~/.config', '*.config')
s.load()
self.assertEqual('fighting',
s.section('chimp').find_by_key('activity'))
self.assertEqual('loving',
s.section('bonobo').find_by_key('activity'))
self.assertEqual('eggs', s.section('chimp').find_by_key('snack'))
self.assertEqual('kiwi', s.section('bonobo').find_by_key('snack'))
def archive(clazz, address, revision, base_name, output_filename,
untracked = False, override_gitignore = None, debug = False):
'git archive with additional support to include untracked files for local repos.'
tmp_repo_dir = temp_file.make_temp_dir(delete = not debug)
if path.isdir(address):
excludes = git_ignore.read_gitignore_file(address)
file_copy.copy_tree(address, tmp_repo_dir, excludes = excludes)
if override_gitignore:
file_util.save(path.join(address, '.gitignore'), content = override_gitignore)
if untracked:
git_exe.call_git(tmp_repo_dir, [ 'add', '-A' ])
git_exe.call_git(tmp_repo_dir, [ 'commit', '-m', '"add untracked files just for tmp repo"' ])
else:
if untracked:
raise git_error('untracked can only be True for local repos.')
clazz.clone(address, tmp_repo_dir)
output_filename = path.abspath(output_filename)
file_util.mkdir(path.dirname(output_filename))
args = [
'archive',
'--format=tgz',
'--prefix=%s-%s/' % (base_name, revision),
'-o',
output_filename,
revision
]
rv = git_exe.call_git(tmp_repo_dir, args)
return rv
def test_use_framework(self):
test = self._make_test_shell_framework()
test.repo.tag('1.2.3', push=True)
test.framework.update('1.2.3')
my_script_content = '''
_this_file="$( command readlink "$BASH_SOURCE" )" || _this_file="$BASH_SOURCE"
_root="${_this_file%/*}"
if [ "$_root" == "$_this_file" ]; then
_root=.
fi
_WHERE="$( command cd -P "$_root" > /dev/null && command pwd -P )"
unset _this_file
unset _root
source ${_WHERE}/bes_shell_framework/bes_bash.bash
bes_env_path_append PATH /foo/bin
'''
my_script_path = path.join(test.tmp_dir, 'my_script.sh')
file_util.save(my_script_path, content=my_script_content, mode=0o755)
ed = env_dir(test.tmp_dir, files=['my_script.sh'])
expected = {
'PATH': '%s:/foo/bin' % (os_env.DEFAULT_SYSTEM_PATH),
}
actual = ed.transform_env({'PATH': os_env.DEFAULT_SYSTEM_PATH})
self.assertEqual(expected, actual)
def __init__(self, filename, options=None):
check.check_string(filename)
check.check_simple_config_options(options, allow_none=True)
self._options = options or simple_config_options()
self._filename = path.abspath(filename)
if not path.isfile(self._filename):
file_util.save(self._filename, content='')
def _config(self):
if not path.exists(self._filename):
file_util.save(self._filename, content='', codec='utf-8')
return simple_config.from_file(
self._filename,
check_env_vars=False,
entry_parser=self._parse_ssh_config_entry,
entry_formatter=self._ssh_config_entry_formatter)
def save_file(self, filename, content, codec = 'utf-8', mode = None, add = True, commit = True):
if add and not commit:
raise ValueError('If add is True then commit should be True as well.')
p = self.file_path(filename)
file_util.save(p, content = content, mode = mode)
if add:
self.add([ filename ])
if commit:
msg = 'add or change {}'.format(filename)
self.commit(msg, [ filename ])
def __init__(self, filename, formatter=None, backup=False):
check.check_string(filename)
check.check_properties_file_formatter(formatter, allow_none=True)
check.check_bool(backup)
self.filename = path.abspath(filename)
self._formatter = formatter or properties._FORMATTER_YAML
self._backup = backup
if not path.isfile(self.filename):
file_util.save(self.filename, content='')
def _make_temp_archive_dmg(clazz, items, filename, mode):
tmp_dir = temp_file.make_temp_dir()
for item in items:
assert item
assert item.arcname
file_util.save(path.join(tmp_dir, item.arcname),
content=item.content)
tmp_dmg = temp_file.make_temp_file()
cmd = 'hdiutil create -srcfolder %s -ov -format UDZO %s' % (tmp_dir,
filename)
execute.execute(cmd)
file_util.remove(tmp_dir)
def _make_temp_archive_xz(clazz, items, filename, mode):
tmp_dir = temp_file.make_temp_dir()
for item in items:
assert item
assert item.arcname
file_util.save(path.join(tmp_dir, item.arcname),
content=item.content)
tmp_xz = temp_file.make_temp_file()
manifest_content = '\n'.join([item.arcname for item in items])
manifest = temp_file.make_temp_file(content=manifest_content)
cmd = 'tar Jcf %s -C %s -T %s' % (filename, tmp_dir, manifest)
execute.execute(cmd)
file_util.remove(tmp_dir)
def test_self_dependency(self):
content = '''\
ape extends ape
something: yes
'''
tmp_dir = self.make_temp_dir()
tmp_file = path.join(tmp_dir, 'apes.config')
file_util.save(tmp_file, content=content)
s = SCL(tmp_dir, '*.config')
with self.assertRaises(ERROR) as ctx:
s.load()
self.assertTrue(
'Self dependency for "ape"' in ctx.exception.message)
def test_update_egg_directory(self):
tmp_dir = temp_file.make_temp_dir()
eggs = [
'foo-1.2.3-py2.7.egg',
'bar-6.6.6-py2.7.egg',
'baz-10.11.12-py2.7.egg',
]
for egg in eggs:
file_util.save(path.join(tmp_dir, egg), content = '%s\n' % (egg))
setup_tools.update_egg_directory(tmp_dir)
easy_install_dot_pth_path = path.join(tmp_dir, setup_tools.EASY_INSTALL_DOT_PTH_FILENAME)
actual_eggs = setup_tools.read_easy_install_pth(easy_install_dot_pth_path)
self.assertEqual( sorted(eggs), sorted(actual_eggs) )
def test_missing_dependency(self):
content = '''\
ape extends missing_link
something: yes
'''
tmp_dir = self.make_temp_dir()
tmp_file = path.join(tmp_dir, 'apes.config')
file_util.save(tmp_file, content=content)
s = SCL(tmp_dir, '*.config')
s.load()
with self.assertRaises(ERROR) as ctx:
s.section('ape')
self.assertTrue('Missing dependency for ape: missing_link' in
ctx.exception.message)
def add_file(self, filename, content, codec = 'utf-8', mode = None, commit = True, push = False,
commit_message = None):
p = self.file_path(filename)
assert not path.isfile(p)
file_util.save(p, content = content, codec = codec, mode = mode)
self.add( [ filename ])
result = None
if commit:
commit_message = commit_message or 'add {}'.format(filename)
self.commit(commit_message, [ filename ])
result = self.last_commit_hash(short_hash = True)
if push:
self.push()
return result
def test_archive_local_repo_untracked(self):
tmp_repo = self._create_tmp_repo()
new_files = self._create_tmp_files(tmp_repo)
git.add(tmp_repo, new_files)
git.commit(tmp_repo, 'nomsg\n', '.')
file_util.save(path.join(tmp_repo, 'kiwi.txt'),
content='this is kiwi.txt\n')
tmp_archive = self.make_temp_file()
git.archive(tmp_repo, 'master', 'foo', tmp_archive, untracked=True)
self.assertEqual([
'foo-master/',
'foo-master/bar.txt',
'foo-master/foo.txt',
'foo-master/kiwi.txt',
], archiver.members(tmp_archive))
def _git_update_op(repo):
# This is called from operation_with_reset below.
fp = repo.file_path(filename)
first_time = not path.exists(fp)
# This sets the content to '' the first time, but None would allow update_func to
# distinguish the true first time from a later time where it happened to become empty. On the
# other hand, None would make the update_func more complex if all you want to do is
# concatenate.
contents = '' if first_time else file_util.read(fp, codec=codec)
updated_contents = update_func(contents)
file_util.save(fp, updated_contents, codec=codec)
if first_time:
# Stage the new file.
repo.add([filename])
def update_site_dot_py(clazz, d):
if not path.exists(d):
return
if not path.isdir(d):
raise RuntimeError('Not a directory: %s' % (d))
site_py_path = path.join(d, clazz.SITE_DOT_PY_FILENAME)
old_content = None
if path.exists(site_py_path):
if not path.isfile(site_py_path):
raise RuntimeError('Not a regular file: %s' % (site_py_path))
old_content = file_util.read(site_py_path)
if old_content == clazz.SITE_DOT_PY_CONTENT:
return
file_util.save(site_py_path,
content=clazz.SITE_DOT_PY_CONTENT,
mode=0o644)
def test_cyclic_dependency(self):
content = '''\
ape extends bonobo
something: yes
bonobo extends ape
something: yes
'''
tmp_dir = self.make_temp_dir()
tmp_file = path.join(tmp_dir, 'apes.config')
file_util.save(tmp_file, content=content)
s = SCL(tmp_dir, '*.config')
s.load()
with self.assertRaises(ERROR) as ctx:
s.section('ape')
self.assertTrue('Cyclic dependencies found: ape bonobo' in
ctx.exception.message)
def update_egg_directory(clazz, d):
if not path.exists(d):
return
if not path.isdir(d):
raise RuntimeError('Not a directory: %s' % (d))
eggs = clazz.list_eggs(d)
eggs = ['./%s' % (egg) for egg in eggs]
eggs_content = '\n'.join(eggs)
easy_install_dot_pth = path.join(d,
clazz.EASY_INSTALL_DOT_PTH_FILENAME)
easy_install_dot_pth_content = clazz.EASY_INSTALL_DOT_PTH_TEMPLATE % (
eggs_content)
file_util.save(easy_install_dot_pth,
content=easy_install_dot_pth_content,
mode=0o644)
clazz.update_site_dot_py(d)
def test_read_gitignore(self):
tmp_repo = self._create_tmp_repo()
new_files = self._create_tmp_files(tmp_repo)
git.add(tmp_repo, new_files)
git.commit(tmp_repo, 'nomsg\n', '.')
self.assertEqual(None, git.read_gitignore(tmp_repo))
file_util.save(path.join(tmp_repo, '.gitignore'),
content='foo.txt\nbar.txt\nBUILD\n*~\n')
git.add(tmp_repo, '.gitignore')
git.commit(tmp_repo, 'add .gitignore\n', '.')
self.assertEqual([
'foo.txt',
'bar.txt',
'BUILD',
'*~',
], git.read_gitignore(tmp_repo))
def test_duplicate_section(self):
content = '''\
organism
respiration: unknown
food: false
organism
respiration: unknown
food: true
'''
tmp_dir = self.make_temp_dir()
tmp_file = path.join(tmp_dir, 'organisms.config')
file_util.save(tmp_file, content=content)
s = SCL(tmp_dir, '*.config')
with self.assertRaises(ERROR) as ctx:
s.load()
self.assertTrue(
'Duplicate config section' in ctx.exception.message)
def test_which(self):
'Test which() Looks like a windows only test but works on unix as well.'
tmp_dir = self.make_temp_dir()
bin_dir = path.join(tmp_dir, 'bin')
content = '@echo off\n\recho kiwi\n\rexit 0\n\r'
temp_bat = file_util.save(path.join(bin_dir, 'kiwi_tool.bat'), content = content, mode = 0o0755)
self.assertEqual( None, file_path.which('kiwi_tool.bat') )
with env_override.path_append(bin_dir) as env:
expected_path = path.join(bin_dir, 'kiwi_tool.bat')
self.assertEqual( expected_path, file_path.which('kiwi_tool.bat') )
def test_trash_one(self):
ctx = self._make_context(timeout=0.100)
ctx.trash.start()
tmp = file_util.save(path.join(ctx.stuff_dir, 'foo.txt'),
content='foo\n')
self.assertEqual(['foo.txt'],
dir_util.list(ctx.stuff_dir, relative=True))
ctx.trash.trash(tmp)
time.sleep(0.250)
self.assertEqual([], dir_util.list(ctx.stuff_dir, relative=True))
ctx.trash.stop()
def _task_build(self, descriptor):
check.check_dim_task_descriptor(descriptor)
build_script = self._find_script(descriptor.step_name)
script_args = descriptor.script_args
bes_version_args = ['bes_version={}'.format(self.bes_version)]
log = path.join(self.logs_dir, descriptor.log_filename())
cmd = [self.egoist, 'script', 'run', build_script, descriptor.system
] + script_args + bes_version_args
self.blurb('building {} => {}'.format(descriptor.docker_tag,
path.relpath(log)))
rv = execute.execute(cmd,
raise_error=False,
non_blocking=descriptor.options.follow,
stderr_to_stdout=True,
cwd=os.getcwd())
file_util.save(log, content=rv.stdout)
success = rv.exit_code == 0
if not success and not descriptor.options.follow and descriptor.options.log:
print(rv.stdout)
return dim_task_result(success, log, descriptor)
def test_which_unix(self):
'Test which() in unix.'
tmp_dir = self.make_temp_dir()
bin_dir = path.join(tmp_dir, 'bin')
content = '!#/bin/bash\nechoecho kiwi\nexit 0\n'
temp_exe = file_util.save(path.join(bin_dir, 'fruit_kiwi_tool'),
content=content,
mode=0o0755)
self.assertEqual(None, which.which('fruit_kiwi_tool'))
with env_override.path_append(bin_dir) as env:
expected_path = path.join(bin_dir, 'fruit_kiwi_tool')
self.assertEqual(expected_path, which.which('fruit_kiwi_tool'))
def install_public_key(self, public_key, filename, add_authorized_key):
'''
Install a public ssh key pair with the following pattern:
${dit_ssh_dir}/${filename}.pub
'''
check.check_string(public_key)
check.check_string(filename)
check.check_bool(add_authorized_key)
public_key_filename = path.join(self._dot_ssh_dir, filename + '.pub')
file_util.save(public_key_filename,
content=public_key,
mode=0o0600,
codec='utf-8')
if add_authorized_key:
authorized_key = ssh_authorized_key.parse_text(public_key)
self.add_authorized_key(authorized_key.key_type,
authorized_key.key,
authorized_key.annotation)
return public_key_filename
def test_which_windows(self):
'Test which() in unix.'
tmp_dir = self.make_temp_dir()
bin_dir = path.join(tmp_dir, 'bin')
content = '@echo off\n\recho kiwi\n\rexit 0\n\r'
temp_bat = file_util.save(path.join(bin_dir, 'fruit_kiwi_tool.bat'),
content=content,
mode=0o0755)
self.assertEqual(None, which.which('fruit_kiwi_tool.bat'))
with env_override.path_append(bin_dir) as env:
expected_path = path.join(bin_dir, 'fruit_kiwi_tool.bat')
self.assertEqual(expected_path, which.which('fruit_kiwi_tool.bat'))
self.assertEqual(expected_path, which.which('fruit_kiwi_tool'))
def install_key_pair(self, public_key, private_key, filename):
'''
Install a ssh key pair with the following patterns:
${dit_ssh_dir}/${filename}
${dit_ssh_dir}/${filename}.pub
'''
check.check_string(public_key)
check.check_string(private_key)
check.check_string(filename)
public_key_filename = path.join(self._dot_ssh_dir, filename + '.pub')
private_key_filename = path.join(self._dot_ssh_dir, filename)
file_util.save(public_key_filename,
content=public_key,
mode=0o0600,
codec='utf-8')
file_util.save(private_key_filename,
content=private_key,
mode=0o0600,
codec='utf-8')
return self._installed_key_pair(public_key_filename,
private_key_filename)