def test_file_or_dir(self):
tmp_file = temp_file.make_temp_dir()
tmp_dir = temp_file.make_temp_dir()
self.assertTrue(
file_type.matches(tmp_file, file_type.DIR | file_type.FILE))
self.assertTrue(
file_type.matches(tmp_dir, file_type.DIR | file_type.FILE))
def _make_context(self, niceness_level=None, timeout=None, deleter=None):
tmp_dir = temp_file.make_temp_dir()
trash_dir = path.join(tmp_dir, 'trash')
stuff_dir = path.join(tmp_dir, 'stuff')
trash = file_trash(temp_file.make_temp_dir(),
niceness_level=niceness_level,
timeout=timeout,
deleter=deleter)
return self._context(trash_dir, stuff_dir, trash)
def install_package(self, package_filename):
'Install a python package directly. Not always supported.'
check.check_string(package_filename)
self._log.log_method_d()
log_dir = temp_file.make_temp_dir(prefix='python_install_',
suffix='.dir',
delete=not self.options.debug)
install_log = path.join(log_dir, 'install.log')
self._log.log_d('install_package: install_log={}'.format(install_log))
cmd = [
package_filename,
'/log',
install_log,
'/quiet',
'/silent',
'InstallAllUsers=1',
'PrependPath=0',
'Shortcuts=0',
'AssociateFiles=0',
'Include_doc=0',
'Include_launcher=0',
'InstallLauncherAllUsers=0',
]
self._log.log_d('install_package: command={}'.format(' '.join(cmd)))
rv = execute.execute(cmd, stderr_to_stdout=True, raise_error=False)
self._log.log_d('install_package: exit_code={} output={}'.format(
rv.exit_code, rv.stdout))
if rv.exit_code != 0:
print(file_util.read(install_log, codec='utf-8'))
def __init__(self, root_dir = None, debug = False, items = None, users = None):
self.root_dir = root_dir or temp_file.make_temp_dir(delete = not debug)
if items:
self.write_temp_content(items)
self.server = None
self.port = None
self.users = users
def archive(clazz, address, revision, base_name, output_filename,
untracked = False, override_gitignore = None, debug = False):
'git archive with additional support to include untracked files for local repos.'
tmp_repo_dir = temp_file.make_temp_dir(delete = not debug)
if path.isdir(address):
excludes = git_ignore.read_gitignore_file(address)
file_copy.copy_tree(address, tmp_repo_dir, excludes = excludes)
if override_gitignore:
file_util.save(path.join(address, '.gitignore'), content = override_gitignore)
if untracked:
git_exe.call_git(tmp_repo_dir, [ 'add', '-A' ])
git_exe.call_git(tmp_repo_dir, [ 'commit', '-m', '"add untracked files just for tmp repo"' ])
else:
if untracked:
raise git_error('untracked can only be True for local repos.')
clazz.clone(address, tmp_repo_dir)
output_filename = path.abspath(output_filename)
file_util.mkdir(path.dirname(output_filename))
args = [
'archive',
'--format=tgz',
'--prefix=%s-%s/' % (base_name, revision),
'-o',
output_filename,
revision
]
rv = git_exe.call_git(tmp_repo_dir, args)
return rv
def _init_remote(self, content, prefix, commit_message=None):
if prefix:
remote_prefix = '{}remote-'.format(prefix)
else:
remote_prefix = 'remote-'
self._remote_repo = self._make_temp_repo(
init_args=['--bare', '--shared'],
debug=self._debug,
prefix=remote_prefix,
commit_message=commit_message)
if prefix:
local_prefix = '{}local-'.format(prefix)
else:
local_prefix = 'local-'
tmp_dir = temp_file.make_temp_dir(delete=not self._debug,
prefix=local_prefix)
if self._debug:
print('git_temp_repo: tmp_dir: %s' % (tmp_dir))
self._local_repo = git_repo(tmp_dir, address=self._remote_repo.root)
self._local_repo.clone()
if content:
self._local_repo.write_temp_content(items=content,
commit=True,
commit_message=commit_message)
self._local_repo.push('origin', 'master')
self.root = self._local_repo.root
self.address = self._remote_repo.root
self._transplant_methods(self._local_repo)
def _do_download_with_temp_ssh(clazz, address, revision, output_filename,
base_name, download_options, parsed_address,
ssh_credentials):
domain_name = parsed_address.service
clazz._log.log_d('_do_download_with_temp_ssh: domain_name={}'.format(domain_name))
temp_home = temp_file.make_temp_dir(delete = not download_options.debug)
if download_options.debug:
print('temp_home={}'.format(temp_home))
temp_ssh_dir = path.join(temp_home, '.ssh')
clazz._log.log_d('_do_download_with_temp_ssh: temp_ssh_dir={}'.format(temp_ssh_dir))
cm = ssh_config_manager(temp_ssh_dir)
installed = cm.install_key_pair_for_host(ssh_credentials.public_key,
ssh_credentials.private_key,
domain_name,
username = ssh_credentials.username,
host_key_type = ssh_credentials.host_key_type,
include_ip_address = True,
include_comment = True)
# use a temporary HOME with custom git and ssh config just for this download
with env_override.temp_home(use_temp_home = temp_home) as env:
git_config.set_identity('test', '*****@*****.**')
ssh_config_file = path.join(temp_ssh_dir, 'config')
ssh_command = 'ssh -F {}'.format(ssh_config_file)
git_config.set_value('core.sshCommand', ssh_command)
clazz._do_download(address, revision, output_filename, base_name, download_options)
file_util.remove(temp_home)
def _downlod_url(clazz, url, debug=False):
if not url_util.exists(url):
raise python_installer_error(
'No python.org package found: "{}"'.format(url))
tmp_dir = temp_file.make_temp_dir(suffix='-python-download',
delete=not debug)
basename = path.basename(url)
tmp_package = path.join(tmp_dir, basename)
url_util.download_to_file(url, tmp_package)
if debug:
print('tmp python package download: {}'.format(tmp_package))
expected_checksum = clazz._fetch_checksum(url)
if not expected_checksum:
raise python_installer_error(
'Failed to determine checksum for: {}'.format(url))
actual_checksum = file_util.checksum('md5', tmp_package)
if expected_checksum != actual_checksum:
msg = '''
CHECKSUM MISMATCH: url={url}
CHECKSUM MISMATCH: expected={expected}
CHECKSUM MISMATCH: actual={actual}
CHECKSUM MISMATCH: filename={filename}
CHECKSUM MISMATCH: run with --debug to keep and debug the download
'''.format(url=url,
expected=expected_checksum,
actual=actual_checksum,
filename=tmp_package)
raise python_installer_error(msg)
return tmp_package
def test_file_sync_change_one(self):
tmp_src_dir1 = self._make_temp_content([
'file foo.txt "first foo.txt\n"',
'file subdir/bar.txt "bar.txt\n"',
'file subdir/subberdir/baz.txt "baz.txt\n"',
'file emptyfile.txt',
'dir emptydir',
])
tmp_src_dir2 = self._make_temp_content([
'file foo.txt "second foo.txt\n"',
'file subdir/bar.txt "bar.txt\n"',
'file subdir/subberdir/baz.txt "baz.txt\n"',
'file emptyfile.txt',
'dir emptydir',
])
tmp_dst_dir = temp_file.make_temp_dir()
file_sync.sync(tmp_src_dir1, tmp_dst_dir)
file_sync.sync(tmp_src_dir2, tmp_dst_dir)
expected = [
self.native_filename('emptyfile.txt'),
self.native_filename('foo.txt'),
self.native_filename('subdir/bar.txt'),
self.native_filename('subdir/subberdir/baz.txt'),
]
self.assertEqual( expected, file_find.find(tmp_dst_dir, relative = True) )
self.assertEqual( 'second foo.txt\n', file_util.read(path.join(tmp_dst_dir, 'foo.txt'), codec = 'utf8') )
def test_compressed_no_uncompress(self):
tmp_dir = self._make_temp_content([
'file foo.txt "this is foo.txt\n"',
'file subdir/bar.txt "bar.txt\n"',
'file subdir/subberdir/baz.txt "this is baz.txt\n"',
'file emptyfile.txt',
'dir emptydir',
])
server = web_server_controller(file_web_server)
server.start(root_dir=tmp_dir)
port = server.address[1]
cache = http_download_cache(temp_file.make_temp_dir(), compressed=True)
url1 = self._make_url(port, 'foo.txt')
self.assertFalse(cache.has_url(url1))
self.assertEqual(0, cache.download_count)
cached_filename = cache.get_url(url1, uncompress=False)
tmp_uncompressed_file = self.make_temp_file()
compressed_file.uncompress(cached_filename, tmp_uncompressed_file)
self.assertEqual("this is foo.txt\n",
file_util.read(tmp_uncompressed_file, codec='utf-8'))
cached_filename = cache.get_url(url1, uncompress=False)
tmp_uncompressed_file = self.make_temp_file()
compressed_file.uncompress(cached_filename, tmp_uncompressed_file)
self.assertEqual("this is foo.txt\n",
file_util.read(tmp_uncompressed_file, codec='utf-8'))
server.stop()
def lfs_invalid_files(clazz, address, branch):
'''
Return a list of files that are GIT LFS invalid. That means:
- They match the .gitattributes
- They are not GIT LFS pointers but plain git files
'''
check.check_string(address)
check.check_string(branch)
'''\
Cloning into 'lfs-test'...
remote: Counting objects: 96, done.
remote: Compressing objects: 100% (89/89), done.
remote: Total 96 (delta 18), reused 0 (delta 0)
Receiving objects: 100% (96/96), 76.11 KiB | 1.14 MiB/s, done.
Resolving deltas: 100% (18/18), done.
Filtering content: 100% (2/2), 20.00 KiB | 7.00 KiB/s, done.
Encountered 3 file(s) that should have been pointers, but weren't:
honeydew melon.bytes
lemon.bytes
subdir/organic banana.bytes
'''
tmp_dir = temp_file.make_temp_dir()
repo = git_repo(tmp_dir, address=address)
options = git_clone_options()
options.branch = branch
clone_rv, sub_rv = repo.clone(options=options)
print('clone_rv: {}'.format(clone_rv))
print(' sub_rv: {}'.format(sub_rv))
return []
def search(clazz, tarball, pattern, ignore_case=False, whole_word=False):
'Return the output of either ag (silver searcher) or grep for the contents of an archive.'
ag = which.which('ag')
if ag:
cmd = [ag]
else:
grep = which.which('grep')
if not grep:
raise RuntimeError('No grep or ag found.')
cmd = [grep, '-r']
if ignore_case:
cmd.append('-i')
if whole_word:
cmd.append('-w')
cmd.extend([pattern, '.'])
tmp_dir = temp_file.make_temp_dir()
archiver.extract(tarball, tmp_dir, strip_common_ancestor=True)
result = execute.execute(cmd,
cwd=tmp_dir,
shell=True,
raise_error=False)
file_util.remove(tmp_dir)
return result
def test_basic(self):
# This is the document to write to the repository.
tmp_source_doc = self.make_temp_file(content='abc', suffix='-doc.txt')
# Here's a repository to put it in.
r = git_temp_repo(debug=self.DEBUG)
r.add_file('dummy.txt', content='dummy')
r.push('origin', 'master')
tmp_db_dir = self.make_temp_dir()
args = [
'git_repo_document',
'update',
tmp_source_doc,
r.address,
'master',
'--working-dir',
tmp_db_dir,
]
rv = self.run_program(self._program, args)
self.assertEqual(0, rv.exit_code)
# Also check doc is in repo. It will have the same name as the source doc.
filename = path.basename(tmp_source_doc)
r.pull()
contents = r.read_file(filename)
self.assertEqual(contents, 'abc')
# Since we now have a file in the repo, let's also test whether load_document can read it.
# Here's an auto-delete temp directory for the document DB's local repository.
tmp_db_dir2 = temp_file.make_temp_dir(delete=not self.DEBUG)
# By default, the CLI will put the output file in a file with the same name as the file in
# the repo, in the current directory.
tmp_target_filepath = path.join(getcwd(), filename)
if path.exists(tmp_target_filepath):
remove(tmp_target_filepath)
# Run the CLI.
args = [
'git_repo_document',
'load',
filename,
r.address,
'master',
'--working-dir',
tmp_db_dir2,
]
rv = self.run_program(self._program, args)
self.assertEqual(0, rv.exit_code)
# See if the file and contents are there.
actual = file_util.read(tmp_target_filepath)
self.assertEqual(actual, b'abc')
# This will cause an exception and fail the test if the file wasn't created by the CLI.
remove(tmp_target_filepath)
def recreate_temp_file(clazz, archive, base_dir, delete=True):
'Recreate the archive to a temp file.'
tmp_dir = temp_file.make_temp_dir(delete=True)
clazz.extract_all(archive, tmp_dir)
return clazz.create_temp_file(
archive_extension.extension_for_filename(archive),
tmp_dir,
base_dir=base_dir)
def make_temp_cloned_repo(self, prefix=None):
tmp_dir = temp_file.make_temp_dir(delete=not self._debug,
prefix=prefix)
if self._debug:
print('git_temp_repo: tmp_dir: %s' % (tmp_dir))
r = git_repo(tmp_dir, address=self._remote_repo.root)
r.clone()
return r
def _tmp_example_dmg(self, filename):
tmp_dir = temp_file.make_temp_dir(delete=not self.DEBUG)
tmp_example = path.join(tmp_dir, filename)
if self.DEBUG:
print('tmp_example: %s' % (tmp_example))
file_util.copy(self.data_path(filename),
tmp_example,
use_hard_link=True)
return tmp_example
def _do_download(clazz, address, revision, output_filename,
base_name, download_options):
clazz._log.log_d('_do_download: home={} real_home={}'.format(path.expanduser('~'), user.HOME))
tmp_root_dir = temp_file.make_temp_dir(delete = not download_options.debug)
if download_options.debug:
print('tmp_root_dir={}'.format(tmp_root_dir))
repo = git_repo(tmp_root_dir, address = address)
repo.clone(options = download_options)
repo.archive_to_file(base_name, revision, output_filename, archive_format = 'tgz')
def test_file_sync_with_mode(self):
tmp_src_dir = self._make_temp_content([
'file foo.txt "foo.txt\n" 644',
'file bar.sh "#!/bin/bash\necho bar\n" 755',
])
tmp_dst_dir = temp_file.make_temp_dir()
file_sync.sync(tmp_src_dir, tmp_dst_dir)
self.assertEqual( 0o0755, file_util.mode(path.join(tmp_dst_dir, 'bar.sh')) )
self.assertEqual( 0o0644, file_util.mode(path.join(tmp_dst_dir, 'foo.txt')) )
def test_extract_all(self):
tmp_dir = temp_file.make_temp_dir()
info1 = dmg.info()
dmg.extract(self._tmp_example_dmg('example.dmg'), tmp_dir)
info2 = dmg.info()
files = file_find.find(tmp_dir,
relative=True,
file_type=file_find.FILE_OR_LINK)
self.assertEqual(['foo.txt', 'link_to_foo.sh', 'subdir/bar.txt'],
files)
def clone_to_temp_dir(clazz, address, options=None, debug=False):
'Clone a git address to a temp dir'
tmp_dir = temp_file.make_temp_dir(delete=not debug)
clazz._LOG.log_d('clone_to_temp_dir: tmp_dir={}'.format(tmp_dir))
if debug:
print('clone_to_temp_dir: tmp_dir={}'.format(tmp_dir))
repo = git_repo(tmp_dir, address=address)
repo.clone(options=options)
assert repo.root == tmp_dir
return repo
def extract_member_to_file(self, member, filename):
tmp_dir = temp_file.make_temp_dir()
tmp_member = path.join(tmp_dir, member)
self.extract(tmp_dir, include=[member])
if not path.exists(tmp_member):
raise RuntimeError('Failed to extract member from {}: {}'.format(
self.filename, member))
if not path.isfile(tmp_member):
raise RuntimeError('Member is not a file {}: {}'.format(
self.filename, member))
file_util.rename(tmp_member, filename)
def _make_temp_archive_dmg(clazz, items, filename, mode):
tmp_dir = temp_file.make_temp_dir()
for item in items:
assert item
assert item.arcname
file_util.save(path.join(tmp_dir, item.arcname),
content=item.content)
tmp_dmg = temp_file.make_temp_file()
cmd = 'hdiutil create -srcfolder %s -ov -format UDZO %s' % (tmp_dir,
filename)
execute.execute(cmd)
file_util.remove(tmp_dir)
def _make_temp_archive_xz(clazz, items, filename, mode):
tmp_dir = temp_file.make_temp_dir()
for item in items:
assert item
assert item.arcname
file_util.save(path.join(tmp_dir, item.arcname),
content=item.content)
tmp_xz = temp_file.make_temp_file()
manifest_content = '\n'.join([item.arcname for item in items])
manifest = temp_file.make_temp_file(content=manifest_content)
cmd = 'tar Jcf %s -C %s -T %s' % (filename, tmp_dir, manifest)
execute.execute(cmd)
file_util.remove(tmp_dir)
def extract_all_temp_dir(clazz,
filename,
base_dir=None,
strip_common_ancestor=False,
strip_head=None,
delete=True):
tmp_dir = temp_file.make_temp_dir(delete=delete)
clazz.extract_all(filename,
tmp_dir,
base_dir=base_dir,
strip_common_ancestor=strip_common_ancestor,
strip_head=strip_head)
return tmp_dir
def run(self):
args = self.parser.parse_args()
blurb.set_verbose(args.verbose)
args.python_versions = dim_python.resolve_python_versions(
args.python_versions)
args.systems = dim_system.resolve_systems(args.systems)
args.steps = dim_step.resolve_steps(args.steps)
self.logs_dir = path.join(os.getcwd(), 'BUILD', 'logs')
self.egoist = self._find_egoist()
self.temp_dir = temp_file.make_temp_dir(suffix='build')
self.repo = git_repo('.', find_root=True)
if args.print_ecr_images:
self._print_ecr_images()
return 0
self.bes_version = args.bes_version or self.repo.greatest_remote_tag()
self.blurb(' steps: {}'.format(' '.join(args.steps)))
self.blurb(' systems: {}'.format(' '.join(args.systems)))
self.blurb(' python_versions: {}'.format(' '.join(
args.python_versions)))
self.blurb(' build: {}'.format(args.build))
self.blurb(' test: {}'.format(args.test))
self.blurb(' publish_egoist: {}'.format(args.publish_egoist))
self.blurb(' publish_image: {}'.format(args.publish_image))
self.blurb(' clean: {}'.format(args.clean))
self.blurb(' very_clean: {}'.format(args.very_clean))
self.blurb('bes_version: {}'.format(self.bes_version))
if args.clean and args.very_clean:
self.blurb('Only one of --clean or --very-clean should be given.')
return 1
if args.clean:
self._clean()
return 0
if args.very_clean:
self._very_clean()
return 0
result = self._do_run(args)
if result.success:
self.blurb('SUCCESS')
return 0
for failed_entry in result.failed_entries:
self.blurb('FAILED: {} => {}'.format(
failed_entry.descriptor.docker_tag, failed_entry.log))
return 1
def diff_contents(clazz, archive1, archive2, strip_common_ancestor=False):
'Return the output of diffing the contents of 2 archives.'
tmp_dir = temp_file.make_temp_dir(delete=True)
tmp_dir1 = path.join(tmp_dir, 'a')
tmp_dir2 = path.join(tmp_dir, 'b')
archiver.extract_all(archive1,
tmp_dir1,
strip_common_ancestor=strip_common_ancestor)
archiver.extract_all(archive2,
tmp_dir2,
strip_common_ancestor=strip_common_ancestor)
cmd = ['diff', '-u', '-r', tmp_dir1, tmp_dir2]
rv = execute.execute(cmd, raise_error=False, stderr_to_stdout=True)
return rv
def test_update_egg_directory(self):
tmp_dir = temp_file.make_temp_dir()
eggs = [
'foo-1.2.3-py2.7.egg',
'bar-6.6.6-py2.7.egg',
'baz-10.11.12-py2.7.egg',
]
for egg in eggs:
file_util.save(path.join(tmp_dir, egg), content = '%s\n' % (egg))
setup_tools.update_egg_directory(tmp_dir)
easy_install_dot_pth_path = path.join(tmp_dir, setup_tools.EASY_INSTALL_DOT_PTH_FILENAME)
actual_eggs = setup_tools.read_easy_install_pth(easy_install_dot_pth_path)
self.assertEqual( sorted(eggs), sorted(actual_eggs) )
def _init_local(self, content, prefix, commit_message=None):
tmp_dir = temp_file.make_temp_dir(delete=not self._debug,
prefix=prefix)
if self._debug:
print('git_temp_repo: tmp_dir: %s' % (tmp_dir))
self._local_repo = git_repo(tmp_dir, address=None)
self._local_repo.init()
if content:
self._local_repo.write_temp_content(items=content,
commit=True,
commit_message=commit_message)
self.root = self._local_repo.root
self.address = None
self._transplant_methods(self._local_repo)
def archive_to_file(clazz, root, prefix, revision, output_filename,
archive_format = None, short_hash = True):
'git archive to a archive file.'
prefix = file_util.ensure_rsep(prefix)
archive_format = archive_format or 'tgz'
output_filename = path.abspath(output_filename)
file_util.ensure_file_dir(output_filename)
if short_hash:
if clazz.is_long_hash(revision):
revision = clazz.short_hash(root, revision)
clazz.log.log_d('archive_to_file: revision={} output_filename={} archive_format={}'.format(revision, output_filename, archive_format))
tmp_dir = temp_file.make_temp_dir()
clazz.archive_to_dir(root, revision, tmp_dir)
archiver.create(output_filename, tmp_dir, base_dir = prefix, extension = archive_format)
def uninstall(self, version):
'''Uninstall a python by version or full_version.'
Uninstall a python version using any of these forms:
major.minor.revision
major.minor
'''
version = python_version.check_version_or_full_version(version)
installed_versions = self.installed_versions()
self._log.log_d('uninstall: installed_versions: {}'.format(
installed_versions.to_string()))
matching_versions = installed_versions.filter_by_version(version)
matching_versions.sort()
self._log.log_d('uninstall: matching_versions: {}'.format(
matching_versions.to_string()))
if not matching_versions:
raise python_installer_error('Not installed: {}'.format(version))
if len(matching_versions) != 1:
raise python_installer_error(
'Somehow multiple installed versions found for {}: {}'.format(
versiom, matching_versions.to_string()))
full_version = matching_versions[0]
self._log.log_d('uninstall: full_version: {}'.format(full_version))
assert full_version
old_package = self.download(full_version)
self._log.log_d('uninstall: old_package={}'.format(old_package))
log_dir = temp_file.make_temp_dir(prefix='python_install_',
suffix='.dir',
delete=not self.options.debug)
uninstall_log = path.join(log_dir, 'uninstall.log')
self._log.log_d('uninstall: uninstall_log={}'.format(uninstall_log))
cmd = [
old_package,
'/uninstall',
'/log',
uninstall_log,
'/quiet',
'/silent',
]
self._log.log_d('uninstall: command={}'.format(' '.join(cmd)))
rv = execute.execute(cmd, stderr_to_stdout=True, raise_error=False)
self._log.log_d('uninstall: exit_code={} output={}'.format(
rv.exit_code, rv.stdout))
if rv.exit_code != 0:
print(file_util.read(uninstall_log, codec='utf-8'))
