def _zip_files(filename, paths):
"""Creates a zip file that contains the specified files."""
# Set allowZip64=True so that large zip files can be handled.
with zipfile.ZipFile(filename, 'w', compression=zipfile.ZIP_DEFLATED,
allowZip64=True) as f:
for path in set(paths):
if path.startswith(build_common.get_stripped_dir()):
# When archiving a stripped file, use the path of the corresponding
# unstripped file as archive name
arcname = os.path.join(
build_common.get_build_dir(),
os.path.relpath(path, build_common.get_stripped_dir()))
f.write(path, arcname=arcname)
else:
f.write(path)
def _zip_files(filename, paths):
"""Creates a zip file that contains the specified files."""
# Set allowZip64=True so that large zip files can be handled.
with zipfile.ZipFile(filename,
'w',
compression=zipfile.ZIP_DEFLATED,
allowZip64=True) as f:
for path in set(paths):
if path.startswith(build_common.get_stripped_dir()):
# When archiving a stripped file, use the path of the corresponding
# unstripped file as archive name
arcname = os.path.join(
build_common.get_build_dir(),
os.path.relpath(path, build_common.get_stripped_dir()))
f.write(path, arcname=arcname)
else:
f.write(path)
def _has_stripped_binary(self, path):
"""Returns True if a stripped binary corresponding to |path| is found."""
relpath = os.path.relpath(path, build_common.get_build_dir())
if relpath.startswith('../'):
# The given file is not under build directory.
return False
# Returns True if there is a stripped file corresponding to the |path|.
return os.path.isfile(
os.path.join(build_common.get_stripped_dir(), relpath))
def _has_stripped_binary(self, path):
"""Returns True if a stripped binary corresponding to |path| is found."""
relpath = os.path.relpath(path, build_common.get_build_dir())
if relpath.startswith('../'):
# The given file is not under build directory.
return False
# Returns True if there is a stripped file corresponding to the |path|.
return os.path.isfile(
os.path.join(build_common.get_stripped_dir(), relpath))
def rsync(self, source_paths, remote_dest_root, exclude_paths=None):
"""Runs rsync command to copy files to remote host.
Sends |source_paths| to |remote_dest_root| directory in the remote machine.
|exclude_paths| can be used to exclude files or directories from the
sending list.
The files which are under |remote_dest_root| but not in the sending list
will be deleted.
Note:
- Known editor temporary files would never be sent.
- .pyc files in remote machine will *not* be deleted.
- If debug info is enabled and therer is a corresponding stripped binary,
the stripped binary will be sent, instead of original (unstripped)
binary.
- Files newly created in the remote machine will be deleted. Specifically,
if a file in the host machine is deleted, the corresponding file in the
remote machine is also deleted after the rsync.
Args:
source_paths: a list of paths to be sent. Each path can be a file or
a directory. If the path is directory, all files under the
directory will be sent.
remote_dest_root: the path to the destination directory in the
remote machine.
exclude_paths: an optional list of paths to be excluded from the
sending path list. Similar to |source_paths|, if a path is
directory, all paths under the directory will be excluded.
"""
filter_list = (
self._build_rsync_filter_list(source_paths, exclude_paths or []))
rsync_options = [
# The remote files need to be writable and executable by chronos. This
# option sets read, write, and execute permissions to all users.
'--chmod=a=rwx',
'--compress',
'--copy-links',
'--delete',
'--delete-excluded',
'--inplace',
'--perms',
'--progress',
'--recursive',
'--rsh=' + ' '.join(['ssh'] + self._build_shared_ssh_command_options()),
'--times',
]
dest = '%s@%s:%s' % (self._user, self._remote, remote_dest_root)
# Checks both whether to enable debug info and the existence of the stripped
# directory because build bots using test bundle may use the configure
# option with debug info enabled but binaries are not available in the
# stripped directory.
unstripped_paths = None
if (OPTIONS.is_debug_info_enabled() and
os.path.exists(build_common.get_stripped_dir())):
# When debug info is enabled, copy the corresponding stripped binaries if
# available to save the disk space on ChromeOS.
list_output = _run_command(
subprocess.check_output,
['rsync'] + filter_list + ['.', dest] + rsync_options +
['--list-only'])
paths = [
line.rsplit(' ', 1)[-1] for line in list_output.splitlines()[1:]]
unstripped_paths = [
path for path in paths if self._has_stripped_binary(path)]
# here, prepend filter rules to "protect" and "exclude" the files which
# have the corresponding stripped binary.
# Note: the stripped binraies will be sync'ed by the second rsync
# command, so it is necessary to "protect" here, too. Otherwise, the
# files will be deleted at the first rsync.
filter_list = list(itertools.chain.from_iterable(
(('--filter', 'P /' + path, '--exclude', '/' + path)
for path in unstripped_paths))) + filter_list
# Copy files to remote machine.
_run_command(subprocess.check_call,
['rsync'] + filter_list + ['.', dest] + rsync_options)
if unstripped_paths:
# Copy strippted binaries to the build/ directory in the remote machine
# directly.
stripped_binary_relative_paths = [
os.path.relpath(path, build_common.get_build_dir())
for path in unstripped_paths]
logging.debug('rsync stripped_binaries: %s', ', '.join(unstripped_paths))
dest_build = os.path.join(dest, build_common.get_build_dir())
# rsync results in error if the parent directory of the destination
# directory does not exist, so ensure it exists.
self.run('mkdir -p ' + dest_build.rsplit(':', 1)[-1])
_run_command(_check_call_with_input,
['rsync', '--files-from=-', build_common.get_stripped_dir(),
dest_build] + rsync_options,
input='\n'.join(stripped_binary_relative_paths))
def rsync(self, source_paths, remote_dest_root, exclude_paths=None):
"""Runs rsync command to copy files to remote host.
Sends |source_paths| to |remote_dest_root| directory in the remote machine.
|exclude_paths| can be used to exclude files or directories from the
sending list.
The files which are under |remote_dest_root| but not in the sending list
will be deleted.
Note:
- Known editor temporary files would never be sent.
- .pyc files in remote machine will *not* be deleted.
- If debug info is enabled and therer is a corresponding stripped binary,
the stripped binary will be sent, instead of original (unstripped)
binary.
- Files newly created in the remote machine will be deleted. Specifically,
if a file in the host machine is deleted, the corresponding file in the
remote machine is also deleted after the rsync.
Args:
source_paths: a list of paths to be sent. Each path can be a file or
a directory. If the path is directory, all files under the
directory will be sent.
remote_dest_root: the path to the destination directory in the
remote machine.
exclude_paths: an optional list of paths to be excluded from the
sending path list. Similar to |source_paths|, if a path is
directory, all paths under the directory will be excluded.
"""
filter_list = (self._build_rsync_filter_list(source_paths,
exclude_paths or []))
rsync_options = [
# The remote files need to be writable and executable by chronos. This
# option sets read, write, and execute permissions to all users.
'--chmod=a=rwx',
'--compress',
'--copy-links',
'--delete',
'--delete-excluded',
'--inplace',
'--perms',
'--progress',
'--recursive',
'--rsh=' +
' '.join(['ssh'] + self._build_shared_ssh_command_options()),
'--times',
]
dest = '%s@%s:%s' % (self._user, self._remote, remote_dest_root)
# Checks both whether to enable debug info and the existence of the stripped
# directory because build bots using test bundle may use the configure
# option with debug info enabled but binaries are not available in the
# stripped directory.
unstripped_paths = None
if (OPTIONS.is_debug_info_enabled()
and os.path.exists(build_common.get_stripped_dir())):
# When debug info is enabled, copy the corresponding stripped binaries if
# available to save the disk space on ChromeOS.
list_output = _run_command(subprocess.check_output,
['rsync'] + filter_list + ['.', dest] +
rsync_options + ['--list-only'])
paths = [
line.rsplit(' ', 1)[-1]
for line in list_output.splitlines()[1:]
]
unstripped_paths = [
path for path in paths if self._has_stripped_binary(path)
]
# here, prepend filter rules to "protect" and "exclude" the files which
# have the corresponding stripped binary.
# Note: the stripped binraies will be sync'ed by the second rsync
# command, so it is necessary to "protect" here, too. Otherwise, the
# files will be deleted at the first rsync.
filter_list = list(
itertools.chain.from_iterable(
(('--filter', 'P /' + path, '--exclude', '/' + path)
for path in unstripped_paths))) + filter_list
# Copy files to remote machine.
_run_command(subprocess.check_call,
['rsync'] + filter_list + ['.', dest] + rsync_options)
if unstripped_paths:
# Copy strippted binaries to the build/ directory in the remote machine
# directly.
stripped_binary_relative_paths = [
os.path.relpath(path, build_common.get_build_dir())
for path in unstripped_paths
]
logging.debug('rsync stripped_binaries: %s',
', '.join(unstripped_paths))
dest_build = os.path.join(dest, build_common.get_build_dir())
# rsync results in error if the parent directory of the destination
# directory does not exist, so ensure it exists.
self.run('mkdir -p ' + dest_build.rsplit(':', 1)[-1])
_run_command(_check_call_with_input,
[
'rsync', '--files-from=-',
build_common.get_stripped_dir(), dest_build
] + rsync_options,
input='\n'.join(stripped_binary_relative_paths))