def testAddAllFilesInPath(self):
tf = TempFileSystem()
tf.Make()
flist = filelist.FileList()
flist.AddAllFilesInPath(tf.root_in_tmp, tf.root_in_tmp)
self.assertTrue(flist.symlink_dir_list)
self.assertTrue(flist.file_list)
def testExtractTo(self):
flist = filelist.FileList()
tf = filelist_test.TempFileSystem()
tf.Clear()
tf.Make()
flist.AddFile(tf.root_in_tmp, tf.test_txt)
flist.AddSymLink(tf.root_in_tmp, tf.sym_dir)
bundle_zip = os.path.join(tf.root_tmp, 'bundle.zip')
car = cobalt_archive.CobaltArchive(bundle_zip)
car.MakeArchive(platform_name='fake',
platform_sdk_version='fake_sdk',
config='devel',
file_list=flist)
out_dir = os.path.join(tf.root_tmp, 'out')
car.ExtractTo(out_dir)
out_from_dir = os.path.join(out_dir, 'from_dir')
out_from_dir_lnk = os.path.join(out_dir, 'from_dir_lnk')
self.assertEqual(filelist.GetFileType(out_from_dir),
filelist.TYPE_DIRECTORY)
self.assertEqual(filelist.GetFileType(out_from_dir_lnk),
filelist.TYPE_SYMLINK_DIR)
resolved_from_link_path = os.path.join(
out_dir, port_symlink.ReadSymLink(out_from_dir_lnk))
self.assertEqual(os.path.abspath(out_from_dir),
os.path.abspath(resolved_from_link_path))
def testExecutionAttribute(self):
flist = filelist.FileList()
tf = filelist_test.TempFileSystem()
tf.Make()
# Execution bit seems to turn off the read bit, so we just set all
# read/write/execute bit for the user.
write_flags = stat.S_IXUSR | stat.S_IWUSR | stat.S_IRUSR
os.chmod(tf.test_txt, write_flags)
self.assertNotEqual(
0, write_flags & cobalt_archive._GetFilePermissions(tf.test_txt))
flist.AddFile(tf.root_tmp, tf.test_txt)
bundle_zip = os.path.join(tf.root_tmp, 'bundle.zip')
car = cobalt_archive.CobaltArchive(bundle_zip)
car.MakeArchive(platform_name='fake',
platform_sdk_version='fake_sdk',
config='devel',
file_list=flist)
# Now grab the json file and check that the file appears in the
# executable_file list.
json_str = car.ReadFile(
'__cobalt_archive/finalize_decompression/decompress.json')
decompress_dict = json.loads(json_str)
executable_files = decompress_dict.get('executable_files')
# Expect that the executable file appears in the executable_files.
self.assertTrue(executable_files)
archive_path = os.path.relpath(tf.test_txt, tf.root_tmp)
self.assertIn(archive_path, executable_files)
out_dir = os.path.join(tf.root_tmp, 'out')
car.ExtractTo(output_dir=out_dir)
out_file = os.path.join(out_dir, tf.test_txt)
self.assertTrue(_LongPathExists(out_file))
perms = cobalt_archive._GetFilePermissions(out_file)
self.assertTrue(perms & stat.S_IXUSR)
def testExtractFileWithLongFileName(self):
"""Tests that a long file name can be archived and extracted."""
flist = filelist.FileList()
tf = filelist_test.TempFileSystem()
tf.Clear()
tf.Make()
self.assertTrue(os.path.exists(tf.root_in_tmp))
suffix_path = os.path.join(
'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa',
'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa',
'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa',
'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa',
'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa',
'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa',
'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa',
'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa',
'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa', 'test.txt')
input_dst = os.path.join(tf.root_in_tmp, suffix_path)
out_dir = os.path.join(tf.root_tmp, 'out')
output_dst = os.path.join(out_dir, suffix_path)
_MoveFileWithLongPath(tf.test_txt, input_dst)
self.assertTrue(_LongPathExists(input_dst))
flist.AddFile(tf.root_in_tmp, input_dst)
bundle_zip = os.path.join(tf.root_tmp, 'bundle.zip')
car = cobalt_archive.CobaltArchive(bundle_zip)
car.MakeArchive(platform_name='fake',
platform_sdk_version='fake_sdk',
config='devel',
file_list=flist)
car.ExtractTo(out_dir)
self.assertTrue(_LongPathExists(output_dst))
def testAddSymlink(self):
tf = TempFileSystem()
tf.Make()
flist = filelist.FileList()
flist.AddFile(tf.root_tmp, tf.sym_dir)
flist.Print()
self.assertTrue(flist.symlink_dir_list)
self.assertFalse(flist.file_list)
def _MakeCobaltArchiveFromSource(output_archive_path,
platform_name,
config,
platform_sdk_version,
additional_buildinfo_dict,
include_black_box_tests):
"""Finds necessary files and makes an archive."""
_MakeDirs(os.path.dirname(output_archive_path))
out_directory = paths.BuildOutputDirectory(platform_name, config)
root_dir = os.path.abspath(
os.path.normpath(os.path.join(out_directory, '..', '..')))
flist = filelist.FileList()
inc_paths = _GetDeployPaths(platform_name, config)
logging.info('Adding binary files to bundle...')
for path in inc_paths:
path = os.path.join(out_directory, path)
if not os.path.exists(path):
logging.info('Skipping deploy path %s because it does not exist.',
path)
continue
logging.info(' adding %s', os.path.abspath(path))
flist.AddAllFilesInPath(root_dir=root_dir, sub_path=path)
logging.info('...done')
launcher_tools_path = os.path.join(
os.path.dirname(output_archive_path),
'____app_launcher')
if os.path.exists(launcher_tools_path):
port_symlink.Rmtree(launcher_tools_path)
logging.info('Adding app_launcher_files to bundle in %s',
os.path.abspath(launcher_tools_path))
try:
CopyAppLauncherTools(repo_root=paths.REPOSITORY_ROOT,
dest_root=launcher_tools_path,
additional_glob_patterns=[],
include_black_box_tests=include_black_box_tests)
flist.AddAllFilesInPath(root_dir=launcher_tools_path,
sub_path=launcher_tools_path)
logging.info('...done')
MakeCobaltArchiveFromFileList(
output_archive_path,
input_file_list=flist,
platform_name=platform_name,
platform_sdk_version=platform_sdk_version,
config=config,
additional_buildinfo_dict=additional_buildinfo_dict)
logging.info('...done')
finally:
port_symlink.Rmtree(launcher_tools_path)
def testAddAllFilesInPath(self):
tf = TempFileSystem()
tf.Make()
tf.MakeLongPathFile()
flist = filelist.FileList()
flist.AddAllFilesInPath(tf.root_in_tmp, tf.root_in_tmp)
self.assertTrue(flist.symlink_dir_list)
expected_file_list = [[
tf.test_txt, os.path.join('from_dir', 'test.txt')
],
[
tf.long_path_txt,
os.path.join('from_dir', LONG_SUB_DIRS,
'test2.txt')
]]
self.assertEqual(flist.file_list, expected_file_list)
def testFoldIdenticalFiles(self):
tf_root = filelist_test.TempFileSystem('bundler_fold')
tf_root.Clear()
tf1 = filelist_test.TempFileSystem(os.path.join('bundler_fold', '1'))
tf2 = filelist_test.TempFileSystem(os.path.join('bundler_fold', '2'))
tf1.Make()
tf2.Make()
flist = filelist.FileList()
subdirs = [tf1.root_in_tmp, tf2.root_in_tmp]
flist.AddAllFilesInPaths(tf_root.root_tmp, subdirs)
flist.Print()
identical_files = [tf1.test_txt, tf2.test_txt]
physical_files, copy_files = cobalt_archive._FoldIdenticalFiles(
identical_files)
self.assertEqual(tf1.test_txt, physical_files[0])
self.assertIn(tf1.test_txt, copy_files[0][0])
self.assertIn(tf2.test_txt, copy_files[0][1])
def testAddRelativeSymlink(self):
"""Tests the that adding a relative symlink works as expected."""
tf = TempFileSystem()
tf.Make()
flist = filelist.FileList()
in2 = os.path.join(tf.root_in_tmp, 'in2')
target_path = os.path.relpath(tf.from_dir, in2)
# Sanity check that target_path is relative.
self.assertIn('..', target_path)
port_symlink.MakeSymLink(target_path, in2)
self.assertTrue(port_symlink.IsSymLink(in2))
read_back_target_path = port_symlink.ReadSymLink(in2)
self.assertIn('..', read_back_target_path)
flist.AddFile(tf.root_tmp, in2)
flist.Print()
self.assertTrue(flist.symlink_dir_list)
symlink_entry = flist.symlink_dir_list[0][1:]
expected = [os.path.join('in', 'in2'), os.path.join('in', 'from_dir')]
self.assertEqual(expected, symlink_entry)
def testAddRelativeSymlink(self):
"""Tests that adding a relative symlink works as expected."""
tf = TempFileSystem()
tf.Make()
flist = filelist.FileList()
in2 = os.path.join(tf.root_in_tmp, 'subdir', 'in2')
target_path = os.path.relpath(tf.from_dir, os.path.dirname(in2))
# Sanity check that target_path is relative.
self.assertEqual(target_path, os.path.join('..', 'from_dir'))
# Create the link and check that it points to the correct folder.
port_symlink.MakeSymLink(target_path, in2)
self.assertTrue(port_symlink.IsSymLink(in2))
self.assertEqual(port_symlink.ReadSymLink(in2), target_path)
self.assertEqual(os.listdir(in2), ['test.txt'])
# Add the symlink to flist and check its content.
flist.AddFile(tf.root_tmp, in2)
flist.Print()
self.assertTrue(flist.symlink_dir_list)
expected = [
tf.root_tmp,
os.path.join('in', 'subdir', 'in2'),
os.path.join('in', 'from_dir')
]
self.assertEqual(flist.symlink_dir_list[0], expected)
def testMakesDeployInfo(self):
flist = filelist.FileList()
tf = filelist_test.TempFileSystem()
tf.Clear()
tf.Make()
bundle_zip = os.path.join(tf.root_tmp, 'bundle.zip')
car = cobalt_archive.CobaltArchive(bundle_zip)
car.MakeArchive(platform_name='fake',
platform_sdk_version='fake_sdk',
config='devel',
file_list=flist)
out_dir = os.path.join(tf.root_tmp, 'out')
car.ExtractTo(out_dir)
out_metadata_file = os.path.join(out_dir,
cobalt_archive._OUT_METADATA_PATH)
self.assertEqual(filelist.GetFileType(out_metadata_file),
filelist.TYPE_FILE)
with open(out_metadata_file) as fd:
text = fd.read()
js = json.loads(text)
self.assertTrue(js)
self.assertEqual(js['sdk_version'], 'fake_sdk')
self.assertEqual(js['platform'], 'fake')
self.assertEqual(js['config'], 'devel')
def testAddFile(self):
flist = filelist.FileList()
flist.AddFile(root_path=r'd1/d2', file_path=r'd1/d2/test.txt')
self.assertEqual(flist.file_list, [['d1/d2/test.txt', 'test.txt']])
def MakeArchive(self,
platform_name,
platform_sdk_version,
config,
file_list, # class FileList
additional_buildinfo_dict=None):
"""Creates an archive for the given platform and config."""
logging.info('Making cobalt archive...')
is_windows = port_symlink.IsWindows()
if additional_buildinfo_dict is None:
additional_buildinfo_dict = {}
if config not in GetAllConfigs():
raise ValueError('Expected %s to be one of %s'
% (config, GetAllConfigs()))
additional_buildinfo_dict = dict(additional_buildinfo_dict) # Copy
build_info_str = _GenerateBuildInfoStr(
platform_name=platform_name,
platform_sdk_version=platform_sdk_version,
config=config,
additional_buildinfo_dict=additional_buildinfo_dict)
with zipfile.ZipFile(self.archive_zip_path, mode='w',
compression=zipfile.ZIP_DEFLATED,
allowZip64=True) as zf:
# Copy the cobalt_archive_content directory into the root of the archive.
content_file_list = filelist.FileList()
content_file_list.AddAllFilesInPath(root_dir=_SRC_CONTENT_PATH,
sub_path=_SRC_CONTENT_PATH)
for file_path, archive_path in content_file_list.file_list:
# Skip the fake metadata.json file because the real one
# is a generated in it's place.
if os.path.basename(file_path) == 'metadata.json':
continue
zf.write(file_path, arcname=archive_path)
# Write out the metadata.
zf.writestr(_OUT_METADATA_PATH, build_info_str)
if file_list.file_list:
logging.info(' Compressing %d files', len(file_list.file_list))
executable_files = []
n_file_list = len(file_list.file_list)
progress_set = set()
for i in range(n_file_list):
# Logging every 5% increment during compression step.
prog = int((float(i)/n_file_list) * 100)
if prog not in progress_set:
progress_set.add(prog)
logging.info(' Compressed %d%%...', prog)
file_path, archive_path = file_list.file_list[i]
if not is_windows:
perms = _GetFilePermissions(file_path)
if (stat.S_IXUSR) & perms:
executable_files.append(archive_path)
# TODO: Use and implement _FoldIdenticalFiles() to reduce
# duplicate files. This will help platforms like nxswitch which include
# a lot of duplicate files for the sdk.
try:
zf.write(file_path, arcname=archive_path)
except WindowsError: # pylint: disable=undefined-variable
# Happens for long file path names.
zf.write(cobalt_archive_extract.ToWinUncPath(file_path),
arcname=archive_path)
if file_list.symlink_dir_list:
logging.info(' Compressing %d symlinks',
len(file_list.symlink_dir_list))
# Generate the decompress.json file used by decompress.py.
# Removes the first element which is the root directory, which is not
# important for symlink creation.
symlink_dir_list = [l[1:] for l in file_list.symlink_dir_list]
# Replace '\\' with '/'
symlink_dir_list = [_ToUnixPaths(l) for l in symlink_dir_list]
decompress_json_str = _JsonDumpPrettyPrint({
'symlink_dir': symlink_dir_list,
'symlink_dir_doc': '[link_dir_path, target_dir_path]',
'executable_files': executable_files,
})
zf.writestr(_OUT_DECOMP_JSON, decompress_json_str)
logging.info('Done...')