def _DoSetRevisionCmd(arguments):
package_name = arguments.setrevision__package
revision_num = arguments.setrevision__revision
revision_desc = revision_info.RevisionInfo(arguments.packages_desc)
package_targets = arguments.packages_desc.GetPackageTargetsForPackage(
package_name
)
for package_target in package_targets:
remote_package_key = package_locations.GetRemotePackageKey(
arguments.packages_desc.IsSharedPackage(package_name),
revision_num,
package_target,
package_name
)
temp_package_file = os.path.join(
arguments.revisions_dir,
os.path.basename(remote_package_key) + TEMP_SUFFIX)
pynacl.file_tools.MakeParentDirectoryIfAbsent(temp_package_file)
url = arguments.gsd_store.GetFile(remote_package_key, temp_package_file)
if url is None:
raise IOError('Could not download package file: %s' % remote_package_key)
package_desc = package_info.PackageInfo(temp_package_file)
logging.info('Setting %s:%s to revision %s',
package_target, package_name, revision_num)
revision_desc.SetTargetRevision(
package_name,
package_target,
package_desc
)
revision_file = package_locations.GetRevisionFile(
arguments.revisions_dir,
package_name
)
pynacl.file_tools.MakeParentDirectoryIfAbsent(revision_file)
revision_desc.SaveRevisionFile(revision_file)
CleanTempFiles(arguments.revisions_dir)
def test_DownloadArchiveMismatchFails(self):
# Check download archive fails when the hash does not match expected hash.
with pynacl.working_directory.TemporaryWorkingDirectory() as work_dir:
mock_tar = self.GenerateMockFile(work_dir)
fake_url = 'http://www.fake.com/archive.tar'
self._fake_downloader.StoreURL(fake_url, mock_tar)
package_desc = package_info.PackageInfo()
archive_desc = archive_info.ArchiveInfo(name='invalid_name.tar',
hash='invalid_hash',
url=fake_url)
package_desc.AppendArchive(archive_desc)
tar_dir = os.path.join(work_dir, 'tar_dir')
self.assertRaises(error.Error,
package_version.DownloadPackageArchives,
tar_dir,
'mismatch_target',
'mismatch_name',
package_desc,
downloader=self._fake_downloader.Download)
def test_RevisionFileMustSetAllTargets(self):
# Tests that a revision file fails if not all package targets are set.
package = package_info.PackageInfo()
package.AppendArchive(
archive_info.ArchiveInfo(name='test_name', hash='hash_value'))
package_targets = self._packages.GetPackageTargetsForPackage(
TEST_MULTI_PACKAGE_PACKAGE_TARGET)
self.assertTrue(
len(package_targets) > 0,
'Invalid test data, multiple package targets expected')
revision = revision_info.RevisionInfo(self._packages)
revision.SetRevisionNumber('123abc')
revision.SetTargetRevision(TEST_MULTI_PACKAGE_PACKAGE_TARGET,
package_targets[0], package)
with pynacl.working_directory.TemporaryWorkingDirectory() as work_dir:
revision_file = os.path.join(work_dir, 'incomplete_revision.json')
self.assertRaises(error.Error, revision.SaveRevisionFile,
revision_file)
def test_CleanupExtraFiles(self):
# Test that the cleanup function properly cleans up extra files
with pynacl.working_directory.TemporaryWorkingDirectory() as work_dir:
mock_tar = self.GenerateMockFile(work_dir)
tar_dir = os.path.join(work_dir, 'tar_dir')
package_target = 'custom_package_target'
package_name = 'custom_package'
package_desc = self.GeneratePackageInfo([mock_tar], )
package_file = package_locations.GetLocalPackageFile(
tar_dir, package_target, package_name)
package_desc.SavePackageFile(package_file)
self.CopyToLocalArchiveFile(mock_tar, tar_dir)
package_dir = os.path.dirname(package_file)
archive_file = package_locations.GetLocalPackageArchiveFile(
tar_dir, os.path.basename(mock_tar),
archive_info.GetArchiveHash(mock_tar))
extra_file = self.GenerateMockFile(tar_dir)
extra_file2 = self.GenerateMockFile(package_dir)
extra_dir = os.path.join(tar_dir, 'extra_dir')
os.makedirs(extra_dir)
extra_file3 = self.GenerateMockFile(extra_dir)
package_version.CleanupTarDirectory(tar_dir)
# Make sure all the key files were not deleted
self.assertTrue(os.path.isfile(package_file))
self.assertTrue(os.path.isfile(archive_file))
# Make sure package file can be loaded and nothing vital was deleted.
new_package_desc = package_info.PackageInfo(package_file)
# Make sure all the extra directories and files were deleted
self.assertFalse(os.path.isfile(extra_file))
self.assertFalse(os.path.isfile(extra_file2))
self.assertFalse(os.path.isfile(extra_file3))
self.assertFalse(os.path.isdir(extra_dir))
def test_AddArchive(self):
# Check that we can successfully add an archive.
archive_name = 'test_archive'
archive_hash = 'test_archive_hash'
archive_url = 'test_archive_url'
tar_src_dir = 'test_archive_dir'
extract_dir = 'test_extraction_dir'
package = package_info.PackageInfo()
archive = archive_info.ArchiveInfo(archive_name, archive_hash, archive_url,
tar_src_dir, extract_dir)
package.AppendArchive(archive)
archive_list = package.GetArchiveList()
self.assertEqual(len(archive_list), 1)
archive_item = archive_list[0].GetArchiveData()
self.assertEqual(archive_item.name, archive_name)
self.assertEqual(archive_item.hash, archive_hash)
self.assertEqual(archive_item.url, archive_url)
self.assertEqual(archive_item.tar_src_dir, tar_src_dir)
self.assertEqual(archive_item.extract_dir, extract_dir)
def test_UploadKeepsArchiveURL(self):
# Checks if the archive URL is kept after a package upload.
with pynacl.working_directory.TemporaryWorkingDirectory() as work_dir:
mock_tar = self.GenerateMockFile(work_dir)
mock_url = 'http://www.mock.com/mock.tar'
package_desc = self.GeneratePackageInfo(
[mock_tar], url_dict={mock_tar: mock_url})
package_file = os.path.join(work_dir, 'package_file.json')
package_desc.SavePackageFile(package_file)
tar_dir = os.path.join(work_dir, 'tar_dir')
package_target = 'custom_package_target'
package_name = 'custom_package'
package_revision = 10
package_version.UploadPackage(self._fake_storage,
package_revision,
tar_dir,
package_target,
package_name,
False,
custom_package_file=package_file)
self.assertEqual(self._fake_storage.WriteCount(), 2,
"Package did not get properly uploaded")
remote_package_key = package_locations.GetRemotePackageKey(
False, package_revision, package_target, package_name)
downloaded_package = os.path.join(work_dir,
'download_package.json')
package_info.DownloadPackageInfoFiles(
downloaded_package,
remote_package_key,
downloader=self._fake_storage.GetFile)
downloaded_package_desc = package_info.PackageInfo(
downloaded_package)
# Verify everything (including URL) still matches.
self.assertEqual(downloaded_package_desc, package_desc)
def GeneratePackageInfo(self,
archive_list,
url_dict={},
src_dir_dict={},
dir_dict={},
log_url_dict={}):
"""Generates a package_info.PackageInfo object for list of archives."
Args:
archive_list: List of file paths where package archives sit.
url_dict: dict of archive file path to URL if url exists.
src_dir_dict: dict of archive file path to source tar dir if exists.
dir_dict: dict of archive file path to root dir if exists.
"""
package_desc = package_info.PackageInfo()
for archive_file in archive_list:
archive_name = os.path.basename(archive_file)
if os.path.isfile(archive_file):
archive_hash = archive_info.GetArchiveHash(archive_file)
else:
archive_hash = 'invalid'
archive_url = url_dict.get(archive_file, None)
archive_src_tar_dir = src_dir_dict.get(archive_file, '')
archive_dir = dir_dict.get(archive_file, '')
archive_log_url = log_url_dict.get(archive_file, None)
archive_desc = archive_info.ArchiveInfo(
name=archive_name,
hash=archive_hash,
url=archive_url,
tar_src_dir=archive_src_tar_dir,
extract_dir=archive_dir,
log_url=archive_log_url)
package_desc.AppendArchive(archive_desc)
return package_desc
def test_ArchivePackageArchives(self):
# Check if we can archive a list of archives to the tar directory.
with pynacl.working_directory.TemporaryWorkingDirectory() as work_dir:
mock_tar1 = self.GenerateMockFile(work_dir,
mock_file='file1.tar',
contents='mock contents 1')
mock_tar2 = self.GenerateMockFile(work_dir,
mock_file='file2.tar',
contents='mock contents 2')
tar_dir = os.path.join(work_dir, 'tar_dir')
package_target = 'test_package_archives'
package_name = 'package_archives'
package_version.ArchivePackageArchives(tar_dir, package_target,
package_name,
[mock_tar1, mock_tar2])
package_file = package_locations.GetLocalPackageFile(
tar_dir, package_target, package_name)
expected_package_desc = self.GeneratePackageInfo(
[mock_tar1, mock_tar2])
package_desc = package_info.PackageInfo(package_file)
self.assertEqual(expected_package_desc, package_desc)
def ExtractPackageTargets(package_target_packages, tar_dir, dest_dir,
downloader=None):
"""Extracts package targets from the tar directory to the destination.
Each package archive within a package will be verified before being
extracted. If a package archive does not exist or does not match the hash
stored within the package file, it will be re-downloaded before being
extracted.
Args:
package_target_packages: List of tuples of package target and package names.
tar_dir: Source tar directory where package archives live.
dest_dir: Root destination directory where packages will be extracted to.
downloader: function which takes a url and a file path for downloading.
"""
if downloader is None:
downloader = gsd_storage.HttpDownload
for package_target, package_name in package_target_packages:
package_file = package_locations.GetLocalPackageFile(
tar_dir,
package_target,
package_name
)
package_desc = package_info.PackageInfo(package_file)
dest_package_dir = package_locations.GetFullDestDir(
dest_dir,
package_target,
package_name
)
dest_package_file = package_locations.GetDestPackageFile(
dest_dir,
package_target,
package_name
)
# Only do the extraction if the extract packages do not match.
if os.path.isfile(dest_package_file):
dest_package_desc = package_info.PackageInfo(dest_package_file)
if dest_package_desc == package_desc:
logging.debug('Skipping extraction for package (%s)', package_name)
continue
if os.path.isdir(dest_package_dir):
logging.info('Deleting old package directory: %s', dest_package_dir)
pynacl.file_tools.RemoveDir(dest_package_dir)
logging.info('Extracting package (%s) to directory: %s',
package_name, dest_package_dir)
for archive_desc in package_desc.GetArchiveList():
archive_file = package_locations.GetLocalPackageArchiveFile(
tar_dir,
package_target,
package_name,
archive_desc.name
)
# Upon extraction, some files may not be downloaded (or have stale files),
# we need to check the hash of each file and attempt to download it if
# they do not match.
archive_hash = package_info.GetArchiveHash(archive_file)
if archive_hash != archive_desc.hash:
logging.warn('Expected archive missing, downloading: %s',
archive_desc.name)
if archive_desc.url is None:
raise IOError('Invalid archive file and URL: %s' % archive_file)
pynacl.file_tools.MakeParentDirectoryIfAbsent(archive_file)
downloader(archive_desc.url, archive_file)
archive_hash = package_info.GetArchiveHash(archive_file)
if archive_hash != archive_desc.hash:
raise IOError('Downloaded archive file does not match hash.'
' [%s] Expected %s, received %s.' %
(archive_file, archive_desc.hash, archive_hash))
destination_dir = os.path.join(dest_package_dir, archive_desc.extract_dir)
logging.info('Extracting %s to %s...', archive_desc.name, destination_dir)
temp_dir = os.path.join(destination_dir, '.tmp')
pynacl.file_tools.RemoveDir(temp_dir)
os.makedirs(temp_dir)
with tarfile.TarFile(archive_file, 'r') as f:
f.extractall(temp_dir)
temp_src_dir = os.path.join(temp_dir, archive_desc.tar_src_dir)
pynacl.file_tools.MoveAndMergeDirTree(temp_src_dir, destination_dir)
pynacl.file_tools.RemoveDir(temp_dir)
pynacl.file_tools.MakeParentDirectoryIfAbsent(dest_package_file)
shutil.copy(package_file, dest_package_file)
def UploadPackage(storage, revision, tar_dir, package_target, package_name,
is_shared_package, annotate=False, custom_package_file=None):
"""Uploads a local package file to the supplied cloud storage object.
By default local package files are expected to be found in the standardized
location within the tar directory, however a custom package file may be
specified to upload from a different location. Package archives that do not
have their URL field set will automaticaly have the archives uploaded so that
someone accessing the package file from the cloud storage will also have
access to the package archives.
Args:
storage: Cloud storage object which supports PutFile and GetFile.
revision: SVN Revision number the package should be associated with.
tar_dir: Root tar directory where archives live.
package_target: Package target of the package to archive.
package_name: Package name of the package to archive.
is_shared_package: Is this package shared among all package targets?
custom_package_file: File location for a custom package file.
Returns:
Returns remote download key for the uploaded package file.
"""
if annotate:
print '@@@BUILD_STEP upload_package@@@'
if custom_package_file is not None:
local_package_file = custom_package_file
else:
local_package_file = package_locations.GetLocalPackageFile(
tar_dir,
package_target,
package_name
)
# Upload the package file and also upload any local package archives so
# that they are downloadable.
package_desc = package_info.PackageInfo(local_package_file)
upload_package_desc = package_info.PackageInfo()
for archive_desc in package_desc.GetArchiveList():
url = archive_desc.url
if url is None:
archive_file = package_locations.GetLocalPackageArchiveFile(
tar_dir,
package_target,
package_name,
archive_desc.name)
archive_hash = package_info.GetArchiveHash(archive_file)
if archive_hash is None:
raise IOError('Missing Archive File: %s' % archive_file)
elif archive_hash != archive_desc.hash:
raise IOError(
'Archive hash does not match package hash: %s' % archive_file
+ '\n Archive Hash: %s' % archive_hash
+ '\n Package Hash: %s' % archive_desc.hash
)
logging.warn('Missing archive URL: %s', archive_desc.name)
logging.warn('Uploading archive to be publically available...')
remote_archive_key = package_locations.GetRemotePackageArchiveKey(
archive_desc.name,
archive_desc.hash
)
url = storage.PutFile(
archive_file,
remote_archive_key,
clobber=True
)
if annotate:
print '@@@STEP_LINK@download@%s@@@' % url
upload_package_desc.AppendArchive(
archive_desc.name,
archive_desc.hash,
url=url,
tar_src_dir=archive_desc.tar_src_dir,
extract_dir=archive_desc.extract_dir
)
upload_package_file = local_package_file + '.upload'
pynacl.file_tools.MakeParentDirectoryIfAbsent(upload_package_file)
upload_package_desc.SavePackageFile(upload_package_file)
logging.info('Uploading package information: %s', package_name)
remote_package_key = package_locations.GetRemotePackageKey(
is_shared_package,
revision,
package_target,
package_name
)
url = storage.PutFile(upload_package_file, remote_package_key)
print '@@@STEP_LINK@download@%s@@@' % url
return remote_package_key
def ArchivePackageArchives(tar_dir, package_target, package_name, archives):
"""Archives local package archives to the tar directory.
Args:
tar_dir: Root tar directory where archives live.
package_target: Package target of the package to archive.
package_name: Package name of the package to archive.
archives: List of archive file paths where archives currently live.
Returns:
Returns the local package file that was archived.
"""
local_package_file = package_locations.GetLocalPackageFile(
tar_dir,
package_target,
package_name
)
archive_list = []
package_desc = package_info.PackageInfo()
for archive in archives:
archive_url = None
if '@' in archive:
archive, archive_url = archive.split('@', 1)
extract_param = ''
tar_src_dir = ''
extract_dir = ''
if ',' in archive:
archive, extract_param = archive.split(',', 1)
if ':' in extract_param:
tar_src_dir, extract_dir = extract_param.split(':', 1)
else:
tar_src_dir = extract_param
archive_hash = package_info.GetArchiveHash(archive)
if archive_hash is None:
raise IOError('Invalid package: %s.' % archive)
archive_name = os.path.basename(archive)
archive_list.append(archive)
package_desc.AppendArchive(
archive_name,
archive_hash,
url=archive_url,
tar_src_dir=tar_src_dir,
extract_dir=extract_dir
)
# We do not need to archive the package if it already matches. But if the
# local package file is invalid or does not match, then we should recreate
# the json file.
if os.path.isfile(local_package_file):
try:
current_package_desc = package_info.PackageInfo(local_package_file)
if current_package_desc == package_desc:
return
except ValueError:
pass
# Copy each of the packages over to the tar directory first.
for archive_file in archive_list:
archive_name = os.path.basename(archive_file)
local_archive_file = package_locations.GetLocalPackageArchiveFile(
tar_dir,
package_target,
package_name,
archive_name
)
logging.info('Archiving file: %s', archive_file)
pynacl.file_tools.MakeParentDirectoryIfAbsent(local_archive_file)
shutil.copyfile(archive_file, local_archive_file)
# Once all the copying is completed, update the local packages file.
logging.info('Package "%s" archived: %s', package_name, local_package_file)
pynacl.file_tools.MakeParentDirectoryIfAbsent(local_package_file)
package_desc.SavePackageFile(local_package_file)
return local_package_file
def DownloadPackageArchives(tar_dir, package_target, package_name, package_desc,
downloader=None):
"""Downloads package archives from the cloud to the tar directory.
Args:
tar_dir: Root tar directory where archives will be downloaded to.
package_target: Package target of the package to download.
package_name: Package name of the package to download.
package_desc: package_info object of the package to download.
downloader: function which takes a url and a file path for downloading.
Returns:
The list of files that were downloaded.
"""
downloaded_files = []
if downloader is None:
downloader = gsd_storage.HttpDownload
local_package_file = package_locations.GetLocalPackageFile(
tar_dir,
package_target,
package_name
)
# To ensure that we do not redownload extra archives that we already have,
# create a dictionary of old package archives that contains the hash of each
# package archive.
old_archives = {}
if os.path.isfile(local_package_file):
old_package_desc = package_info.PackageInfo(local_package_file)
old_archives_list = old_package_desc.GetArchiveList()
old_archive_names = [archive.name for archive in old_archives_list]
for archive_name in old_archive_names:
archive_file = package_locations.GetLocalPackageArchiveFile(
tar_dir,
package_target,
package_name,
archive_name
)
archive_hash = package_info.GetArchiveHash(archive_file)
if archive_hash is not None:
old_archives[archive_name] = archive_hash
# Download packages information file along with each of the package
# archives described in the information file. Also keep track of what
# new package names matches old package names. We will have to delete
# stale package names after we are finished.
for archive_info in package_desc.GetArchiveList():
old_hash = old_archives.get(archive_info.name, None)
if old_hash is not None:
old_archives.pop(archive_info.name)
if archive_info.hash == old_hash:
logging.debug('Skipping matching archive: %s', archive_info.name)
continue
local_archive_file = package_locations.GetLocalPackageArchiveFile(
tar_dir,
package_target,
package_name,
archive_info.name
)
pynacl.file_tools.MakeParentDirectoryIfAbsent(local_archive_file)
if archive_info.url is None:
raise IOError('Error, no URL for archive: %s' % archive_info.name)
logging.info('Downloading package archive: %s', archive_info.name)
downloader(archive_info.url, local_archive_file)
verified_hash = package_info.GetArchiveHash(local_archive_file)
if verified_hash != archive_info.hash:
raise IOError('Package hash check failed: %s != %s' %
(verified_hash, archive_info.hash))
downloaded_files.append(local_archive_file)
# Delete any stale left over packages.
for old_archive in old_archives:
archive_file = package_locations.GetLocalPackageArchiveFile(
tar_dir,
package_target,
package_name,
old_archive
)
os.unlink(archive_file)
return downloaded_files
def parse_info(self, line, file_num):
"""解析一行日志并完成提取'卡顿时长'和'绘制总帧数'.
参数:
line: 需解析的某一行日志.
file_num: log文件序号
返回值:
解析结束返回True
"""
if self.__state == IDLE:
"""解析log中Stats since"""
since = info.PackageInfo.parse_since(line)
if since != info.PARSE_NOT_FOUND:
if self.__package_dict.has_key(
long(since)) and file_num != LAST_FILE_NUM:
"""the package since has exist, do not parse, wait over"""
self.__state = CATON_TIME
elif not self.__package_dict.has_key(
long(since)) and file_num == LAST_FILE_NUM:
"""since the package is the last, do not parse, wait over"""
self.__state = CATON_TIME
else:
"""create new package info"""
self.__cur_package = info.PackageInfo(
self.package_name,
package_since=long(since),
file_num=file_num)
self.__state = TOTAL_FRAMES
elif self.__state == TOTAL_FRAMES:
"""统计总绘制帧数."""
total_frames = info.PackageInfo.parse_total_frame(line)
if total_frames != info.PARSE_NOT_FOUND and self.__cur_package:
self.__cur_package.total_frame = int(total_frames)
self.__state = CATON_TIME
elif self.__state == CATON_TIME:
"""统计卡顿绘制时长"""
caton_time = info.PackageInfo.parse_caton(line)
if caton_time != info.PARSE_NOT_FOUND:
if self.__cur_package:
self.__cur_package.caton_time = int(caton_time)
if self.__package_dict.has_key(self.__cur_package.package_since) \
and file_num == LAST_FILE_NUM:
"""减掉log文件中第一轮数据(测试前数据)"""
sub_success = self.__package_dict[
self.__cur_package.package_since].subtract(
self.__cur_package)
if not sub_success:
print 'Fail to remove the first round data'
elif not self.__package_dict.has_key(
self.__cur_package.package_since):
"""检查是否已经加入，如果没有再加入到字典中"""
self.__package_dict[self.__cur_package.
package_since] = self.__cur_package
if self.__last_file_num != file_num:
self.__last_file_num = file_num
else:
"""一个log文件中同一个包名的绘制信息出现多次"""
self.__has_two_statics_one_file = True
self.reset_state()
return True
return False
def ExtractPackageTargets(package_target_packages, tar_dir, dest_dir,
downloader=None, skip_missing=False, quiet=False):
"""Extracts package targets from the tar directory to the destination.
Each package archive within a package will be verified before being
extracted. If a package archive does not exist or does not match the hash
stored within the package file, it will be re-downloaded before being
extracted.
Args:
package_target_packages: List of tuples of package target and package names.
tar_dir: Source tar directory where package archives live.
dest_dir: Root destination directory where packages will be extracted to.
downloader: function which takes a url and a file path for downloading.
"""
if downloader is None:
downloader = pynacl.gsd_storage.HttpDownload
for package_target, package_name in package_target_packages:
package_file = package_locations.GetLocalPackageFile(tar_dir,
package_target,
package_name)
package_desc = package_info.PackageInfo(package_file,
skip_missing=skip_missing)
dest_package_dir = package_locations.GetFullDestDir(dest_dir,
package_target,
package_name)
dest_package_file = package_locations.GetDestPackageFile(dest_dir,
package_target,
package_name)
# Only do the extraction if the extract packages do not match.
if os.path.isfile(dest_package_file):
try:
dest_package_desc = package_info.PackageInfo(dest_package_file)
if dest_package_desc == package_desc:
logging.debug('Skipping extraction for package (%s)', package_name)
continue
except:
# Destination package file cannot be trusted, if invalid re-extract.
pass
# Delete the old package file before we extract.
os.unlink(dest_package_file)
if os.path.isdir(dest_package_dir):
logging.debug('Deleting old package directory: %s', dest_package_dir)
pynacl.file_tools.RemoveDir(dest_package_dir)
logging.info('Extracting package (%s) to directory: %s',
package_name, dest_package_dir)
archive_list = package_desc.GetArchiveList()
num_archives = len(archive_list)
for index, archive_obj in enumerate(archive_list):
archive_desc = archive_obj.GetArchiveData()
archive_file = package_locations.GetLocalPackageArchiveFile(
tar_dir,
package_target,
package_name,
archive_desc.name
)
# Upon extraction, some files may not be downloaded (or have stale files),
# we need to check the hash of each file and attempt to download it if
# they do not match.
archive_hash = archive_info.GetArchiveHash(archive_file)
if archive_hash != archive_desc.hash:
if archive_desc.url is None:
if skip_missing:
logging.info('Skipping extraction of missing archive: %s' %
archive_file)
continue
raise IOError('Invalid archive file and URL: %s' % archive_file)
logging.warn('Expected archive missing, downloading: %s',
archive_desc.name)
pynacl.file_tools.MakeParentDirectoryIfAbsent(archive_file)
downloader(archive_desc.url, archive_file)
archive_hash = archive_info.GetArchiveHash(archive_file)
if archive_hash != archive_desc.hash:
raise IOError('Downloaded archive file does not match hash.'
' [%s] Expected %s, received %s.' %
(archive_file, archive_desc.hash, archive_hash))
destination_dir = os.path.join(dest_package_dir, archive_desc.extract_dir)
logging.info('Extracting %s (%d/%d)' %
(archive_desc.name, index+1, num_archives))
temp_dir = os.path.join(destination_dir, '.tmp')
pynacl.file_tools.RemoveDir(temp_dir)
os.makedirs(temp_dir)
tar_output = not quiet
tar = cygtar.CygTar(archive_file, 'r:*', verbose=tar_output)
curdir = os.getcwd()
os.chdir(temp_dir)
try:
tar.Extract()
tar.Close()
finally:
os.chdir(curdir)
temp_src_dir = os.path.join(temp_dir, archive_desc.tar_src_dir)
pynacl.file_tools.MoveAndMergeDirTree(temp_src_dir, destination_dir)
pynacl.file_tools.RemoveDir(temp_dir)
pynacl.file_tools.MakeParentDirectoryIfAbsent(dest_package_file)
package_desc.SavePackageFile(dest_package_file)
def ArchivePackageArchives(tar_dir, package_target, package_name, archives,
extra_archives=[]):
"""Archives local package archives to the tar directory.
Args:
tar_dir: Root tar directory where archives live.
package_target: Package target of the package to archive.
package_name: Package name of the package to archive.
archives: List of archive file paths where archives currently live.
extra_archives: Extra archives that are expected to be build elsewhere.
Returns:
Returns the local package file that was archived.
"""
local_package_file = package_locations.GetLocalPackageFile(tar_dir,
package_target,
package_name)
valid_archive_files = set()
archive_list = []
package_desc = package_info.PackageInfo()
package_archives = ([(archive, False) for archive in archives] +
[(archive, True) for archive in extra_archives])
for archive, skip_missing in package_archives:
archive_url = None
if '@' in archive:
archive, archive_url = archive.split('@', 1)
extract_param = ''
tar_src_dir = ''
extract_dir = ''
if ',' in archive:
archive, extract_param = archive.split(',', 1)
if ':' in extract_param:
tar_src_dir, extract_dir = extract_param.split(':', 1)
else:
tar_src_dir = extract_param
archive_hash = archive_info.GetArchiveHash(archive)
archive_name = os.path.basename(archive)
archive_desc = archive_info.ArchiveInfo(archive_name,
archive_hash,
url=archive_url,
tar_src_dir=tar_src_dir,
extract_dir=extract_dir)
package_desc.AppendArchive(archive_desc)
if archive_hash is None:
if skip_missing:
logging.info('Skipping archival of missing file: %s', archive)
continue
raise IOError('Invalid package: %s.' % archive)
archive_list.append(archive)
archive_basename = os.path.basename(archive)
archive_json = archive_basename + '.json'
valid_archive_files.update([archive_basename, archive_json])
# Delete any stale archive files
local_archive_dir = package_locations.GetLocalPackageArchiveDir(
tar_dir,
package_target,
package_name)
if os.path.isdir(local_archive_dir):
for dir_item in os.listdir(local_archive_dir):
if dir_item in valid_archive_files:
continue
item_path = os.path.join(local_archive_dir, dir_item)
if os.path.isdir(item_path):
pynacl.file_tools.RemoveDir(item_path)
else:
pynacl.file_tools.RemoveFile(item_path)
# We do not need to archive the package if it already matches. But if the
# local package file is invalid or does not match, then we should recreate
# the json file.
if os.path.isfile(local_package_file):
try:
current_package_desc = package_info.PackageInfo(local_package_file,
skip_missing=True)
if current_package_desc == package_desc:
return
except ValueError:
pass
# Copy each of the packages over to the tar directory first.
for archive_file in archive_list:
archive_name = os.path.basename(archive_file)
local_archive_file = package_locations.GetLocalPackageArchiveFile(
tar_dir,
package_target,
package_name,
archive_name)
logging.info('Archiving file: %s', archive_file)
pynacl.file_tools.MakeParentDirectoryIfAbsent(local_archive_file)
shutil.copyfile(archive_file, local_archive_file)
# Once all the copying is completed, update the local packages file.
logging.info('Package "%s" archived: %s', package_name, local_package_file)
pynacl.file_tools.MakeParentDirectoryIfAbsent(local_package_file)
package_desc.SavePackageFile(local_package_file)
return local_package_file
def DownloadPackageArchives(tar_dir, package_target, package_name, package_desc,
downloader=None, revision_num=None):
"""Downloads package archives from the cloud to the tar directory.
Args:
tar_dir: Root tar directory where archives will be downloaded to.
package_target: Package target of the package to download.
package_name: Package name of the package to download.
package_desc: package_info object of the package to download.
downloader: function which takes a url and a file path for downloading.
Returns:
The list of files that were downloaded.
"""
downloaded_files = []
if downloader is None:
downloader = pynacl.gsd_storage.HttpDownload
local_package_file = package_locations.GetLocalPackageFile(tar_dir,
package_target,
package_name)
# To ensure that we do not redownload extra archives that we already have,
# create a dictionary of old package archives that contains the hash of each
# package archive.
old_archives = {}
if os.path.isfile(local_package_file):
try:
old_package_desc = package_info.PackageInfo(local_package_file)
old_archives_list = old_package_desc.GetArchiveList()
old_archive_names = [archive.GetArchiveData().name
for archive
in old_archives_list]
for archive_name in old_archive_names:
archive_file = package_locations.GetLocalPackageArchiveFile(
tar_dir,
package_target,
package_name,
archive_name
)
archive_hash = archive_info.GetArchiveHash(archive_file)
if archive_hash is not None:
old_archives[archive_name] = archive_hash
except:
# Nothing can be trusted here anymore, delete all package archives.
archive_directory = package_locations.GetLocalPackageArchiveDir(
tar_dir,
package_target,
package_name
)
os.unlink(local_package_file)
pynacl.file_tools.RemoveDir(archive_directory)
# Download packages information file along with each of the package
# archives described in the information file. Also keep track of what
# new package names matches old package names. We will have to delete
# stale package names after we are finished.
update_archives = []
for archive_obj in package_desc.GetArchiveList():
archive_desc = archive_obj.GetArchiveData()
old_hash = old_archives.get(archive_desc.name, None)
if old_hash is not None:
old_archives.pop(archive_desc.name)
if archive_desc.hash == old_hash:
logging.debug('Skipping matching archive: %s', archive_desc.name)
continue
update_archives.append(archive_obj)
if update_archives:
logging.info('--Syncing %s to revision %s--' % (package_name, revision_num))
num_archives = len(update_archives)
for index, archive_obj in enumerate(update_archives):
archive_desc = archive_obj.GetArchiveData()
local_archive_file = package_locations.GetLocalPackageArchiveFile(
tar_dir,
package_target,
package_name,
archive_desc.name
)
pynacl.file_tools.MakeParentDirectoryIfAbsent(local_archive_file)
if archive_desc.url is None:
raise IOError('Error, no URL for archive: %s' % archive_desc.name)
logging.info('Downloading package archive: %s (%d/%d)' %
(archive_desc.name, index+1, num_archives))
try:
downloader(archive_desc.url, local_archive_file)
except Exception as e:
raise IOError('Could not download URL (%s): %s' %
(archive_desc.url, e))
verified_hash = archive_info.GetArchiveHash(local_archive_file)
if verified_hash != archive_desc.hash:
raise IOError('Package hash check failed: %s != %s' %
(verified_hash, archive_desc.hash))
downloaded_files.append(local_archive_file)
# Delete any stale left over packages.
for old_archive in old_archives:
archive_file = package_locations.GetLocalPackageArchiveFile(
tar_dir,
package_target,
package_name,
old_archive)
os.unlink(archive_file)
# Save the package file so we know what we currently have.
package_desc.SavePackageFile(local_package_file)
return downloaded_files
