def testUpdateMinimumCloudStorageInfo(self): dep_info1 = dependency_info.DependencyInfo('dep1', 'platform1', 'config_path1') cs_info2 = cloud_storage_info.CloudStorageInfo( cs_bucket='cs_bucket2', cs_hash='cs_hash2', download_path='download_path2', cs_remote_path='cs_remote_path2') dep_info2 = dependency_info.DependencyInfo('dep1', 'platform1', 'config_path2', cloud_storage_info=cs_info2) dep_info3 = dependency_info.DependencyInfo('dep1', 'platform1', 'config_path3') cs_info4 = cloud_storage_info.CloudStorageInfo( cs_bucket='cs_bucket4', cs_hash='cs_hash4', download_path='download_path4', cs_remote_path='cs_remote_path4') dep_info4 = dependency_info.DependencyInfo('dep1', 'platform1', 'config_path4', cloud_storage_info=cs_info4) self.assertEqual('dep1', dep_info1.dependency) self.assertEqual('platform1', dep_info1.platform) self.assertEqual(['config_path1'], dep_info1.config_paths) dep_info1.Update(dep_info2) self.assertFalse(dep_info1.local_paths) self.assertEqual('dep1', dep_info1.dependency) self.assertEqual('platform1', dep_info1.platform) self.assertEqual(['config_path1', 'config_path2'], dep_info1.config_paths) cs_info = dep_info1._cloud_storage_info self.assertEqual(cs_info, cs_info2) self.assertEqual('cs_bucket2', cs_info._cs_bucket) self.assertEqual('cs_hash2', cs_info._cs_hash) self.assertEqual('download_path2', cs_info._download_path) self.assertEqual('cs_remote_path2', cs_info._cs_remote_path) dep_info1.Update(dep_info3) self.assertEqual('dep1', dep_info1.dependency) self.assertEqual('platform1', dep_info1.platform) self.assertEqual(['config_path1', 'config_path2', 'config_path3'], dep_info1.config_paths) self.assertFalse(dep_info1.local_paths) cs_info = dep_info1._cloud_storage_info self.assertEqual(cs_info, cs_info2) self.assertEqual('cs_bucket2', cs_info._cs_bucket) self.assertEqual('cs_hash2', cs_info._cs_hash) self.assertEqual('download_path2', cs_info._download_path) self.assertEqual('cs_remote_path2', cs_info._cs_remote_path) self.assertRaises(ValueError, dep_info1.Update, dep_info4)
def testUpdateCloudStorageInfoWithVersions(self): dep_info1 = dependency_info.DependencyInfo('dep1', 'platform1', 'config_file1') dep_info2 = dependency_info.DependencyInfo( 'dep1', 'platform1', 'config_file2', cs_bucket='cs_bucket2', cs_hash='cs_hash2', download_path='download_path2', cs_remote_path='cs_remote_path2', version_in_cs='2.1.1') dep_info3 = dependency_info.DependencyInfo('dep1', 'platform1', 'config_file3') dep_info4 = dependency_info.DependencyInfo( 'dep1', 'platform1', 'config_file4', cs_bucket='cs_bucket4', cs_hash='cs_hash4', download_path='download_path4', cs_remote_path='cs_remote_path4') dep_info5 = dependency_info.DependencyInfo( 'dep1', 'platform1', 'config_file5', cs_bucket='cs_bucket5', cs_hash='cs_hash5', download_path='download_path5', cs_remote_path='cs_remote_path5') dep_info1.Update(dep_info2) self.assertEqual('cs_bucket2', dep_info1.cs_bucket) self.assertEqual('cs_hash2', dep_info1.cs_hash) self.assertEqual('download_path2', dep_info1.download_path) self.assertEqual('cs_remote_path2', dep_info1.cs_remote_path) self.assertEqual('2.1.1', dep_info1.version_in_cs) self.assertFalse(dep_info1.local_paths) dep_info1.Update(dep_info3) self.assertEqual('cs_bucket2', dep_info1.cs_bucket) self.assertEqual('cs_hash2', dep_info1.cs_hash) self.assertEqual('download_path2', dep_info1.download_path) self.assertEqual('cs_remote_path2', dep_info1.cs_remote_path) self.assertEqual('2.1.1', dep_info1.version_in_cs) self.assertFalse(dep_info1.local_paths) self.assertRaises(ValueError, dep_info1.Update, dep_info4) self.assertEqual('cs_bucket2', dep_info1.cs_bucket) self.assertEqual('cs_hash2', dep_info1.cs_hash) self.assertEqual('download_path2', dep_info1.download_path) self.assertEqual('cs_remote_path2', dep_info1.cs_remote_path) self.assertEqual('2.1.1', dep_info1.version_in_cs) self.assertFalse(dep_info1.local_paths) self.assertRaises(ValueError, dep_info1.Update, dep_info5)
def testAppendConflictingLocalFiles(self): dep_info1 = dependency_info.DependencyInfo( 'dep1', 'platform1', 'config_file1', local_paths=['path0', 'path1', 'path3', 'path5', 'path6']) dep_info2 = dependency_info.DependencyInfo( 'dep1', 'platform1', 'config_file2', local_paths=['path0', 'path2', 'path4', 'path5']) expected_local_paths = [ 'path0', 'path1', 'path3', 'path5', 'path6', 'path2', 'path4' ] dep_info1.Update(dep_info2) self.assertEquals(expected_local_paths, dep_info1.local_paths)
def testInitLocalPaths(self): dep_info = dependency_info.DependencyInfo( 'dep', 'platform', 'config_path', local_paths=['path0', 'path1']) self.assertEqual('dep', dep_info.dependency) self.assertEqual('platform', dep_info.platform) self.assertEqual(['config_path'], dep_info.config_paths) self.assertEqual(['path0', 'path1'], dep_info.local_paths) self.assertFalse(dep_info.has_cloud_storage_info)
def testUpdateAllInfo(self): dep_info1 = dependency_info.DependencyInfo('dep1', 'platform1', 'config_file1', local_paths=['path1']) dep_info2 = dependency_info.DependencyInfo( 'dep1', 'platform1', 'config_file2', cs_bucket='cs_bucket', cs_hash='cs_hash', download_path='download_path', cs_remote_path='cs_remote_path', local_paths=['path2']) dep_info3 = dependency_info.DependencyInfo('dep1', 'platform1', 'config_file3', local_paths=['path3']) dep_info4 = dependency_info.DependencyInfo( 'dep1', 'platform1', 'config_file4', cs_bucket='cs_bucket', cs_hash='cs_hash', download_path='download_path', cs_remote_path='cs_remote_path', local_paths=['path4']) dep_info1.Update(dep_info2) self.assertEqual('cs_bucket', dep_info1.cs_bucket) self.assertEqual('cs_hash', dep_info1.cs_hash) self.assertEqual('download_path', dep_info1.download_path) self.assertEqual('cs_remote_path', dep_info1.cs_remote_path) self.assertEqual(['path1', 'path2'], dep_info1.local_paths) dep_info1.Update(dep_info3) self.assertEqual('cs_bucket', dep_info1.cs_bucket) self.assertEqual('cs_hash', dep_info1.cs_hash) self.assertEqual('download_path', dep_info1.download_path) self.assertEqual('cs_remote_path', dep_info1.cs_remote_path) self.assertEqual(['path1', 'path2', 'path3'], dep_info1.local_paths) self.assertRaises(ValueError, dep_info1.Update, dep_info4)
def testUpdateRequiredArgsConflicts(self): dep_info1 = dependency_info.DependencyInfo( 'dep1', 'platform1', 'config_file1', local_paths=['path0', 'path1']) dep_info2 = dependency_info.DependencyInfo( 'dep1', 'platform2', 'config_file2', local_paths=['path0', 'path2']) dep_info3 = dependency_info.DependencyInfo( 'dep2', 'platform1', 'config_file3', local_paths=['path0', 'path3']) self.assertRaises(ValueError, dep_info1.Update, dep_info2) self.assertRaises(ValueError, dep_info1.Update, dep_info3) self.assertRaises(ValueError, dep_info3.Update, dep_info2)
def testInitAllInfo(self): dep_info = dependency_info.DependencyInfo( 'dep', 'platform', 'config_file', cs_bucket='cs_bucket', cs_hash='cs_hash', download_path='download_path', cs_remote_path='cs_remote_path', local_paths=['path0', 'path1']) self.assertEqual('dep', dep_info.dependency) self.assertEqual('platform', dep_info.platform) self.assertEqual(['config_file'], dep_info.config_files) self.assertEqual('cs_hash', dep_info.cs_hash) self.assertEqual('cs_bucket', dep_info.cs_bucket) self.assertEqual('cs_remote_path', dep_info.cs_remote_path) self.assertEqual('download_path', dep_info.download_path) self.assertEqual(['path0', 'path1'], dep_info.local_paths) self.assertFalse(dep_info.version_in_cs) dep_info = dependency_info.DependencyInfo( 'dep', 'platform', 'config_file', cs_bucket='cs_bucket', cs_hash='cs_hash', download_path='download_path', cs_remote_path='cs_remote_path', version_in_cs='version_in_cs', local_paths=['path0', 'path1']) self.assertEqual('dep', dep_info.dependency) self.assertEqual('platform', dep_info.platform) self.assertEqual(['config_file'], dep_info.config_files) self.assertEqual('cs_hash', dep_info.cs_hash) self.assertEqual('cs_bucket', dep_info.cs_bucket) self.assertEqual('cs_remote_path', dep_info.cs_remote_path) self.assertEqual('download_path', dep_info.download_path) self.assertEqual('version_in_cs', dep_info.version_in_cs) self.assertEqual(['path0', 'path1'], dep_info.local_paths)
def testInitAllInfo(self): cs_info = cloud_storage_info.CloudStorageInfo('cs_bucket', 'cs_hash', 'dowload_path', 'cs_remote_path') dep_info = dependency_info.DependencyInfo('dep', 'platform', 'config_path', cloud_storage_info=cs_info) self.assertEqual('dep', dep_info.dependency) self.assertEqual('platform', dep_info.platform) self.assertEqual(['config_path'], dep_info.config_paths) self.assertFalse(dep_info.local_paths) self.assertTrue(dep_info.has_cloud_storage_info)
def testInitLocalPaths(self): dep_info = dependency_info.DependencyInfo( 'dep', 'platform', 'config_file', local_paths=['path0', 'path1']) self.assertEqual('dep', dep_info.dependency) self.assertEqual('platform', dep_info.platform) self.assertEqual(['config_file'], dep_info.config_files) self.assertEqual(['path0', 'path1'], dep_info.local_paths) self.assertFalse(dep_info.version_in_cs) self.assertFalse(dep_info.cs_hash) self.assertFalse(dep_info.cs_bucket) self.assertFalse(dep_info.cs_remote_path) self.assertFalse(dep_info.download_path) self.assertFalse(dep_info.unzip_location) self.assertFalse(dep_info.path_within_archive)
def testUpdateCloudStorageInfoNoVersions(self): dep_info1 = dependency_info.DependencyInfo('dep1', 'platform1', 'config_file1') dep_info2 = dependency_info.DependencyInfo( 'dep1', 'platform1', 'config_file2', cs_bucket='cs_bucket', cs_hash='cs_hash', download_path='download_path', cs_remote_path='cs_remote_path') dep_info3 = dependency_info.DependencyInfo('dep1', 'platform1', 'config_file3') dep_info4 = dependency_info.DependencyInfo( 'dep1', 'platform1', 'config_file4', cs_bucket='cs_bucket', cs_hash='cs_hash', download_path='download_path', cs_remote_path='cs_remote_path') dep_info1.Update(dep_info2) self.assertEqual('cs_bucket', dep_info1.cs_bucket) self.assertEqual('cs_hash', dep_info1.cs_hash) self.assertEqual('download_path', dep_info1.download_path) self.assertEqual('cs_remote_path', dep_info1.cs_remote_path) self.assertFalse(dep_info1.local_paths) dep_info1.Update(dep_info3) self.assertEqual('cs_bucket', dep_info1.cs_bucket) self.assertEqual('cs_hash', dep_info1.cs_hash) self.assertEqual('download_path', dep_info1.download_path) self.assertEqual('cs_remote_path', dep_info1.cs_remote_path) self.assertFalse(dep_info1.local_paths) self.assertRaises(ValueError, dep_info1.Update, dep_info4)
def testInitRequiredInfo(self): # Must have a dependency, platform and file_path. self.assertRaises(ValueError, dependency_info.DependencyInfo, None, None, None) self.assertRaises(ValueError, dependency_info.DependencyInfo, 'dep', None, None) self.assertRaises(ValueError, dependency_info.DependencyInfo, None, 'plat', None) self.assertRaises(ValueError, dependency_info.DependencyInfo, None, None, 'config_path') # Empty DependencyInfo. empty_di = dependency_info.DependencyInfo('dep', 'plat', 'config_path') self.assertEqual('dep', empty_di.dependency) self.assertEqual('plat', empty_di.platform) self.assertEqual(['config_path'], empty_di.config_paths) self.assertFalse(empty_di.local_paths) self.assertFalse(empty_di.has_cloud_storage_info)
def testInitRequiredInfo(self): # Must have a dependency, platform and file_path. self.assertRaises(ValueError, dependency_info.DependencyInfo, None, None, None) self.assertRaises(ValueError, dependency_info.DependencyInfo, 'dep', None, None) self.assertRaises(ValueError, dependency_info.DependencyInfo, None, 'plat', None) self.assertRaises(ValueError, dependency_info.DependencyInfo, None, None, 'config_file') # Empty DependencyInfo. empty_di = dependency_info.DependencyInfo('dep', 'plat', 'config_file') self.assertFalse(empty_di.cs_bucket) self.assertFalse(empty_di.cs_hash) self.assertFalse(empty_di.download_path) self.assertFalse(empty_di.cs_remote_path) self.assertFalse(empty_di.local_paths) self.assertEqual('dep', empty_di.dependency) self.assertEqual('plat', empty_di.platform) self.assertEqual(['config_file'], empty_di.config_files)
def IterDependencyInfo(self): """ Yields a DependencyInfo for each dependency/platform pair. Raises: ReadWriteError: If called when the config is writable. ValueError: If any of the dependencies contain partial information for downloading from cloud_storage. (See dependency_info.py) """ if self._writable: raise exceptions.ReadWriteError( 'Trying to read dependency info from a writable config. File for ' 'config: %s' % self._config_path) base_path = os.path.dirname(self._config_path) for dependency in self._config_data: dependency_dict = self._config_data.get(dependency) platforms_dict = dependency_dict.get('file_info', {}) for platform in platforms_dict: platform_info = platforms_dict.get(platform) local_info = None local_paths = platform_info.get('local_paths', []) if local_paths: paths = [] for path in local_paths: path = self._FormatPath(path) paths.append( os.path.abspath(os.path.join(base_path, path))) local_info = local_path_info.LocalPathInfo(paths) cs_info = None cs_bucket = dependency_dict.get('cloud_storage_bucket') cs_base_folder = dependency_dict.get( 'cloud_storage_base_folder', '') download_path = platform_info.get('download_path') if download_path: download_path = self._FormatPath(download_path) download_path = os.path.abspath( os.path.join(base_path, download_path)) cs_hash = platform_info.get('cloud_storage_hash') if not cs_hash: raise exceptions.ConfigError( 'Dependency %s has cloud storage info on platform %s, but is ' 'missing a cloud storage hash.', dependency, platform) cs_remote_path = self._CloudStorageRemotePath( dependency, cs_hash, cs_base_folder) version_in_cs = platform_info.get('version_in_cs') zip_info = None path_within_archive = platform_info.get( 'path_within_archive') if path_within_archive: unzip_path = os.path.abspath( os.path.join( os.path.dirname(download_path), '%s_%s_%s' % (dependency, platform, cs_hash))) zip_info = archive_info.ArchiveInfo( download_path, unzip_path, path_within_archive) cs_info = cloud_storage_info.CloudStorageInfo( cs_bucket, cs_hash, download_path, cs_remote_path, version_in_cs=version_in_cs, archive_info=zip_info) dep_info = dependency_info.DependencyInfo( dependency, platform, self._config_path, local_path_info=local_info, cloud_storage_info=cs_info) yield dep_info
def testInitWithArchivePath(self): self.assertRaises(ValueError, dependency_info.DependencyInfo, 'dep', 'plat', 'config_file', path_within_archive='path_within_archive') self.assertRaises(ValueError, dependency_info.DependencyInfo, 'dep', 'plat', 'config_file', path_within_archive='path_within_archive', local_paths=['path2']) self.assertRaises(ValueError, dependency_info.DependencyInfo, 'dep', 'plat', 'config_file', cs_bucket='cs_bucket', cs_hash='cs_hash', cs_remote_path='cs_remote_path', path_within_archive='path_within_archive', local_paths=['path2']) self.assertRaises(ValueError, dependency_info.DependencyInfo, 'dep', 'plat', 'config_file', cs_bucket='cs_bucket', cs_hash='cs_hash', cs_remote_path='cs_remote_path', version_in_cs='version', path_within_archive='path_within_archive', local_paths=['path2']) dep_info = dependency_info.DependencyInfo( 'dep', 'platform', 'config_file', cs_bucket='cs_bucket', cs_hash='cs_hash', download_path='download_path', cs_remote_path='cs_remote_path', path_within_archive='path_within_archive') self.assertEqual('dep', dep_info.dependency) self.assertEqual('platform', dep_info.platform) self.assertEqual(['config_file'], dep_info.config_files) self.assertEqual('cs_hash', dep_info.cs_hash) self.assertEqual('cs_bucket', dep_info.cs_bucket) self.assertEqual('cs_remote_path', dep_info.cs_remote_path) self.assertEqual('download_path', dep_info.download_path) self.assertEqual('path_within_archive', dep_info.path_within_archive) self.assertFalse(dep_info.local_paths) self.assertFalse(dep_info.version_in_cs) dep_info = dependency_info.DependencyInfo( 'dep', 'platform', 'config_file', cs_bucket='cs_bucket', cs_hash='cs_hash', download_path='download_path', cs_remote_path='cs_remote_path', path_within_archive='path_within_archive', local_paths=['path']) self.assertEqual('dep', dep_info.dependency) self.assertEqual('platform', dep_info.platform) self.assertEqual(['config_file'], dep_info.config_files) self.assertEqual('cs_hash', dep_info.cs_hash) self.assertEqual('cs_bucket', dep_info.cs_bucket) self.assertEqual('cs_remote_path', dep_info.cs_remote_path) self.assertEqual('download_path', dep_info.download_path) self.assertEqual('path_within_archive', dep_info.path_within_archive) self.assertEqual(['path'], dep_info.local_paths) self.assertFalse(dep_info.version_in_cs) dep_info = dependency_info.DependencyInfo( 'dep', 'platform', 'config_file', cs_bucket='cs_bucket', cs_hash='cs_hash', download_path='download_path', cs_remote_path='cs_remote_path', version_in_cs='version_in_cs', path_within_archive='path_within_archive', local_paths=['path']) self.assertEqual('dep', dep_info.dependency) self.assertEqual('platform', dep_info.platform) self.assertEqual(['config_file'], dep_info.config_files) self.assertEqual('cs_hash', dep_info.cs_hash) self.assertEqual('cs_bucket', dep_info.cs_bucket) self.assertEqual('cs_remote_path', dep_info.cs_remote_path) self.assertEqual('download_path', dep_info.download_path) self.assertEqual('path_within_archive', dep_info.path_within_archive) self.assertEqual(['path'], dep_info.local_paths) self.assertEqual('version_in_cs', dep_info.version_in_cs)
def testUpdateMaxCloudStorageInfo(self): dep_info1 = dependency_info.DependencyInfo('dep1', 'platform1', 'config_path1') zip_info2 = archive_info.ArchiveInfo('archive_path2', 'unzip_path2', 'path_withing_archive2') cs_info2 = cloud_storage_info.CloudStorageInfo('cs_bucket2', 'cs_hash2', 'download_path2', 'cs_remote_path2', version_in_cs='2.1.1', archive_info=zip_info2) dep_info2 = dependency_info.DependencyInfo('dep1', 'platform1', 'config_path2', cloud_storage_info=cs_info2) dep_info3 = dependency_info.DependencyInfo('dep1', 'platform1', 'config_path3') zip_info4 = archive_info.ArchiveInfo('archive_path4', 'unzip_path4', 'path_withing_archive4') cs_info4 = cloud_storage_info.CloudStorageInfo('cs_bucket4', 'cs_hash4', 'download_path4', 'cs_remote_path4', version_in_cs='4.2.1', archive_info=zip_info4) dep_info4 = dependency_info.DependencyInfo('dep1', 'platform1', 'config_path4', cloud_storage_info=cs_info4) self.assertEqual('dep1', dep_info1.dependency) self.assertEqual('platform1', dep_info1.platform) self.assertEqual(['config_path1'], dep_info1.config_paths) dep_info1.Update(dep_info2) self.assertFalse(dep_info1.local_paths) self.assertEqual('dep1', dep_info1.dependency) self.assertEqual('platform1', dep_info1.platform) self.assertEqual(['config_path1', 'config_path2'], dep_info1.config_paths) cs_info = dep_info1._cloud_storage_info self.assertEqual(cs_info, cs_info2) self.assertEqual('cs_bucket2', cs_info._cs_bucket) self.assertEqual('cs_hash2', cs_info._cs_hash) self.assertEqual('download_path2', cs_info._download_path) self.assertEqual('cs_remote_path2', cs_info._cs_remote_path) self.assertEqual('cs_remote_path2', cs_info._cs_remote_path) dep_info1.Update(dep_info3) self.assertEqual('dep1', dep_info1.dependency) self.assertEqual('platform1', dep_info1.platform) self.assertEqual(['config_path1', 'config_path2', 'config_path3'], dep_info1.config_paths) self.assertFalse(dep_info1.local_paths) cs_info = dep_info1._cloud_storage_info self.assertEqual(cs_info, cs_info2) self.assertEqual('cs_bucket2', cs_info._cs_bucket) self.assertEqual('cs_hash2', cs_info._cs_hash) self.assertEqual('download_path2', cs_info._download_path) self.assertEqual('cs_remote_path2', cs_info._cs_remote_path) self.assertRaises(ValueError, dep_info1.Update, dep_info4)
def IterDependencyInfo(self): """ Yields a DependencyInfo for each dependency/platform pair. Raises: ReadWriteError: If called when the config is writable. ValueError: If any of the dependencies contain partial information for downloading from cloud_storage. (See dependency_info.py) """ if self._writable: raise exceptions.ReadWriteError( 'Trying to read dependency info from a writable config. File for ' 'config: %s' % self._config_path) base_path = os.path.dirname(self._config_path) for dependency in self._config_data: dependency_dict = self._config_data.get(dependency) cs_bucket = dependency_dict.get('cloud_storage_bucket') cs_base_folder = dependency_dict.get('cloud_storage_base_folder', '') platforms_dict = dependency_dict.get('file_info', {}) for platform in platforms_dict: platform_info = platforms_dict.get(platform) local_paths = platform_info.get('local_paths', []) if local_paths: paths = [] for path in local_paths: path = self._FormatPath(path) paths.append( os.path.abspath(os.path.join(base_path, path))) local_paths = paths download_path = platform_info.get('download_path', None) if download_path: download_path = self._FormatPath(download_path) download_path = os.path.abspath( os.path.join(base_path, download_path)) cs_remote_path = None cs_hash = platform_info.get('cloud_storage_hash', None) if cs_hash: cs_remote_file = '%s_%s' % (dependency, cs_hash) cs_remote_path = cs_remote_file if not cs_base_folder else ( '%s/%s' % (cs_base_folder, cs_remote_file)) version_in_cs = platform_info.get('version_in_cs', None) path_within_archive = platform_info.get( 'path_within_archive', None) if any([ download_path, cs_remote_path, cs_hash, version_in_cs, path_within_archive ]): dep_info = dependency_info.DependencyInfo( dependency, platform, self._config_path, cs_bucket=cs_bucket, cs_remote_path=cs_remote_path, download_path=download_path, cs_hash=cs_hash, version_in_cs=version_in_cs, path_within_archive=path_within_archive, local_paths=local_paths) else: dep_info = dependency_info.DependencyInfo( dependency, platform, self._config_path, local_paths=local_paths) yield dep_info