def testThen(self): def assertIs42(val): self.assertEqual(val, 42) return val then = Future(value=42).Then(assertIs42) # Shouldn't raise an error. self.assertEqual(42, then.Get()) # Test raising an error. then = Future(value=41).Then(assertIs42) self.assertRaises(AssertionError, then.Get) # Test setting up an error handler. def handle(error): if isinstance(error, ValueError): return 'Caught' raise error def raiseValueError(): raise ValueError def raiseException(): raise Exception then = Future(callback=raiseValueError).Then(assertIs42, handle) self.assertEqual('Caught', then.Get()) then = Future(callback=raiseException).Then(assertIs42, handle) self.assertRaises(Exception, then.Get) # Test chains of thens. addOne = lambda val: val + 1 then = Future(value=40).Then(addOne).Then(addOne).Then(assertIs42) # Shouldn't raise an error. self.assertEqual(42, then.Get()) # Test error in chain. then = Future(value=40).Then(addOne).Then(assertIs42).Then(addOne) self.assertRaises(AssertionError, then.Get) # Test handle error in chain. def raiseValueErrorWithVal(val): raise ValueError then = Future(value=40).Then(addOne).Then(raiseValueErrorWithVal).Then( addOne, handle).Then(lambda val: val + ' me') self.assertEquals(then.Get(), 'Caught me') # Test multiple handlers. def myHandle(error): if isinstance(error, AssertionError): return 10 raise error then = Future(value=40).Then(assertIs42).Then(addOne, handle).Then( addOne, myHandle) self.assertEquals(then.Get(), 10)
class GithubFileSystem(FileSystem): """FileSystem implementation which fetches resources from github. """ def __init__(self, fetcher, object_store, blobstore): self._fetcher = fetcher self._object_store = object_store self._blobstore = blobstore self._version = None self._GetZip(self.Stat(ZIP_KEY).version) def _GetZip(self, version): blob = self._blobstore.Get(_MakeKey(version), blobstore.BLOBSTORE_GITHUB) if blob is not None: self._zip_file = Future(value=ZipFile(StringIO(blob))) else: self._zip_file = Future( delegate=_AsyncFetchFutureZip(self._fetcher, self._blobstore, version, key_to_delete=self._version)) self._version = version def _ReadFile(self, path): zip_file = self._zip_file.Get() prefix = zip_file.namelist()[0][:-1] return zip_file.read(prefix + path) def _ListDir(self, path): filenames = self._zip_file.Get().namelist() # Take out parent directory name (GoogleChrome-chrome-app-samples-c78a30f) filenames = [f[len(filenames[0]) - 1:] for f in filenames] # Remove the path of the directory we're listing from the filenames. filenames = [f[len(path):] for f in filenames if f != path and f.startswith(path)] # Remove all files not directly in this directory. return [f for f in filenames if f[:-1].count('/') == 0] def Read(self, paths, binary=False): version = self.Stat(ZIP_KEY).version if version != self._version: self._GetZip(version) result = {} for path in paths: if path.endswith('/'): result[path] = self._ListDir(path) else: result[path] = self._ReadFile(path) return Future(value=result) def Stat(self, path): version = self._object_store.Get(path, object_store.GITHUB_STAT).Get() if version is not None: return StatInfo(version) version = json.loads( self._fetcher.Fetch('commits/HEAD').content)['commit']['tree']['sha'] self._object_store.Set(path, version, object_store.GITHUB_STAT) return StatInfo(version)
def testDelegateValue(self): called = [ False, ] def callback(): self.assertFalse(called[0]) called[0] = True return 42 future = Future(callback=callback) self.assertEqual(42, future.Get()) self.assertEqual(42, future.Get())
def testDelegateValue(self): assertFalse = self.assertFalse class delegate(object): def __init__(self): self._get_called = False def Get(self): assertFalse(self._get_called) self._get_called = True return 42 future = Future(delegate=delegate()) self.assertEqual(42, future.Get()) self.assertEqual(42, future.Get())
class GithubFileSystem(FileSystem): @staticmethod def CreateChromeAppsSamples(object_store_creator): return GithubFileSystem( '%s/GoogleChrome/chrome-app-samples' % url_constants.GITHUB_REPOS, AppEngineBlobstore(), object_store_creator) def __init__(self, url, blobstore, object_store_creator): # If we key the password store on the app version then the whole advantage # of having it in the first place is greatly lessened (likewise it should # always start populated). password_store = object_store_creator.Create( GithubFileSystem, app_version=None, category='password', start_empty=False) if USERNAME is None: password_data = password_store.GetMulti(('username', 'password')).Get() self._username, self._password = (password_data.get('username'), password_data.get('password')) else: password_store.SetMulti({'username': USERNAME, 'password': PASSWORD}) self._username, self._password = (USERNAME, PASSWORD) self._url = url self._fetcher = AppEngineUrlFetcher(url) self._blobstore = blobstore self._stat_object_store = object_store_creator.Create(GithubFileSystem) self._version = None self._GetZip(self.Stat(ZIP_KEY).version) def _GetZip(self, version): try: blob = self._blobstore.Get(_MakeBlobstoreKey(version), BLOBSTORE_GITHUB) except blobstore.BlobNotFoundError: self._zip_file = Future(value=None) return if blob is not None: try: self._zip_file = Future(value=ZipFile(StringIO(blob))) except BadZipfile as e: self._blobstore.Delete(_MakeBlobstoreKey(version), BLOBSTORE_GITHUB) logging.error('Bad github zip file: %s' % e) self._zip_file = Future(value=None) else: self._zip_file = Future( callback=_GetAsyncFetchCallback(self._fetcher, self._username, self._password, self._blobstore, version, key_to_delete=self._version)) self._version = version def _ReadFile(self, path): try: zip_file = self._zip_file.Get() except Exception as e: logging.error('Github ReadFile error: %s' % e) return '' if zip_file is None: logging.error('Bad github zip file.') return '' prefix = zip_file.namelist()[0] return zip_file.read(prefix + path) def _ListDir(self, path): try: zip_file = self._zip_file.Get() except Exception as e: logging.error('Github ListDir error: %s' % e) return [] if zip_file is None: logging.error('Bad github zip file.') return [] filenames = zip_file.namelist() # Take out parent directory name (GoogleChrome-chrome-app-samples-c78a30f) filenames = [f[len(filenames[0]):] for f in filenames] # Remove the path of the directory we're listing from the filenames. filenames = [f[len(path):] for f in filenames if f != path and f.startswith(path)] # Remove all files not directly in this directory. return [f for f in filenames if f[:-1].count('/') == 0] def Read(self, paths, skip_not_found=False): version = self.Stat(ZIP_KEY).version if version != self._version: self._GetZip(version) result = {} for path in paths: if IsDirectory(path): result[path] = self._ListDir(path) else: result[path] = self._ReadFile(path) return Future(value=result) def _DefaultStat(self, path): version = 0 # TODO(kalman): we should replace all of this by wrapping the # GithubFileSystem in a CachingFileSystem. A lot of work has been put into # CFS to be robust, and GFS is missing out. # For example: the following line is wrong, but it could be moot. self._stat_object_store.Set(path, version) return StatInfo(version) def Stat(self, path): version = self._stat_object_store.Get(path).Get() if version is not None: return StatInfo(version) try: result = self._fetcher.Fetch('commits/HEAD', username=USERNAME, password=PASSWORD) except urlfetch.DownloadError as e: logging.warning('GithubFileSystem Stat: %s' % e) return self._DefaultStat(path) # Check if Github authentication failed. if result.status_code == 401: logging.warning('Github authentication failed for %s, falling back to ' 'unauthenticated.' % USERNAME) try: result = self._fetcher.Fetch('commits/HEAD') except urlfetch.DownloadError as e: logging.warning('GithubFileSystem Stat: %s' % e) return self._DefaultStat(path) # Parse response JSON - but sometimes github gives us invalid JSON. try: version = json.loads(result.content)['sha'] self._stat_object_store.Set(path, version) return StatInfo(version) except StandardError as e: logging.warning( ('%s: got invalid or unexpected JSON from github. Response status ' + 'was %s, content %s') % (e, result.status_code, result.content)) return self._DefaultStat(path) def GetIdentity(self): return '%s@%s' % (self.__class__.__name__, StringIdentity(self._url))
class GithubFileSystem(FileSystem): '''Allows reading from a github.com repository. ''' @staticmethod def Create(owner, repo, object_store_creator): '''Creates a GithubFileSystem that corresponds to a single github repository specified by |owner| and |repo|. ''' return GithubFileSystem( url_constants.GITHUB_REPOS, owner, repo, object_store_creator, AppEngineUrlFetcher) @staticmethod def ForTest(repo, fake_fetcher, path=None, object_store_creator=None): '''Creates a GithubFIleSystem that can be used for testing. It reads zip files and commit data from server2/test_data/github_file_system/test_owner instead of github.com. It reads from files specified by |repo|. ''' return GithubFileSystem( path if path is not None else 'test_data/github_file_system', 'test_owner', repo, object_store_creator or ObjectStoreCreator.ForTest(), fake_fetcher) def __init__(self, base_url, owner, repo, object_store_creator, Fetcher): self._repo_key = '%s/%s' % (owner, repo) self._repo_url = '%s/%s/%s' % (base_url, owner, repo) self._blobstore = blobstore.AppEngineBlobstore() # Lookup the chrome github api credentials. self._username, self._password = _LoadCredentials(object_store_creator) self._fetcher = Fetcher(self._repo_url) self._stat_cache = object_store_creator.Create( GithubFileSystem, category='stat-cache') self._repo_zip = Future(value=None) def _GetNamelist(self): '''Returns a list of all file names in a repository zip file. ''' zipfile = self._repo_zip.Get() if zipfile is None: return [] return zipfile.namelist() def _GetVersion(self): '''Returns the currently cached version of the repository. The version is a 'sha' hash value. ''' return self._stat_cache.Get(self._repo_key).Get() def _FetchLiveVersion(self): '''Fetches the current repository version from github.com and returns it. The version is a 'sha' hash value. ''' # TODO(kalman): Do this asynchronously (use FetchAsync). result = self._fetcher.Fetch( 'commits/HEAD', username=self._username, password=self._password) try: return json.loads(result.content)['commit']['tree']['sha'] except (KeyError, ValueError): logging.warn('Error parsing JSON from repo %s' % self._repo_url) def Refresh(self): '''Compares the cached and live stat versions to see if the cached repository is out of date. If it is, an async fetch is started and a Future is returned. When this Future is evaluated, the fetch will be completed and the results cached. If no update is needed, None will be returned. ''' version = self._FetchLiveVersion() repo_zip_url = self._repo_url + '/zipball' def persist_fetch(fetch): '''Completes |fetch| and stores the results in blobstore. ''' try: blob = fetch.Get().content except urlfetch.DownloadError: logging.error( '%s: Failed to download zip file from repository %s' % repo_zip_url) else: try: zipfile = ZipFile(StringIO(blob)) except BadZipfile as error: logging.error( '%s: Bad zip file returned from url %s' % (error, repo_zip_url)) else: self._blobstore.Set(repo_zip_url, blob, _GITHUB_REPOS_NAMESPACE) self._repo_zip = Future(value=zipfile) self._stat_cache.Set(self._repo_key, version) # If the cached and live stat versions are different fetch the new repo. if version != self._stat_cache.Get('stat').Get(): fetch = self._fetcher.FetchAsync( 'zipball', username=self._username, password=self._password) return Future(delegate=Gettable(lambda: persist_fetch(fetch))) return Future(value=None) def Read(self, paths, binary=False): '''Returns a directory mapping |paths| to the contents of the file at each path. If path ends with a '/', it is treated as a directory and is mapped to a list of filenames in that directory. |binary| is ignored. ''' names = self._GetNamelist() if not names: # No files in this repository. def raise_file_not_found(): raise FileNotFoundError('No paths can be found, repository is empty') return Future(delegate=Gettable(raise_file_not_found)) else: prefix = names[0].split('/')[0] reads = {} for path in paths: full_path = posixpath.join(prefix, path) if path == '' or path.endswith('/'): # If path is a directory... trimmed_paths = [] for f in filter(lambda s: s.startswith(full_path), names): if not '/' in f[len(full_path):-1] and not f == full_path: trimmed_paths.append(f[len(full_path):]) reads[path] = trimmed_paths else: try: reads[path] = self._repo_zip.Get().read(full_path) except KeyError as error: return Future(exc_info=(FileNotFoundError, FileNotFoundError(error), sys.exc_info()[2])) return Future(value=reads) def Stat(self, path): '''Stats |path| returning its version as as StatInfo object. If |path| ends with a '/', it is assumed to be a directory and the StatInfo object returned includes child_versions for all paths in the directory. File paths do not include the name of the zip file, which is arbitrary and useless to consumers. Because the repository will only be downloaded once per server version, all stat versions are always 0. ''' # Trim off the zip file's name. path = path.lstrip('/') trimmed = [f.split('/', 1)[1] for f in self._GetNamelist()] if path not in trimmed: raise FileNotFoundError("No stat found for '%s' in %s" % (path, trimmed)) version = self._GetVersion() child_paths = {} if path == '' or path.endswith('/'): # Deal with a directory for f in filter(lambda s: s.startswith(path), trimmed): filename = f[len(path):] if not '/' in filename and not f == path: child_paths[filename] = StatInfo(version) return StatInfo(version, child_paths or None) def GetIdentity(self): return '%s' % StringIdentity(self.__class__.__name__ + self._repo_key) def __repr__(self): return '<%s: key=%s, url=%s>' % (type(self).__name__, self._repo_key, self._repo_url)
class GithubFileSystem(FileSystem): """FileSystem implementation which fetches resources from github. """ def __init__(self, fetcher, object_store, blobstore): self._fetcher = fetcher self._object_store = object_store self._blobstore = blobstore self._version = None self._GetZip(self.Stat(ZIP_KEY).version) def _GetZip(self, version): blob = self._blobstore.Get(_MakeKey(version), blobstore.BLOBSTORE_GITHUB) if blob is not None: try: self._zip_file = Future(value=ZipFile(StringIO(blob))) except BadZipfile as e: self._blobstore.Delete(_MakeKey(version), blobstore.BLOBSTORE_GITHUB) logging.error('Bad github zip file: %s' % e) self._zip_file = Future(value=None) else: self._zip_file = Future( delegate=_AsyncFetchFutureZip(self._fetcher, self._blobstore, version, key_to_delete=self._version)) self._version = version def _ReadFile(self, path): zip_file = self._zip_file.Get() if zip_file is None: logging.error('Bad github zip file.') return '' prefix = zip_file.namelist()[0][:-1] return zip_file.read(prefix + path) def _ListDir(self, path): zip_file = self._zip_file.Get() if zip_file is None: logging.error('Bad github zip file.') return [] filenames = zip_file.namelist() # Take out parent directory name (GoogleChrome-chrome-app-samples-c78a30f) filenames = [f[len(filenames[0]) - 1:] for f in filenames] # Remove the path of the directory we're listing from the filenames. filenames = [ f[len(path):] for f in filenames if f != path and f.startswith(path) ] # Remove all files not directly in this directory. return [f for f in filenames if f[:-1].count('/') == 0] def Read(self, paths, binary=False): version = self.Stat(ZIP_KEY).version if version != self._version: self._GetZip(version) result = {} for path in paths: if path.endswith('/'): result[path] = self._ListDir(path) else: result[path] = self._ReadFile(path) return Future(value=result) def Stat(self, path): version = self._object_store.Get(path, object_store.GITHUB_STAT).Get() if version is not None: return StatInfo(version) version = (json.loads(self._fetcher.Fetch('commits/HEAD').content).get( 'commit', {}).get('tree', {}).get('sha', None)) # Check if the JSON was valid, and set to 0 if not. if version is not None: self._object_store.Set(path, version, object_store.GITHUB_STAT) else: logging.warning('Problem fetching commit hash from github.') version = 0 # Cache for a minute so we don't try to keep fetching bad data. self._object_store.Set(path, version, object_store.GITHUB_STAT, time=60) return StatInfo(version)
def testValue(self): future = Future(value=42) self.assertEqual(42, future.Get()) self.assertEqual(42, future.Get())
class GithubFileSystem(FileSystem): """FileSystem implementation which fetches resources from github. """ def __init__(self, fetcher, blobstore): self._fetcher = fetcher self._stat_object_store = (ObjectStoreCreator.SharedFactory( GetAppVersion()).Create(GithubFileSystem).Create()) self._blobstore = blobstore self._version = None self._GetZip(self.Stat(ZIP_KEY).version) def _GetZip(self, version): blob = self._blobstore.Get(_MakeBlobstoreKey(version), blobstore.BLOBSTORE_GITHUB) if blob is not None: try: self._zip_file = Future(value=ZipFile(StringIO(blob))) except BadZipfile as e: self._blobstore.Delete(_MakeBlobstoreKey(version), blobstore.BLOBSTORE_GITHUB) logging.error('Bad github zip file: %s' % e) self._zip_file = Future(value=None) else: self._zip_file = Future( delegate=_AsyncFetchFutureZip(self._fetcher, self._blobstore, version, key_to_delete=self._version)) self._version = version def _ReadFile(self, path): try: zip_file = self._zip_file.Get() except Exception as e: logging.error('Github ReadFile error: %s' % e) return '' if zip_file is None: logging.error('Bad github zip file.') return '' prefix = zip_file.namelist()[0][:-1] return zip_file.read(prefix + path) def _ListDir(self, path): try: zip_file = self._zip_file.Get() except Exception as e: logging.error('Github ListDir error: %s' % e) return [] if zip_file is None: logging.error('Bad github zip file.') return [] filenames = zip_file.namelist() # Take out parent directory name (GoogleChrome-chrome-app-samples-c78a30f) filenames = [f[len(filenames[0]) - 1:] for f in filenames] # Remove the path of the directory we're listing from the filenames. filenames = [ f[len(path):] for f in filenames if f != path and f.startswith(path) ] # Remove all files not directly in this directory. return [f for f in filenames if f[:-1].count('/') == 0] def Read(self, paths, binary=False): version = self.Stat(ZIP_KEY).version if version != self._version: self._GetZip(version) result = {} for path in paths: if path.endswith('/'): result[path] = self._ListDir(path) else: result[path] = self._ReadFile(path) return Future(value=result) def _DefaultStat(self, path): version = 0 # TODO(kalman): we should replace all of this by wrapping the # GithubFileSystem in a CachingFileSystem. A lot of work has been put into # CFS to be robust, and GFS is missing out. # For example: the following line is wrong, but it could be moot. self._stat_object_store.Set(path, version) return StatInfo(version) def Stat(self, path): version = self._stat_object_store.Get(path).Get() if version is not None: return StatInfo(version) try: result = self._fetcher.Fetch('commits/HEAD', username=USERNAME, password=PASSWORD) except urlfetch.DownloadError as e: logging.error('GithubFileSystem Stat: %s' % e) return self._DefaultStat(path) # Check if Github authentication failed. if result.status_code == 401: logging.error( 'Github authentication failed for %s, falling back to ' 'unauthenticated.' % USERNAME) try: result = self._fetcher.Fetch('commits/HEAD') except urlfetch.DownloadError as e: logging.error('GithubFileSystem Stat: %s' % e) return self._DefaultStat(path) version = (json.loads(result.content).get('commit', {}).get('tree', {}).get('sha', None)) # Check if the JSON was valid, and set to 0 if not. if version is not None: self._stat_object_store.Set(path, version) else: logging.warning('Problem fetching commit hash from github.') return self._DefaultStat(path) return StatInfo(version)
class GithubFileSystem(FileSystem): @staticmethod def Create(object_store_creator): return GithubFileSystem(AppEngineUrlFetcher(url_constants.GITHUB_URL), blobstore.AppEngineBlobstore(), object_store_creator) def __init__(self, fetcher, blobstore, object_store_creator): # Password store doesn't depend on channel, and if we don't cancel the app # version then the whole advantage of having it in the first place is # greatly lessened (likewise it should always start populated). password_store = object_store_creator.Create(GithubFileSystem, channel=None, app_version=None, category='password', start_empty=False) if USERNAME is None: password_data = password_store.GetMulti( ('username', 'password')).Get() self._username, self._password = (password_data.get('username'), password_data.get('password')) else: password_store.SetMulti({ 'username': USERNAME, 'password': PASSWORD }) self._username, self._password = (USERNAME, PASSWORD) self._fetcher = fetcher self._blobstore = blobstore # Github has no knowledge of Chrome channels, set channel to None. self._stat_object_store = object_store_creator.Create(GithubFileSystem, channel=None) self._version = None self._GetZip(self.Stat(ZIP_KEY).version) def _GetZip(self, version): blob = self._blobstore.Get(_MakeBlobstoreKey(version), blobstore.BLOBSTORE_GITHUB) if blob is not None: try: self._zip_file = Future(value=ZipFile(StringIO(blob))) except BadZipfile as e: self._blobstore.Delete(_MakeBlobstoreKey(version), blobstore.BLOBSTORE_GITHUB) logging.error('Bad github zip file: %s' % e) self._zip_file = Future(value=None) else: self._zip_file = Future( delegate=_AsyncFetchFutureZip(self._fetcher, self._username, self._password, self._blobstore, version, key_to_delete=self._version)) self._version = version def _ReadFile(self, path): try: zip_file = self._zip_file.Get() except Exception as e: logging.error('Github ReadFile error: %s' % e) return '' if zip_file is None: logging.error('Bad github zip file.') return '' prefix = zip_file.namelist()[0][:-1] return zip_file.read(prefix + path) def _ListDir(self, path): try: zip_file = self._zip_file.Get() except Exception as e: logging.error('Github ListDir error: %s' % e) return [] if zip_file is None: logging.error('Bad github zip file.') return [] filenames = zip_file.namelist() # Take out parent directory name (GoogleChrome-chrome-app-samples-c78a30f) filenames = [f[len(filenames[0]) - 1:] for f in filenames] # Remove the path of the directory we're listing from the filenames. filenames = [ f[len(path):] for f in filenames if f != path and f.startswith(path) ] # Remove all files not directly in this directory. return [f for f in filenames if f[:-1].count('/') == 0] def Read(self, paths, binary=False): version = self.Stat(ZIP_KEY).version if version != self._version: self._GetZip(version) result = {} for path in paths: if path.endswith('/'): result[path] = self._ListDir(path) else: result[path] = self._ReadFile(path) return Future(value=result) def _DefaultStat(self, path): version = 0 # TODO(kalman): we should replace all of this by wrapping the # GithubFileSystem in a CachingFileSystem. A lot of work has been put into # CFS to be robust, and GFS is missing out. # For example: the following line is wrong, but it could be moot. self._stat_object_store.Set(path, version) return StatInfo(version) def Stat(self, path): version = self._stat_object_store.Get(path).Get() if version is not None: return StatInfo(version) try: result = self._fetcher.Fetch('commits/HEAD', username=USERNAME, password=PASSWORD) except urlfetch.DownloadError as e: logging.error('GithubFileSystem Stat: %s' % e) return self._DefaultStat(path) # Check if Github authentication failed. if result.status_code == 401: logging.error( 'Github authentication failed for %s, falling back to ' 'unauthenticated.' % USERNAME) try: result = self._fetcher.Fetch('commits/HEAD') except urlfetch.DownloadError as e: logging.error('GithubFileSystem Stat: %s' % e) return self._DefaultStat(path) version = (json.loads(result.content).get('commit', {}).get('tree', {}).get('sha', None)) # Check if the JSON was valid, and set to 0 if not. if version is not None: self._stat_object_store.Set(path, version) else: logging.warning('Problem fetching commit hash from github.') return self._DefaultStat(path) return StatInfo(version)