def quit(self, b_blocking=True): #@ReservedAssignment f_done = Future() def async_quit(): # ask all threads to exit def finish(): self.logger.Debug('quiting AO thread') return 1 for _ in xrange(self.n_threads): self.enq(finish) # wait for them for t in self.threads: t.join() # run passive object's quit method and hook BaseAO.quit(self) self.run_hook(RunStage.AO_Stop,ao=self) # signal we're done self.logger.Debug('quit finished') f_done.set(None) t = Thread(target = async_quit) t.setDaemon(True) t.start() if b_blocking: f_done.get() else: return f_done
def maybe_future(x): """Converts ``x`` into a `.Future`. the facebook makeFuture, create a fullfilled future If ``x`` is already a `.Future`, it is simply returned; otherwise it is wrapped in a new `.Future`. This is suitable for use as ``result = yield gen.maybe_future(f())`` when you don't know whether ``f()`` returns a `.Future` or not. """ if is_future(x): return x else: fut = Future() fut.set_result(x) return fut
def sleep(duration): """Return a `.Future` that resolves after the given number of seconds. When used with ``yield`` in a coroutine, this is a non-blocking analogue to `time.sleep` (which should not be used in coroutines because it is blocking):: yield gen.sleep(0.5) Note that calling this function on its own does nothing; you must wait on the `.Future` it returns (usually by yielding it). """ f = Future() IOLoop.current().call_later(duration, lambda: f.set_result(None)) return f
def __init__(self,n_threads,ao, mth): self.lock = RLock() self.finished = Future().set_name('AO Visitor') self.n_visits = 0 self.n_threads = n_threads self.ao = ao self.mth = mth
def connect(self, port, ip = '127.0.0.1'): '''Nonblocking connect to remote address''' if self.__future is not None: raise RuntimeError("already connecting") if self.__state != Connector.NONE: raise RuntimeError("already connected") self.__sock = socket.socket() self.__sock.setblocking(False) self.__peeraddr = (ip, port) self.__future = Future() try: self.__sock.connect(self.__peeraddr) except socket.error as serr: if serr.errno != os.errno.EINPROGRESS and serr.errno != os.errno.EWOULDBLOCK: logging.exception("connect error is not EINPROGRESS: %s", str(serr.errno)) self.__future.set_exception(ConnectError(str(serr.errno))) self.close() return self.__future else: self.__state = Connector.CONNECTING else: self.__state = Connector.CONNECTED logging.info("connect immediately success %s", str(self.fileno())) self._onSuccess() return self.__future from ioloop import IOLoop self.__loop.register(self, IOLoop.WRITE_EVENT) return self.__future
def _internal_invoke(self,method,a,kw,wrapper_level=None): method,wrapper_info = self._get_callable(method,wrapper_level) AO_cmd_meta_data = kw.pop('AO_cmd_meta_data',{}) call_type = getattr(method,'call_type',MethodCallType.Async) f = Future() cmd = BaseAO.Cmd(call_type,f,self,method,a,kw,wrapper_info,AO_cmd_meta_data) f.set_name(cmd.qualified_name_no_wrappers) if call_type == MethodCallType.Sync: # ignores queue and executes immediately + blocks cmd() return f.get() else: self.enq(cmd) if hasattr(method,'sync_facade'): # does not ignore the queue but blocks return f.get() else: return f
def start(self, config): self._protocol.subscribe_channel(self._new_channel) # Blockchain future = Future() self._chain.start(config["database"], future) ec, = future.get() if ec: print >> sys.stderr, "Couldn't start blockchain:", str(ec) return False # Transaction pool self._txpool.start() # Session future = Future() self._session.start(future) ec, = future.get() if ec: print >> sys.stderr, "Couldn't start session:", str(ec) return False return True
def multi_future(children, quiet_exceptions=()): """Wait for multiple asynchronous futures in parallel. like collectAll""" if isinstance(children, dict): keys = list(children.keys()) children = children.values() else: keys = None children = list(map(convert_yielded, children)) assert all(is_future(i) for i in children) unfinished_children = set(children) future = Future() # it's a collectAll future if not children: future.set_result({} if keys is not None else []) def callback(f_child): unfinished_children.remove(f_child) if not unfinished_children: # all child futures are done! result_list = [] for f in children: try: result_list.append(f.result()) except Exception as e: if future.done(): if not isinstance(e, quiet_exceptions): logging.error("Multiple exceptions in yield list") else: future.set_exc_info(sys.exc_info()) if not future.done(): if keys is not None: future.set_result(dict(zip(keys, result_list))) else: future.set_result(result_list) listening = set() for child in children: if child not in listening: listening.add(child) child.add_done_callback(callback) # when child future is done, tell parent future return future
class AsyncRequest(object): def __init__(self, host, url, port, timeout=5): self.sock = socket.socket() self.sock.settimeout(timeout) self.sock.setblocking(False) self.host = host self.url = url self.port = port self.method = None def get(self): self.method = "GET" self.request = "{} {} HTTP/1.0\r\nHost: {}\r\n\r\n".format(self.method, self.url, self.host) return self def process(self): if self.method is None: self.get() try: self.sock.connect((self.host, self.port)) except BlockingIOError: pass self.f = Future() selector.register(self.sock.fileno(), EVENT_WRITE, self.on_connected) yield self.f selector.unregister(self.sock.fileno()) self.sock.send(self.request.encode("ascii")) chunk = yield from read_all(self.sock) return chunk def on_connected(self, key, mask): self.f.set_result(None)
def GetMulti(self, keys): db_futures = dict( (k, db.get_async( PersistentObjectStoreItem.CreateKey(self._namespace, k))) for k in keys) def resolve(): return dict((key, future.get_result().GetValue()) for key, future in db_futures.iteritems() if future.get_result() is not None) return Future(callback=resolve)
def StatAsync(self, path): '''Stats the directory given, or if a file is given, stats the file's parent directory to get info about the file. ''' # Always stat the parent directory, since it will have the stat of the child # anyway, and this gives us an entire directory's stat info at once. dir_path, file_path = posixpath.split(path) dir_path = ToDirectory(dir_path) def make_stat_info(dir_stat): '''Converts a dir stat into the correct resulting StatInfo; if the Stat was for a file, the StatInfo should just contain that file. ''' if path == dir_path: return dir_stat # Was a file stat. Extract that file. file_version = dir_stat.child_versions.get(file_path) if file_version is None: raise FileNotFoundError('No stat found for %s in %s (found %s)' % (path, dir_path, dir_stat.child_versions)) return StatInfo(file_version) def raise_cache_miss(path): raise FileNotFoundError('Got cache miss when trying to stat %s' % path) dir_stat = self._stat_cache.Get(dir_path).Get() if dir_stat is not None: return Future(callback=lambda: make_stat_info(dir_stat)) if self._fail_on_miss: logging.warning('Bailing on stat cache miss for %s' % dir_path) return Future(callback=lambda: raise_cache_miss(dir_path)) def next(dir_stat): assert dir_stat is not None # should have raised a FileNotFoundError # We only ever need to cache the dir stat. self._stat_cache.Set(dir_path, dir_stat) return make_stat_info(dir_stat) return self._MemoizedStatAsyncFromFileSystem(dir_path).Then(next)
def GetAPIFeatures(self): api_features = self._object_store.Get('api_features').Get() if api_features is not None: return Future(value=api_features) api_features_future = self._api_cache.GetFeatures() manifest_features_future = self._manifest_cache.GetFeatures() permission_features_future = self._permission_cache.GetFeatures() def resolve(): api_features = api_features_future.Get() manifest_features = manifest_features_future.Get() permission_features = permission_features_future.Get() # TODO(rockot): Handle inter-API dependencies more gracefully. # Not yet a problem because there is only one such case (windows -> tabs). # If we don't store this value before annotating platforms, inter-API # dependencies will lead to infinite recursion. for feature in api_features.itervalues(): _AddPlatformsAndChannelsFromDependencies( feature, api_features, manifest_features, permission_features) self._object_store.Set('api_features', api_features) return api_features return Future(callback=resolve)
def GetFromFileListing(self, path): '''Calls |compilation_function| on the listing of the files at |path|. Assumes that the path given is to a directory. ''' AssertIsDirectory(path) try: version = self._file_system.Stat(path).version except FileNotFoundError: return Future(exc_info=sys.exc_info()) cache_entry = self._Get(self._list_object_store, path).Get() if (cache_entry is not None) and (version == cache_entry.version): return Future(value=cache_entry.cache_data) def compile_(files): cache_data = self._compilation_function(path, files) self._Set(self._list_object_store, path, _CacheEntry(cache_data, version)) return cache_data return self._RecursiveList(path).Then(compile_)
def FetchAsync(self, url): """Fetches a file asynchronously, and returns a Future with the result. """ rpc = urlfetch.create_rpc() if self._base_path is not None: urlfetch.make_fetch_call(rpc, self._base_path + '/' + url, headers={'Cache-Control': 'max-age=0'}) else: urlfetch.make_fetch_call(rpc, url, headers={'Cache-Control': 'max-age=0'}) return Future(delegate=_AsyncFetchDelegate(rpc))
def GetFromFile(self, path): '''Calls |compilation_function| on the contents of the file at |path|. If |binary| is True then the file will be read as binary - but this will only apply for the first time the file is fetched; if already cached, |binary| will be ignored. ''' AssertIsFile(path) try: version = self._file_system.Stat(path).version except FileNotFoundError: return Future(exc_info=sys.exc_info()) cache_entry = self._file_object_store.Get(path).Get() if (cache_entry is not None) and (version == cache_entry.version): return Future(value=cache_entry._cache_data) def next(files): cache_data = self._compilation_function(path, files) self._file_object_store.Set(path, _CacheEntry(cache_data, version)) return cache_data return self._file_system.ReadSingle(path).Then(next)
def Refresh(self, path): def safe(name, action, callback): '''Safely runs |callback| for a ContentProvider called |name| by swallowing exceptions and turning them into a None return value. It's important to run all ContentProvider Refreshes even if some of them fail. ''' try: return callback() except: if not _IGNORE_MISSING_CONTENT_PROVIDERS[0]: logging.error( 'Error %s Refresh for ContentProvider "%s":\n%s' % (action, name, traceback.format_exc())) return None config = self._GetConfig()[path] provider = self._CreateContentProvider(path, config) future = safe(path, 'initializing', self._CreateContentProvider(path, config).Refresh) if future is None: return Future(callback=lambda: True) return Future(callback=lambda: safe(path, 'resolving', future.Get))
def _GetImpl(self, api_name): handlebar_dict_future = self._GetSchemaModel(api_name) def resolve(): handlebar_dict = handlebar_dict_future.Get() # Parsing samples on the preview server takes seconds and doesn't add # anything. Don't do it. if not IsPreviewServer(): handlebar_dict['samples'] = _LazySamplesGetter( handlebar_dict['name'], self._samples) return handlebar_dict return Future(callback=resolve)
def Read(self, paths, skip_not_found=False): args = None if self._revision is not None: # |fetcher| gets from svn.chromium.org which uses p= for version. args = 'p=%s' % self._revision def apply_args(path): return path if args is None else '%s?%s' % (path, args) def list_dir(directory): dom = xml.parseString(directory) files = [ elem.childNodes[0].data for elem in dom.getElementsByTagName('a') ] if '..' in files: files.remove('..') return files # A list of tuples of the form (path, Future). fetches = [(path, self._file_fetcher.FetchAsync(apply_args(path))) for path in paths] def resolve(): value = {} for path, future in fetches: try: result = future.Get() except Exception as e: if skip_not_found and IsDownloadError(e): continue exc_type = (FileNotFoundError if IsDownloadError(e) else FileSystemError) raise exc_type( '%s fetching %s for Get: %s' % (type(e).__name__, path, traceback.format_exc())) if result.status_code == 404: if skip_not_found: continue raise FileNotFoundError( 'Got 404 when fetching %s for Get, content %s' % (path, result.content)) if result.status_code != 200: raise FileSystemError( 'Got %s when fetching %s for Get, content %s' % (result.status_code, path, result.content)) if path.endswith('/'): value[path] = list_dir(result.content) else: value[path] = result.content return value return Future(callback=resolve)
def with_timeout(abstimeout, future, io_loop=None, quiet_exceptions=()): """Wraps a `.Future` in a timeout. Raises `TimeoutError` if the input future does not complete before ``timeout``, which may be specified in any form allowed by `.IOLoop.add_timeout` (i.e. a `datetime.timedelta` or an absolute time relative to `.IOLoop.time`) If the wrapped `.Future` fails after it has timed out, the exception will be logged unless it is of a type contained in ``quiet_exceptions`` (which may be an exception type or a sequence of types). Currently only supports Futures, not other `YieldPoint` classes. """ result = Future() chain_future(future, result) if io_loop is None: io_loop = IOLoop.current() def error_callback(future): try: future.result() except Exception as e: if not isinstance(e, quiet_exceptions): logging.error("Exception in Future %r after timeout", future, exc_info=True) def timeout_callback(): result.set_exception(TimeoutError("Timeout")) # In case the wrapped future goes on to fail, log it. future.add_done_callback(error_callback) timeout_handle = io_loop.call_at(abstimeout, timeout_callback) if isinstance(future, Future): # We know this future will resolve on the IOLoop, so we don't # need the extra thread-safety of IOLoop.add_future future.add_done_callback( lambda future: io_loop.remove_timeout(timeout_handle)) else: # concurrent.futures.Futures may resolve on any thread, so we # need to route them back to the IOLoop. def cancel_on_ioloop(fut): IOLoop.current().call_next_tick( lambda fut: IOLoop.current().remove_timeout(timeout_handle)) future.add_done_callback(cancel_on_ioloop) return result
def Exists(self, path): '''Returns a Future to the existence of |path|; True if |path| exists, False if not. This method will not throw a FileNotFoundError unlike the Read* methods, however it may still throw a FileSystemError. There are several ways to implement this method via the interface but this method exists to do so in a canonical and most efficient way for caching. ''' AssertIsValid(path) if path == '': # There is always a root directory. return Future(value=True) parent, base = SplitParent(path) list_future = self.ReadSingle(ToDirectory(parent)) def resolve(): try: return base in list_future.Get() except FileNotFoundError: return False return Future(callback=resolve)
def get_feed(self, urls): future_calls = [Future(feedparser.parse) for url in urls] feeds = [future_obj() for future_obj in future_calls] entries = [] for feed in feeds: entries.extend(feed.entries) sorted_entries = filter( todays_catch, sorted(entries, key=lambda entry: entry.published_parsed)) print len(sorted_entries) return sorted_entries[::-1]
def Read(self, paths, binary=False): patched_files = set() added, deleted, modified = self._patcher.GetPatchedFiles() if set(paths) & set(deleted): raise FileNotFoundError('Files are removed from the patch.') patched_files |= (set(added) | set(modified)) dir_paths = set(path for path in paths if path.endswith('/')) file_paths = set(paths) - dir_paths patched_paths = file_paths & patched_files unpatched_paths = file_paths - patched_files return Future(delegate=_AsyncFetchFuture( self._host_file_system.Read(unpatched_paths, binary), self._patcher.Apply(patched_paths, self._host_file_system, binary), self._TryReadDirectory(dir_paths, binary), self))
class Visitor(object): def __init__(self,n_threads,ao, mth): self.lock = RLock() self.finished = Future().set_name('AO Visitor') self.n_visits = 0 self.n_threads = n_threads self.ao = ao self.mth = mth def visit(self): """ each thread will execute exactly one visit call """ with self.lock: self.n_visits += 1 b_last = (self.n_visits == self.n_threads) rc = mth() if b_last: self.finished.set(rc) # This blocks the thread until we're done with all threads # It's necessary so we don't dequeue another thread's visit command # It also ensures that commands enqueued after visit started don't start executing until it's finished # which makes visit act in a quiesced state. self.finished.get()
def Refresh(self): '''Compares the cached and live stat versions to see if the cached repository is out of date. If it is, an async fetch is started and a Future is returned. When this Future is evaluated, the fetch will be completed and the results cached. If no update is needed, None will be returned. ''' version = self._FetchLiveVersion() repo_zip_url = self._repo_url + '/zipball' def persist_fetch(fetch): '''Completes |fetch| and stores the results in blobstore. ''' try: blob = fetch.Get().content except urlfetch.DownloadError: logging.error( '%s: Failed to download zip file from repository %s' % repo_zip_url) else: try: zipfile = ZipFile(StringIO(blob)) except BadZipfile as error: logging.error( '%s: Bad zip file returned from url %s' % (error, repo_zip_url)) else: self._blobstore.Set(repo_zip_url, blob, _GITHUB_REPOS_NAMESPACE) self._repo_zip = Future(value=zipfile) self._stat_cache.Set(self._repo_key, version) # If the cached and live stat versions are different fetch the new repo. if version != self._stat_cache.Get('stat').Get(): fetch = self._fetcher.FetchAsync( 'zipball', username=self._username, password=self._password) return Future(delegate=Gettable(lambda: persist_fetch(fetch))) return Future(value=None)
def Read(self, paths, binary=False): '''Reads a list of files. If a file is in memcache and it is not out of date, it is returned. Otherwise, the file is retrieved from the file system. ''' read_object_store = (self._read_binary_object_store if binary else self._read_object_store) read_values = read_object_store.GetMulti(paths).Get() stat_values = self._stat_object_store.GetMulti(paths).Get() results = {} # maps path to read value uncached = {} # maps path to stat value for path in paths: stat_value = stat_values.get(path) if stat_value is None: # TODO(cduvall): do a concurrent Stat with the missing stat values. try: stat_value = self.Stat(path) except: return Future(exc_info=sys.exc_info()) read_value = read_values.get(path) if read_value is None: uncached[path] = stat_value continue read_data, read_version = read_value if stat_value.version != read_version: uncached[path] = stat_value continue results[path] = read_data if not uncached: return Future(value=results) return Future(delegate=_AsyncUncachedFuture( self._file_system.Read(uncached.keys(), binary=binary), uncached, results, self, read_object_store))
def Read(self, paths, binary=False): '''Reads |paths| from |_file_system|, then applies the most recent update from |_updates|, if any. ''' self._read_count += 1 future_result = self._file_system.Read(paths, binary=binary) try: result = future_result.Get() except: return future_result for path in result.iterkeys(): _, update = self._GetMostRecentUpdate(path) if update is not None: result[path] = update return Future(value=result)
def FetchAsync(self, url, username=None, password=None): """Fetches a file asynchronously, and returns a Future with the result. """ rpc = urlfetch.create_rpc(deadline=20) headers = _MakeHeaders(username, password) import logging if self._base_path is not None: logging.info('%s/%s' % (self._base_path, url)) urlfetch.make_fetch_call(rpc, '%s/%s' % (self._base_path, url), headers=headers) else: logging.info(url) urlfetch.make_fetch_call(rpc, url, headers=headers) return Future(delegate=_AsyncFetchDelegate(rpc))
def fetch_from_blobstore(): '''Returns a Future which resolves to the _GithubZipFile for this repo fetched from blobstore. ''' blob = self._blobstore.Get(repo_url, _GITHUB_REPOS_NAMESPACE) if blob is None: return FileSystemError.RaiseInFuture( 'No blob for %s found in datastore' % repo_key) repo_zip = _GithubZipFile.Create(repo_key, blob) if repo_zip is None: return FileSystemError.RaiseInFuture( 'Blob for %s was corrupted in blobstore!?' % repo_key) return Future(value=repo_zip)
def Read(self, paths, binary=False): # Maintain reverse mapping so the result can be mapped to the original # paths given (the result from |file_system| will include |root| in the # result, which would be wrong). prefixed_paths = {} def prefix(path): prefixed = posixpath.join(self._root, path) prefixed_paths[prefixed] = path return prefixed future_result = self._file_system.Read( tuple(prefix(path) for path in paths), binary=binary) def resolve(): return dict((prefixed_paths[path], content) for path, content in future_result.Get().iteritems()) return Future(delegate=Gettable(resolve))
def Read(self, paths): def resolve(): result = {} for path in paths: AssertIsValid(path) full_path = os.path.join( self._base_path, _ConvertToFilepath(path).lstrip(os.sep)) if path == '' or path.endswith('/'): result[path] = _ListDir(full_path) else: result[path] = _ReadFile(full_path) return result return Future(delegate=Gettable(resolve))
def Read(self, paths, binary=False): '''Reads |paths| from |_file_system|, then applies the most recent update from |_updates|, if any. ''' self._read_count += 1 future_result = self._file_system.Read(paths, binary=binary) def resolve(): self._read_resolve_count += 1 result = future_result.Get() for path in result.iterkeys(): _, update = self._GetMostRecentUpdate(path) if update is not None: result[path] = update return result return Future(delegate=Gettable(resolve))
def GetMulti(self, keys, time=CACHE_TIMEOUT): keys = keys[:] mapping = {} for key in keys: cache_entry = self._cache.get(key, None) if cache_entry is None or cache_entry.HasExpired(): mapping[key] = None else: mapping[key] = cache_entry.value keys.remove(key) future = self._object_store.GetMulti(keys, time=time) return Future(delegate=_AsyncGetFuture(self._cache, time, future, mapping))
def Read(self, paths, skip_not_found=False): def resolve(): result = {} for path in paths: AssertIsValid(path) full_path = os.path.join( self._base_path, _ConvertToFilepath(path).lstrip(os.sep)) if path == '' or path.endswith('/'): result[path] = _ListDir(full_path) else: result[path] = _ReadFile(full_path) return result return Future(callback=resolve)
def _GetSchemaModel(self, api_name): jsc_model_future = self._model_cache.Get(api_name) model_future = self._api_models.GetModel(api_name) def resolve(): jsc_model = jsc_model_future.Get() if jsc_model is None: jsc_model = _JSCModel(model_future.Get(), self._availability_finder, self._json_cache, self._template_cache, self._features_bundle, self._LoadEventByName).ToDict() self._model_cache.Set(api_name, jsc_model) return jsc_model return Future(callback=resolve)
def GetModel(self, api_name): # By default |api_name| is assumed to be given without a path or extension, # so combinations of known paths and extension types will be searched. api_extensions = ('.json', '.idl') api_paths = API_PATHS # Callers sometimes include a file extension and/or prefix path with the # |api_name| argument. We believe them and narrow the search space # accordingly. name, ext = posixpath.splitext(api_name) if ext in api_extensions: api_extensions = (ext, ) api_name = name for api_path in api_paths: if api_name.startswith(api_path): api_name = api_name[len(api_path):] api_paths = (api_path, ) break # API names are given as declarativeContent and app.window but file names # will be declarative_content and app_window. file_name = UnixName(api_name).replace('.', '_') # Devtools APIs are in API/devtools/ not API/, and have their # "devtools" names removed from the file names. basename = posixpath.basename(file_name) if 'devtools_' in basename: file_name = posixpath.join( 'devtools', file_name.replace(basename, basename.replace('devtools_', ''))) futures = [ self._model_cache.GetFromFile( posixpath.join(path, '%s%s' % (file_name, ext))) for ext in api_extensions for path in api_paths ] def resolve(): for future in futures: try: return future.Get() # Either the file wasn't found or there was no schema for the file except (FileNotFoundError, ValueError): pass # Propagate the first error if neither were found. futures[0].Get() return Future(callback=resolve)
def _FindFileForPath(self, path): '''Finds the real file backing |path|. This may require looking for the correct file extension, or looking for an 'index' file if it's a directory. Returns None if no path is found. ''' AssertIsValid(path) _, ext = posixpath.splitext(path) if ext: # There was already an extension, trust that it's a path. Elsewhere # up the stack this will be caught if it's not. return Future(value=path) def find_file_with_name(name): '''Tries to find a file in the file system called |name| with one of the default extensions of this content provider. If none is found, returns None. ''' paths = [name + ext for ext in self._default_extensions] def get_first_path_which_exists(existence): for exists, path in zip(existence, paths): if exists: return path return None return (All(self.file_system.Exists(path) for path in paths).Then(get_first_path_which_exists)) def find_index_file(): '''Tries to find an index file in |path|, if |path| is a directory. If not, or if there is no index file, returns None. ''' def get_index_if_directory_exists(directory_exists): if not directory_exists: return None return find_file_with_name(Join(path, 'index')) return (self.file_system.Exists( ToDirectory(path)).Then(get_index_if_directory_exists)) # Try to find a file with the right name. If not, and it's a directory, # look for an index file in that directory. If nothing at all is found, # return the original |path| - its nonexistence will be caught up the stack. return (find_file_with_name(path).Then( lambda found: found or find_index_file()).Then( lambda found: found or path))
def Read(self, paths): '''Reads |paths| from |_file_system|, then applies the most recent update from |_updates|, if any. ''' self._read_count += 1 future_result = self._file_system.Read(paths) def resolve(): self._read_resolve_count += 1 result = future_result.Get() for path in result.iterkeys(): update = self._GetMostRecentUpdate(path) if update is not None: result[path] = update return result return Future(callback=resolve)
def FetchAsyncImpl(self, url, headers): def process_result(result): if result.status_code == 429: if self._retries_left == 0: logging.error('Still throttled. Giving up.') return result self._retries_left -= 1 logging.info('Throttled. Trying again in %s seconds.' % _RETRY_DELAY_SECONDS) time.sleep(_RETRY_DELAY_SECONDS) return self.FetchAsync(url, username, password, access_token).Get() return result rpc = urlfetch.create_rpc(deadline=20) urlfetch.make_fetch_call(rpc, url, headers=headers) return Future(callback=lambda: process_result(rpc.get_result()))
def resolve(): result = {} for path in paths: AssertIsValid(path) full_path = os.path.join( self._base_path, _ConvertToFilepath(path).lstrip(os.sep)) if path == '' or path.endswith('/'): result[path] = _ListDir(full_path) else: try: result[path] = _ReadFile(full_path) except FileNotFoundError: if skip_not_found: continue return Future(exc_info=sys.exc_info()) return result
def _LoadCache(self): cached_future = self._cache.GetMulti( ('canonical_paths', 'simplified_paths_map')) def resolve(): # |canonical_paths| is the pre-calculated set of canonical paths. # |simplified_paths_map| is a lazily populated mapping of simplified file # names to a list of full paths that contain them. For example, # - browseraction: [extensions/browserAction.html] # - storage: [apps/storage.html, extensions/storage.html] cached = cached_future.Get() canonical_paths, simplified_paths_map = ( cached.get('canonical_paths'), cached.get('simplified_paths_map')) if canonical_paths is None: assert simplified_paths_map is None canonical_paths = set() simplified_paths_map = defaultdict(list) for base, dirs, files in self._file_system.Walk(''): for path in dirs + files: path_without_ext, ext = posixpath.splitext(path) canonical_path = posixpath.join(base, path_without_ext) if (ext not in self._strip_extensions or path == SITE_VERIFICATION_FILE): canonical_path += ext canonical_paths.add(canonical_path) simplified_paths_map[_Normalize( path, splittext=True)].append(canonical_path) # Store |simplified_paths_map| sorted. Ties in length are broken by # taking the shortest, lexicographically smallest path. for path_list in simplified_paths_map.itervalues(): path_list.sort(key=lambda p: (len(p), p)) self._cache.SetMulti({ 'canonical_paths': canonical_paths, 'simplified_paths_map': simplified_paths_map, }) else: assert simplified_paths_map is not None return canonical_paths, simplified_paths_map return Future(callback=resolve)
def _apply_hook(self, future_hook, func_set_result, func_continue, stage_entered): if not future_hook: future_hook = Future.preset(None).set_name('AO empty future hook') def completion(future_hook): if stage_entered != None: self.ao.method_call_hook(stage_entered,self.call_type,self) val,exc = future_hook.get_val_or_error() if exc is not None: func_set_result(exc=exc) elif isinstance(val,AsyncRV): func_set_result(val=val.val) else: assertions.fail_unless(val is None, "Hook returned a value, but not through AsyncRV", val=val) func_continue() if self.call_type == MethodCallType.Coroutine: future_hook.attach_observer(partial(self._async_completion,stage_entered,completion)) # Coroutine - don't block AO thread while waiting for hook's future else: future_hook.wait() completion(future_hook)
def process(self): if self.method is None: self.get() try: self.sock.connect((self.host, self.port)) except BlockingIOError: pass self.f = Future() selector.register(self.sock.fileno(), EVENT_WRITE, self.on_connected) yield self.f selector.unregister(self.sock.fileno()) self.sock.send(self.request.encode("ascii")) chunk = yield from read_all(self.sock) return chunk
def __init__(self, coro): super().__init__() self.coro = coro f = Future() f.set_result(None) self.step(f)
When used with ``yield`` in a coroutine, this is a non-blocking analogue to `time.sleep` (which should not be used in coroutines because it is blocking):: yield gen.sleep(0.5) Note that calling this function on its own does nothing; you must wait on the `.Future` it returns (usually by yielding it). """ f = Future() IOLoop.current().call_later(duration, lambda: f.set_result(None)) return f _null_future = Future() _null_future.set_result(None) moment = Future() moment.set_result(None) moment.__doc__ = \ """A special object which may be yielded to allow the IOLoop to run for one iteration. This is not needed in normal use but it can be helpful in long-running coroutines that are likely to yield Futures that are ready instantly. Usage: ``yield gen.moment`` """ class Runner(object):
def progress_caller(increment): """Syntactic sugaring for progressing caller without doing anything """ return Future.preset(None).progress_caller(increment)
def progress_caller_to(endpoint): """Syntactic sugaring for progressing caller to endpoint without doing anything """ return Future.preset(None).progress_caller_to(endpoint)
def test_simple(self): future = Future(identity, x=5) self.assertEqual(5, future.run())
def sleep(timeout): future = Future() event = SleepEvent(timeout) event.set_callback(lambda: future.done()) return future
class Connector: ''' Tcp connector''' NONE = 0 CONNECTING = 1 CONNECTED = 2 def __init__(self, loop): self.__sock = None self.__peeraddr = None self.__state = Connector.NONE self.__loop = loop self.__future = None from handler import NewConnectionCallback self.__newConnectionCallback = NewConnectionCallback() def setNewConnCallback(self, callback): self.__newConnectionCallback = callback def fileno(self): return self.__sock.fileno() def connect(self, port, ip = '127.0.0.1'): '''Nonblocking connect to remote address''' if self.__future is not None: raise RuntimeError("already connecting") if self.__state != Connector.NONE: raise RuntimeError("already connected") self.__sock = socket.socket() self.__sock.setblocking(False) self.__peeraddr = (ip, port) self.__future = Future() try: self.__sock.connect(self.__peeraddr) except socket.error as serr: if serr.errno != os.errno.EINPROGRESS and serr.errno != os.errno.EWOULDBLOCK: logging.exception("connect error is not EINPROGRESS: %s", str(serr.errno)) self.__future.set_exception(ConnectError(str(serr.errno))) self.close() return self.__future else: self.__state = Connector.CONNECTING else: self.__state = Connector.CONNECTED logging.info("connect immediately success %s", str(self.fileno())) self._onSuccess() return self.__future from ioloop import IOLoop self.__loop.register(self, IOLoop.WRITE_EVENT) return self.__future def close(self): if self.__sock is None: return try: self.__sock.shutdown(socket.SHUT_WR) self.__sock.close() self.__sock = None except socket.error as serr: pass self.__state = Connector.NONE def onError(self): self.__future.set_exception(ConnectError()) self.close() def onReadable(self): raise RuntimeError("connector should not readable") def onWritable(self): err = self.__sock.getsockopt(socket.SOL_SOCKET, socket.SO_ERROR) if err != 0: logging.error("Connector onWritable error %s", os.strerror(err)) self.__future.set_exception(ConnectError()) self.close() return False self._onSuccess() return True def _onSuccess(self): self.__state = Connector.CONNECTED from connection import Connection conn = Connection(loop = self.__loop, socket = self.__sock, remote = self.__peeraddr) self.__newConnectionCallback(conn) from ioloop import IOLoop self.__loop.modify(conn, IOLoop.READ_EVENT | IOLoop.WRITE_EVENT) conn.onConnect() self.__sock = None self.__future.set_result(conn)
def checkpoint(name): """Syntactic sugaring for declaring hook points """ return Future.preset(None).set_name(name)