def quit(self, b_blocking=True): #@ReservedAssignment
f_done = Future()
def async_quit():
# ask all threads to exit
def finish():
self.logger.Debug('quiting AO thread')
return 1
for _ in xrange(self.n_threads):
self.enq(finish)
# wait for them
for t in self.threads:
t.join()
# run passive object's quit method and hook
BaseAO.quit(self)
self.run_hook(RunStage.AO_Stop,ao=self)
# signal we're done
self.logger.Debug('quit finished')
f_done.set(None)
t = Thread(target = async_quit)
t.setDaemon(True)
t.start()
if b_blocking:
f_done.get()
else:
return f_done
def maybe_future(x):
"""Converts ``x`` into a `.Future`.
the facebook makeFuture, create a fullfilled future
If ``x`` is already a `.Future`, it is simply returned; otherwise
it is wrapped in a new `.Future`. This is suitable for use as
``result = yield gen.maybe_future(f())`` when you don't know whether
``f()`` returns a `.Future` or not.
"""
if is_future(x):
return x
else:
fut = Future()
fut.set_result(x)
return fut
def sleep(duration):
"""Return a `.Future` that resolves after the given number of seconds.
When used with ``yield`` in a coroutine, this is a non-blocking
analogue to `time.sleep` (which should not be used in coroutines
because it is blocking)::
yield gen.sleep(0.5)
Note that calling this function on its own does nothing; you must
wait on the `.Future` it returns (usually by yielding it).
"""
f = Future()
IOLoop.current().call_later(duration, lambda: f.set_result(None))
return f
def __init__(self,n_threads,ao, mth):
self.lock = RLock()
self.finished = Future().set_name('AO Visitor')
self.n_visits = 0
self.n_threads = n_threads
self.ao = ao
self.mth = mth
def connect(self, port, ip = '127.0.0.1'):
'''Nonblocking connect to remote address'''
if self.__future is not None:
raise RuntimeError("already connecting")
if self.__state != Connector.NONE:
raise RuntimeError("already connected")
self.__sock = socket.socket()
self.__sock.setblocking(False)
self.__peeraddr = (ip, port)
self.__future = Future()
try:
self.__sock.connect(self.__peeraddr)
except socket.error as serr:
if serr.errno != os.errno.EINPROGRESS and serr.errno != os.errno.EWOULDBLOCK:
logging.exception("connect error is not EINPROGRESS: %s", str(serr.errno))
self.__future.set_exception(ConnectError(str(serr.errno)))
self.close()
return self.__future
else:
self.__state = Connector.CONNECTING
else:
self.__state = Connector.CONNECTED
logging.info("connect immediately success %s", str(self.fileno()))
self._onSuccess()
return self.__future
from ioloop import IOLoop
self.__loop.register(self, IOLoop.WRITE_EVENT)
return self.__future
def _internal_invoke(self,method,a,kw,wrapper_level=None):
method,wrapper_info = self._get_callable(method,wrapper_level)
AO_cmd_meta_data = kw.pop('AO_cmd_meta_data',{})
call_type = getattr(method,'call_type',MethodCallType.Async)
f = Future()
cmd = BaseAO.Cmd(call_type,f,self,method,a,kw,wrapper_info,AO_cmd_meta_data)
f.set_name(cmd.qualified_name_no_wrappers)
if call_type == MethodCallType.Sync: # ignores queue and executes immediately + blocks
cmd()
return f.get()
else:
self.enq(cmd)
if hasattr(method,'sync_facade'): # does not ignore the queue but blocks
return f.get()
else:
return f
def start(self, config):
self._protocol.subscribe_channel(self._new_channel)
# Blockchain
future = Future()
self._chain.start(config["database"], future)
ec, = future.get()
if ec:
print >> sys.stderr, "Couldn't start blockchain:", str(ec)
return False
# Transaction pool
self._txpool.start()
# Session
future = Future()
self._session.start(future)
ec, = future.get()
if ec:
print >> sys.stderr, "Couldn't start session:", str(ec)
return False
return True
def multi_future(children, quiet_exceptions=()):
"""Wait for multiple asynchronous futures in parallel. like collectAll"""
if isinstance(children, dict):
keys = list(children.keys())
children = children.values()
else:
keys = None
children = list(map(convert_yielded, children))
assert all(is_future(i) for i in children)
unfinished_children = set(children)
future = Future() # it's a collectAll future
if not children:
future.set_result({} if keys is not None else [])
def callback(f_child):
unfinished_children.remove(f_child)
if not unfinished_children:
# all child futures are done!
result_list = []
for f in children:
try:
result_list.append(f.result())
except Exception as e:
if future.done():
if not isinstance(e, quiet_exceptions):
logging.error("Multiple exceptions in yield list")
else:
future.set_exc_info(sys.exc_info())
if not future.done():
if keys is not None:
future.set_result(dict(zip(keys, result_list)))
else:
future.set_result(result_list)
listening = set()
for child in children:
if child not in listening:
listening.add(child)
child.add_done_callback(callback) # when child future is done, tell parent future
return future
class AsyncRequest(object):
def __init__(self, host, url, port, timeout=5):
self.sock = socket.socket()
self.sock.settimeout(timeout)
self.sock.setblocking(False)
self.host = host
self.url = url
self.port = port
self.method = None
def get(self):
self.method = "GET"
self.request = "{} {} HTTP/1.0\r\nHost: {}\r\n\r\n".format(self.method, self.url, self.host)
return self
def process(self):
if self.method is None:
self.get()
try:
self.sock.connect((self.host, self.port))
except BlockingIOError:
pass
self.f = Future()
selector.register(self.sock.fileno(), EVENT_WRITE, self.on_connected)
yield self.f
selector.unregister(self.sock.fileno())
self.sock.send(self.request.encode("ascii"))
chunk = yield from read_all(self.sock)
return chunk
def on_connected(self, key, mask):
self.f.set_result(None)
def GetMulti(self, keys):
db_futures = dict(
(k,
db.get_async(
PersistentObjectStoreItem.CreateKey(self._namespace, k)))
for k in keys)
def resolve():
return dict((key, future.get_result().GetValue())
for key, future in db_futures.iteritems()
if future.get_result() is not None)
return Future(callback=resolve)
def StatAsync(self, path):
'''Stats the directory given, or if a file is given, stats the file's parent
directory to get info about the file.
'''
# Always stat the parent directory, since it will have the stat of the child
# anyway, and this gives us an entire directory's stat info at once.
dir_path, file_path = posixpath.split(path)
dir_path = ToDirectory(dir_path)
def make_stat_info(dir_stat):
'''Converts a dir stat into the correct resulting StatInfo; if the Stat
was for a file, the StatInfo should just contain that file.
'''
if path == dir_path:
return dir_stat
# Was a file stat. Extract that file.
file_version = dir_stat.child_versions.get(file_path)
if file_version is None:
raise FileNotFoundError('No stat found for %s in %s (found %s)' %
(path, dir_path, dir_stat.child_versions))
return StatInfo(file_version)
def raise_cache_miss(path):
raise FileNotFoundError('Got cache miss when trying to stat %s' % path)
dir_stat = self._stat_cache.Get(dir_path).Get()
if dir_stat is not None:
return Future(callback=lambda: make_stat_info(dir_stat))
if self._fail_on_miss:
logging.warning('Bailing on stat cache miss for %s' % dir_path)
return Future(callback=lambda: raise_cache_miss(dir_path))
def next(dir_stat):
assert dir_stat is not None # should have raised a FileNotFoundError
# We only ever need to cache the dir stat.
self._stat_cache.Set(dir_path, dir_stat)
return make_stat_info(dir_stat)
return self._MemoizedStatAsyncFromFileSystem(dir_path).Then(next)
def GetAPIFeatures(self):
api_features = self._object_store.Get('api_features').Get()
if api_features is not None:
return Future(value=api_features)
api_features_future = self._api_cache.GetFeatures()
manifest_features_future = self._manifest_cache.GetFeatures()
permission_features_future = self._permission_cache.GetFeatures()
def resolve():
api_features = api_features_future.Get()
manifest_features = manifest_features_future.Get()
permission_features = permission_features_future.Get()
# TODO(rockot): Handle inter-API dependencies more gracefully.
# Not yet a problem because there is only one such case (windows -> tabs).
# If we don't store this value before annotating platforms, inter-API
# dependencies will lead to infinite recursion.
for feature in api_features.itervalues():
_AddPlatformsAndChannelsFromDependencies(
feature, api_features, manifest_features, permission_features)
self._object_store.Set('api_features', api_features)
return api_features
return Future(callback=resolve)
def GetFromFileListing(self, path):
'''Calls |compilation_function| on the listing of the files at |path|.
Assumes that the path given is to a directory.
'''
AssertIsDirectory(path)
try:
version = self._file_system.Stat(path).version
except FileNotFoundError:
return Future(exc_info=sys.exc_info())
cache_entry = self._Get(self._list_object_store, path).Get()
if (cache_entry is not None) and (version == cache_entry.version):
return Future(value=cache_entry.cache_data)
def compile_(files):
cache_data = self._compilation_function(path, files)
self._Set(self._list_object_store, path,
_CacheEntry(cache_data, version))
return cache_data
return self._RecursiveList(path).Then(compile_)
def FetchAsync(self, url):
"""Fetches a file asynchronously, and returns a Future with the result.
"""
rpc = urlfetch.create_rpc()
if self._base_path is not None:
urlfetch.make_fetch_call(rpc,
self._base_path + '/' + url,
headers={'Cache-Control': 'max-age=0'})
else:
urlfetch.make_fetch_call(rpc,
url,
headers={'Cache-Control': 'max-age=0'})
return Future(delegate=_AsyncFetchDelegate(rpc))
def GetFromFile(self, path):
'''Calls |compilation_function| on the contents of the file at |path|. If
|binary| is True then the file will be read as binary - but this will only
apply for the first time the file is fetched; if already cached, |binary|
will be ignored.
'''
AssertIsFile(path)
try:
version = self._file_system.Stat(path).version
except FileNotFoundError:
return Future(exc_info=sys.exc_info())
cache_entry = self._file_object_store.Get(path).Get()
if (cache_entry is not None) and (version == cache_entry.version):
return Future(value=cache_entry._cache_data)
def next(files):
cache_data = self._compilation_function(path, files)
self._file_object_store.Set(path, _CacheEntry(cache_data, version))
return cache_data
return self._file_system.ReadSingle(path).Then(next)
def Refresh(self, path):
def safe(name, action, callback):
'''Safely runs |callback| for a ContentProvider called |name| by
swallowing exceptions and turning them into a None return value. It's
important to run all ContentProvider Refreshes even if some of them fail.
'''
try:
return callback()
except:
if not _IGNORE_MISSING_CONTENT_PROVIDERS[0]:
logging.error(
'Error %s Refresh for ContentProvider "%s":\n%s' %
(action, name, traceback.format_exc()))
return None
config = self._GetConfig()[path]
provider = self._CreateContentProvider(path, config)
future = safe(path, 'initializing',
self._CreateContentProvider(path, config).Refresh)
if future is None:
return Future(callback=lambda: True)
return Future(callback=lambda: safe(path, 'resolving', future.Get))
def _GetImpl(self, api_name):
handlebar_dict_future = self._GetSchemaModel(api_name)
def resolve():
handlebar_dict = handlebar_dict_future.Get()
# Parsing samples on the preview server takes seconds and doesn't add
# anything. Don't do it.
if not IsPreviewServer():
handlebar_dict['samples'] = _LazySamplesGetter(
handlebar_dict['name'], self._samples)
return handlebar_dict
return Future(callback=resolve)
def Read(self, paths, skip_not_found=False):
args = None
if self._revision is not None:
# |fetcher| gets from svn.chromium.org which uses p= for version.
args = 'p=%s' % self._revision
def apply_args(path):
return path if args is None else '%s?%s' % (path, args)
def list_dir(directory):
dom = xml.parseString(directory)
files = [
elem.childNodes[0].data
for elem in dom.getElementsByTagName('a')
]
if '..' in files:
files.remove('..')
return files
# A list of tuples of the form (path, Future).
fetches = [(path, self._file_fetcher.FetchAsync(apply_args(path)))
for path in paths]
def resolve():
value = {}
for path, future in fetches:
try:
result = future.Get()
except Exception as e:
if skip_not_found and IsDownloadError(e): continue
exc_type = (FileNotFoundError
if IsDownloadError(e) else FileSystemError)
raise exc_type(
'%s fetching %s for Get: %s' %
(type(e).__name__, path, traceback.format_exc()))
if result.status_code == 404:
if skip_not_found: continue
raise FileNotFoundError(
'Got 404 when fetching %s for Get, content %s' %
(path, result.content))
if result.status_code != 200:
raise FileSystemError(
'Got %s when fetching %s for Get, content %s' %
(result.status_code, path, result.content))
if path.endswith('/'):
value[path] = list_dir(result.content)
else:
value[path] = result.content
return value
return Future(callback=resolve)
def with_timeout(abstimeout, future, io_loop=None, quiet_exceptions=()):
"""Wraps a `.Future` in a timeout.
Raises `TimeoutError` if the input future does not complete before
``timeout``, which may be specified in any form allowed by
`.IOLoop.add_timeout` (i.e. a `datetime.timedelta` or an absolute time
relative to `.IOLoop.time`)
If the wrapped `.Future` fails after it has timed out, the exception
will be logged unless it is of a type contained in ``quiet_exceptions``
(which may be an exception type or a sequence of types).
Currently only supports Futures, not other `YieldPoint` classes.
"""
result = Future()
chain_future(future, result)
if io_loop is None:
io_loop = IOLoop.current()
def error_callback(future):
try:
future.result()
except Exception as e:
if not isinstance(e, quiet_exceptions):
logging.error("Exception in Future %r after timeout",
future,
exc_info=True)
def timeout_callback():
result.set_exception(TimeoutError("Timeout"))
# In case the wrapped future goes on to fail, log it.
future.add_done_callback(error_callback)
timeout_handle = io_loop.call_at(abstimeout, timeout_callback)
if isinstance(future, Future):
# We know this future will resolve on the IOLoop, so we don't
# need the extra thread-safety of IOLoop.add_future
future.add_done_callback(
lambda future: io_loop.remove_timeout(timeout_handle))
else:
# concurrent.futures.Futures may resolve on any thread, so we
# need to route them back to the IOLoop.
def cancel_on_ioloop(fut):
IOLoop.current().call_next_tick(
lambda fut: IOLoop.current().remove_timeout(timeout_handle))
future.add_done_callback(cancel_on_ioloop)
return result
def Exists(self, path):
'''Returns a Future to the existence of |path|; True if |path| exists,
False if not. This method will not throw a FileNotFoundError unlike
the Read* methods, however it may still throw a FileSystemError.
There are several ways to implement this method via the interface but this
method exists to do so in a canonical and most efficient way for caching.
'''
AssertIsValid(path)
if path == '':
# There is always a root directory.
return Future(value=True)
parent, base = SplitParent(path)
list_future = self.ReadSingle(ToDirectory(parent))
def resolve():
try:
return base in list_future.Get()
except FileNotFoundError:
return False
return Future(callback=resolve)
def get_feed(self, urls):
future_calls = [Future(feedparser.parse) for url in urls]
feeds = [future_obj() for future_obj in future_calls]
entries = []
for feed in feeds:
entries.extend(feed.entries)
sorted_entries = filter(
todays_catch,
sorted(entries, key=lambda entry: entry.published_parsed))
print len(sorted_entries)
return sorted_entries[::-1]
def Read(self, paths, binary=False):
patched_files = set()
added, deleted, modified = self._patcher.GetPatchedFiles()
if set(paths) & set(deleted):
raise FileNotFoundError('Files are removed from the patch.')
patched_files |= (set(added) | set(modified))
dir_paths = set(path for path in paths if path.endswith('/'))
file_paths = set(paths) - dir_paths
patched_paths = file_paths & patched_files
unpatched_paths = file_paths - patched_files
return Future(delegate=_AsyncFetchFuture(
self._host_file_system.Read(unpatched_paths, binary),
self._patcher.Apply(patched_paths, self._host_file_system, binary),
self._TryReadDirectory(dir_paths, binary), self))
class Visitor(object):
def __init__(self,n_threads,ao, mth):
self.lock = RLock()
self.finished = Future().set_name('AO Visitor')
self.n_visits = 0
self.n_threads = n_threads
self.ao = ao
self.mth = mth
def visit(self):
""" each thread will execute exactly one visit call """
with self.lock:
self.n_visits += 1
b_last = (self.n_visits == self.n_threads)
rc = mth()
if b_last:
self.finished.set(rc)
# This blocks the thread until we're done with all threads
# It's necessary so we don't dequeue another thread's visit command
# It also ensures that commands enqueued after visit started don't start executing until it's finished
# which makes visit act in a quiesced state.
self.finished.get()
def Refresh(self):
'''Compares the cached and live stat versions to see if the cached
repository is out of date. If it is, an async fetch is started and a
Future is returned. When this Future is evaluated, the fetch will be
completed and the results cached.
If no update is needed, None will be returned.
'''
version = self._FetchLiveVersion()
repo_zip_url = self._repo_url + '/zipball'
def persist_fetch(fetch):
'''Completes |fetch| and stores the results in blobstore.
'''
try:
blob = fetch.Get().content
except urlfetch.DownloadError:
logging.error(
'%s: Failed to download zip file from repository %s' % repo_zip_url)
else:
try:
zipfile = ZipFile(StringIO(blob))
except BadZipfile as error:
logging.error(
'%s: Bad zip file returned from url %s' % (error, repo_zip_url))
else:
self._blobstore.Set(repo_zip_url, blob, _GITHUB_REPOS_NAMESPACE)
self._repo_zip = Future(value=zipfile)
self._stat_cache.Set(self._repo_key, version)
# If the cached and live stat versions are different fetch the new repo.
if version != self._stat_cache.Get('stat').Get():
fetch = self._fetcher.FetchAsync(
'zipball', username=self._username, password=self._password)
return Future(delegate=Gettable(lambda: persist_fetch(fetch)))
return Future(value=None)
def Read(self, paths, binary=False):
'''Reads a list of files. If a file is in memcache and it is not out of
date, it is returned. Otherwise, the file is retrieved from the file system.
'''
read_object_store = (self._read_binary_object_store if binary else
self._read_object_store)
read_values = read_object_store.GetMulti(paths).Get()
stat_values = self._stat_object_store.GetMulti(paths).Get()
results = {} # maps path to read value
uncached = {} # maps path to stat value
for path in paths:
stat_value = stat_values.get(path)
if stat_value is None:
# TODO(cduvall): do a concurrent Stat with the missing stat values.
try:
stat_value = self.Stat(path)
except:
return Future(exc_info=sys.exc_info())
read_value = read_values.get(path)
if read_value is None:
uncached[path] = stat_value
continue
read_data, read_version = read_value
if stat_value.version != read_version:
uncached[path] = stat_value
continue
results[path] = read_data
if not uncached:
return Future(value=results)
return Future(delegate=_AsyncUncachedFuture(
self._file_system.Read(uncached.keys(), binary=binary),
uncached,
results,
self,
read_object_store))
def Read(self, paths, binary=False):
'''Reads |paths| from |_file_system|, then applies the most recent update
from |_updates|, if any.
'''
self._read_count += 1
future_result = self._file_system.Read(paths, binary=binary)
try:
result = future_result.Get()
except:
return future_result
for path in result.iterkeys():
_, update = self._GetMostRecentUpdate(path)
if update is not None:
result[path] = update
return Future(value=result)
def FetchAsync(self, url, username=None, password=None):
"""Fetches a file asynchronously, and returns a Future with the result.
"""
rpc = urlfetch.create_rpc(deadline=20)
headers = _MakeHeaders(username, password)
import logging
if self._base_path is not None:
logging.info('%s/%s' % (self._base_path, url))
urlfetch.make_fetch_call(rpc,
'%s/%s' % (self._base_path, url),
headers=headers)
else:
logging.info(url)
urlfetch.make_fetch_call(rpc, url, headers=headers)
return Future(delegate=_AsyncFetchDelegate(rpc))
def fetch_from_blobstore():
'''Returns a Future which resolves to the _GithubZipFile for this repo
fetched from blobstore.
'''
blob = self._blobstore.Get(repo_url, _GITHUB_REPOS_NAMESPACE)
if blob is None:
return FileSystemError.RaiseInFuture(
'No blob for %s found in datastore' % repo_key)
repo_zip = _GithubZipFile.Create(repo_key, blob)
if repo_zip is None:
return FileSystemError.RaiseInFuture(
'Blob for %s was corrupted in blobstore!?' % repo_key)
return Future(value=repo_zip)
def Read(self, paths, binary=False):
# Maintain reverse mapping so the result can be mapped to the original
# paths given (the result from |file_system| will include |root| in the
# result, which would be wrong).
prefixed_paths = {}
def prefix(path):
prefixed = posixpath.join(self._root, path)
prefixed_paths[prefixed] = path
return prefixed
future_result = self._file_system.Read(
tuple(prefix(path) for path in paths), binary=binary)
def resolve():
return dict((prefixed_paths[path], content)
for path, content in future_result.Get().iteritems())
return Future(delegate=Gettable(resolve))
def Read(self, paths):
def resolve():
result = {}
for path in paths:
AssertIsValid(path)
full_path = os.path.join(
self._base_path,
_ConvertToFilepath(path).lstrip(os.sep))
if path == '' or path.endswith('/'):
result[path] = _ListDir(full_path)
else:
result[path] = _ReadFile(full_path)
return result
return Future(delegate=Gettable(resolve))
def Read(self, paths, binary=False):
'''Reads |paths| from |_file_system|, then applies the most recent update
from |_updates|, if any.
'''
self._read_count += 1
future_result = self._file_system.Read(paths, binary=binary)
def resolve():
self._read_resolve_count += 1
result = future_result.Get()
for path in result.iterkeys():
_, update = self._GetMostRecentUpdate(path)
if update is not None:
result[path] = update
return result
return Future(delegate=Gettable(resolve))
def GetMulti(self, keys, time=CACHE_TIMEOUT):
keys = keys[:]
mapping = {}
for key in keys:
cache_entry = self._cache.get(key, None)
if cache_entry is None or cache_entry.HasExpired():
mapping[key] = None
else:
mapping[key] = cache_entry.value
keys.remove(key)
future = self._object_store.GetMulti(keys, time=time)
return Future(delegate=_AsyncGetFuture(self._cache,
time,
future,
mapping))
def Read(self, paths, skip_not_found=False):
def resolve():
result = {}
for path in paths:
AssertIsValid(path)
full_path = os.path.join(
self._base_path,
_ConvertToFilepath(path).lstrip(os.sep))
if path == '' or path.endswith('/'):
result[path] = _ListDir(full_path)
else:
result[path] = _ReadFile(full_path)
return result
return Future(callback=resolve)
def _GetSchemaModel(self, api_name):
jsc_model_future = self._model_cache.Get(api_name)
model_future = self._api_models.GetModel(api_name)
def resolve():
jsc_model = jsc_model_future.Get()
if jsc_model is None:
jsc_model = _JSCModel(model_future.Get(),
self._availability_finder,
self._json_cache, self._template_cache,
self._features_bundle,
self._LoadEventByName).ToDict()
self._model_cache.Set(api_name, jsc_model)
return jsc_model
return Future(callback=resolve)
def GetModel(self, api_name):
# By default |api_name| is assumed to be given without a path or extension,
# so combinations of known paths and extension types will be searched.
api_extensions = ('.json', '.idl')
api_paths = API_PATHS
# Callers sometimes include a file extension and/or prefix path with the
# |api_name| argument. We believe them and narrow the search space
# accordingly.
name, ext = posixpath.splitext(api_name)
if ext in api_extensions:
api_extensions = (ext, )
api_name = name
for api_path in api_paths:
if api_name.startswith(api_path):
api_name = api_name[len(api_path):]
api_paths = (api_path, )
break
# API names are given as declarativeContent and app.window but file names
# will be declarative_content and app_window.
file_name = UnixName(api_name).replace('.', '_')
# Devtools APIs are in API/devtools/ not API/, and have their
# "devtools" names removed from the file names.
basename = posixpath.basename(file_name)
if 'devtools_' in basename:
file_name = posixpath.join(
'devtools',
file_name.replace(basename, basename.replace('devtools_', '')))
futures = [
self._model_cache.GetFromFile(
posixpath.join(path, '%s%s' % (file_name, ext)))
for ext in api_extensions for path in api_paths
]
def resolve():
for future in futures:
try:
return future.Get()
# Either the file wasn't found or there was no schema for the file
except (FileNotFoundError, ValueError):
pass
# Propagate the first error if neither were found.
futures[0].Get()
return Future(callback=resolve)
def _FindFileForPath(self, path):
'''Finds the real file backing |path|. This may require looking for the
correct file extension, or looking for an 'index' file if it's a directory.
Returns None if no path is found.
'''
AssertIsValid(path)
_, ext = posixpath.splitext(path)
if ext:
# There was already an extension, trust that it's a path. Elsewhere
# up the stack this will be caught if it's not.
return Future(value=path)
def find_file_with_name(name):
'''Tries to find a file in the file system called |name| with one of the
default extensions of this content provider.
If none is found, returns None.
'''
paths = [name + ext for ext in self._default_extensions]
def get_first_path_which_exists(existence):
for exists, path in zip(existence, paths):
if exists:
return path
return None
return (All(self.file_system.Exists(path)
for path in paths).Then(get_first_path_which_exists))
def find_index_file():
'''Tries to find an index file in |path|, if |path| is a directory.
If not, or if there is no index file, returns None.
'''
def get_index_if_directory_exists(directory_exists):
if not directory_exists:
return None
return find_file_with_name(Join(path, 'index'))
return (self.file_system.Exists(
ToDirectory(path)).Then(get_index_if_directory_exists))
# Try to find a file with the right name. If not, and it's a directory,
# look for an index file in that directory. If nothing at all is found,
# return the original |path| - its nonexistence will be caught up the stack.
return (find_file_with_name(path).Then(
lambda found: found or find_index_file()).Then(
lambda found: found or path))
def Read(self, paths):
'''Reads |paths| from |_file_system|, then applies the most recent update
from |_updates|, if any.
'''
self._read_count += 1
future_result = self._file_system.Read(paths)
def resolve():
self._read_resolve_count += 1
result = future_result.Get()
for path in result.iterkeys():
update = self._GetMostRecentUpdate(path)
if update is not None:
result[path] = update
return result
return Future(callback=resolve)
def FetchAsyncImpl(self, url, headers):
def process_result(result):
if result.status_code == 429:
if self._retries_left == 0:
logging.error('Still throttled. Giving up.')
return result
self._retries_left -= 1
logging.info('Throttled. Trying again in %s seconds.' %
_RETRY_DELAY_SECONDS)
time.sleep(_RETRY_DELAY_SECONDS)
return self.FetchAsync(url, username, password,
access_token).Get()
return result
rpc = urlfetch.create_rpc(deadline=20)
urlfetch.make_fetch_call(rpc, url, headers=headers)
return Future(callback=lambda: process_result(rpc.get_result()))
def resolve():
result = {}
for path in paths:
AssertIsValid(path)
full_path = os.path.join(
self._base_path,
_ConvertToFilepath(path).lstrip(os.sep))
if path == '' or path.endswith('/'):
result[path] = _ListDir(full_path)
else:
try:
result[path] = _ReadFile(full_path)
except FileNotFoundError:
if skip_not_found:
continue
return Future(exc_info=sys.exc_info())
return result
def _LoadCache(self):
cached_future = self._cache.GetMulti(
('canonical_paths', 'simplified_paths_map'))
def resolve():
# |canonical_paths| is the pre-calculated set of canonical paths.
# |simplified_paths_map| is a lazily populated mapping of simplified file
# names to a list of full paths that contain them. For example,
# - browseraction: [extensions/browserAction.html]
# - storage: [apps/storage.html, extensions/storage.html]
cached = cached_future.Get()
canonical_paths, simplified_paths_map = (
cached.get('canonical_paths'),
cached.get('simplified_paths_map'))
if canonical_paths is None:
assert simplified_paths_map is None
canonical_paths = set()
simplified_paths_map = defaultdict(list)
for base, dirs, files in self._file_system.Walk(''):
for path in dirs + files:
path_without_ext, ext = posixpath.splitext(path)
canonical_path = posixpath.join(base, path_without_ext)
if (ext not in self._strip_extensions
or path == SITE_VERIFICATION_FILE):
canonical_path += ext
canonical_paths.add(canonical_path)
simplified_paths_map[_Normalize(
path, splittext=True)].append(canonical_path)
# Store |simplified_paths_map| sorted. Ties in length are broken by
# taking the shortest, lexicographically smallest path.
for path_list in simplified_paths_map.itervalues():
path_list.sort(key=lambda p: (len(p), p))
self._cache.SetMulti({
'canonical_paths':
canonical_paths,
'simplified_paths_map':
simplified_paths_map,
})
else:
assert simplified_paths_map is not None
return canonical_paths, simplified_paths_map
return Future(callback=resolve)
def _apply_hook(self, future_hook, func_set_result, func_continue, stage_entered):
if not future_hook:
future_hook = Future.preset(None).set_name('AO empty future hook')
def completion(future_hook):
if stage_entered != None:
self.ao.method_call_hook(stage_entered,self.call_type,self)
val,exc = future_hook.get_val_or_error()
if exc is not None:
func_set_result(exc=exc)
elif isinstance(val,AsyncRV):
func_set_result(val=val.val)
else:
assertions.fail_unless(val is None, "Hook returned a value, but not through AsyncRV", val=val)
func_continue()
if self.call_type == MethodCallType.Coroutine:
future_hook.attach_observer(partial(self._async_completion,stage_entered,completion)) # Coroutine - don't block AO thread while waiting for hook's future
else:
future_hook.wait()
completion(future_hook)
def process(self):
if self.method is None:
self.get()
try:
self.sock.connect((self.host, self.port))
except BlockingIOError:
pass
self.f = Future()
selector.register(self.sock.fileno(), EVENT_WRITE, self.on_connected)
yield self.f
selector.unregister(self.sock.fileno())
self.sock.send(self.request.encode("ascii"))
chunk = yield from read_all(self.sock)
return chunk
def __init__(self, coro):
super().__init__()
self.coro = coro
f = Future()
f.set_result(None)
self.step(f)
When used with ``yield`` in a coroutine, this is a non-blocking
analogue to `time.sleep` (which should not be used in coroutines
because it is blocking)::
yield gen.sleep(0.5)
Note that calling this function on its own does nothing; you must
wait on the `.Future` it returns (usually by yielding it).
"""
f = Future()
IOLoop.current().call_later(duration, lambda: f.set_result(None))
return f
_null_future = Future()
_null_future.set_result(None)
moment = Future()
moment.set_result(None)
moment.__doc__ = \
"""A special object which may be yielded to allow the IOLoop to run for
one iteration.
This is not needed in normal use but it can be helpful in long-running
coroutines that are likely to yield Futures that are ready instantly.
Usage: ``yield gen.moment``
"""
class Runner(object):
def progress_caller(increment):
"""Syntactic sugaring for progressing caller without doing anything
"""
return Future.preset(None).progress_caller(increment)
def progress_caller_to(endpoint):
"""Syntactic sugaring for progressing caller to endpoint without doing anything
"""
return Future.preset(None).progress_caller_to(endpoint)
def test_simple(self):
future = Future(identity, x=5)
self.assertEqual(5, future.run())
def sleep(timeout):
future = Future()
event = SleepEvent(timeout)
event.set_callback(lambda: future.done())
return future
class Connector:
''' Tcp connector'''
NONE = 0
CONNECTING = 1
CONNECTED = 2
def __init__(self, loop):
self.__sock = None
self.__peeraddr = None
self.__state = Connector.NONE
self.__loop = loop
self.__future = None
from handler import NewConnectionCallback
self.__newConnectionCallback = NewConnectionCallback()
def setNewConnCallback(self, callback):
self.__newConnectionCallback = callback
def fileno(self):
return self.__sock.fileno()
def connect(self, port, ip = '127.0.0.1'):
'''Nonblocking connect to remote address'''
if self.__future is not None:
raise RuntimeError("already connecting")
if self.__state != Connector.NONE:
raise RuntimeError("already connected")
self.__sock = socket.socket()
self.__sock.setblocking(False)
self.__peeraddr = (ip, port)
self.__future = Future()
try:
self.__sock.connect(self.__peeraddr)
except socket.error as serr:
if serr.errno != os.errno.EINPROGRESS and serr.errno != os.errno.EWOULDBLOCK:
logging.exception("connect error is not EINPROGRESS: %s", str(serr.errno))
self.__future.set_exception(ConnectError(str(serr.errno)))
self.close()
return self.__future
else:
self.__state = Connector.CONNECTING
else:
self.__state = Connector.CONNECTED
logging.info("connect immediately success %s", str(self.fileno()))
self._onSuccess()
return self.__future
from ioloop import IOLoop
self.__loop.register(self, IOLoop.WRITE_EVENT)
return self.__future
def close(self):
if self.__sock is None:
return
try:
self.__sock.shutdown(socket.SHUT_WR)
self.__sock.close()
self.__sock = None
except socket.error as serr:
pass
self.__state = Connector.NONE
def onError(self):
self.__future.set_exception(ConnectError())
self.close()
def onReadable(self):
raise RuntimeError("connector should not readable")
def onWritable(self):
err = self.__sock.getsockopt(socket.SOL_SOCKET, socket.SO_ERROR)
if err != 0:
logging.error("Connector onWritable error %s", os.strerror(err))
self.__future.set_exception(ConnectError())
self.close()
return False
self._onSuccess()
return True
def _onSuccess(self):
self.__state = Connector.CONNECTED
from connection import Connection
conn = Connection(loop = self.__loop, socket = self.__sock, remote = self.__peeraddr)
self.__newConnectionCallback(conn)
from ioloop import IOLoop
self.__loop.modify(conn, IOLoop.READ_EVENT | IOLoop.WRITE_EVENT)
conn.onConnect()
self.__sock = None
self.__future.set_result(conn)
def checkpoint(name):
"""Syntactic sugaring for declaring hook points
"""
return Future.preset(None).set_name(name)
