def _next_source(resp, idurl, sources, pos):
global _LastTimeCached
if _Debug:
lg.args(_DebugLevel,
resp=resp,
idurl=idurl,
pos=pos,
sources=len(sources))
if pos >= len(sources):
lg.warn('[cache failed] %r and also %d other sources' %
(idurl, len(sources)))
defer_results = caching().pop(idurl, [])
for result in defer_results:
if result and not result.called:
reactor.callLater(
0, result.errback,
Exception('cache failed from multiple sources'
)) # @UndefinedVariable
p2p_stats.count_identity_cache(idurl, 0)
_LastTimeCached.pop(idurl, None)
defer_results = []
del defer_results
return None
next_idurl = sources[pos]
next_idurl = id_url.to_original(next_idurl)
if _Debug:
lg.args(_DebugLevel, next_idurl=next_idurl)
d = net_misc.getPageTwisted(url=next_idurl, timeout=timeout)
d.addCallback(_success, idurl)
d.addErrback(_next_source, idurl, sources, pos + 1)
return None
def getPageSuccess(src, idurl):
"""
This is called when requested identity source gets received.
"""
UpdateAfterChecking(idurl, src)
p2p_stats.count_identity_cache(idurl, len(src))
return src
def _success(src, idurl):
global _LastTimeCached
idurl = id_url.to_original(idurl)
defer_results = caching().pop(idurl, [])
if _Debug:
lg.args(_DebugLevel,
src=type(src),
idurl=idurl,
defer_results=len(defer_results))
if UpdateAfterChecking(idurl, src):
for result in defer_results:
if result and not result.called:
reactor.callLater(0, result.callback,
src) # @UndefinedVariable
if _Debug:
lg.out(_DebugLevel, '[cached] %s' % idurl)
p2p_stats.count_identity_cache(idurl, len(src))
_LastTimeCached[idurl] = time.time()
else:
for result in defer_results:
if result and not result.called:
reactor.callLater(0, result.errback,
Exception(src)) # @UndefinedVariable
lg.warn('[cache error] %s is not valid' % idurl)
p2p_stats.count_identity_cache(idurl, 0)
_LastTimeCached.pop(idurl, None)
defer_results = []
del defer_results
return src
def getPageFail(x, idurl):
"""
This is called when identity request is failed.
"""
if _Debug:
lg.out(6, "identitycache.getPageFail NETERROR in request to " + idurl)
p2p_stats.count_identity_cache(idurl, 0)
return x
def _getPageFail(x, idurl):
global _CachingTasks
result = _CachingTasks.pop(idurl)
if result:
result.errback(x)
else:
lg.warn('caching task for %s was not found' % idurl)
p2p_stats.count_identity_cache(idurl, 0)
if _Debug:
lg.warn('[cache failed] %s : %s' % (idurl, x.getErrorMessage(), ))
return None
def _fail(err, idurl):
global _CachingTasks
idurl = id_url.to_original(idurl)
result = _CachingTasks.pop(idurl)
if not try_other_sources:
if result:
result.errback(err)
else:
lg.warn('caching task for %s was not found' % idurl)
p2p_stats.count_identity_cache(idurl, 0)
lg.warn('[cache failed] %s : %s' % (
idurl,
err.getErrorMessage(),
))
return None
latest_idurl, latest_rev = id_url.get_latest_revision(idurl)
latest_ident = None
sources = []
if latest_idurl:
latest_ident = identitydb.get_ident(latest_idurl)
if latest_ident:
sources = latest_ident.getSources(as_fields=False)
if sources:
if idurl in sources:
sources = sources.remove(idurl)
if sources:
lg.warn('[cache failed] %s : %s but will try %d more sources' % (
idurl,
err.getErrorMessage(),
len(sources),
))
_next_source(err, sources, 0, result)
return result
if result:
result.errback(err)
else:
lg.warn('caching task for %s was not found' % idurl)
p2p_stats.count_identity_cache(idurl, 0)
lg.warn('[cache failed] and also no other sources found %s : %s' % (
idurl,
err.getErrorMessage(),
))
return None
def _success(src, idurl):
global _CachingTasks
global _LastTimeCached
idurl = id_url.to_original(idurl)
result = _CachingTasks.pop(idurl, None)
if not result:
lg.warn('caching task for %s was not found' % idurl)
if UpdateAfterChecking(idurl, src):
if result:
result.callback(src)
lg.out(_DebugLevel, '[cached] %s' % idurl)
p2p_stats.count_identity_cache(idurl, len(src))
_LastTimeCached[idurl] = time.time()
else:
if result:
result.errback(Exception(src))
lg.warn('[cache error] %s is not valid' % idurl)
p2p_stats.count_identity_cache(idurl, 0)
return src
def _getPageSuccess(src, idurl):
global _CachingTasks
result = _CachingTasks.pop(idurl, None)
if not result:
lg.warn('caching task for %s was not found' % idurl)
return src
if UpdateAfterChecking(idurl, src):
if result:
result.callback(src)
if _Debug:
lg.out(14, '[cached] %s' % idurl)
p2p_stats.count_identity_cache(idurl, len(src))
else:
if result:
result.errback(Exception(src))
if _Debug:
lg.warn('[cache error] %s is not valid' % idurl)
p2p_stats.count_identity_cache(idurl, 0)
return src
def _fail(err, idurl):
global _LastTimeCached
idurl = id_url.to_original(idurl)
if _Debug:
lg.args(_DebugLevel, err=err, idurl=idurl)
if not try_other_sources:
p2p_stats.count_identity_cache(idurl, 0)
_LastTimeCached.pop(idurl, None)
lg.warn('[cache failed] %s : %s' % (
idurl,
err.getErrorMessage(),
))
defer_results = caching().pop(idurl, [])
for result in defer_results:
if result and not result.called:
reactor.callLater(0, result.errback,
err) # @UndefinedVariable
defer_results = []
del defer_results
return None
sources = []
latest_ident = None
latest_idurl, _ = id_url.get_latest_revision(idurl)
if not latest_idurl:
latest_idurl = idurl
if latest_idurl:
latest_ident = identitydb.get_ident(latest_idurl)
if latest_ident:
sources.extend(list(latest_ident.getSources(as_originals=True)))
if not sources:
pub_key = id_url.known().get(latest_idurl)
if pub_key:
known_sources = id_url.sources(pub_key)
for another_idurl in reversed(known_sources):
if another_idurl != latest_idurl and another_idurl != idurl:
if another_idurl not in sources:
sources.append(another_idurl)
if idurl in sources:
sources.remove(idurl)
if _Debug:
lg.args(_DebugLevel,
idurl=idurl,
latest_idurl=latest_idurl,
latest_ident=latest_ident,
sources=sources)
if sources:
lg.warn('[cache failed] %s : %s but will try %d more sources' % (
idurl,
err.getErrorMessage(),
len(sources),
))
_next_source(None, idurl, sources, 0)
return None
p2p_stats.count_identity_cache(idurl, 0)
_LastTimeCached.pop(idurl, None)
lg.warn('[cache failed] and also no other sources found %s : %s' % (
idurl,
err.getErrorMessage(),
))
defer_results = caching().pop(idurl, [])
if _Debug:
lg.args(_DebugLevel,
known=len(id_url.known().keys()),
defer_results=len(defer_results))
for result in defer_results:
if result and not result.called:
reactor.callLater(0, result.errback, err) # @UndefinedVariable
defer_results = []
del defer_results
return None