def _next_source(resp, idurl, sources, pos): global _LastTimeCached if _Debug: lg.args(_DebugLevel, resp=resp, idurl=idurl, pos=pos, sources=len(sources)) if pos >= len(sources): lg.warn('[cache failed] %r and also %d other sources' % (idurl, len(sources))) defer_results = caching().pop(idurl, []) for result in defer_results: if result and not result.called: reactor.callLater( 0, result.errback, Exception('cache failed from multiple sources' )) # @UndefinedVariable p2p_stats.count_identity_cache(idurl, 0) _LastTimeCached.pop(idurl, None) defer_results = [] del defer_results return None next_idurl = sources[pos] next_idurl = id_url.to_original(next_idurl) if _Debug: lg.args(_DebugLevel, next_idurl=next_idurl) d = net_misc.getPageTwisted(url=next_idurl, timeout=timeout) d.addCallback(_success, idurl) d.addErrback(_next_source, idurl, sources, pos + 1) return None
def getPageSuccess(src, idurl): """ This is called when requested identity source gets received. """ UpdateAfterChecking(idurl, src) p2p_stats.count_identity_cache(idurl, len(src)) return src
def _success(src, idurl): global _LastTimeCached idurl = id_url.to_original(idurl) defer_results = caching().pop(idurl, []) if _Debug: lg.args(_DebugLevel, src=type(src), idurl=idurl, defer_results=len(defer_results)) if UpdateAfterChecking(idurl, src): for result in defer_results: if result and not result.called: reactor.callLater(0, result.callback, src) # @UndefinedVariable if _Debug: lg.out(_DebugLevel, '[cached] %s' % idurl) p2p_stats.count_identity_cache(idurl, len(src)) _LastTimeCached[idurl] = time.time() else: for result in defer_results: if result and not result.called: reactor.callLater(0, result.errback, Exception(src)) # @UndefinedVariable lg.warn('[cache error] %s is not valid' % idurl) p2p_stats.count_identity_cache(idurl, 0) _LastTimeCached.pop(idurl, None) defer_results = [] del defer_results return src
def getPageFail(x, idurl): """ This is called when identity request is failed. """ if _Debug: lg.out(6, "identitycache.getPageFail NETERROR in request to " + idurl) p2p_stats.count_identity_cache(idurl, 0) return x
def _getPageFail(x, idurl): global _CachingTasks result = _CachingTasks.pop(idurl) if result: result.errback(x) else: lg.warn('caching task for %s was not found' % idurl) p2p_stats.count_identity_cache(idurl, 0) if _Debug: lg.warn('[cache failed] %s : %s' % (idurl, x.getErrorMessage(), )) return None
def _fail(err, idurl): global _CachingTasks idurl = id_url.to_original(idurl) result = _CachingTasks.pop(idurl) if not try_other_sources: if result: result.errback(err) else: lg.warn('caching task for %s was not found' % idurl) p2p_stats.count_identity_cache(idurl, 0) lg.warn('[cache failed] %s : %s' % ( idurl, err.getErrorMessage(), )) return None latest_idurl, latest_rev = id_url.get_latest_revision(idurl) latest_ident = None sources = [] if latest_idurl: latest_ident = identitydb.get_ident(latest_idurl) if latest_ident: sources = latest_ident.getSources(as_fields=False) if sources: if idurl in sources: sources = sources.remove(idurl) if sources: lg.warn('[cache failed] %s : %s but will try %d more sources' % ( idurl, err.getErrorMessage(), len(sources), )) _next_source(err, sources, 0, result) return result if result: result.errback(err) else: lg.warn('caching task for %s was not found' % idurl) p2p_stats.count_identity_cache(idurl, 0) lg.warn('[cache failed] and also no other sources found %s : %s' % ( idurl, err.getErrorMessage(), )) return None
def _success(src, idurl): global _CachingTasks global _LastTimeCached idurl = id_url.to_original(idurl) result = _CachingTasks.pop(idurl, None) if not result: lg.warn('caching task for %s was not found' % idurl) if UpdateAfterChecking(idurl, src): if result: result.callback(src) lg.out(_DebugLevel, '[cached] %s' % idurl) p2p_stats.count_identity_cache(idurl, len(src)) _LastTimeCached[idurl] = time.time() else: if result: result.errback(Exception(src)) lg.warn('[cache error] %s is not valid' % idurl) p2p_stats.count_identity_cache(idurl, 0) return src
def _getPageSuccess(src, idurl): global _CachingTasks result = _CachingTasks.pop(idurl, None) if not result: lg.warn('caching task for %s was not found' % idurl) return src if UpdateAfterChecking(idurl, src): if result: result.callback(src) if _Debug: lg.out(14, '[cached] %s' % idurl) p2p_stats.count_identity_cache(idurl, len(src)) else: if result: result.errback(Exception(src)) if _Debug: lg.warn('[cache error] %s is not valid' % idurl) p2p_stats.count_identity_cache(idurl, 0) return src
def _fail(err, idurl): global _LastTimeCached idurl = id_url.to_original(idurl) if _Debug: lg.args(_DebugLevel, err=err, idurl=idurl) if not try_other_sources: p2p_stats.count_identity_cache(idurl, 0) _LastTimeCached.pop(idurl, None) lg.warn('[cache failed] %s : %s' % ( idurl, err.getErrorMessage(), )) defer_results = caching().pop(idurl, []) for result in defer_results: if result and not result.called: reactor.callLater(0, result.errback, err) # @UndefinedVariable defer_results = [] del defer_results return None sources = [] latest_ident = None latest_idurl, _ = id_url.get_latest_revision(idurl) if not latest_idurl: latest_idurl = idurl if latest_idurl: latest_ident = identitydb.get_ident(latest_idurl) if latest_ident: sources.extend(list(latest_ident.getSources(as_originals=True))) if not sources: pub_key = id_url.known().get(latest_idurl) if pub_key: known_sources = id_url.sources(pub_key) for another_idurl in reversed(known_sources): if another_idurl != latest_idurl and another_idurl != idurl: if another_idurl not in sources: sources.append(another_idurl) if idurl in sources: sources.remove(idurl) if _Debug: lg.args(_DebugLevel, idurl=idurl, latest_idurl=latest_idurl, latest_ident=latest_ident, sources=sources) if sources: lg.warn('[cache failed] %s : %s but will try %d more sources' % ( idurl, err.getErrorMessage(), len(sources), )) _next_source(None, idurl, sources, 0) return None p2p_stats.count_identity_cache(idurl, 0) _LastTimeCached.pop(idurl, None) lg.warn('[cache failed] and also no other sources found %s : %s' % ( idurl, err.getErrorMessage(), )) defer_results = caching().pop(idurl, []) if _Debug: lg.args(_DebugLevel, known=len(id_url.known().keys()), defer_results=len(defer_results)) for result in defer_results: if result and not result.called: reactor.callLater(0, result.errback, err) # @UndefinedVariable defer_results = [] del defer_results return None