def doRequestMyIdentity(self, arg): global _WorkingIDURL idurl = _WorkingIDURL lg.out(4, 'identity_restorer.doRequestMyIdentity %s %s' % (idurl, type(idurl))) net_misc.getPageTwisted(idurl).addCallbacks( lambda src: self.automat('my-id-received', src), lambda err: self.automat('my-id-failed', err))
def doPingServers(self, arg): """ Action method. """ lg.out(4, 'id_registrator.doPingServers %d in list' % len(self.discovered_servers)) def _cb(htmlsrc, id_server_host): lg.out(4, ' RESPONDED: %s' % id_server_host) if self.preferred_servers and id_server_host in self.preferred_servers: self.good_servers.insert(0, id_server_host) else: self.good_servers.append(id_server_host) self.discovered_servers.remove(id_server_host) self.automat('id-server-response', (id_server_host, htmlsrc)) def _eb(err, id_server_host): lg.out(4, ' FAILED: %s' % id_server_host) self.discovered_servers.remove(id_server_host) self.automat('id-server-failed', (id_server_host, err)) for host in self.discovered_servers: webport, tcpport = known_servers.by_host().get(host, (settings.IdentityWebPort(), settings.IdentityServerPort())) if webport == 80: webport = '' server_url = nameurl.UrlMake('http', host, webport, '') d = net_misc.getPageTwisted(server_url, timeout=10) d.addCallback(_cb, host) d.addErrback(_eb, host)
def download_info(): def _done(src, result): lg.out(6, 'os_windows_update.download_info.done ') lines = src.split('\n') files_dict = {} for line in lines: words = line.split(' ') if len(words) < 2: continue files_dict[words[1].strip()] = words[0].strip() result.callback(files_dict) return src def _fail(x, result): lg.out(1, 'os_windows_update.download_info FAILED') result.errback(Exception('error downloading info')) return x repo, locationURL = misc.ReadRepoLocation() url = locationURL + settings.FilesDigestsFilename() lg.out(6, 'os_windows_update.download_info ' + str(url)) result = Deferred() d = net_misc.getPageTwisted(url) d.addCallback(_done, result) d.addErrback(_fail, result) return result
def doRequestServers(self, arg): """ Action method. """ login = bpio.ReadTextFile(settings.UserNameFilename()) def _cb(xmlsrc, idurl, host): lg.out(4, ' EXIST: %s' % idurl) self.registrations.remove(idurl) self.automat('id-exist', idurl) def _eb(err, idurl, host): lg.out(4, ' NOT EXIST: %s' % idurl) if self.preferred_server and self.preferred_server == host: self.free_idurls.insert(0, idurl) else: self.free_idurls.append(idurl) self.registrations.remove(idurl) self.automat('id-not-exist', idurl) for host in self.good_servers: webport, tcpport = known_servers.by_host().get( host, (settings.IdentityWebPort(), settings.IdentityServerPort())) if webport == 80: webport = '' idurl = nameurl.UrlMake('http', host, webport, login + '.xml') lg.out(4, ' %s' % idurl) d = net_misc.getPageTwisted(idurl, timeout=10) d.addCallback(_cb, idurl, host) d.addErrback(_eb, idurl, host) self.registrations.append(idurl) lg.out(4, 'id_registrator.doRequestServers login=%s registrations=%d' % (login, len(self.registrations)))
def _next_source(err, sources, pos, ret): if pos >= len(sources): lg.warn('[cache failed] %r from %d sources' % (idurl, len(sources))) if ret: ret.errback(Exception('cache failed from %d sources' % len(sources))) return None next_idurl = sources[pos] next_idurl = id_url.to_original(next_idurl) if _Debug: lg.out(_DebugLevel, 'identitycache.immediatelyCaching._next_source %r from %r : %r' % (pos, sources, next_idurl, )) if next_idurl in _CachingTasks: if _Debug: lg.out(_DebugLevel, 'identitycache.immediatelyCaching already have next task for %r' % next_idurl) d = _CachingTasks[next_idurl] else: if _Debug: lg.out(_DebugLevel, 'identitycache.immediatelyCaching will try another source of %r : %r' % (idurl, next_idurl)) _CachingTasks[next_idurl] = Deferred() _CachingTasks[next_idurl].addErrback(on_caching_task_failed, next_idurl) d = net_misc.getPageTwisted(next_idurl, timeout) d.addCallback(_success, next_idurl) if ret: d.addCallback(ret.callback) d.addErrback(_next_source, sources, pos+1, ret) return None
def doRequestMyIdentity(self, *args, **kwargs): """ Action method. """ if _Debug: lg.out(_DebugLevel, 'id_rotator.doRequestMyIdentity') def _cb(results): if _Debug: lg.args(_DebugLevel, results=results) # TODO: validate my identity in all responses self.automat('my-id-exist', results) return results def _eb(err): lg.err(err) self.automat('my-id-not-exist', err) return None dl = [] for idurl in my_id.getLocalIdentity().getSources(as_originals=True): dl.append(net_misc.getPageTwisted(idurl, timeout=10)) d = DeferredList(dl) d.addCallback(_cb) d.addErrback(_eb)
def _next_source(resp, idurl, sources, pos): global _LastTimeCached if _Debug: lg.args(_DebugLevel, resp=resp, idurl=idurl, pos=pos, sources=len(sources)) if pos >= len(sources): lg.warn('[cache failed] %r and also %d other sources' % (idurl, len(sources))) defer_results = caching().pop(idurl, []) for result in defer_results: if result and not result.called: reactor.callLater( 0, result.errback, Exception('cache failed from multiple sources' )) # @UndefinedVariable p2p_stats.count_identity_cache(idurl, 0) _LastTimeCached.pop(idurl, None) defer_results = [] del defer_results return None next_idurl = sources[pos] next_idurl = id_url.to_original(next_idurl) if _Debug: lg.args(_DebugLevel, next_idurl=next_idurl) d = net_misc.getPageTwisted(url=next_idurl, timeout=timeout) d.addCallback(_success, idurl) d.addErrback(_next_source, idurl, sources, pos + 1) return None
def doPingServers(self, arg): """ Action method. """ lg.out(4, 'id_registrator.doPingServers %d in list' % len(self.discovered_servers)) def _cb(htmlsrc, id_server_host): lg.out(4, ' RESPONDED: %s' % id_server_host) if self.preferred_server and id_server_host == self.preferred_server: self.good_servers.insert(0, id_server_host) else: self.good_servers.append(id_server_host) self.discovered_servers.remove(id_server_host) self.automat('id-server-response', (id_server_host, htmlsrc)) def _eb(err, id_server_host): lg.out(4, ' FAILED: %s' % id_server_host) self.discovered_servers.remove(id_server_host) self.automat('id-server-failed', (id_server_host, err)) for host in self.discovered_servers: webport, tcpport = known_servers.by_host().get(host, (settings.IdentityWebPort(), settings.IdentityServerPort())) if webport == 80: webport = '' server_url = nameurl.UrlMake('http', host, webport, '') d = net_misc.getPageTwisted(server_url, timeout=10) d.addCallback(_cb, host) d.addErrback(_eb, host)
def pageRequestTwisted(idurl, timeout=0): """ Request an HTML page - this can be an user identity. """ d = net_misc.getPageTwisted(idurl, timeout) d.addCallback(getPageSuccess, idurl) d.addErrback(getPageFail, idurl) return d
def _start_one(idurl, ignore_errors): if _Debug: lg.args(_DebugLevel, idurl=idurl) defer_obj = add_callback(idurl, ignore_errors=ignore_errors) d = net_misc.getPageTwisted(idurl, timeout) d.addCallback(_success, idurl) d.addErrback(_fail, idurl) return defer_obj
def _do_verify_my_sources(self): my_sources = my_id.getLocalIdentity().getSources(as_originals=True) dl = [] for idurl_bin in my_sources: d = net_misc.getPageTwisted(idurl_bin, timeout=5) dl.append(d) d = DeferredList(dl, consumeErrors=True) d.addCallback(self._do_check_ping_results) d.addErrback(lambda err: self.automat('ping-failed', err))
def _server_replied(htmlsrc, host, pos): if _Debug: lg.out(_DebugLevel, 'id_rotator.doSelectNewIDServer._server_replied %r' % host) webport, _ = target_servers[host] if webport == 80: webport = '' new_idurl = nameurl.UrlMake('http', strng.to_text(host), webport, my_id.getIDName() + '.xml') d = net_misc.getPageTwisted(new_idurl, timeout=10) d.addCallback(_new_idurl_exist, new_idurl, pos) d.addErrback(_new_idurl_not_exist, new_idurl)
def _ping_one_server(pos): host = target_hosts[pos] webport, tcpport = target_servers[host] if webport == 80: webport = '' tcpport = int(tcpport) server_url = nameurl.UrlMake('http', strng.to_text(host), webport, '') if _Debug: lg.out(_DebugLevel, 'id_rotator.doSelectNewIDServer._ping_one_server at %s known tcp port is %d' % ( server_url, tcpport, )) d = net_misc.getPageTwisted(server_url, timeout=10) d.addCallback(_server_replied, host, pos) d.addErrback(_server_failed, host, pos)
def doRequestServers(self, *args, **kwargs): """ Action method. """ login = bpio.ReadTextFile(settings.UserNameFilename()) def _cb(xmlsrc, idurl, host): if not xmlsrc: if self.preferred_servers and host in self.preferred_servers: if self.preferred_servers[0] == host: self.free_idurls.insert(0, idurl) else: self.free_idurls.insert(1, idurl) else: self.free_idurls.append(idurl) self.registrations.remove(idurl) self.automat('id-not-exist', idurl) else: lg.out(4, ' EXIST: %s' % idurl) self.registrations.remove(idurl) self.automat('id-exist', idurl) def _eb(err, idurl, host): lg.out(4, ' NOT EXIST: %s' % idurl) if self.preferred_servers and host in self.preferred_servers: if self.preferred_servers[0] == host: self.free_idurls.insert(0, idurl) else: self.free_idurls.insert(1, idurl) else: self.free_idurls.append(idurl) self.registrations.remove(idurl) self.automat('id-not-exist', idurl) for host in self.good_servers: webport, tcpport = known_servers.by_host().get( host, (settings.IdentityWebPort(), settings.IdentityServerPort())) if webport == 80: webport = '' idurl = nameurl.UrlMake('http', strng.to_text(host), webport, login + '.xml') lg.out(4, ' %s' % idurl) d = net_misc.getPageTwisted(idurl, timeout=7) d.addCallback(_cb, idurl, host) d.addErrback(_eb, idurl, host) self.registrations.append(idurl) lg.out( 4, 'id_registrator.doRequestServers login=%s registrations=%d' % (login, len(self.registrations)))
def doRequestMyIdentity(self, arg): """ Action method. """ lg.out(8, 'id_registrator.doRequestMyIdentity') def _cb(src): self.automat('my-id-exist', src) def _eb(err): self.automat('my-id-not-exist', err) for idurl in self.new_identity.sources: lg.out(8, ' %s' % idurl) d = net_misc.getPageTwisted(idurl, timeout=20) d.addCallback(_cb) d.addErrback(_eb)
def immediatelyCaching(idurl, timeout=10): """ A smart method to cache some identity and get results in callbacks. """ global _CachingTasks if idurl in _CachingTasks: return _CachingTasks[idurl] def _getPageSuccess(src, idurl): global _CachingTasks result = _CachingTasks.pop(idurl, None) if not result: lg.warn('caching task for %s was not found' % idurl) return src if UpdateAfterChecking(idurl, src): if result: result.callback(src) if _Debug: lg.out(14, '[cached] %s' % idurl) p2p_stats.count_identity_cache(idurl, len(src)) else: if result: result.errback(Exception(src)) if _Debug: lg.warn('[cache error] %s is not valid' % idurl) p2p_stats.count_identity_cache(idurl, 0) return src def _getPageFail(x, idurl): global _CachingTasks result = _CachingTasks.pop(idurl) if result: result.errback(x) else: lg.warn('caching task for %s was not found' % idurl) p2p_stats.count_identity_cache(idurl, 0) if _Debug: lg.warn('[cache failed] %s : %s' % (idurl, x.getErrorMessage(), )) return None _CachingTasks[idurl] = Deferred() d = net_misc.getPageTwisted(idurl, timeout) d.addCallback(_getPageSuccess, idurl) d.addErrback(_getPageFail, idurl) if _Debug: lg.out(14, 'identitycache.immediatelyCaching %s' % idurl) return _CachingTasks[idurl]
def main(): def _ok(x): print 'ok', x reactor.stop() def _fail(x): print 'fail', x reactor.stop() from lib import net_misc from main import settings settings.init() settings.update_proxy_settings() idurl = 'http://p2p-id.ru/atg314.xml' r = net_misc.getPageTwisted(idurl) r.addCallback(_ok) r.addErrback(_fail) reactor.run()
def doRequestMyIdentity(self, *args, **kwargs): """ Action method. """ lg.out(8, 'id_registrator.doRequestMyIdentity') def _cb(src): self.automat('my-id-exist', src) def _eb(err): self.automat('my-id-not-exist', err) for idurl in self.new_identity.sources: lg.out(8, ' %s' % idurl) d = net_misc.getPageTwisted(idurl, timeout=20) d.addCallback(_cb) d.addErrback(_eb)
def _fallback_and_ping_my_servers(self): """ Just ping all my id servers by sending a HTTP request to the "main page". """ my_sources = my_id.getLocalIdentity().getSources(as_originals=True) id_servers = [] for idurl_bin in my_sources: proto, host, port, _ = nameurl.UrlParse(idurl_bin) id_servers.append(nameurl.UrlMake(proto, host, port, '')) if _Debug: lg.args(_DebugLevel, id_servers=id_servers) dl = [] for url in id_servers: d = net_misc.getPageTwisted(url, timeout=5) dl.append(d) d = DeferredList(dl, consumeErrors=True) d.addCallback(lambda result: self.automat('ping-done', [])) d.addErrback(self._fallback_check_network_connected)
def doRequestMyIdentity(self, *args, **kwargs): """ Action method. """ lg.out(8, 'id_registrator.doRequestMyIdentity') def _cb(src): # TODO: validate my identity and make sure other servers also stored self.automat('my-id-exist', src) def _eb(err): # TODO: this is actually can be an issue if one server failed but others are fine... self.automat('my-id-not-exist', err) for idurl in self.new_identity.getSources(as_originals=True): lg.out(8, ' %s' % idurl) d = net_misc.getPageTwisted(idurl, timeout=7) d.addCallback(_cb) d.addErrback(_eb)
def main(): def _ok(x): print "ok", x reactor.stop() def _fail(x): print "fail", x reactor.stop() from lib import net_misc from main import settings settings.init() settings.update_proxy_settings() idurl = "http://p2p-id.ru/atg314.xml" r = net_misc.getPageTwisted(idurl) r.addCallback(_ok) r.addErrback(_fail) reactor.run()
def main(): def _ok(x): print('ok', x) reactor.stop() # @UndefinedVariable def _fail(x): print('fail', x) reactor.stop() # @UndefinedVariable from lib import net_misc from main import settings settings.init() settings.update_proxy_settings() url = 'http://localhost:8084' r = net_misc.getPageTwisted(url) r.addCallback(_ok) r.addErrback(_fail) reactor.run() # @UndefinedVariable settings.shutdown()
def immediatelyCaching(idurl, timeout=0): """ A smart method to start caching some identity and get results in callbacks. """ global _CachingTasks if idurl in _CachingTasks: return _CachingTasks[idurl] def _getPageSuccess(src, idurl, res): global _CachingTasks _CachingTasks.pop(idurl) if UpdateAfterChecking(idurl, src): res.callback(src) if _Debug: lg.out(14, ' [cached] %s' % idurl) else: res.errback(Exception(src)) if _Debug: lg.out(14, ' [cache error] %s' % idurl) return src def _getPageFail(x, idurl, res): global _CachingTasks _CachingTasks.pop(idurl) res.errback(x) if _Debug: lg.out(14, ' [cache failed] %s' % idurl) return None result = Deferred() d = net_misc.getPageTwisted(idurl, timeout) d.addCallback(_getPageSuccess, idurl, result) d.addErrback(_getPageFail, idurl, result) _CachingTasks[idurl] = result if _Debug: lg.out(14, 'identitycache.immediatelyCaching %s' % idurl) return result
def download_version(): repo, locationURL = misc.ReadRepoLocation() url = locationURL + settings.CurrentVersionDigestsFilename() lg.out(6, 'os_windows_update.download_version ' + str(url)) return net_misc.getPageTwisted(url)
def immediatelyCaching(idurl, timeout=10, try_other_sources=True): """ A smart method to cache some identity and get results in callbacks. """ global _CachingTasks global _LastTimeCached idurl = id_url.to_original(idurl) if not idurl: raise Exception('can not cache, idurl is empty') if idurl in _CachingTasks and not _CachingTasks[idurl].called: if _Debug: lg.out( _DebugLevel, 'identitycache.immediatelyCaching already have a task for %r' % idurl) return _CachingTasks[idurl] if _Debug: lg.out( _DebugLevel, 'identitycache.immediatelyCaching started new task for %r' % idurl) def _success(src, idurl): global _CachingTasks global _LastTimeCached idurl = id_url.to_original(idurl) result = _CachingTasks.pop(idurl, None) if not result: lg.warn('caching task for %s was not found' % idurl) if UpdateAfterChecking(idurl, src): if result: result.callback(src) lg.out(_DebugLevel, '[cached] %s' % idurl) p2p_stats.count_identity_cache(idurl, len(src)) _LastTimeCached[idurl] = time.time() else: if result: result.errback(Exception(src)) lg.warn('[cache error] %s is not valid' % idurl) p2p_stats.count_identity_cache(idurl, 0) return src def _next_source(err, sources, pos, ret): if pos >= len(sources): lg.warn('[cache failed] %r from %d sources' % (idurl, len(sources))) if ret: ret.errback( Exception('cache failed from %d sources' % len(sources))) return None next_idurl = sources[pos] next_idurl = id_url.to_original(next_idurl) if _Debug: lg.out( _DebugLevel, 'identitycache.immediatelyCaching._next_source %r from %r : %r' % ( pos, sources, next_idurl, )) if next_idurl in _CachingTasks: if _Debug: lg.out( _DebugLevel, 'identitycache.immediatelyCaching already have next task for %r' % next_idurl) d = _CachingTasks[next_idurl] else: if _Debug: lg.out( _DebugLevel, 'identitycache.immediatelyCaching will try another source of %r : %r' % (idurl, next_idurl)) _CachingTasks[next_idurl] = Deferred() _CachingTasks[next_idurl].addErrback(on_caching_task_failed, next_idurl) d = net_misc.getPageTwisted(next_idurl, timeout) d.addCallback(_success, next_idurl) if ret: d.addCallback(ret.callback) d.addErrback(_next_source, sources, pos + 1, ret) return None def _fail(err, idurl): global _CachingTasks idurl = id_url.to_original(idurl) result = _CachingTasks.pop(idurl) if not try_other_sources: if result: result.errback(err) else: lg.warn('caching task for %s was not found' % idurl) p2p_stats.count_identity_cache(idurl, 0) lg.warn('[cache failed] %s : %s' % ( idurl, err.getErrorMessage(), )) return None latest_idurl, latest_rev = id_url.get_latest_revision(idurl) latest_ident = None sources = [] if latest_idurl: latest_ident = identitydb.get_ident(latest_idurl) if latest_ident: sources = latest_ident.getSources(as_fields=False) if sources: if idurl in sources: sources = sources.remove(idurl) if sources: lg.warn('[cache failed] %s : %s but will try %d more sources' % ( idurl, err.getErrorMessage(), len(sources), )) _next_source(err, sources, 0, result) return result if result: result.errback(err) else: lg.warn('caching task for %s was not found' % idurl) p2p_stats.count_identity_cache(idurl, 0) lg.warn('[cache failed] and also no other sources found %s : %s' % ( idurl, err.getErrorMessage(), )) return None idurl = id_url.to_original(idurl) _CachingTasks[idurl] = Deferred() _CachingTasks[idurl].addErrback(on_caching_task_failed, idurl) d = net_misc.getPageTwisted(idurl, timeout) d.addCallback(_success, idurl) d.addErrback(_fail, idurl) return _CachingTasks[idurl]
def http_stun(result_defer=None): from userid import known_servers identity_servers = known_servers.by_host() if not identity_servers: if _Debug: lg.out( _DebugLevel, 'stun_client.http_stun SKIP, no known identity servers found' ) return False one_host = random.choice(list(identity_servers.keys())) one_port = identity_servers[one_host][0] one_url = nameurl.UrlMake('http', one_host, one_port) if _Debug: lg.out(_DebugLevel, 'stun_client.http_stun GO with one node : %r' % one_url) def _check_body(html_response): ret = { 'result': 'stun-failed', 'type': None, 'ip': None, 'details': [ 'unknown client host from response', ], } mo = re.search(b'\<\!\-\-CLIENT_HOST\=([\d\.]+):(\d+)\-\-\>', html_response) if not mo: if _Debug: lg.out( _DebugLevel, 'stun_client.http_stun FAILED : unknown client host from response' ) if result_defer: result_defer.callback(ret) return None ret = { 'result': 'stun-success', 'type': 'unknown', 'ip': strng.to_text(mo.group(1)), 'details': [], } if _Debug: lg.out(_DebugLevel, 'stun_client.http_stun SUCCESS : %r' % ret) if result_defer: result_defer.callback(ret) return None def _request_failed(err): if _Debug: lg.out(_DebugLevel, 'stun_client.http_stun FAILED : %r' % err) ret = { 'result': 'stun-failed', 'type': None, 'ip': None, 'details': [ err.getErrorMessage(), ], } if result_defer: result_defer.callback(ret) return None d = net_misc.getPageTwisted(one_url) d.addCallback(_check_body) d.addErrback(_request_failed) return True