def doRequestMyIdentity(self, arg):
global _WorkingIDURL
idurl = _WorkingIDURL
lg.out(4, 'identity_restorer.doRequestMyIdentity %s %s' % (idurl, type(idurl)))
net_misc.getPageTwisted(idurl).addCallbacks(
lambda src: self.automat('my-id-received', src),
lambda err: self.automat('my-id-failed', err))
def doRequestMyIdentity(self, arg):
global _WorkingIDURL
idurl = _WorkingIDURL
lg.out(4, 'identity_restorer.doRequestMyIdentity %s %s' % (idurl, type(idurl)))
net_misc.getPageTwisted(idurl).addCallbacks(
lambda src: self.automat('my-id-received', src),
lambda err: self.automat('my-id-failed', err))
def doPingServers(self, arg):
"""
Action method.
"""
lg.out(4, 'id_registrator.doPingServers %d in list' % len(self.discovered_servers))
def _cb(htmlsrc, id_server_host):
lg.out(4, ' RESPONDED: %s' % id_server_host)
if self.preferred_servers and id_server_host in self.preferred_servers:
self.good_servers.insert(0, id_server_host)
else:
self.good_servers.append(id_server_host)
self.discovered_servers.remove(id_server_host)
self.automat('id-server-response', (id_server_host, htmlsrc))
def _eb(err, id_server_host):
lg.out(4, ' FAILED: %s' % id_server_host)
self.discovered_servers.remove(id_server_host)
self.automat('id-server-failed', (id_server_host, err))
for host in self.discovered_servers:
webport, tcpport = known_servers.by_host().get(host,
(settings.IdentityWebPort(), settings.IdentityServerPort()))
if webport == 80:
webport = ''
server_url = nameurl.UrlMake('http', host, webport, '')
d = net_misc.getPageTwisted(server_url, timeout=10)
d.addCallback(_cb, host)
d.addErrback(_eb, host)
def download_info():
def _done(src, result):
lg.out(6, 'os_windows_update.download_info.done ')
lines = src.split('\n')
files_dict = {}
for line in lines:
words = line.split(' ')
if len(words) < 2:
continue
files_dict[words[1].strip()] = words[0].strip()
result.callback(files_dict)
return src
def _fail(x, result):
lg.out(1, 'os_windows_update.download_info FAILED')
result.errback(Exception('error downloading info'))
return x
repo, locationURL = misc.ReadRepoLocation()
url = locationURL + settings.FilesDigestsFilename()
lg.out(6, 'os_windows_update.download_info ' + str(url))
result = Deferred()
d = net_misc.getPageTwisted(url)
d.addCallback(_done, result)
d.addErrback(_fail, result)
return result
def doRequestServers(self, arg):
"""
Action method.
"""
login = bpio.ReadTextFile(settings.UserNameFilename())
def _cb(xmlsrc, idurl, host):
lg.out(4, ' EXIST: %s' % idurl)
self.registrations.remove(idurl)
self.automat('id-exist', idurl)
def _eb(err, idurl, host):
lg.out(4, ' NOT EXIST: %s' % idurl)
if self.preferred_server and self.preferred_server == host:
self.free_idurls.insert(0, idurl)
else:
self.free_idurls.append(idurl)
self.registrations.remove(idurl)
self.automat('id-not-exist', idurl)
for host in self.good_servers:
webport, tcpport = known_servers.by_host().get(
host, (settings.IdentityWebPort(), settings.IdentityServerPort()))
if webport == 80:
webport = ''
idurl = nameurl.UrlMake('http', host, webport, login + '.xml')
lg.out(4, ' %s' % idurl)
d = net_misc.getPageTwisted(idurl, timeout=10)
d.addCallback(_cb, idurl, host)
d.addErrback(_eb, idurl, host)
self.registrations.append(idurl)
lg.out(4, 'id_registrator.doRequestServers login=%s registrations=%d' % (login, len(self.registrations)))
def _next_source(err, sources, pos, ret):
if pos >= len(sources):
lg.warn('[cache failed] %r from %d sources' % (idurl, len(sources)))
if ret:
ret.errback(Exception('cache failed from %d sources' % len(sources)))
return None
next_idurl = sources[pos]
next_idurl = id_url.to_original(next_idurl)
if _Debug:
lg.out(_DebugLevel, 'identitycache.immediatelyCaching._next_source %r from %r : %r' % (pos, sources, next_idurl, ))
if next_idurl in _CachingTasks:
if _Debug:
lg.out(_DebugLevel, 'identitycache.immediatelyCaching already have next task for %r' % next_idurl)
d = _CachingTasks[next_idurl]
else:
if _Debug:
lg.out(_DebugLevel, 'identitycache.immediatelyCaching will try another source of %r : %r' % (idurl, next_idurl))
_CachingTasks[next_idurl] = Deferred()
_CachingTasks[next_idurl].addErrback(on_caching_task_failed, next_idurl)
d = net_misc.getPageTwisted(next_idurl, timeout)
d.addCallback(_success, next_idurl)
if ret:
d.addCallback(ret.callback)
d.addErrback(_next_source, sources, pos+1, ret)
return None
def doRequestMyIdentity(self, *args, **kwargs):
"""
Action method.
"""
if _Debug:
lg.out(_DebugLevel, 'id_rotator.doRequestMyIdentity')
def _cb(results):
if _Debug:
lg.args(_DebugLevel, results=results)
# TODO: validate my identity in all responses
self.automat('my-id-exist', results)
return results
def _eb(err):
lg.err(err)
self.automat('my-id-not-exist', err)
return None
dl = []
for idurl in my_id.getLocalIdentity().getSources(as_originals=True):
dl.append(net_misc.getPageTwisted(idurl, timeout=10))
d = DeferredList(dl)
d.addCallback(_cb)
d.addErrback(_eb)
def _next_source(resp, idurl, sources, pos):
global _LastTimeCached
if _Debug:
lg.args(_DebugLevel,
resp=resp,
idurl=idurl,
pos=pos,
sources=len(sources))
if pos >= len(sources):
lg.warn('[cache failed] %r and also %d other sources' %
(idurl, len(sources)))
defer_results = caching().pop(idurl, [])
for result in defer_results:
if result and not result.called:
reactor.callLater(
0, result.errback,
Exception('cache failed from multiple sources'
)) # @UndefinedVariable
p2p_stats.count_identity_cache(idurl, 0)
_LastTimeCached.pop(idurl, None)
defer_results = []
del defer_results
return None
next_idurl = sources[pos]
next_idurl = id_url.to_original(next_idurl)
if _Debug:
lg.args(_DebugLevel, next_idurl=next_idurl)
d = net_misc.getPageTwisted(url=next_idurl, timeout=timeout)
d.addCallback(_success, idurl)
d.addErrback(_next_source, idurl, sources, pos + 1)
return None
def download_info():
def _done(src, result):
lg.out(6, 'os_windows_update.download_info.done ')
lines = src.split('\n')
files_dict = {}
for line in lines:
words = line.split(' ')
if len(words) < 2:
continue
files_dict[words[1].strip()] = words[0].strip()
result.callback(files_dict)
return src
def _fail(x, result):
lg.out(1, 'os_windows_update.download_info FAILED')
result.errback(Exception('error downloading info'))
return x
repo, locationURL = misc.ReadRepoLocation()
url = locationURL + settings.FilesDigestsFilename()
lg.out(6, 'os_windows_update.download_info ' + str(url))
result = Deferred()
d = net_misc.getPageTwisted(url)
d.addCallback(_done, result)
d.addErrback(_fail, result)
return result
def doPingServers(self, arg):
"""
Action method.
"""
lg.out(4, 'id_registrator.doPingServers %d in list' % len(self.discovered_servers))
def _cb(htmlsrc, id_server_host):
lg.out(4, ' RESPONDED: %s' % id_server_host)
if self.preferred_server and id_server_host == self.preferred_server:
self.good_servers.insert(0, id_server_host)
else:
self.good_servers.append(id_server_host)
self.discovered_servers.remove(id_server_host)
self.automat('id-server-response', (id_server_host, htmlsrc))
def _eb(err, id_server_host):
lg.out(4, ' FAILED: %s' % id_server_host)
self.discovered_servers.remove(id_server_host)
self.automat('id-server-failed', (id_server_host, err))
for host in self.discovered_servers:
webport, tcpport = known_servers.by_host().get(host,
(settings.IdentityWebPort(), settings.IdentityServerPort()))
if webport == 80:
webport = ''
server_url = nameurl.UrlMake('http', host, webport, '')
d = net_misc.getPageTwisted(server_url, timeout=10)
d.addCallback(_cb, host)
d.addErrback(_eb, host)
def pageRequestTwisted(idurl, timeout=0):
"""
Request an HTML page - this can be an user identity.
"""
d = net_misc.getPageTwisted(idurl, timeout)
d.addCallback(getPageSuccess, idurl)
d.addErrback(getPageFail, idurl)
return d
def _start_one(idurl, ignore_errors):
if _Debug:
lg.args(_DebugLevel, idurl=idurl)
defer_obj = add_callback(idurl, ignore_errors=ignore_errors)
d = net_misc.getPageTwisted(idurl, timeout)
d.addCallback(_success, idurl)
d.addErrback(_fail, idurl)
return defer_obj
def pageRequestTwisted(idurl, timeout=0):
"""
Request an HTML page - this can be an user identity.
"""
d = net_misc.getPageTwisted(idurl, timeout)
d.addCallback(getPageSuccess, idurl)
d.addErrback(getPageFail, idurl)
return d
def _do_verify_my_sources(self):
my_sources = my_id.getLocalIdentity().getSources(as_originals=True)
dl = []
for idurl_bin in my_sources:
d = net_misc.getPageTwisted(idurl_bin, timeout=5)
dl.append(d)
d = DeferredList(dl, consumeErrors=True)
d.addCallback(self._do_check_ping_results)
d.addErrback(lambda err: self.automat('ping-failed', err))
def _server_replied(htmlsrc, host, pos):
if _Debug:
lg.out(_DebugLevel, 'id_rotator.doSelectNewIDServer._server_replied %r' % host)
webport, _ = target_servers[host]
if webport == 80:
webport = ''
new_idurl = nameurl.UrlMake('http', strng.to_text(host), webport, my_id.getIDName() + '.xml')
d = net_misc.getPageTwisted(new_idurl, timeout=10)
d.addCallback(_new_idurl_exist, new_idurl, pos)
d.addErrback(_new_idurl_not_exist, new_idurl)
def _ping_one_server(pos):
host = target_hosts[pos]
webport, tcpport = target_servers[host]
if webport == 80:
webport = ''
tcpport = int(tcpport)
server_url = nameurl.UrlMake('http', strng.to_text(host), webport, '')
if _Debug:
lg.out(_DebugLevel, 'id_rotator.doSelectNewIDServer._ping_one_server at %s known tcp port is %d' % (
server_url, tcpport, ))
d = net_misc.getPageTwisted(server_url, timeout=10)
d.addCallback(_server_replied, host, pos)
d.addErrback(_server_failed, host, pos)
def doRequestServers(self, *args, **kwargs):
"""
Action method.
"""
login = bpio.ReadTextFile(settings.UserNameFilename())
def _cb(xmlsrc, idurl, host):
if not xmlsrc:
if self.preferred_servers and host in self.preferred_servers:
if self.preferred_servers[0] == host:
self.free_idurls.insert(0, idurl)
else:
self.free_idurls.insert(1, idurl)
else:
self.free_idurls.append(idurl)
self.registrations.remove(idurl)
self.automat('id-not-exist', idurl)
else:
lg.out(4, ' EXIST: %s' % idurl)
self.registrations.remove(idurl)
self.automat('id-exist', idurl)
def _eb(err, idurl, host):
lg.out(4, ' NOT EXIST: %s' % idurl)
if self.preferred_servers and host in self.preferred_servers:
if self.preferred_servers[0] == host:
self.free_idurls.insert(0, idurl)
else:
self.free_idurls.insert(1, idurl)
else:
self.free_idurls.append(idurl)
self.registrations.remove(idurl)
self.automat('id-not-exist', idurl)
for host in self.good_servers:
webport, tcpport = known_servers.by_host().get(
host,
(settings.IdentityWebPort(), settings.IdentityServerPort()))
if webport == 80:
webport = ''
idurl = nameurl.UrlMake('http', strng.to_text(host), webport,
login + '.xml')
lg.out(4, ' %s' % idurl)
d = net_misc.getPageTwisted(idurl, timeout=7)
d.addCallback(_cb, idurl, host)
d.addErrback(_eb, idurl, host)
self.registrations.append(idurl)
lg.out(
4, 'id_registrator.doRequestServers login=%s registrations=%d' %
(login, len(self.registrations)))
def doRequestMyIdentity(self, arg):
"""
Action method.
"""
lg.out(8, 'id_registrator.doRequestMyIdentity')
def _cb(src):
self.automat('my-id-exist', src)
def _eb(err):
self.automat('my-id-not-exist', err)
for idurl in self.new_identity.sources:
lg.out(8, ' %s' % idurl)
d = net_misc.getPageTwisted(idurl, timeout=20)
d.addCallback(_cb)
d.addErrback(_eb)
def immediatelyCaching(idurl, timeout=10):
"""
A smart method to cache some identity and get results in callbacks.
"""
global _CachingTasks
if idurl in _CachingTasks:
return _CachingTasks[idurl]
def _getPageSuccess(src, idurl):
global _CachingTasks
result = _CachingTasks.pop(idurl, None)
if not result:
lg.warn('caching task for %s was not found' % idurl)
return src
if UpdateAfterChecking(idurl, src):
if result:
result.callback(src)
if _Debug:
lg.out(14, '[cached] %s' % idurl)
p2p_stats.count_identity_cache(idurl, len(src))
else:
if result:
result.errback(Exception(src))
if _Debug:
lg.warn('[cache error] %s is not valid' % idurl)
p2p_stats.count_identity_cache(idurl, 0)
return src
def _getPageFail(x, idurl):
global _CachingTasks
result = _CachingTasks.pop(idurl)
if result:
result.errback(x)
else:
lg.warn('caching task for %s was not found' % idurl)
p2p_stats.count_identity_cache(idurl, 0)
if _Debug:
lg.warn('[cache failed] %s : %s' % (idurl, x.getErrorMessage(), ))
return None
_CachingTasks[idurl] = Deferred()
d = net_misc.getPageTwisted(idurl, timeout)
d.addCallback(_getPageSuccess, idurl)
d.addErrback(_getPageFail, idurl)
if _Debug:
lg.out(14, 'identitycache.immediatelyCaching %s' % idurl)
return _CachingTasks[idurl]
def main():
def _ok(x):
print 'ok', x
reactor.stop()
def _fail(x):
print 'fail', x
reactor.stop()
from lib import net_misc
from main import settings
settings.init()
settings.update_proxy_settings()
idurl = 'http://p2p-id.ru/atg314.xml'
r = net_misc.getPageTwisted(idurl)
r.addCallback(_ok)
r.addErrback(_fail)
reactor.run()
def doRequestMyIdentity(self, *args, **kwargs):
"""
Action method.
"""
lg.out(8, 'id_registrator.doRequestMyIdentity')
def _cb(src):
self.automat('my-id-exist', src)
def _eb(err):
self.automat('my-id-not-exist', err)
for idurl in self.new_identity.sources:
lg.out(8, ' %s' % idurl)
d = net_misc.getPageTwisted(idurl, timeout=20)
d.addCallback(_cb)
d.addErrback(_eb)
def _fallback_and_ping_my_servers(self):
"""
Just ping all my id servers by sending a HTTP request to the "main page".
"""
my_sources = my_id.getLocalIdentity().getSources(as_originals=True)
id_servers = []
for idurl_bin in my_sources:
proto, host, port, _ = nameurl.UrlParse(idurl_bin)
id_servers.append(nameurl.UrlMake(proto, host, port, ''))
if _Debug:
lg.args(_DebugLevel, id_servers=id_servers)
dl = []
for url in id_servers:
d = net_misc.getPageTwisted(url, timeout=5)
dl.append(d)
d = DeferredList(dl, consumeErrors=True)
d.addCallback(lambda result: self.automat('ping-done', []))
d.addErrback(self._fallback_check_network_connected)
def doRequestMyIdentity(self, *args, **kwargs):
"""
Action method.
"""
lg.out(8, 'id_registrator.doRequestMyIdentity')
def _cb(src):
# TODO: validate my identity and make sure other servers also stored
self.automat('my-id-exist', src)
def _eb(err):
# TODO: this is actually can be an issue if one server failed but others are fine...
self.automat('my-id-not-exist', err)
for idurl in self.new_identity.getSources(as_originals=True):
lg.out(8, ' %s' % idurl)
d = net_misc.getPageTwisted(idurl, timeout=7)
d.addCallback(_cb)
d.addErrback(_eb)
def main():
def _ok(x):
print "ok", x
reactor.stop()
def _fail(x):
print "fail", x
reactor.stop()
from lib import net_misc
from main import settings
settings.init()
settings.update_proxy_settings()
idurl = "http://p2p-id.ru/atg314.xml"
r = net_misc.getPageTwisted(idurl)
r.addCallback(_ok)
r.addErrback(_fail)
reactor.run()
def main():
def _ok(x):
print('ok', x)
reactor.stop() # @UndefinedVariable
def _fail(x):
print('fail', x)
reactor.stop() # @UndefinedVariable
from lib import net_misc
from main import settings
settings.init()
settings.update_proxy_settings()
url = 'http://localhost:8084'
r = net_misc.getPageTwisted(url)
r.addCallback(_ok)
r.addErrback(_fail)
reactor.run() # @UndefinedVariable
settings.shutdown()
def immediatelyCaching(idurl, timeout=0):
"""
A smart method to start caching some identity and get results in callbacks.
"""
global _CachingTasks
if idurl in _CachingTasks:
return _CachingTasks[idurl]
def _getPageSuccess(src, idurl, res):
global _CachingTasks
_CachingTasks.pop(idurl)
if UpdateAfterChecking(idurl, src):
res.callback(src)
if _Debug:
lg.out(14, ' [cached] %s' % idurl)
else:
res.errback(Exception(src))
if _Debug:
lg.out(14, ' [cache error] %s' % idurl)
return src
def _getPageFail(x, idurl, res):
global _CachingTasks
_CachingTasks.pop(idurl)
res.errback(x)
if _Debug:
lg.out(14, ' [cache failed] %s' % idurl)
return None
result = Deferred()
d = net_misc.getPageTwisted(idurl, timeout)
d.addCallback(_getPageSuccess, idurl, result)
d.addErrback(_getPageFail, idurl, result)
_CachingTasks[idurl] = result
if _Debug:
lg.out(14, 'identitycache.immediatelyCaching %s' % idurl)
return result
def download_version():
repo, locationURL = misc.ReadRepoLocation()
url = locationURL + settings.CurrentVersionDigestsFilename()
lg.out(6, 'os_windows_update.download_version ' + str(url))
return net_misc.getPageTwisted(url)
def download_version():
repo, locationURL = misc.ReadRepoLocation()
url = locationURL + settings.CurrentVersionDigestsFilename()
lg.out(6, 'os_windows_update.download_version ' + str(url))
return net_misc.getPageTwisted(url)
def immediatelyCaching(idurl, timeout=10, try_other_sources=True):
"""
A smart method to cache some identity and get results in callbacks.
"""
global _CachingTasks
global _LastTimeCached
idurl = id_url.to_original(idurl)
if not idurl:
raise Exception('can not cache, idurl is empty')
if idurl in _CachingTasks and not _CachingTasks[idurl].called:
if _Debug:
lg.out(
_DebugLevel,
'identitycache.immediatelyCaching already have a task for %r' %
idurl)
return _CachingTasks[idurl]
if _Debug:
lg.out(
_DebugLevel,
'identitycache.immediatelyCaching started new task for %r' % idurl)
def _success(src, idurl):
global _CachingTasks
global _LastTimeCached
idurl = id_url.to_original(idurl)
result = _CachingTasks.pop(idurl, None)
if not result:
lg.warn('caching task for %s was not found' % idurl)
if UpdateAfterChecking(idurl, src):
if result:
result.callback(src)
lg.out(_DebugLevel, '[cached] %s' % idurl)
p2p_stats.count_identity_cache(idurl, len(src))
_LastTimeCached[idurl] = time.time()
else:
if result:
result.errback(Exception(src))
lg.warn('[cache error] %s is not valid' % idurl)
p2p_stats.count_identity_cache(idurl, 0)
return src
def _next_source(err, sources, pos, ret):
if pos >= len(sources):
lg.warn('[cache failed] %r from %d sources' %
(idurl, len(sources)))
if ret:
ret.errback(
Exception('cache failed from %d sources' % len(sources)))
return None
next_idurl = sources[pos]
next_idurl = id_url.to_original(next_idurl)
if _Debug:
lg.out(
_DebugLevel,
'identitycache.immediatelyCaching._next_source %r from %r : %r'
% (
pos,
sources,
next_idurl,
))
if next_idurl in _CachingTasks:
if _Debug:
lg.out(
_DebugLevel,
'identitycache.immediatelyCaching already have next task for %r'
% next_idurl)
d = _CachingTasks[next_idurl]
else:
if _Debug:
lg.out(
_DebugLevel,
'identitycache.immediatelyCaching will try another source of %r : %r'
% (idurl, next_idurl))
_CachingTasks[next_idurl] = Deferred()
_CachingTasks[next_idurl].addErrback(on_caching_task_failed,
next_idurl)
d = net_misc.getPageTwisted(next_idurl, timeout)
d.addCallback(_success, next_idurl)
if ret:
d.addCallback(ret.callback)
d.addErrback(_next_source, sources, pos + 1, ret)
return None
def _fail(err, idurl):
global _CachingTasks
idurl = id_url.to_original(idurl)
result = _CachingTasks.pop(idurl)
if not try_other_sources:
if result:
result.errback(err)
else:
lg.warn('caching task for %s was not found' % idurl)
p2p_stats.count_identity_cache(idurl, 0)
lg.warn('[cache failed] %s : %s' % (
idurl,
err.getErrorMessage(),
))
return None
latest_idurl, latest_rev = id_url.get_latest_revision(idurl)
latest_ident = None
sources = []
if latest_idurl:
latest_ident = identitydb.get_ident(latest_idurl)
if latest_ident:
sources = latest_ident.getSources(as_fields=False)
if sources:
if idurl in sources:
sources = sources.remove(idurl)
if sources:
lg.warn('[cache failed] %s : %s but will try %d more sources' % (
idurl,
err.getErrorMessage(),
len(sources),
))
_next_source(err, sources, 0, result)
return result
if result:
result.errback(err)
else:
lg.warn('caching task for %s was not found' % idurl)
p2p_stats.count_identity_cache(idurl, 0)
lg.warn('[cache failed] and also no other sources found %s : %s' % (
idurl,
err.getErrorMessage(),
))
return None
idurl = id_url.to_original(idurl)
_CachingTasks[idurl] = Deferred()
_CachingTasks[idurl].addErrback(on_caching_task_failed, idurl)
d = net_misc.getPageTwisted(idurl, timeout)
d.addCallback(_success, idurl)
d.addErrback(_fail, idurl)
return _CachingTasks[idurl]
def http_stun(result_defer=None):
from userid import known_servers
identity_servers = known_servers.by_host()
if not identity_servers:
if _Debug:
lg.out(
_DebugLevel,
'stun_client.http_stun SKIP, no known identity servers found'
)
return False
one_host = random.choice(list(identity_servers.keys()))
one_port = identity_servers[one_host][0]
one_url = nameurl.UrlMake('http', one_host, one_port)
if _Debug:
lg.out(_DebugLevel,
'stun_client.http_stun GO with one node : %r' % one_url)
def _check_body(html_response):
ret = {
'result': 'stun-failed',
'type': None,
'ip': None,
'details': [
'unknown client host from response',
],
}
mo = re.search(b'\<\!\-\-CLIENT_HOST\=([\d\.]+):(\d+)\-\-\>',
html_response)
if not mo:
if _Debug:
lg.out(
_DebugLevel,
'stun_client.http_stun FAILED : unknown client host from response'
)
if result_defer:
result_defer.callback(ret)
return None
ret = {
'result': 'stun-success',
'type': 'unknown',
'ip': strng.to_text(mo.group(1)),
'details': [],
}
if _Debug:
lg.out(_DebugLevel, 'stun_client.http_stun SUCCESS : %r' % ret)
if result_defer:
result_defer.callback(ret)
return None
def _request_failed(err):
if _Debug:
lg.out(_DebugLevel, 'stun_client.http_stun FAILED : %r' % err)
ret = {
'result': 'stun-failed',
'type': None,
'ip': None,
'details': [
err.getErrorMessage(),
],
}
if result_defer:
result_defer.callback(ret)
return None
d = net_misc.getPageTwisted(one_url)
d.addCallback(_check_body)
d.addErrback(_request_failed)
return True
