def _do_send_my_identity(self):
"""
Send my updated identity to the identity servers to register it.
"""
from transport.tcp import tcp_node
sendfilename = settings.LocalIdentityFilename()
my_sources = my_id.getLocalIdentity().getSources(as_originals=True)
dlist = []
if _Debug:
lg.out(
_DebugLevel,
'id_rotator._do_send_my_identity my_sources=%r' % my_sources)
for idurl_bin in my_sources:
_, host, _, _ = nameurl.UrlParse(idurl_bin)
tcpport = None
if host in self.preferred_servers:
tcpport = int(self.preferred_servers[host][1])
if not tcpport and host in self.known_servers:
tcpport = int(self.known_servers[host][1])
if not tcpport:
tcpport = settings.IdentityServerPort()
srvhost = net_misc.pack_address((
host,
tcpport,
))
if _Debug:
lg.out(_DebugLevel, ' sending to %r via TCP' % srvhost)
dlist.append(
tcp_node.send(
sendfilename,
srvhost,
'Identity',
keep_alive=False,
))
return DeferredList(dlist, fireOnOneCallback=True)
def _do_send_my_identity(self):
"""
Send my updated identity to the identity servers to register it.
"""
my_sources = my_id.getLocalIdentity().getSources(as_originals=True)
payload = my_id.getLocalIdentity().serialize(as_text=False)
dlist = []
if _Debug:
lg.out(_DebugLevel, 'id_rotator._do_send_my_identity my_sources=%r' % my_sources)
for idurl_bin in my_sources:
_, host, _webport, filename = nameurl.UrlParse(idurl_bin)
webport = None
if host in self.preferred_servers:
webport = int(self.preferred_servers[host][0])
if not webport and host in self.known_servers:
webport = int(self.known_servers[host][0])
if not webport:
webport = _webport
url = net_misc.pack_address((host, webport, ), proto='http')
dlist.append(net_misc.http_post_data(
url=url,
data=payload,
connectTimeout=15,
))
if _Debug:
lg.args(_DebugLevel, url=url, filename=filename, size=len(payload))
return DeferredList(dlist, fireOnOneCallback=True)
def idremove(url):
"""
Remove identity from cache, also update indexes.
Not remove local file.
"""
global _IdentityCache
global _IdentityCacheIDs
global _IdentityCacheModifiedTime
global _Contact2IDURL
global _IDURL2Contacts
global _IPPort2IDURL
idobj = _IdentityCache.pop(url, None)
identid = _IdentityCacheIDs.pop(url, None)
_IdentityCacheModifiedTime.pop(url, None)
_IDURL2Contacts.pop(url, None)
if idobj is not None:
for contact in idobj.getContacts():
_Contact2IDURL.pop(contact, None)
try:
proto, host, port, fname = nameurl.UrlParse(contact)
ipport = (host, int(port))
_IPPort2IDURL.pop(ipport, None)
except:
pass
fire_cache_updated_callbacks(single_item=(identid, None, None))
return idobj
def SendServers():
"""
My identity file can be stored in different locations, see the "sources"
field.
So I can use different identity servers to store more secure and reliable. This
method will send my identity file to all my identity servers via
transport_tcp.
"""
from transport.tcp import tcp_node
sendfile, sendfilename = tmpfile.make("propagate")
os.close(sendfile)
LocalIdentity = my_id.getLocalIdentity()
bpio.WriteTextFile(sendfilename, LocalIdentity.serialize(as_text=True))
dlist = []
for idurl in LocalIdentity.getSources(as_originals=True):
# sources for out identity are servers we need to send to
protocol, host, port, filename = nameurl.UrlParse(idurl)
# TODO: rebuild identity-server logic to be able to send my identity via HTTP POST instead of TCP and
# get rid of second TCP port at all
webport, tcpport = known_servers.by_host().get(host, (
# by default use "expected" port numbers
settings.IdentityWebPort(), settings.IdentityServerPort()))
normalized_address = net_misc.normalize_address((host, int(tcpport), ))
dlist.append(tcp_node.send(
sendfilename, normalized_address, 'Identity', keep_alive=False,
))
if _Debug:
lg.args(_DebugLevel, normalized_address=normalized_address, filename=filename)
dl = DeferredList(dlist, consumeErrors=True)
return dl
def setContactHost(self, host, index):
"""
This is to set only host part of the contact.
"""
protocol, _, port, filename = nameurl.UrlParse(self.contacts[index])
url = nameurl.UrlMake(protocol, host, port, filename)
self.contacts[index] = strng.to_bin(url).strip()
def test1():
"""
Some tests.
"""
from userid import my_id
myidentity = my_id.getLocalIdentity()
print 'getIP =', myidentity.getIP()
if myidentity.Valid():
print "myidentity is Valid!!!!"
else:
print "myidentity is not Valid"
my_id.saveLocalIdentity() # sign and save
raise Exception("myidentity is not Valid")
print "myidentity.contacts"
print myidentity.contacts
print "len myidentity.contacts "
print len(myidentity.contacts)
print "len myidentity.contacts[0] "
print myidentity.contacts[0]
con = myidentity.getContact()
print "con:", con, type(con)
protocol, machine, port, filename = nameurl.UrlParse(con)
print protocol, machine, port, filename
print "identity.main serialize:\n", myidentity.serialize()
for index in range(myidentity.getContactsNumber()):
proto, host, port, filename = myidentity.getContactParts(index)
print '[%s] [%s] [%s] [%s]' % (proto, host, port, filename)
def push_contact(idurl):
global _Contacts
global _PingDelayDict
global _CurrentDelay
ident = identitycache.FromCache(idurl)
if ident is None:
lg.err('"%s" not in the cache' % idurl)
return None
http_contact = ident.getProtoContact('http')
if http_contact is None:
if _Debug:
lg.out(
_DebugLevel * 2,
'http_node.add_contact SKIP "%s" : no http contacts found in identity'
% idurl)
return None
_, host, port, _ = nameurl.UrlParse(http_contact)
new_item = False
if idurl in _Contacts:
new_item = True
_Contacts[idurl] = (host, port)
_PingDelayDict[idurl] = _CurrentDelay
if new_item:
if _Debug:
lg.out(
_DebugLevel, 'http_node.add_contact ADDED "%s" on %s:%s' %
(idurl, host, port))
else:
if _Debug:
lg.out(
_DebugLevel, 'http_node.add_contact UPDATED "%s" on %s:%s' %
(idurl, host, port))
return idurl
def _send_new_identity(self):
"""
Send created identity to the identity servers to register it.
"""
if _Debug:
lg.out(_DebugLevel, 'id_registrator._send_new_identity')
dlist = []
payload = self.new_identity.serialize(as_text=False)
for idurl in self.new_identity.getSources(as_originals=True):
self.free_idurls.remove(strng.to_bin(idurl))
_, host, webport, filename = nameurl.UrlParse(idurl)
url = net_misc.pack_address((
host,
webport,
), proto='http')
dlist.append(
net_misc.http_post_data(
url=url,
data=payload,
connectTimeout=15,
))
if _Debug:
lg.args(_DebugLevel,
url=url,
filename=filename,
size=len(payload))
return DeferredList(dlist, fireOnOneCallback=True)
def getIDName(self, index=0):
"""
Return an account name - this is just a user name taken from IDURL:
"veselin" for "http://id.bitdust.io/veselin.xml"
"""
protocol, host, port, filename = nameurl.UrlParse(self.getIDURL(index))
return filename.strip()[0:-4]
def SendServers():
"""
My identity file can be stored in different locations, see the "sources"
field.
So I can use different identity servers to store more secure. This
method will send my identity file to all my identity servers via
transport_tcp.
"""
from transport.tcp import tcp_node
sendfile, sendfilename = tmpfile.make("propagate")
os.close(sendfile)
LocalIdentity = my_id.getLocalIdentity()
bpio.WriteFile(sendfilename, LocalIdentity.serialize())
dlist = []
for idurl in LocalIdentity.sources:
# sources for out identity are servers we need to send to
protocol, host, port, filename = nameurl.UrlParse(idurl)
# if host == settings.IdentityServerName():
# host = '67.207.147.183'
webport, tcpport = known_servers.by_host().get(
host, (settings.IdentityWebPort(), settings.IdentityServerPort()))
# srvhost = '%s:%d' % (host, int(tcpport))
dlist.append(
tcp_node.send(sendfilename, (host, int(tcpport)), 'Identity',
True))
# dlist.append(gateway.send_file_single('tcp', srvhost, sendfilename, 'Identity'))
dl = DeferredList(dlist, consumeErrors=True)
return dl
def _send_new_identity(self):
"""
Send created identity to the identity server to register it.
TODO: need to close transport and gateway after that
"""
lg.out(4, 'id_registrator._send_new_identity ')
from transport import gateway
from transport import network_transport
from transport.tcp import tcp_interface
gateway.init()
interface = tcp_interface.GateInterface()
transport = network_transport.NetworkTransport('tcp', interface)
transport.automat('init', gateway.listener())
transport.automat('start')
gateway.start()
sendfilename = settings.LocalIdentityFilename() + '.new'
dlist = []
for idurl in self.new_identity.sources:
self.free_idurls.remove(strng.to_bin(idurl))
_, host, _, _ = nameurl.UrlParse(idurl)
_, tcpport = known_servers.by_host().get(
host,
(settings.IdentityWebPort(), settings.IdentityServerPort()))
srvhost = net_misc.pack_address((
host,
tcpport,
))
dlist.append(
gateway.send_file_single(idurl, 'tcp', srvhost, sendfilename,
'Identity'))
# assert len(self.free_idurls) == 0
return DeferredList(dlist, fireOnOneCallback=True)
def _send_new_identity(self):
"""
Send created identity to the identity servers to register it.
"""
if _Debug:
lg.out(_DebugLevel, 'id_registrator._send_new_identity')
from transport.tcp import tcp_node
sendfilename = settings.LocalIdentityFilename() + '.new'
dlist = []
for idurl in self.new_identity.getSources(as_originals=True):
self.free_idurls.remove(strng.to_bin(idurl))
_, host, _, _ = nameurl.UrlParse(idurl)
_, tcpport = known_servers.by_host().get(
host,
(settings.IdentityWebPort(), settings.IdentityServerPort()))
srvhost = net_misc.pack_address((
host,
tcpport,
))
if _Debug:
lg.out(_DebugLevel, ' sending to %r via TCP' % srvhost)
dlist.append(
tcp_node.send(
sendfilename,
srvhost,
'Identity',
keep_alive=False,
))
return DeferredList(dlist, fireOnOneCallback=True)
def setContactHost(self, host, index):
"""
This is to set only host part of the contact.
"""
protocol, host_, port, filename = nameurl.UrlParse(self.contacts[index])
url = nameurl.UrlMake(protocol, host, port, filename)
self.contacts[index] = url.encode("ascii").strip()
def setContactPort(self, index, newport):
"""
This is useful when listening port get changed.
"""
protocol, host, _, filename = nameurl.UrlParse(self.contacts[index])
url = nameurl.UrlMake(protocol, host, newport, filename)
self.contacts[index] = strng.to_bin(url).strip()
def setContactPort(self, index, newport):
"""
This is useful when listening port get changed.
"""
protocol, host, port, filename = nameurl.UrlParse(self.contacts[index])
url = nameurl.UrlMake(protocol, host, newport, filename)
self.contacts[index] = url.encode("ascii").strip()
def isCorrect(self):
"""
Do some checking on the object fields.
"""
if len(self.contacts) == 0:
return False
if len(self.sources) == 0:
return False
if not self.publickey:
return False
if not self.signature:
return False
if not self.revision:
return False
original_sources = self.getSources(as_originals=True)
if len(set(original_sources)) != len(self.sources):
lg.warn('original identity sources are duplicated: %r' %
original_sources)
return False
if len(original_sources) > settings.MaximumIdentitySources():
lg.warn('too much sources')
return False
if len(original_sources) < settings.MinimumIdentitySources():
lg.warn('too few sources')
return False
try:
int(self.revision)
except:
lg.warn('identity revision: %s' % self.revision)
return False
names = set()
for source in original_sources:
if not source:
lg.warn('found empty source')
return False
proto, host, port, filename = nameurl.UrlParse(source)
if filename.count('/'):
lg.warn("incorrect identity name: %s" % filename)
return False
name, justxml = filename.split('.')
names.add(name)
# SECURITY check that name is simple
if justxml != "xml":
lg.warn("incorrect identity name: %s" % filename)
return False
if len(name) > settings.MaximumUsernameLength():
lg.warn("incorrect identity name: %s" % filename)
return False
if len(name) < settings.MinimumUsernameLength():
lg.warn("incorrect identity name: %s" % filename)
return False
for c in name:
if c not in settings.LegalUsernameChars():
lg.warn("incorrect identity name: %s" % filename)
return False
if len(names) > 1:
lg.warn('names are not consistent: %s' % str(names))
return False
return True
def getContactProto(self, index):
"""
Return a proto part of the contact at given position.
"""
c = self.getContact(index)
if c is None:
return None
return nameurl.UrlParse(c)[0]
def getContactsByProto(self):
"""
Return a dictionary of all contacts where keys are protos.
"""
d = {}
for i in range(len(self.contacts)):
proto, x, x, x = nameurl.UrlParse(self.contacts[i])
d[proto] = self.contacts[i]
return d
def getProtoParts(self, proto):
"""
See ``getProtoContact``, return a tuple for given ``proto``: (proto,
host, port, filename)
"""
contact = self.getProtoContact(proto)
if contact is None:
return None, None, None, None
return nameurl.UrlParse(contact)
def idurl_to_id(idurl):
"""
"""
proto, host, port, filename = nameurl.UrlParse(idurl)
assert proto == 'http'
user_id = filename.replace('.xml', '') + '@' + host
if port and port not in ['80', 80, ]:
user_id += ':%s' % str(port)
return user_id
def getIDHost(self, index=0):
"""
Return a server host name where that identity is stored:
"id.bitdust.io" for "http://id.bitdust.io/veselin.xml".
"""
protocol, host, port, filename = nameurl.UrlParse(self.getIDURL(index))
if port:
host += ':' + str(port)
return host
def SubstitutePacketID(
packet_id,
idurl=None,
customer=None,
key_id=None,
path=None,
key_alias=None,
):
from lib import nameurl
g = ParseGlobalID(packet_id, as_field=False)
if key_id is not None:
g['key_id'] = key_id
key_alias, _, customer = key_id.rpartition('$')
if key_alias is not None:
g['key_alias'] = key_alias
if path is not None:
g['path'] = path
if idurl is not None:
g['idurl'] = idurl
g['customer'] = UrlToGlobalID(g['idurl'])
_, idhost, port, filename = nameurl.UrlParse(idurl)
if port:
idhost += '_' + str(port)
g['user'] = filename.strip()[0:-4]
if key_id is None:
g['key_id'] = _FORMAT_GLOBAL_ID_KEY_USER.format(
key_alias=g['key_alias'],
user=g['customer'],
)
if customer is not None:
g['customer'] = customer
g['idurl'] = glob2idurl(g['customer'], as_field=False)
_, idhost, port, filename = nameurl.UrlParse(idurl)
if port:
idhost += '_' + str(port)
g['user'] = filename.strip()[0:-4]
if key_id is None:
g['key_id'] = _FORMAT_GLOBAL_ID_KEY_USER.format(
key_alias=g['key_alias'],
user=g['customer'],
)
return MakeGlobalID(**g)
def setProtoContact(self, proto, contact):
"""
Found a contact with given ``proto`` and set its value or append a new
contact.
"""
for i in range(0, len(self.contacts)):
proto_, host, port, filename = nameurl.UrlParse(self.contacts[i])
if proto_.strip() == proto.strip():
self.contacts[i] = contact
return
self.contacts.append(contact)
def setProtoContact(self, proto, contact):
"""
Found a contact with given ``proto`` and set its value or append a new
contact.
"""
for i in range(0, len(self.contacts)):
proto_, _, _, _ = nameurl.UrlParse(self.contacts[i])
if proto_.strip() == strng.to_bin(proto).strip():
self.contacts[i] = strng.to_bin(contact)
return
self.contacts.append(strng.to_bin(contact))
def getProtoOrder(self):
"""
Return a list of "proto" parts of all contacts. In other words return a
list of all supported protocols.
This keeps the order of the protos - this is a sort of priority of the transports.
"""
orderL = []
for c in self.contacts:
proto, host, port, filename = nameurl.UrlParse(c)
orderL.append(proto)
return orderL
def UrlToGlobalID(url):
"""
"""
if not url:
return url
from lib import nameurl
_, host, port, filename = nameurl.UrlParse(url)
if filename.count('.'):
username = filename.split('.')[0]
if port:
host = '%s_%s' % (host, port)
return '%s@%s' % (username, host)
def isCorrect(self):
"""
Do some checking on the object fields.
"""
if len(self.contacts) == 0:
return False
if len(self.sources) == 0:
return False
if self.publickey == '':
return False
if self.signature == '':
return False
if self.revision == '':
return False
if len(self.sources) > settings.MaximumIdentitySources():
lg.warn('too much sources')
return False
if len(self.sources) < settings.MinimumIdentitySources():
lg.warn('too few sources')
return False
try:
int(self.revision)
except:
lg.warn('identity revision: %s' % self.revision)
return False
names = set()
for source in self.sources:
proto, host, port, filename = nameurl.UrlParse(source)
if filename.count('/'):
lg.warn("identity name: %s" % filename)
return False
name, justxml = filename.split('.')
names.add(name)
# SECURITY check that name is simple
if justxml != "xml":
lg.warn("identity name: %s" % filename)
return False
if len(name) > settings.MaximumUsernameLength():
lg.warn("identity name: %s" % filename)
return False
if len(name) < settings.MinimumUsernameLength():
lg.warn("identity name: %s" % filename)
return False
for c in name:
if c not in settings.LegalUsernameChars():
lg.warn("identity name: %s" % filename)
return False
if len(names) > 1:
lg.warn('names are not consistant: %s' % str(names))
return False
return True
def UrlToGlobalID(url, include_key=False):
"""
"""
if not url:
return url
from lib import nameurl
_, host, port, filename = nameurl.UrlParse(url)
if filename.count('.'):
username = filename.split('.')[0]
if port:
host = '%s_%s' % (host, port)
if include_key:
username = '******' % username
return '%s@%s' % (username, host)
def idset(idurl, id_obj):
"""
Important method - need to call that to update indexes.
"""
global _Contact2IDURL
global _IDURL2Contacts
global _IPPort2IDURL
global _IdentityCache
global _IdentityCacheIDs
global _IdentityCacheCounter
global _IdentityCacheModifiedTime
idurl = id_url.to_original(idurl)
if not has_idurl(idurl):
if _Debug:
lg.out(_DebugLevel, 'identitydb.idset new identity: %r' % idurl)
_IdentityCache[idurl] = id_obj
_IdentityCacheModifiedTime[idurl] = time.time()
identid = _IdentityCacheIDs.get(idurl, None)
if identid is None:
identid = _IdentityCacheCounter
_IdentityCacheCounter += 1
_IdentityCacheIDs[idurl] = identid
for contact in id_obj.getContacts():
if contact not in _Contact2IDURL:
_Contact2IDURL[contact] = set()
# else:
# if len(_Contact2IDURL[contact]) >= 1 and idurl not in _Contact2IDURL[contact]:
# lg.warn('another user have same contact: ' + str(list(_Contact2IDURL[contact])))
_Contact2IDURL[contact].add(idurl)
if idurl not in _IDURL2Contacts:
_IDURL2Contacts[idurl] = set()
_IDURL2Contacts[idurl].add(contact)
try:
proto, host, port, fname = nameurl.UrlParse(contact)
ipport = (host, int(port))
_IPPort2IDURL[ipport] = idurl
except:
pass
# TODO: when identity contacts changed - need to remove old items from _Contact2IDURL
fire_cache_updated_callbacks(single_item=(identid, idurl, id_obj))
if _Debug:
lg.out(_DebugLevel, 'identitydb.idset %r' % idurl)
# now make sure we properly handle changes in the sources of that identity
try:
id_url.identity_cached(id_obj)
except:
lg.exc()
def MakeGlobalID(
idurl=None,
user=None,
idhost=None,
customer=None,
key_alias=None,
path=None,
version=None,
key_id=None,
):
"""
Based on input parameters returns string like this:
[email protected]:animals/cat.png#F20160313043757PM
"""
output_format = _FORMAT_GLOBAL_ID_KEY_USER
out = ''
if key_id:
out = key_id
else:
if customer:
if not idurl:
idurl = GlobalUserToIDURL(customer)
if customer.count('$'):
key_alias, _, _ = customer.rpartition('$')
if customer.count('!'):
user_and_key, _, _ = customer.rpartition('@')
_, _, key_alias = user_and_key.rpartition('!')
if idurl:
from lib import nameurl
_, idhost, port, filename = nameurl.UrlParse(idurl)
if port:
idhost += '_' + str(port)
user = filename.strip()[0:-4]
if key_alias:
out = output_format.format(user=user, key_alias=key_alias)
else:
out = user
out += '@{}'.format(idhost)
if path:
out += ':{}'.format(path)
if version:
out += '#{}'.format(version)
return strng.to_text(out)
