def _do_send_my_identity(self): """ Send my updated identity to the identity servers to register it. """ from transport.tcp import tcp_node sendfilename = settings.LocalIdentityFilename() my_sources = my_id.getLocalIdentity().getSources(as_originals=True) dlist = [] if _Debug: lg.out( _DebugLevel, 'id_rotator._do_send_my_identity my_sources=%r' % my_sources) for idurl_bin in my_sources: _, host, _, _ = nameurl.UrlParse(idurl_bin) tcpport = None if host in self.preferred_servers: tcpport = int(self.preferred_servers[host][1]) if not tcpport and host in self.known_servers: tcpport = int(self.known_servers[host][1]) if not tcpport: tcpport = settings.IdentityServerPort() srvhost = net_misc.pack_address(( host, tcpport, )) if _Debug: lg.out(_DebugLevel, ' sending to %r via TCP' % srvhost) dlist.append( tcp_node.send( sendfilename, srvhost, 'Identity', keep_alive=False, )) return DeferredList(dlist, fireOnOneCallback=True)
def _do_send_my_identity(self): """ Send my updated identity to the identity servers to register it. """ my_sources = my_id.getLocalIdentity().getSources(as_originals=True) payload = my_id.getLocalIdentity().serialize(as_text=False) dlist = [] if _Debug: lg.out(_DebugLevel, 'id_rotator._do_send_my_identity my_sources=%r' % my_sources) for idurl_bin in my_sources: _, host, _webport, filename = nameurl.UrlParse(idurl_bin) webport = None if host in self.preferred_servers: webport = int(self.preferred_servers[host][0]) if not webport and host in self.known_servers: webport = int(self.known_servers[host][0]) if not webport: webport = _webport url = net_misc.pack_address((host, webport, ), proto='http') dlist.append(net_misc.http_post_data( url=url, data=payload, connectTimeout=15, )) if _Debug: lg.args(_DebugLevel, url=url, filename=filename, size=len(payload)) return DeferredList(dlist, fireOnOneCallback=True)
def idremove(url): """ Remove identity from cache, also update indexes. Not remove local file. """ global _IdentityCache global _IdentityCacheIDs global _IdentityCacheModifiedTime global _Contact2IDURL global _IDURL2Contacts global _IPPort2IDURL idobj = _IdentityCache.pop(url, None) identid = _IdentityCacheIDs.pop(url, None) _IdentityCacheModifiedTime.pop(url, None) _IDURL2Contacts.pop(url, None) if idobj is not None: for contact in idobj.getContacts(): _Contact2IDURL.pop(contact, None) try: proto, host, port, fname = nameurl.UrlParse(contact) ipport = (host, int(port)) _IPPort2IDURL.pop(ipport, None) except: pass fire_cache_updated_callbacks(single_item=(identid, None, None)) return idobj
def SendServers(): """ My identity file can be stored in different locations, see the "sources" field. So I can use different identity servers to store more secure and reliable. This method will send my identity file to all my identity servers via transport_tcp. """ from transport.tcp import tcp_node sendfile, sendfilename = tmpfile.make("propagate") os.close(sendfile) LocalIdentity = my_id.getLocalIdentity() bpio.WriteTextFile(sendfilename, LocalIdentity.serialize(as_text=True)) dlist = [] for idurl in LocalIdentity.getSources(as_originals=True): # sources for out identity are servers we need to send to protocol, host, port, filename = nameurl.UrlParse(idurl) # TODO: rebuild identity-server logic to be able to send my identity via HTTP POST instead of TCP and # get rid of second TCP port at all webport, tcpport = known_servers.by_host().get(host, ( # by default use "expected" port numbers settings.IdentityWebPort(), settings.IdentityServerPort())) normalized_address = net_misc.normalize_address((host, int(tcpport), )) dlist.append(tcp_node.send( sendfilename, normalized_address, 'Identity', keep_alive=False, )) if _Debug: lg.args(_DebugLevel, normalized_address=normalized_address, filename=filename) dl = DeferredList(dlist, consumeErrors=True) return dl
def setContactHost(self, host, index): """ This is to set only host part of the contact. """ protocol, _, port, filename = nameurl.UrlParse(self.contacts[index]) url = nameurl.UrlMake(protocol, host, port, filename) self.contacts[index] = strng.to_bin(url).strip()
def test1(): """ Some tests. """ from userid import my_id myidentity = my_id.getLocalIdentity() print 'getIP =', myidentity.getIP() if myidentity.Valid(): print "myidentity is Valid!!!!" else: print "myidentity is not Valid" my_id.saveLocalIdentity() # sign and save raise Exception("myidentity is not Valid") print "myidentity.contacts" print myidentity.contacts print "len myidentity.contacts " print len(myidentity.contacts) print "len myidentity.contacts[0] " print myidentity.contacts[0] con = myidentity.getContact() print "con:", con, type(con) protocol, machine, port, filename = nameurl.UrlParse(con) print protocol, machine, port, filename print "identity.main serialize:\n", myidentity.serialize() for index in range(myidentity.getContactsNumber()): proto, host, port, filename = myidentity.getContactParts(index) print '[%s] [%s] [%s] [%s]' % (proto, host, port, filename)
def push_contact(idurl): global _Contacts global _PingDelayDict global _CurrentDelay ident = identitycache.FromCache(idurl) if ident is None: lg.err('"%s" not in the cache' % idurl) return None http_contact = ident.getProtoContact('http') if http_contact is None: if _Debug: lg.out( _DebugLevel * 2, 'http_node.add_contact SKIP "%s" : no http contacts found in identity' % idurl) return None _, host, port, _ = nameurl.UrlParse(http_contact) new_item = False if idurl in _Contacts: new_item = True _Contacts[idurl] = (host, port) _PingDelayDict[idurl] = _CurrentDelay if new_item: if _Debug: lg.out( _DebugLevel, 'http_node.add_contact ADDED "%s" on %s:%s' % (idurl, host, port)) else: if _Debug: lg.out( _DebugLevel, 'http_node.add_contact UPDATED "%s" on %s:%s' % (idurl, host, port)) return idurl
def _send_new_identity(self): """ Send created identity to the identity servers to register it. """ if _Debug: lg.out(_DebugLevel, 'id_registrator._send_new_identity') dlist = [] payload = self.new_identity.serialize(as_text=False) for idurl in self.new_identity.getSources(as_originals=True): self.free_idurls.remove(strng.to_bin(idurl)) _, host, webport, filename = nameurl.UrlParse(idurl) url = net_misc.pack_address(( host, webport, ), proto='http') dlist.append( net_misc.http_post_data( url=url, data=payload, connectTimeout=15, )) if _Debug: lg.args(_DebugLevel, url=url, filename=filename, size=len(payload)) return DeferredList(dlist, fireOnOneCallback=True)
def getIDName(self, index=0): """ Return an account name - this is just a user name taken from IDURL: "veselin" for "http://id.bitdust.io/veselin.xml" """ protocol, host, port, filename = nameurl.UrlParse(self.getIDURL(index)) return filename.strip()[0:-4]
def SendServers(): """ My identity file can be stored in different locations, see the "sources" field. So I can use different identity servers to store more secure. This method will send my identity file to all my identity servers via transport_tcp. """ from transport.tcp import tcp_node sendfile, sendfilename = tmpfile.make("propagate") os.close(sendfile) LocalIdentity = my_id.getLocalIdentity() bpio.WriteFile(sendfilename, LocalIdentity.serialize()) dlist = [] for idurl in LocalIdentity.sources: # sources for out identity are servers we need to send to protocol, host, port, filename = nameurl.UrlParse(idurl) # if host == settings.IdentityServerName(): # host = '67.207.147.183' webport, tcpport = known_servers.by_host().get( host, (settings.IdentityWebPort(), settings.IdentityServerPort())) # srvhost = '%s:%d' % (host, int(tcpport)) dlist.append( tcp_node.send(sendfilename, (host, int(tcpport)), 'Identity', True)) # dlist.append(gateway.send_file_single('tcp', srvhost, sendfilename, 'Identity')) dl = DeferredList(dlist, consumeErrors=True) return dl
def _send_new_identity(self): """ Send created identity to the identity server to register it. TODO: need to close transport and gateway after that """ lg.out(4, 'id_registrator._send_new_identity ') from transport import gateway from transport import network_transport from transport.tcp import tcp_interface gateway.init() interface = tcp_interface.GateInterface() transport = network_transport.NetworkTransport('tcp', interface) transport.automat('init', gateway.listener()) transport.automat('start') gateway.start() sendfilename = settings.LocalIdentityFilename() + '.new' dlist = [] for idurl in self.new_identity.sources: self.free_idurls.remove(strng.to_bin(idurl)) _, host, _, _ = nameurl.UrlParse(idurl) _, tcpport = known_servers.by_host().get( host, (settings.IdentityWebPort(), settings.IdentityServerPort())) srvhost = net_misc.pack_address(( host, tcpport, )) dlist.append( gateway.send_file_single(idurl, 'tcp', srvhost, sendfilename, 'Identity')) # assert len(self.free_idurls) == 0 return DeferredList(dlist, fireOnOneCallback=True)
def _send_new_identity(self): """ Send created identity to the identity servers to register it. """ if _Debug: lg.out(_DebugLevel, 'id_registrator._send_new_identity') from transport.tcp import tcp_node sendfilename = settings.LocalIdentityFilename() + '.new' dlist = [] for idurl in self.new_identity.getSources(as_originals=True): self.free_idurls.remove(strng.to_bin(idurl)) _, host, _, _ = nameurl.UrlParse(idurl) _, tcpport = known_servers.by_host().get( host, (settings.IdentityWebPort(), settings.IdentityServerPort())) srvhost = net_misc.pack_address(( host, tcpport, )) if _Debug: lg.out(_DebugLevel, ' sending to %r via TCP' % srvhost) dlist.append( tcp_node.send( sendfilename, srvhost, 'Identity', keep_alive=False, )) return DeferredList(dlist, fireOnOneCallback=True)
def setContactHost(self, host, index): """ This is to set only host part of the contact. """ protocol, host_, port, filename = nameurl.UrlParse(self.contacts[index]) url = nameurl.UrlMake(protocol, host, port, filename) self.contacts[index] = url.encode("ascii").strip()
def setContactPort(self, index, newport): """ This is useful when listening port get changed. """ protocol, host, _, filename = nameurl.UrlParse(self.contacts[index]) url = nameurl.UrlMake(protocol, host, newport, filename) self.contacts[index] = strng.to_bin(url).strip()
def setContactPort(self, index, newport): """ This is useful when listening port get changed. """ protocol, host, port, filename = nameurl.UrlParse(self.contacts[index]) url = nameurl.UrlMake(protocol, host, newport, filename) self.contacts[index] = url.encode("ascii").strip()
def isCorrect(self): """ Do some checking on the object fields. """ if len(self.contacts) == 0: return False if len(self.sources) == 0: return False if not self.publickey: return False if not self.signature: return False if not self.revision: return False original_sources = self.getSources(as_originals=True) if len(set(original_sources)) != len(self.sources): lg.warn('original identity sources are duplicated: %r' % original_sources) return False if len(original_sources) > settings.MaximumIdentitySources(): lg.warn('too much sources') return False if len(original_sources) < settings.MinimumIdentitySources(): lg.warn('too few sources') return False try: int(self.revision) except: lg.warn('identity revision: %s' % self.revision) return False names = set() for source in original_sources: if not source: lg.warn('found empty source') return False proto, host, port, filename = nameurl.UrlParse(source) if filename.count('/'): lg.warn("incorrect identity name: %s" % filename) return False name, justxml = filename.split('.') names.add(name) # SECURITY check that name is simple if justxml != "xml": lg.warn("incorrect identity name: %s" % filename) return False if len(name) > settings.MaximumUsernameLength(): lg.warn("incorrect identity name: %s" % filename) return False if len(name) < settings.MinimumUsernameLength(): lg.warn("incorrect identity name: %s" % filename) return False for c in name: if c not in settings.LegalUsernameChars(): lg.warn("incorrect identity name: %s" % filename) return False if len(names) > 1: lg.warn('names are not consistent: %s' % str(names)) return False return True
def getContactProto(self, index): """ Return a proto part of the contact at given position. """ c = self.getContact(index) if c is None: return None return nameurl.UrlParse(c)[0]
def getContactsByProto(self): """ Return a dictionary of all contacts where keys are protos. """ d = {} for i in range(len(self.contacts)): proto, x, x, x = nameurl.UrlParse(self.contacts[i]) d[proto] = self.contacts[i] return d
def getProtoParts(self, proto): """ See ``getProtoContact``, return a tuple for given ``proto``: (proto, host, port, filename) """ contact = self.getProtoContact(proto) if contact is None: return None, None, None, None return nameurl.UrlParse(contact)
def idurl_to_id(idurl): """ """ proto, host, port, filename = nameurl.UrlParse(idurl) assert proto == 'http' user_id = filename.replace('.xml', '') + '@' + host if port and port not in ['80', 80, ]: user_id += ':%s' % str(port) return user_id
def getIDHost(self, index=0): """ Return a server host name where that identity is stored: "id.bitdust.io" for "http://id.bitdust.io/veselin.xml". """ protocol, host, port, filename = nameurl.UrlParse(self.getIDURL(index)) if port: host += ':' + str(port) return host
def SubstitutePacketID( packet_id, idurl=None, customer=None, key_id=None, path=None, key_alias=None, ): from lib import nameurl g = ParseGlobalID(packet_id, as_field=False) if key_id is not None: g['key_id'] = key_id key_alias, _, customer = key_id.rpartition('$') if key_alias is not None: g['key_alias'] = key_alias if path is not None: g['path'] = path if idurl is not None: g['idurl'] = idurl g['customer'] = UrlToGlobalID(g['idurl']) _, idhost, port, filename = nameurl.UrlParse(idurl) if port: idhost += '_' + str(port) g['user'] = filename.strip()[0:-4] if key_id is None: g['key_id'] = _FORMAT_GLOBAL_ID_KEY_USER.format( key_alias=g['key_alias'], user=g['customer'], ) if customer is not None: g['customer'] = customer g['idurl'] = glob2idurl(g['customer'], as_field=False) _, idhost, port, filename = nameurl.UrlParse(idurl) if port: idhost += '_' + str(port) g['user'] = filename.strip()[0:-4] if key_id is None: g['key_id'] = _FORMAT_GLOBAL_ID_KEY_USER.format( key_alias=g['key_alias'], user=g['customer'], ) return MakeGlobalID(**g)
def setProtoContact(self, proto, contact): """ Found a contact with given ``proto`` and set its value or append a new contact. """ for i in range(0, len(self.contacts)): proto_, host, port, filename = nameurl.UrlParse(self.contacts[i]) if proto_.strip() == proto.strip(): self.contacts[i] = contact return self.contacts.append(contact)
def setProtoContact(self, proto, contact): """ Found a contact with given ``proto`` and set its value or append a new contact. """ for i in range(0, len(self.contacts)): proto_, _, _, _ = nameurl.UrlParse(self.contacts[i]) if proto_.strip() == strng.to_bin(proto).strip(): self.contacts[i] = strng.to_bin(contact) return self.contacts.append(strng.to_bin(contact))
def getProtoOrder(self): """ Return a list of "proto" parts of all contacts. In other words return a list of all supported protocols. This keeps the order of the protos - this is a sort of priority of the transports. """ orderL = [] for c in self.contacts: proto, host, port, filename = nameurl.UrlParse(c) orderL.append(proto) return orderL
def UrlToGlobalID(url): """ """ if not url: return url from lib import nameurl _, host, port, filename = nameurl.UrlParse(url) if filename.count('.'): username = filename.split('.')[0] if port: host = '%s_%s' % (host, port) return '%s@%s' % (username, host)
def isCorrect(self): """ Do some checking on the object fields. """ if len(self.contacts) == 0: return False if len(self.sources) == 0: return False if self.publickey == '': return False if self.signature == '': return False if self.revision == '': return False if len(self.sources) > settings.MaximumIdentitySources(): lg.warn('too much sources') return False if len(self.sources) < settings.MinimumIdentitySources(): lg.warn('too few sources') return False try: int(self.revision) except: lg.warn('identity revision: %s' % self.revision) return False names = set() for source in self.sources: proto, host, port, filename = nameurl.UrlParse(source) if filename.count('/'): lg.warn("identity name: %s" % filename) return False name, justxml = filename.split('.') names.add(name) # SECURITY check that name is simple if justxml != "xml": lg.warn("identity name: %s" % filename) return False if len(name) > settings.MaximumUsernameLength(): lg.warn("identity name: %s" % filename) return False if len(name) < settings.MinimumUsernameLength(): lg.warn("identity name: %s" % filename) return False for c in name: if c not in settings.LegalUsernameChars(): lg.warn("identity name: %s" % filename) return False if len(names) > 1: lg.warn('names are not consistant: %s' % str(names)) return False return True
def UrlToGlobalID(url, include_key=False): """ """ if not url: return url from lib import nameurl _, host, port, filename = nameurl.UrlParse(url) if filename.count('.'): username = filename.split('.')[0] if port: host = '%s_%s' % (host, port) if include_key: username = '******' % username return '%s@%s' % (username, host)
def idset(idurl, id_obj): """ Important method - need to call that to update indexes. """ global _Contact2IDURL global _IDURL2Contacts global _IPPort2IDURL global _IdentityCache global _IdentityCacheIDs global _IdentityCacheCounter global _IdentityCacheModifiedTime idurl = id_url.to_original(idurl) if not has_idurl(idurl): if _Debug: lg.out(_DebugLevel, 'identitydb.idset new identity: %r' % idurl) _IdentityCache[idurl] = id_obj _IdentityCacheModifiedTime[idurl] = time.time() identid = _IdentityCacheIDs.get(idurl, None) if identid is None: identid = _IdentityCacheCounter _IdentityCacheCounter += 1 _IdentityCacheIDs[idurl] = identid for contact in id_obj.getContacts(): if contact not in _Contact2IDURL: _Contact2IDURL[contact] = set() # else: # if len(_Contact2IDURL[contact]) >= 1 and idurl not in _Contact2IDURL[contact]: # lg.warn('another user have same contact: ' + str(list(_Contact2IDURL[contact]))) _Contact2IDURL[contact].add(idurl) if idurl not in _IDURL2Contacts: _IDURL2Contacts[idurl] = set() _IDURL2Contacts[idurl].add(contact) try: proto, host, port, fname = nameurl.UrlParse(contact) ipport = (host, int(port)) _IPPort2IDURL[ipport] = idurl except: pass # TODO: when identity contacts changed - need to remove old items from _Contact2IDURL fire_cache_updated_callbacks(single_item=(identid, idurl, id_obj)) if _Debug: lg.out(_DebugLevel, 'identitydb.idset %r' % idurl) # now make sure we properly handle changes in the sources of that identity try: id_url.identity_cached(id_obj) except: lg.exc()
def MakeGlobalID( idurl=None, user=None, idhost=None, customer=None, key_alias=None, path=None, version=None, key_id=None, ): """ Based on input parameters returns string like this: [email protected]:animals/cat.png#F20160313043757PM """ output_format = _FORMAT_GLOBAL_ID_KEY_USER out = '' if key_id: out = key_id else: if customer: if not idurl: idurl = GlobalUserToIDURL(customer) if customer.count('$'): key_alias, _, _ = customer.rpartition('$') if customer.count('!'): user_and_key, _, _ = customer.rpartition('@') _, _, key_alias = user_and_key.rpartition('!') if idurl: from lib import nameurl _, idhost, port, filename = nameurl.UrlParse(idurl) if port: idhost += '_' + str(port) user = filename.strip()[0:-4] if key_alias: out = output_format.format(user=user, key_alias=key_alias) else: out = user out += '@{}'.format(idhost) if path: out += ':{}'.format(path) if version: out += '#{}'.format(version) return strng.to_text(out)