def doCheckOverride(self, arg): """ Action method. """ target = arg.CreatorID idsrc = arg.Payload try: new_ident = identity.identity(xmlsrc=idsrc) except: lg.out(_DebugLevel, 'payload: [%s]' % idsrc) lg.exc() return if not new_ident.isCorrect() or not new_ident.Valid(): lg.warn('incoming identity is not valid') return if not self._is_my_contacts_present_in_identity(new_ident): cur_contacts = [] try: cur_contacts = identity.identity( xmlsrc=identitycache.ReadOverriddenIdentityXMLSource(target) ).getContacts() except: lg.exc() return if _Debug: lg.out(_DebugLevel, 'proxy_router.doCheckOverride override identity for %s' % arg.CreatorID) lg.out(_DebugLevel, ' current override contacts is : %s' % cur_contacts) lg.out(_DebugLevel, ' new contacts is : %s' % new_ident.getContacts()) identitycache.OverrideIdentity(arg.CreatorID, idsrc) else: if _Debug: lg.out(_DebugLevel, 'proxy_router.doCheckOverride skip override, found my contacts in identity from %s' % arg.CreatorID) lg.out(_DebugLevel, ' known contacts is : %s' % new_ident.getContacts())
def _do_set_contacts_override(self, *args, **kwargs): if _Debug: lg.out( _DebugLevel, 'proxy_router.doSetContactsOverride identity for %s' % args[0].CreatorID) user_id = args[0].CreatorID idsrc = args[0].Payload try: new_ident = identity.identity(xmlsrc=idsrc) except: lg.out(_DebugLevel, 'payload: [%s]' % idsrc) lg.exc() return if not new_ident.isCorrect() or not new_ident.Valid(): lg.warn('incoming identity is not valid') return current_overridden_identity = identitycache.ReadOverriddenIdentityXMLSource( user_id) try: current_contacts = identity.identity( xmlsrc=current_overridden_identity).getContacts() except: current_contacts = [] identitycache.StopOverridingIdentity(user_id) result = identitycache.OverrideIdentity(args[0].CreatorID, idsrc) if _Debug: lg.out( _DebugLevel, ' current overridden contacts is : %s' % current_contacts) lg.out( _DebugLevel, ' new override contacts will be : %s' % new_ident.getContacts()) lg.out(_DebugLevel, ' result=%s' % result)
def update(idurl, xml_src): """ This is a correct method to update an identity in the local cache. PREPRO need to check that date or version is after old one so not vulnerable to replay attacks. """ idurl = strng.to_bin(idurl) try: newid = identity.identity(xmlsrc=xml_src) except: lg.exc() return False if not newid.isCorrect(): lg.out(1, "identitydb.update ERROR incorrect identity : %r" % idurl) return False try: if not newid.Valid(): lg.out(1, "identitydb.update ERROR identity not Valid : %r" % idurl) return False except: lg.exc() return False filename = os.path.join(settings.IdentityCacheDir(), nameurl.UrlFilename(idurl)) if os.path.exists(filename): oldidentityxml = bpio.ReadTextFile(filename) oldidentity = identity.identity(xmlsrc=oldidentityxml) if oldidentity.publickey != newid.publickey: lg.out( 1, "identitydb.update ERROR new publickey does not match old, SECURITY VIOLATION : %r" % idurl) return False if oldidentity.signature != newid.signature: lg.out( 6, 'identitydb.update have new data for %r' % nameurl.GetName(idurl)) else: idset(idurl, newid) return True # publickeys match so we can update it bpio.WriteTextFile(filename, xml_src) idset(idurl, newid) return True
def update(idurl, xml_src): """ This is a correct method to update an identity in the local cache. PREPRO need to check that date or version is after old one so not vulnerable to replay attacks. """ idurl = id_url.to_original(idurl) try: newid = identity.identity(xmlsrc=xml_src) except: lg.exc() return False if not newid.isCorrect(): lg.err("incorrect identity : %r" % idurl) return False try: if not newid.Valid(): lg.err("identity not valid : %r" % idurl) return False except: lg.exc() return False filename = os.path.join(settings.IdentityCacheDir(), nameurl.UrlFilename(idurl)) if os.path.exists(filename): oldidentityxml = bpio.ReadTextFile(filename) oldidentity = identity.identity(xmlsrc=oldidentityxml) if oldidentity.publickey != newid.publickey: # TODO: SECURITY add some kind of black list to be able to block certain IP's if the DDoS me lg.err( "new public key does not match with old, SECURITY VIOLATION : %r" % idurl) return False if oldidentity.signature != newid.signature: if _Debug: lg.out( _DebugLevel, 'identitydb.update have new data for %r' % nameurl.GetName(idurl)) else: idset(idurl, newid) return True # publickeys match so we can update it bpio.WriteTextFile(filename, xml_src) idset(idurl, newid) return True
def loadLocalIdentity(): """ The core method. The file [BitDust data dir]/metadata/localidentity keeps the user identity in XML format. Do read the local file and set into object in memory. """ global _LocalIdentity global _LocalIDURL global _LocalName xmlid = '' filename = bpio.portablePath(settings.LocalIdentityFilename()) if os.path.exists(filename): xmlid = bpio.ReadTextFile(filename) lg.out(6, 'my_id.loadLocalIdentity %d bytes read from\n %s' % (len(xmlid), filename)) if xmlid == '': lg.out(2, "my_id.loadLocalIdentity SKIPPED, local identity in %s is EMPTY !!!" % filename) return lid = identity.identity(xmlsrc=xmlid) if not lid.isCorrect(): lg.out(2, "my_id.loadLocalIdentity ERROR loaded identity is not Correct") return if not lid.Valid(): lg.out(2, "my_id.loadLocalIdentity ERROR loaded identity is not Valid") return setLocalIdentity(lid) # _LocalIdentity = lid # _LocalIDURL = lid.getIDURL() # _LocalName = lid.getIDName() setTransportOrder(getOrderFromContacts(_LocalIdentity)) lg.out(6, "my_id.loadLocalIdentity my name is [%s]" % lid.getIDName())
def _new_idurl_exist(idsrc, new_idurl, pos): if _Debug: lg.out(_DebugLevel, 'id_rotator.doSelectNewIDServer._new_idurl_exist %r already with same name' % new_idurl) latest_revision = my_id.getLocalIdentity().getRevisionValue() try: existing_identity_with_same_name = identity.identity(xmlsrc=idsrc) if not existing_identity_with_same_name.isCorrect(): raise Exception('remote identity not correct at position %r' % pos) if not existing_identity_with_same_name.Valid(): raise Exception('remote identity not valid at position %r' % pos) except: lg.exc() if pos + 1 >= len(target_hosts): self.automat('no-id-servers-found') else: _ping_one_server(pos + 1) return if existing_identity_with_same_name.getPublicKey() == my_id.getLocalIdentity().getPublicKey(): if latest_revision <= existing_identity_with_same_name.getRevisionValue(): self.new_revision = max(self.new_revision or -1, existing_identity_with_same_name.getRevisionValue() + 1) lg.info('found my own identity on "old" ID server and will re-use that source again: %r' % new_idurl) if new_idurl not in self.possible_sources: self.possible_sources.append(new_idurl) self.automat('found-new-id-source', new_idurl) return if pos + 1 >= len(target_hosts): self.automat('no-id-servers-found') else: _ping_one_server(pos + 1)
def _load_routes(self): src = config.conf().getData('services/proxy-server/current-routes') if src is None: lg.warn('setting [services/proxy-server/current-routes] not exist') return try: dct = serialization.BytesToDict(strng.to_bin(src), keys_to_text=True, values_to_text=True) except: dct = {} for k, v in dct.items(): self.routes[strng.to_bin(k)] = v ident = identity.identity(xmlsrc=v['identity']) if not self._is_my_contacts_present_in_identity(ident): if _Debug: lg.out(_DebugLevel, ' DO OVERRIDE identity for %s' % k) identitycache.OverrideIdentity(k, v['identity']) else: if _Debug: lg.out(_DebugLevel, ' skip overriding %s' % k) if _Debug: lg.out( _DebugLevel, 'proxy_router._load_routes %d routes total' % len(self.routes))
def get(url): """ A smart way to get identity from cache. If not cached in memory but found on disk - will cache from disk. """ if has_idurl(url): return idget(url) else: try: partfilename = nameurl.UrlFilename(url) except: lg.out(1, "identitydb.get ERROR %s is incorrect" % str(url)) return None filename = os.path.join(settings.IdentityCacheDir(), partfilename) if not os.path.exists(filename): lg.out(6, "identitydb.get file %s not exist" % os.path.basename(filename)) return None idxml = bpio.ReadTextFile(filename) if idxml: idobj = identity.identity(xmlsrc=idxml) url2 = idobj.getIDURL() if url == url2: idset(url, idobj) return idobj else: lg.out(1, "identitydb.get ERROR url=%s url2=%s" % (url, url2)) return None lg.out(6, "identitydb.get %s not found" % nameurl.GetName(url)) return None
def _check_update_original_identity(self): from logs import lg from lib import misc from main.config import conf from userid import identity orig_ident_xmlsrc = conf().getData( 'services/proxy-transport/my-original-identity', '').strip() current_router_idurl = conf().getString( 'services/proxy-transport/current-router', '').strip() if not orig_ident_xmlsrc: if current_router_idurl: lg.warn( 'current-router is %s, but my-original-identity is empty' % current_router_idurl) self._reset_my_original_identity() return orig_ident = identity.identity(xmlsrc=orig_ident_xmlsrc) if not orig_ident.isCorrect() or not orig_ident.Valid(): lg.warn('my original identity is not valid') self._reset_my_original_identity() return externalIP = misc.readExternalIP() if externalIP and orig_ident.getIP() != externalIP: lg.warn('external IP was changed : restore my original identity') self._reset_my_original_identity() return if not current_router_idurl: lg.warn('original identity is correct, but current router is empty') self._reset_my_original_identity()
def get(url): """ A smart way to get identity from cache. If not cached in memory but found on disk - will cache from disk. """ if has_idurl(url): return idget(url) else: try: partfilename = nameurl.UrlFilename(url) except: lg.out(1, "identitydb.get ERROR %s is incorrect" % str(url)) return None filename = os.path.join(settings.IdentityCacheDir(), partfilename) if not os.path.exists(filename): lg.out( 6, "identitydb.get file %s not exist" % os.path.basename(filename)) return None idxml = bpio.ReadTextFile(filename) if idxml: idobj = identity.identity(xmlsrc=idxml) url2 = idobj.getIDURL() if url == url2: idset(url, idobj) return idobj else: lg.out(1, "identitydb.get ERROR url=%s url2=%s" % (url, url2)) return None lg.out(6, "identitydb.get %s not found" % nameurl.GetName(url)) return None
def loadLocalIdentity(): """ The core method. The file [BitDust data dir]/metadata/localidentity keeps the user identity in XML format. Do read the local file and set into object in memory. """ global _LocalIdentity xmlid = '' filename = bpio.portablePath(settings.LocalIdentityFilename()) if os.path.exists(filename): xmlid = bpio.ReadTextFile(filename) if _Debug: lg.out(_DebugLevel, 'my_id.loadLocalIdentity %d bytes read from %s' % (len(xmlid), filename)) if not xmlid: if _Debug: lg.out(_DebugLevel, "my_id.loadLocalIdentity SKIPPED, local identity in %s is EMPTY !!!" % filename) return False lid = identity.identity(xmlsrc=xmlid) if not lid.isCorrect(): if _Debug: lg.out(_DebugLevel, "my_id.loadLocalIdentity ERROR loaded identity is not Correct") return False if not lid.Valid(): if _Debug: lg.out(_DebugLevel, "my_id.loadLocalIdentity ERROR loaded identity is not Valid") return False setLocalIdentity(lid) setTransportOrder(getOrderFromContacts(_LocalIdentity)) if _Debug: lg.out(_DebugLevel, "my_id.loadLocalIdentity my global id is %s" % getGlobalID()) return True
def get_latest_ident(pub_key): global _KnownUsers from userid import identity user_path = _KnownUsers.get(pub_key) if not user_path: return None user_identity_files = sorted(map(int, os.listdir(user_path))) if len(user_identity_files) == 0: lg.warn('identity history is broken, public key is known, but no identity files found') latest_revision = -1 latest_ident = None known_revisions = set() for_cleanup = [] for id_file in user_identity_files: identity_file_path = os.path.join(user_path, strng.to_text(id_file)) xmlsrc = local_fs.ReadBinaryFile(identity_file_path) one_id_obj = identity.identity(xmlsrc=xmlsrc) if not one_id_obj.isCorrect(): lg.warn('identity history is broken, identity in the file %r is not correct' % identity_file_path) for_cleanup.append(identity_file_path) continue if not one_id_obj.Valid(): lg.warn('identity history is broken, identity in the file %r is not valid' % identity_file_path) for_cleanup.append(identity_file_path) continue if pub_key != one_id_obj.getPublicKey(): lg.err('identity history is broken, public key not matching in the file %r' % identity_file_path) for_cleanup.append(identity_file_path) continue known_revisions.add(one_id_obj.getRevisionValue()) if one_id_obj.getRevisionValue() > latest_revision: latest_revision = one_id_obj.getRevisionValue() latest_ident = one_id_obj return latest_ident
def setUp(self): try: bpio.rmdir_recursive('/tmp/.bitdust_tmp') except Exception: pass lg.set_debug_level(30) settings.init(base_dir='/tmp/.bitdust_tmp') self.my_current_key = None try: os.makedirs('/tmp/.bitdust_tmp/metadata/') except: pass try: os.makedirs('/tmp/.bitdust_tmp/identitycache/') except: pass fout = open('/tmp/_some_priv_key', 'w') fout.write(_some_priv_key) fout.close() fout = open(settings.LocalIdentityFilename(), 'w') fout.write(_some_identity_xml) fout.close() self.assertTrue(key.LoadMyKey(keyfilename='/tmp/_some_priv_key')) self.assertTrue(my_id.loadLocalIdentity()) self.bob_ident = identity.identity(xmlsrc=_another_identity_xml) identitycache.UpdateAfterChecking(idurl=self.bob_ident.getIDURL(), xml_src=_another_identity_xml)
def _do_send_identity_to_router(self, identity_source, failed_event): try: identity_obj = identity.identity(xmlsrc=identity_source) except: lg.exc() return if _Debug: lg.out(_DebugLevel, 'proxy_receiver._do_send_identity_to_router to %s' % self.router_idurl) lg.out(_DebugLevel, ' contacts=%r, sources=%r' % ( identity_obj.contacts, identity_obj.getSources(as_originals=True))) newpacket = signed.Packet( Command=commands.Identity(), OwnerID=my_id.getIDURL(), CreatorID=my_id.getIDURL(), PacketID=('proxy_receiver:%s' % packetid.UniqueID()), Payload=identity_obj.serialize(), RemoteID=self.router_idurl, ) packet_out.create( newpacket, wide=True, callbacks={ commands.Ack(): lambda response, info: self.automat('ack-received', (response, info)), commands.Fail(): lambda x: self.automat(failed_event), None: lambda pkt_out: self.automat('ack-timeout', pkt_out), 'failed': lambda pkt_out, error_message: self.automat('sending-failed', (pkt_out, error_message)), }, keep_alive=True, response_timeout=30, )
def _do_send_identity_to_router(self, identity_source, failed_event): try: identity_obj = identity.identity(xmlsrc=identity_source) except: lg.exc() return if _Debug: lg.out(_DebugLevel, 'proxy_receiver.doSendMyIdentity to %s' % self.router_idurl) lg.out(_DebugLevel, ' contacts=%s, sources=%s' % (identity_obj.contacts, identity_obj.sources)) newpacket = signed.Packet( commands.Identity(), my_id.getLocalID(), my_id.getLocalID(), commands.Identity(), identity_source, self.router_idurl, ) packet_out.create( newpacket, wide=True, callbacks={ commands.Ack(): lambda response, info: self.automat('ack-received', (response, info)), commands.Fail(): lambda x: self.automat(failed_event), }, keep_alive=True, )
def _check_install(self): """ Return True if Private Key and local identity files exists and both is valid. """ if _Debug: lg.out(_DebugLevel, 'initializer._check_install') from userid import identity from crypt import key keyfilename = settings.KeyFileName() keyfilenamelocation = settings.KeyFileNameLocation() if os.path.exists(keyfilenamelocation): keyfilename = bpio.ReadTextFile(keyfilenamelocation) if not os.path.exists(keyfilename): keyfilename = settings.KeyFileName() idfilename = settings.LocalIdentityFilename() if not os.path.exists(keyfilename) or not os.path.exists(idfilename): if _Debug: lg.out(_DebugLevel, 'initializer._check_install local key or local id not exists') return False current_key = bpio.ReadTextFile(keyfilename) current_id = bpio.ReadTextFile(idfilename) if not current_id: if _Debug: lg.out(_DebugLevel, 'initializer._check_install local identity is empty ') return False if not current_key: if _Debug: lg.out(_DebugLevel, 'initializer._check_install private key is empty ') return False try: key.InitMyKey() except: if _Debug: lg.out(_DebugLevel, 'initializer._check_install fail loading private key ') return False try: ident = identity.identity(xmlsrc=current_id) except: if _Debug: lg.out(_DebugLevel, 'initializer._check_install fail init local identity ') return False if not ident.isCorrect(): lg.err('local identity is not correct !!!') return False try: res = ident.Valid() except: if _Debug: lg.out(_DebugLevel, 'failed to validate local identity') return False if not res: lg.err('local identity is not valid !!!') return False if _Debug: lg.out(_DebugLevel, 'initializer._check_install SUCCESS!!!') return True
def test_identity_valid(self): from userid import identity some_identity = identity.identity(xmlsrc=_some_identity_xml) self.assertTrue(some_identity.isCorrect()) self.assertTrue(some_identity.Valid()) self.assertEqual(some_identity.getIDURL().to_bin(), b'http://127.0.0.1:8084/alice.xml') self.assertEqual(some_identity.getIDURL().to_id(), '[email protected]_8084')
def Identity(newpacket, send_ack=True): """ Normal node or Identity server is sending us a new copy of an identity for a contact of ours. Checks that identity is signed correctly. Sending requests to cache all sources (other identity servers) holding that identity. """ # TODO: move to service_gateway newxml = newpacket.Payload newidentity = identity.identity(xmlsrc=newxml) # SECURITY # check that identity is signed correctly # old public key matches new one # this is done in `UpdateAfterChecking()` idurl = newidentity.getIDURL() if not identitycache.HasKey(idurl): lg.info('received new identity: %s' % idurl) if not identitycache.UpdateAfterChecking(idurl, newxml): lg.warn("ERROR has non-Valid identity") return False # Now that we have ID we can check packet if not newpacket.Valid(): # If not valid do nothing lg.warn("not Valid packet from %s" % idurl) return False # TODO: after receiving full list of identity sources we can call ALL OF THEM or those which are not cached yet. # this way we can be sure that even if first source (server holding your public key) is not responding # other sources still can give you required user info: public key, contacts, etc.. # TODO: we can also consolidate few "idurl" sources for every public key - basically identify user by public key # something like: # for source in identitycache.FromCache(idurl).getSources(): # if source not in identitycache.FromCache(idurl): # d = identitycache.immediatelyCaching(source) # d.addCallback(lambda xml_src: identitycache.UpdateAfterChecking(idurl, xml_src)) # d.addErrback(lambda err: lg.warn('caching filed: %s' % err)) if not send_ack: if _Debug: lg.out( _DebugLevel, "p2p_service.Identity idurl=%s skip sending Ack()" % idurl) return True if newpacket.OwnerID == idurl: if _Debug: lg.out( _DebugLevel, "p2p_service.Identity idurl=%s sending wide Ack()" % idurl) else: if _Debug: lg.out( _DebugLevel, "p2p_service.Identity idurl=%s but packet ownerID=%s sending wide Ack()" % ( idurl, newpacket.OwnerID, )) # wide=True : a small trick to respond to all his contacts reactor.callLater(0, SendAck, newpacket, wide=True) # @UndefinedVariable return True
def update(url, xml_src): """ This is a correct method to update an identity in the database. PREPRO need to check that date or version is after old one so not vulnerable to replay attacks. """ try: newid = identity.identity(xmlsrc=xml_src) except: lg.exc() return False if not newid.isCorrect(): lg.out(1, "identitydb.update ERROR: incorrect identity " + str(url)) return False try: if not newid.Valid(): lg.out(1, "identitydb.update ERROR identity not Valid" + str(url)) return False except: lg.exc() return False filename = os.path.join(settings.IdentityCacheDir(), nameurl.UrlFilename(url)) if os.path.exists(filename): oldidentityxml = bpio.ReadTextFile(filename) oldidentity = identity.identity(xmlsrc=oldidentityxml) if oldidentity.publickey != newid.publickey: lg.out(1, "identitydb.update ERROR new publickey does not match old : SECURITY VIOLATION " + url) return False if oldidentity.signature != newid.signature: lg.out(6, 'identitydb.update have new data for ' + nameurl.GetName(url)) else: idset(url, newid) return True bpio.WriteFile(filename, xml_src) # publickeys match so we can update it idset(url, newid) return True
def _check_reset_original_identity(self): from logs import lg from lib import misc from lib import strng from main.config import conf from userid import identity from userid import my_id from userid import id_url orig_ident_xmlsrc = conf().getData( 'services/proxy-transport/my-original-identity', '').strip() current_router_idurl = conf().getString( 'services/proxy-transport/current-router', '').strip() if current_router_idurl: current_router_idurl = id_url.field( current_router_idurl.split(' ')[0]) if not orig_ident_xmlsrc: if current_router_idurl: lg.warn( '"current-router" is %s, but "my-original-identity" is empty' % current_router_idurl) else: lg.warn('"current-router" and "my-original-identity" is empty') self._reset_my_original_identity() return orig_ident = identity.identity(xmlsrc=orig_ident_xmlsrc) if not orig_ident.isCorrect() or not orig_ident.Valid(): lg.warn('"my-original-identity" config has not valid value') self._reset_my_original_identity() return if orig_ident.getIDURL() != my_id.getIDURL(): lg.warn( '"my-original-identity" source is not equal to local identity source' ) self._reset_my_original_identity() return externalIP = strng.to_bin(misc.readExternalIP()) if externalIP and strng.to_bin(orig_ident.getIP()) != externalIP: lg.warn( 'external IP was changed : reset "my-original-identity" config' ) self._reset_my_original_identity() return if not current_router_idurl: lg.warn( '"my-original-identity" config is correct, but current router is empty' ) self._reset_my_original_identity() all_orig_contacts_present_in_local_identity = True for orig_contact in orig_ident.getContacts(): if orig_contact not in my_id.getLocalIdentity().getContacts(): all_orig_contacts_present_in_local_identity = False if all_orig_contacts_present_in_local_identity: lg.warn( 'all of "my-original-identity" contacts is found in local identity: need to RESET!' ) self._reset_my_original_identity()
def doCheckOverride(self, arg): """ Action method. """ if _Debug: lg.out( _DebugLevel, 'proxy_router.doCheckOverride identity for %s' % arg.CreatorID) user_id = arg.CreatorID idsrc = arg.Payload try: new_ident = identity.identity(xmlsrc=idsrc) except: lg.out(_DebugLevel, 'payload: [%s]' % idsrc) lg.exc() return if not new_ident.isCorrect() or not new_ident.Valid(): lg.warn('incoming identity is not valid') return current_overridden_identity = identitycache.ReadOverriddenIdentityXMLSource( user_id) try: current_contacts = identity.identity( xmlsrc=current_overridden_identity).getContacts() except: current_contacts = [] if _Debug: lg.out(_DebugLevel, ' current override contacts is : %s' % current_contacts) identitycache.StopOverridingIdentity(user_id) if not self._is_my_contacts_present_in_identity(new_ident): if _Debug: lg.out(_DebugLevel, ' DO OVERRIDE identity for %s' % arg.CreatorID) lg.out(_DebugLevel, ' new contacts is : %s' % new_ident.getContacts()) identitycache.OverrideIdentity(arg.CreatorID, idsrc) else: if _Debug: lg.out(_DebugLevel, ' SKIP OVERRIDE identity from %s' % arg.CreatorID) lg.out(_DebugLevel, ' new contacts was : %s' % new_ident.getContacts())
def FromCache(idurl): """ Get identity object from cache. """ if IsOverridden(idurl): overridden_xmlsrc = ReadOverriddenIdentityXMLSource(idurl) if overridden_xmlsrc: if _Debug: lg.out(14, ' returning overridden identity (%d bytes) for %s' % (len(overridden_xmlsrc), idurl)) return identity.identity(xmlsrc=overridden_xmlsrc) return identitydb.get(idurl)
def isMyIdentityValid(self, arg): """ Condition method. """ id_from_server = identity.identity(xmlsrc=arg) if not id_from_server.isCorrect(): return False if not id_from_server.Valid(): return False equal = self.new_identity.serialize() == id_from_server.serialize() return equal
def Identity(newpacket): """ Normal node or Identity server is sending us a new copy of an identity for a contact of ours. Checks that identity is signed correctly. Sending requests to cache all sources (other identity servers) holding that identity. """ # TODO: move to service_gateway newxml = newpacket.Payload newidentity = identity.identity(xmlsrc=newxml) # SECURITY # check that identity is signed correctly # old public key matches new one # this is done in `UpdateAfterChecking()` idurl = newidentity.getIDURL() if not identitycache.HasKey(idurl): lg.warn('received new identity: %s' % idurl) if not identitycache.UpdateAfterChecking(idurl, newxml): lg.warn("ERROR has non-Valid identity") return False # Now that we have ID we can check packet if not newpacket.Valid(): # If not valid do nothing lg.warn("not Valid packet from %s" % idurl) return False if newpacket.OwnerID == idurl: # TODO: this needs to be moved to a service # wide=True : a small trick to respond to all contacts if we receive pings if _Debug: lg.out( _DebugLevel, "p2p_service.Identity idurl=%s ... also sent WIDE Acks" % nameurl.GetName(idurl)) else: if _Debug: lg.out( _DebugLevel, "p2p_service.Identity idurl=%s, but packet ownerID=%s ... also sent WIDE Acks" % ( nameurl.GetName(idurl), newpacket.OwnerID, )) reactor.callLater(0, SendAck, newpacket, wide=True) # SendAck(newpacket, wide=True) # TODO: after receiving the full identity sources we can call ALL OF them if some are not cached yet. # this way we can be sure that even if first source (server holding your public key) is not availabble # other sources still can give you required user info: public key, contacts, etc.. # something like: # for source in identitycache.FromCache(idurl).getSources(): # if source not in identitycache.FromCache(idurl): # d = identitycache.immediatelyCaching(source) # d.addCallback(lambda xml_src: identitycache.UpdateAfterChecking(idurl, xml_src)) # d.addErrback(lambda err: lg.warn('caching filed: %s' % err)) return True
def _check_install(self): """ Return True if Private Key and local identity files exists and both is valid. """ lg.out(2, 'initializer._check_install') from userid import identity from crypt import key keyfilename = settings.KeyFileName() keyfilenamelocation = settings.KeyFileNameLocation() if os.path.exists(keyfilenamelocation): keyfilename = bpio.ReadTextFile(keyfilenamelocation) if not os.path.exists(keyfilename): keyfilename = settings.KeyFileName() idfilename = settings.LocalIdentityFilename() if not os.path.exists(keyfilename) or not os.path.exists(idfilename): lg.out( 2, 'initializer._check_install local key or local id not exists') return False current_key = bpio.ReadBinaryFile(keyfilename) current_id = bpio.ReadBinaryFile(idfilename) if current_id == '': lg.out(2, 'initializer._check_install local identity is empty ') return False if current_key == '': lg.out(2, 'initializer._check_install private key is empty ') return False try: key.InitMyKey() except: lg.out(2, 'initializer._check_install fail loading private key ') return False try: ident = identity.identity(xmlsrc=current_id) except: lg.out(2, 'initializer._check_install fail init local identity ') return False try: res = ident.Valid() and ident.isCorrect() except: lg.out( 2, 'initializer._check_install wrong data in local identity ') return False if not res: lg.out(2, 'initializer._check_install local identity is not valid ') return False lg.out(2, 'initializer._check_install done') return True
def doUpdateRouterID(self, *args, **kwargs): """ Action method. """ newpacket, _ = args[0] newxml = newpacket.Payload newidentity = identity.identity(xmlsrc=newxml) cachedidentity = identitycache.FromCache(self.router_idurl) if self.router_idurl != newidentity.getIDURL(): lg.warn('router idurl is unrecognized from response %r != %r' % (self.router_idurl, newidentity.getIDURL(), )) return if newidentity.serialize() != cachedidentity.serialize(): lg.warn('cached identity is not same, router identity changed') self.router_identity = newidentity
def doUpdateRouterID(self, arg): """ Action method. """ newpacket, _ = arg newxml = newpacket.Payload newidentity = identity.identity(xmlsrc=newxml) cachedidentity = identitycache.FromCache(self.router_idurl) if self.router_idurl != newidentity.getIDURL(): lg.warn('router_idurl != newidentity.getIDURL()') return if newidentity.serialize() != cachedidentity.serialize(): lg.warn('cached identity is not same, router identity changed') self.router_identity = newidentity
def doProcessInboxPacket(self, arg): """ Action method. """ newpacket, info, _, _ = arg block = encrypted.Unserialize(newpacket.Payload) if block is None: lg.out(2, 'proxy_receiver.doProcessInboxPacket ERROR reading data from %s' % newpacket.CreatorID) return try: session_key = key.DecryptLocalPrivateKey(block.EncryptedSessionKey) padded_data = key.DecryptWithSessionKey(session_key, block.EncryptedData) inpt = cStringIO.StringIO(padded_data[:int(block.Length)]) data = inpt.read() except: lg.out(2, 'proxy_receiver.doProcessInboxPacket ERROR reading data from %s' % newpacket.CreatorID) lg.exc() try: inpt.close() except: pass return inpt.close() routed_packet = signed.Unserialize(data) if not routed_packet: lg.out(2, 'proxy_receiver.doProcessInboxPacket ERROR unserialize packet failed from %s' % newpacket.CreatorID) return if routed_packet.Command == commands.Identity(): newidentity = identity.identity(xmlsrc=routed_packet.Payload) idurl = newidentity.getIDURL() if not identitycache.HasKey(idurl): lg.warn('received new identity: %s' % idurl) if not identitycache.UpdateAfterChecking(idurl, routed_packet.Payload): lg.warn("ERROR has non-Valid identity") return if not routed_packet.Valid(): lg.out(2, 'proxy_receiver.doProcessInboxPacket ERROR invalid packet from %s' % newpacket.CreatorID) return self.traffic_in += len(data) if _Debug: lg.out(_DebugLevel, '<<<Relay-IN %s from %s://%s with %d bytes' % ( str(routed_packet), info.proto, info.host, len(data))) packet_in.process(routed_packet, info) del block del data del padded_data del inpt del session_key del routed_packet
def doUpdateRouterID(self, arg): """ Action method. """ newpacket, _ = arg newxml = newpacket.Payload newidentity = identity.identity(xmlsrc=newxml) cachedidentity = identitycache.FromCache(self.router_idurl) if self.router_idurl != newidentity.getIDURL(): lg.warn('router_idurl != newidentity.getIDURL()') return if newidentity.serialize() != cachedidentity.serialize(): lg.warn('cached identity is not same') return self.router_identity = newidentity
def _do_check_ping_results(self, ping_results): """ """ self.alive_idurls = [] my_sources = my_id.getLocalIdentity().getSources(as_originals=True) local_revision = my_id.getLocalIdentity().getRevisionValue() latest_revision = -1 pos = -1 for result, remote_identity_src in ping_results: pos += 1 idurl_bin = my_sources[pos] if not result: self.alive_idurls.append(None) continue try: remote_ident = identity.identity(xmlsrc=remote_identity_src) if not remote_ident.isCorrect(): raise Exception( 'remote identity not correct at position %r' % pos) if not remote_ident.Valid(): raise Exception( 'remote identity not valid at position %r' % pos) if latest_revision == -1: latest_revision = remote_ident.getRevisionValue() if latest_revision <= remote_ident.getRevisionValue(): latest_revision = remote_ident.getRevisionValue() except: lg.exc() self.alive_idurls.append(None) continue self.alive_idurls.append(idurl_bin) if not self.new_revision: self.new_revision = max(local_revision, latest_revision) + 1 if _Debug: lg.args(_DebugLevel, new_revision=self.new_revision, alive_idurls=self.alive_idurls) if not self.alive_idurls or not list(filter(None, self.alive_idurls)): # so all my id sources are down # if no alive sources found then probably network is down # and we should not do anything at the moment # but we must also check in that case if any of my ID servers are still alive, but only my identity was removed # otherwise we can get situation when all my id servers are UP, but my identity just expired and id_rotator do nothing to fix that self._fallback_and_ping_my_servers() return self.automat('ping-done', self.alive_idurls)
def isMyIdentityValid(self, *args, **kwargs): """ Condition method. """ id_from_server = identity.identity(xmlsrc=args[0]) if not id_from_server.isCorrect(): lg.warn('my identity is not correct') return False if not id_from_server.Valid(): lg.warn('my identity is not valid') return False if self.new_identity.serialize() != id_from_server.serialize(): lg.warn( 'my identity source is different than copy received from id server' ) return False return True
def _load_routes(self): src = config.conf().getData('services/proxy-server/current-routes') if src is None: lg.warn('setting [services/proxy-server/current-routes] not exist') return try: dct = json.loads(src) except: dct = {} for k, v in dct.items(): self.routes[k] = v ident = identity.identity(xmlsrc=v['identity']) if not self._is_my_contacts_present_in_identity(ident): identitycache.OverrideIdentity(k, v['identity']) else: if _Debug: lg.out(_DebugLevel, ' skip overriding %s' % k) if _Debug: lg.out(_DebugLevel, 'proxy_router._load_routes %d routes total' % len(self.routes))
def _verify(xmlsrc=None): if not xmlsrc: lg.err('my current identity server not healthy') self.NeedPropagate = True self.automat('check-synchronize') return remote_ident = identity.identity(xmlsrc=xmlsrc) if not remote_ident.isCorrect() or not remote_ident.Valid(): lg.warn('my current identity server responded with bad identity file') self.NeedPropagate = True self.automat('check-synchronize') return if remote_ident.getIDURL(as_original=True) != my_idurl: lg.warn('my current identity server responded with unknown identity') self.NeedPropagate = True self.automat('check-synchronize') return if _Debug: lg.dbg(_DebugLevel, 'my current identity server is healthy')
def get_ident(idurl): """ A smart way to get identity from cache. If not cached in memory but found locally - read it from disk. """ idurl = id_url.to_original(idurl) if has_idurl(idurl): return idget(idurl) try: partfilename = nameurl.UrlFilename(idurl) except: if _Debug: lg.out(_DebugLevel, "identitydb.get_ident ERROR %r is incorrect" % idurl) return None if not partfilename: if _Debug: lg.out(_DebugLevel, "identitydb.get_ident ERROR %r is empty" % idurl) return None filename = os.path.join(settings.IdentityCacheDir(), partfilename) if not os.path.exists(filename): if _Debug: lg.out( _DebugLevel, "identitydb.get_ident file %r not exist" % os.path.basename(filename)) return None idxml = bpio.ReadTextFile(filename) if not idxml: if _Debug: lg.out( _DebugLevel, "identitydb.get_ident %s not found" % nameurl.GetName(idurl)) return None idobj = identity.identity(xmlsrc=idxml) idurl_orig = idobj.getIDURL() if idurl == idurl_orig.original(): idset(idurl, idobj) return idobj lg.err("not found identity object idurl=%r idurl_orig=%r" % (idurl, idurl_orig)) return None
def Identity(newpacket): """ Contact or identity server is sending us a new copy of an identity for a contact of ours. Checks that identity is signed correctly. """ newxml = newpacket.Payload newidentity = identity.identity(xmlsrc=newxml) # SECURITY - check that identity is signed correctly # if not newidentity.Valid(): # lg.out(1,"p2p_service.Identity ERROR has non-Valid identity") # return idurl = newidentity.getIDURL() if not identitycache.UpdateAfterChecking(idurl, newxml): lg.warn("ERROR has non-Valid identity") return False # if contacts.isKnown(idurl): # This checks that old public key matches new # identitycache.UpdateAfterChecking(idurl, newxml) # else: # TODO # may be we need to make some temporary storage # for identities who we did not know yet # just to be able to receive packets from them # identitycache.UpdateAfterChecking(idurl, newxml) # Now that we have ID we can check packet if not newpacket.Valid(): # If not valid do nothing lg.warn("not Valid packet from %s" % idurl) # TODO: send Fail ? return False if newpacket.OwnerID == idurl: # wide=True : a small trick to respond to all contacts if we receive pings SendAck(newpacket, wide=True) if _Debug: lg.out(_DebugLevel, "p2p_service.Identity from [%s], sent wide Acks" % nameurl.GetName(idurl)) else: if _Debug: lg.out(_DebugLevel, "p2p_service.Identity from [%s]" % nameurl.GetName(idurl)) return True
def doProcessRequest(self, arg): """ Action method. """ global _MaxRoutesNumber request, _ = arg user_id = request.CreatorID if request.Command == commands.RequestService(): if len(self.routes) >= _MaxRoutesNumber: if _Debug: lg.out( _DebugLevel, 'proxy_server.doProcessRequest RequestService rejected: too many routes' ) lg.out(_DebugLevel, ' %s' % pprint.pformat(self.routes)) p2p_service.SendAck(request, 'rejected', wide=True) else: try: service_info = request.Payload idsrc = service_info.lstrip('service_proxy_server').strip() cached_id = identity.identity(xmlsrc=idsrc) except: lg.out(_DebugLevel, 'payload: [%s]' % request.Payload) lg.exc() return if not cached_id.isCorrect() or not cached_id.Valid(): lg.warn('incoming identity is not valid') return oldnew = '' if user_id not in self.routes.keys(): # accept new route oldnew = 'NEW' self.routes[user_id] = {} else: # accept existing router oldnew = 'OLD' if not self._is_my_contacts_present_in_identity(cached_id): if _Debug: lg.out(_DebugLevel, ' DO OVERRIDE identity for %s' % user_id) identitycache.OverrideIdentity(user_id, idsrc) else: if _Debug: lg.out( _DebugLevel, ' SKIP OVERRIDE identity for %s' % user_id) self.routes[user_id]['time'] = time.time() self.routes[user_id]['identity'] = idsrc self.routes[user_id]['publickey'] = cached_id.publickey self.routes[user_id][ 'contacts'] = cached_id.getContactsAsTuples() self.routes[user_id]['address'] = [] self._write_route(user_id) self.acks.append( p2p_service.SendAck(request, 'accepted', wide=True, packetid=request.PacketID)) if _Debug: lg.out( _DebugLevel, 'proxy_server.doProcessRequest !!!!!!! ACCEPTED %s ROUTE for %s' % (oldnew, user_id)) elif request.Command == commands.CancelService(): if user_id in self.routes: # cancel existing route self._remove_route(user_id) self.routes.pop(user_id) identitycache.StopOverridingIdentity(user_id) p2p_service.SendAck(request, 'accepted', wide=True) if _Debug: lg.out( _DebugLevel, 'proxy_server.doProcessRequest !!!!!!! CANCELLED ROUTE for %s' % user_id) else: p2p_service.SendAck(request, 'rejected', wide=True) if _Debug: lg.out( _DebugLevel, 'proxy_server.doProcessRequest CancelService rejected : %s is not found in routes' % user_id) lg.out(_DebugLevel, ' %s' % pprint.pformat(self.routes)) else: p2p_service.SendFail(request, 'wrong command or payload') # , wide=True)
def buildDefaultIdentity(name='', ip='', idurls=[]): """ Use some local settings and config files to create some new identity. Nice to provide a user name or it will have a form like: [ip_address]_[date]. """ if not ip: ip = misc.readExternalIP() if not name: name = ip.replace('.', '-') + '_' + time.strftime('%M%S') lg.out(4, 'my_id.buildDefaultIdentity: %s %s' % (name, ip)) # create a new identity object # it is stored in memory and another copy on disk drive ident = identity.identity(xmlsrc=identity.default_identity_src) # this is my IDURL address # you can have many IDURL locations for same identity # just need to keep all them synchronized # this is identity propagate procedure, see p2p/propagate.py if len(idurls) == 0: idurls.append(b'http://127.0.0.1/%s.xml' % strng.to_bin(name.lower())) for idurl in idurls: ident.sources.append(strng.to_bin(idurl.strip())) # create a full list of needed transport methods # to be able to accept incoming traffic from other nodes new_contacts, new_order = buildProtoContacts(ident) if len(new_contacts) == 0: if settings.enableTCP() and settings.enableTCPreceiving(): new_contacts['tcp'] = b'tcp://' + strng.to_bin( ip) + b':' + strng.to_bin(str(settings.getTCPPort())) new_order.append('tcp') if settings.enableUDP() and settings.enableUDPreceiving(): _, servername, _, _ = nameurl.UrlParse(ident.sources[0]) new_contacts['udp'] = b'udp://' + strng.to_bin( name.lower()) + b'@' + strng.to_bin(servername) new_order.append('udp') if settings.enableHTTP() and settings.enableHTTPreceiving(): new_contacts['http'] = b'http://' + strng.to_bin( ip) + b':' + strng.to_bin(str(settings.getHTTPPort())) new_order.append('http') # erase current contacts from my identity ident.clearContacts() # add contacts data to the local identity for proto in new_order: contact = new_contacts.get(proto, None) if contact is None: lg.warn('proto %s was not found in contacts' % proto) continue ident.setProtoContact(proto, contact) # set other info # ident.certificates = [] ident.setDate(time.strftime('%b %d, %Y')) ident.setPostage(1) ident.setRevision(0) ident.setVersion('') # TODO: put latest git commit hash here # update software version number # version_number = bpio.ReadTextFile(settings.VersionNumberFile()).strip() # repo, location = misc.ReadRepoLocation() # ident.version = (version_number.strip() + ' ' + repo.strip() + ' ' + bpio.osinfo().strip()).strip() # build a version info # vernum = bpio.ReadTextFile(settings.VersionNumberFile()) # repo, location = misc.ReadRepoLocation() # ident.version = (vernum.strip() + ' ' + repo.strip() + ' ' + bpio.osinfo().strip()).strip() # put my public key in my identity ident.setPublicKey(key.MyPublicKey()) # generate signature ident.sign() # validate new identity if not ident.Valid(): lg.warn('generated identity is not valid !!!') return ident
def doProcessRequest(self, arg): """ Action method. """ global _MaxRoutesNumber request, _ = arg target = request.CreatorID if request.Command == commands.RequestService(): if len(self.routes) >= _MaxRoutesNumber: if _Debug: lg.out(_DebugLevel, 'proxy_server.doProcessRequest RequestService rejected: too many routes') lg.out(_DebugLevel, ' %s' % pprint.pformat(self.routes)) p2p_service.SendAck(request, 'rejected', wide=True) else: try: service_info = request.Payload idsrc = service_info.lstrip('service_proxy_server').strip() cached_id = identity.identity(xmlsrc=idsrc) except: lg.out(_DebugLevel, 'payload: [%s]' % request.Payload) lg.exc() return if not cached_id.isCorrect() or not cached_id.Valid(): lg.warn('incoming identity is not valid') return oldnew = '' if target not in self.routes.keys(): # accept new route oldnew = 'NEW' self.routes[target] = {} else: # accept existing router oldnew = 'OLD' if not self._is_my_contacts_present_in_identity(cached_id): identitycache.OverrideIdentity(target, idsrc) else: if _Debug: lg.out(_DebugLevel, ' skip overriding %s' % target) self.routes[target]['time'] = time.time() self.routes[target]['identity'] = idsrc self.routes[target]['publickey'] = cached_id.publickey self.routes[target]['contacts'] = cached_id.getContactsAsTuples() self.routes[target]['address'] = [] self._write_route(target) self.acks.append( p2p_service.SendAck( request, 'accepted', wide=True, packetid=request.PacketID)) if _Debug: lg.out(_DebugLevel, 'proxy_server.doProcessRequest !!!!!!! ACCEPTED %s ROUTE for %s' % (oldnew, target)) elif request.Command == commands.CancelService(): if target in self.routes: # cancel existing route self._remove_route(target) self.routes.pop(target) identitycache.StopOverridingIdentity(target) p2p_service.SendAck(request, 'accepted', wide=True) if _Debug: lg.out(_DebugLevel, 'proxy_server.doProcessRequest !!!!!!! CANCELLED ROUTE for %s' % target) else: p2p_service.SendAck(request, 'rejected', wide=True) if _Debug: lg.out(_DebugLevel, 'proxy_server.doProcessRequest CancelService rejected : %s is not found in routes' % target) lg.out(_DebugLevel, ' %s' % pprint.pformat(self.routes)) else: p2p_service.SendFail(request, 'wrong command or payload') # , wide=True)
def doVerifyAndRestore(self, arg): global _WorkingKey lg.out(4, 'identity_restorer.doVerifyAndRestore') remote_identity_src = arg if os.path.isfile(settings.KeyFileName()): lg.out(4, 'identity_restorer.doVerifyAndRestore will backup and remove ' + settings.KeyFileName()) bpio.backup_and_remove(settings.KeyFileName()) if os.path.isfile(settings.LocalIdentityFilename()): lg.out(4, 'identity_restorer.doVerifyAndRestore will backup and remove ' + settings.LocalIdentityFilename()) bpio.backup_and_remove(settings.LocalIdentityFilename()) try: remote_ident = identity.identity(xmlsrc=remote_identity_src) local_ident = identity.identity(xmlsrc=remote_identity_src) except: # lg.exc() reactor.callLater(0.1, self.automat, 'restore-failed', ('remote identity have incorrect format', 'red')) return lg.out(4, 'identity_restorer.doVerifyAndRestore checking remote identity') try: res = remote_ident.isCorrect() except: lg.exc() res = False if not res: lg.out(4, 'identity_restorer.doVerifyAndRestore remote identity is not correct FAILED!!!!') reactor.callLater(0.1, self.automat, 'restore-failed', ('remote identity format is not correct', 'red')) return lg.out(4, 'identity_restorer.doVerifyAndRestore validate remote identity') try: res = remote_ident.Valid() except: lg.exc() res = False if not res: lg.out(4, 'identity_restorer.doVerifyAndRestore validate remote identity FAILED!!!!') reactor.callLater(0.1, self.automat, 'restore-failed', ('remote identity is not valid', 'red')) return key.ForgetMyKey() bpio.WriteFile(settings.KeyFileName(), _WorkingKey) try: key.InitMyKey() except: key.ForgetMyKey() # lg.exc() try: os.remove(settings.KeyFileName()) except: pass reactor.callLater(0.1, self.automat, 'restore-failed', ('private key is not valid', 'red')) return try: local_ident.sign() except: # lg.exc() reactor.callLater(0.1, self.automat, 'restore-failed', ('error while signing identity', 'red')) return if remote_ident.signature != local_ident.signature: reactor.callLater(0.1, self.automat, 'restore-failed', ('signature did not match, key verification failed!', 'red')) return my_id.setLocalIdentity(local_ident) my_id.saveLocalIdentity() bpio.WriteFile(settings.UserNameFilename(), my_id.getIDName()) if os.path.isfile(settings.KeyFileName() + '.backup'): lg.out(4, 'identity_restorer.doVerifyAndRestore will remove backup file for ' + settings.KeyFileName()) bpio.remove_backuped_file(settings.KeyFileName()) if os.path.isfile(settings.LocalIdentityFilename() + '.backup'): lg.out(4, 'identity_restorer.doVerifyAndRestore will remove backup file for ' + settings.LocalIdentityFilename()) bpio.remove_backuped_file(settings.LocalIdentityFilename()) reactor.callLater(0.1, self.automat, 'restore-success')