def doReportSuccess(self, *args, **kwargs): """ Action method. """ try: min_port = min( [addr[1] for addr in list(self.stun_results.values())]) max_port = max( [addr[1] for addr in list(self.stun_results.values())]) my_ip = strng.to_text(list(self.stun_results.values())[0][0]) if min_port == max_port: result = ('stun-success', 'non-symmetric', my_ip, min_port) else: result = ('stun-success', 'symmetric', my_ip, self.stun_results) self.my_address = (my_ip, min_port) except: lg.exc() result = ('stun-failed', None, None, []) self.my_address = None if self.my_address: bpio.WriteTextFile(settings.ExternalIPFilename(), self.my_address[0]) bpio.WriteTextFile(settings.ExternalUDPPortFilename(), str(self.my_address[1])) if _Debug: lg.out( _DebugLevel, 'stun_client.doReportSuccess based on %d nodes: %s' % (len(self.stun_results), str(self.my_address))) if _Debug: lg.out(_DebugLevel + 10, ' %s' % str(result)) for cb in self.callbacks: cb(result[0], result[1], result[2], result[3]) self.callbacks = []
def register_key(key_id, key_object_or_string, keys_folder=None): """ """ if key_id in known_keys(): lg.warn('key %s already exists' % key_id) return None if isinstance(key_object_or_string, six.string_types): lg.out( 4, 'my_keys.register_key %s from %d bytes openssh_input_string' % (key_id, len(key_object_or_string))) key_object = unserialize_key_to_object(key_object_or_string) if not key_object: lg.warn( 'invalid openssh string, unserialize_key_to_object() failed') return None else: lg.out(4, 'my_keys.register_key %s from object' % key_id) key_object = key_object_or_string known_keys()[key_id] = key_object if not keys_folder: keys_folder = settings.KeyStoreDir() if key_object.isPublic(): key_string = key_object.toPublicString() key_filepath = os.path.join(keys_folder, key_id + '.public') bpio.WriteTextFile(key_filepath, key_string) else: key_string = key_object.toPrivateString() key_filepath = os.path.join(keys_folder, key_id + '.private') bpio.WriteTextFile(key_filepath, key_string) if _Debug: lg.out(_DebugLevel, ' key %s added, saved to %s' % (key_id, key_filepath)) return key_filepath
def SendServers(): """ My identity file can be stored in different locations, see the "sources" field. So I can use different identity servers to store more secure. This method will send my identity file to all my identity servers via transport_tcp. """ from transport.tcp import tcp_node sendfile, sendfilename = tmpfile.make("propagate") os.close(sendfile) LocalIdentity = my_id.getLocalIdentity() bpio.WriteTextFile(sendfilename, LocalIdentity.serialize()) dlist = [] for idurl in LocalIdentity.sources: # sources for out identity are servers we need to send to protocol, host, port, filename = nameurl.UrlParse(idurl) # if host == settings.IdentityServerName(): # host = '67.207.147.183' webport, tcpport = known_servers.by_host().get( host, (settings.IdentityWebPort(), settings.IdentityServerPort())) # srvhost = '%s:%d' % (host, int(tcpport)) dlist.append( tcp_node.send(sendfilename, (host, int(tcpport)), 'Identity', keep_alive=False)) # dlist.append(gateway.send_file_single('tcp', srvhost, sendfilename, 'Identity')) dl = DeferredList(dlist, consumeErrors=True) return dl
def save_key(key_id, keys_folder=None): """ """ key_object = known_keys()[key_id] if key_object is None: lg.warn('can not save key %s because it is not loaded yet' % key_id) return False if not keys_folder: keys_folder = settings.KeyStoreDir() if key_object.isPublic(): key_filepath = os.path.join(keys_folder, key_id + '.public') key_dict = key_object.toDict(include_private=False) key_string = jsn.dumps(key_dict, indent=1, separators=(',', ':')) else: key_filepath = os.path.join(keys_folder, key_id + '.private') key_dict = key_object.toDict(include_private=True) key_string = jsn.dumps(key_dict, indent=1, separators=(',', ':')) if not bpio.WriteTextFile(key_filepath, key_string): lg.warn('failed saving key %r to %r' % ( key_id, key_filepath, )) return False if _Debug: lg.out( _DebugLevel, 'my_keys.save_key stored key %r locally in %r' % ( key_id, key_filepath, )) return True
def SendServers(): """ My identity file can be stored in different locations, see the "sources" field. So I can use different identity servers to store more secure and reliable. This method will send my identity file to all my identity servers via transport_tcp. """ from transport.tcp import tcp_node sendfile, sendfilename = tmpfile.make("propagate") os.close(sendfile) LocalIdentity = my_id.getLocalIdentity() bpio.WriteTextFile(sendfilename, LocalIdentity.serialize(as_text=True)) dlist = [] for idurl in LocalIdentity.getSources(as_originals=True): # sources for out identity are servers we need to send to protocol, host, port, filename = nameurl.UrlParse(idurl) # TODO: rebuild identity-server logic to be able to send my identity via HTTP POST instead of TCP and # get rid of second TCP port at all webport, tcpport = known_servers.by_host().get(host, ( # by default use "expected" port numbers settings.IdentityWebPort(), settings.IdentityServerPort())) normalized_address = net_misc.normalize_address((host, int(tcpport), )) dlist.append(tcp_node.send( sendfilename, normalized_address, 'Identity', keep_alive=False, )) if _Debug: lg.args(_DebugLevel, normalized_address=normalized_address, filename=filename) dl = DeferredList(dlist, consumeErrors=True) return dl
def WriteIndex(filepath=None, encoding='utf-8'): """ Write index data base to the local file .bitdust/metadata/index. """ global _LoadingFlag if _LoadingFlag: return if filepath is None: filepath = settings.BackupIndexFilePath() json_data = {} for customer_idurl in backup_fs.known_customers(): customer_id = customer_idurl.to_id() json_data[customer_id] = backup_fs.Serialize( iterID=backup_fs.fsID(customer_idurl), to_json=True, encoding=encoding, ) src = '%d\n' % revision() src += jsn.dumps( json_data, indent=1, separators=(',', ':'), encoding=encoding, ) if _Debug: lg.args(_DebugLevel, size=len(src), filepath=filepath) return bpio.WriteTextFile(filepath, src)
def _create_new_identity(self): """ Generate new Private key and new identity file. Reads some extra info from config files. """ login = strng.to_bin(bpio.ReadTextFile(settings.UserNameFilename())) externalIP = strng.to_bin(misc.readExternalIP()) or b'127.0.0.1' if self.free_idurls[0].count(b'127.0.0.1'): externalIP = b'127.0.0.1' lg.out( 4, 'id_registrator._create_new_identity %s %s ' % (login, externalIP)) key.InitMyKey() if not key.isMyKeyReady(): key.GenerateNewKey() lg.out(4, ' my key is ready') ident = my_id.buildDefaultIdentity(name=login, ip=externalIP, idurls=self.free_idurls) # my_id.rebuildLocalIdentity( # identity_object=ident, revision_up=True, save_identity=False) # localIP = bpio.ReadTextFile(settings.LocalIPFilename()) my_identity_xmlsrc = ident.serialize(as_text=True) newfilename = settings.LocalIdentityFilename() + '.new' bpio.WriteTextFile(newfilename, my_identity_xmlsrc) self.new_identity = ident lg.out( 4, ' wrote %d bytes to %s' % (len(my_identity_xmlsrc), newfilename))
def saveLocalIdentity(): """ Save identity object from memory into local file. Do sign the identity than serialize to write to the file. """ global _LocalIdentity if not isLocalIdentityReady(): lg.warn("ERROR local identity not exist!") return False if not _LocalIdentity.isCorrect(): lg.warn('local identity is not correct') return False _LocalIdentity.sign() if not _LocalIdentity.Valid(): lg.err('local identity is not valid') return False xmlid = _LocalIdentity.serialize(as_text=True) filename = bpio.portablePath(settings.LocalIdentityFilename()) bpio.WriteTextFile(filename, xmlid) setTransportOrder(getOrderFromContacts(_LocalIdentity)) events.send('local-identity-written', data=dict(idurl=_LocalIdentity.getIDURL(), filename=filename)) if _Debug: lg.out(_DebugLevel, "my_id.saveLocalIdentity %d bytes wrote to %s" % (len(xmlid), filename)) return True
def _create_new_identity(self): """ Generate new Private key and new identity file. Reads some extra info from config files. """ login = strng.to_bin(bpio.ReadTextFile(settings.UserNameFilename())) externalIP = strng.to_bin(misc.readExternalIP()) or b'127.0.0.1' if self.free_idurls[0].count(b'127.0.0.1'): externalIP = b'127.0.0.1' lg.out( 4, 'id_registrator._create_new_identity %s %s ' % (login, externalIP)) my_id.forgetLocalIdentity() my_id.eraseLocalIdentity(do_backup=True) key.ForgetMyKey(erase_file=True, do_backup=True) key.InitMyKey() if not key.isMyKeyReady(): key.GenerateNewKey() lg.out(4, ' my key is ready') ident = my_id.buildDefaultIdentity(name=login, ip=externalIP, idurls=self.free_idurls) my_identity_xmlsrc = ident.serialize(as_text=True) newfilename = settings.LocalIdentityFilename() + '.new' bpio.WriteTextFile(newfilename, my_identity_xmlsrc) try: id_url.identity_cached(ident) except: lg.exc() self.new_identity = ident lg.out( 4, ' wrote %d bytes to %s' % (len(my_identity_xmlsrc), newfilename))
def WriteIndex(filepath=None, encoding='utf-8'): """ Write index data base to the local file .bitdust/metadata/index. """ global _LoadingFlag if _LoadingFlag: return if filepath is None: filepath = settings.BackupIndexFilePath() json_data = {} # json_data = backup_fs.Serialize(to_json=True, encoding=encoding) for customer_idurl in backup_fs.known_customers(): customer_id = global_id.UrlToGlobalID(customer_idurl) json_data[customer_id] = backup_fs.Serialize( iterID=backup_fs.fsID(customer_idurl), to_json=True, encoding=encoding, ) src = '%d\n' % revision() src += json.dumps(json_data, indent=1, separators=(',', ':'), encoding=encoding) if _Debug: import pprint lg.out(_DebugLevel, pprint.pformat(json_data)) return bpio.WriteTextFile(filepath, src)
def doSaveMyName(self, *args, **kwargs): """ Action method. """ login = kwargs['username'] self.preferred_servers = [ s.strip() for s in kwargs.get('preferred_servers', []) ] if not self.known_servers: self.known_servers = known_servers.by_host() if not self.preferred_servers: try: for srv in str(config.conf().getData( 'services/identity-propagate/preferred-servers') ).split(','): if srv.strip(): self.preferred_servers.append(srv.strip()) except: pass self.min_servers = max( settings.MinimumIdentitySources(), config.conf().getInt('services/identity-propagate/min-servers') or settings.MinimumIdentitySources()) self.max_servers = min( settings.MaximumIdentitySources(), config.conf().getInt('services/identity-propagate/max-servers') or settings.MaximumIdentitySources()) lg.out(4, 'id_registrator.doSaveMyName [%s]' % login) lg.out(4, ' known_servers=%s' % self.known_servers) lg.out(4, ' preferred_servers=%s' % self.preferred_servers) lg.out(4, ' min_servers=%s' % self.min_servers) lg.out(4, ' max_servers=%s' % self.max_servers) bpio.WriteTextFile(settings.UserNameFilename(), login)
def state_changed(self, oldstate, newstate, event, *args, **kwargs): """ This method intended to catch the moment when automat's state were changed. """ if newstate in ['CONNECTED', 'DISCONNECTED', 'NO_SERVICE']: supplierPath = settings.SupplierPath( self.supplier_idurl, customer_idurl=self.customer_idurl) if not os.path.isdir(supplierPath): try: os.makedirs(supplierPath) except: lg.exc() return bpio.WriteTextFile( settings.SupplierServiceFilename( self.supplier_idurl, customer_idurl=self.customer_idurl), newstate, ) if newstate == 'CONNECTED': if not self._supplier_connected_event_sent: self._supplier_connected_event_sent = True events.send('supplier-connected', data=dict( supplier_idurl=self.supplier_idurl, customer_idurl=self.customer_idurl, needed_bytes=self.needed_bytes, key_id=self.key_id, )) if newstate in [ 'DISCONNECTED', 'NO_SERVICE', ]: self._supplier_connected_event_sent = False
def doDetectLocalIP(self, *args, **kwargs): """ Action method. """ localip = net_misc.getLocalIp() bpio.WriteTextFile(settings.LocalIPFilename(), localip) lg.out(4, 'id_registrator.doDetectLocalIP [%s]' % localip) self.automat('local-ip-detected')
def save(result): lg.out(4, ' external IP : %s' % result) if result['result'] != 'stun-success': self.automat('stun-failed') return ip = result['ip'] bpio.WriteTextFile(settings.ExternalIPFilename(), ip) self.automat('stun-success', ip)
def writeSupplierData(supplier_idurl, filename, data, customer_idurl): """ Writes to a config file for given supplier. """ dirPath = settings.SupplierPath(supplier_idurl, customer_idurl) if not os.path.isdir(dirPath): os.makedirs(dirPath) path = settings.SupplierPath(supplier_idurl, customer_idurl, filename) return bpio.WriteTextFile(path, data)
def init(): """ Got a filename for today, check if already exists, read today file, start counting. """ global CountTimeIn global CountTimeOut lg.out(4, 'bandwidth.init') fin = filenameIN() fout = filenameOUT() if not os.path.isfile(fin): bpio.WriteTextFile(fin, '') if not os.path.isfile(fout): bpio.WriteTextFile(fout, '') read_bandwidthIN() read_bandwidthOUT() CountTimeIn = time.time() CountTimeOut = time.time() reactor.addSystemEventTrigger('before', 'shutdown', save) # @UndefinedVariable
def doReportSuccess(self, *args, **kwargs): """ Action method. """ try: min_port = min( [addr[1] for addr in list(self.stun_results.values())]) max_port = max( [addr[1] for addr in list(self.stun_results.values())]) my_ip = strng.to_text(list(self.stun_results.values())[0][0]) if min_port == max_port: result = ('stun-success', 'non-symmetric', my_ip, min_port) else: result = ('stun-success', 'symmetric', my_ip, self.stun_results) self.my_address = (my_ip, min_port) except: lg.exc() result = ('stun-failed', None, None, []) self.my_address = None if _Debug: lg.out( _DebugLevel, 'stun_client.doReportSuccess based on %d nodes: %r' % ( len(self.stun_results), result, )) if self.my_address: current_external_ip = misc.readExternalIP() if current_external_ip != self.my_address[0]: events.send('my-external-ip-changed', data=dict( old=current_external_ip, new=self.my_address[0], )) bpio.WriteTextFile(settings.ExternalIPFilename(), self.my_address[0]) bpio.WriteTextFile(settings.ExternalUDPPortFilename(), str(self.my_address[1])) for cb in self.callbacks: cb(result[0], result[1], result[2], result[3]) self.callbacks = []
def update(idurl, xml_src): """ This is a correct method to update an identity in the local cache. PREPRO need to check that date or version is after old one so not vulnerable to replay attacks. """ idurl = strng.to_bin(idurl) try: newid = identity.identity(xmlsrc=xml_src) except: lg.exc() return False if not newid.isCorrect(): lg.out(1, "identitydb.update ERROR incorrect identity : %r" % idurl) return False try: if not newid.Valid(): lg.out(1, "identitydb.update ERROR identity not Valid : %r" % idurl) return False except: lg.exc() return False filename = os.path.join(settings.IdentityCacheDir(), nameurl.UrlFilename(idurl)) if os.path.exists(filename): oldidentityxml = bpio.ReadTextFile(filename) oldidentity = identity.identity(xmlsrc=oldidentityxml) if oldidentity.publickey != newid.publickey: lg.out( 1, "identitydb.update ERROR new publickey does not match old, SECURITY VIOLATION : %r" % idurl) return False if oldidentity.signature != newid.signature: lg.out( 6, 'identitydb.update have new data for %r' % nameurl.GetName(idurl)) else: idset(idurl, newid) return True # publickeys match so we can update it bpio.WriteTextFile(filename, xml_src) idset(idurl, newid) return True
def save_latest_local_key_id(keys_folder=None): global _LatestLocalKeyID if not keys_folder: keys_folder = settings.KeyStoreDir() latest_local_key_id_filepath = os.path.join(keys_folder, 'latest_local_key_id') if not bpio.WriteTextFile(latest_local_key_id_filepath, '{}'.format(_LatestLocalKeyID)): lg.warn('failed saving latest_local_key_id to %r' % latest_local_key_id_filepath) return False return True
def setTransportOrder(orderL): """ Validate transports and save the list in the [BitDust data dir]\metadata\torder. It is useful to remember the priority of used transports. """ orderl = orderL orderL = validateTransports(orderL) orderTxt = ' '.join(orderl) lg.out(8, 'my_id.setTransportOrder: ' + str(orderTxt)) bpio.WriteTextFile(settings.DefaultTransportOrderFilename(), orderTxt)
def update(idurl, xml_src): """ This is a correct method to update an identity in the local cache. PREPRO need to check that date or version is after old one so not vulnerable to replay attacks. """ idurl = id_url.to_original(idurl) try: newid = identity.identity(xmlsrc=xml_src) except: lg.exc() return False if not newid.isCorrect(): lg.err("incorrect identity : %r" % idurl) return False try: if not newid.Valid(): lg.err("identity not valid : %r" % idurl) return False except: lg.exc() return False filename = os.path.join(settings.IdentityCacheDir(), nameurl.UrlFilename(idurl)) if os.path.exists(filename): oldidentityxml = bpio.ReadTextFile(filename) oldidentity = identity.identity(xmlsrc=oldidentityxml) if oldidentity.publickey != newid.publickey: # TODO: SECURITY add some kind of black list to be able to block certain IP's if the DDoS me lg.err( "new public key does not match with old, SECURITY VIOLATION : %r" % idurl) return False if oldidentity.signature != newid.signature: if _Debug: lg.out( _DebugLevel, 'identitydb.update have new data for %r' % nameurl.GetName(idurl)) else: idset(idurl, newid) return True # publickeys match so we can update it bpio.WriteTextFile(filename, xml_src) idset(idurl, newid) return True
def init(json_rpc_port=None): global _JsonRPCServer if _Debug: lg.out(4, 'api_jsonrpc_server.init') if _JsonRPCServer: lg.warn('already started') return from main import settings from system import bpio if not json_rpc_port: json_rpc_port = settings.getJsonRPCServerPort() bpio.WriteTextFile(settings.LocalJsonRPCPortFilename(), str(json_rpc_port)) # TODO: add protection: accept connections only from local host: 127.0.0.1 _JsonRPCServer = reactor.listenTCP(json_rpc_port, server.Site(BitDustJsonRPCServer())) # @UndefinedVariable if _Debug: lg.out(4, ' started on port %d' % json_rpc_port)
def step0(): lg.out(4, 'os_windows_update.step0') global _UpdatingInProgress if _UpdatingInProgress: lg.out(6, 'os_windows_update.step0 _UpdatingInProgress is True, skip.') return repo, locationURL = misc.ReadRepoLocation() src = bpio.ReadTextFile(settings.RepoFile()) if src == '': bpio.WriteTextFile(settings.RepoFile(), u'%s\n%s' % (repo, locationURL)) _UpdatingInProgress = True d = download_version() d.addCallback(step1) d.addErrback(fail)
def state_changed(self, oldstate, newstate, event, *args, **kwargs): """ This method intended to catch the moment when automat's state were changed. """ if newstate in ['CONNECTED', 'DISCONNECTED', 'NO_SERVICE']: supplierPath = settings.SupplierPath(self.supplier_idurl, customer_idurl=self.customer_idurl) if not os.path.isdir(supplierPath): try: os.makedirs(supplierPath) except: lg.exc() return bpio.WriteTextFile( settings.SupplierServiceFilename(self.supplier_idurl, customer_idurl=self.customer_idurl), newstate, )
def GenerateNewKey(keyfilename=None): global _MyKeyObject if keyfilename is None: keyfilename = settings.KeyFileName() if os.path.exists(keyfilename + '_location'): newkeyfilename = bpio.ReadTextFile(keyfilename + '_location').strip() if os.path.exists(newkeyfilename): keyfilename = newkeyfilename if _Debug: lg.out(_DebugLevel, 'key.InitMyKey generate new private key') _MyKeyObject = rsa_key.RSAKey() _MyKeyObject.generate(settings.getPrivateKeySize()) keystring = _MyKeyObject.toPrivateString() bpio.WriteTextFile(keyfilename, keystring) if _Debug: lg.out(_DebugLevel, ' wrote %d bytes to %s' % (len(keystring), keyfilename)) del keystring gc.collect()
def generate_key(key_id, key_size=4096, keys_folder=None): """ """ if key_id in known_keys(): lg.warn('key "%s" already exists' % key_id) return None lg.out(4, 'my_keys.generate_key "%s" of %d bits' % (key_id, key_size)) key_object = rsa_key.RSAKey() key_object.generate(key_size) known_keys()[key_id] = key_object if not keys_folder: keys_folder = settings.KeyStoreDir() key_string = key_object.toPrivateString() key_filepath = os.path.join(keys_folder, key_id + '.private') bpio.WriteTextFile(key_filepath, key_string) if _Debug: lg.out(_DebugLevel, ' key %s generated, saved to %s' % (key_id, key_filepath)) return key_object
def saveLocalIdentity(): """ Save identity object from memory into local file. Do sign the identity than serialize to write to the file. """ global _LocalIdentity if not isLocalIdentityReady(): lg.warn("ERROR local identity not exist!") return if not _LocalIdentity.isCorrect(): lg.warn('local identity is not correct') return _LocalIdentity.sign() xmlid = _LocalIdentity.serialize() filename = bpio.portablePath(settings.LocalIdentityFilename()) bpio.WriteTextFile(filename, xmlid) lg.out( 6, "my_id.saveLocalIdentity %d bytes wrote to %s" % (len(xmlid), filename))
def init(ftp_port=None): global _FTPServer lg.out(4, 'ftp_server.init') if _FTPServer: lg.warn('already started') return if not ftp_port: ftp_port = settings.getFTPServerPort() if not os.path.isfile(settings.FTPServerCredentialsFile()): bpio.WriteTextFile(settings.FTPServerCredentialsFile(), 'bitdust:bitdust') # TODO: add protection: accept connections only from local host: 127.0.0.1 _FTPServer = reactor.listenTCP( ftp_port, BitDustFTPFactory( Portal(FTPRealm('./'), [ AllowAnonymousAccess(), FilePasswordDB(settings.FTPServerCredentialsFile()), ]), )) lg.out(4, ' started on port %d' % ftp_port)
def save_keys_local(keys_folder=None): """ """ if not keys_folder: keys_folder = settings.KeyStoreDir() if _Debug: lg.out( _DebugLevel, 'my_keys.save_keys_local will store all known keys in %s' % keys_folder) count = 0 for key_id, key_object in known_keys().items(): if key_object.isPublic(): key_filepath = os.path.join(keys_folder, key_id + '.public') else: key_filepath = os.path.join(keys_folder, key_id + '.private') key_string = key_object.toPrivateString() bpio.WriteTextFile(key_filepath, key_string) count += 1 if _Debug: lg.out(_DebugLevel, ' %d keys saved' % count)
def _create_new_identity(self): """ Generate new Private key and new identity file. Reads some extra info from config files. """ login = strng.to_bin(bpio.ReadTextFile(settings.UserNameFilename())) externalIP = strng.to_bin(misc.readExternalIP()) or '127.0.0.1' lg.out( 4, 'id_registrator._create_new_identity %s %s ' % (login, externalIP)) key.InitMyKey() lg.out(4, ' my key is ready') ident = my_id.buildDefaultIdentity(name=login, ip=externalIP, idurls=self.free_idurls) # localIP = bpio.ReadTextFile(settings.LocalIPFilename()) my_identity_xmlsrc = ident.serialize() newfilename = settings.LocalIdentityFilename() + '.new' bpio.WriteTextFile(newfilename, my_identity_xmlsrc) self.new_identity = ident lg.out( 4, ' wrote %d bytes to %s' % (len(my_identity_xmlsrc), newfilename))