def handleReceivedObject(self, streamNumber, hashid): """Handling received object""" for i in network.connectionpool.BMConnectionPool().connections(): if not i.fullyEstablished: continue try: del i.objectsNewToMe[hashid] except KeyError: if streamNumber in i.streams and ( not Dandelion().hasHash(hashid) or Dandelion().objectChildStem(hashid) == i): with i.objectsNewToThemLock: i.objectsNewToThem[hashid] = time.time() # update stream number, # which we didn't have when we just received the dinv # also resets expiration of the stem mode Dandelion().setHashStream(hashid, streamNumber) if i == self: try: with i.objectsNewToThemLock: del i.objectsNewToThem[hashid] except KeyError: pass self.objectsNewToMe.setLastObject()
def run(self): while not state.shutdown: chunk = [] while True: # Dandelion fluff trigger by expiration Dandelion().expire() try: data = invQueue.get(False) chunk.append((data[0], data[1])) # locally generated if len(data) == 2 or data[2] is None: self.handleLocallyGenerated(data[0], data[1]) except Queue.Empty: break if chunk: for connection in BMConnectionPool().inboundConnections.values() + \ BMConnectionPool().outboundConnections.values(): fluffs = [] stems = [] for inv in chunk: if inv[0] not in connection.streams: continue try: with connection.objectsNewToThemLock: del connection.objectsNewToThem[inv[1]] except KeyError: continue try: if connection == Dandelion().objectChildStem( inv[1]): # Fluff trigger by RNG # auto-ignore if config set to 0, i.e. dandelion is off if randint(1, 100) >= state.dandelion: fluffs.append(inv[1]) # send a dinv only if the stem node supports dandelion elif connection.services & protocol.NODE_DANDELION > 0: stems.append(inv[1]) else: fluffs.append(inv[1]) except KeyError: fluffs.append(inv[1]) if fluffs: shuffle(fluffs) connection.append_write_buf(protocol.CreatePacket('inv', \ addresses.encodeVarint(len(fluffs)) + "".join(fluffs))) if stems: shuffle(stems) connection.append_write_buf(protocol.CreatePacket('dinv', \ addresses.encodeVarint(len(stems)) + "".join(stems))) invQueue.iterate() for i in range(len(chunk)): invQueue.task_done() if Dandelion().refresh < time(): Dandelion().reRandomiseStems() self.stop.wait(1)
def handleLocallyGenerated(self, stream, hashId): Dandelion().addHash(hashId, stream=stream) for connection in BMConnectionPool().inboundConnections.values() + \ BMConnectionPool().outboundConnections.values(): if state.dandelion and connection != Dandelion().objectChildStem(hashId): continue connection.objectsNewToThem[hashId] = time()
def bm_command_getdata(self): items = self.decode_payload_content("l32s") # skip? if time.time() < self.skipUntil: return True #TODO make this more asynchronous helper_random.randomshuffle(items) for i in map(str, items): if Dandelion().hasHash(i) and \ self != Dandelion().objectChildStem(i): self.antiIntersectionDelay() logger.info( '%s asked for a stem object we didn\'t offer to it.', self.destination) break else: try: self.append_write_buf( protocol.CreatePacket('object', Inventory()[i].payload)) except KeyError: self.antiIntersectionDelay() logger.info('%s asked for an object we don\'t have.', self.destination) break # I think that aborting after the first missing/stem object is more secure # when using random reordering, as the recipient won't know exactly which objects we refuse to deliver return True
def bm_command_object(self): """Incoming object, process it""" objectOffset = self.payloadOffset nonce, expiresTime, objectType, version, streamNumber = \ self.decode_payload_content("QQIvv") self.object = BMObject(nonce, expiresTime, objectType, version, streamNumber, self.payload, self.payloadOffset) if len(self.payload) - self.payloadOffset > MAX_OBJECT_PAYLOAD_SIZE: logger.info( 'The payload length of this object is too large (%d bytes).' ' Ignoring it.', len(self.payload) - self.payloadOffset) raise BMProtoExcessiveDataError() try: self.object.checkProofOfWorkSufficient() self.object.checkEOLSanity() self.object.checkAlreadyHave() except (BMObjectExpiredError, BMObjectAlreadyHaveError, BMObjectInsufficientPOWError): BMProto.stopDownloadingObject(self.object.inventoryHash) raise try: self.object.checkStream() except BMObjectUnwantedStreamError: acceptmismatch = BMConfigParser().get("inventory", "acceptmismatch") BMProto.stopDownloadingObject(self.object.inventoryHash, acceptmismatch) if not acceptmismatch: raise try: self.object.checkObjectByType() objectProcessorQueue.put( (self.object.objectType, buffer(self.object.data))) except BMObjectInvalidError: BMProto.stopDownloadingObject(self.object.inventoryHash, True) else: try: del missingObjects[self.object.inventoryHash] except KeyError: pass if self.object.inventoryHash in Inventory() and Dandelion().hasHash( self.object.inventoryHash): Dandelion().removeHash(self.object.inventoryHash, "cycle detection") Inventory()[self.object.inventoryHash] = ( self.object.objectType, self.object.streamNumber, buffer(self.payload[objectOffset:]), self.object.expiresTime, buffer(self.object.tag)) self.handleReceivedObject(self.object.streamNumber, self.object.inventoryHash) invQueue.put((self.object.streamNumber, self.object.inventoryHash, self.destination)) return True
def handleLocallyGenerated(stream, hashId): """Locally generated inventory items require special handling""" Dandelion().addHash(hashId, stream=stream) for connection in BMConnectionPool().connections(): if state.dandelion and connection != \ Dandelion().objectChildStem(hashId): continue connection.objectsNewToThem[hashId] = time()
def run(self): while not self._stopped: uploaded = 0 # Choose uploading peers randomly connections = BMConnectionPool().establishedConnections() helper_random.randomshuffle(connections) for i in connections: now = time.time() # avoid unnecessary delay if i.skipUntil >= now: continue if len(i.write_buf) > self.maxBufSize: continue try: request = i.pendingUpload.randomKeys( RandomTrackingDict.maxPending) except KeyError: continue payload = bytearray() chunk_count = 0 for chunk in request: del i.pendingUpload[chunk] if Dandelion().hasHash(chunk) and \ i != Dandelion().objectChildStem(chunk): i.antiIntersectionDelay() self.logger.info( '%s asked for a stem object we didn\'t offer to it.', i.destination) break try: payload.extend( protocol.CreatePacket('object', Inventory()[chunk].payload)) chunk_count += 1 except KeyError: i.antiIntersectionDelay() self.logger.info( '%s asked for an object we don\'t have.', i.destination) break if not chunk_count: continue i.append_write_buf(payload) self.logger.debug('%s:%i Uploading %i objects', i.destination.host, i.destination.port, chunk_count) uploaded += chunk_count if not uploaded: self.stop.wait(1)
def handleReceivedInventory(self, hashId): if haveBloom: self.invBloom.add(hashId) try: with self.objectsNewToThemLock: del self.objectsNewToThem[hashId] except KeyError: pass # Fluff trigger by cycle detection if hashId not in Inventory() or hashId in Dandelion().hashMap: if hashId in Dandelion().hashMap: Dandelion().fluffTrigger(hashId) if hashId not in missingObjects: missingObjects[hashId] = time.time() with self.objectsNewToMeLock: self.objectsNewToMe[hashId] = True
def sendBigInv(self): def sendChunk(): if objectCount == 0: return logger.debug('Sending huge inv message with %i objects to just this one peer', objectCount) self.append_write_buf(protocol.CreatePacket('inv', addresses.encodeVarint(objectCount) + payload)) # Select all hashes for objects in this stream. bigInvList = {} for stream in self.streams: # may lock for a long time, but I think it's better than thousands of small locks with self.objectsNewToThemLock: for objHash in Inventory().unexpired_hashes_by_stream(stream): # don't advertise stem objects on bigInv if Dandelion().hasHash(objHash): continue bigInvList[objHash] = 0 #self.objectsNewToThem[objHash] = time.time() objectCount = 0 payload = b'' # Now let us start appending all of these hashes together. They will be # sent out in a big inv message to our new peer. for hash, storedValue in bigInvList.items(): payload += hash objectCount += 1 if objectCount >= BMProto.maxObjectCount: sendChunk() payload = b'' objectCount = 0 # flush sendChunk()
def handle_close(self): if self.isOutbound and not self.fullyEstablished: knownnodes.decreaseRating(self.destination) if self.fullyEstablished: UISignalQueue.put(('updateNetworkStatusTab', (self.isOutbound, False, self.destination))) if self.isOutbound: Dandelion().maybeRemoveStem(self) BMProto.handle_close(self)
def checkAlreadyHave(self): """ Check if we already have the object (so that we don't duplicate it in inventory or advertise it unnecessarily) """ # if it's a stem duplicate, pretend we don't have it if Dandelion().hasHash(self.inventoryHash): return if self.inventoryHash in Inventory(): raise BMObjectAlreadyHaveError()
def _command_inv(self, dandelion=False): items = self.decode_payload_content("l32s") if len(items) > MAX_OBJECT_COUNT: logger.error('Too many items in %sinv message!', 'd' if dandelion else '') raise BMProtoExcessiveDataError() # ignore dinv if dandelion turned off if dandelion and not state.dandelion: return True for i in map(str, items): if i in Inventory() and not Dandelion().hasHash(i): continue if dandelion and not Dandelion().hasHash(i): Dandelion().addHash(i, self) self.handleReceivedInventory(i) return True
def handleReceivedObject(self, streamNumber, hashid, connection=None): for i in self.inboundConnections.values( ) + self.outboundConnections.values(): if not isinstance(i, network.bmproto.BMProto): continue if not i.fullyEstablished: continue try: with i.objectsNewToMeLock: del i.objectsNewToMe[hashid] except KeyError: with i.objectsNewToThemLock: i.objectsNewToThem[hashid] = time.time() if i == connection: try: with i.objectsNewToThemLock: del i.objectsNewToThem[hashid] except KeyError: pass if hashid in Dandelion().fluff: Dandelion().removeHash(hashid)
def _command_inv(self, dandelion=False): items = self.decode_payload_content("l32s") if len(items) > BMProto.maxObjectCount: logger.error("Too many items in %sinv message!", "d" if dandelion else "") raise BMProtoExcessiveDataError() else: pass # ignore dinv if dandelion turned off if dandelion and not state.dandelion: return True for i in map(str, items): if i in Inventory() and not Dandelion().hasHash(i): continue if dandelion and not Dandelion().hasHash(i): Dandelion().addHash(i, self) self.handleReceivedInventory(i) return True
def set_connection_fully_established(self): if not self.isOutbound and not self.local: shared.clientHasReceivedIncomingConnections = True UISignalQueue.put(('setStatusIcon', 'green')) UISignalQueue.put(('updateNetworkStatusTab', (self.isOutbound, True, self.destination))) self.antiIntersectionDelay(True) self.fullyEstablished = True if self.isOutbound: knownnodes.increaseRating(self.destination) if self.isOutbound: Dandelion().maybeAddStem(self) self.sendAddr() self.sendBigInv()
def handle_close(self): """Callback for connection being closed.""" host_is_global = self.isOutbound or not self.local and not state.socksIP if self.fullyEstablished: UISignalQueue.put(( 'updateNetworkStatusTab', (self.isOutbound, False, self.destination) )) if host_is_global: knownnodes.addKnownNode( self.streams, self.destination, time.time()) Dandelion().maybeRemoveStem(self) elif host_is_global: knownnodes.decreaseRating(self.destination) BMProto.handle_close(self)
def set_connection_fully_established(self): """Initiate inventory synchronisation.""" if not self.isOutbound and not self.local: state.clientHasReceivedIncomingConnections = True UISignalQueue.put(('setStatusIcon', 'green')) UISignalQueue.put(( 'updateNetworkStatusTab', (self.isOutbound, True, self.destination) )) self.antiIntersectionDelay(True) self.fullyEstablished = True # The connection having host suitable for knownnodes if self.isOutbound or not self.local and not state.socksIP: knownnodes.increaseRating(self.destination) knownnodes.addKnownNode( self.streams, self.destination, time.time()) Dandelion().maybeAddStem(self) self.sendAddr() self.sendBigInv()
def sendBigInv(self): """ Send hashes of all inventory objects, chunked as the protocol has a per-command limit. """ def sendChunk(): """Send one chunk of inv entries in one command""" if objectCount == 0: return logger.debug( 'Sending huge inv message with %i objects to just this' ' one peer', objectCount) self.append_write_buf(protocol.CreatePacket( 'inv', addresses.encodeVarint(objectCount) + payload)) # Select all hashes for objects in this stream. bigInvList = {} for stream in self.streams: # may lock for a long time, but I think it's better than # thousands of small locks with self.objectsNewToThemLock: for objHash in Inventory().unexpired_hashes_by_stream(stream): # don't advertise stem objects on bigInv if Dandelion().hasHash(objHash): continue bigInvList[objHash] = 0 objectCount = 0 payload = b'' # Now let us start appending all of these hashes together. # They will be sent out in a big inv message to our new peer. for obj_hash, _ in bigInvList.items(): payload += obj_hash objectCount += 1 # Remove -1 below when sufficient time has passed for users to # upgrade to versions of PyBitmessage that accept inv with 50,000 # items if objectCount >= MAX_OBJECT_COUNT - 1: sendChunk() payload = b'' objectCount = 0 # flush sendChunk()
def bm_command_dinv(self): """ Dandelion stem announce """ items = self.decode_payload_content("l32s") if len(items) >= BMProto.maxObjectCount: logger.error("Too many items in dinv message!") raise BMProtoExcessiveDataError() else: pass # ignore command if dandelion turned off if BMConfigParser().safeGetBoolean("network", "dandelion") == 0: return True for i in map(str, items): self.handleReceivedInventory(i) Dandelion().addHash(i, self) return True
def start(self): _fixSocket() daemon = BMConfigParser().safeGetBoolean('bitmessagesettings', 'daemon') try: opts, args = getopt.getopt(sys.argv[1:], "hcd", ["help", "curses", "daemon"]) except getopt.GetoptError: self.usage() sys.exit(2) for opt, arg in opts: if opt in ("-h", "--help"): self.usage() sys.exit() elif opt in ("-d", "--daemon"): daemon = True elif opt in ("-c", "--curses"): state.curses = True # is the application already running? If yes then exit. shared.thisapp = singleinstance("", daemon) if daemon: with shared.printLock: print('Running as a daemon. Send TERM signal to end.') self.daemonize() self.setSignalHandler() helper_threading.set_thread_name("PyBitmessage") state.dandelion = BMConfigParser().safeGetInt('network', 'dandelion') # dandelion requires outbound connections, without them, stem objects will get stuck forever if state.dandelion and not BMConfigParser().safeGetBoolean( 'bitmessagesettings', 'sendoutgoingconnections'): state.dandelion = 0 helper_bootstrap.knownNodes() # Start the address generation thread addressGeneratorThread = addressGenerator() addressGeneratorThread.daemon = True # close the main program even if there are threads left addressGeneratorThread.start() # Start the thread that calculates POWs singleWorkerThread = singleWorker() singleWorkerThread.daemon = True # close the main program even if there are threads left singleWorkerThread.start() # Start the SQL thread sqlLookup = sqlThread() sqlLookup.daemon = False # DON'T close the main program even if there are threads left. The closeEvent should command this thread to exit gracefully. sqlLookup.start() Inventory() # init Dandelion( ) # init, needs to be early because other thread may access it early # SMTP delivery thread if daemon and BMConfigParser().safeGet("bitmessagesettings", "smtpdeliver", '') != '': smtpDeliveryThread = smtpDeliver() smtpDeliveryThread.start() # SMTP daemon thread if daemon and BMConfigParser().safeGetBoolean("bitmessagesettings", "smtpd"): smtpServerThread = smtpServer() smtpServerThread.start() # Start the thread that calculates POWs objectProcessorThread = objectProcessor() objectProcessorThread.daemon = False # DON'T close the main program even the thread remains. This thread checks the shutdown variable after processing each object. objectProcessorThread.start() # Start the cleanerThread singleCleanerThread = singleCleaner() singleCleanerThread.daemon = True # close the main program even if there are threads left singleCleanerThread.start() shared.reloadMyAddressHashes() shared.reloadBroadcastSendersForWhichImWatching() if BMConfigParser().safeGetBoolean('bitmessagesettings', 'apienabled'): try: apiNotifyPath = BMConfigParser().get('bitmessagesettings', 'apinotifypath') except: apiNotifyPath = '' if apiNotifyPath != '': with shared.printLock: print('Trying to call', apiNotifyPath) call([apiNotifyPath, "startingUp"]) singleAPIThread = singleAPI() singleAPIThread.daemon = True # close the main program even if there are threads left singleAPIThread.start() BMConnectionPool() asyncoreThread = BMNetworkThread() asyncoreThread.daemon = True asyncoreThread.start() for i in range(BMConfigParser().getint("threads", "receive")): receiveQueueThread = ReceiveQueueThread(i) receiveQueueThread.daemon = True receiveQueueThread.start() announceThread = AnnounceThread() announceThread.daemon = True announceThread.start() state.invThread = InvThread() state.invThread.daemon = True state.invThread.start() state.addrThread = AddrThread() state.addrThread.daemon = True state.addrThread.start() state.downloadThread = DownloadThread() state.downloadThread.daemon = True state.downloadThread.start() connectToStream(1) if BMConfigParser().safeGetBoolean('bitmessagesettings', 'upnp'): import upnp upnpThread = upnp.uPnPThread() upnpThread.start() if daemon == False and BMConfigParser().safeGetBoolean( 'bitmessagesettings', 'daemon') == False: if state.curses == False: if not depends.check_pyqt(): print( 'PyBitmessage requires PyQt unless you want to run it as a daemon and interact with it using the API. You can download PyQt from http://www.riverbankcomputing.com/software/pyqt/download or by searching Google for \'PyQt Download\'. If you want to run in daemon mode, see https://bitmessage.org/wiki/Daemon' ) print( 'You can also run PyBitmessage with the new curses interface by providing \'-c\' as a commandline argument.' ) sys.exit() import bitmessageqt bitmessageqt.run() else: if True: # if depends.check_curses(): print('Running with curses') import bitmessagecurses bitmessagecurses.runwrapper() else: BMConfigParser().remove_option('bitmessagesettings', 'dontconnect') if daemon: while state.shutdown == 0: sleep(1)
def run(self): gc.disable() timeWeLastClearedInventoryAndPubkeysTables = 0 try: shared.maximumLengthOfTimeToBotherResendingMessages = ( float(BMConfigParser().get('bitmessagesettings', 'stopresendingafterxdays')) * 24 * 60 * 60) + (float(BMConfigParser().get( 'bitmessagesettings', 'stopresendingafterxmonths')) * (60 * 60 * 24 * 365) / 12) except: # Either the user hasn't set stopresendingafterxdays and stopresendingafterxmonths yet or the options are missing from the config file. shared.maximumLengthOfTimeToBotherResendingMessages = float('inf') # initial wait if state.shutdown == 0: self.stop.wait(singleCleaner.cycleLength) while state.shutdown == 0: queues.UISignalQueue.put( ('updateStatusBar', 'Doing housekeeping (Flushing inventory in memory to disk...)' )) Inventory().flush() queues.UISignalQueue.put(('updateStatusBar', '')) # If we are running as a daemon then we are going to fill up the UI # queue which will never be handled by a UI. We should clear it to # save memory. if BMConfigParser().safeGetBoolean('bitmessagesettings', 'daemon'): queues.UISignalQueue.queue.clear() if timeWeLastClearedInventoryAndPubkeysTables < int( time.time()) - 7380: timeWeLastClearedInventoryAndPubkeysTables = int(time.time()) Inventory().clean() # pubkeys sqlExecute( '''DELETE FROM pubkeys WHERE time<? AND usedpersonally='no' ''', int(time.time()) - shared.lengthOfTimeToHoldOnToAllPubkeys) # Let us resend getpubkey objects if we have not yet heard a pubkey, and also msg objects if we have not yet heard an acknowledgement queryreturn = sqlQuery( '''select toaddress, ackdata, status FROM sent WHERE ((status='awaitingpubkey' OR status='msgsent') AND folder='sent' AND sleeptill<? AND senttime>?) ''', int(time.time()), int(time.time()) - shared.maximumLengthOfTimeToBotherResendingMessages) for row in queryreturn: if len(row) < 2: logger.error( 'Something went wrong in the singleCleaner thread: a query did not return the requested fields. ' + repr(row)) self.stop.wait(3) break toAddress, ackData, status = row if status == 'awaitingpubkey': resendPubkeyRequest(toAddress) elif status == 'msgsent': resendMsg(ackData) # cleanup old nodes now = int(time.time()) with knownnodes.knownNodesLock: for stream in knownnodes.knownNodes: keys = knownnodes.knownNodes[stream].keys() for node in keys: try: # scrap old nodes if now - knownnodes.knownNodes[stream][node][ "lastseen"] > 2419200: # 28 days shared.needToWriteKnownNodesToDisk = True del knownnodes.knownNodes[stream][node] continue # scrap old nodes with low rating if now - knownnodes.knownNodes[stream][node][ "lastseen"] > 10800 and knownnodes.knownNodes[ stream][node][ "rating"] <= knownnodes.knownNodesForgetRating: shared.needToWriteKnownNodesToDisk = True del knownnodes.knownNodes[stream][node] continue except TypeError: print "Error in %s" % (str(node)) keys = [] # Let us write out the knowNodes to disk if there is anything new to write out. if shared.needToWriteKnownNodesToDisk: try: knownnodes.saveKnownNodes() except Exception as err: if "Errno 28" in str(err): logger.fatal( '(while receiveDataThread knownnodes.needToWriteKnownNodesToDisk) Alert: Your disk or data storage volume is full. ' ) queues.UISignalQueue.put(('alert', ( tr._translate("MainWindow", "Disk full"), tr._translate( "MainWindow", 'Alert: Your disk or data storage volume is full. Bitmessage will now exit.' ), True))) if shared.daemon: os._exit(0) shared.needToWriteKnownNodesToDisk = False # # clear download queues # for thread in threading.enumerate(): # if thread.isAlive() and hasattr(thread, 'downloadQueue'): # thread.downloadQueue.clear() # inv/object tracking for connection in BMConnectionPool().inboundConnections.values( ) + BMConnectionPool().outboundConnections.values(): connection.clean() # dandelion fluff trigger by expiration Dandelion().expire() # discovery tracking exp = time.time() - singleCleaner.expireDiscoveredPeers reaper = (k for k, v in state.discoveredPeers.items() if v < exp) for k in reaper: try: del state.discoveredPeers[k] except KeyError: pass # TODO: cleanup pending upload / download gc.collect() if state.shutdown == 0: self.stop.wait(singleCleaner.cycleLength)
def start(self): _fixSocket() daemon = BMConfigParser().safeGetBoolean('bitmessagesettings', 'daemon') try: opts, args = getopt.getopt(sys.argv[1:], "hcdt", ["help", "curses", "daemon", "test"]) except getopt.GetoptError: self.usage() sys.exit(2) for opt, arg in opts: if opt in ("-h", "--help"): self.usage() sys.exit() elif opt in ("-d", "--daemon"): daemon = True state.enableGUI = False # run without a UI elif opt in ("-c", "--curses"): state.curses = True elif opt in ("-t", "--test"): state.testmode = True if os.path.isfile(os.path.join(state.appdata, 'unittest.lock')): daemon = True state.enableGUI = False # run without a UI # Fallback: in case when no api command was issued state.last_api_response = datetime.now() # Apply special settings config = BMConfigParser() config.set('bitmessagesettings', 'apienabled', 'true') config.set('bitmessagesettings', 'apiusername', 'username') config.set('bitmessagesettings', 'apipassword', 'password') config.set( 'bitmessagesettings', 'apinotifypath', os.path.join(app_dir, 'tests', 'apinotify_handler.py')) # is the application already running? If yes then exit. shared.thisapp = singleinstance("", daemon) if daemon: with shared.printLock: print('Running as a daemon. Send TERM signal to end.') self.daemonize() self.setSignalHandler() helper_threading.set_thread_name("PyBitmessage") state.dandelion = BMConfigParser().safeGetInt('network', 'dandelion') # dandelion requires outbound connections, without them, stem objects will get stuck forever if state.dandelion and not BMConfigParser().safeGetBoolean( 'bitmessagesettings', 'sendoutgoingconnections'): state.dandelion = 0 helper_bootstrap.knownNodes() # Not needed if objproc is disabled if state.enableObjProc: # Start the address generation thread addressGeneratorThread = addressGenerator() addressGeneratorThread.daemon = True # close the main program even if there are threads left addressGeneratorThread.start() # Start the thread that calculates POWs singleWorkerThread = singleWorker() singleWorkerThread.daemon = True # close the main program even if there are threads left singleWorkerThread.start() # Start the SQL thread sqlLookup = sqlThread() sqlLookup.daemon = False # DON'T close the main program even if there are threads left. The closeEvent should command this thread to exit gracefully. sqlLookup.start() Inventory() # init Dandelion( ) # init, needs to be early because other thread may access it early # Enable object processor and SMTP only if objproc enabled if state.enableObjProc: # SMTP delivery thread if daemon and BMConfigParser().safeGet("bitmessagesettings", "smtpdeliver", '') != '': smtpDeliveryThread = smtpDeliver() smtpDeliveryThread.start() # SMTP daemon thread if daemon and BMConfigParser().safeGetBoolean( "bitmessagesettings", "smtpd"): smtpServerThread = smtpServer() smtpServerThread.start() # Start the thread that calculates POWs objectProcessorThread = objectProcessor() objectProcessorThread.daemon = False # DON'T close the main program even the thread remains. This thread checks the shutdown variable after processing each object. objectProcessorThread.start() # Start the cleanerThread singleCleanerThread = singleCleaner() singleCleanerThread.daemon = True # close the main program even if there are threads left singleCleanerThread.start() # Not needed if objproc disabled if state.enableObjProc: shared.reloadMyAddressHashes() shared.reloadBroadcastSendersForWhichImWatching() # API is also objproc dependent if BMConfigParser().safeGetBoolean('bitmessagesettings', 'apienabled'): singleAPIThread = singleAPI() singleAPIThread.daemon = True # close the main program even if there are threads left singleAPIThread.start() # start network components if networking is enabled if state.enableNetwork: BMConnectionPool() asyncoreThread = BMNetworkThread() asyncoreThread.daemon = True asyncoreThread.start() for i in range(BMConfigParser().getint("threads", "receive")): receiveQueueThread = ReceiveQueueThread(i) receiveQueueThread.daemon = True receiveQueueThread.start() announceThread = AnnounceThread() announceThread.daemon = True announceThread.start() state.invThread = InvThread() state.invThread.daemon = True state.invThread.start() state.addrThread = AddrThread() state.addrThread.daemon = True state.addrThread.start() state.downloadThread = DownloadThread() state.downloadThread.daemon = True state.downloadThread.start() connectToStream(1) if BMConfigParser().safeGetBoolean('bitmessagesettings', 'upnp'): import upnp upnpThread = upnp.uPnPThread() upnpThread.start() else: # Populate with hardcoded value (same as connectToStream above) state.streamsInWhichIAmParticipating.append(1) if daemon is False and state.enableGUI: # FIXME redundant? if state.curses is False: if not depends.check_pyqt(): sys.exit( 'PyBitmessage requires PyQt unless you want' ' to run it as a daemon and interact with it' ' using the API. You can download PyQt from ' 'http://www.riverbankcomputing.com/software/pyqt/download' ' or by searching Google for \'PyQt Download\'.' ' If you want to run in daemon mode, see ' 'https://bitmessage.org/wiki/Daemon\n' 'You can also run PyBitmessage with' ' the new curses interface by providing' ' \'-c\' as a commandline argument.') import bitmessageqt bitmessageqt.run() else: if True: # if depends.check_curses(): print('Running with curses') import bitmessagecurses bitmessagecurses.runwrapper() else: BMConfigParser().remove_option('bitmessagesettings', 'dontconnect') if daemon: while state.shutdown == 0: sleep(1) if state.testmode and 30 <= \ (datetime.now() - state.last_api_response).seconds: self.stop() elif not state.enableGUI: from tests import core test_core_result = core.run() state.enableGUI = True self.stop() sys.exit('Core tests failed!' if test_core_result.errors or test_core_result.failures else 0)
def checkAlreadyHave(self): # if it's a stem duplicate, pretend we don't have it if self.inventoryHash in Dandelion().hashMap: return if self.inventoryHash in Inventory(): raise BMObjectAlreadyHaveError()
def start(self): _fixSocket() daemon = BMConfigParser().safeGetBoolean( 'bitmessagesettings', 'daemon') try: opts, args = getopt.getopt( sys.argv[1:], "hcdtsa", ["help", "curses", "daemon", "test", "apitest"]) except getopt.GetoptError: self.usage() sys.exit(2) for opt, arg in opts: if opt in ("-h", "--help"): self.usage() sys.exit() elif opt in ("-d", "--daemon"): daemon = True state.enableGUI = False # run without a UI elif opt in ("-c", "--curses"): state.curses = True elif opt in ("-t", "--test"): state.testmode = daemon = True state.enableGUI = False # run without a UI elif opt in ("-a", "--apitest"): from bmsxmlrpc.server import TestAPIThread server = TestAPIThread() server.start() # is the application already running? If yes then exit. if state.enableGUI and not state.curses and not depends.check_pyqt(): sys.exit( 'PyBitmessage requires PyQt unless you want' ' to run it as a daemon and interact with it' ' using the API. You can download PyQt from ' 'http://www.riverbankcomputing.com/software/pyqt/download' ' or by searching Google for \'PyQt Download\'.' ' If you want to run in daemon mode, see ' 'https://bitmessage.org/wiki/Daemon\n' 'You can also run PyBitmessage with' ' the new curses interface by providing' ' \'-c\' as a commandline argument.' ) shared.thisapp = singleinstance("", daemon) if daemon and not state.testmode: with shared.printLock: logger.warn('Running as a daemon. Send TERM signal to end.') self.daemonize() self.setSignalHandler() helper_threading.set_thread_name("PyBitmessage") state.dandelion = BMConfigParser().safeGetInt('network', 'dandelion') # dandelion requires outbound connections, without them, # stem objects will get stuck forever if state.dandelion and not BMConfigParser().safeGetBoolean( 'bitmessagesettings', 'sendoutgoingconnections'): state.dandelion = 0 knownnodes.readKnownNodes() # Not needed if objproc is disabled if state.enableObjProc: # Start the address generation thread addressGeneratorThread = addressGenerator() # close the main program even if there are threads left addressGeneratorThread.daemon = True addressGeneratorThread.start() # Start the thread that calculates POWs singleWorkerThread = singleWorker() # close the main program even if there are threads left singleWorkerThread.daemon = True singleWorkerThread.start() # Start the SQL thread sqlLookup = sqlThread() # DON'T close the main program even if there are threads left. # The closeEvent should command this thread to exit gracefully. sqlLookup.daemon = False sqlLookup.start() Inventory() # init # init, needs to be early because other thread may access it early Dandelion() # Enable object processor and SMTP only if objproc enabled if state.enableObjProc: # SMTP delivery thread if daemon and BMConfigParser().safeGet( "bitmessagesettings", "smtpdeliver", '') != '': from class_smtpDeliver import smtpDeliver smtpDeliveryThread = smtpDeliver() smtpDeliveryThread.start() # SMTP daemon thread if daemon and BMConfigParser().safeGetBoolean( "bitmessagesettings", "smtpd"): from class_smtpServer import smtpServer smtpServerThread = smtpServer() smtpServerThread.start() # Start the thread that calculates POWs objectProcessorThread = objectProcessor() # DON'T close the main program even the thread remains. # This thread checks the shutdown variable after processing # each object. objectProcessorThread.daemon = False objectProcessorThread.start() # Start the cleanerThread singleCleanerThread = singleCleaner() # close the main program even if there are threads left singleCleanerThread.daemon = True singleCleanerThread.start() # Not needed if objproc disabled if state.enableObjProc: shared.reloadMyAddressHashes() shared.reloadBroadcastSendersForWhichImWatching() # API is also objproc dependent if BMConfigParser().safeGetBoolean( 'bitmessagesettings', 'apienabled'): try: apiNotifyPath = BMConfigParser().get( 'bitmessagesettings', 'apinotifypath') except: apiNotifyPath = '' if apiNotifyPath != '': with shared.printLock: print('Trying to call', apiNotifyPath) call([apiNotifyPath, "startingUp"]) from bmsxmlrpc.server import singleAPI singleAPIThread = singleAPI() # close the main program even if there are threads left singleAPIThread.daemon = True singleAPIThread.start() # start network components if networking is enabled if state.enableNetwork: BMConnectionPool() asyncoreThread = BMNetworkThread() asyncoreThread.daemon = True asyncoreThread.start() for i in range(BMConfigParser().getint("threads", "receive")): receiveQueueThread = ReceiveQueueThread(i) receiveQueueThread.daemon = True receiveQueueThread.start() announceThread = AnnounceThread() announceThread.daemon = True announceThread.start() state.invThread = InvThread() state.invThread.daemon = True state.invThread.start() state.addrThread = AddrThread() state.addrThread.daemon = True state.addrThread.start() state.downloadThread = DownloadThread() state.downloadThread.daemon = True state.downloadThread.start() connectToStream(1) if BMConfigParser().safeGetBoolean( 'bitmessagesettings', 'upnp'): import upnp upnpThread = upnp.uPnPThread() upnpThread.start() else: # Populate with hardcoded value (same as connectToStream above) state.streamsInWhichIAmParticipating.append(1) if not daemon and state.enableGUI: if state.curses: if not depends.check_curses(): sys.exit() logger.info('Running with curses') import bitmessagecurses bitmessagecurses.runwrapper() elif state.kivy: BMConfigParser().remove_option('bitmessagesettings', 'dontconnect') from bitmessagekivy.mpybit import NavigateApp NavigateApp().run() else: import bitmessageqt bitmessageqt.run() else: BMConfigParser().remove_option('bitmessagesettings', 'dontconnect') if daemon: if state.testmode: sleep(30) # make testing self.stop() while state.shutdown == 0: sleep(1)
def reRandomiseDandelionStems(self): # Choose 2 peers randomly # TODO: handle streams Dandelion().reRandomiseStems(self.outboundConnections.values())
def run(self): while not state.shutdown: chunk = [] while True: try: data = invQueue.get(False) chunk.append((data[0], data[1])) # locally generated if len(data) == 2: Dandelion().addHash(data[1], None) BMConnectionPool().handleReceivedObject(data[0], data[1]) # came over the network else: source = BMConnectionPool().getConnectionByAddr(data[2]) BMConnectionPool().handleReceivedObject(data[0], data[1], source) except Queue.Empty: break # connection not found, handle it as if generated locally except KeyError: BMConnectionPool().handleReceivedObject(data[0], data[1]) if chunk: for connection in BMConnectionPool().inboundConnections.values() + \ BMConnectionPool().outboundConnections.values(): fluffs = [] stems = [] for inv in chunk: if inv[0] not in connection.streams: continue try: with connection.objectsNewToThemLock: del connection.objectsNewToThem[inv[1]] except KeyError: continue try: if connection == Dandelion().hashMap[inv[1]]: # Fluff trigger by RNG # auto-ignore if config set to 0, i.e. dandelion is off # send a normal inv if stem node doesn't support dandelion if randint(1, 100) < BMConfigParser().safeGetBoolean("network", "dandelion") and \ connection.services | protocol.NODE_DANDELION > 0: stems.append(inv[1]) else: fluffs.append(inv[1]) except KeyError: fluffs.append(inv[1]) if fluffs: shuffle(fluffs) connection.append_write_buf(protocol.CreatePacket('inv', \ addresses.encodeVarint(len(fluffs)) + "".join(fluffs))) if stems: shuffle(stems) connection.append_write_buf(protocol.CreatePacket('dinv', \ addresses.encodeVarint(len(stems)) + "".join(stems))) invQueue.iterate() for i in range(len(chunk)): invQueue.task_done() if Dandelion().refresh < time(): BMConnectionPool().reRandomiseDandelionStems() self.stop.wait(1)