def decodeWalletImportFormat(WIFstring):
# pylint: disable=inconsistent-return-statements
"""
Convert private key from base58 that's used in the config file to
8-bit binary string
"""
fullString = arithmetic.changebase(WIFstring, 58, 256)
privkey = fullString[:-4]
if fullString[-4:] != \
hashlib.sha256(hashlib.sha256(privkey).digest()).digest()[:4]:
logger.critical(
'Major problem! When trying to decode one of your'
' private keys, the checksum failed. Here are the first'
' 6 characters of the PRIVATE key: %s',
str(WIFstring)[:6])
os._exit(0) # pylint: disable=protected-access
# return ""
elif privkey[0] == '\x80': # checksum passed
return privkey[1:]
logger.critical(
'Major problem! When trying to decode one of your private keys,'
' the checksum passed but the key doesn\'t begin with hex 80.'
' Here is the PRIVATE key: %s', WIFstring)
os._exit(0) # pylint: disable=protected-access
def run(self):
while True:
objectType, data = queues.objectProcessorQueue.get()
self.checkackdata(data)
try:
if objectType == 0: # getpubkey
self.processgetpubkey(data)
elif objectType == 1: # pubkey
self.processpubkey(data)
elif objectType == 2: # msg
self.processmsg(data)
elif objectType == 3: # broadcast
self.processbroadcast(data)
# is more of a command, not an object type. Is used to get
# this thread past the queue.get() so that it will check
# the shutdown variable.
elif objectType == 'checkShutdownVariable':
pass
else:
if isinstance(objectType, int):
logger.info(
'Don\'t know how to handle object type 0x%08X',
objectType)
else:
logger.info('Don\'t know how to handle object type %s',
objectType)
except helper_msgcoding.DecompressionSizeException as e:
logger.error(
'The object is too big after decompression (stopped'
' decompressing at %ib, your configured limit %ib).'
' Ignoring', e.size,
BMConfigParser().safeGetInt("zlib", "maxsize"))
except varintDecodeError as e:
logger.debug(
'There was a problem with a varint while processing an'
' object. Some details: %s', e)
except Exception:
logger.critical(
'Critical error within objectProcessorThread: \n',
exc_info=True)
if state.shutdown:
# Wait just a moment for most of the connections to close
time.sleep(.5)
numberOfObjectsThatWereInTheObjectProcessorQueue = 0
with SqlBulkExecute() as sql:
while queues.objectProcessorQueue.curSize > 0:
objectType, data = queues.objectProcessorQueue.get()
sql.execute(
'INSERT INTO objectprocessorqueue VALUES (?,?)',
objectType, data)
numberOfObjectsThatWereInTheObjectProcessorQueue += 1
logger.debug(
'Saved %s objects from the objectProcessorQueue to'
' disk. objectProcessorThread exiting.',
numberOfObjectsThatWereInTheObjectProcessorQueue)
state.shutdown = 2
break
def run(self):
while True:
objectType, data = shared.objectProcessorQueue.get()
try:
if objectType == 0: # getpubkey
self.processgetpubkey(data)
elif objectType == 1: #pubkey
self.processpubkey(data)
elif objectType == 2: #msg
self.processmsg(data)
elif objectType == 3: #broadcast
self.processbroadcast(data)
elif objectType == 'checkShutdownVariable': # is more of a command, not an object type. Is used to get this thread past the queue.get() so that it will check the shutdown variable.
pass
else:
logger.critical('Error! Bug! The class_objectProcessor was passed an object type it doesn\'t recognize: %s' % str(objectType))
except varintDecodeError as e:
logger.debug("There was a problem with a varint while processing an object. Some details: %s" % e)
except Exception as e:
logger.critical("Critical error within objectProcessorThread: \n%s" % traceback.format_exc())
if shared.shutdown:
time.sleep(.5) # Wait just a moment for most of the connections to close
numberOfObjectsThatWereInTheObjectProcessorQueue = 0
with SqlBulkExecute() as sql:
while shared.objectProcessorQueue.curSize > 0:
objectType, data = shared.objectProcessorQueue.get()
sql.execute('''INSERT INTO objectprocessorqueue VALUES (?,?)''',
objectType,data)
numberOfObjectsThatWereInTheObjectProcessorQueue += 1
logger.debug('Saved %s objects from the objectProcessorQueue to disk. objectProcessorThread exiting.' % str(numberOfObjectsThatWereInTheObjectProcessorQueue))
shared.shutdown = 2
break
def lookupAppdataFolder():
APPNAME = "PyBitmessage"
from os import path, environ
if sys.platform == 'darwin':
if "HOME" in environ:
dataFolder = path.join(os.environ["HOME"], "Library/Application Support/", APPNAME) + '/'
else:
logger.critical('Could not find home folder, please report this message and your '
'OS X version to the BitMessage Github.')
sys.exit()
elif 'win32' in sys.platform or 'win64' in sys.platform:
dataFolder = path.join(environ['APPDATA'], APPNAME) + '\\'
else:
from shutil import move
try:
dataFolder = path.join(environ["XDG_CONFIG_HOME"], APPNAME)
except KeyError:
dataFolder = path.join(environ["HOME"], ".config", APPNAME)
# Migrate existing data to the proper location if this is an existing install
try:
logger.info("Moving data folder to %s" % (dataFolder))
move(path.join(environ["HOME"], ".%s" % APPNAME), dataFolder)
except IOError:
pass
dataFolder = dataFolder + '/'
return dataFolder
def run(self):
while True:
objectType, data = shared.objectProcessorQueue.get()
try:
if objectType == 0: # getpubkey
self.processgetpubkey(data)
elif objectType == 1: #pubkey
self.processpubkey(data)
elif objectType == 2: #msg
self.processmsg(data)
elif objectType == 3: #broadcast
self.processbroadcast(data)
elif objectType == 'checkShutdownVariable': # is more of a command, not an object type. Is used to get this thread past the queue.get() so that it will check the shutdown variable.
pass
else:
logger.critical('Error! Bug! The class_objectProcessor was passed an object type it doesn\'t recognize: %s' % str(objectType))
except varintDecodeError as e:
logger.debug("There was a problem with a varint while processing an object. Some details: %s" % e)
except Exception as e:
logger.critical("Critical error within objectProcessorThread: \n%s" % traceback.format_exc())
if shared.shutdown:
time.sleep(.5) # Wait just a moment for most of the connections to close
numberOfObjectsThatWereInTheObjectProcessorQueue = 0
with SqlBulkExecute() as sql:
while shared.objectProcessorQueue.curSize > 0:
objectType, data = shared.objectProcessorQueue.get()
sql.execute('''INSERT INTO objectprocessorqueue VALUES (?,?)''',
objectType,data)
numberOfObjectsThatWereInTheObjectProcessorQueue += 1
logger.debug('Saved %s objects from the objectProcessorQueue to disk. objectProcessorThread exiting.' % str(numberOfObjectsThatWereInTheObjectProcessorQueue))
shared.shutdown = 2
break
def checkAndShareObjectWithPeers(data):
"""
This function is called after either receiving an object off of the wire
or after receiving one as ackdata.
Returns the length of time that we should reserve to process this message
if we are receiving it off of the wire.
"""
if len(data) > 2**18:
logger.info(
'The payload length of this object is too large (%s bytes). Ignoring it.',
len(data))
return 0
# Let us check to make sure that the proof of work is sufficient.
if not isProofOfWorkSufficient(data):
logger.info('Proof of work is insufficient.')
return 0
endOfLifeTime, = unpack('>Q', data[8:16])
if endOfLifeTime - int(
time.time()
) > 28 * 24 * 60 * 60 + 10800: # The TTL may not be larger than 28 days + 3 hours of wiggle room
logger.info(
'This object\'s End of Life time is too far in the future. Ignoring it. Time is %s',
endOfLifeTime)
return 0
if endOfLifeTime - int(time.time(
)) < -3600: # The EOL time was more than an hour ago. That's too much.
logger.info(
'This object\'s End of Life time was more than an hour ago. Ignoring the object. Time is %s',
endOfLifeTime)
return 0
intObjectType, = unpack('>I', data[16:20])
try:
if intObjectType == 0:
_checkAndShareGetpubkeyWithPeers(data)
return 0.1
elif intObjectType == 1:
_checkAndSharePubkeyWithPeers(data)
return 0.1
elif intObjectType == 2:
_checkAndShareMsgWithPeers(data)
return 0.6
elif intObjectType == 3:
_checkAndShareBroadcastWithPeers(data)
return 0.6
else:
_checkAndShareUndefinedObjectWithPeers(data)
return 0.6
except varintDecodeError as e:
logger.debug(
"There was a problem with a varint while checking to see whether it was appropriate to share an object with peers. Some details: %s",
e)
except Exception as e:
logger.critical(
'There was a problem while checking to see whether it was appropriate to share an object with peers. This is definitely a bug! \n%s',
traceback.format_exc())
return 0
def checkAndShareObjectWithPeers(data):
"""
This function is called after either receiving an object off of the wire
or after receiving one as ackdata.
Returns the length of time that we should reserve to process this message
if we are receiving it off of the wire.
"""
if len(data) > 2 ** 18:
logger.info('The payload length of this object is too large (%s bytes). Ignoring it.', len(data))
return 0
# Let us check to make sure that the proof of work is sufficient.
if not isProofOfWorkSufficient(data):
logger.info('Proof of work is insufficient.')
return 0
endOfLifeTime, = unpack('>Q', data[8:16])
if endOfLifeTime - int(time.time()) > 28 * 24 * 60 * 60 + 10800: # The TTL may not be larger than 28 days + 3 hours of wiggle room
logger.info('This object\'s End of Life time is too far in the future. Ignoring it. Time is %s', endOfLifeTime)
return 0
if endOfLifeTime - int(time.time()) < - 3600: # The EOL time was more than an hour ago. That's too much.
logger.info('This object\'s End of Life time was more than an hour ago. Ignoring the object. Time is %s', endOfLifeTime)
return 0
intObjectType, = unpack('>I', data[16:20])
try:
if intObjectType == 0:
_checkAndShareGetpubkeyWithPeers(data)
return 0.1
elif intObjectType == 1:
_checkAndSharePubkeyWithPeers(data)
return 0.1
elif intObjectType == 2:
_checkAndShareMsgWithPeers(data)
return 0.6
elif intObjectType == 3:
_checkAndShareBroadcastWithPeers(data)
return 0.6
else:
_checkAndShareUndefinedObjectWithPeers(data)
return 0.6
except varintDecodeError as e:
logger.debug("There was a problem with a varint while checking to see whether it was appropriate to share an object with peers. Some details: %s", e)
except Exception as e:
logger.critical('There was a problem while checking to see whether it was appropriate to share an object with peers. This is definitely a bug! \n%s', traceback.format_exc())
return 0
def decodeWalletImportFormat(WIFstring):
fullString = arithmetic.changebase(WIFstring, 58, 256)
privkey = fullString[:-4]
if fullString[-4:] != \
hashlib.sha256(hashlib.sha256(privkey).digest()).digest()[:4]:
logger.critical(
'Major problem! When trying to decode one of your'
' private keys, the checksum failed. Here are the first'
' 6 characters of the PRIVATE key: %s',
str(WIFstring)[:6])
os._exit(0)
# return ""
elif privkey[0] == '\x80': # checksum passed
return privkey[1:]
logger.critical(
'Major problem! When trying to decode one of your private keys,'
' the checksum passed but the key doesn\'t begin with hex 80.'
' Here is the PRIVATE key: %s', WIFstring)
os._exit(0)
def run(self):
while True:
objectType, data = queues.objectProcessorQueue.get()
self.checkackdata(data)
try:
if objectType == 0: # getpubkey
self.processgetpubkey(data)
elif objectType == 1: #pubkey
self.processpubkey(data)
elif objectType == 2: #msg
self.processmsg(data)
elif objectType == 3: #broadcast
self.processbroadcast(data)
elif objectType == 'checkShutdownVariable': # is more of a command, not an object type. Is used to get this thread past the queue.get() so that it will check the shutdown variable.
pass
else:
if isinstance(objectType, int):
logger.info('Don\'t know how to handle object type 0x%08X', objectType)
else:
logger.info('Don\'t know how to handle object type %s', objectType)
except helper_msgcoding.DecompressionSizeException as e:
logger.error("The object is too big after decompression (stopped decompressing at %ib, your configured limit %ib). Ignoring", e.size, BMConfigParser().safeGetInt("zlib", "maxsize"))
except varintDecodeError as e:
logger.debug("There was a problem with a varint while processing an object. Some details: %s" % e)
except Exception as e:
logger.critical("Critical error within objectProcessorThread: \n%s" % traceback.format_exc())
if state.shutdown:
time.sleep(.5) # Wait just a moment for most of the connections to close
numberOfObjectsThatWereInTheObjectProcessorQueue = 0
with SqlBulkExecute() as sql:
while queues.objectProcessorQueue.curSize > 0:
objectType, data = queues.objectProcessorQueue.get()
sql.execute('''INSERT INTO objectprocessorqueue VALUES (?,?)''',
objectType,data)
numberOfObjectsThatWereInTheObjectProcessorQueue += 1
logger.debug('Saved %s objects from the objectProcessorQueue to disk. objectProcessorThread exiting.' % str(numberOfObjectsThatWereInTheObjectProcessorQueue))
state.shutdown = 2
break
def decryptAndCheckPubkeyPayload(data, address):
"""
Version 4 pubkeys are encrypted. This function is run when we already have the
address to which we want to try to send a message. The 'data' may come either
off of the wire or we might have had it already in our inventory when we tried
to send a msg to this particular address.
"""
try:
status, addressVersion, streamNumber, ripe = decodeAddress(address)
readPosition = 20 # bypass the nonce, time, and object type
embeddedAddressVersion, varintLength = decodeVarint(
data[readPosition:readPosition + 10])
readPosition += varintLength
embeddedStreamNumber, varintLength = decodeVarint(
data[readPosition:readPosition + 10])
readPosition += varintLength
storedData = data[
20:
readPosition] # We'll store the address version and stream number (and some more) in the pubkeys table.
if addressVersion != embeddedAddressVersion:
logger.info(
'Pubkey decryption was UNsuccessful due to address version mismatch.'
)
return 'failed'
if streamNumber != embeddedStreamNumber:
logger.info(
'Pubkey decryption was UNsuccessful due to stream number mismatch.'
)
return 'failed'
tag = data[readPosition:readPosition + 32]
readPosition += 32
signedData = data[
8:
readPosition] # the time through the tag. More data is appended onto signedData below after the decryption.
encryptedData = data[readPosition:]
# Let us try to decrypt the pubkey
toAddress, cryptorObject = state.neededPubkeys[tag]
if toAddress != address:
logger.critical(
'decryptAndCheckPubkeyPayload failed due to toAddress mismatch. This is very peculiar. toAddress: %s, address %s',
toAddress, address)
# the only way I can think that this could happen is if someone encodes their address data two different ways.
# That sort of address-malleability should have been caught by the UI or API and an error given to the user.
return 'failed'
try:
decryptedData = cryptorObject.decrypt(encryptedData)
except:
# Someone must have encrypted some data with a different key
# but tagged it with a tag for which we are watching.
logger.info('Pubkey decryption was unsuccessful.')
return 'failed'
readPosition = 0
bitfieldBehaviors = decryptedData[readPosition:readPosition + 4]
readPosition += 4
publicSigningKey = '\x04' + decryptedData[readPosition:readPosition +
64]
readPosition += 64
publicEncryptionKey = '\x04' + decryptedData[
readPosition:readPosition + 64]
readPosition += 64
specifiedNonceTrialsPerByte, specifiedNonceTrialsPerByteLength = decodeVarint(
decryptedData[readPosition:readPosition + 10])
readPosition += specifiedNonceTrialsPerByteLength
specifiedPayloadLengthExtraBytes, specifiedPayloadLengthExtraBytesLength = decodeVarint(
decryptedData[readPosition:readPosition + 10])
readPosition += specifiedPayloadLengthExtraBytesLength
storedData += decryptedData[:readPosition]
signedData += decryptedData[:readPosition]
signatureLength, signatureLengthLength = decodeVarint(
decryptedData[readPosition:readPosition + 10])
readPosition += signatureLengthLength
signature = decryptedData[readPosition:readPosition + signatureLength]
if highlevelcrypto.verify(signedData, signature,
hexlify(publicSigningKey)):
logger.info(
'ECDSA verify passed (within decryptAndCheckPubkeyPayload)')
else:
logger.info(
'ECDSA verify failed (within decryptAndCheckPubkeyPayload)')
return 'failed'
sha = hashlib.new('sha512')
sha.update(publicSigningKey + publicEncryptionKey)
ripeHasher = hashlib.new('ripemd160')
ripeHasher.update(sha.digest())
embeddedRipe = ripeHasher.digest()
if embeddedRipe != ripe:
# Although this pubkey object had the tag were were looking for and was
# encrypted with the correct encryption key, it doesn't contain the
# correct pubkeys. Someone is either being malicious or using buggy software.
logger.info(
'Pubkey decryption was UNsuccessful due to RIPE mismatch.')
return 'failed'
# Everything checked out. Insert it into the pubkeys table.
logger.info(
'within decryptAndCheckPubkeyPayload, addressVersion: %s, streamNumber: %s \n\
ripe %s\n\
publicSigningKey in hex: %s\n\
publicEncryptionKey in hex: %s', addressVersion,
streamNumber, hexlify(ripe), hexlify(publicSigningKey),
hexlify(publicEncryptionKey))
t = (address, addressVersion, storedData, int(time.time()), 'yes')
sqlExecute('''INSERT INTO pubkeys VALUES (?,?,?,?,?)''', *t)
return 'successful'
except varintDecodeError as e:
logger.info(
'Pubkey decryption was UNsuccessful due to a malformed varint.')
return 'failed'
except Exception as e:
logger.critical(
'Pubkey decryption was UNsuccessful because of an unhandled exception! This is definitely a bug! \n%s',
traceback.format_exc())
return 'failed'
from class_addressGenerator import addressGenerator
from debug import logger
# Helper Functions
import helper_bootstrap
import helper_generic
from subprocess import call
import time
# OSX python version check
import sys
if 'win' in sys.platform:
if float("{1}.{2}".format(*sys.version_info)) < 7.5:
msg = "You should use python 2.7.5 or greater. Your version: %s", "{0}.{1}.{2}".format(*sys.version_info)
logger.critical(msg)
print msg
sys.exit(0)
def connectToStream(streamNumber):
shared.streamsInWhichIAmParticipating[streamNumber] = 'no data'
selfInitiatedConnections[streamNumber] = {}
shared.inventorySets[streamNumber] = set()
queryData = sqlQuery('''SELECT hash FROM inventory WHERE streamnumber=?''', streamNumber)
for row in queryData:
shared.inventorySets[streamNumber].add(row[0])
if isOurOperatingSystemLimitedToHavingVeryFewHalfOpenConnections():
# Some XP and Vista systems can only have 10 outgoing connections at a time.
maximumNumberOfHalfOpenConnections = 9
# Helper Functions
import helper_bootstrap
import helper_generic
from subprocess import call
import time
# OSX python version check
import sys
if sys.platform == 'darwin':
if float("{1}.{2}".format(*sys.version_info)) < 7.5:
msg = "You should use python 2.7.5 or greater. Your version: %s", "{0}.{1}.{2}".format(
*sys.version_info)
logger.critical(msg)
print msg
sys.exit(0)
def connectToStream(streamNumber):
shared.streamsInWhichIAmParticipating[streamNumber] = 'no data'
selfInitiatedConnections[streamNumber] = {}
shared.inventorySets[streamNumber] = set()
queryData = sqlQuery('''SELECT hash FROM inventory WHERE streamnumber=?''',
streamNumber)
for row in queryData:
shared.inventorySets[streamNumber].add(row[0])
if isOurOperatingSystemLimitedToHavingVeryFewHalfOpenConnections():
# Some XP and Vista systems can only have 10 outgoing connections at a time.
from class_sqlThread import *
from class_singleCleaner import *
from class_singleWorker import *
from class_outgoingSynSender import *
from class_singleListener import *
from class_addressGenerator import *
from debug import logger
# Helper Functions
import helper_bootstrap
import proofofwork
import sys
if sys.platform == 'darwin':
if float("{1}.{2}".format(*sys.version_info)) < 7.5:
logger.critical("You should use python 2.7.5 or greater. Your version: %s", "{0}.{1}.{2}".format(*sys.version_info))
sys.exit(0)
def connectToStream(streamNumber):
selfInitiatedConnections[streamNumber] = {}
if sys.platform[0:3] == 'win':
maximumNumberOfHalfOpenConnections = 9
else:
maximumNumberOfHalfOpenConnections = 32
for i in range(maximumNumberOfHalfOpenConnections):
a = outgoingSynSender()
a.setup(streamNumber, selfInitiatedConnections)
a.start()
class APIError(Exception):
def processData(self):
if len(self.data) < protocol.Header.size: # if so little of the data has arrived that we can't even read the checksum then wait for more data.
return
magic,command,payloadLength,checksum = protocol.Header.unpack(self.data[:protocol.Header.size])
if magic != 0xE9BEB4D9:
self.data = ""
return
if payloadLength > 1600100: # ~1.6 MB which is the maximum possible size of an inv message.
logger.info('The incoming message, which we have not yet download, is too large. Ignoring it. (unfortunately there is no way to tell the other node to stop sending it except to disconnect.) Message size: %s' % payloadLength)
self.data = self.data[payloadLength + protocol.Header.size:]
del magic,command,payloadLength,checksum # we don't need these anymore and better to clean them now before the recursive call rather than after
self.processData()
return
if len(self.data) < payloadLength + protocol.Header.size: # check if the whole message has arrived yet.
return
payload = self.data[protocol.Header.size:payloadLength + protocol.Header.size]
if checksum != hashlib.sha512(payload).digest()[0:4]: # test the checksum in the message.
logger.error('Checksum incorrect. Clearing this message.')
self.data = self.data[payloadLength + protocol.Header.size:]
del magic,command,payloadLength,checksum,payload # better to clean up before the recursive call
self.processData()
return
# The time we've last seen this node is obviously right now since we
# just received valid data from it. So update the knownNodes list so
# that other peers can be made aware of its existance.
if self.initiatedConnection and self.connectionIsOrWasFullyEstablished: # The remote port is only something we should share with others if it is the remote node's incoming port (rather than some random operating-system-assigned outgoing port).
with knownnodes.knownNodesLock:
for stream in self.streamNumber:
knownnodes.knownNodes[stream][self.peer] = int(time.time())
#Strip the nulls
command = command.rstrip('\x00')
logger.debug('remoteCommand ' + repr(command) + ' from ' + str(self.peer))
try:
#TODO: Use a dispatcher here
if command == 'error':
self.recerror(payload)
elif not self.connectionIsOrWasFullyEstablished:
if command == 'version':
self.recversion(payload)
elif command == 'verack':
self.recverack()
else:
if command == 'addr':
self.recaddr(payload)
elif command == 'inv':
self.recinv(payload)
elif command == 'getdata':
self.recgetdata(payload)
elif command == 'object':
self.recobject(payload)
elif command == 'ping':
self.sendpong(payload)
elif command == 'pong':
pass
else:
logger.info("Unknown command %s, ignoring", command)
except varintDecodeError as e:
logger.debug("There was a problem with a varint while processing a message from the wire. Some details: %s" % e)
except Exception as e:
logger.critical("Critical error in a receiveDataThread: \n%s" % traceback.format_exc())
del payload
self.data = self.data[payloadLength + protocol.Header.size:] # take this message out and then process the next message
if self.data == '': # if there are no more messages
toRequest = []
try:
for i in range(len(self.downloadQueue.pending), 100):
while True:
hashId = self.downloadQueue.get(False)
if not hashId in Inventory():
toRequest.append(hashId)
break
# don't track download for duplicates
self.downloadQueue.task_done(hashId)
except Queue.Empty:
pass
if len(toRequest) > 0:
self.sendgetdata(toRequest)
self.processData()
def processData(self):
if len(self.data) < shared.Header.size: # if so little of the data has arrived that we can't even read the checksum then wait for more data.
return
magic,command,payloadLength,checksum = shared.Header.unpack(self.data[:shared.Header.size])
if magic != 0xE9BEB4D9:
self.data = ""
return
if payloadLength > 1600100: # ~1.6 MB which is the maximum possible size of an inv message.
logger.info('The incoming message, which we have not yet download, is too large. Ignoring it. (unfortunately there is no way to tell the other node to stop sending it except to disconnect.) Message size: %s' % payloadLength)
self.data = self.data[payloadLength + shared.Header.size:]
del magic,command,payloadLength,checksum # we don't need these anymore and better to clean them now before the recursive call rather than after
self.processData()
return
if len(self.data) < payloadLength + shared.Header.size: # check if the whole message has arrived yet.
return
payload = self.data[shared.Header.size:payloadLength + shared.Header.size]
if checksum != hashlib.sha512(payload).digest()[0:4]: # test the checksum in the message.
print 'Checksum incorrect. Clearing this message.'
self.data = self.data[payloadLength + shared.Header.size:]
del magic,command,payloadLength,checksum,payload # better to clean up before the recursive call
self.processData()
return
# The time we've last seen this node is obviously right now since we
# just received valid data from it. So update the knownNodes list so
# that other peers can be made aware of its existance.
if self.initiatedConnection and self.connectionIsOrWasFullyEstablished: # The remote port is only something we should share with others if it is the remote node's incoming port (rather than some random operating-system-assigned outgoing port).
shared.knownNodesLock.acquire()
shared.knownNodes[self.streamNumber][self.peer] = int(time.time())
shared.knownNodesLock.release()
#Strip the nulls
command = command.rstrip('\x00')
with shared.printLock:
print 'remoteCommand', repr(command), ' from', self.peer
try:
#TODO: Use a dispatcher here
if command == 'error':
self.recerror(payload)
elif not self.connectionIsOrWasFullyEstablished:
if command == 'version':
self.recversion(payload)
elif command == 'verack':
self.recverack()
else:
if command == 'addr':
self.recaddr(payload)
elif command == 'inv':
self.recinv(payload)
elif command == 'getdata':
self.recgetdata(payload)
elif command == 'object':
self.recobject(payload)
elif command == 'ping':
self.sendpong(payload)
#elif command == 'pong':
# pass
except varintDecodeError as e:
logger.debug("There was a problem with a varint while processing a message from the wire. Some details: %s" % e)
except Exception as e:
logger.critical("Critical error in a receiveDataThread: \n%s" % traceback.format_exc())
del payload
self.data = self.data[payloadLength + shared.Header.size:] # take this message out and then process the next message
if self.data == '': # if there are no more messages
while len(self.objectsThatWeHaveYetToGetFromThisPeer) > 0:
shared.numberOfInventoryLookupsPerformed += 1
objectHash, = random.sample(
self.objectsThatWeHaveYetToGetFromThisPeer, 1)
if objectHash in shared.inventory:
with shared.printLock:
print 'Inventory (in memory) already has object listed in inv message.'
del self.objectsThatWeHaveYetToGetFromThisPeer[
objectHash]
elif shared.isInSqlInventory(objectHash):
if shared.verbose >= 3:
with shared.printLock:
print 'Inventory (SQL on disk) already has object listed in inv message.'
del self.objectsThatWeHaveYetToGetFromThisPeer[
objectHash]
else:
# We don't have the object in our inventory. Let's request it.
self.sendgetdata(objectHash)
del self.objectsThatWeHaveYetToGetFromThisPeer[
objectHash] # It is possible that the remote node might not respond with the object. In that case, we'll very likely get it from someone else anyway.
if len(self.objectsThatWeHaveYetToGetFromThisPeer) == 0:
with shared.printLock:
print '(concerning', str(self.peer) + ')', 'number of objectsThatWeHaveYetToGetFromThisPeer is now 0'
try:
del shared.numberOfObjectsThatWeHaveYetToGetPerPeer[
self.peer] # this data structure is maintained so that we can keep track of how many total objects, across all connections, are currently outstanding. If it goes too high it can indicate that we are under attack by multiple nodes working together.
except:
pass
break
if len(self.objectsThatWeHaveYetToGetFromThisPeer) == 0:
# We had objectsThatWeHaveYetToGetFromThisPeer but the loop ran, they were all in our inventory, and now we don't have any to get anymore.
with shared.printLock:
print '(concerning', str(self.peer) + ')', 'number of objectsThatWeHaveYetToGetFromThisPeer is now 0'
try:
del shared.numberOfObjectsThatWeHaveYetToGetPerPeer[
self.peer] # this data structure is maintained so that we can keep track of how many total objects, across all connections, are currently outstanding. If it goes too high it can indicate that we are under attack by multiple nodes working together.
except:
pass
if len(self.objectsThatWeHaveYetToGetFromThisPeer) > 0:
with shared.printLock:
print '(concerning', str(self.peer) + ')', 'number of objectsThatWeHaveYetToGetFromThisPeer is now', len(self.objectsThatWeHaveYetToGetFromThisPeer)
shared.numberOfObjectsThatWeHaveYetToGetPerPeer[self.peer] = len(
self.objectsThatWeHaveYetToGetFromThisPeer) # this data structure is maintained so that we can keep track of how many total objects, across all connections, are currently outstanding. If it goes too high it can indicate that we are under attack by multiple nodes working together.
self.processData()
def decryptAndCheckPubkeyPayload(data, address):
"""
Version 4 pubkeys are encrypted. This function is run when we already have the
address to which we want to try to send a message. The 'data' may come either
off of the wire or we might have had it already in our inventory when we tried
to send a msg to this particular address.
"""
try:
status, addressVersion, streamNumber, ripe = decodeAddress(address)
readPosition = 20 # bypass the nonce, time, and object type
embeddedAddressVersion, varintLength = decodeVarint(data[readPosition:readPosition + 10])
readPosition += varintLength
embeddedStreamNumber, varintLength = decodeVarint(data[readPosition:readPosition + 10])
readPosition += varintLength
storedData = data[20:readPosition] # We'll store the address version and stream number (and some more) in the pubkeys table.
if addressVersion != embeddedAddressVersion:
logger.info('Pubkey decryption was UNsuccessful due to address version mismatch.')
return 'failed'
if streamNumber != embeddedStreamNumber:
logger.info('Pubkey decryption was UNsuccessful due to stream number mismatch.')
return 'failed'
tag = data[readPosition:readPosition + 32]
readPosition += 32
signedData = data[8:readPosition] # the time through the tag. More data is appended onto signedData below after the decryption.
encryptedData = data[readPosition:]
# Let us try to decrypt the pubkey
toAddress, cryptorObject = state.neededPubkeys[tag]
if toAddress != address:
logger.critical('decryptAndCheckPubkeyPayload failed due to toAddress mismatch. This is very peculiar. toAddress: %s, address %s', toAddress, address)
# the only way I can think that this could happen is if someone encodes their address data two different ways.
# That sort of address-malleability should have been caught by the UI or API and an error given to the user.
return 'failed'
try:
decryptedData = cryptorObject.decrypt(encryptedData)
except:
# Someone must have encrypted some data with a different key
# but tagged it with a tag for which we are watching.
logger.info('Pubkey decryption was unsuccessful.')
return 'failed'
readPosition = 0
bitfieldBehaviors = decryptedData[readPosition:readPosition + 4]
readPosition += 4
publicSigningKey = '\x04' + decryptedData[readPosition:readPosition + 64]
readPosition += 64
publicEncryptionKey = '\x04' + decryptedData[readPosition:readPosition + 64]
readPosition += 64
specifiedNonceTrialsPerByte, specifiedNonceTrialsPerByteLength = decodeVarint(
decryptedData[readPosition:readPosition + 10])
readPosition += specifiedNonceTrialsPerByteLength
specifiedPayloadLengthExtraBytes, specifiedPayloadLengthExtraBytesLength = decodeVarint(
decryptedData[readPosition:readPosition + 10])
readPosition += specifiedPayloadLengthExtraBytesLength
storedData += decryptedData[:readPosition]
signedData += decryptedData[:readPosition]
signatureLength, signatureLengthLength = decodeVarint(
decryptedData[readPosition:readPosition + 10])
readPosition += signatureLengthLength
signature = decryptedData[readPosition:readPosition + signatureLength]
if highlevelcrypto.verify(signedData, signature, hexlify(publicSigningKey)):
logger.info('ECDSA verify passed (within decryptAndCheckPubkeyPayload)')
else:
logger.info('ECDSA verify failed (within decryptAndCheckPubkeyPayload)')
return 'failed'
sha = hashlib.new('sha512')
sha.update(publicSigningKey + publicEncryptionKey)
ripeHasher = hashlib.new('ripemd160')
ripeHasher.update(sha.digest())
embeddedRipe = ripeHasher.digest()
if embeddedRipe != ripe:
# Although this pubkey object had the tag were were looking for and was
# encrypted with the correct encryption key, it doesn't contain the
# correct pubkeys. Someone is either being malicious or using buggy software.
logger.info('Pubkey decryption was UNsuccessful due to RIPE mismatch.')
return 'failed'
# Everything checked out. Insert it into the pubkeys table.
logger.info('within decryptAndCheckPubkeyPayload, addressVersion: %s, streamNumber: %s \n\
ripe %s\n\
publicSigningKey in hex: %s\n\
publicEncryptionKey in hex: %s', addressVersion,
streamNumber,
hexlify(ripe),
hexlify(publicSigningKey),
hexlify(publicEncryptionKey)
)
t = (address, addressVersion, storedData, int(time.time()), 'yes')
sqlExecute('''INSERT INTO pubkeys VALUES (?,?,?,?,?)''', *t)
return 'successful'
except varintDecodeError as e:
logger.info('Pubkey decryption was UNsuccessful due to a malformed varint.')
return 'failed'
except Exception as e:
logger.critical('Pubkey decryption was UNsuccessful because of an unhandled exception! This is definitely a bug! \n%s', traceback.format_exc())
return 'failed'
def processData(self):
if len(
self.data
) < protocol.Header.size: # if so little of the data has arrived that we can't even read the checksum then wait for more data.
return
magic, command, payloadLength, checksum = protocol.Header.unpack(
self.data[:protocol.Header.size])
if magic != 0xE9BEB4D9:
self.data = ""
return
if payloadLength > 1600100: # ~1.6 MB which is the maximum possible size of an inv message.
logger.info(
'The incoming message, which we have not yet download, is too large. Ignoring it. (unfortunately there is no way to tell the other node to stop sending it except to disconnect.) Message size: %s'
% payloadLength)
self.data = self.data[payloadLength + protocol.Header.size:]
del magic, command, payloadLength, checksum # we don't need these anymore and better to clean them now before the recursive call rather than after
self.processData()
return
if len(
self.data
) < payloadLength + protocol.Header.size: # check if the whole message has arrived yet.
return
payload = self.data[protocol.Header.size:payloadLength +
protocol.Header.size]
if checksum != hashlib.sha512(
payload).digest()[0:4]: # test the checksum in the message.
logger.error('Checksum incorrect. Clearing this message.')
self.data = self.data[payloadLength + protocol.Header.size:]
del magic, command, payloadLength, checksum, payload # better to clean up before the recursive call
self.processData()
return
# The time we've last seen this node is obviously right now since we
# just received valid data from it. So update the knownNodes list so
# that other peers can be made aware of its existance.
if self.initiatedConnection and self.connectionIsOrWasFullyEstablished: # The remote port is only something we should share with others if it is the remote node's incoming port (rather than some random operating-system-assigned outgoing port).
with knownnodes.knownNodesLock:
for stream in self.streamNumber:
knownnodes.knownNodes[stream][self.peer] = int(time.time())
#Strip the nulls
command = command.rstrip('\x00')
logger.debug('remoteCommand ' + repr(command) + ' from ' +
str(self.peer))
try:
#TODO: Use a dispatcher here
if command == 'error':
self.recerror(payload)
elif not self.connectionIsOrWasFullyEstablished:
if command == 'version':
self.recversion(payload)
elif command == 'verack':
self.recverack()
else:
if command == 'addr':
self.recaddr(payload)
elif command == 'inv':
self.recinv(payload)
elif command == 'getdata':
self.recgetdata(payload)
elif command == 'object':
self.recobject(payload)
elif command == 'ping':
self.sendpong(payload)
elif command == 'pong':
pass
else:
logger.info("Unknown command %s, ignoring", command)
except varintDecodeError as e:
logger.debug(
"There was a problem with a varint while processing a message from the wire. Some details: %s"
% e)
except Exception as e:
logger.critical("Critical error in a receiveDataThread: \n%s" %
traceback.format_exc())
del payload
self.data = self.data[
payloadLength + protocol.Header.
size:] # take this message out and then process the next message
if self.data == '': # if there are no more messages
try:
self.sendgetdata(PendingDownload().pull(100))
except Queue.Full:
pass
self.processData()
