def run(self):
self.conn = sqlite3.connect(state.appdata + 'messages.dat')
self.conn.text_factory = str
self.cur = self.conn.cursor()
self.cur.execute('PRAGMA secure_delete = true')
try:
self.cur.execute(
'''CREATE TABLE inbox (msgid blob, toaddress text, fromaddress text, subject text, received text, message text, folder text, encodingtype int, read bool, sighash blob, UNIQUE(msgid) ON CONFLICT REPLACE)''' )
self.cur.execute(
'''CREATE TABLE sent (msgid blob, toaddress text, toripe blob, fromaddress text, subject text, message text, ackdata blob, senttime integer, lastactiontime integer, sleeptill integer, status text, retrynumber integer, folder text, encodingtype int, ttl int)''' )
self.cur.execute(
'''CREATE TABLE subscriptions (label text, address text, enabled bool)''' )
self.cur.execute(
'''CREATE TABLE addressbook (label text, address text)''' )
self.cur.execute(
'''CREATE TABLE blacklist (label text, address text, enabled bool)''' )
self.cur.execute(
'''CREATE TABLE whitelist (label text, address text, enabled bool)''' )
self.cur.execute(
'''CREATE TABLE pubkeys (address text, addressversion int, transmitdata blob, time int, usedpersonally text, UNIQUE(address) ON CONFLICT REPLACE)''' )
self.cur.execute(
'''CREATE TABLE inventory (hash blob, objecttype int, streamnumber int, payload blob, expirestime integer, tag blob, UNIQUE(hash) ON CONFLICT REPLACE)''' )
self.cur.execute(
'''INSERT INTO subscriptions VALUES('Bitmessage new releases/announcements','BM-GtovgYdgs7qXPkoYaRgrLFuFKz1SFpsw',1)''')
self.cur.execute(
'''CREATE TABLE settings (key blob, value blob, UNIQUE(key) ON CONFLICT REPLACE)''' )
self.cur.execute( '''INSERT INTO settings VALUES('version','10')''')
self.cur.execute( '''INSERT INTO settings VALUES('lastvacuumtime',?)''', (
int(time.time()),))
self.cur.execute(
'''CREATE TABLE objectprocessorqueue (objecttype int, data blob, UNIQUE(objecttype, data) ON CONFLICT REPLACE)''' )
self.conn.commit()
logger.info('Created messages database file')
except Exception as err:
if str(err) == 'table inbox already exists':
logger.debug('Database file already exists.')
else:
sys.stderr.write(
'ERROR trying to create database file (message.dat). Error message: %s\n' % str(err))
os._exit(0)
# If the settings version is equal to 2 or 3 then the
# sqlThread will modify the pubkeys table and change
# the settings version to 4.
settingsversion = BMConfigParser().getint(
'bitmessagesettings', 'settingsversion')
# People running earlier versions of PyBitmessage do not have the
# usedpersonally field in their pubkeys table. Let's add it.
if settingsversion == 2:
item = '''ALTER TABLE pubkeys ADD usedpersonally text DEFAULT 'no' '''
parameters = ''
self.cur.execute(item, parameters)
self.conn.commit()
settingsversion = 3
# People running earlier versions of PyBitmessage do not have the
# encodingtype field in their inbox and sent tables or the read field
# in the inbox table. Let's add them.
if settingsversion == 3:
item = '''ALTER TABLE inbox ADD encodingtype int DEFAULT '2' '''
parameters = ''
self.cur.execute(item, parameters)
item = '''ALTER TABLE inbox ADD read bool DEFAULT '1' '''
parameters = ''
self.cur.execute(item, parameters)
item = '''ALTER TABLE sent ADD encodingtype int DEFAULT '2' '''
parameters = ''
self.cur.execute(item, parameters)
self.conn.commit()
settingsversion = 4
BMConfigParser().set(
'bitmessagesettings', 'settingsversion', str(settingsversion))
BMConfigParser().save()
helper_startup.updateConfig()
# From now on, let us keep a 'version' embedded in the messages.dat
# file so that when we make changes to the database, the database
# version we are on can stay embedded in the messages.dat file. Let us
# check to see if the settings table exists yet.
item = '''SELECT name FROM sqlite_master WHERE type='table' AND name='settings';'''
parameters = ''
self.cur.execute(item, parameters)
if self.cur.fetchall() == []:
# The settings table doesn't exist. We need to make it.
logger.debug(
"In messages.dat database, creating new 'settings' table.")
self.cur.execute(
'''CREATE TABLE settings (key text, value blob, UNIQUE(key) ON CONFLICT REPLACE)''' )
self.cur.execute( '''INSERT INTO settings VALUES('version','1')''')
self.cur.execute( '''INSERT INTO settings VALUES('lastvacuumtime',?)''', (
int(time.time()),))
logger.debug('In messages.dat database, removing an obsolete field from the pubkeys table.')
self.cur.execute(
'''CREATE TEMPORARY TABLE pubkeys_backup(hash blob, transmitdata blob, time int, usedpersonally text, UNIQUE(hash) ON CONFLICT REPLACE);''')
self.cur.execute(
'''INSERT INTO pubkeys_backup SELECT hash, transmitdata, time, usedpersonally FROM pubkeys;''')
self.cur.execute( '''DROP TABLE pubkeys''')
self.cur.execute(
'''CREATE TABLE pubkeys (hash blob, transmitdata blob, time int, usedpersonally text, UNIQUE(hash) ON CONFLICT REPLACE)''' )
self.cur.execute(
'''INSERT INTO pubkeys SELECT hash, transmitdata, time, usedpersonally FROM pubkeys_backup;''')
self.cur.execute( '''DROP TABLE pubkeys_backup;''')
logger.debug('Deleting all pubkeys from inventory. They will be redownloaded and then saved with the correct times.')
self.cur.execute(
'''delete from inventory where objecttype = 'pubkey';''')
logger.debug('replacing Bitmessage announcements mailing list with a new one.')
self.cur.execute(
'''delete from subscriptions where address='BM-BbkPSZbzPwpVcYZpU4yHwf9ZPEapN5Zx' ''')
self.cur.execute(
'''INSERT INTO subscriptions VALUES('Bitmessage new releases/announcements','BM-GtovgYdgs7qXPkoYaRgrLFuFKz1SFpsw',1)''')
logger.debug('Commiting.')
self.conn.commit()
logger.debug('Vacuuming message.dat. You might notice that the file size gets much smaller.')
self.cur.execute( ''' VACUUM ''')
# After code refactoring, the possible status values for sent messages
# have changed.
self.cur.execute(
'''update sent set status='doingmsgpow' where status='doingpow' ''')
self.cur.execute(
'''update sent set status='msgsent' where status='sentmessage' ''')
self.cur.execute(
'''update sent set status='doingpubkeypow' where status='findingpubkey' ''')
self.cur.execute(
'''update sent set status='broadcastqueued' where status='broadcastpending' ''')
self.conn.commit()
# Let's get rid of the first20bytesofencryptedmessage field in
# the inventory table.
item = '''SELECT value FROM settings WHERE key='version';'''
parameters = ''
self.cur.execute(item, parameters)
if int(self.cur.fetchall()[0][0]) == 2:
logger.debug(
'In messages.dat database, removing an obsolete field from'
' the inventory table.')
self.cur.execute(
'''CREATE TEMPORARY TABLE inventory_backup(hash blob, objecttype text, streamnumber int, payload blob, receivedtime integer, UNIQUE(hash) ON CONFLICT REPLACE);''')
self.cur.execute(
'''INSERT INTO inventory_backup SELECT hash, objecttype, streamnumber, payload, receivedtime FROM inventory;''')
self.cur.execute( '''DROP TABLE inventory''')
self.cur.execute(
'''CREATE TABLE inventory (hash blob, objecttype text, streamnumber int, payload blob, receivedtime integer, UNIQUE(hash) ON CONFLICT REPLACE)''' )
self.cur.execute(
'''INSERT INTO inventory SELECT hash, objecttype, streamnumber, payload, receivedtime FROM inventory_backup;''')
self.cur.execute( '''DROP TABLE inventory_backup;''')
item = '''update settings set value=? WHERE key='version';'''
parameters = (3,)
self.cur.execute(item, parameters)
# Add a new column to the inventory table to store tags.
item = '''SELECT value FROM settings WHERE key='version';'''
parameters = ''
self.cur.execute(item, parameters)
currentVersion = int(self.cur.fetchall()[0][0])
if currentVersion == 1 or currentVersion == 3:
logger.debug(
'In messages.dat database, adding tag field to'
' the inventory table.')
item = '''ALTER TABLE inventory ADD tag blob DEFAULT '' '''
parameters = ''
self.cur.execute(item, parameters)
item = '''update settings set value=? WHERE key='version';'''
parameters = (4,)
self.cur.execute(item, parameters)
# Add a new column to the pubkeys table to store the address version.
# We're going to trash all of our pubkeys and let them be redownloaded.
item = '''SELECT value FROM settings WHERE key='version';'''
parameters = ''
self.cur.execute(item, parameters)
currentVersion = int(self.cur.fetchall()[0][0])
if currentVersion == 4:
self.cur.execute('''DROP TABLE pubkeys''')
self.cur.execute(
'''CREATE TABLE pubkeys (hash blob, addressversion int, transmitdata blob, time int, usedpersonally text, UNIQUE(hash, addressversion) ON CONFLICT REPLACE)''')
self.cur.execute(
'''delete from inventory where objecttype = 'pubkey';''')
item = '''update settings set value=? WHERE key='version';'''
parameters = (5,)
self.cur.execute(item, parameters)
# Add a new table: objectprocessorqueue with which to hold objects
# that have yet to be processed if the user shuts down Bitmessage.
item = '''SELECT value FROM settings WHERE key='version';'''
parameters = ''
self.cur.execute(item, parameters)
currentVersion = int(self.cur.fetchall()[0][0])
if currentVersion == 5:
self.cur.execute('''DROP TABLE knownnodes''')
self.cur.execute(
'''CREATE TABLE objectprocessorqueue (objecttype text, data blob, UNIQUE(objecttype, data) ON CONFLICT REPLACE)''')
item = '''update settings set value=? WHERE key='version';'''
parameters = (6,)
self.cur.execute(item, parameters)
# changes related to protocol v3
# In table inventory and objectprocessorqueue, objecttype is now
# an integer (it was a human-friendly string previously)
item = '''SELECT value FROM settings WHERE key='version';'''
parameters = ''
self.cur.execute(item, parameters)
currentVersion = int(self.cur.fetchall()[0][0])
if currentVersion == 6:
logger.debug(
'In messages.dat database, dropping and recreating'
' the inventory table.')
self.cur.execute( '''DROP TABLE inventory''')
self.cur.execute( '''CREATE TABLE inventory (hash blob, objecttype int, streamnumber int, payload blob, expirestime integer, tag blob, UNIQUE(hash) ON CONFLICT REPLACE)''' )
self.cur.execute( '''DROP TABLE objectprocessorqueue''')
self.cur.execute( '''CREATE TABLE objectprocessorqueue (objecttype int, data blob, UNIQUE(objecttype, data) ON CONFLICT REPLACE)''' )
item = '''update settings set value=? WHERE key='version';'''
parameters = (7,)
self.cur.execute(item, parameters)
logger.debug(
'Finished dropping and recreating the inventory table.')
# The format of data stored in the pubkeys table has changed. Let's
# clear it, and the pubkeys from inventory, so that they'll
# be re-downloaded.
item = '''SELECT value FROM settings WHERE key='version';'''
parameters = ''
self.cur.execute(item, parameters)
currentVersion = int(self.cur.fetchall()[0][0])
if currentVersion == 7:
logger.debug(
'In messages.dat database, clearing pubkeys table'
' because the data format has been updated.')
self.cur.execute(
'''delete from inventory where objecttype = 1;''')
self.cur.execute(
'''delete from pubkeys;''')
# Any sending messages for which we *thought* that we had
# the pubkey must be rechecked.
self.cur.execute(
'''UPDATE sent SET status='msgqueued' WHERE status='doingmsgpow' or status='badkey';''')
query = '''update settings set value=? WHERE key='version';'''
parameters = (8,)
self.cur.execute(query, parameters)
logger.debug('Finished clearing currently held pubkeys.')
# Add a new column to the inbox table to store the hash of
# the message signature. We'll use this as temporary message UUID
# in order to detect duplicates.
item = '''SELECT value FROM settings WHERE key='version';'''
parameters = ''
self.cur.execute(item, parameters)
currentVersion = int(self.cur.fetchall()[0][0])
if currentVersion == 8:
logger.debug(
'In messages.dat database, adding sighash field to'
' the inbox table.')
item = '''ALTER TABLE inbox ADD sighash blob DEFAULT '' '''
parameters = ''
self.cur.execute(item, parameters)
item = '''update settings set value=? WHERE key='version';'''
parameters = (9,)
self.cur.execute(item, parameters)
# We'll also need a `sleeptill` field and a `ttl` field. Also we
# can combine the pubkeyretrynumber and msgretrynumber into one.
item = '''SELECT value FROM settings WHERE key='version';'''
parameters = ''
self.cur.execute(item, parameters)
currentVersion = int(self.cur.fetchall()[0][0])
if currentVersion == 9:
logger.info(
'In messages.dat database, making TTL-related changes:'
' combining the pubkeyretrynumber and msgretrynumber'
' fields into the retrynumber field and adding the'
' sleeptill and ttl fields...')
self.cur.execute(
'''CREATE TEMPORARY TABLE sent_backup (msgid blob, toaddress text, toripe blob, fromaddress text, subject text, message text, ackdata blob, lastactiontime integer, status text, retrynumber integer, folder text, encodingtype int)''' )
self.cur.execute(
'''INSERT INTO sent_backup SELECT msgid, toaddress, toripe, fromaddress, subject, message, ackdata, lastactiontime, status, 0, folder, encodingtype FROM sent;''')
self.cur.execute( '''DROP TABLE sent''')
self.cur.execute(
'''CREATE TABLE sent (msgid blob, toaddress text, toripe blob, fromaddress text, subject text, message text, ackdata blob, senttime integer, lastactiontime integer, sleeptill int, status text, retrynumber integer, folder text, encodingtype int, ttl int)''' )
self.cur.execute(
'''INSERT INTO sent SELECT msgid, toaddress, toripe, fromaddress, subject, message, ackdata, lastactiontime, lastactiontime, 0, status, 0, folder, encodingtype, 216000 FROM sent_backup;''')
self.cur.execute( '''DROP TABLE sent_backup''')
logger.info('In messages.dat database, finished making TTL-related changes.')
logger.debug('In messages.dat database, adding address field to the pubkeys table.')
# We're going to have to calculate the address for each row in the pubkeys
# table. Then we can take out the hash field.
self.cur.execute('''ALTER TABLE pubkeys ADD address text DEFAULT '' ''')
self.cur.execute('''SELECT hash, addressversion FROM pubkeys''')
queryResult = self.cur.fetchall()
from addresses import encodeAddress
for row in queryResult:
addressHash, addressVersion = row
address = encodeAddress(addressVersion, 1, hash)
item = '''UPDATE pubkeys SET address=? WHERE hash=?;'''
parameters = (address, addressHash)
self.cur.execute(item, parameters)
# Now we can remove the hash field from the pubkeys table.
self.cur.execute(
'''CREATE TEMPORARY TABLE pubkeys_backup (address text, addressversion int, transmitdata blob, time int, usedpersonally text, UNIQUE(address) ON CONFLICT REPLACE)''' )
self.cur.execute(
'''INSERT INTO pubkeys_backup SELECT address, addressversion, transmitdata, time, usedpersonally FROM pubkeys;''')
self.cur.execute( '''DROP TABLE pubkeys''')
self.cur.execute(
'''CREATE TABLE pubkeys (address text, addressversion int, transmitdata blob, time int, usedpersonally text, UNIQUE(address) ON CONFLICT REPLACE)''' )
self.cur.execute(
'''INSERT INTO pubkeys SELECT address, addressversion, transmitdata, time, usedpersonally FROM pubkeys_backup;''')
self.cur.execute( '''DROP TABLE pubkeys_backup''')
logger.debug('In messages.dat database, done adding address field to the pubkeys table and removing the hash field.')
self.cur.execute('''update settings set value=10 WHERE key='version';''')
# Are you hoping to add a new option to the keys.dat file of existing
# Bitmessage users or modify the SQLite database? Add it right
# above this line!
try:
testpayload = '\x00\x00'
t = ('1234', 1, testpayload, '12345678', 'no')
self.cur.execute( '''INSERT INTO pubkeys VALUES(?,?,?,?,?)''', t)
self.conn.commit()
self.cur.execute(
'''SELECT transmitdata FROM pubkeys WHERE address='1234' ''')
queryreturn = self.cur.fetchall()
for row in queryreturn:
transmitdata, = row
self.cur.execute('''DELETE FROM pubkeys WHERE address='1234' ''')
self.conn.commit()
if transmitdata == '':
logger.fatal('Problem: The version of SQLite you have cannot store Null values. Please download and install the latest revision of your version of Python (for example, the latest Python 2.7 revision) and try again.\n')
logger.fatal('PyBitmessage will now exit very abruptly. You may now see threading errors related to this abrupt exit but the problem you need to solve is related to SQLite.\n\n')
os._exit(0)
except Exception as err:
if str(err) == 'database or disk is full':
logger.fatal('(While null value test) Alert: Your disk or data storage volume is full. sqlThread will now exit.')
queues.UISignalQueue.put(('alert', (tr._translate("MainWindow", "Disk full"), tr._translate("MainWindow", 'Alert: Your disk or data storage volume is full. Bitmessage will now exit.'), True)))
os._exit(0)
else:
logger.error(err)
# Let us check to see the last time we vaccumed the messages.dat file.
# If it has been more than a month let's do it now.
item = '''SELECT value FROM settings WHERE key='lastvacuumtime';'''
parameters = ''
self.cur.execute(item, parameters)
queryreturn = self.cur.fetchall()
for row in queryreturn:
value, = row
if int(value) < int(time.time()) - 86400:
logger.info('It has been a long time since the messages.dat file has been vacuumed. Vacuuming now...')
try:
self.cur.execute( ''' VACUUM ''')
except Exception as err:
if str(err) == 'database or disk is full':
logger.fatal('(While VACUUM) Alert: Your disk or data storage volume is full. sqlThread will now exit.')
queues.UISignalQueue.put(('alert', (tr._translate("MainWindow", "Disk full"), tr._translate("MainWindow", 'Alert: Your disk or data storage volume is full. Bitmessage will now exit.'), True)))
os._exit(0)
item = '''update settings set value=? WHERE key='lastvacuumtime';'''
parameters = (int(time.time()),)
self.cur.execute(item, parameters)
state.sqlReady = True
while True:
item = helper_sql.sqlSubmitQueue.get()
if item == 'commit':
try:
self.conn.commit()
except Exception as err:
if str(err) == 'database or disk is full':
logger.fatal('(While committing) Alert: Your disk or data storage volume is full. sqlThread will now exit.')
queues.UISignalQueue.put(('alert', (tr._translate("MainWindow", "Disk full"), tr._translate("MainWindow", 'Alert: Your disk or data storage volume is full. Bitmessage will now exit.'), True)))
os._exit(0)
elif item == 'exit':
self.conn.close()
logger.info('sqlThread exiting gracefully.')
return
elif item == 'movemessagstoprog':
logger.debug('the sqlThread is moving the messages.dat file to the local program directory.')
try:
self.conn.commit()
except Exception as err:
if str(err) == 'database or disk is full':
logger.fatal('(while movemessagstoprog) Alert: Your disk or data storage volume is full. sqlThread will now exit.')
queues.UISignalQueue.put(('alert', (tr._translate("MainWindow", "Disk full"), tr._translate("MainWindow", 'Alert: Your disk or data storage volume is full. Bitmessage will now exit.'), True)))
os._exit(0)
self.conn.close()
shutil.move(
paths.lookupAppdataFolder() + 'messages.dat', paths.lookupExeFolder() + 'messages.dat')
self.conn = sqlite3.connect(paths.lookupExeFolder() + 'messages.dat')
self.conn.text_factory = str
self.cur = self.conn.cursor()
elif item == 'movemessagstoappdata':
logger.debug('the sqlThread is moving the messages.dat file to the Appdata folder.')
try:
self.conn.commit()
except Exception as err:
if str(err) == 'database or disk is full':
logger.fatal('(while movemessagstoappdata) Alert: Your disk or data storage volume is full. sqlThread will now exit.')
queues.UISignalQueue.put(('alert', (tr._translate("MainWindow", "Disk full"), tr._translate("MainWindow", 'Alert: Your disk or data storage volume is full. Bitmessage will now exit.'), True)))
os._exit(0)
self.conn.close()
shutil.move(
paths.lookupExeFolder() + 'messages.dat', paths.lookupAppdataFolder() + 'messages.dat')
self.conn = sqlite3.connect(paths.lookupAppdataFolder() + 'messages.dat')
self.conn.text_factory = str
self.cur = self.conn.cursor()
elif item == 'deleteandvacuume':
self.cur.execute('''delete from inbox where folder='trash' ''')
self.cur.execute('''delete from sent where folder='trash' ''')
self.conn.commit()
try:
self.cur.execute( ''' VACUUM ''')
except Exception as err:
if str(err) == 'database or disk is full':
logger.fatal('(while deleteandvacuume) Alert: Your disk or data storage volume is full. sqlThread will now exit.')
queues.UISignalQueue.put(('alert', (tr._translate("MainWindow", "Disk full"), tr._translate("MainWindow", 'Alert: Your disk or data storage volume is full. Bitmessage will now exit.'), True)))
os._exit(0)
else:
parameters = helper_sql.sqlSubmitQueue.get()
rowcount = 0
# print 'item', item
# print 'parameters', parameters
try:
self.cur.execute(item, parameters)
rowcount = self.cur.rowcount
except Exception as err:
if str(err) == 'database or disk is full':
logger.fatal('(while cur.execute) Alert: Your disk or data storage volume is full. sqlThread will now exit.')
queues.UISignalQueue.put(('alert', (tr._translate("MainWindow", "Disk full"), tr._translate("MainWindow", 'Alert: Your disk or data storage volume is full. Bitmessage will now exit.'), True)))
os._exit(0)
else:
logger.fatal('Major error occurred when trying to execute a SQL statement within the sqlThread. Please tell Atheros about this error message or post it in the forum! Error occurred while trying to execute statement: "%s" Here are the parameters; you might want to censor this data with asterisks (***) as it can contain private information: %s. Here is the actual error message thrown by the sqlThread: %s', str(item), str(repr(parameters)), str(err))
logger.fatal('This program shall now abruptly exit!')
os._exit(0)
helper_sql.sqlReturnQueue.put((self.cur.fetchall(), rowcount))
def run(self):
while state.shutdown == 0:
queueValue = queues.addressGeneratorQueue.get()
nonceTrialsPerByte = 0
payloadLengthExtraBytes = 0
live = True
if queueValue[0] == 'createChan':
command, addressVersionNumber, streamNumber, label, \
deterministicPassphrase, live = queueValue
eighteenByteRipe = False
numberOfAddressesToMake = 1
numberOfNullBytesDemandedOnFrontOfRipeHash = 1
elif queueValue[0] == 'joinChan':
command, chanAddress, label, deterministicPassphrase, \
live = queueValue
eighteenByteRipe = False
addressVersionNumber = decodeAddress(chanAddress)[1]
streamNumber = decodeAddress(chanAddress)[2]
numberOfAddressesToMake = 1
numberOfNullBytesDemandedOnFrontOfRipeHash = 1
elif len(queueValue) == 7:
command, addressVersionNumber, streamNumber, label, \
numberOfAddressesToMake, deterministicPassphrase, \
eighteenByteRipe = queueValue
try:
numberOfNullBytesDemandedOnFrontOfRipeHash = \
BMConfigParser().getint(
'bitmessagesettings',
'numberofnullbytesonaddress'
)
except:
if eighteenByteRipe:
numberOfNullBytesDemandedOnFrontOfRipeHash = 2
else:
# the default
numberOfNullBytesDemandedOnFrontOfRipeHash = 1
elif len(queueValue) == 9:
command, addressVersionNumber, streamNumber, label, \
numberOfAddressesToMake, deterministicPassphrase, \
eighteenByteRipe, nonceTrialsPerByte, \
payloadLengthExtraBytes = queueValue
try:
numberOfNullBytesDemandedOnFrontOfRipeHash = \
BMConfigParser().getint(
'bitmessagesettings',
'numberofnullbytesonaddress'
)
except:
if eighteenByteRipe:
numberOfNullBytesDemandedOnFrontOfRipeHash = 2
else:
# the default
numberOfNullBytesDemandedOnFrontOfRipeHash = 1
elif queueValue[0] == 'stopThread':
break
else:
logger.error(
'Programming error: A structure with the wrong number'
' of values was passed into the addressGeneratorQueue.'
' Here is the queueValue: %r\n', queueValue)
if addressVersionNumber < 3 or addressVersionNumber > 4:
logger.error(
'Program error: For some reason the address generator'
' queue has been given a request to create at least'
' one version %s address which it cannot do.\n',
addressVersionNumber)
if nonceTrialsPerByte == 0:
nonceTrialsPerByte = BMConfigParser().getint(
'bitmessagesettings', 'defaultnoncetrialsperbyte')
if nonceTrialsPerByte < \
defaults.networkDefaultProofOfWorkNonceTrialsPerByte:
nonceTrialsPerByte = \
defaults.networkDefaultProofOfWorkNonceTrialsPerByte
if payloadLengthExtraBytes == 0:
payloadLengthExtraBytes = BMConfigParser().getint(
'bitmessagesettings', 'defaultpayloadlengthextrabytes')
if payloadLengthExtraBytes < \
defaults.networkDefaultPayloadLengthExtraBytes:
payloadLengthExtraBytes = \
defaults.networkDefaultPayloadLengthExtraBytes
if command == 'createRandomAddress':
queues.UISignalQueue.put(
('updateStatusBar',
tr._translate("MainWindow",
"Generating one new address")))
# This next section is a little bit strange. We're going
# to generate keys over and over until we find one
# that starts with either \x00 or \x00\x00. Then when
# we pack them into a Bitmessage address, we won't store
# the \x00 or \x00\x00 bytes thus making the address shorter.
startTime = time.time()
numberOfAddressesWeHadToMakeBeforeWeFoundOneWithTheCorrectRipePrefix = 0
potentialPrivSigningKey = OpenSSL.rand(32)
potentialPubSigningKey = highlevelcrypto.pointMult(
potentialPrivSigningKey)
while True:
numberOfAddressesWeHadToMakeBeforeWeFoundOneWithTheCorrectRipePrefix += 1
potentialPrivEncryptionKey = OpenSSL.rand(32)
potentialPubEncryptionKey = highlevelcrypto.pointMult(
potentialPrivEncryptionKey)
sha = hashlib.new('sha512')
sha.update(potentialPubSigningKey +
potentialPubEncryptionKey)
ripe = RIPEMD160Hash(sha.digest()).digest()
if (ripe[:numberOfNullBytesDemandedOnFrontOfRipeHash] ==
'\x00' *
numberOfNullBytesDemandedOnFrontOfRipeHash):
break
logger.info('Generated address with ripe digest: %s',
hexlify(ripe))
try:
logger.info(
'Address generator calculated %s addresses at %s'
' addresses per second before finding one with'
' the correct ripe-prefix.',
numberOfAddressesWeHadToMakeBeforeWeFoundOneWithTheCorrectRipePrefix,
numberOfAddressesWeHadToMakeBeforeWeFoundOneWithTheCorrectRipePrefix
/ (time.time() - startTime))
except ZeroDivisionError:
# The user must have a pretty fast computer.
# time.time() - startTime equaled zero.
pass
address = encodeAddress(addressVersionNumber, streamNumber,
ripe)
# An excellent way for us to store our keys
# is in Wallet Import Format. Let us convert now.
# https://en.bitcoin.it/wiki/Wallet_import_format
privSigningKey = '\x80' + potentialPrivSigningKey
checksum = hashlib.sha256(
hashlib.sha256(privSigningKey).digest()).digest()[0:4]
privSigningKeyWIF = arithmetic.changebase(
privSigningKey + checksum, 256, 58)
privEncryptionKey = '\x80' + potentialPrivEncryptionKey
checksum = hashlib.sha256(
hashlib.sha256(privEncryptionKey).digest()).digest()[0:4]
privEncryptionKeyWIF = arithmetic.changebase(
privEncryptionKey + checksum, 256, 58)
BMConfigParser().add_section(address)
BMConfigParser().set(address, 'label', label)
BMConfigParser().set(address, 'enabled', 'true')
BMConfigParser().set(address, 'decoy', 'false')
BMConfigParser().set(address, 'noncetrialsperbyte',
str(nonceTrialsPerByte))
BMConfigParser().set(address, 'payloadlengthextrabytes',
str(payloadLengthExtraBytes))
BMConfigParser().set(address, 'privsigningkey',
privSigningKeyWIF)
BMConfigParser().set(address, 'privencryptionkey',
privEncryptionKeyWIF)
BMConfigParser().save()
# The API and the join and create Chan functionality
# both need information back from the address generator.
queues.apiAddressGeneratorReturnQueue.put(address)
queues.UISignalQueue.put(
('updateStatusBar',
tr._translate(
"MainWindow",
"Done generating address. Doing work necessary"
" to broadcast it...")))
queues.UISignalQueue.put(
('writeNewAddressToTable', (label, address, streamNumber)))
shared.reloadMyAddressHashes()
if addressVersionNumber == 3:
queues.workerQueue.put(('sendOutOrStoreMyV3Pubkey', ripe))
elif addressVersionNumber == 4:
queues.workerQueue.put(
('sendOutOrStoreMyV4Pubkey', address))
elif command == 'createDeterministicAddresses' \
or command == 'getDeterministicAddress' \
or command == 'createChan' or command == 'joinChan':
if len(deterministicPassphrase) == 0:
logger.warning(
'You are creating deterministic'
' address(es) using a blank passphrase.'
' Bitmessage will do it but it is rather stupid.')
if command == 'createDeterministicAddresses':
queues.UISignalQueue.put(
('updateStatusBar',
tr._translate("MainWindow",
"Generating %1 new addresses.").arg(
str(numberOfAddressesToMake))))
signingKeyNonce = 0
encryptionKeyNonce = 1
# We fill out this list no matter what although we only
# need it if we end up passing the info to the API.
listOfNewAddressesToSendOutThroughTheAPI = []
for _ in range(numberOfAddressesToMake):
# This next section is a little bit strange. We're
# going to generate keys over and over until we find
# one that has a RIPEMD hash that starts with either
# \x00 or \x00\x00. Then when we pack them into a
# Bitmessage address, we won't store the \x00 or
# \x00\x00 bytes thus making the address shorter.
startTime = time.time()
numberOfAddressesWeHadToMakeBeforeWeFoundOneWithTheCorrectRipePrefix = 0
while True:
numberOfAddressesWeHadToMakeBeforeWeFoundOneWithTheCorrectRipePrefix += 1
potentialPrivSigningKey = hashlib.sha512(
deterministicPassphrase +
encodeVarint(signingKeyNonce)).digest()[:32]
potentialPrivEncryptionKey = hashlib.sha512(
deterministicPassphrase +
encodeVarint(encryptionKeyNonce)).digest()[:32]
potentialPubSigningKey = highlevelcrypto.pointMult(
potentialPrivSigningKey)
potentialPubEncryptionKey = highlevelcrypto.pointMult(
potentialPrivEncryptionKey)
signingKeyNonce += 2
encryptionKeyNonce += 2
sha = hashlib.new('sha512')
sha.update(potentialPubSigningKey +
potentialPubEncryptionKey)
ripe = RIPEMD160Hash(sha.digest()).digest()
if (ripe[:numberOfNullBytesDemandedOnFrontOfRipeHash]
== '\x00' *
numberOfNullBytesDemandedOnFrontOfRipeHash):
break
logger.info('Generated address with ripe digest: %s',
hexlify(ripe))
try:
logger.info(
'Address generator calculated %s addresses'
' at %s addresses per second before finding'
' one with the correct ripe-prefix.',
numberOfAddressesWeHadToMakeBeforeWeFoundOneWithTheCorrectRipePrefix,
numberOfAddressesWeHadToMakeBeforeWeFoundOneWithTheCorrectRipePrefix
/ (time.time() - startTime))
except ZeroDivisionError:
# The user must have a pretty fast computer.
# time.time() - startTime equaled zero.
pass
address = encodeAddress(addressVersionNumber, streamNumber,
ripe)
saveAddressToDisk = True
# If we are joining an existing chan, let us check
# to make sure it matches the provided Bitmessage address
if command == 'joinChan':
if address != chanAddress:
listOfNewAddressesToSendOutThroughTheAPI.append(
'chan name does not match address')
saveAddressToDisk = False
if command == 'getDeterministicAddress':
saveAddressToDisk = False
if saveAddressToDisk and live:
# An excellent way for us to store our keys is
# in Wallet Import Format. Let us convert now.
# https://en.bitcoin.it/wiki/Wallet_import_format
privSigningKey = '\x80' + potentialPrivSigningKey
checksum = hashlib.sha256(
hashlib.sha256(
privSigningKey).digest()).digest()[0:4]
privSigningKeyWIF = arithmetic.changebase(
privSigningKey + checksum, 256, 58)
privEncryptionKey = '\x80' + \
potentialPrivEncryptionKey
checksum = hashlib.sha256(
hashlib.sha256(
privEncryptionKey).digest()).digest()[0:4]
privEncryptionKeyWIF = arithmetic.changebase(
privEncryptionKey + checksum, 256, 58)
try:
BMConfigParser().add_section(address)
addressAlreadyExists = False
except:
addressAlreadyExists = True
if addressAlreadyExists:
logger.info(
'%s already exists. Not adding it again.',
address)
queues.UISignalQueue.put(
('updateStatusBar',
tr._translate(
"MainWindow",
"%1 is already in 'Your Identities'."
" Not adding it again.").arg(address)))
else:
logger.debug('label: %s', label)
BMConfigParser().set(address, 'label', label)
BMConfigParser().set(address, 'enabled', 'true')
BMConfigParser().set(address, 'decoy', 'false')
if command == 'joinChan' \
or command == 'createChan':
BMConfigParser().set(address, 'chan', 'true')
BMConfigParser().set(address, 'noncetrialsperbyte',
str(nonceTrialsPerByte))
BMConfigParser().set(address,
'payloadlengthextrabytes',
str(payloadLengthExtraBytes))
BMConfigParser().set(address, 'privSigningKey',
privSigningKeyWIF)
BMConfigParser().set(address, 'privEncryptionKey',
privEncryptionKeyWIF)
BMConfigParser().save()
queues.UISignalQueue.put(
('writeNewAddressToTable',
(label, address, str(streamNumber))))
listOfNewAddressesToSendOutThroughTheAPI.append(
address)
shared.myECCryptorObjects[ripe] = \
highlevelcrypto.makeCryptor(
hexlify(potentialPrivEncryptionKey))
shared.myAddressesByHash[ripe] = address
tag = hashlib.sha512(
hashlib.sha512(
encodeVarint(addressVersionNumber) +
encodeVarint(streamNumber) +
ripe).digest()).digest()[32:]
shared.myAddressesByTag[tag] = address
if addressVersionNumber == 3:
# If this is a chan address,
# the worker thread won't send out
# the pubkey over the network.
queues.workerQueue.put(
('sendOutOrStoreMyV3Pubkey', ripe))
elif addressVersionNumber == 4:
queues.workerQueue.put(
('sendOutOrStoreMyV4Pubkey', address))
queues.UISignalQueue.put(
('updateStatusBar',
tr._translate("MainWindow",
"Done generating address")))
elif saveAddressToDisk and not live \
and not BMConfigParser().has_section(address):
listOfNewAddressesToSendOutThroughTheAPI.append(
address)
# Done generating addresses.
if command == 'createDeterministicAddresses' \
or command == 'joinChan' or command == 'createChan':
queues.apiAddressGeneratorReturnQueue.put(
listOfNewAddressesToSendOutThroughTheAPI)
elif command == 'getDeterministicAddress':
queues.apiAddressGeneratorReturnQueue.put(address)
else:
raise Exception(
"Error in the addressGenerator thread. Thread was" +
" given a command it could not understand: " + command)
queues.addressGeneratorQueue.task_done()
def run(self):
self.conn = sqlite3.connect(state.appdata + 'messages.dat')
self.conn.text_factory = str
self.cur = self.conn.cursor()
self.cur.execute('PRAGMA secure_delete = true')
try:
self.cur.execute(
'''CREATE TABLE inbox (msgid blob, toaddress text, fromaddress text, subject text, received text, message text, folder text, encodingtype int, read bool, sighash blob, UNIQUE(msgid) ON CONFLICT REPLACE)'''
)
self.cur.execute(
'''CREATE TABLE sent (msgid blob, toaddress text, toripe blob, fromaddress text, subject text, message text, ackdata blob, senttime integer, lastactiontime integer, sleeptill integer, status text, retrynumber integer, folder text, encodingtype int, ttl int)'''
)
self.cur.execute(
'''CREATE TABLE subscriptions (label text, address text, enabled bool)'''
)
self.cur.execute(
'''CREATE TABLE addressbook (label text, address text)''')
self.cur.execute(
'''CREATE TABLE blacklist (label text, address text, enabled bool)'''
)
self.cur.execute(
'''CREATE TABLE whitelist (label text, address text, enabled bool)'''
)
self.cur.execute(
'''CREATE TABLE pubkeys (address text, addressversion int, transmitdata blob, time int, usedpersonally text, UNIQUE(address) ON CONFLICT REPLACE)'''
)
self.cur.execute(
'''CREATE TABLE inventory (hash blob, objecttype int, streamnumber int, payload blob, expirestime integer, tag blob, UNIQUE(hash) ON CONFLICT REPLACE)'''
)
self.cur.execute(
'''INSERT INTO subscriptions VALUES('Bitmessage new releases/announcements','BM-GtovgYdgs7qXPkoYaRgrLFuFKz1SFpsw',1)'''
)
self.cur.execute(
'''CREATE TABLE settings (key blob, value blob, UNIQUE(key) ON CONFLICT REPLACE)'''
)
self.cur.execute('''INSERT INTO settings VALUES('version','10')''')
self.cur.execute(
'''INSERT INTO settings VALUES('lastvacuumtime',?)''',
(int(time.time()), ))
self.cur.execute(
'''CREATE TABLE objectprocessorqueue (objecttype int, data blob, UNIQUE(objecttype, data) ON CONFLICT REPLACE)'''
)
self.conn.commit()
logger.info('Created messages database file')
except Exception as err:
if str(err) == 'table inbox already exists':
logger.debug('Database file already exists.')
else:
sys.stderr.write(
'ERROR trying to create database file (message.dat). Error message: %s\n'
% str(err))
os._exit(0)
if BMConfigParser().getint('bitmessagesettings',
'settingsversion') == 1:
BMConfigParser().set('bitmessagesettings', 'settingsversion', '2')
# If the settings version is equal to 2 or 3 then the
# sqlThread will modify the pubkeys table and change
# the settings version to 4.
BMConfigParser().set('bitmessagesettings', 'socksproxytype',
'none')
BMConfigParser().set('bitmessagesettings', 'sockshostname',
'localhost')
BMConfigParser().set('bitmessagesettings', 'socksport', '9050')
BMConfigParser().set('bitmessagesettings', 'socksauthentication',
'false')
BMConfigParser().set('bitmessagesettings', 'socksusername', '')
BMConfigParser().set('bitmessagesettings', 'sockspassword', '')
BMConfigParser().set('bitmessagesettings', 'sockslisten', 'false')
BMConfigParser().set('bitmessagesettings', 'keysencrypted',
'false')
BMConfigParser().set('bitmessagesettings', 'messagesencrypted',
'false')
# People running earlier versions of PyBitmessage do not have the
# usedpersonally field in their pubkeys table. Let's add it.
if BMConfigParser().getint('bitmessagesettings',
'settingsversion') == 2:
item = '''ALTER TABLE pubkeys ADD usedpersonally text DEFAULT 'no' '''
parameters = ''
self.cur.execute(item, parameters)
self.conn.commit()
BMConfigParser().set('bitmessagesettings', 'settingsversion', '3')
# People running earlier versions of PyBitmessage do not have the
# encodingtype field in their inbox and sent tables or the read field
# in the inbox table. Let's add them.
if BMConfigParser().getint('bitmessagesettings',
'settingsversion') == 3:
item = '''ALTER TABLE inbox ADD encodingtype int DEFAULT '2' '''
parameters = ''
self.cur.execute(item, parameters)
item = '''ALTER TABLE inbox ADD read bool DEFAULT '1' '''
parameters = ''
self.cur.execute(item, parameters)
item = '''ALTER TABLE sent ADD encodingtype int DEFAULT '2' '''
parameters = ''
self.cur.execute(item, parameters)
self.conn.commit()
BMConfigParser().set('bitmessagesettings', 'settingsversion', '4')
if BMConfigParser().getint('bitmessagesettings',
'settingsversion') == 4:
BMConfigParser().set(
'bitmessagesettings', 'defaultnoncetrialsperbyte',
str(defaults.networkDefaultProofOfWorkNonceTrialsPerByte))
BMConfigParser().set(
'bitmessagesettings', 'defaultpayloadlengthextrabytes',
str(defaults.networkDefaultPayloadLengthExtraBytes))
BMConfigParser().set('bitmessagesettings', 'settingsversion', '5')
if BMConfigParser().getint('bitmessagesettings',
'settingsversion') == 5:
BMConfigParser().set('bitmessagesettings',
'maxacceptablenoncetrialsperbyte', '0')
BMConfigParser().set('bitmessagesettings',
'maxacceptablepayloadlengthextrabytes', '0')
BMConfigParser().set('bitmessagesettings', 'settingsversion', '6')
# From now on, let us keep a 'version' embedded in the messages.dat
# file so that when we make changes to the database, the database
# version we are on can stay embedded in the messages.dat file. Let us
# check to see if the settings table exists yet.
item = '''SELECT name FROM sqlite_master WHERE type='table' AND name='settings';'''
parameters = ''
self.cur.execute(item, parameters)
if self.cur.fetchall() == []:
# The settings table doesn't exist. We need to make it.
logger.debug(
'In messages.dat database, creating new \'settings\' table.')
self.cur.execute(
'''CREATE TABLE settings (key text, value blob, UNIQUE(key) ON CONFLICT REPLACE)'''
)
self.cur.execute('''INSERT INTO settings VALUES('version','1')''')
self.cur.execute(
'''INSERT INTO settings VALUES('lastvacuumtime',?)''',
(int(time.time()), ))
logger.debug(
'In messages.dat database, removing an obsolete field from the pubkeys table.'
)
self.cur.execute(
'''CREATE TEMPORARY TABLE pubkeys_backup(hash blob, transmitdata blob, time int, usedpersonally text, UNIQUE(hash) ON CONFLICT REPLACE);'''
)
self.cur.execute(
'''INSERT INTO pubkeys_backup SELECT hash, transmitdata, time, usedpersonally FROM pubkeys;'''
)
self.cur.execute('''DROP TABLE pubkeys''')
self.cur.execute(
'''CREATE TABLE pubkeys (hash blob, transmitdata blob, time int, usedpersonally text, UNIQUE(hash) ON CONFLICT REPLACE)'''
)
self.cur.execute(
'''INSERT INTO pubkeys SELECT hash, transmitdata, time, usedpersonally FROM pubkeys_backup;'''
)
self.cur.execute('''DROP TABLE pubkeys_backup;''')
logger.debug(
'Deleting all pubkeys from inventory. They will be redownloaded and then saved with the correct times.'
)
self.cur.execute(
'''delete from inventory where objecttype = 'pubkey';''')
logger.debug(
'replacing Bitmessage announcements mailing list with a new one.'
)
self.cur.execute(
'''delete from subscriptions where address='BM-BbkPSZbzPwpVcYZpU4yHwf9ZPEapN5Zx' '''
)
self.cur.execute(
'''INSERT INTO subscriptions VALUES('Bitmessage new releases/announcements','BM-GtovgYdgs7qXPkoYaRgrLFuFKz1SFpsw',1)'''
)
logger.debug('Commiting.')
self.conn.commit()
logger.debug(
'Vacuuming message.dat. You might notice that the file size gets much smaller.'
)
self.cur.execute(''' VACUUM ''')
# After code refactoring, the possible status values for sent messages
# have changed.
self.cur.execute(
'''update sent set status='doingmsgpow' where status='doingpow' '''
)
self.cur.execute(
'''update sent set status='msgsent' where status='sentmessage' '''
)
self.cur.execute(
'''update sent set status='doingpubkeypow' where status='findingpubkey' '''
)
self.cur.execute(
'''update sent set status='broadcastqueued' where status='broadcastpending' '''
)
self.conn.commit()
if not BMConfigParser().has_option('bitmessagesettings',
'sockslisten'):
BMConfigParser().set('bitmessagesettings', 'sockslisten', 'false')
"""# Add a new column to the inventory table to store the first 20 bytes of encrypted messages to support Android app
item = '''SELECT value FROM settings WHERE key='version';'''
parameters = ''
self.cur.execute(item, parameters)
if int(self.cur.fetchall()[0][0]) == 1:
print 'upgrading database'
item = '''ALTER TABLE inventory ADD first20bytesofencryptedmessage blob DEFAULT '' '''
parameters = ''
self.cur.execute(item, parameters)
item = '''update settings set value=? WHERE key='version';'''
parameters = (2,)
self.cur.execute(item, parameters)"""
# Let's get rid of the first20bytesofencryptedmessage field in the inventory table.
item = '''SELECT value FROM settings WHERE key='version';'''
parameters = ''
self.cur.execute(item, parameters)
if int(self.cur.fetchall()[0][0]) == 2:
logger.debug(
'In messages.dat database, removing an obsolete field from the inventory table.'
)
self.cur.execute(
'''CREATE TEMPORARY TABLE inventory_backup(hash blob, objecttype text, streamnumber int, payload blob, receivedtime integer, UNIQUE(hash) ON CONFLICT REPLACE);'''
)
self.cur.execute(
'''INSERT INTO inventory_backup SELECT hash, objecttype, streamnumber, payload, receivedtime FROM inventory;'''
)
self.cur.execute('''DROP TABLE inventory''')
self.cur.execute(
'''CREATE TABLE inventory (hash blob, objecttype text, streamnumber int, payload blob, receivedtime integer, UNIQUE(hash) ON CONFLICT REPLACE)'''
)
self.cur.execute(
'''INSERT INTO inventory SELECT hash, objecttype, streamnumber, payload, receivedtime FROM inventory_backup;'''
)
self.cur.execute('''DROP TABLE inventory_backup;''')
item = '''update settings set value=? WHERE key='version';'''
parameters = (3, )
self.cur.execute(item, parameters)
# Add a new column to the inventory table to store tags.
item = '''SELECT value FROM settings WHERE key='version';'''
parameters = ''
self.cur.execute(item, parameters)
currentVersion = int(self.cur.fetchall()[0][0])
if currentVersion == 1 or currentVersion == 3:
logger.debug(
'In messages.dat database, adding tag field to the inventory table.'
)
item = '''ALTER TABLE inventory ADD tag blob DEFAULT '' '''
parameters = ''
self.cur.execute(item, parameters)
item = '''update settings set value=? WHERE key='version';'''
parameters = (4, )
self.cur.execute(item, parameters)
if not BMConfigParser().has_option('bitmessagesettings', 'userlocale'):
BMConfigParser().set('bitmessagesettings', 'userlocale', 'system')
if not BMConfigParser().has_option('bitmessagesettings',
'sendoutgoingconnections'):
BMConfigParser().set('bitmessagesettings',
'sendoutgoingconnections', 'True')
# Raise the default required difficulty from 1 to 2
# With the change to protocol v3, this is obsolete.
if BMConfigParser().getint('bitmessagesettings',
'settingsversion') == 6:
"""if int(shared.config.get('bitmessagesettings','defaultnoncetrialsperbyte')) == defaults.networkDefaultProofOfWorkNonceTrialsPerByte:
shared.config.set('bitmessagesettings','defaultnoncetrialsperbyte', str(defaults.networkDefaultProofOfWorkNonceTrialsPerByte * 2))
"""
BMConfigParser().set('bitmessagesettings', 'settingsversion', '7')
# Add a new column to the pubkeys table to store the address version.
# We're going to trash all of our pubkeys and let them be redownloaded.
item = '''SELECT value FROM settings WHERE key='version';'''
parameters = ''
self.cur.execute(item, parameters)
currentVersion = int(self.cur.fetchall()[0][0])
if currentVersion == 4:
self.cur.execute('''DROP TABLE pubkeys''')
self.cur.execute(
'''CREATE TABLE pubkeys (hash blob, addressversion int, transmitdata blob, time int, usedpersonally text, UNIQUE(hash, addressversion) ON CONFLICT REPLACE)'''
)
self.cur.execute(
'''delete from inventory where objecttype = 'pubkey';''')
item = '''update settings set value=? WHERE key='version';'''
parameters = (5, )
self.cur.execute(item, parameters)
if not BMConfigParser().has_option('bitmessagesettings',
'useidenticons'):
BMConfigParser().set('bitmessagesettings', 'useidenticons', 'True')
if not BMConfigParser().has_option(
'bitmessagesettings', 'identiconsuffix'): # acts as a salt
BMConfigParser(
).set('bitmessagesettings', 'identiconsuffix', ''.join(
random.choice(
"123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz"
) for x in xrange(12)
)) # a twelve character pseudo-password to salt the identicons
#Add settings to support no longer resending messages after a certain period of time even if we never get an ack
if BMConfigParser().getint('bitmessagesettings',
'settingsversion') == 7:
BMConfigParser().set('bitmessagesettings',
'stopresendingafterxdays', '')
BMConfigParser().set('bitmessagesettings',
'stopresendingafterxmonths', '')
BMConfigParser().set('bitmessagesettings', 'settingsversion', '8')
# Add a new table: objectprocessorqueue with which to hold objects
# that have yet to be processed if the user shuts down Bitmessage.
item = '''SELECT value FROM settings WHERE key='version';'''
parameters = ''
self.cur.execute(item, parameters)
currentVersion = int(self.cur.fetchall()[0][0])
if currentVersion == 5:
self.cur.execute('''DROP TABLE knownnodes''')
self.cur.execute(
'''CREATE TABLE objectprocessorqueue (objecttype text, data blob, UNIQUE(objecttype, data) ON CONFLICT REPLACE)'''
)
item = '''update settings set value=? WHERE key='version';'''
parameters = (6, )
self.cur.execute(item, parameters)
# changes related to protocol v3
# In table inventory and objectprocessorqueue, objecttype is now an integer (it was a human-friendly string previously)
item = '''SELECT value FROM settings WHERE key='version';'''
parameters = ''
self.cur.execute(item, parameters)
currentVersion = int(self.cur.fetchall()[0][0])
if currentVersion == 6:
logger.debug(
'In messages.dat database, dropping and recreating the inventory table.'
)
self.cur.execute('''DROP TABLE inventory''')
self.cur.execute(
'''CREATE TABLE inventory (hash blob, objecttype int, streamnumber int, payload blob, expirestime integer, tag blob, UNIQUE(hash) ON CONFLICT REPLACE)'''
)
self.cur.execute('''DROP TABLE objectprocessorqueue''')
self.cur.execute(
'''CREATE TABLE objectprocessorqueue (objecttype int, data blob, UNIQUE(objecttype, data) ON CONFLICT REPLACE)'''
)
item = '''update settings set value=? WHERE key='version';'''
parameters = (7, )
self.cur.execute(item, parameters)
logger.debug(
'Finished dropping and recreating the inventory table.')
# With the change to protocol version 3, reset the user-settable difficulties to 1
if BMConfigParser().getint('bitmessagesettings',
'settingsversion') == 8:
BMConfigParser().set(
'bitmessagesettings', 'defaultnoncetrialsperbyte',
str(defaults.networkDefaultProofOfWorkNonceTrialsPerByte))
BMConfigParser().set(
'bitmessagesettings', 'defaultpayloadlengthextrabytes',
str(defaults.networkDefaultPayloadLengthExtraBytes))
previousTotalDifficulty = int(BMConfigParser().getint(
'bitmessagesettings', 'maxacceptablenoncetrialsperbyte')) / 320
previousSmallMessageDifficulty = int(BMConfigParser().getint(
'bitmessagesettings',
'maxacceptablepayloadlengthextrabytes')) / 14000
BMConfigParser().set('bitmessagesettings',
'maxacceptablenoncetrialsperbyte',
str(previousTotalDifficulty * 1000))
BMConfigParser().set('bitmessagesettings',
'maxacceptablepayloadlengthextrabytes',
str(previousSmallMessageDifficulty * 1000))
BMConfigParser().set('bitmessagesettings', 'settingsversion', '9')
# Adjust the required POW values for each of this user's addresses to conform to protocol v3 norms.
if BMConfigParser().getint('bitmessagesettings',
'settingsversion') == 9:
for addressInKeysFile in BMConfigParser().addressses():
try:
previousTotalDifficulty = float(BMConfigParser().getint(
addressInKeysFile, 'noncetrialsperbyte')) / 320
previousSmallMessageDifficulty = float(
BMConfigParser().getint(
addressInKeysFile,
'payloadlengthextrabytes')) / 14000
if previousTotalDifficulty <= 2:
previousTotalDifficulty = 1
if previousSmallMessageDifficulty < 1:
previousSmallMessageDifficulty = 1
BMConfigParser().set(
addressInKeysFile, 'noncetrialsperbyte',
str(int(previousTotalDifficulty * 1000)))
BMConfigParser().set(
addressInKeysFile, 'payloadlengthextrabytes',
str(int(previousSmallMessageDifficulty * 1000)))
except:
continue
BMConfigParser().set('bitmessagesettings', 'maxdownloadrate', '0')
BMConfigParser().set('bitmessagesettings', 'maxuploadrate', '0')
BMConfigParser().set('bitmessagesettings', 'settingsversion', '10')
BMConfigParser().save()
# sanity check
if BMConfigParser().getint('bitmessagesettings',
'maxacceptablenoncetrialsperbyte') == 0:
BMConfigParser().set(
'bitmessagesettings', 'maxacceptablenoncetrialsperbyte',
str(defaults.ridiculousDifficulty *
defaults.networkDefaultProofOfWorkNonceTrialsPerByte))
if BMConfigParser().getint(
'bitmessagesettings',
'maxacceptablepayloadlengthextrabytes') == 0:
BMConfigParser().set(
'bitmessagesettings', 'maxacceptablepayloadlengthextrabytes',
str(defaults.ridiculousDifficulty *
defaults.networkDefaultPayloadLengthExtraBytes))
# The format of data stored in the pubkeys table has changed. Let's
# clear it, and the pubkeys from inventory, so that they'll be re-downloaded.
item = '''SELECT value FROM settings WHERE key='version';'''
parameters = ''
self.cur.execute(item, parameters)
currentVersion = int(self.cur.fetchall()[0][0])
if currentVersion == 7:
logger.debug(
'In messages.dat database, clearing pubkeys table because the data format has been updated.'
)
self.cur.execute('''delete from inventory where objecttype = 1;''')
self.cur.execute('''delete from pubkeys;''')
# Any sending messages for which we *thought* that we had the pubkey must
# be rechecked.
self.cur.execute(
'''UPDATE sent SET status='msgqueued' WHERE status='doingmsgpow' or status='badkey';'''
)
query = '''update settings set value=? WHERE key='version';'''
parameters = (8, )
self.cur.execute(query, parameters)
logger.debug('Finished clearing currently held pubkeys.')
# Add a new column to the inbox table to store the hash of the message signature.
# We'll use this as temporary message UUID in order to detect duplicates.
item = '''SELECT value FROM settings WHERE key='version';'''
parameters = ''
self.cur.execute(item, parameters)
currentVersion = int(self.cur.fetchall()[0][0])
if currentVersion == 8:
logger.debug(
'In messages.dat database, adding sighash field to the inbox table.'
)
item = '''ALTER TABLE inbox ADD sighash blob DEFAULT '' '''
parameters = ''
self.cur.execute(item, parameters)
item = '''update settings set value=? WHERE key='version';'''
parameters = (9, )
self.cur.execute(item, parameters)
# TTL is now user-specifiable. Let's add an option to save whatever the user selects.
if not BMConfigParser().has_option('bitmessagesettings', 'ttl'):
BMConfigParser().set('bitmessagesettings', 'ttl', '367200')
# We'll also need a `sleeptill` field and a `ttl` field. Also we can combine
# the pubkeyretrynumber and msgretrynumber into one.
item = '''SELECT value FROM settings WHERE key='version';'''
parameters = ''
self.cur.execute(item, parameters)
currentVersion = int(self.cur.fetchall()[0][0])
if currentVersion == 9:
logger.info(
'In messages.dat database, making TTL-related changes: combining the pubkeyretrynumber and msgretrynumber fields into the retrynumber field and adding the sleeptill and ttl fields...'
)
self.cur.execute(
'''CREATE TEMPORARY TABLE sent_backup (msgid blob, toaddress text, toripe blob, fromaddress text, subject text, message text, ackdata blob, lastactiontime integer, status text, retrynumber integer, folder text, encodingtype int)'''
)
self.cur.execute(
'''INSERT INTO sent_backup SELECT msgid, toaddress, toripe, fromaddress, subject, message, ackdata, lastactiontime, status, 0, folder, encodingtype FROM sent;'''
)
self.cur.execute('''DROP TABLE sent''')
self.cur.execute(
'''CREATE TABLE sent (msgid blob, toaddress text, toripe blob, fromaddress text, subject text, message text, ackdata blob, senttime integer, lastactiontime integer, sleeptill int, status text, retrynumber integer, folder text, encodingtype int, ttl int)'''
)
self.cur.execute(
'''INSERT INTO sent SELECT msgid, toaddress, toripe, fromaddress, subject, message, ackdata, lastactiontime, lastactiontime, 0, status, 0, folder, encodingtype, 216000 FROM sent_backup;'''
)
self.cur.execute('''DROP TABLE sent_backup''')
logger.info(
'In messages.dat database, finished making TTL-related changes.'
)
logger.debug(
'In messages.dat database, adding address field to the pubkeys table.'
)
# We're going to have to calculate the address for each row in the pubkeys
# table. Then we can take out the hash field.
self.cur.execute(
'''ALTER TABLE pubkeys ADD address text DEFAULT '' ''')
self.cur.execute('''SELECT hash, addressversion FROM pubkeys''')
queryResult = self.cur.fetchall()
from addresses import encodeAddress
for row in queryResult:
hash, addressVersion = row
address = encodeAddress(addressVersion, 1, hash)
item = '''UPDATE pubkeys SET address=? WHERE hash=?;'''
parameters = (address, hash)
self.cur.execute(item, parameters)
# Now we can remove the hash field from the pubkeys table.
self.cur.execute(
'''CREATE TEMPORARY TABLE pubkeys_backup (address text, addressversion int, transmitdata blob, time int, usedpersonally text, UNIQUE(address) ON CONFLICT REPLACE)'''
)
self.cur.execute(
'''INSERT INTO pubkeys_backup SELECT address, addressversion, transmitdata, time, usedpersonally FROM pubkeys;'''
)
self.cur.execute('''DROP TABLE pubkeys''')
self.cur.execute(
'''CREATE TABLE pubkeys (address text, addressversion int, transmitdata blob, time int, usedpersonally text, UNIQUE(address) ON CONFLICT REPLACE)'''
)
self.cur.execute(
'''INSERT INTO pubkeys SELECT address, addressversion, transmitdata, time, usedpersonally FROM pubkeys_backup;'''
)
self.cur.execute('''DROP TABLE pubkeys_backup''')
logger.debug(
'In messages.dat database, done adding address field to the pubkeys table and removing the hash field.'
)
self.cur.execute(
'''update settings set value=10 WHERE key='version';''')
if not BMConfigParser().has_option('bitmessagesettings',
'onionhostname'):
BMConfigParser().set('bitmessagesettings', 'onionhostname', '')
if not BMConfigParser().has_option('bitmessagesettings', 'onionport'):
BMConfigParser().set('bitmessagesettings', 'onionport', '8444')
if not BMConfigParser().has_option('bitmessagesettings',
'onionbindip'):
BMConfigParser().set('bitmessagesettings', 'onionbindip',
'127.0.0.1')
if not BMConfigParser().has_option('bitmessagesettings',
'hidetrayconnectionnotifications'):
BMConfigParser().set('bitmessagesettings',
'hidetrayconnectionnotifications', 'false')
if BMConfigParser().has_option('bitmessagesettings',
'maxoutboundconnections'):
try:
if BMConfigParser().getint('bitmessagesettings',
'maxoutboundconnections') < 1:
raise ValueError
except ValueError as err:
BMConfigParser().remove_option('bitmessagesettings',
'maxoutboundconnections')
logger.error(
'Your maximum outbound connections must be a number.')
if not BMConfigParser().has_option('bitmessagesettings',
'maxoutboundconnections'):
logger.info('Setting maximum outbound connections to 8.')
BMConfigParser().set('bitmessagesettings',
'maxoutboundconnections', '8')
BMConfigParser().save()
# Are you hoping to add a new option to the keys.dat file of existing
# Bitmessage users or modify the SQLite database? Add it right above this line!
try:
testpayload = '\x00\x00'
t = ('1234', 1, testpayload, '12345678', 'no')
self.cur.execute('''INSERT INTO pubkeys VALUES(?,?,?,?,?)''', t)
self.conn.commit()
self.cur.execute(
'''SELECT transmitdata FROM pubkeys WHERE address='1234' ''')
queryreturn = self.cur.fetchall()
for row in queryreturn:
transmitdata, = row
self.cur.execute('''DELETE FROM pubkeys WHERE address='1234' ''')
self.conn.commit()
if transmitdata == '':
logger.fatal(
'Problem: The version of SQLite you have cannot store Null values. Please download and install the latest revision of your version of Python (for example, the latest Python 2.7 revision) and try again.\n'
)
logger.fatal(
'PyBitmessage will now exit very abruptly. You may now see threading errors related to this abrupt exit but the problem you need to solve is related to SQLite.\n\n'
)
os._exit(0)
except Exception as err:
if str(err) == 'database or disk is full':
logger.fatal(
'(While null value test) Alert: Your disk or data storage volume is full. sqlThread will now exit.'
)
queues.UISignalQueue.put(('alert', (
tr._translate("MainWindow", "Disk full"),
tr._translate(
"MainWindow",
'Alert: Your disk or data storage volume is full. Bitmessage will now exit.'
), True)))
os._exit(0)
else:
logger.error(err)
# Let us check to see the last time we vaccumed the messages.dat file.
# If it has been more than a month let's do it now.
item = '''SELECT value FROM settings WHERE key='lastvacuumtime';'''
parameters = ''
self.cur.execute(item, parameters)
queryreturn = self.cur.fetchall()
for row in queryreturn:
value, = row
if int(value) < int(time.time()) - 86400:
logger.info(
'It has been a long time since the messages.dat file has been vacuumed. Vacuuming now...'
)
try:
self.cur.execute(''' VACUUM ''')
except Exception as err:
if str(err) == 'database or disk is full':
logger.fatal(
'(While VACUUM) Alert: Your disk or data storage volume is full. sqlThread will now exit.'
)
queues.UISignalQueue.put(('alert', (
tr._translate("MainWindow", "Disk full"),
tr._translate(
"MainWindow",
'Alert: Your disk or data storage volume is full. Bitmessage will now exit.'
), True)))
os._exit(0)
item = '''update settings set value=? WHERE key='lastvacuumtime';'''
parameters = (int(time.time()), )
self.cur.execute(item, parameters)
state.sqlReady = True
while True:
item = helper_sql.sqlSubmitQueue.get()
if item == 'commit':
try:
self.conn.commit()
except Exception as err:
if str(err) == 'database or disk is full':
logger.fatal(
'(While committing) Alert: Your disk or data storage volume is full. sqlThread will now exit.'
)
queues.UISignalQueue.put(('alert', (
tr._translate("MainWindow", "Disk full"),
tr._translate(
"MainWindow",
'Alert: Your disk or data storage volume is full. Bitmessage will now exit.'
), True)))
os._exit(0)
elif item == 'exit':
self.conn.close()
logger.info('sqlThread exiting gracefully.')
return
elif item == 'deleteandvacuume':
self.cur.execute('''delete from inbox where folder='trash' ''')
self.cur.execute('''delete from sent where folder='trash' ''')
self.conn.commit()
try:
self.cur.execute(''' VACUUM ''')
except Exception as err:
if str(err) == 'database or disk is full':
logger.fatal(
'(while deleteandvacuume) Alert: Your disk or data storage volume is full. sqlThread will now exit.'
)
queues.UISignalQueue.put(('alert', (
tr._translate("MainWindow", "Disk full"),
tr._translate(
"MainWindow",
'Alert: Your disk or data storage volume is full. Bitmessage will now exit.'
), True)))
os._exit(0)
else:
parameters = helper_sql.sqlSubmitQueue.get()
rowcount = 0
# print 'item', item
# print 'parameters', parameters
try:
self.cur.execute(item, parameters)
rowcount = self.cur.rowcount
except Exception as err:
if str(err) == 'database or disk is full':
logger.fatal(
'(while cur.execute) Alert: Your disk or data storage volume is full. sqlThread will now exit.'
)
queues.UISignalQueue.put(('alert', (
tr._translate("MainWindow", "Disk full"),
tr._translate(
"MainWindow",
'Alert: Your disk or data storage volume is full. Bitmessage will now exit.'
), True)))
os._exit(0)
else:
logger.fatal(
'Major error occurred when trying to execute a SQL statement within the sqlThread. Please tell Atheros about this error message or post it in the forum! Error occurred while trying to execute statement: "%s" Here are the parameters; you might want to censor this data with asterisks (***) as it can contain private information: %s. Here is the actual error message thrown by the sqlThread: %s',
str(item), str(repr(parameters)), str(err))
logger.fatal('This program shall now abruptly exit!')
os._exit(0)
helper_sql.sqlReturnQueue.put((self.cur.fetchall(), rowcount))
def run(self):
self.conn = sqlite3.connect(shared.appdata + 'messages.dat')
self.conn.text_factory = str
self.cur = self.conn.cursor()
self.cur.execute('PRAGMA secure_delete = true')
try:
self.cur.execute(
'''CREATE TABLE inbox (msgid blob, toaddress text, fromaddress text, subject text, received text, message text, folder text, encodingtype int, read bool, sighash blob, UNIQUE(msgid) ON CONFLICT REPLACE)''' )
self.cur.execute(
'''CREATE TABLE sent (msgid blob, toaddress text, toripe blob, fromaddress text, subject text, message text, ackdata blob, senttime integer, lastactiontime integer, sleeptill integer, status text, retrynumber integer, folder text, encodingtype int, ttl int)''' )
self.cur.execute(
'''CREATE TABLE subscriptions (label text, address text, enabled bool)''' )
self.cur.execute(
'''CREATE TABLE addressbook (label text, address text)''' )
self.cur.execute(
'''CREATE TABLE blacklist (label text, address text, enabled bool)''' )
self.cur.execute(
'''CREATE TABLE whitelist (label text, address text, enabled bool)''' )
self.cur.execute(
'''CREATE TABLE pubkeys (address text, addressversion int, transmitdata blob, time int, usedpersonally text, UNIQUE(address) ON CONFLICT REPLACE)''' )
self.cur.execute(
'''CREATE TABLE inventory (hash blob, objecttype int, streamnumber int, payload blob, expirestime integer, tag blob, UNIQUE(hash) ON CONFLICT REPLACE)''' )
self.cur.execute(
'''INSERT INTO subscriptions VALUES('Bitmessage new releases/announcements','BM-GtovgYdgs7qXPkoYaRgrLFuFKz1SFpsw',1)''')
self.cur.execute(
'''CREATE TABLE settings (key blob, value blob, UNIQUE(key) ON CONFLICT REPLACE)''' )
self.cur.execute( '''INSERT INTO settings VALUES('version','10')''')
self.cur.execute( '''INSERT INTO settings VALUES('lastvacuumtime',?)''', (
int(time.time()),))
self.cur.execute(
'''CREATE TABLE objectprocessorqueue (objecttype int, data blob, UNIQUE(objecttype, data) ON CONFLICT REPLACE)''' )
self.conn.commit()
logger.info('Created messages database file')
except Exception as err:
if str(err) == 'table inbox already exists':
logger.debug('Database file already exists.')
else:
sys.stderr.write(
'ERROR trying to create database file (message.dat). Error message: %s\n' % str(err))
os._exit(0)
if shared.config.getint('bitmessagesettings', 'settingsversion') == 1:
shared.config.set('bitmessagesettings', 'settingsversion', '2')
# If the settings version is equal to 2 or 3 then the
# sqlThread will modify the pubkeys table and change
# the settings version to 4.
shared.config.set('bitmessagesettings', 'socksproxytype', 'none')
shared.config.set('bitmessagesettings', 'sockshostname', 'localhost')
shared.config.set('bitmessagesettings', 'socksport', '9050')
shared.config.set('bitmessagesettings', 'socksauthentication', 'false')
shared.config.set('bitmessagesettings', 'socksusername', '')
shared.config.set('bitmessagesettings', 'sockspassword', '')
shared.config.set('bitmessagesettings', 'sockslisten', 'false')
shared.config.set('bitmessagesettings', 'keysencrypted', 'false')
shared.config.set('bitmessagesettings', 'messagesencrypted', 'false')
# People running earlier versions of PyBitmessage do not have the
# usedpersonally field in their pubkeys table. Let's add it.
if shared.config.getint('bitmessagesettings', 'settingsversion') == 2:
item = '''ALTER TABLE pubkeys ADD usedpersonally text DEFAULT 'no' '''
parameters = ''
self.cur.execute(item, parameters)
self.conn.commit()
shared.config.set('bitmessagesettings', 'settingsversion', '3')
# People running earlier versions of PyBitmessage do not have the
# encodingtype field in their inbox and sent tables or the read field
# in the inbox table. Let's add them.
if shared.config.getint('bitmessagesettings', 'settingsversion') == 3:
item = '''ALTER TABLE inbox ADD encodingtype int DEFAULT '2' '''
parameters = ''
self.cur.execute(item, parameters)
item = '''ALTER TABLE inbox ADD read bool DEFAULT '1' '''
parameters = ''
self.cur.execute(item, parameters)
item = '''ALTER TABLE sent ADD encodingtype int DEFAULT '2' '''
parameters = ''
self.cur.execute(item, parameters)
self.conn.commit()
shared.config.set('bitmessagesettings', 'settingsversion', '4')
if shared.config.getint('bitmessagesettings', 'settingsversion') == 4:
shared.config.set('bitmessagesettings', 'defaultnoncetrialsperbyte', str(
shared.networkDefaultProofOfWorkNonceTrialsPerByte))
shared.config.set('bitmessagesettings', 'defaultpayloadlengthextrabytes', str(
shared.networkDefaultPayloadLengthExtraBytes))
shared.config.set('bitmessagesettings', 'settingsversion', '5')
if shared.config.getint('bitmessagesettings', 'settingsversion') == 5:
shared.config.set(
'bitmessagesettings', 'maxacceptablenoncetrialsperbyte', '0')
shared.config.set(
'bitmessagesettings', 'maxacceptablepayloadlengthextrabytes', '0')
shared.config.set('bitmessagesettings', 'settingsversion', '6')
# From now on, let us keep a 'version' embedded in the messages.dat
# file so that when we make changes to the database, the database
# version we are on can stay embedded in the messages.dat file. Let us
# check to see if the settings table exists yet.
item = '''SELECT name FROM sqlite_master WHERE type='table' AND name='settings';'''
parameters = ''
self.cur.execute(item, parameters)
if self.cur.fetchall() == []:
# The settings table doesn't exist. We need to make it.
logger.debug('In messages.dat database, creating new \'settings\' table.')
self.cur.execute(
'''CREATE TABLE settings (key text, value blob, UNIQUE(key) ON CONFLICT REPLACE)''' )
self.cur.execute( '''INSERT INTO settings VALUES('version','1')''')
self.cur.execute( '''INSERT INTO settings VALUES('lastvacuumtime',?)''', (
int(time.time()),))
logger.debug('In messages.dat database, removing an obsolete field from the pubkeys table.')
self.cur.execute(
'''CREATE TEMPORARY TABLE pubkeys_backup(hash blob, transmitdata blob, time int, usedpersonally text, UNIQUE(hash) ON CONFLICT REPLACE);''')
self.cur.execute(
'''INSERT INTO pubkeys_backup SELECT hash, transmitdata, time, usedpersonally FROM pubkeys;''')
self.cur.execute( '''DROP TABLE pubkeys''')
self.cur.execute(
'''CREATE TABLE pubkeys (hash blob, transmitdata blob, time int, usedpersonally text, UNIQUE(hash) ON CONFLICT REPLACE)''' )
self.cur.execute(
'''INSERT INTO pubkeys SELECT hash, transmitdata, time, usedpersonally FROM pubkeys_backup;''')
self.cur.execute( '''DROP TABLE pubkeys_backup;''')
logger.debug('Deleting all pubkeys from inventory. They will be redownloaded and then saved with the correct times.')
self.cur.execute(
'''delete from inventory where objecttype = 'pubkey';''')
logger.debug('replacing Bitmessage announcements mailing list with a new one.')
self.cur.execute(
'''delete from subscriptions where address='BM-BbkPSZbzPwpVcYZpU4yHwf9ZPEapN5Zx' ''')
self.cur.execute(
'''INSERT INTO subscriptions VALUES('Bitmessage new releases/announcements','BM-GtovgYdgs7qXPkoYaRgrLFuFKz1SFpsw',1)''')
logger.debug('Commiting.')
self.conn.commit()
logger.debug('Vacuuming message.dat. You might notice that the file size gets much smaller.')
self.cur.execute( ''' VACUUM ''')
# After code refactoring, the possible status values for sent messages
# have changed.
self.cur.execute(
'''update sent set status='doingmsgpow' where status='doingpow' ''')
self.cur.execute(
'''update sent set status='msgsent' where status='sentmessage' ''')
self.cur.execute(
'''update sent set status='doingpubkeypow' where status='findingpubkey' ''')
self.cur.execute(
'''update sent set status='broadcastqueued' where status='broadcastpending' ''')
self.conn.commit()
if not shared.config.has_option('bitmessagesettings', 'sockslisten'):
shared.config.set('bitmessagesettings', 'sockslisten', 'false')
ensureNamecoinOptions()
"""# Add a new column to the inventory table to store the first 20 bytes of encrypted messages to support Android app
item = '''SELECT value FROM settings WHERE key='version';'''
parameters = ''
self.cur.execute(item, parameters)
if int(self.cur.fetchall()[0][0]) == 1:
print 'upgrading database'
item = '''ALTER TABLE inventory ADD first20bytesofencryptedmessage blob DEFAULT '' '''
parameters = ''
self.cur.execute(item, parameters)
item = '''update settings set value=? WHERE key='version';'''
parameters = (2,)
self.cur.execute(item, parameters)"""
# Let's get rid of the first20bytesofencryptedmessage field in the inventory table.
item = '''SELECT value FROM settings WHERE key='version';'''
parameters = ''
self.cur.execute(item, parameters)
if int(self.cur.fetchall()[0][0]) == 2:
logger.debug('In messages.dat database, removing an obsolete field from the inventory table.')
self.cur.execute(
'''CREATE TEMPORARY TABLE inventory_backup(hash blob, objecttype text, streamnumber int, payload blob, receivedtime integer, UNIQUE(hash) ON CONFLICT REPLACE);''')
self.cur.execute(
'''INSERT INTO inventory_backup SELECT hash, objecttype, streamnumber, payload, receivedtime FROM inventory;''')
self.cur.execute( '''DROP TABLE inventory''')
self.cur.execute(
'''CREATE TABLE inventory (hash blob, objecttype text, streamnumber int, payload blob, receivedtime integer, UNIQUE(hash) ON CONFLICT REPLACE)''' )
self.cur.execute(
'''INSERT INTO inventory SELECT hash, objecttype, streamnumber, payload, receivedtime FROM inventory_backup;''')
self.cur.execute( '''DROP TABLE inventory_backup;''')
item = '''update settings set value=? WHERE key='version';'''
parameters = (3,)
self.cur.execute(item, parameters)
# Add a new column to the inventory table to store tags.
item = '''SELECT value FROM settings WHERE key='version';'''
parameters = ''
self.cur.execute(item, parameters)
currentVersion = int(self.cur.fetchall()[0][0])
if currentVersion == 1 or currentVersion == 3:
logger.debug('In messages.dat database, adding tag field to the inventory table.')
item = '''ALTER TABLE inventory ADD tag blob DEFAULT '' '''
parameters = ''
self.cur.execute(item, parameters)
item = '''update settings set value=? WHERE key='version';'''
parameters = (4,)
self.cur.execute(item, parameters)
if not shared.config.has_option('bitmessagesettings', 'userlocale'):
shared.config.set('bitmessagesettings', 'userlocale', 'system')
if not shared.config.has_option('bitmessagesettings', 'sendoutgoingconnections'):
shared.config.set('bitmessagesettings', 'sendoutgoingconnections', 'True')
# Raise the default required difficulty from 1 to 2
# With the change to protocol v3, this is obsolete.
if shared.config.getint('bitmessagesettings', 'settingsversion') == 6:
"""if int(shared.config.get('bitmessagesettings','defaultnoncetrialsperbyte')) == shared.networkDefaultProofOfWorkNonceTrialsPerByte:
shared.config.set('bitmessagesettings','defaultnoncetrialsperbyte', str(shared.networkDefaultProofOfWorkNonceTrialsPerByte * 2))
"""
shared.config.set('bitmessagesettings', 'settingsversion', '7')
# Add a new column to the pubkeys table to store the address version.
# We're going to trash all of our pubkeys and let them be redownloaded.
item = '''SELECT value FROM settings WHERE key='version';'''
parameters = ''
self.cur.execute(item, parameters)
currentVersion = int(self.cur.fetchall()[0][0])
if currentVersion == 4:
self.cur.execute( '''DROP TABLE pubkeys''')
self.cur.execute(
'''CREATE TABLE pubkeys (hash blob, addressversion int, transmitdata blob, time int, usedpersonally text, UNIQUE(hash, addressversion) ON CONFLICT REPLACE)''' )
self.cur.execute(
'''delete from inventory where objecttype = 'pubkey';''')
item = '''update settings set value=? WHERE key='version';'''
parameters = (5,)
self.cur.execute(item, parameters)
if not shared.config.has_option('bitmessagesettings', 'useidenticons'):
shared.config.set('bitmessagesettings', 'useidenticons', 'True')
if not shared.config.has_option('bitmessagesettings', 'identiconsuffix'): # acts as a salt
shared.config.set('bitmessagesettings', 'identiconsuffix', ''.join(random.choice("123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz") for x in range(12))) # a twelve character pseudo-password to salt the identicons
#Add settings to support no longer resending messages after a certain period of time even if we never get an ack
if shared.config.getint('bitmessagesettings', 'settingsversion') == 7:
shared.config.set(
'bitmessagesettings', 'stopresendingafterxdays', '')
shared.config.set(
'bitmessagesettings', 'stopresendingafterxmonths', '')
shared.config.set('bitmessagesettings', 'settingsversion', '8')
# Add a new table: objectprocessorqueue with which to hold objects
# that have yet to be processed if the user shuts down Bitmessage.
item = '''SELECT value FROM settings WHERE key='version';'''
parameters = ''
self.cur.execute(item, parameters)
currentVersion = int(self.cur.fetchall()[0][0])
if currentVersion == 5:
self.cur.execute( '''DROP TABLE knownnodes''')
self.cur.execute(
'''CREATE TABLE objectprocessorqueue (objecttype text, data blob, UNIQUE(objecttype, data) ON CONFLICT REPLACE)''' )
item = '''update settings set value=? WHERE key='version';'''
parameters = (6,)
self.cur.execute(item, parameters)
# changes related to protocol v3
# In table inventory and objectprocessorqueue, objecttype is now an integer (it was a human-friendly string previously)
item = '''SELECT value FROM settings WHERE key='version';'''
parameters = ''
self.cur.execute(item, parameters)
currentVersion = int(self.cur.fetchall()[0][0])
if currentVersion == 6:
logger.debug('In messages.dat database, dropping and recreating the inventory table.')
self.cur.execute( '''DROP TABLE inventory''')
self.cur.execute( '''CREATE TABLE inventory (hash blob, objecttype int, streamnumber int, payload blob, expirestime integer, tag blob, UNIQUE(hash) ON CONFLICT REPLACE)''' )
self.cur.execute( '''DROP TABLE objectprocessorqueue''')
self.cur.execute( '''CREATE TABLE objectprocessorqueue (objecttype int, data blob, UNIQUE(objecttype, data) ON CONFLICT REPLACE)''' )
item = '''update settings set value=? WHERE key='version';'''
parameters = (7,)
self.cur.execute(item, parameters)
logger.debug('Finished dropping and recreating the inventory table.')
# With the change to protocol version 3, reset the user-settable difficulties to 1
if shared.config.getint('bitmessagesettings', 'settingsversion') == 8:
shared.config.set('bitmessagesettings','defaultnoncetrialsperbyte', str(shared.networkDefaultProofOfWorkNonceTrialsPerByte))
shared.config.set('bitmessagesettings','defaultpayloadlengthextrabytes', str(shared.networkDefaultPayloadLengthExtraBytes))
previousTotalDifficulty = int(shared.config.getint('bitmessagesettings', 'maxacceptablenoncetrialsperbyte')) / 320
previousSmallMessageDifficulty = int(shared.config.getint('bitmessagesettings', 'maxacceptablepayloadlengthextrabytes')) / 14000
shared.config.set('bitmessagesettings','maxacceptablenoncetrialsperbyte', str(previousTotalDifficulty * 1000))
shared.config.set('bitmessagesettings','maxacceptablepayloadlengthextrabytes', str(previousSmallMessageDifficulty * 1000))
shared.config.set('bitmessagesettings', 'settingsversion', '9')
# Adjust the required POW values for each of this user's addresses to conform to protocol v3 norms.
if shared.config.getint('bitmessagesettings', 'settingsversion') == 9:
for addressInKeysFile in shared.config.sections():
try:
previousTotalDifficulty = float(shared.config.getint(addressInKeysFile, 'noncetrialsperbyte')) / 320
previousSmallMessageDifficulty = float(shared.config.getint(addressInKeysFile, 'payloadlengthextrabytes')) / 14000
if previousTotalDifficulty <= 2:
previousTotalDifficulty = 1
if previousSmallMessageDifficulty < 1:
previousSmallMessageDifficulty = 1
shared.config.set(addressInKeysFile,'noncetrialsperbyte', str(int(previousTotalDifficulty * 1000)))
shared.config.set(addressInKeysFile,'payloadlengthextrabytes', str(int(previousSmallMessageDifficulty * 1000)))
except:
continue
shared.config.set('bitmessagesettings', 'maxdownloadrate', '0')
shared.config.set('bitmessagesettings', 'maxuploadrate', '0')
shared.config.set('bitmessagesettings', 'settingsversion', '10')
shared.writeKeysFile()
# sanity check
if shared.config.getint('bitmessagesettings', 'maxacceptablenoncetrialsperbyte') == 0:
shared.config.set('bitmessagesettings','maxacceptablenoncetrialsperbyte', str(shared.ridiculousDifficulty * shared.networkDefaultProofOfWorkNonceTrialsPerByte))
if shared.config.getint('bitmessagesettings', 'maxacceptablepayloadlengthextrabytes') == 0:
shared.config.set('bitmessagesettings','maxacceptablepayloadlengthextrabytes', str(shared.ridiculousDifficulty * shared.networkDefaultPayloadLengthExtraBytes))
# The format of data stored in the pubkeys table has changed. Let's
# clear it, and the pubkeys from inventory, so that they'll be re-downloaded.
item = '''SELECT value FROM settings WHERE key='version';'''
parameters = ''
self.cur.execute(item, parameters)
currentVersion = int(self.cur.fetchall()[0][0])
if currentVersion == 7:
logger.debug('In messages.dat database, clearing pubkeys table because the data format has been updated.')
self.cur.execute(
'''delete from inventory where objecttype = 1;''')
self.cur.execute(
'''delete from pubkeys;''')
# Any sending messages for which we *thought* that we had the pubkey must
# be rechecked.
self.cur.execute(
'''UPDATE sent SET status='msgqueued' WHERE status='doingmsgpow' or status='badkey';''')
query = '''update settings set value=? WHERE key='version';'''
parameters = (8,)
self.cur.execute(query, parameters)
logger.debug('Finished clearing currently held pubkeys.')
# Add a new column to the inbox table to store the hash of the message signature.
# We'll use this as temporary message UUID in order to detect duplicates.
item = '''SELECT value FROM settings WHERE key='version';'''
parameters = ''
self.cur.execute(item, parameters)
currentVersion = int(self.cur.fetchall()[0][0])
if currentVersion == 8:
logger.debug('In messages.dat database, adding sighash field to the inbox table.')
item = '''ALTER TABLE inbox ADD sighash blob DEFAULT '' '''
parameters = ''
self.cur.execute(item, parameters)
item = '''update settings set value=? WHERE key='version';'''
parameters = (9,)
self.cur.execute(item, parameters)
# TTL is now user-specifiable. Let's add an option to save whatever the user selects.
if not shared.config.has_option('bitmessagesettings', 'ttl'):
shared.config.set('bitmessagesettings', 'ttl', '367200')
# We'll also need a `sleeptill` field and a `ttl` field. Also we can combine
# the pubkeyretrynumber and msgretrynumber into one.
item = '''SELECT value FROM settings WHERE key='version';'''
parameters = ''
self.cur.execute(item, parameters)
currentVersion = int(self.cur.fetchall()[0][0])
if currentVersion == 9:
logger.info('In messages.dat database, making TTL-related changes: combining the pubkeyretrynumber and msgretrynumber fields into the retrynumber field and adding the sleeptill and ttl fields...')
self.cur.execute(
'''CREATE TEMPORARY TABLE sent_backup (msgid blob, toaddress text, toripe blob, fromaddress text, subject text, message text, ackdata blob, lastactiontime integer, status text, retrynumber integer, folder text, encodingtype int)''' )
self.cur.execute(
'''INSERT INTO sent_backup SELECT msgid, toaddress, toripe, fromaddress, subject, message, ackdata, lastactiontime, status, 0, folder, encodingtype FROM sent;''')
self.cur.execute( '''DROP TABLE sent''')
self.cur.execute(
'''CREATE TABLE sent (msgid blob, toaddress text, toripe blob, fromaddress text, subject text, message text, ackdata blob, senttime integer, lastactiontime integer, sleeptill int, status text, retrynumber integer, folder text, encodingtype int, ttl int)''' )
self.cur.execute(
'''INSERT INTO sent SELECT msgid, toaddress, toripe, fromaddress, subject, message, ackdata, lastactiontime, lastactiontime, 0, status, 0, folder, encodingtype, 216000 FROM sent_backup;''')
self.cur.execute( '''DROP TABLE sent_backup''')
logger.info('In messages.dat database, finished making TTL-related changes.')
logger.debug('In messages.dat database, adding address field to the pubkeys table.')
# We're going to have to calculate the address for each row in the pubkeys
# table. Then we can take out the hash field.
self.cur.execute('''ALTER TABLE pubkeys ADD address text DEFAULT '' ''')
self.cur.execute('''SELECT hash, addressversion FROM pubkeys''')
queryResult = self.cur.fetchall()
from addresses import encodeAddress
for row in queryResult:
hash, addressVersion = row
address = encodeAddress(addressVersion, 1, hash)
item = '''UPDATE pubkeys SET address=? WHERE hash=?;'''
parameters = (address, hash)
self.cur.execute(item, parameters)
# Now we can remove the hash field from the pubkeys table.
self.cur.execute(
'''CREATE TEMPORARY TABLE pubkeys_backup (address text, addressversion int, transmitdata blob, time int, usedpersonally text, UNIQUE(address) ON CONFLICT REPLACE)''' )
self.cur.execute(
'''INSERT INTO pubkeys_backup SELECT address, addressversion, transmitdata, time, usedpersonally FROM pubkeys;''')
self.cur.execute( '''DROP TABLE pubkeys''')
self.cur.execute(
'''CREATE TABLE pubkeys (address text, addressversion int, transmitdata blob, time int, usedpersonally text, UNIQUE(address) ON CONFLICT REPLACE)''' )
self.cur.execute(
'''INSERT INTO pubkeys SELECT address, addressversion, transmitdata, time, usedpersonally FROM pubkeys_backup;''')
self.cur.execute( '''DROP TABLE pubkeys_backup''')
logger.debug('In messages.dat database, done adding address field to the pubkeys table and removing the hash field.')
self.cur.execute('''update settings set value=10 WHERE key='version';''')
# Are you hoping to add a new option to the keys.dat file of existing
# Bitmessage users or modify the SQLite database? Add it right above this line!
try:
testpayload = '\x00\x00'
t = ('1234', 1, testpayload, '12345678', 'no')
self.cur.execute( '''INSERT INTO pubkeys VALUES(?,?,?,?,?)''', t)
self.conn.commit()
self.cur.execute(
'''SELECT transmitdata FROM pubkeys WHERE address='1234' ''')
queryreturn = self.cur.fetchall()
for row in queryreturn:
transmitdata, = row
self.cur.execute('''DELETE FROM pubkeys WHERE address='1234' ''')
self.conn.commit()
if transmitdata == '':
logger.fatal('Problem: The version of SQLite you have cannot store Null values. Please download and install the latest revision of your version of Python (for example, the latest Python 2.7 revision) and try again.\n')
logger.fatal('PyBitmessage will now exit very abruptly. You may now see threading errors related to this abrupt exit but the problem you need to solve is related to SQLite.\n\n')
os._exit(0)
except Exception as err:
if str(err) == 'database or disk is full':
logger.fatal('(While null value test) Alert: Your disk or data storage volume is full. sqlThread will now exit.')
shared.UISignalQueue.put(('alert', (tr._translate("MainWindow", "Disk full"), tr._translate("MainWindow", 'Alert: Your disk or data storage volume is full. Bitmessage will now exit.'), True)))
if shared.daemon:
os._exit(0)
else:
return
else:
logger.error(err)
# Let us check to see the last time we vaccumed the messages.dat file.
# If it has been more than a month let's do it now.
item = '''SELECT value FROM settings WHERE key='lastvacuumtime';'''
parameters = ''
self.cur.execute(item, parameters)
queryreturn = self.cur.fetchall()
for row in queryreturn:
value, = row
if int(value) < int(time.time()) - 86400:
logger.info('It has been a long time since the messages.dat file has been vacuumed. Vacuuming now...')
try:
self.cur.execute( ''' VACUUM ''')
except Exception as err:
if str(err) == 'database or disk is full':
logger.fatal('(While VACUUM) Alert: Your disk or data storage volume is full. sqlThread will now exit.')
shared.UISignalQueue.put(('alert', (tr._translate("MainWindow", "Disk full"), tr._translate("MainWindow", 'Alert: Your disk or data storage volume is full. Bitmessage will now exit.'), True)))
if shared.daemon:
os._exit(0)
else:
return
item = '''update settings set value=? WHERE key='lastvacuumtime';'''
parameters = (int(time.time()),)
self.cur.execute(item, parameters)
while True:
item = shared.sqlSubmitQueue.get()
if item == 'commit':
try:
self.conn.commit()
except Exception as err:
if str(err) == 'database or disk is full':
logger.fatal('(While committing) Alert: Your disk or data storage volume is full. sqlThread will now exit.')
shared.UISignalQueue.put(('alert', (tr._translate("MainWindow", "Disk full"), tr._translate("MainWindow", 'Alert: Your disk or data storage volume is full. Bitmessage will now exit.'), True)))
if shared.daemon:
os._exit(0)
else:
return
elif item == 'exit':
self.conn.close()
logger.info('sqlThread exiting gracefully.')
return
elif item == 'movemessagstoprog':
logger.debug('the sqlThread is moving the messages.dat file to the local program directory.')
try:
self.conn.commit()
except Exception as err:
if str(err) == 'database or disk is full':
logger.fatal('(while movemessagstoprog) Alert: Your disk or data storage volume is full. sqlThread will now exit.')
shared.UISignalQueue.put(('alert', (tr._translate("MainWindow", "Disk full"), tr._translate("MainWindow", 'Alert: Your disk or data storage volume is full. Bitmessage will now exit.'), True)))
if shared.daemon:
os._exit(0)
else:
return
self.conn.close()
shutil.move(
shared.lookupAppdataFolder() + 'messages.dat', shared.lookupExeFolder() + 'messages.dat')
self.conn = sqlite3.connect(shared.lookupExeFolder() + 'messages.dat')
self.conn.text_factory = str
self.cur = self.conn.cursor()
elif item == 'movemessagstoappdata':
logger.debug('the sqlThread is moving the messages.dat file to the Appdata folder.')
try:
self.conn.commit()
except Exception as err:
if str(err) == 'database or disk is full':
logger.fatal('(while movemessagstoappdata) Alert: Your disk or data storage volume is full. sqlThread will now exit.')
shared.UISignalQueue.put(('alert', (tr._translate("MainWindow", "Disk full"), tr._translate("MainWindow", 'Alert: Your disk or data storage volume is full. Bitmessage will now exit.'), True)))
if shared.daemon:
os._exit(0)
else:
return
self.conn.close()
shutil.move(
shared.lookupExeFolder() + 'messages.dat', shared.lookupAppdataFolder() + 'messages.dat')
self.conn = sqlite3.connect(shared.lookupAppdataFolder() + 'messages.dat')
self.conn.text_factory = str
self.cur = self.conn.cursor()
elif item == 'deleteandvacuume':
self.cur.execute('''delete from inbox where folder='trash' ''')
self.cur.execute('''delete from sent where folder='trash' ''')
self.conn.commit()
try:
self.cur.execute( ''' VACUUM ''')
except Exception as err:
if str(err) == 'database or disk is full':
logger.fatal('(while deleteandvacuume) Alert: Your disk or data storage volume is full. sqlThread will now exit.')
shared.UISignalQueue.put(('alert', (tr._translate("MainWindow", "Disk full"), tr._translate("MainWindow", 'Alert: Your disk or data storage volume is full. Bitmessage will now exit.'), True)))
if shared.daemon:
os._exit(0)
else:
return
else:
parameters = shared.sqlSubmitQueue.get()
rowcount = 0
# print 'item', item
# print 'parameters', parameters
try:
self.cur.execute(item, parameters)
rowcount = self.cur.rowcount
except Exception as err:
if str(err) == 'database or disk is full':
logger.fatal('(while cur.execute) Alert: Your disk or data storage volume is full. sqlThread will now exit.')
shared.UISignalQueue.put(('alert', (tr._translate("MainWindow", "Disk full"), tr._translate("MainWindow", 'Alert: Your disk or data storage volume is full. Bitmessage will now exit.'), True)))
if shared.daemon:
os._exit(0)
else:
return
else:
logger.fatal('Major error occurred when trying to execute a SQL statement within the sqlThread. Please tell Atheros about this error message or post it in the forum! Error occurred while trying to execute statement: "%s" Here are the parameters; you might want to censor this data with asterisks (***) as it can contain private information: %s. Here is the actual error message thrown by the sqlThread: %s', str(item), str(repr(parameters)), str(err))
logger.fatal('This program shall now abruptly exit!')
os._exit(0)
shared.sqlReturnQueue.put((self.cur.fetchall(), rowcount))
def processbroadcast(self, data):
messageProcessingStartTime = time.time()
shared.numberOfBroadcastsProcessed += 1
queues.UISignalQueue.put((
'updateNumberOfBroadcastsProcessed', 'no data'))
inventoryHash = calculateInventoryHash(data)
readPosition = 20 # bypass the nonce, time, and object type
broadcastVersion, broadcastVersionLength = decodeVarint(
data[readPosition:readPosition + 9])
readPosition += broadcastVersionLength
if broadcastVersion < 4 or broadcastVersion > 5:
logger.info('Cannot decode incoming broadcast versions less than 4 or higher than 5. Assuming the sender isn\'t being silly, you should upgrade Bitmessage because this message shall be ignored.')
return
cleartextStreamNumber, cleartextStreamNumberLength = decodeVarint(
data[readPosition:readPosition + 10])
readPosition += cleartextStreamNumberLength
if broadcastVersion == 4:
"""
v4 broadcasts are encrypted the same way the msgs are encrypted. To see if we are interested in a
v4 broadcast, we try to decrypt it. This was replaced with v5 broadcasts which include a tag which
we check instead, just like we do with v4 pubkeys.
"""
signedData = data[8:readPosition]
initialDecryptionSuccessful = False
for key, cryptorObject in shared.MyECSubscriptionCryptorObjects.items():
try:
if initialDecryptionSuccessful: # continue decryption attempts to avoid timing attacks
cryptorObject.decrypt(data[readPosition:])
else:
decryptedData = cryptorObject.decrypt(data[readPosition:])
toRipe = key # This is the RIPE hash of the sender's pubkey. We need this below to compare to the RIPE hash of the sender's address to verify that it was encrypted by with their key rather than some other key.
initialDecryptionSuccessful = True
logger.info('EC decryption successful using key associated with ripe hash: %s' % hexlify(key))
except Exception as err:
pass
# print 'cryptorObject.decrypt Exception:', err
if not initialDecryptionSuccessful:
# This is not a broadcast I am interested in.
logger.debug('Length of time program spent failing to decrypt this v4 broadcast: %s seconds.' % (time.time() - messageProcessingStartTime,))
return
elif broadcastVersion == 5:
embeddedTag = data[readPosition:readPosition+32]
readPosition += 32
if embeddedTag not in shared.MyECSubscriptionCryptorObjects:
logger.debug('We\'re not interested in this broadcast.')
return
# We are interested in this broadcast because of its tag.
signedData = data[8:readPosition] # We're going to add some more data which is signed further down.
cryptorObject = shared.MyECSubscriptionCryptorObjects[embeddedTag]
try:
decryptedData = cryptorObject.decrypt(data[readPosition:])
logger.debug('EC decryption successful')
except Exception as err:
logger.debug('Broadcast version %s decryption Unsuccessful.' % broadcastVersion)
return
# At this point this is a broadcast I have decrypted and am
# interested in.
readPosition = 0
sendersAddressVersion, sendersAddressVersionLength = decodeVarint(
decryptedData[readPosition:readPosition + 9])
if broadcastVersion == 4:
if sendersAddressVersion < 2 or sendersAddressVersion > 3:
logger.warning('Cannot decode senderAddressVersion other than 2 or 3. Assuming the sender isn\'t being silly, you should upgrade Bitmessage because this message shall be ignored.')
return
elif broadcastVersion == 5:
if sendersAddressVersion < 4:
logger.info('Cannot decode senderAddressVersion less than 4 for broadcast version number 5. Assuming the sender isn\'t being silly, you should upgrade Bitmessage because this message shall be ignored.')
return
readPosition += sendersAddressVersionLength
sendersStream, sendersStreamLength = decodeVarint(
decryptedData[readPosition:readPosition + 9])
if sendersStream != cleartextStreamNumber:
logger.info('The stream number outside of the encryption on which the POW was completed doesn\'t match the stream number inside the encryption. Ignoring broadcast.')
return
readPosition += sendersStreamLength
behaviorBitfield = decryptedData[readPosition:readPosition + 4]
readPosition += 4
sendersPubSigningKey = '\x04' + \
decryptedData[readPosition:readPosition + 64]
readPosition += 64
sendersPubEncryptionKey = '\x04' + \
decryptedData[readPosition:readPosition + 64]
readPosition += 64
if sendersAddressVersion >= 3:
requiredAverageProofOfWorkNonceTrialsPerByte, varintLength = decodeVarint(
decryptedData[readPosition:readPosition + 10])
readPosition += varintLength
logger.debug('sender\'s requiredAverageProofOfWorkNonceTrialsPerByte is %s' % requiredAverageProofOfWorkNonceTrialsPerByte)
requiredPayloadLengthExtraBytes, varintLength = decodeVarint(
decryptedData[readPosition:readPosition + 10])
readPosition += varintLength
logger.debug('sender\'s requiredPayloadLengthExtraBytes is %s' % requiredPayloadLengthExtraBytes)
endOfPubkeyPosition = readPosition
sha = hashlib.new('sha512')
sha.update(sendersPubSigningKey + sendersPubEncryptionKey)
ripeHasher = hashlib.new('ripemd160')
ripeHasher.update(sha.digest())
calculatedRipe = ripeHasher.digest()
if broadcastVersion == 4:
if toRipe != calculatedRipe:
logger.info('The encryption key used to encrypt this message doesn\'t match the keys inbedded in the message itself. Ignoring message.')
return
elif broadcastVersion == 5:
calculatedTag = hashlib.sha512(hashlib.sha512(encodeVarint(
sendersAddressVersion) + encodeVarint(sendersStream) + calculatedRipe).digest()).digest()[32:]
if calculatedTag != embeddedTag:
logger.debug('The tag and encryption key used to encrypt this message doesn\'t match the keys inbedded in the message itself. Ignoring message.')
return
messageEncodingType, messageEncodingTypeLength = decodeVarint(
decryptedData[readPosition:readPosition + 9])
if messageEncodingType == 0:
return
readPosition += messageEncodingTypeLength
messageLength, messageLengthLength = decodeVarint(
decryptedData[readPosition:readPosition + 9])
readPosition += messageLengthLength
message = decryptedData[readPosition:readPosition + messageLength]
readPosition += messageLength
readPositionAtBottomOfMessage = readPosition
signatureLength, signatureLengthLength = decodeVarint(
decryptedData[readPosition:readPosition + 9])
readPosition += signatureLengthLength
signature = decryptedData[
readPosition:readPosition + signatureLength]
signedData += decryptedData[:readPositionAtBottomOfMessage]
if not highlevelcrypto.verify(signedData, signature, hexlify(sendersPubSigningKey)):
logger.debug('ECDSA verify failed')
return
logger.debug('ECDSA verify passed')
sigHash = hashlib.sha512(hashlib.sha512(signature).digest()).digest()[32:] # Used to detect and ignore duplicate messages in our inbox
fromAddress = encodeAddress(
sendersAddressVersion, sendersStream, calculatedRipe)
logger.info('fromAddress: %s' % fromAddress)
# Let's store the public key in case we want to reply to this person.
sqlExecute('''INSERT INTO pubkeys VALUES (?,?,?,?,?)''',
fromAddress,
sendersAddressVersion,
decryptedData[:endOfPubkeyPosition],
int(time.time()),
'yes')
# Check to see whether we happen to be awaiting this
# pubkey in order to send a message. If we are, it will do the POW
# and send it.
self.possibleNewPubkey(fromAddress)
fromAddress = encodeAddress(
sendersAddressVersion, sendersStream, calculatedRipe)
logger.debug('fromAddress: ' + fromAddress)
decodedMessage = helper_msgcoding.MsgDecode(messageEncodingType, message)
subject = decodedMessage.subject
body = decodedMessage.body
toAddress = '[Broadcast subscribers]'
if helper_inbox.isMessageAlreadyInInbox(sigHash):
logger.info('This broadcast is already in our inbox. Ignoring it.')
return
t = (inventoryHash, toAddress, fromAddress, subject, int(
time.time()), body, 'inbox', messageEncodingType, 0, sigHash)
helper_inbox.insert(t)
queues.UISignalQueue.put(('displayNewInboxMessage', (
inventoryHash, toAddress, fromAddress, subject, body)))
# If we are behaving as an API then we might need to run an
# outside command to let some program know that a new message
# has arrived.
if BMConfigParser().safeGetBoolean('bitmessagesettings', 'apienabled'):
try:
apiNotifyPath = BMConfigParser().get(
'bitmessagesettings', 'apinotifypath')
except:
apiNotifyPath = ''
if apiNotifyPath != '':
call([apiNotifyPath, "newBroadcast"])
# Display timing data
logger.info('Time spent processing this interesting broadcast: %s' % (time.time() - messageProcessingStartTime,))
def processmsg(self, data):
messageProcessingStartTime = time.time()
shared.numberOfMessagesProcessed += 1
queues.UISignalQueue.put((
'updateNumberOfMessagesProcessed', 'no data'))
readPosition = 20 # bypass the nonce, time, and object type
msgVersion, msgVersionLength = decodeVarint(data[readPosition:readPosition + 9])
if msgVersion != 1:
logger.info('Cannot understand message versions other than one. Ignoring message.')
return
readPosition += msgVersionLength
streamNumberAsClaimedByMsg, streamNumberAsClaimedByMsgLength = decodeVarint(
data[readPosition:readPosition + 9])
readPosition += streamNumberAsClaimedByMsgLength
inventoryHash = calculateInventoryHash(data)
initialDecryptionSuccessful = False
# This is not an acknowledgement bound for me. See if it is a message
# bound for me by trying to decrypt it with my private keys.
for key, cryptorObject in shared.myECCryptorObjects.items():
try:
if initialDecryptionSuccessful: # continue decryption attempts to avoid timing attacks
cryptorObject.decrypt(data[readPosition:])
else:
decryptedData = cryptorObject.decrypt(data[readPosition:])
toRipe = key # This is the RIPE hash of my pubkeys. We need this below to compare to the destination_ripe included in the encrypted data.
initialDecryptionSuccessful = True
logger.info('EC decryption successful using key associated with ripe hash: %s.' % hexlify(key))
except Exception as err:
pass
if not initialDecryptionSuccessful:
# This is not a message bound for me.
logger.info('Length of time program spent failing to decrypt this message: %s seconds.' % (time.time() - messageProcessingStartTime,))
return
# This is a message bound for me.
toAddress = shared.myAddressesByHash[
toRipe] # Look up my address based on the RIPE hash.
readPosition = 0
sendersAddressVersionNumber, sendersAddressVersionNumberLength = decodeVarint(
decryptedData[readPosition:readPosition + 10])
readPosition += sendersAddressVersionNumberLength
if sendersAddressVersionNumber == 0:
logger.info('Cannot understand sendersAddressVersionNumber = 0. Ignoring message.')
return
if sendersAddressVersionNumber > 4:
logger.info('Sender\'s address version number %s not yet supported. Ignoring message.' % sendersAddressVersionNumber)
return
if len(decryptedData) < 170:
logger.info('Length of the unencrypted data is unreasonably short. Sanity check failed. Ignoring message.')
return
sendersStreamNumber, sendersStreamNumberLength = decodeVarint(
decryptedData[readPosition:readPosition + 10])
if sendersStreamNumber == 0:
logger.info('sender\'s stream number is 0. Ignoring message.')
return
readPosition += sendersStreamNumberLength
behaviorBitfield = decryptedData[readPosition:readPosition + 4]
readPosition += 4
pubSigningKey = '\x04' + decryptedData[
readPosition:readPosition + 64]
readPosition += 64
pubEncryptionKey = '\x04' + decryptedData[
readPosition:readPosition + 64]
readPosition += 64
if sendersAddressVersionNumber >= 3:
requiredAverageProofOfWorkNonceTrialsPerByte, varintLength = decodeVarint(
decryptedData[readPosition:readPosition + 10])
readPosition += varintLength
logger.info('sender\'s requiredAverageProofOfWorkNonceTrialsPerByte is %s' % requiredAverageProofOfWorkNonceTrialsPerByte)
requiredPayloadLengthExtraBytes, varintLength = decodeVarint(
decryptedData[readPosition:readPosition + 10])
readPosition += varintLength
logger.info('sender\'s requiredPayloadLengthExtraBytes is %s' % requiredPayloadLengthExtraBytes)
endOfThePublicKeyPosition = readPosition # needed for when we store the pubkey in our database of pubkeys for later use.
if toRipe != decryptedData[readPosition:readPosition + 20]:
logger.info('The original sender of this message did not send it to you. Someone is attempting a Surreptitious Forwarding Attack.\n\
See: http://world.std.com/~dtd/sign_encrypt/sign_encrypt7.html \n\
your toRipe: %s\n\
embedded destination toRipe: %s' % (hexlify(toRipe), hexlify(decryptedData[readPosition:readPosition + 20]))
)
return
readPosition += 20
messageEncodingType, messageEncodingTypeLength = decodeVarint(
decryptedData[readPosition:readPosition + 10])
readPosition += messageEncodingTypeLength
messageLength, messageLengthLength = decodeVarint(
decryptedData[readPosition:readPosition + 10])
readPosition += messageLengthLength
message = decryptedData[readPosition:readPosition + messageLength]
# print 'First 150 characters of message:', repr(message[:150])
readPosition += messageLength
ackLength, ackLengthLength = decodeVarint(
decryptedData[readPosition:readPosition + 10])
readPosition += ackLengthLength
ackData = decryptedData[readPosition:readPosition + ackLength]
readPosition += ackLength
positionOfBottomOfAckData = readPosition # needed to mark the end of what is covered by the signature
signatureLength, signatureLengthLength = decodeVarint(
decryptedData[readPosition:readPosition + 10])
readPosition += signatureLengthLength
signature = decryptedData[
readPosition:readPosition + signatureLength]
signedData = data[8:20] + encodeVarint(1) + encodeVarint(streamNumberAsClaimedByMsg) + decryptedData[:positionOfBottomOfAckData]
if not highlevelcrypto.verify(signedData, signature, hexlify(pubSigningKey)):
logger.debug('ECDSA verify failed')
return
logger.debug('ECDSA verify passed')
sigHash = hashlib.sha512(hashlib.sha512(signature).digest()).digest()[32:] # Used to detect and ignore duplicate messages in our inbox
# calculate the fromRipe.
sha = hashlib.new('sha512')
sha.update(pubSigningKey + pubEncryptionKey)
ripe = hashlib.new('ripemd160')
ripe.update(sha.digest())
fromAddress = encodeAddress(
sendersAddressVersionNumber, sendersStreamNumber, ripe.digest())
# Let's store the public key in case we want to reply to this
# person.
sqlExecute(
'''INSERT INTO pubkeys VALUES (?,?,?,?,?)''',
fromAddress,
sendersAddressVersionNumber,
decryptedData[:endOfThePublicKeyPosition],
int(time.time()),
'yes')
# Check to see whether we happen to be awaiting this
# pubkey in order to send a message. If we are, it will do the POW
# and send it.
self.possibleNewPubkey(fromAddress)
# If this message is bound for one of my version 3 addresses (or
# higher), then we must check to make sure it meets our demanded
# proof of work requirement. If this is bound for one of my chan
# addresses then we skip this check; the minimum network POW is
# fine.
if decodeAddress(toAddress)[1] >= 3 and not BMConfigParser().safeGetBoolean(toAddress, 'chan'): # If the toAddress version number is 3 or higher and not one of my chan addresses:
if not shared.isAddressInMyAddressBookSubscriptionsListOrWhitelist(fromAddress): # If I'm not friendly with this person:
requiredNonceTrialsPerByte = BMConfigParser().getint(
toAddress, 'noncetrialsperbyte')
requiredPayloadLengthExtraBytes = BMConfigParser().getint(
toAddress, 'payloadlengthextrabytes')
if not protocol.isProofOfWorkSufficient(data, requiredNonceTrialsPerByte, requiredPayloadLengthExtraBytes):
logger.info('Proof of work in msg is insufficient only because it does not meet our higher requirement.')
return
blockMessage = False # Gets set to True if the user shouldn't see the message according to black or white lists.
if BMConfigParser().get('bitmessagesettings', 'blackwhitelist') == 'black': # If we are using a blacklist
queryreturn = sqlQuery(
'''SELECT label FROM blacklist where address=? and enabled='1' ''',
fromAddress)
if queryreturn != []:
logger.info('Message ignored because address is in blacklist.')
blockMessage = True
else: # We're using a whitelist
queryreturn = sqlQuery(
'''SELECT label FROM whitelist where address=? and enabled='1' ''',
fromAddress)
if queryreturn == []:
logger.info('Message ignored because address not in whitelist.')
blockMessage = True
toLabel = BMConfigParser().get(toAddress, 'label')
if toLabel == '':
toLabel = toAddress
decodedMessage = helper_msgcoding.MsgDecode(messageEncodingType, message)
subject = decodedMessage.subject
body = decodedMessage.body
# Let us make sure that we haven't already received this message
if helper_inbox.isMessageAlreadyInInbox(sigHash):
logger.info('This msg is already in our inbox. Ignoring it.')
blockMessage = True
if not blockMessage:
if messageEncodingType != 0:
t = (inventoryHash, toAddress, fromAddress, subject, int(
time.time()), body, 'inbox', messageEncodingType, 0, sigHash)
helper_inbox.insert(t)
queues.UISignalQueue.put(('displayNewInboxMessage', (
inventoryHash, toAddress, fromAddress, subject, body)))
# If we are behaving as an API then we might need to run an
# outside command to let some program know that a new message
# has arrived.
if BMConfigParser().safeGetBoolean('bitmessagesettings', 'apienabled'):
try:
apiNotifyPath = BMConfigParser().get(
'bitmessagesettings', 'apinotifypath')
except:
apiNotifyPath = ''
if apiNotifyPath != '':
call([apiNotifyPath, "newMessage"])
# Let us now check and see whether our receiving address is
# behaving as a mailing list
if BMConfigParser().safeGetBoolean(toAddress, 'mailinglist') and messageEncodingType != 0:
try:
mailingListName = BMConfigParser().get(
toAddress, 'mailinglistname')
except:
mailingListName = ''
# Let us send out this message as a broadcast
subject = self.addMailingListNameToSubject(
subject, mailingListName)
# Let us now send this message out as a broadcast
message = time.strftime("%a, %Y-%m-%d %H:%M:%S UTC", time.gmtime(
)) + ' Message ostensibly from ' + fromAddress + ':\n\n' + body
fromAddress = toAddress # The fromAddress for the broadcast that we are about to send is the toAddress (my address) for the msg message we are currently processing.
ackdataForBroadcast = OpenSSL.rand(
32) # We don't actually need the ackdataForBroadcast for acknowledgement since this is a broadcast message but we can use it to update the user interface when the POW is done generating.
toAddress = '[Broadcast subscribers]'
ripe = ''
# We really should have a discussion about how to
# set the TTL for mailing list broadcasts. This is obviously
# hard-coded.
TTL = 2*7*24*60*60 # 2 weeks
t = ('',
toAddress,
ripe,
fromAddress,
subject,
message,
ackdataForBroadcast,
int(time.time()), # sentTime (this doesn't change)
int(time.time()), # lastActionTime
0,
'broadcastqueued',
0,
'sent',
messageEncodingType,
TTL)
helper_sent.insert(t)
queues.UISignalQueue.put(('displayNewSentMessage', (
toAddress, '[Broadcast subscribers]', fromAddress, subject, message, ackdataForBroadcast)))
queues.workerQueue.put(('sendbroadcast', ''))
# Don't send ACK if invalid, blacklisted senders, invisible messages, disabled or chan
if self.ackDataHasAValidHeader(ackData) and \
not blockMessage and \
messageEncodingType != 0 and \
not BMConfigParser().safeGetBoolean(toAddress, 'dontsendack') and \
not BMConfigParser().safeGetBoolean(toAddress, 'chan'):
shared.checkAndShareObjectWithPeers(ackData[24:])
# Display timing data
timeRequiredToAttemptToDecryptMessage = time.time(
) - messageProcessingStartTime
shared.successfullyDecryptMessageTimings.append(
timeRequiredToAttemptToDecryptMessage)
sum = 0
for item in shared.successfullyDecryptMessageTimings:
sum += item
logger.debug('Time to decrypt this message successfully: %s\n\
Average time for all message decryption successes since startup: %s.' %
(timeRequiredToAttemptToDecryptMessage, sum / len(shared.successfullyDecryptMessageTimings))
)
def processpubkey(self, data):
pubkeyProcessingStartTime = time.time()
shared.numberOfPubkeysProcessed += 1
queues.UISignalQueue.put((
'updateNumberOfPubkeysProcessed', 'no data'))
embeddedTime, = unpack('>Q', data[8:16])
readPosition = 20 # bypass the nonce, time, and object type
addressVersion, varintLength = decodeVarint(
data[readPosition:readPosition + 10])
readPosition += varintLength
streamNumber, varintLength = decodeVarint(
data[readPosition:readPosition + 10])
readPosition += varintLength
if addressVersion == 0:
logger.debug('(Within processpubkey) addressVersion of 0 doesn\'t make sense.')
return
if addressVersion > 4 or addressVersion == 1:
logger.info('This version of Bitmessage cannot handle version %s addresses.' % addressVersion)
return
if addressVersion == 2:
if len(data) < 146: # sanity check. This is the minimum possible length.
logger.debug('(within processpubkey) payloadLength less than 146. Sanity check failed.')
return
bitfieldBehaviors = data[readPosition:readPosition + 4]
readPosition += 4
publicSigningKey = data[readPosition:readPosition + 64]
# Is it possible for a public key to be invalid such that trying to
# encrypt or sign with it will cause an error? If it is, it would
# be easiest to test them here.
readPosition += 64
publicEncryptionKey = data[readPosition:readPosition + 64]
if len(publicEncryptionKey) < 64:
logger.debug('publicEncryptionKey length less than 64. Sanity check failed.')
return
readPosition += 64
dataToStore = data[20:readPosition] # The data we'll store in the pubkeys table.
sha = hashlib.new('sha512')
sha.update(
'\x04' + publicSigningKey + '\x04' + publicEncryptionKey)
ripeHasher = hashlib.new('ripemd160')
ripeHasher.update(sha.digest())
ripe = ripeHasher.digest()
logger.debug('within recpubkey, addressVersion: %s, streamNumber: %s \n\
ripe %s\n\
publicSigningKey in hex: %s\n\
publicEncryptionKey in hex: %s' % (addressVersion,
streamNumber,
hexlify(ripe),
hexlify(publicSigningKey),
hexlify(publicEncryptionKey)
)
)
address = encodeAddress(addressVersion, streamNumber, ripe)
queryreturn = sqlQuery(
'''SELECT usedpersonally FROM pubkeys WHERE address=? AND usedpersonally='yes' ''', address)
if queryreturn != []: # if this pubkey is already in our database and if we have used it personally:
logger.info('We HAVE used this pubkey personally. Updating time.')
t = (address, addressVersion, dataToStore, int(time.time()), 'yes')
else:
logger.info('We have NOT used this pubkey personally. Inserting in database.')
t = (address, addressVersion, dataToStore, int(time.time()), 'no')
sqlExecute('''INSERT INTO pubkeys VALUES (?,?,?,?,?)''', *t)
self.possibleNewPubkey(address)
if addressVersion == 3:
if len(data) < 170: # sanity check.
logger.warning('(within processpubkey) payloadLength less than 170. Sanity check failed.')
return
bitfieldBehaviors = data[readPosition:readPosition + 4]
readPosition += 4
publicSigningKey = '\x04' + data[readPosition:readPosition + 64]
readPosition += 64
publicEncryptionKey = '\x04' + data[readPosition:readPosition + 64]
readPosition += 64
specifiedNonceTrialsPerByte, specifiedNonceTrialsPerByteLength = decodeVarint(
data[readPosition:readPosition + 10])
readPosition += specifiedNonceTrialsPerByteLength
specifiedPayloadLengthExtraBytes, specifiedPayloadLengthExtraBytesLength = decodeVarint(
data[readPosition:readPosition + 10])
readPosition += specifiedPayloadLengthExtraBytesLength
endOfSignedDataPosition = readPosition
dataToStore = data[20:readPosition] # The data we'll store in the pubkeys table.
signatureLength, signatureLengthLength = decodeVarint(
data[readPosition:readPosition + 10])
readPosition += signatureLengthLength
signature = data[readPosition:readPosition + signatureLength]
if highlevelcrypto.verify(data[8:endOfSignedDataPosition], signature, hexlify(publicSigningKey)):
logger.debug('ECDSA verify passed (within processpubkey)')
else:
logger.warning('ECDSA verify failed (within processpubkey)')
return
sha = hashlib.new('sha512')
sha.update(publicSigningKey + publicEncryptionKey)
ripeHasher = hashlib.new('ripemd160')
ripeHasher.update(sha.digest())
ripe = ripeHasher.digest()
logger.debug('within recpubkey, addressVersion: %s, streamNumber: %s \n\
ripe %s\n\
publicSigningKey in hex: %s\n\
publicEncryptionKey in hex: %s' % (addressVersion,
streamNumber,
hexlify(ripe),
hexlify(publicSigningKey),
hexlify(publicEncryptionKey)
)
)
address = encodeAddress(addressVersion, streamNumber, ripe)
queryreturn = sqlQuery('''SELECT usedpersonally FROM pubkeys WHERE address=? AND usedpersonally='yes' ''', address)
if queryreturn != []: # if this pubkey is already in our database and if we have used it personally:
logger.info('We HAVE used this pubkey personally. Updating time.')
t = (address, addressVersion, dataToStore, int(time.time()), 'yes')
else:
logger.info('We have NOT used this pubkey personally. Inserting in database.')
t = (address, addressVersion, dataToStore, int(time.time()), 'no')
sqlExecute('''INSERT INTO pubkeys VALUES (?,?,?,?,?)''', *t)
self.possibleNewPubkey(address)
if addressVersion == 4:
if len(data) < 350: # sanity check.
logger.debug('(within processpubkey) payloadLength less than 350. Sanity check failed.')
return
tag = data[readPosition:readPosition + 32]
if tag not in state.neededPubkeys:
logger.info('We don\'t need this v4 pubkey. We didn\'t ask for it.')
return
# Let us try to decrypt the pubkey
toAddress, cryptorObject = state.neededPubkeys[tag]
if shared.decryptAndCheckPubkeyPayload(data, toAddress) == 'successful':
# At this point we know that we have been waiting on this pubkey.
# This function will command the workerThread to start work on
# the messages that require it.
self.possibleNewPubkey(toAddress)
# Display timing data
timeRequiredToProcessPubkey = time.time(
) - pubkeyProcessingStartTime
logger.debug('Time required to process this pubkey: %s' % timeRequiredToProcessPubkey)
