def run(self):
timeWeLastClearedInventoryAndPubkeysTables = 0
try:
shared.maximumLengthOfTimeToBotherResendingMessages = (float(shared.config.get('bitmessagesettings', 'stopresendingafterxdays')) * 24 * 60 * 60) + (float(shared.config.get('bitmessagesettings', 'stopresendingafterxmonths')) * (60 * 60 * 24 *365)/12)
except:
# Either the user hasn't set stopresendingafterxdays and stopresendingafterxmonths yet or the options are missing from the config file.
shared.maximumLengthOfTimeToBotherResendingMessages = float('inf')
while shared.shutdown == 0:
shared.UISignalQueue.put((
'updateStatusBar', 'Doing housekeeping (Flushing inventory in memory to disk...)'))
with shared.inventoryLock: # If you use both the inventoryLock and the sqlLock, always use the inventoryLock OUTSIDE of the sqlLock.
with SqlBulkExecute() as sql:
for hash, storedValue in shared.inventory.items():
objectType, streamNumber, payload, expiresTime, tag = storedValue
sql.execute(
'''INSERT INTO inventory VALUES (?,?,?,?,?,?)''',
hash,
objectType,
streamNumber,
payload,
expiresTime,
tag)
del shared.inventory[hash]
shared.UISignalQueue.put(('updateStatusBar', ''))
shared.broadcastToSendDataQueues((
0, 'pong', 'no data')) # commands the sendData threads to send out a pong message if they haven't sent anything else in the last five minutes. The socket timeout-time is 10 minutes.
# If we are running as a daemon then we are going to fill up the UI
# queue which will never be handled by a UI. We should clear it to
# save memory.
if shared.safeConfigGetBoolean('bitmessagesettings', 'daemon'):
shared.UISignalQueue.queue.clear()
if timeWeLastClearedInventoryAndPubkeysTables < int(time.time()) - 7380:
timeWeLastClearedInventoryAndPubkeysTables = int(time.time())
sqlExecute(
'''DELETE FROM inventory WHERE expirestime<? ''',
int(time.time()) - (60 * 60 * 3))
# pubkeys
sqlExecute(
'''DELETE FROM pubkeys WHERE time<? AND usedpersonally='no' ''',
int(time.time()) - shared.lengthOfTimeToHoldOnToAllPubkeys)
# Let us resend getpubkey objects if we have not yet heard a pubkey, and also msg objects if we have not yet heard an acknowledgement
queryreturn = sqlQuery(
'''select toaddress, ackdata, status FROM sent WHERE ((status='awaitingpubkey' OR status='msgsent') AND folder='sent' AND sleeptill<? AND senttime>?) ''',
int(time.time()),
int(time.time()) - shared.maximumLengthOfTimeToBotherResendingMessages)
for row in queryreturn:
if len(row) < 2:
logger.error('Something went wrong in the singleCleaner thread: a query did not return the requested fields. ' + repr(row))
self.stop.wait(3)
break
toAddress, ackData, status = row
if status == 'awaitingpubkey':
resendPubkeyRequest(toAddress)
elif status == 'msgsent':
resendMsg(ackData)
# Let's also clear and reload shared.inventorySets to keep it from
# taking up an unnecessary amount of memory.
for streamNumber in shared.inventorySets:
shared.inventorySets[streamNumber] = set()
queryData = sqlQuery('''SELECT hash FROM inventory WHERE streamnumber=?''', streamNumber)
for row in queryData:
shared.inventorySets[streamNumber].add(row[0])
with shared.inventoryLock:
for hash, storedValue in shared.inventory.items():
objectType, streamNumber, payload, expiresTime, tag = storedValue
if not streamNumber in shared.inventorySets:
shared.inventorySets[streamNumber] = set()
shared.inventorySets[streamNumber].add(hash)
# Let us write out the knowNodes to disk if there is anything new to write out.
if shared.needToWriteKnownNodesToDisk:
shared.knownNodesLock.acquire()
output = open(shared.appdata + 'knownnodes.dat', 'wb')
try:
pickle.dump(shared.knownNodes, output)
output.close()
except Exception as err:
if "Errno 28" in str(err):
logger.fatal('(while receiveDataThread shared.needToWriteKnownNodesToDisk) Alert: Your disk or data storage volume is full. ')
shared.UISignalQueue.put(('alert', (tr.translateText("MainWindow", "Disk full"), tr.translateText("MainWindow", 'Alert: Your disk or data storage volume is full. Bitmessage will now exit.'), True)))
if shared.daemon:
os._exit(0)
shared.knownNodesLock.release()
shared.needToWriteKnownNodesToDisk = False
self.stop.wait(300)
def run(self):
self.conn = sqlite3.connect(state.appdata + 'messages.dat')
self.conn.text_factory = str
self.cur = self.conn.cursor()
self.cur.execute('PRAGMA secure_delete = true')
try:
self.cur.execute(
'''CREATE TABLE inbox (msgid blob, toaddress text, fromaddress text, subject text, received text, message text, folder text, encodingtype int, read bool, sighash blob, UNIQUE(msgid) ON CONFLICT REPLACE)''' )
self.cur.execute(
'''CREATE TABLE sent (msgid blob, toaddress text, toripe blob, fromaddress text, subject text, message text, ackdata blob, senttime integer, lastactiontime integer, sleeptill integer, status text, retrynumber integer, folder text, encodingtype int, ttl int)''' )
self.cur.execute(
'''CREATE TABLE subscriptions (label text, address text, enabled bool)''' )
self.cur.execute(
'''CREATE TABLE addressbook (label text, address text)''' )
self.cur.execute(
'''CREATE TABLE blacklist (label text, address text, enabled bool)''' )
self.cur.execute(
'''CREATE TABLE whitelist (label text, address text, enabled bool)''' )
self.cur.execute(
'''CREATE TABLE pubkeys (address text, addressversion int, transmitdata blob, time int, usedpersonally text, UNIQUE(address) ON CONFLICT REPLACE)''' )
self.cur.execute(
'''CREATE TABLE inventory (hash blob, objecttype int, streamnumber int, payload blob, expirestime integer, tag blob, UNIQUE(hash) ON CONFLICT REPLACE)''' )
self.cur.execute(
'''INSERT INTO subscriptions VALUES('Bitmessage new releases/announcements','BM-GtovgYdgs7qXPkoYaRgrLFuFKz1SFpsw',1)''')
self.cur.execute(
'''CREATE TABLE settings (key blob, value blob, UNIQUE(key) ON CONFLICT REPLACE)''' )
self.cur.execute( '''INSERT INTO settings VALUES('version','10')''')
self.cur.execute( '''INSERT INTO settings VALUES('lastvacuumtime',?)''', (
int(time.time()),))
self.cur.execute(
'''CREATE TABLE objectprocessorqueue (objecttype int, data blob, UNIQUE(objecttype, data) ON CONFLICT REPLACE)''' )
self.conn.commit()
logger.info('Created messages database file')
except Exception as err:
if str(err) == 'table inbox already exists':
logger.debug('Database file already exists.')
else:
sys.stderr.write(
'ERROR trying to create database file (message.dat). Error message: %s\n' % str(err))
os._exit(0)
# If the settings version is equal to 2 or 3 then the
# sqlThread will modify the pubkeys table and change
# the settings version to 4.
settingsversion = BMConfigParser().getint(
'bitmessagesettings', 'settingsversion')
# People running earlier versions of PyBitmessage do not have the
# usedpersonally field in their pubkeys table. Let's add it.
if settingsversion == 2:
item = '''ALTER TABLE pubkeys ADD usedpersonally text DEFAULT 'no' '''
parameters = ''
self.cur.execute(item, parameters)
self.conn.commit()
settingsversion = 3
# People running earlier versions of PyBitmessage do not have the
# encodingtype field in their inbox and sent tables or the read field
# in the inbox table. Let's add them.
if settingsversion == 3:
item = '''ALTER TABLE inbox ADD encodingtype int DEFAULT '2' '''
parameters = ''
self.cur.execute(item, parameters)
item = '''ALTER TABLE inbox ADD read bool DEFAULT '1' '''
parameters = ''
self.cur.execute(item, parameters)
item = '''ALTER TABLE sent ADD encodingtype int DEFAULT '2' '''
parameters = ''
self.cur.execute(item, parameters)
self.conn.commit()
settingsversion = 4
BMConfigParser().set(
'bitmessagesettings', 'settingsversion', str(settingsversion))
BMConfigParser().save()
helper_startup.updateConfig()
# From now on, let us keep a 'version' embedded in the messages.dat
# file so that when we make changes to the database, the database
# version we are on can stay embedded in the messages.dat file. Let us
# check to see if the settings table exists yet.
item = '''SELECT name FROM sqlite_master WHERE type='table' AND name='settings';'''
parameters = ''
self.cur.execute(item, parameters)
if self.cur.fetchall() == []:
# The settings table doesn't exist. We need to make it.
logger.debug(
"In messages.dat database, creating new 'settings' table.")
self.cur.execute(
'''CREATE TABLE settings (key text, value blob, UNIQUE(key) ON CONFLICT REPLACE)''' )
self.cur.execute( '''INSERT INTO settings VALUES('version','1')''')
self.cur.execute( '''INSERT INTO settings VALUES('lastvacuumtime',?)''', (
int(time.time()),))
logger.debug('In messages.dat database, removing an obsolete field from the pubkeys table.')
self.cur.execute(
'''CREATE TEMPORARY TABLE pubkeys_backup(hash blob, transmitdata blob, time int, usedpersonally text, UNIQUE(hash) ON CONFLICT REPLACE);''')
self.cur.execute(
'''INSERT INTO pubkeys_backup SELECT hash, transmitdata, time, usedpersonally FROM pubkeys;''')
self.cur.execute( '''DROP TABLE pubkeys''')
self.cur.execute(
'''CREATE TABLE pubkeys (hash blob, transmitdata blob, time int, usedpersonally text, UNIQUE(hash) ON CONFLICT REPLACE)''' )
self.cur.execute(
'''INSERT INTO pubkeys SELECT hash, transmitdata, time, usedpersonally FROM pubkeys_backup;''')
self.cur.execute( '''DROP TABLE pubkeys_backup;''')
logger.debug('Deleting all pubkeys from inventory. They will be redownloaded and then saved with the correct times.')
self.cur.execute(
'''delete from inventory where objecttype = 'pubkey';''')
logger.debug('replacing Bitmessage announcements mailing list with a new one.')
self.cur.execute(
'''delete from subscriptions where address='BM-BbkPSZbzPwpVcYZpU4yHwf9ZPEapN5Zx' ''')
self.cur.execute(
'''INSERT INTO subscriptions VALUES('Bitmessage new releases/announcements','BM-GtovgYdgs7qXPkoYaRgrLFuFKz1SFpsw',1)''')
logger.debug('Commiting.')
self.conn.commit()
logger.debug('Vacuuming message.dat. You might notice that the file size gets much smaller.')
self.cur.execute( ''' VACUUM ''')
# After code refactoring, the possible status values for sent messages
# have changed.
self.cur.execute(
'''update sent set status='doingmsgpow' where status='doingpow' ''')
self.cur.execute(
'''update sent set status='msgsent' where status='sentmessage' ''')
self.cur.execute(
'''update sent set status='doingpubkeypow' where status='findingpubkey' ''')
self.cur.execute(
'''update sent set status='broadcastqueued' where status='broadcastpending' ''')
self.conn.commit()
# Let's get rid of the first20bytesofencryptedmessage field in
# the inventory table.
item = '''SELECT value FROM settings WHERE key='version';'''
parameters = ''
self.cur.execute(item, parameters)
if int(self.cur.fetchall()[0][0]) == 2:
logger.debug(
'In messages.dat database, removing an obsolete field from'
' the inventory table.')
self.cur.execute(
'''CREATE TEMPORARY TABLE inventory_backup(hash blob, objecttype text, streamnumber int, payload blob, receivedtime integer, UNIQUE(hash) ON CONFLICT REPLACE);''')
self.cur.execute(
'''INSERT INTO inventory_backup SELECT hash, objecttype, streamnumber, payload, receivedtime FROM inventory;''')
self.cur.execute( '''DROP TABLE inventory''')
self.cur.execute(
'''CREATE TABLE inventory (hash blob, objecttype text, streamnumber int, payload blob, receivedtime integer, UNIQUE(hash) ON CONFLICT REPLACE)''' )
self.cur.execute(
'''INSERT INTO inventory SELECT hash, objecttype, streamnumber, payload, receivedtime FROM inventory_backup;''')
self.cur.execute( '''DROP TABLE inventory_backup;''')
item = '''update settings set value=? WHERE key='version';'''
parameters = (3,)
self.cur.execute(item, parameters)
# Add a new column to the inventory table to store tags.
item = '''SELECT value FROM settings WHERE key='version';'''
parameters = ''
self.cur.execute(item, parameters)
currentVersion = int(self.cur.fetchall()[0][0])
if currentVersion == 1 or currentVersion == 3:
logger.debug(
'In messages.dat database, adding tag field to'
' the inventory table.')
item = '''ALTER TABLE inventory ADD tag blob DEFAULT '' '''
parameters = ''
self.cur.execute(item, parameters)
item = '''update settings set value=? WHERE key='version';'''
parameters = (4,)
self.cur.execute(item, parameters)
# Add a new column to the pubkeys table to store the address version.
# We're going to trash all of our pubkeys and let them be redownloaded.
item = '''SELECT value FROM settings WHERE key='version';'''
parameters = ''
self.cur.execute(item, parameters)
currentVersion = int(self.cur.fetchall()[0][0])
if currentVersion == 4:
self.cur.execute('''DROP TABLE pubkeys''')
self.cur.execute(
'''CREATE TABLE pubkeys (hash blob, addressversion int, transmitdata blob, time int, usedpersonally text, UNIQUE(hash, addressversion) ON CONFLICT REPLACE)''')
self.cur.execute(
'''delete from inventory where objecttype = 'pubkey';''')
item = '''update settings set value=? WHERE key='version';'''
parameters = (5,)
self.cur.execute(item, parameters)
# Add a new table: objectprocessorqueue with which to hold objects
# that have yet to be processed if the user shuts down Bitmessage.
item = '''SELECT value FROM settings WHERE key='version';'''
parameters = ''
self.cur.execute(item, parameters)
currentVersion = int(self.cur.fetchall()[0][0])
if currentVersion == 5:
self.cur.execute('''DROP TABLE knownnodes''')
self.cur.execute(
'''CREATE TABLE objectprocessorqueue (objecttype text, data blob, UNIQUE(objecttype, data) ON CONFLICT REPLACE)''')
item = '''update settings set value=? WHERE key='version';'''
parameters = (6,)
self.cur.execute(item, parameters)
# changes related to protocol v3
# In table inventory and objectprocessorqueue, objecttype is now
# an integer (it was a human-friendly string previously)
item = '''SELECT value FROM settings WHERE key='version';'''
parameters = ''
self.cur.execute(item, parameters)
currentVersion = int(self.cur.fetchall()[0][0])
if currentVersion == 6:
logger.debug(
'In messages.dat database, dropping and recreating'
' the inventory table.')
self.cur.execute( '''DROP TABLE inventory''')
self.cur.execute( '''CREATE TABLE inventory (hash blob, objecttype int, streamnumber int, payload blob, expirestime integer, tag blob, UNIQUE(hash) ON CONFLICT REPLACE)''' )
self.cur.execute( '''DROP TABLE objectprocessorqueue''')
self.cur.execute( '''CREATE TABLE objectprocessorqueue (objecttype int, data blob, UNIQUE(objecttype, data) ON CONFLICT REPLACE)''' )
item = '''update settings set value=? WHERE key='version';'''
parameters = (7,)
self.cur.execute(item, parameters)
logger.debug(
'Finished dropping and recreating the inventory table.')
# The format of data stored in the pubkeys table has changed. Let's
# clear it, and the pubkeys from inventory, so that they'll
# be re-downloaded.
item = '''SELECT value FROM settings WHERE key='version';'''
parameters = ''
self.cur.execute(item, parameters)
currentVersion = int(self.cur.fetchall()[0][0])
if currentVersion == 7:
logger.debug(
'In messages.dat database, clearing pubkeys table'
' because the data format has been updated.')
self.cur.execute(
'''delete from inventory where objecttype = 1;''')
self.cur.execute(
'''delete from pubkeys;''')
# Any sending messages for which we *thought* that we had
# the pubkey must be rechecked.
self.cur.execute(
'''UPDATE sent SET status='msgqueued' WHERE status='doingmsgpow' or status='badkey';''')
query = '''update settings set value=? WHERE key='version';'''
parameters = (8,)
self.cur.execute(query, parameters)
logger.debug('Finished clearing currently held pubkeys.')
# Add a new column to the inbox table to store the hash of
# the message signature. We'll use this as temporary message UUID
# in order to detect duplicates.
item = '''SELECT value FROM settings WHERE key='version';'''
parameters = ''
self.cur.execute(item, parameters)
currentVersion = int(self.cur.fetchall()[0][0])
if currentVersion == 8:
logger.debug(
'In messages.dat database, adding sighash field to'
' the inbox table.')
item = '''ALTER TABLE inbox ADD sighash blob DEFAULT '' '''
parameters = ''
self.cur.execute(item, parameters)
item = '''update settings set value=? WHERE key='version';'''
parameters = (9,)
self.cur.execute(item, parameters)
# We'll also need a `sleeptill` field and a `ttl` field. Also we
# can combine the pubkeyretrynumber and msgretrynumber into one.
item = '''SELECT value FROM settings WHERE key='version';'''
parameters = ''
self.cur.execute(item, parameters)
currentVersion = int(self.cur.fetchall()[0][0])
if currentVersion == 9:
logger.info(
'In messages.dat database, making TTL-related changes:'
' combining the pubkeyretrynumber and msgretrynumber'
' fields into the retrynumber field and adding the'
' sleeptill and ttl fields...')
self.cur.execute(
'''CREATE TEMPORARY TABLE sent_backup (msgid blob, toaddress text, toripe blob, fromaddress text, subject text, message text, ackdata blob, lastactiontime integer, status text, retrynumber integer, folder text, encodingtype int)''' )
self.cur.execute(
'''INSERT INTO sent_backup SELECT msgid, toaddress, toripe, fromaddress, subject, message, ackdata, lastactiontime, status, 0, folder, encodingtype FROM sent;''')
self.cur.execute( '''DROP TABLE sent''')
self.cur.execute(
'''CREATE TABLE sent (msgid blob, toaddress text, toripe blob, fromaddress text, subject text, message text, ackdata blob, senttime integer, lastactiontime integer, sleeptill int, status text, retrynumber integer, folder text, encodingtype int, ttl int)''' )
self.cur.execute(
'''INSERT INTO sent SELECT msgid, toaddress, toripe, fromaddress, subject, message, ackdata, lastactiontime, lastactiontime, 0, status, 0, folder, encodingtype, 216000 FROM sent_backup;''')
self.cur.execute( '''DROP TABLE sent_backup''')
logger.info('In messages.dat database, finished making TTL-related changes.')
logger.debug('In messages.dat database, adding address field to the pubkeys table.')
# We're going to have to calculate the address for each row in the pubkeys
# table. Then we can take out the hash field.
self.cur.execute('''ALTER TABLE pubkeys ADD address text DEFAULT '' ''')
self.cur.execute('''SELECT hash, addressversion FROM pubkeys''')
queryResult = self.cur.fetchall()
from addresses import encodeAddress
for row in queryResult:
addressHash, addressVersion = row
address = encodeAddress(addressVersion, 1, hash)
item = '''UPDATE pubkeys SET address=? WHERE hash=?;'''
parameters = (address, addressHash)
self.cur.execute(item, parameters)
# Now we can remove the hash field from the pubkeys table.
self.cur.execute(
'''CREATE TEMPORARY TABLE pubkeys_backup (address text, addressversion int, transmitdata blob, time int, usedpersonally text, UNIQUE(address) ON CONFLICT REPLACE)''' )
self.cur.execute(
'''INSERT INTO pubkeys_backup SELECT address, addressversion, transmitdata, time, usedpersonally FROM pubkeys;''')
self.cur.execute( '''DROP TABLE pubkeys''')
self.cur.execute(
'''CREATE TABLE pubkeys (address text, addressversion int, transmitdata blob, time int, usedpersonally text, UNIQUE(address) ON CONFLICT REPLACE)''' )
self.cur.execute(
'''INSERT INTO pubkeys SELECT address, addressversion, transmitdata, time, usedpersonally FROM pubkeys_backup;''')
self.cur.execute( '''DROP TABLE pubkeys_backup''')
logger.debug('In messages.dat database, done adding address field to the pubkeys table and removing the hash field.')
self.cur.execute('''update settings set value=10 WHERE key='version';''')
# Are you hoping to add a new option to the keys.dat file of existing
# Bitmessage users or modify the SQLite database? Add it right
# above this line!
try:
testpayload = '\x00\x00'
t = ('1234', 1, testpayload, '12345678', 'no')
self.cur.execute( '''INSERT INTO pubkeys VALUES(?,?,?,?,?)''', t)
self.conn.commit()
self.cur.execute(
'''SELECT transmitdata FROM pubkeys WHERE address='1234' ''')
queryreturn = self.cur.fetchall()
for row in queryreturn:
transmitdata, = row
self.cur.execute('''DELETE FROM pubkeys WHERE address='1234' ''')
self.conn.commit()
if transmitdata == '':
logger.fatal('Problem: The version of SQLite you have cannot store Null values. Please download and install the latest revision of your version of Python (for example, the latest Python 2.7 revision) and try again.\n')
logger.fatal('PyBitmessage will now exit very abruptly. You may now see threading errors related to this abrupt exit but the problem you need to solve is related to SQLite.\n\n')
os._exit(0)
except Exception as err:
if str(err) == 'database or disk is full':
logger.fatal('(While null value test) Alert: Your disk or data storage volume is full. sqlThread will now exit.')
queues.UISignalQueue.put(('alert', (tr._translate("MainWindow", "Disk full"), tr._translate("MainWindow", 'Alert: Your disk or data storage volume is full. Bitmessage will now exit.'), True)))
os._exit(0)
else:
logger.error(err)
# Let us check to see the last time we vaccumed the messages.dat file.
# If it has been more than a month let's do it now.
item = '''SELECT value FROM settings WHERE key='lastvacuumtime';'''
parameters = ''
self.cur.execute(item, parameters)
queryreturn = self.cur.fetchall()
for row in queryreturn:
value, = row
if int(value) < int(time.time()) - 86400:
logger.info('It has been a long time since the messages.dat file has been vacuumed. Vacuuming now...')
try:
self.cur.execute( ''' VACUUM ''')
except Exception as err:
if str(err) == 'database or disk is full':
logger.fatal('(While VACUUM) Alert: Your disk or data storage volume is full. sqlThread will now exit.')
queues.UISignalQueue.put(('alert', (tr._translate("MainWindow", "Disk full"), tr._translate("MainWindow", 'Alert: Your disk or data storage volume is full. Bitmessage will now exit.'), True)))
os._exit(0)
item = '''update settings set value=? WHERE key='lastvacuumtime';'''
parameters = (int(time.time()),)
self.cur.execute(item, parameters)
state.sqlReady = True
while True:
item = helper_sql.sqlSubmitQueue.get()
if item == 'commit':
try:
self.conn.commit()
except Exception as err:
if str(err) == 'database or disk is full':
logger.fatal('(While committing) Alert: Your disk or data storage volume is full. sqlThread will now exit.')
queues.UISignalQueue.put(('alert', (tr._translate("MainWindow", "Disk full"), tr._translate("MainWindow", 'Alert: Your disk or data storage volume is full. Bitmessage will now exit.'), True)))
os._exit(0)
elif item == 'exit':
self.conn.close()
logger.info('sqlThread exiting gracefully.')
return
elif item == 'movemessagstoprog':
logger.debug('the sqlThread is moving the messages.dat file to the local program directory.')
try:
self.conn.commit()
except Exception as err:
if str(err) == 'database or disk is full':
logger.fatal('(while movemessagstoprog) Alert: Your disk or data storage volume is full. sqlThread will now exit.')
queues.UISignalQueue.put(('alert', (tr._translate("MainWindow", "Disk full"), tr._translate("MainWindow", 'Alert: Your disk or data storage volume is full. Bitmessage will now exit.'), True)))
os._exit(0)
self.conn.close()
shutil.move(
paths.lookupAppdataFolder() + 'messages.dat', paths.lookupExeFolder() + 'messages.dat')
self.conn = sqlite3.connect(paths.lookupExeFolder() + 'messages.dat')
self.conn.text_factory = str
self.cur = self.conn.cursor()
elif item == 'movemessagstoappdata':
logger.debug('the sqlThread is moving the messages.dat file to the Appdata folder.')
try:
self.conn.commit()
except Exception as err:
if str(err) == 'database or disk is full':
logger.fatal('(while movemessagstoappdata) Alert: Your disk or data storage volume is full. sqlThread will now exit.')
queues.UISignalQueue.put(('alert', (tr._translate("MainWindow", "Disk full"), tr._translate("MainWindow", 'Alert: Your disk or data storage volume is full. Bitmessage will now exit.'), True)))
os._exit(0)
self.conn.close()
shutil.move(
paths.lookupExeFolder() + 'messages.dat', paths.lookupAppdataFolder() + 'messages.dat')
self.conn = sqlite3.connect(paths.lookupAppdataFolder() + 'messages.dat')
self.conn.text_factory = str
self.cur = self.conn.cursor()
elif item == 'deleteandvacuume':
self.cur.execute('''delete from inbox where folder='trash' ''')
self.cur.execute('''delete from sent where folder='trash' ''')
self.conn.commit()
try:
self.cur.execute( ''' VACUUM ''')
except Exception as err:
if str(err) == 'database or disk is full':
logger.fatal('(while deleteandvacuume) Alert: Your disk or data storage volume is full. sqlThread will now exit.')
queues.UISignalQueue.put(('alert', (tr._translate("MainWindow", "Disk full"), tr._translate("MainWindow", 'Alert: Your disk or data storage volume is full. Bitmessage will now exit.'), True)))
os._exit(0)
else:
parameters = helper_sql.sqlSubmitQueue.get()
rowcount = 0
# print 'item', item
# print 'parameters', parameters
try:
self.cur.execute(item, parameters)
rowcount = self.cur.rowcount
except Exception as err:
if str(err) == 'database or disk is full':
logger.fatal('(while cur.execute) Alert: Your disk or data storage volume is full. sqlThread will now exit.')
queues.UISignalQueue.put(('alert', (tr._translate("MainWindow", "Disk full"), tr._translate("MainWindow", 'Alert: Your disk or data storage volume is full. Bitmessage will now exit.'), True)))
os._exit(0)
else:
logger.fatal('Major error occurred when trying to execute a SQL statement within the sqlThread. Please tell Atheros about this error message or post it in the forum! Error occurred while trying to execute statement: "%s" Here are the parameters; you might want to censor this data with asterisks (***) as it can contain private information: %s. Here is the actual error message thrown by the sqlThread: %s', str(item), str(repr(parameters)), str(err))
logger.fatal('This program shall now abruptly exit!')
os._exit(0)
helper_sql.sqlReturnQueue.put((self.cur.fetchall(), rowcount))
def run(self):
timeWeLastClearedInventoryAndPubkeysTables = 0
while True:
shared.UISignalQueue.put((
'updateStatusBar', 'Doing housekeeping (Flushing inventory in memory to disk...)'))
with shared.inventoryLock: # If you use both the inventoryLock and the sqlLock, always use the inventoryLock OUTSIDE of the sqlLock.
with SqlBulkExecute() as sql:
for hash, storedValue in shared.inventory.items():
objectType, streamNumber, payload, receivedTime, tag = storedValue
if int(time.time()) - 3600 > receivedTime:
sql.execute(
'''INSERT INTO inventory VALUES (?,?,?,?,?,?)''',
hash,
objectType,
streamNumber,
payload,
receivedTime,
tag)
del shared.inventory[hash]
shared.UISignalQueue.put(('updateStatusBar', ''))
shared.broadcastToSendDataQueues((
0, 'pong', 'no data')) # commands the sendData threads to send out a pong message if they haven't sent anything else in the last five minutes. The socket timeout-time is 10 minutes.
# If we are running as a daemon then we are going to fill up the UI
# queue which will never be handled by a UI. We should clear it to
# save memory.
if shared.safeConfigGetBoolean('bitmessagesettings', 'daemon'):
shared.UISignalQueue.queue.clear()
if timeWeLastClearedInventoryAndPubkeysTables < int(time.time()) - 7380:
timeWeLastClearedInventoryAndPubkeysTables = int(time.time())
# inventory (moves data from the inventory data structure to
# the on-disk sql database)
# inventory (clears pubkeys after 28 days and everything else
# after 2 days and 12 hours)
sqlExecute(
'''DELETE FROM inventory WHERE (receivedtime<? AND objecttype<>'pubkey') OR (receivedtime<? AND objecttype='pubkey') ''',
int(time.time()) - shared.lengthOfTimeToLeaveObjectsInInventory,
int(time.time()) - shared.lengthOfTimeToHoldOnToAllPubkeys)
# pubkeys
sqlExecute(
'''DELETE FROM pubkeys WHERE time<? AND usedpersonally='no' ''',
int(time.time()) - shared.lengthOfTimeToHoldOnToAllPubkeys)
queryreturn = sqlQuery(
'''select toaddress, toripe, fromaddress, subject, message, ackdata, lastactiontime, status, pubkeyretrynumber, msgretrynumber FROM sent WHERE ((status='awaitingpubkey' OR status='msgsent') AND folder='sent') ''') # If the message's folder='trash' then we'll ignore it.
for row in queryreturn:
if len(row) < 5:
with shared.printLock:
sys.stderr.write(
'Something went wrong in the singleCleaner thread: a query did not return the requested fields. ' + repr(row))
time.sleep(3)
break
toaddress, toripe, fromaddress, subject, message, ackdata, lastactiontime, status, pubkeyretrynumber, msgretrynumber = row
if status == 'awaitingpubkey':#start:UI setting to stop trying to send messages after X hours/days/months
if int(shared.config.get('bitmessagesettings', 'timeperiod'))> -1:#The default value of timeperiod is -1.
if (int(time.time()) - lastactiontime) > (shared.maximumAgeOfAnObjectThatIAmWillingToAccept * (2 ** (pubkeyretrynumber))) and ((int(time.time()) - lastactiontime) < int(shared.config.get('bitmessagesettings', 'timeperiod'))):
resendPubkey(pubkeyretrynumber,toripe)#This will be executed if the user has adjusted the time period with some value
else:
if int(time.time()) - lastactiontime > (shared.maximumAgeOfAnObjectThatIAmWillingToAccept * (2 ** (pubkeyretrynumber))):
resendPubkey(pubkeyretrynumber,toripe)#This will be executed if the time period has its default value -1. Input (blank/blank/blank)
else: # status == msgsent
if int(shared.config.get('bitmessagesettings', 'timeperiod'))> -1:
if (int(time.time()) - lastactiontime) > (shared.maximumAgeOfAnObjectThatIAmWillingToAccept * (2 ** (msgretrynumber))) and ((int(time.time()) - lastactiontime) < int(shared.config.get('bitmessagesettings', 'timeperiod'))):
resendMsg(msgretrynumber,ackdata)
else:
if int(time.time()) - lastactiontime > (shared.maximumAgeOfAnObjectThatIAmWillingToAccept * (2 ** (msgretrynumber))):
resendMsg(msgretrynumber,ackdata)
#end
# Let's also clear and reload shared.inventorySets to keep it from
# taking up an unnecessary amount of memory.
for streamNumber in shared.inventorySets:
shared.inventorySets[streamNumber] = set()
queryData = sqlQuery('''SELECT hash FROM inventory WHERE streamnumber=?''', streamNumber)
for row in queryData:
shared.inventorySets[streamNumber].add(row[0])
with shared.inventoryLock:
for hash, storedValue in shared.inventory.items():
objectType, streamNumber, payload, receivedTime, tag = storedValue
if streamNumber in shared.inventorySets:
shared.inventorySets[streamNumber].add(hash)
# Let us write out the knowNodes to disk if there is anything new to write out.
if shared.needToWriteKnownNodesToDisk:
shared.knownNodesLock.acquire()
output = open(shared.appdata + 'knownnodes.dat', 'wb')
try:
pickle.dump(shared.knownNodes, output)
output.close()
except Exception as err:
if "Errno 28" in str(err):
logger.fatal('(while receiveDataThread shared.needToWriteKnownNodesToDisk) Alert: Your disk or data storage volume is full. ')
shared.UISignalQueue.put(('alert', (tr.translateText("MainWindow", "Disk full"), tr.translateText("MainWindow", 'Alert: Your disk or data storage volume is full. Bitmessage will now exit.'), True)))
if shared.daemon:
os._exit(0)
shared.knownNodesLock.release()
shared.needToWriteKnownNodesToDisk = False
time.sleep(300)
def run(self):
gc.disable()
timeWeLastClearedInventoryAndPubkeysTables = 0
try:
shared.maximumLengthOfTimeToBotherResendingMessages = (float(BMConfigParser().get('bitmessagesettings', 'stopresendingafterxdays')) * 24 * 60 * 60) + (float(BMConfigParser().get('bitmessagesettings', 'stopresendingafterxmonths')) * (60 * 60 * 24 *365)/12)
except:
# Either the user hasn't set stopresendingafterxdays and stopresendingafterxmonths yet or the options are missing from the config file.
shared.maximumLengthOfTimeToBotherResendingMessages = float('inf')
# initial wait
if state.shutdown == 0:
self.stop.wait(singleCleaner.cycleLength)
while state.shutdown == 0:
queues.UISignalQueue.put((
'updateStatusBar', 'Doing housekeeping (Flushing inventory in memory to disk...)'))
Inventory().flush()
queues.UISignalQueue.put(('updateStatusBar', ''))
# If we are running as a daemon then we are going to fill up the UI
# queue which will never be handled by a UI. We should clear it to
# save memory.
if BMConfigParser().safeGetBoolean('bitmessagesettings', 'daemon'):
queues.UISignalQueue.queue.clear()
if timeWeLastClearedInventoryAndPubkeysTables < int(time.time()) - 7380:
timeWeLastClearedInventoryAndPubkeysTables = int(time.time())
Inventory().clean()
# pubkeys
sqlExecute(
'''DELETE FROM pubkeys WHERE time<? AND usedpersonally='no' ''',
int(time.time()) - shared.lengthOfTimeToHoldOnToAllPubkeys)
# Let us resend getpubkey objects if we have not yet heard a pubkey, and also msg objects if we have not yet heard an acknowledgement
queryreturn = sqlQuery(
'''select toaddress, ackdata, status FROM sent WHERE ((status='awaitingpubkey' OR status='msgsent') AND folder='sent' AND sleeptill<? AND senttime>?) ''',
int(time.time()),
int(time.time()) - shared.maximumLengthOfTimeToBotherResendingMessages)
for row in queryreturn:
if len(row) < 2:
logger.error('Something went wrong in the singleCleaner thread: a query did not return the requested fields. ' + repr(row))
self.stop.wait(3)
break
toAddress, ackData, status = row
if status == 'awaitingpubkey':
resendPubkeyRequest(toAddress)
elif status == 'msgsent':
resendMsg(ackData)
# cleanup old nodes
now = int(time.time())
with knownnodes.knownNodesLock:
for stream in knownnodes.knownNodes:
keys = knownnodes.knownNodes[stream].keys()
for node in keys:
try:
# scrap old nodes
if now - knownnodes.knownNodes[stream][node]["lastseen"] > 2419200: # 28 days
shared.needToWriteKnownNodesToDisk = True
del knownnodes.knownNodes[stream][node]
continue
# scrap old nodes with low rating
if now - knownnodes.knownNodes[stream][node]["lastseen"] > 10800 and knownnodes.knownNodes[stream][node]["rating"] <= knownnodes.knownNodesForgetRating:
shared.needToWriteKnownNodesToDisk = True
del knownnodes.knownNodes[stream][node]
continue
except TypeError:
print "Error in %s" % (str(node))
keys = []
# Let us write out the knowNodes to disk if there is anything new to write out.
if shared.needToWriteKnownNodesToDisk:
try:
knownnodes.saveKnownNodes()
except Exception as err:
if "Errno 28" in str(err):
logger.fatal('(while receiveDataThread knownnodes.needToWriteKnownNodesToDisk) Alert: Your disk or data storage volume is full. ')
queues.UISignalQueue.put(('alert', (tr._translate("MainWindow", "Disk full"), tr._translate("MainWindow", 'Alert: Your disk or data storage volume is full. Bitmessage will now exit.'), True)))
if shared.daemon:
os._exit(0)
shared.needToWriteKnownNodesToDisk = False
# # clear download queues
# for thread in threading.enumerate():
# if thread.isAlive() and hasattr(thread, 'downloadQueue'):
# thread.downloadQueue.clear()
# inv/object tracking
for connection in BMConnectionPool().inboundConnections.values() + BMConnectionPool().outboundConnections.values():
connection.clean()
# dandelion fluff trigger by expiration
Dandelion().expire()
# discovery tracking
exp = time.time() - singleCleaner.expireDiscoveredPeers
reaper = (k for k, v in state.discoveredPeers.items() if v < exp)
for k in reaper:
try:
del state.discoveredPeers[k]
except KeyError:
pass
# TODO: cleanup pending upload / download
gc.collect()
if state.shutdown == 0:
self.stop.wait(singleCleaner.cycleLength)
def run(self):
self.conn = sqlite3.connect(shared.appdata + 'messages.dat')
self.conn.text_factory = str
self.cur = self.conn.cursor()
try:
self.cur.execute(
'''CREATE TABLE inbox (msgid blob, toaddress text, fromaddress text, subject text, received text, message text, folder text, encodingtype int, read bool, UNIQUE(msgid) ON CONFLICT REPLACE)'''
)
self.cur.execute(
'''CREATE TABLE sent (msgid blob, toaddress text, toripe blob, fromaddress text, subject text, message text, ackdata blob, lastactiontime integer, status text, pubkeyretrynumber integer, msgretrynumber integer, folder text, encodingtype int)'''
)
self.cur.execute(
'''CREATE TABLE subscriptions (label text, address text, enabled bool)'''
)
self.cur.execute(
'''CREATE TABLE addressbook (label text, address text)''')
self.cur.execute(
'''CREATE TABLE blacklist (label text, address text, enabled bool)'''
)
self.cur.execute(
'''CREATE TABLE whitelist (label text, address text, enabled bool)'''
)
"""
Explanation of what is in the pubkeys table:
The hash is the RIPEMD160 hash that is encoded in the Bitmessage address.
transmitdata /was/ literally the data that was included in the Bitmessage pubkey message when it arrived,
except for the 24 byte protocol header- ie, it started with the POW nonce. Since protocol v3, to maintain
backwards compability, the data format of the data on disk is staying the same even though the wire format has changed.
time is the time that the pubkey was broadcast on the network same as with every other type of Bitmessage object.
usedpersonally is set to "yes" if we have used the key personally. This keeps us from deleting it because we may want to
reply to a message in the future. This field is not a bool because we may need more flexability in the future and it doesn't
take up much more space anyway.
"""
self.cur.execute(
'''CREATE TABLE pubkeys (hash blob, addressversion int, transmitdata blob, time int, usedpersonally text, UNIQUE(hash, addressversion) ON CONFLICT REPLACE)'''
)
self.cur.execute(
'''CREATE TABLE inventory (hash blob, objecttype int, streamnumber int, payload blob, expirestime integer, tag blob, UNIQUE(hash) ON CONFLICT REPLACE)'''
)
self.cur.execute(
'''INSERT INTO subscriptions VALUES('Bitmessage new releases/announcements','BM-GtovgYdgs7qXPkoYaRgrLFuFKz1SFpsw',1)'''
)
self.cur.execute(
'''CREATE TABLE settings (key blob, value blob, UNIQUE(key) ON CONFLICT REPLACE)'''
)
self.cur.execute('''INSERT INTO settings VALUES('version','7')''')
self.cur.execute(
'''INSERT INTO settings VALUES('lastvacuumtime',?)''',
(int(time.time()), ))
self.cur.execute(
'''CREATE TABLE objectprocessorqueue (objecttype int, data blob, UNIQUE(objecttype, data) ON CONFLICT REPLACE)'''
)
self.conn.commit()
logger.info('Created messages database file')
except Exception as err:
if str(err) == 'table inbox already exists':
logger.debug('Database file already exists.')
else:
sys.stderr.write(
'ERROR trying to create database file (message.dat). Error message: %s\n'
% str(err))
os._exit(0)
if shared.config.getint('bitmessagesettings', 'settingsversion') == 1:
shared.config.set('bitmessagesettings', 'settingsversion', '2')
# If the settings version is equal to 2 or 3 then the
# sqlThread will modify the pubkeys table and change
# the settings version to 4.
shared.config.set('bitmessagesettings', 'socksproxytype', 'none')
shared.config.set('bitmessagesettings', 'sockshostname',
'localhost')
shared.config.set('bitmessagesettings', 'socksport', '9050')
shared.config.set('bitmessagesettings', 'socksauthentication',
'false')
shared.config.set('bitmessagesettings', 'socksusername', '')
shared.config.set('bitmessagesettings', 'sockspassword', '')
shared.config.set('bitmessagesettings', 'sockslisten', 'false')
shared.config.set('bitmessagesettings', 'keysencrypted', 'false')
shared.config.set('bitmessagesettings', 'messagesencrypted',
'false')
with open(shared.appdata + 'keys.dat', 'wb') as configfile:
shared.config.write(configfile)
# People running earlier versions of PyBitmessage do not have the
# usedpersonally field in their pubkeys table. Let's add it.
if shared.config.getint('bitmessagesettings', 'settingsversion') == 2:
item = '''ALTER TABLE pubkeys ADD usedpersonally text DEFAULT 'no' '''
parameters = ''
self.cur.execute(item, parameters)
self.conn.commit()
shared.config.set('bitmessagesettings', 'settingsversion', '3')
with open(shared.appdata + 'keys.dat', 'wb') as configfile:
shared.config.write(configfile)
# People running earlier versions of PyBitmessage do not have the
# encodingtype field in their inbox and sent tables or the read field
# in the inbox table. Let's add them.
if shared.config.getint('bitmessagesettings', 'settingsversion') == 3:
item = '''ALTER TABLE inbox ADD encodingtype int DEFAULT '2' '''
parameters = ''
self.cur.execute(item, parameters)
item = '''ALTER TABLE inbox ADD read bool DEFAULT '1' '''
parameters = ''
self.cur.execute(item, parameters)
item = '''ALTER TABLE sent ADD encodingtype int DEFAULT '2' '''
parameters = ''
self.cur.execute(item, parameters)
self.conn.commit()
shared.config.set('bitmessagesettings', 'settingsversion', '4')
if shared.config.getint('bitmessagesettings', 'settingsversion') == 4:
shared.config.set(
'bitmessagesettings', 'defaultnoncetrialsperbyte',
str(shared.networkDefaultProofOfWorkNonceTrialsPerByte))
shared.config.set(
'bitmessagesettings', 'defaultpayloadlengthextrabytes',
str(shared.networkDefaultPayloadLengthExtraBytes))
shared.config.set('bitmessagesettings', 'settingsversion', '5')
if shared.config.getint('bitmessagesettings', 'settingsversion') == 5:
shared.config.set('bitmessagesettings',
'maxacceptablenoncetrialsperbyte', '0')
shared.config.set('bitmessagesettings',
'maxacceptablepayloadlengthextrabytes', '0')
shared.config.set('bitmessagesettings', 'settingsversion', '6')
# From now on, let us keep a 'version' embedded in the messages.dat
# file so that when we make changes to the database, the database
# version we are on can stay embedded in the messages.dat file. Let us
# check to see if the settings table exists yet.
item = '''SELECT name FROM sqlite_master WHERE type='table' AND name='settings';'''
parameters = ''
self.cur.execute(item, parameters)
if self.cur.fetchall() == []:
# The settings table doesn't exist. We need to make it.
logger.debug(
'In messages.dat database, creating new \'settings\' table.')
self.cur.execute(
'''CREATE TABLE settings (key text, value blob, UNIQUE(key) ON CONFLICT REPLACE)'''
)
self.cur.execute('''INSERT INTO settings VALUES('version','1')''')
self.cur.execute(
'''INSERT INTO settings VALUES('lastvacuumtime',?)''',
(int(time.time()), ))
logger.debug(
'In messages.dat database, removing an obsolete field from the pubkeys table.'
)
self.cur.execute(
'''CREATE TEMPORARY TABLE pubkeys_backup(hash blob, transmitdata blob, time int, usedpersonally text, UNIQUE(hash) ON CONFLICT REPLACE);'''
)
self.cur.execute(
'''INSERT INTO pubkeys_backup SELECT hash, transmitdata, time, usedpersonally FROM pubkeys;'''
)
self.cur.execute('''DROP TABLE pubkeys''')
self.cur.execute(
'''CREATE TABLE pubkeys (hash blob, transmitdata blob, time int, usedpersonally text, UNIQUE(hash) ON CONFLICT REPLACE)'''
)
self.cur.execute(
'''INSERT INTO pubkeys SELECT hash, transmitdata, time, usedpersonally FROM pubkeys_backup;'''
)
self.cur.execute('''DROP TABLE pubkeys_backup;''')
logger.debug(
'Deleting all pubkeys from inventory. They will be redownloaded and then saved with the correct times.'
)
self.cur.execute(
'''delete from inventory where objecttype = 'pubkey';''')
logger.debug(
'replacing Bitmessage announcements mailing list with a new one.'
)
self.cur.execute(
'''delete from subscriptions where address='BM-BbkPSZbzPwpVcYZpU4yHwf9ZPEapN5Zx' '''
)
self.cur.execute(
'''INSERT INTO subscriptions VALUES('Bitmessage new releases/announcements','BM-GtovgYdgs7qXPkoYaRgrLFuFKz1SFpsw',1)'''
)
logger.debug('Commiting.')
self.conn.commit()
logger.debug(
'Vacuuming message.dat. You might notice that the file size gets much smaller.'
)
self.cur.execute(''' VACUUM ''')
# After code refactoring, the possible status values for sent messages
# have changed.
self.cur.execute(
'''update sent set status='doingmsgpow' where status='doingpow' '''
)
self.cur.execute(
'''update sent set status='msgsent' where status='sentmessage' '''
)
self.cur.execute(
'''update sent set status='doingpubkeypow' where status='findingpubkey' '''
)
self.cur.execute(
'''update sent set status='broadcastqueued' where status='broadcastpending' '''
)
self.conn.commit()
if not shared.config.has_option('bitmessagesettings', 'sockslisten'):
shared.config.set('bitmessagesettings', 'sockslisten', 'false')
ensureNamecoinOptions()
"""# Add a new column to the inventory table to store the first 20 bytes of encrypted messages to support Android app
item = '''SELECT value FROM settings WHERE key='version';'''
parameters = ''
self.cur.execute(item, parameters)
if int(self.cur.fetchall()[0][0]) == 1:
print 'upgrading database'
item = '''ALTER TABLE inventory ADD first20bytesofencryptedmessage blob DEFAULT '' '''
parameters = ''
self.cur.execute(item, parameters)
item = '''update settings set value=? WHERE key='version';'''
parameters = (2,)
self.cur.execute(item, parameters)"""
# Let's get rid of the first20bytesofencryptedmessage field in the inventory table.
item = '''SELECT value FROM settings WHERE key='version';'''
parameters = ''
self.cur.execute(item, parameters)
if int(self.cur.fetchall()[0][0]) == 2:
logger.debug(
'In messages.dat database, removing an obsolete field from the inventory table.'
)
self.cur.execute(
'''CREATE TEMPORARY TABLE inventory_backup(hash blob, objecttype text, streamnumber int, payload blob, receivedtime integer, UNIQUE(hash) ON CONFLICT REPLACE);'''
)
self.cur.execute(
'''INSERT INTO inventory_backup SELECT hash, objecttype, streamnumber, payload, receivedtime FROM inventory;'''
)
self.cur.execute('''DROP TABLE inventory''')
self.cur.execute(
'''CREATE TABLE inventory (hash blob, objecttype text, streamnumber int, payload blob, receivedtime integer, UNIQUE(hash) ON CONFLICT REPLACE)'''
)
self.cur.execute(
'''INSERT INTO inventory SELECT hash, objecttype, streamnumber, payload, receivedtime FROM inventory_backup;'''
)
self.cur.execute('''DROP TABLE inventory_backup;''')
item = '''update settings set value=? WHERE key='version';'''
parameters = (3, )
self.cur.execute(item, parameters)
# Add a new column to the inventory table to store tags.
item = '''SELECT value FROM settings WHERE key='version';'''
parameters = ''
self.cur.execute(item, parameters)
currentVersion = int(self.cur.fetchall()[0][0])
if currentVersion == 1 or currentVersion == 3:
logger.debug(
'In messages.dat database, adding tag field to the inventory table.'
)
item = '''ALTER TABLE inventory ADD tag blob DEFAULT '' '''
parameters = ''
self.cur.execute(item, parameters)
item = '''update settings set value=? WHERE key='version';'''
parameters = (4, )
self.cur.execute(item, parameters)
if not shared.config.has_option('bitmessagesettings', 'userlocale'):
shared.config.set('bitmessagesettings', 'userlocale', 'system')
if not shared.config.has_option('bitmessagesettings',
'sendoutgoingconnections'):
shared.config.set('bitmessagesettings', 'sendoutgoingconnections',
'True')
# Raise the default required difficulty from 1 to 2
# With the change to protocol v3, this is obsolete.
if shared.config.getint('bitmessagesettings', 'settingsversion') == 6:
"""if int(shared.config.get('bitmessagesettings','defaultnoncetrialsperbyte')) == shared.networkDefaultProofOfWorkNonceTrialsPerByte:
shared.config.set('bitmessagesettings','defaultnoncetrialsperbyte', str(shared.networkDefaultProofOfWorkNonceTrialsPerByte * 2))
"""
shared.config.set('bitmessagesettings', 'settingsversion', '7')
# Add a new column to the pubkeys table to store the address version.
# We're going to trash all of our pubkeys and let them be redownloaded.
item = '''SELECT value FROM settings WHERE key='version';'''
parameters = ''
self.cur.execute(item, parameters)
currentVersion = int(self.cur.fetchall()[0][0])
if currentVersion == 4:
self.cur.execute('''DROP TABLE pubkeys''')
self.cur.execute(
'''CREATE TABLE pubkeys (hash blob, addressversion int, transmitdata blob, time int, usedpersonally text, UNIQUE(hash, addressversion) ON CONFLICT REPLACE)'''
)
self.cur.execute(
'''delete from inventory where objecttype = 'pubkey';''')
item = '''update settings set value=? WHERE key='version';'''
parameters = (5, )
self.cur.execute(item, parameters)
if not shared.config.has_option('bitmessagesettings', 'useidenticons'):
shared.config.set('bitmessagesettings', 'useidenticons', 'True')
if not shared.config.has_option('bitmessagesettings',
'identiconsuffix'): # acts as a salt
shared.config.set('bitmessagesettings', 'identiconsuffix', ''.join(
random.choice(
"123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz"
) for x in range(12)
)) # a twelve character pseudo-password to salt the identicons
#Add settings to support no longer resending messages after a certain period of time even if we never get an ack
if shared.config.getint('bitmessagesettings', 'settingsversion') == 7:
shared.config.set('bitmessagesettings', 'stopresendingafterxdays',
'')
shared.config.set('bitmessagesettings',
'stopresendingafterxmonths', '')
#shared.config.set(
shared.config.set('bitmessagesettings', 'settingsversion', '8')
with open(shared.appdata + 'keys.dat', 'wb') as configfile:
shared.config.write(configfile)
# Add a new table: objectprocessorqueue with which to hold objects
# that have yet to be processed if the user shuts down Bitmessage.
item = '''SELECT value FROM settings WHERE key='version';'''
parameters = ''
self.cur.execute(item, parameters)
currentVersion = int(self.cur.fetchall()[0][0])
if currentVersion == 5:
self.cur.execute('''DROP TABLE knownnodes''')
self.cur.execute(
'''CREATE TABLE objectprocessorqueue (objecttype text, data blob, UNIQUE(objecttype, data) ON CONFLICT REPLACE)'''
)
item = '''update settings set value=? WHERE key='version';'''
parameters = (6, )
self.cur.execute(item, parameters)
# changes related to protocol v3
# In table inventory and objectprocessorqueue, objecttype is now an integer (it was a human-friendly string previously)
item = '''SELECT value FROM settings WHERE key='version';'''
parameters = ''
self.cur.execute(item, parameters)
currentVersion = int(self.cur.fetchall()[0][0])
if currentVersion == 6:
logger.debug(
'In messages.dat database, dropping and recreating the inventory table.'
)
self.cur.execute('''DROP TABLE inventory''')
self.cur.execute(
'''CREATE TABLE inventory (hash blob, objecttype int, streamnumber int, payload blob, expirestime integer, tag blob, UNIQUE(hash) ON CONFLICT REPLACE)'''
)
self.cur.execute('''DROP TABLE objectprocessorqueue''')
self.cur.execute(
'''CREATE TABLE objectprocessorqueue (objecttype int, data blob, UNIQUE(objecttype, data) ON CONFLICT REPLACE)'''
)
item = '''update settings set value=? WHERE key='version';'''
parameters = (7, )
self.cur.execute(item, parameters)
logger.debug(
'Finished dropping and recreating the inventory table.')
# With the change to protocol version 3, reset the user-settable difficulties to 1
if shared.config.getint('bitmessagesettings', 'settingsversion') == 8:
shared.config.set(
'bitmessagesettings', 'defaultnoncetrialsperbyte',
str(shared.networkDefaultProofOfWorkNonceTrialsPerByte))
shared.config.set(
'bitmessagesettings', 'defaultpayloadlengthextrabytes',
str(shared.networkDefaultPayloadLengthExtraBytes))
previousTotalDifficulty = int(
shared.config.getint('bitmessagesettings',
'maxacceptablenoncetrialsperbyte')) / 320
previousSmallMessageDifficulty = int(
shared.config.getint(
'bitmessagesettings',
'maxacceptablepayloadlengthextrabytes')) / 14000
shared.config.set('bitmessagesettings',
'maxacceptablenoncetrialsperbyte',
str(previousTotalDifficulty * 1000))
shared.config.set('bitmessagesettings',
'maxacceptablepayloadlengthextrabytes',
str(previousSmallMessageDifficulty * 1000))
shared.config.set('bitmessagesettings', 'settingsversion', '9')
# Adjust the required POW values for each of this user's addresses to conform to protocol v3 norms.
if shared.config.getint('bitmessagesettings', 'settingsversion') == 9:
for addressInKeysFile in shared.config.sections():
try:
previousTotalDifficulty = float(
shared.config.getint(addressInKeysFile,
'noncetrialsperbyte')) / 320
previousSmallMessageDifficulty = float(
shared.config.getint(
addressInKeysFile,
'payloadlengthextrabytes')) / 14000
if previousTotalDifficulty <= 2:
previousTotalDifficulty = 1
if previousSmallMessageDifficulty < 1:
previousSmallMessageDifficulty = 1
shared.config.set(addressInKeysFile, 'noncetrialsperbyte',
str(int(previousTotalDifficulty * 1000)))
shared.config.set(
addressInKeysFile, 'payloadlengthextrabytes',
str(int(previousSmallMessageDifficulty * 1000)))
except:
continue
shared.config.set('bitmessagesettings', 'maxdownloadrate', '0')
shared.config.set('bitmessagesettings', 'maxuploadrate', '0')
shared.config.set('bitmessagesettings', 'settingsversion', '10')
with open(shared.appdata + 'keys.dat', 'wb') as configfile:
shared.config.write(configfile)
# Are you hoping to add a new option to the keys.dat file of existing
# Bitmessage users or modify the SQLite database? Add it right above this line!
try:
testpayload = '\x00\x00'
t = ('1234', 1, testpayload, '12345678', 'no')
self.cur.execute('''INSERT INTO pubkeys VALUES(?,?,?,?,?)''', t)
self.conn.commit()
self.cur.execute(
'''SELECT transmitdata FROM pubkeys WHERE hash='1234' ''')
queryreturn = self.cur.fetchall()
for row in queryreturn:
transmitdata, = row
self.cur.execute('''DELETE FROM pubkeys WHERE hash='1234' ''')
self.conn.commit()
if transmitdata == '':
logger.fatal(
'Problem: The version of SQLite you have cannot store Null values. Please download and install the latest revision of your version of Python (for example, the latest Python 2.7 revision) and try again.\n'
)
logger.fatal(
'PyBitmessage will now exit very abruptly. You may now see threading errors related to this abrupt exit but the problem you need to solve is related to SQLite.\n\n'
)
os._exit(0)
except Exception as err:
if str(err) == 'database or disk is full':
logger.fatal(
'(While null value test) Alert: Your disk or data storage volume is full. sqlThread will now exit.'
)
shared.UISignalQueue.put(('alert', (
tr.translateText("MainWindow", "Disk full"),
tr.translateText(
"MainWindow",
'Alert: Your disk or data storage volume is full. Bitmessage will now exit.'
), True)))
if shared.daemon:
os._exit(0)
else:
return
else:
logger.error(err)
# Let us check to see the last time we vaccumed the messages.dat file.
# If it has been more than a month let's do it now.
item = '''SELECT value FROM settings WHERE key='lastvacuumtime';'''
parameters = ''
self.cur.execute(item, parameters)
queryreturn = self.cur.fetchall()
for row in queryreturn:
value, = row
if int(value) < int(time.time()) - 2592000:
logger.info(
'It has been a long time since the messages.dat file has been vacuumed. Vacuuming now...'
)
try:
self.cur.execute(''' VACUUM ''')
except Exception as err:
if str(err) == 'database or disk is full':
logger.fatal(
'(While VACUUM) Alert: Your disk or data storage volume is full. sqlThread will now exit.'
)
shared.UISignalQueue.put(('alert', (
tr.translateText("MainWindow", "Disk full"),
tr.translateText(
"MainWindow",
'Alert: Your disk or data storage volume is full. Bitmessage will now exit.'
), True)))
if shared.daemon:
os._exit(0)
else:
return
item = '''update settings set value=? WHERE key='lastvacuumtime';'''
parameters = (int(time.time()), )
self.cur.execute(item, parameters)
while True:
item = shared.sqlSubmitQueue.get()
if item == 'commit':
try:
self.conn.commit()
except Exception as err:
if str(err) == 'database or disk is full':
logger.fatal(
'(While committing) Alert: Your disk or data storage volume is full. sqlThread will now exit.'
)
shared.UISignalQueue.put(('alert', (
tr.translateText("MainWindow", "Disk full"),
tr.translateText(
"MainWindow",
'Alert: Your disk or data storage volume is full. Bitmessage will now exit.'
), True)))
if shared.daemon:
os._exit(0)
else:
return
elif item == 'exit':
self.conn.close()
logger.info('sqlThread exiting gracefully.')
return
elif item == 'movemessagstoprog':
logger.debug(
'the sqlThread is moving the messages.dat file to the local program directory.'
)
try:
self.conn.commit()
except Exception as err:
if str(err) == 'database or disk is full':
logger.fatal(
'(while movemessagstoprog) Alert: Your disk or data storage volume is full. sqlThread will now exit.'
)
shared.UISignalQueue.put(('alert', (
tr.translateText("MainWindow", "Disk full"),
tr.translateText(
"MainWindow",
'Alert: Your disk or data storage volume is full. Bitmessage will now exit.'
), True)))
if shared.daemon:
os._exit(0)
else:
return
self.conn.close()
shutil.move(shared.lookupAppdataFolder() + 'messages.dat',
'messages.dat')
self.conn = sqlite3.connect('messages.dat')
self.conn.text_factory = str
self.cur = self.conn.cursor()
elif item == 'movemessagstoappdata':
logger.debug(
'the sqlThread is moving the messages.dat file to the Appdata folder.'
)
try:
self.conn.commit()
except Exception as err:
if str(err) == 'database or disk is full':
logger.fatal(
'(while movemessagstoappdata) Alert: Your disk or data storage volume is full. sqlThread will now exit.'
)
shared.UISignalQueue.put(('alert', (
tr.translateText("MainWindow", "Disk full"),
tr.translateText(
"MainWindow",
'Alert: Your disk or data storage volume is full. Bitmessage will now exit.'
), True)))
if shared.daemon:
os._exit(0)
else:
return
self.conn.close()
shutil.move('messages.dat',
shared.lookupAppdataFolder() + 'messages.dat')
self.conn = sqlite3.connect(shared.appdata + 'messages.dat')
self.conn.text_factory = str
self.cur = self.conn.cursor()
elif item == 'deleteandvacuume':
self.cur.execute('''delete from inbox where folder='trash' ''')
self.cur.execute('''delete from sent where folder='trash' ''')
self.conn.commit()
try:
self.cur.execute(''' VACUUM ''')
except Exception as err:
if str(err) == 'database or disk is full':
logger.fatal(
'(while deleteandvacuume) Alert: Your disk or data storage volume is full. sqlThread will now exit.'
)
shared.UISignalQueue.put(('alert', (
tr.translateText("MainWindow", "Disk full"),
tr.translateText(
"MainWindow",
'Alert: Your disk or data storage volume is full. Bitmessage will now exit.'
), True)))
if shared.daemon:
os._exit(0)
else:
return
else:
parameters = shared.sqlSubmitQueue.get()
# print 'item', item
# print 'parameters', parameters
try:
self.cur.execute(item, parameters)
except Exception as err:
if str(err) == 'database or disk is full':
logger.fatal(
'(while cur.execute) Alert: Your disk or data storage volume is full. sqlThread will now exit.'
)
shared.UISignalQueue.put(('alert', (
tr.translateText("MainWindow", "Disk full"),
tr.translateText(
"MainWindow",
'Alert: Your disk or data storage volume is full. Bitmessage will now exit.'
), True)))
if shared.daemon:
os._exit(0)
else:
return
else:
logger.fatal(
'Major error occurred when trying to execute a SQL statement within the sqlThread. Please tell Atheros about this error message or post it in the forum! Error occurred while trying to execute statement: "%s" Here are the parameters; you might want to censor this data with asterisks (***) as it can contain private information: %s. Here is the actual error message thrown by the sqlThread: %s',
str(item), str(repr(parameters)), str(err))
logger.fatal('This program shall now abruptly exit!')
os._exit(0)
shared.sqlReturnQueue.put(self.cur.fetchall())
def run(self):
self.conn = sqlite3.connect(shared.appdata + 'messages.dat')
self.conn.text_factory = str
self.cur = self.conn.cursor()
try:
self.cur.execute(
'''CREATE TABLE inbox (msgid blob, toaddress text, fromaddress text, subject text, received text, message text, folder text, encodingtype int, read bool, UNIQUE(msgid) ON CONFLICT REPLACE)''' )
self.cur.execute(
'''CREATE TABLE sent (msgid blob, toaddress text, toripe blob, fromaddress text, subject text, message text, ackdata blob, lastactiontime integer, status text, pubkeyretrynumber integer, msgretrynumber integer, folder text, encodingtype int)''' )
self.cur.execute(
'''CREATE TABLE subscriptions (label text, address text, enabled bool)''' )
self.cur.execute(
'''CREATE TABLE addressbook (label text, address text)''' )
self.cur.execute(
'''CREATE TABLE blacklist (label text, address text, enabled bool)''' )
self.cur.execute(
'''CREATE TABLE whitelist (label text, address text, enabled bool)''' )
# Explanation of what is in the pubkeys table:
# The hash is the RIPEMD160 hash that is encoded in the Bitmessage address.
# transmitdata is literally the data that was included in the Bitmessage pubkey message when it arrived, except for the 24 byte protocol header- ie, it starts with the POW nonce.
# time is the time that the pubkey was broadcast on the network same as with every other type of Bitmessage object.
# usedpersonally is set to "yes" if we have used the key
# personally. This keeps us from deleting it because we may want to
# reply to a message in the future. This field is not a bool
# because we may need more flexability in the future and it doesn't
# take up much more space anyway.
self.cur.execute(
'''CREATE TABLE pubkeys (hash blob, addressversion int, transmitdata blob, time int, usedpersonally text, UNIQUE(hash, addressversion) ON CONFLICT REPLACE)''' )
self.cur.execute(
'''CREATE TABLE inventory (hash blob, objecttype text, streamnumber int, payload blob, receivedtime integer, tag blob, UNIQUE(hash) ON CONFLICT REPLACE)''' )
self.cur.execute(
'''INSERT INTO subscriptions VALUES('Bitmessage new releases/announcements','BM-GtovgYdgs7qXPkoYaRgrLFuFKz1SFpsw',1)''')
self.cur.execute(
'''CREATE TABLE settings (key blob, value blob, UNIQUE(key) ON CONFLICT REPLACE)''' )
self.cur.execute( '''INSERT INTO settings VALUES('version','6')''')
self.cur.execute( '''INSERT INTO settings VALUES('lastvacuumtime',?)''', (
int(time.time()),))
self.cur.execute(
'''CREATE TABLE objectprocessorqueue (objecttype text, data blob, UNIQUE(objecttype, data) ON CONFLICT REPLACE)''' )
self.conn.commit()
logger.info('Created messages database file')
except Exception as err:
if str(err) == 'table inbox already exists':
logger.debug('Database file already exists.')
else:
sys.stderr.write(
'ERROR trying to create database file (message.dat). Error message: %s\n' % str(err))
os._exit(0)
if shared.config.getint('bitmessagesettings', 'settingsversion') == 1:
shared.config.set('bitmessagesettings', 'settingsversion', '2')
# If the settings version is equal to 2 or 3 then the
# sqlThread will modify the pubkeys table and change
# the settings version to 4.
shared.config.set('bitmessagesettings', 'socksproxytype', 'none')
shared.config.set('bitmessagesettings', 'sockshostname', 'localhost')
shared.config.set('bitmessagesettings', 'socksport', '9050')
shared.config.set('bitmessagesettings', 'socksauthentication', 'false')
shared.config.set('bitmessagesettings', 'socksusername', '')
shared.config.set('bitmessagesettings', 'sockspassword', '')
shared.config.set('bitmessagesettings', 'sockslisten', 'false')
shared.config.set('bitmessagesettings', 'keysencrypted', 'false')
shared.config.set('bitmessagesettings', 'messagesencrypted', 'false')
with open(shared.appdata + 'keys.dat', 'wb') as configfile:
shared.config.write(configfile)
# People running earlier versions of PyBitmessage do not have the
# usedpersonally field in their pubkeys table. Let's add it.
if shared.config.getint('bitmessagesettings', 'settingsversion') == 2:
item = '''ALTER TABLE pubkeys ADD usedpersonally text DEFAULT 'no' '''
parameters = ''
self.cur.execute(item, parameters)
self.conn.commit()
shared.config.set('bitmessagesettings', 'settingsversion', '3')
with open(shared.appdata + 'keys.dat', 'wb') as configfile:
shared.config.write(configfile)
# People running earlier versions of PyBitmessage do not have the
# encodingtype field in their inbox and sent tables or the read field
# in the inbox table. Let's add them.
if shared.config.getint('bitmessagesettings', 'settingsversion') == 3:
item = '''ALTER TABLE inbox ADD encodingtype int DEFAULT '2' '''
parameters = ''
self.cur.execute(item, parameters)
item = '''ALTER TABLE inbox ADD read bool DEFAULT '1' '''
parameters = ''
self.cur.execute(item, parameters)
item = '''ALTER TABLE sent ADD encodingtype int DEFAULT '2' '''
parameters = ''
self.cur.execute(item, parameters)
self.conn.commit()
shared.config.set('bitmessagesettings', 'settingsversion', '4')
with open(shared.appdata + 'keys.dat', 'wb') as configfile:
shared.config.write(configfile)
if shared.config.getint('bitmessagesettings', 'settingsversion') == 4:
shared.config.set('bitmessagesettings', 'defaultnoncetrialsperbyte', str(
shared.networkDefaultProofOfWorkNonceTrialsPerByte))
shared.config.set('bitmessagesettings', 'defaultpayloadlengthextrabytes', str(
shared.networkDefaultPayloadLengthExtraBytes))
shared.config.set('bitmessagesettings', 'settingsversion', '5')
if shared.config.getint('bitmessagesettings', 'settingsversion') == 5:
shared.config.set(
'bitmessagesettings', 'maxacceptablenoncetrialsperbyte', '0')
shared.config.set(
'bitmessagesettings', 'maxacceptablepayloadlengthextrabytes', '0')
shared.config.set('bitmessagesettings', 'settingsversion', '6')
with open(shared.appdata + 'keys.dat', 'wb') as configfile:
shared.config.write(configfile)
# From now on, let us keep a 'version' embedded in the messages.dat
# file so that when we make changes to the database, the database
# version we are on can stay embedded in the messages.dat file. Let us
# check to see if the settings table exists yet.
item = '''SELECT name FROM sqlite_master WHERE type='table' AND name='settings';'''
parameters = ''
self.cur.execute(item, parameters)
if self.cur.fetchall() == []:
# The settings table doesn't exist. We need to make it.
logger.debug('In messages.dat database, creating new \'settings\' table.')
self.cur.execute(
'''CREATE TABLE settings (key text, value blob, UNIQUE(key) ON CONFLICT REPLACE)''' )
self.cur.execute( '''INSERT INTO settings VALUES('version','1')''')
self.cur.execute( '''INSERT INTO settings VALUES('lastvacuumtime',?)''', (
int(time.time()),))
logger.debug('In messages.dat database, removing an obsolete field from the pubkeys table.')
self.cur.execute(
'''CREATE TEMPORARY TABLE pubkeys_backup(hash blob, transmitdata blob, time int, usedpersonally text, UNIQUE(hash) ON CONFLICT REPLACE);''')
self.cur.execute(
'''INSERT INTO pubkeys_backup SELECT hash, transmitdata, time, usedpersonally FROM pubkeys;''')
self.cur.execute( '''DROP TABLE pubkeys''')
self.cur.execute(
'''CREATE TABLE pubkeys (hash blob, transmitdata blob, time int, usedpersonally text, UNIQUE(hash) ON CONFLICT REPLACE)''' )
self.cur.execute(
'''INSERT INTO pubkeys SELECT hash, transmitdata, time, usedpersonally FROM pubkeys_backup;''')
self.cur.execute( '''DROP TABLE pubkeys_backup;''')
logger.debug('Deleting all pubkeys from inventory. They will be redownloaded and then saved with the correct times.')
self.cur.execute(
'''delete from inventory where objecttype = 'pubkey';''')
logger.debug('replacing Bitmessage announcements mailing list with a new one.')
self.cur.execute(
'''delete from subscriptions where address='BM-BbkPSZbzPwpVcYZpU4yHwf9ZPEapN5Zx' ''')
self.cur.execute(
'''INSERT INTO subscriptions VALUES('Bitmessage new releases/announcements','BM-GtovgYdgs7qXPkoYaRgrLFuFKz1SFpsw',1)''')
logger.debug('Commiting.')
self.conn.commit()
logger.debug('Vacuuming message.dat. You might notice that the file size gets much smaller.')
self.cur.execute( ''' VACUUM ''')
# After code refactoring, the possible status values for sent messages
# have changed.
self.cur.execute(
'''update sent set status='doingmsgpow' where status='doingpow' ''')
self.cur.execute(
'''update sent set status='msgsent' where status='sentmessage' ''')
self.cur.execute(
'''update sent set status='doingpubkeypow' where status='findingpubkey' ''')
self.cur.execute(
'''update sent set status='broadcastqueued' where status='broadcastpending' ''')
self.conn.commit()
if not shared.config.has_option('bitmessagesettings', 'sockslisten'):
shared.config.set('bitmessagesettings', 'sockslisten', 'false')
ensureNamecoinOptions()
"""# Add a new column to the inventory table to store the first 20 bytes of encrypted messages to support Android app
item = '''SELECT value FROM settings WHERE key='version';'''
parameters = ''
self.cur.execute(item, parameters)
if int(self.cur.fetchall()[0][0]) == 1:
print 'upgrading database'
item = '''ALTER TABLE inventory ADD first20bytesofencryptedmessage blob DEFAULT '' '''
parameters = ''
self.cur.execute(item, parameters)
item = '''update settings set value=? WHERE key='version';'''
parameters = (2,)
self.cur.execute(item, parameters)"""
# Let's get rid of the first20bytesofencryptedmessage field in the inventory table.
item = '''SELECT value FROM settings WHERE key='version';'''
parameters = ''
self.cur.execute(item, parameters)
if int(self.cur.fetchall()[0][0]) == 2:
logger.debug('In messages.dat database, removing an obsolete field from the inventory table.')
self.cur.execute(
'''CREATE TEMPORARY TABLE inventory_backup(hash blob, objecttype text, streamnumber int, payload blob, receivedtime integer, UNIQUE(hash) ON CONFLICT REPLACE);''')
self.cur.execute(
'''INSERT INTO inventory_backup SELECT hash, objecttype, streamnumber, payload, receivedtime FROM inventory;''')
self.cur.execute( '''DROP TABLE inventory''')
self.cur.execute(
'''CREATE TABLE inventory (hash blob, objecttype text, streamnumber int, payload blob, receivedtime integer, UNIQUE(hash) ON CONFLICT REPLACE)''' )
self.cur.execute(
'''INSERT INTO inventory SELECT hash, objecttype, streamnumber, payload, receivedtime FROM inventory_backup;''')
self.cur.execute( '''DROP TABLE inventory_backup;''')
item = '''update settings set value=? WHERE key='version';'''
parameters = (3,)
self.cur.execute(item, parameters)
# Add a new column to the inventory table to store tags.
item = '''SELECT value FROM settings WHERE key='version';'''
parameters = ''
self.cur.execute(item, parameters)
currentVersion = int(self.cur.fetchall()[0][0])
if currentVersion == 1 or currentVersion == 3:
logger.debug('In messages.dat database, adding tag field to the inventory table.')
item = '''ALTER TABLE inventory ADD tag blob DEFAULT '' '''
parameters = ''
self.cur.execute(item, parameters)
item = '''update settings set value=? WHERE key='version';'''
parameters = (4,)
self.cur.execute(item, parameters)
if not shared.config.has_option('bitmessagesettings', 'userlocale'):
shared.config.set('bitmessagesettings', 'userlocale', 'system')
if not shared.config.has_option('bitmessagesettings', 'sendoutgoingconnections'):
shared.config.set('bitmessagesettings', 'sendoutgoingconnections', 'True')
# Raise the default required difficulty from 1 to 2
if shared.config.getint('bitmessagesettings', 'settingsversion') == 6:
if int(shared.config.get('bitmessagesettings','defaultnoncetrialsperbyte')) == shared.networkDefaultProofOfWorkNonceTrialsPerByte:
shared.config.set('bitmessagesettings','defaultnoncetrialsperbyte', str(shared.networkDefaultProofOfWorkNonceTrialsPerByte * 2))
shared.config.set('bitmessagesettings', 'settingsversion', '7')
with open(shared.appdata + 'keys.dat', 'wb') as configfile:
shared.config.write(configfile)
# Add a new column to the pubkeys table to store the address version.
# We're going to trash all of our pubkeys and let them be redownloaded.
item = '''SELECT value FROM settings WHERE key='version';'''
parameters = ''
self.cur.execute(item, parameters)
currentVersion = int(self.cur.fetchall()[0][0])
if currentVersion == 4:
self.cur.execute( '''DROP TABLE pubkeys''')
self.cur.execute(
'''CREATE TABLE pubkeys (hash blob, addressversion int, transmitdata blob, time int, usedpersonally text, UNIQUE(hash, addressversion) ON CONFLICT REPLACE)''' )
self.cur.execute(
'''delete from inventory where objecttype = 'pubkey';''')
item = '''update settings set value=? WHERE key='version';'''
parameters = (5,)
self.cur.execute(item, parameters)
if not shared.config.has_option('bitmessagesettings', 'useidenticons'):
shared.config.set('bitmessagesettings', 'useidenticons', 'True')
if not shared.config.has_option('bitmessagesettings', 'identiconsuffix'): # acts as a salt
shared.config.set('bitmessagesettings', 'identiconsuffix', ''.join(random.choice("123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz") for x in range(12))) # a twelve character pseudo-password to salt the identicons
# Since we've added a new config entry, let's write keys.dat to disk.
with open(shared.appdata + 'keys.dat', 'wb') as configfile:
shared.config.write(configfile)
#Adjusting time period to stop sending messages
if shared.config.getint('bitmessagesettings', 'settingsversion') == 7:
shared.config.set(
'bitmessagesettings', 'stopresendingafterxdays', '')
shared.config.set(
'bitmessagesettings', 'stopresendingafterxmonths', '')
#shared.config.set(
shared.config.set('bitmessagesettings', 'settingsversion', '8')
with open(shared.appdata + 'keys.dat', 'wb') as configfile:
shared.config.write(configfile)
# Add a new table: objectprocessorqueue with which to hold objects
# that have yet to be processed if the user shuts down Bitmessage.
item = '''SELECT value FROM settings WHERE key='version';'''
parameters = ''
self.cur.execute(item, parameters)
currentVersion = int(self.cur.fetchall()[0][0])
if currentVersion == 5:
self.cur.execute( '''DROP TABLE knownnodes''')
self.cur.execute(
'''CREATE TABLE objectprocessorqueue (objecttype text, data blob, UNIQUE(objecttype, data) ON CONFLICT REPLACE)''' )
item = '''update settings set value=? WHERE key='version';'''
parameters = (6,)
self.cur.execute(item, parameters)
# Are you hoping to add a new option to the keys.dat file of existing
# Bitmessage users? Add it right above this line!
try:
testpayload = '\x00\x00'
t = ('1234', 1, testpayload, '12345678', 'no')
self.cur.execute( '''INSERT INTO pubkeys VALUES(?,?,?,?,?)''', t)
self.conn.commit()
self.cur.execute(
'''SELECT transmitdata FROM pubkeys WHERE hash='1234' ''')
queryreturn = self.cur.fetchall()
for row in queryreturn:
transmitdata, = row
self.cur.execute('''DELETE FROM pubkeys WHERE hash='1234' ''')
self.conn.commit()
if transmitdata == '':
logger.fatal('Problem: The version of SQLite you have cannot store Null values. Please download and install the latest revision of your version of Python (for example, the latest Python 2.7 revision) and try again.\n')
logger.fatal('PyBitmessage will now exit very abruptly. You may now see threading errors related to this abrupt exit but the problem you need to solve is related to SQLite.\n\n')
os._exit(0)
except Exception as err:
if str(err) == 'database or disk is full':
logger.fatal('(While null value test) Alert: Your disk or data storage volume is full. sqlThread will now exit.')
shared.UISignalQueue.put(('alert', (tr.translateText("MainWindow", "Disk full"), tr.translateText("MainWindow", 'Alert: Your disk or data storage volume is full. Bitmessage will now exit.'), True)))
if shared.daemon:
os._exit(0)
else:
return
else:
logger.error(err)
# Let us check to see the last time we vaccumed the messages.dat file.
# If it has been more than a month let's do it now.
item = '''SELECT value FROM settings WHERE key='lastvacuumtime';'''
parameters = ''
self.cur.execute(item, parameters)
queryreturn = self.cur.fetchall()
for row in queryreturn:
value, = row
if int(value) < int(time.time()) - 2592000:
logger.info('It has been a long time since the messages.dat file has been vacuumed. Vacuuming now...')
try:
self.cur.execute( ''' VACUUM ''')
except Exception as err:
if str(err) == 'database or disk is full':
logger.fatal('(While VACUUM) Alert: Your disk or data storage volume is full. sqlThread will now exit.')
shared.UISignalQueue.put(('alert', (tr.translateText("MainWindow", "Disk full"), tr.translateText("MainWindow", 'Alert: Your disk or data storage volume is full. Bitmessage will now exit.'), True)))
if shared.daemon:
os._exit(0)
else:
return
item = '''update settings set value=? WHERE key='lastvacuumtime';'''
parameters = (int(time.time()),)
self.cur.execute(item, parameters)
while True:
item = shared.sqlSubmitQueue.get()
if item == 'commit':
try:
self.conn.commit()
except Exception as err:
if str(err) == 'database or disk is full':
logger.fatal('(While committing) Alert: Your disk or data storage volume is full. sqlThread will now exit.')
shared.UISignalQueue.put(('alert', (tr.translateText("MainWindow", "Disk full"), tr.translateText("MainWindow", 'Alert: Your disk or data storage volume is full. Bitmessage will now exit.'), True)))
if shared.daemon:
os._exit(0)
else:
return
elif item == 'exit':
self.conn.close()
logger.info('sqlThread exiting gracefully.')
return
elif item == 'movemessagstoprog':
logger.debug('the sqlThread is moving the messages.dat file to the local program directory.')
try:
self.conn.commit()
except Exception as err:
if str(err) == 'database or disk is full':
logger.fatal('(while movemessagstoprog) Alert: Your disk or data storage volume is full. sqlThread will now exit.')
shared.UISignalQueue.put(('alert', (tr.translateText("MainWindow", "Disk full"), tr.translateText("MainWindow", 'Alert: Your disk or data storage volume is full. Bitmessage will now exit.'), True)))
if shared.daemon:
os._exit(0)
else:
return
self.conn.close()
shutil.move(
shared.lookupAppdataFolder() + 'messages.dat', 'messages.dat')
self.conn = sqlite3.connect('messages.dat')
self.conn.text_factory = str
self.cur = self.conn.cursor()
elif item == 'movemessagstoappdata':
logger.debug('the sqlThread is moving the messages.dat file to the Appdata folder.')
try:
self.conn.commit()
except Exception as err:
if str(err) == 'database or disk is full':
logger.fatal('(while movemessagstoappdata) Alert: Your disk or data storage volume is full. sqlThread will now exit.')
shared.UISignalQueue.put(('alert', (tr.translateText("MainWindow", "Disk full"), tr.translateText("MainWindow", 'Alert: Your disk or data storage volume is full. Bitmessage will now exit.'), True)))
if shared.daemon:
os._exit(0)
else:
return
self.conn.close()
shutil.move(
'messages.dat', shared.lookupAppdataFolder() + 'messages.dat')
self.conn = sqlite3.connect(shared.appdata + 'messages.dat')
self.conn.text_factory = str
self.cur = self.conn.cursor()
elif item == 'deleteandvacuume':
self.cur.execute('''delete from inbox where folder='trash' ''')
self.cur.execute('''delete from sent where folder='trash' ''')
self.conn.commit()
try:
self.cur.execute( ''' VACUUM ''')
except Exception as err:
if str(err) == 'database or disk is full':
logger.fatal('(while deleteandvacuume) Alert: Your disk or data storage volume is full. sqlThread will now exit.')
shared.UISignalQueue.put(('alert', (tr.translateText("MainWindow", "Disk full"), tr.translateText("MainWindow", 'Alert: Your disk or data storage volume is full. Bitmessage will now exit.'), True)))
if shared.daemon:
os._exit(0)
else:
return
else:
parameters = shared.sqlSubmitQueue.get()
# print 'item', item
# print 'parameters', parameters
try:
self.cur.execute(item, parameters)
except Exception as err:
if str(err) == 'database or disk is full':
logger.fatal('(while cur.execute) Alert: Your disk or data storage volume is full. sqlThread will now exit.')
shared.UISignalQueue.put(('alert', (tr.translateText("MainWindow", "Disk full"), tr.translateText("MainWindow", 'Alert: Your disk or data storage volume is full. Bitmessage will now exit.'), True)))
if shared.daemon:
os._exit(0)
else:
return
else:
logger.fatal('Major error occurred when trying to execute a SQL statement within the sqlThread. Please tell Atheros about this error message or post it in the forum! Error occurred while trying to execute statement: "%s" Here are the parameters; you might want to censor this data with asterisks (***) as it can contain private information: %s. Here is the actual error message thrown by the sqlThread: %s', str(item), str(repr(parameters)), str(err))
logger.fatal('This program shall now abruptly exit!')
os._exit(0)
shared.sqlReturnQueue.put(self.cur.fetchall())
def run(self):
self.conn = sqlite3.connect(shared.appdata + 'messages.dat')
self.conn.text_factory = str
self.cur = self.conn.cursor()
try:
self.cur.execute(
'''CREATE TABLE inbox (msgid blob, toaddress text, fromaddress text, subject text, received text, message text, folder text, encodingtype int, read bool, UNIQUE(msgid) ON CONFLICT REPLACE)'''
)
self.cur.execute(
'''CREATE TABLE sent (msgid blob, toaddress text, toripe blob, fromaddress text, subject text, message text, ackdata blob, lastactiontime integer, status text, pubkeyretrynumber integer, msgretrynumber integer, folder text, encodingtype int)'''
)
self.cur.execute(
'''CREATE TABLE subscriptions (label text, address text, enabled bool)'''
)
self.cur.execute(
'''CREATE TABLE addressbook (label text, address text)''')
self.cur.execute(
'''CREATE TABLE blacklist (label text, address text, enabled bool)'''
)
self.cur.execute(
'''CREATE TABLE whitelist (label text, address text, enabled bool)'''
)
# Explanation of what is in the pubkeys table:
# The hash is the RIPEMD160 hash that is encoded in the Bitmessage address.
# transmitdata is literally the data that was included in the Bitmessage pubkey message when it arrived, except for the 24 byte protocol header- ie, it starts with the POW nonce.
# time is the time that the pubkey was broadcast on the network same as with every other type of Bitmessage object.
# usedpersonally is set to "yes" if we have used the key
# personally. This keeps us from deleting it because we may want to
# reply to a message in the future. This field is not a bool
# because we may need more flexability in the future and it doesn't
# take up much more space anyway.
self.cur.execute(
'''CREATE TABLE pubkeys (hash blob, transmitdata blob, time int, usedpersonally text, UNIQUE(hash) ON CONFLICT REPLACE)'''
)
self.cur.execute(
'''CREATE TABLE inventory (hash blob, objecttype text, streamnumber int, payload blob, receivedtime integer, first20bytesofencryptedmessage blob, UNIQUE(hash) ON CONFLICT REPLACE)'''
)
self.cur.execute(
'''CREATE TABLE knownnodes (timelastseen int, stream int, services blob, host blob, port blob, UNIQUE(host, stream, port) ON CONFLICT REPLACE)'''
)
# This table isn't used in the program yet but I
# have a feeling that we'll need it.
self.cur.execute(
'''INSERT INTO subscriptions VALUES('Bitmessage new releases/announcements','BM-GtovgYdgs7qXPkoYaRgrLFuFKz1SFpsw',1)'''
)
self.cur.execute(
'''CREATE TABLE settings (key blob, value blob, UNIQUE(key) ON CONFLICT REPLACE)'''
)
self.cur.execute('''INSERT INTO settings VALUES('version','2')''')
self.cur.execute(
'''INSERT INTO settings VALUES('lastvacuumtime',?)''',
(int(time.time()), ))
self.conn.commit()
logger.info('Created messages database file')
except Exception as err:
if str(err) == 'table inbox already exists':
logger.debug('Database file already exists.')
else:
sys.stderr.write(
'ERROR trying to create database file (message.dat). Error message: %s\n'
% str(err))
os._exit(0)
if shared.config.getint('bitmessagesettings', 'settingsversion') == 1:
shared.config.set('bitmessagesettings', 'settingsversion', '2')
# If the settings version is equal to 2 or 3 then the
# sqlThread will modify the pubkeys table and change
# the settings version to 4.
shared.config.set('bitmessagesettings', 'socksproxytype', 'none')
shared.config.set('bitmessagesettings', 'sockshostname',
'localhost')
shared.config.set('bitmessagesettings', 'socksport', '9050')
shared.config.set('bitmessagesettings', 'socksauthentication',
'false')
shared.config.set('bitmessagesettings', 'socksusername', '')
shared.config.set('bitmessagesettings', 'sockspassword', '')
shared.config.set('bitmessagesettings', 'sockslisten', 'false')
shared.config.set('bitmessagesettings', 'keysencrypted', 'false')
shared.config.set('bitmessagesettings', 'messagesencrypted',
'false')
with open(shared.appdata + 'keys.dat', 'wb') as configfile:
shared.config.write(configfile)
# People running earlier versions of PyBitmessage do not have the
# usedpersonally field in their pubkeys table. Let's add it.
if shared.config.getint('bitmessagesettings', 'settingsversion') == 2:
item = '''ALTER TABLE pubkeys ADD usedpersonally text DEFAULT 'no' '''
parameters = ''
self.cur.execute(item, parameters)
self.conn.commit()
shared.config.set('bitmessagesettings', 'settingsversion', '3')
with open(shared.appdata + 'keys.dat', 'wb') as configfile:
shared.config.write(configfile)
# People running earlier versions of PyBitmessage do not have the
# encodingtype field in their inbox and sent tables or the read field
# in the inbox table. Let's add them.
if shared.config.getint('bitmessagesettings', 'settingsversion') == 3:
item = '''ALTER TABLE inbox ADD encodingtype int DEFAULT '2' '''
parameters = ''
self.cur.execute(item, parameters)
item = '''ALTER TABLE inbox ADD read bool DEFAULT '1' '''
parameters = ''
self.cur.execute(item, parameters)
item = '''ALTER TABLE sent ADD encodingtype int DEFAULT '2' '''
parameters = ''
self.cur.execute(item, parameters)
self.conn.commit()
shared.config.set('bitmessagesettings', 'settingsversion', '4')
with open(shared.appdata + 'keys.dat', 'wb') as configfile:
shared.config.write(configfile)
if shared.config.getint('bitmessagesettings', 'settingsversion') == 4:
shared.config.set(
'bitmessagesettings', 'defaultnoncetrialsperbyte',
str(shared.networkDefaultProofOfWorkNonceTrialsPerByte))
shared.config.set(
'bitmessagesettings', 'defaultpayloadlengthextrabytes',
str(shared.networkDefaultPayloadLengthExtraBytes))
shared.config.set('bitmessagesettings', 'settingsversion', '5')
if shared.config.getint('bitmessagesettings', 'settingsversion') == 5:
shared.config.set('bitmessagesettings',
'maxacceptablenoncetrialsperbyte', '0')
shared.config.set('bitmessagesettings',
'maxacceptablepayloadlengthextrabytes', '0')
shared.config.set('bitmessagesettings', 'settingsversion', '6')
with open(shared.appdata + 'keys.dat', 'wb') as configfile:
shared.config.write(configfile)
# From now on, let us keep a 'version' embedded in the messages.dat
# file so that when we make changes to the database, the database
# version we are on can stay embedded in the messages.dat file. Let us
# check to see if the settings table exists yet.
item = '''SELECT name FROM sqlite_master WHERE type='table' AND name='settings';'''
parameters = ''
self.cur.execute(item, parameters)
if self.cur.fetchall() == []:
# The settings table doesn't exist. We need to make it.
logger.debug(
'In messages.dat database, creating new \'settings\' table.')
self.cur.execute(
'''CREATE TABLE settings (key text, value blob, UNIQUE(key) ON CONFLICT REPLACE)'''
)
self.cur.execute('''INSERT INTO settings VALUES('version','1')''')
self.cur.execute(
'''INSERT INTO settings VALUES('lastvacuumtime',?)''',
(int(time.time()), ))
logger.debug(
'In messages.dat database, removing an obsolete field from the pubkeys table.'
)
self.cur.execute(
'''CREATE TEMPORARY TABLE pubkeys_backup(hash blob, transmitdata blob, time int, usedpersonally text, UNIQUE(hash) ON CONFLICT REPLACE);'''
)
self.cur.execute(
'''INSERT INTO pubkeys_backup SELECT hash, transmitdata, time, usedpersonally FROM pubkeys;'''
)
self.cur.execute('''DROP TABLE pubkeys''')
self.cur.execute(
'''CREATE TABLE pubkeys (hash blob, transmitdata blob, time int, usedpersonally text, UNIQUE(hash) ON CONFLICT REPLACE)'''
)
self.cur.execute(
'''INSERT INTO pubkeys SELECT hash, transmitdata, time, usedpersonally FROM pubkeys_backup;'''
)
self.cur.execute('''DROP TABLE pubkeys_backup;''')
logger.debug(
'Deleting all pubkeys from inventory. They will be redownloaded and then saved with the correct times.'
)
self.cur.execute(
'''delete from inventory where objecttype = 'pubkey';''')
logger.debug(
'replacing Bitmessage announcements mailing list with a new one.'
)
self.cur.execute(
'''delete from subscriptions where address='BM-BbkPSZbzPwpVcYZpU4yHwf9ZPEapN5Zx' '''
)
self.cur.execute(
'''INSERT INTO subscriptions VALUES('Bitmessage new releases/announcements','BM-GtovgYdgs7qXPkoYaRgrLFuFKz1SFpsw',1)'''
)
logger.debug('Commiting.')
self.conn.commit()
logger.debug(
'Vacuuming message.dat. You might notice that the file size gets much smaller.'
)
self.cur.execute(''' VACUUM ''')
# After code refactoring, the possible status values for sent messages
# have changed.
self.cur.execute(
'''update sent set status='doingmsgpow' where status='doingpow' '''
)
self.cur.execute(
'''update sent set status='msgsent' where status='sentmessage' '''
)
self.cur.execute(
'''update sent set status='doingpubkeypow' where status='findingpubkey' '''
)
self.cur.execute(
'''update sent set status='broadcastqueued' where status='broadcastpending' '''
)
self.conn.commit()
if not shared.config.has_option('bitmessagesettings', 'sockslisten'):
shared.config.set('bitmessagesettings', 'sockslisten', 'false')
ensureNamecoinOptions()
# Add a new column to the inventory table to store the first 20 bytes of encrypted messages to support Android app
item = '''SELECT value FROM settings WHERE key='version';'''
parameters = ''
self.cur.execute(item, parameters)
if int(self.cur.fetchall()[0][0]) == 1:
print 'upgrading database'
item = '''ALTER TABLE inventory ADD first20bytesofencryptedmessage blob DEFAULT '' '''
parameters = ''
self.cur.execute(item, parameters)
item = '''update settings set value=? WHERE key='version';'''
parameters = (2, )
self.cur.execute(item, parameters)
if not shared.config.has_option('bitmessagesettings', 'userlocale'):
shared.config.set('bitmessagesettings', 'userlocale', 'system')
if not shared.config.has_option('bitmessagesettings',
'sendoutgoingconnections'):
shared.config.set('bitmessagesettings', 'sendoutgoingconnections',
'True')
# Are you hoping to add a new option to the keys.dat file of existing
# Bitmessage users? Add it right above this line!
try:
testpayload = '\x00\x00'
t = ('1234', testpayload, '12345678', 'no')
self.cur.execute('''INSERT INTO pubkeys VALUES(?,?,?,?)''', t)
self.conn.commit()
self.cur.execute(
'''SELECT transmitdata FROM pubkeys WHERE hash='1234' ''')
queryreturn = self.cur.fetchall()
for row in queryreturn:
transmitdata, = row
self.cur.execute('''DELETE FROM pubkeys WHERE hash='1234' ''')
self.conn.commit()
if transmitdata == '':
logger.fatal(
'Problem: The version of SQLite you have cannot store Null values. Please download and install the latest revision of your version of Python (for example, the latest Python 2.7 revision) and try again.\n'
)
logger.fatal(
'PyBitmessage will now exit very abruptly. You may now see threading errors related to this abrupt exit but the problem you need to solve is related to SQLite.\n\n'
)
os._exit(0)
except Exception as err:
logger.error(err)
# Let us check to see the last time we vaccumed the messages.dat file.
# If it has been more than a month let's do it now.
item = '''SELECT value FROM settings WHERE key='lastvacuumtime';'''
parameters = ''
self.cur.execute(item, parameters)
queryreturn = self.cur.fetchall()
for row in queryreturn:
value, = row
if int(value) < int(time.time()) - 2592000:
logger.info(
'It has been a long time since the messages.dat file has been vacuumed. Vacuuming now...'
)
self.cur.execute(''' VACUUM ''')
item = '''update settings set value=? WHERE key='lastvacuumtime';'''
parameters = (int(time.time()), )
self.cur.execute(item, parameters)
while True:
item = shared.sqlSubmitQueue.get()
if item == 'commit':
self.conn.commit()
elif item == 'exit':
self.conn.close()
logger.info('sqlThread exiting gracefully.')
return
elif item == 'movemessagstoprog':
logger.debug(
'the sqlThread is moving the messages.dat file to the local program directory.'
)
self.conn.commit()
self.conn.close()
shutil.move(shared.lookupAppdataFolder() + 'messages.dat',
'messages.dat')
self.conn = sqlite3.connect('messages.dat')
self.conn.text_factory = str
self.cur = self.conn.cursor()
elif item == 'movemessagstoappdata':
logger.debug(
'the sqlThread is moving the messages.dat file to the Appdata folder.'
)
self.conn.commit()
self.conn.close()
shutil.move('messages.dat',
shared.lookupAppdataFolder() + 'messages.dat')
self.conn = sqlite3.connect(shared.appdata + 'messages.dat')
self.conn.text_factory = str
self.cur = self.conn.cursor()
elif item == 'deleteandvacuume':
self.cur.execute('''delete from inbox where folder='trash' ''')
self.cur.execute('''delete from sent where folder='trash' ''')
self.conn.commit()
self.cur.execute(''' VACUUM ''')
else:
parameters = shared.sqlSubmitQueue.get()
# print 'item', item
# print 'parameters', parameters
try:
self.cur.execute(item, parameters)
except Exception as err:
logger.fatal(
'Major error occurred when trying to execute a SQL statement within the sqlThread. Please tell Atheros about this error message or post it in the forum! Error occurred while trying to execute statement: "%s" Here are the parameters; you might want to censor this data with asterisks (***) as it can contain private information: %s. Here is the actual error message thrown by the sqlThread: %s',
str(item), str(repr(parameters)), str(err))
logger.fatal('This program shall now abruptly exit!')
os._exit(0)
shared.sqlReturnQueue.put(self.cur.fetchall())
def run(self):
gc.disable()
timeWeLastClearedInventoryAndPubkeysTables = 0
try:
shared.maximumLengthOfTimeToBotherResendingMessages = (
float(BMConfigParser().get('lmessagesettings',
'stopresendingafterxdays')) * 24 *
60 * 60) + (float(BMConfigParser().get(
'lmessagesettings', 'stopresendingafterxmonths')) *
(60 * 60 * 24 * 365) / 12)
except:
# Either the user hasn't set stopresendingafterxdays and
# stopresendingafterxmonths yet or the options are missing
# from the config file.
shared.maximumLengthOfTimeToBotherResendingMessages = float('inf')
# initial wait
if state.shutdown == 0:
self.stop.wait(singleCleaner.cycleLength)
while state.shutdown == 0:
queues.UISignalQueue.put(
('updateStatusBar',
'Doing housekeeping (Flushing inventory in memory to disk...)'
))
Inventory().flush()
queues.UISignalQueue.put(('updateStatusBar', ''))
# If we are running as a daemon then we are going to fill up the UI
# queue which will never be handled by a UI. We should clear it to
# save memory.
# FIXME redundant?
if shared.thisapp.daemon or not state.enableGUI:
queues.UISignalQueue.queue.clear()
if timeWeLastClearedInventoryAndPubkeysTables < \
int(time.time()) - 7380:
timeWeLastClearedInventoryAndPubkeysTables = int(time.time())
Inventory().clean()
# pubkeys
sqlExecute(
"DELETE FROM pubkeys WHERE time<? AND usedpersonally='no'",
int(time.time()) - shared.lengthOfTimeToHoldOnToAllPubkeys)
# Let us resend getpubkey objects if we have not yet heard
# a pubkey, and also msg objects if we have not yet heard
# an acknowledgement
queryreturn = sqlQuery(
"SELECT toaddress, ackdata, status FROM sent"
" WHERE ((status='awaitingpubkey' OR status='msgsent')"
" AND folder='sent' AND sleeptill<? AND senttime>?)",
int(time.time()),
int(time.time()) -
shared.maximumLengthOfTimeToBotherResendingMessages)
for row in queryreturn:
if len(row) < 2:
logger.error(
'Something went wrong in the singleCleaner thread:'
' a query did not return the requested fields. %r',
row)
self.stop.wait(3)
break
toAddress, ackData, status = row
if status == 'awaitingpubkey':
resendPubkeyRequest(toAddress)
elif status == 'msgsent':
resendMsg(ackData)
# cleanup old nodes
now = int(time.time())
with knownnodes.knownNodesLock:
for stream in knownnodes.knownNodes:
keys = knownnodes.knownNodes[stream].keys()
for node in keys:
try:
# scrap old nodes
if now - knownnodes.knownNodes[stream][node][
"lastseen"] > 2419200: # 28 days
shared.needToWriteKnownNodesToDisk = True
del knownnodes.knownNodes[stream][node]
continue
# scrap old nodes with low rating
if now - knownnodes.knownNodes[stream][node][
"lastseen"] > 10800 and knownnodes.knownNodes[
stream][node][
"rating"] <= knownnodes.knownNodesForgetRating:
shared.needToWriteKnownNodesToDisk = True
del knownnodes.knownNodes[stream][node]
continue
except TypeError:
print "Error in %s" % node
keys = []
# Let us write out the knowNodes to disk
# if there is anything new to write out.
if shared.needToWriteKnownNodesToDisk:
try:
knownnodes.saveKnownNodes()
except Exception as err:
if "Errno 28" in str(err):
logger.fatal('(while receiveDataThread'
' knownnodes.needToWriteKnownNodesToDisk)'
' Alert: Your disk or data storage volume'
' is full. ')
queues.UISignalQueue.put(
('alert',
(tr._translate("MainWindow", "Disk full"),
tr._translate(
"MainWindow",
'Alert: Your disk or data storage volume'
' is full. LMessage will now exit.'), True)))
# FIXME redundant?
if shared.daemon or not state.enableGUI:
os._exit(0)
shared.needToWriteKnownNodesToDisk = False
# # clear download queues
# for thread in threading.enumerate():
# if thread.isAlive() and hasattr(thread, 'downloadQueue'):
# thread.downloadQueue.clear()
# inv/object tracking
for connection in \
BMConnectionPool().inboundConnections.values() + \
BMConnectionPool().outboundConnections.values():
connection.clean()
# discovery tracking
exp = time.time() - singleCleaner.expireDiscoveredPeers
reaper = (k for k, v in state.discoveredPeers.items() if v < exp)
for k in reaper:
try:
del state.discoveredPeers[k]
except KeyError:
pass
# TODO: cleanup pending upload / download
gc.collect()
if state.shutdown == 0:
self.stop.wait(singleCleaner.cycleLength)
def run(self):
timeWeLastClearedInventoryAndPubkeysTables = 0
try:
shared.maximumLengthOfTimeToBotherResendingMessages = (float(BMConfigParser().get('bitmessagesettings', 'stopresendingafterxdays')) * 24 * 60 * 60) + (float(BMConfigParser().get('bitmessagesettings', 'stopresendingafterxmonths')) * (60 * 60 * 24 *365)/12)
except:
# Either the user hasn't set stopresendingafterxdays and stopresendingafterxmonths yet or the options are missing from the config file.
shared.maximumLengthOfTimeToBotherResendingMessages = float('inf')
# initial wait
if state.shutdown == 0:
self.stop.wait(singleCleaner.cycleLength)
while state.shutdown == 0:
queues.UISignalQueue.put((
'updateStatusBar', 'Doing housekeeping (Flushing inventory in memory to disk...)'))
Inventory().flush()
queues.UISignalQueue.put(('updateStatusBar', ''))
protocol.broadcastToSendDataQueues((
0, 'pong', 'no data')) # commands the sendData threads to send out a pong message if they haven't sent anything else in the last five minutes. The socket timeout-time is 10 minutes.
# If we are running as a daemon then we are going to fill up the UI
# queue which will never be handled by a UI. We should clear it to
# save memory.
if BMConfigParser().safeGetBoolean('bitmessagesettings', 'daemon'):
queues.UISignalQueue.queue.clear()
if timeWeLastClearedInventoryAndPubkeysTables < int(time.time()) - 7380:
timeWeLastClearedInventoryAndPubkeysTables = int(time.time())
Inventory().clean()
# pubkeys
sqlExecute(
'''DELETE FROM pubkeys WHERE time<? AND usedpersonally='no' ''',
int(time.time()) - shared.lengthOfTimeToHoldOnToAllPubkeys)
# Let us resend getpubkey objects if we have not yet heard a pubkey, and also msg objects if we have not yet heard an acknowledgement
queryreturn = sqlQuery(
'''select toaddress, ackdata, status FROM sent WHERE ((status='awaitingpubkey' OR status='msgsent') AND folder='sent' AND sleeptill<? AND senttime>?) ''',
int(time.time()),
int(time.time()) - shared.maximumLengthOfTimeToBotherResendingMessages)
for row in queryreturn:
if len(row) < 2:
logger.error('Something went wrong in the singleCleaner thread: a query did not return the requested fields. ' + repr(row))
self.stop.wait(3)
break
toAddress, ackData, status = row
if status == 'awaitingpubkey':
resendPubkeyRequest(toAddress)
elif status == 'msgsent':
resendMsg(ackData)
# cleanup old nodes
now = int(time.time())
toDelete = []
with knownnodes.knownNodesLock:
for stream in knownnodes.knownNodes:
for node in knownnodes.knownNodes[stream].keys():
try:
if now - knownnodes.knownNodes[stream][node]["lastseen"] > 2419200: # 28 days
shared.needToWriteKownNodesToDisk = True
del knownnodes.knownNodes[stream][node]
except TypeError:
print "Error in %s" % (str(node))
# Let us write out the knowNodes to disk if there is anything new to write out.
if shared.needToWriteKnownNodesToDisk:
try:
knownnodes.saveKnownNodes()
except Exception as err:
if "Errno 28" in str(err):
logger.fatal('(while receiveDataThread knownnodes.needToWriteKnownNodesToDisk) Alert: Your disk or data storage volume is full. ')
queues.UISignalQueue.put(('alert', (tr._translate("MainWindow", "Disk full"), tr._translate("MainWindow", 'Alert: Your disk or data storage volume is full. Bitmessage will now exit.'), True)))
if shared.daemon:
os._exit(0)
shared.needToWriteKnownNodesToDisk = False
# clear download queues
for thread in threading.enumerate():
if thread.isAlive() and hasattr(thread, 'downloadQueue'):
thread.downloadQueue.clear()
# inv/object tracking
for connection in BMConnectionPool().inboundConnections.values() + BMConnectionPool().outboundConnections.values():
connection.clean()
# TODO: cleanup pending upload / download
if state.shutdown == 0:
self.stop.wait(singleCleaner.cycleLength)
def run(self):
self.conn = sqlite3.connect(shared.appdata + 'messages.dat')
self.conn.text_factory = str
self.cur = self.conn.cursor()
self.cur.execute('PRAGMA secure_delete = true')
try:
self.cur.execute(
'''CREATE TABLE inbox (msgid blob, toaddress text, fromaddress text, subject text, received text, message text, folder text, encodingtype int, read bool, sighash blob, UNIQUE(msgid) ON CONFLICT REPLACE)''' )
self.cur.execute(
'''CREATE TABLE sent (msgid blob, toaddress text, toripe blob, fromaddress text, subject text, message text, ackdata blob, senttime integer, lastactiontime integer, sleeptill integer, status text, retrynumber integer, folder text, encodingtype int, ttl int)''' )
self.cur.execute(
'''CREATE TABLE subscriptions (label text, address text, enabled bool)''' )
self.cur.execute(
'''CREATE TABLE addressbook (label text, address text)''' )
self.cur.execute(
'''CREATE TABLE blacklist (label text, address text, enabled bool)''' )
self.cur.execute(
'''CREATE TABLE whitelist (label text, address text, enabled bool)''' )
self.cur.execute(
'''CREATE TABLE pubkeys (address text, addressversion int, transmitdata blob, time int, usedpersonally text, UNIQUE(address) ON CONFLICT REPLACE)''' )
self.cur.execute(
'''CREATE TABLE inventory (hash blob, objecttype int, streamnumber int, payload blob, expirestime integer, tag blob, UNIQUE(hash) ON CONFLICT REPLACE)''' )
self.cur.execute(
'''INSERT INTO subscriptions VALUES('Bitmessage new releases/announcements','BM-GtovgYdgs7qXPkoYaRgrLFuFKz1SFpsw',1)''')
self.cur.execute(
'''CREATE TABLE settings (key blob, value blob, UNIQUE(key) ON CONFLICT REPLACE)''' )
self.cur.execute( '''INSERT INTO settings VALUES('version','10')''')
self.cur.execute( '''INSERT INTO settings VALUES('lastvacuumtime',?)''', (
int(time.time()),))
self.cur.execute(
'''CREATE TABLE objectprocessorqueue (objecttype int, data blob, UNIQUE(objecttype, data) ON CONFLICT REPLACE)''' )
self.conn.commit()
logger.info('Created messages database file')
except Exception as err:
if str(err) == 'table inbox already exists':
logger.debug('Database file already exists.')
else:
sys.stderr.write(
'ERROR trying to create database file (message.dat). Error message: %s\n' % str(err))
os._exit(0)
if shared.config.getint('bitmessagesettings', 'settingsversion') == 1:
shared.config.set('bitmessagesettings', 'settingsversion', '2')
# If the settings version is equal to 2 or 3 then the
# sqlThread will modify the pubkeys table and change
# the settings version to 4.
shared.config.set('bitmessagesettings', 'socksproxytype', 'none')
shared.config.set('bitmessagesettings', 'sockshostname', 'localhost')
shared.config.set('bitmessagesettings', 'socksport', '9050')
shared.config.set('bitmessagesettings', 'socksauthentication', 'false')
shared.config.set('bitmessagesettings', 'socksusername', '')
shared.config.set('bitmessagesettings', 'sockspassword', '')
shared.config.set('bitmessagesettings', 'sockslisten', 'false')
shared.config.set('bitmessagesettings', 'keysencrypted', 'false')
shared.config.set('bitmessagesettings', 'messagesencrypted', 'false')
# People running earlier versions of PyBitmessage do not have the
# usedpersonally field in their pubkeys table. Let's add it.
if shared.config.getint('bitmessagesettings', 'settingsversion') == 2:
item = '''ALTER TABLE pubkeys ADD usedpersonally text DEFAULT 'no' '''
parameters = ''
self.cur.execute(item, parameters)
self.conn.commit()
shared.config.set('bitmessagesettings', 'settingsversion', '3')
# People running earlier versions of PyBitmessage do not have the
# encodingtype field in their inbox and sent tables or the read field
# in the inbox table. Let's add them.
if shared.config.getint('bitmessagesettings', 'settingsversion') == 3:
item = '''ALTER TABLE inbox ADD encodingtype int DEFAULT '2' '''
parameters = ''
self.cur.execute(item, parameters)
item = '''ALTER TABLE inbox ADD read bool DEFAULT '1' '''
parameters = ''
self.cur.execute(item, parameters)
item = '''ALTER TABLE sent ADD encodingtype int DEFAULT '2' '''
parameters = ''
self.cur.execute(item, parameters)
self.conn.commit()
shared.config.set('bitmessagesettings', 'settingsversion', '4')
if shared.config.getint('bitmessagesettings', 'settingsversion') == 4:
shared.config.set('bitmessagesettings', 'defaultnoncetrialsperbyte', str(
shared.networkDefaultProofOfWorkNonceTrialsPerByte))
shared.config.set('bitmessagesettings', 'defaultpayloadlengthextrabytes', str(
shared.networkDefaultPayloadLengthExtraBytes))
shared.config.set('bitmessagesettings', 'settingsversion', '5')
if shared.config.getint('bitmessagesettings', 'settingsversion') == 5:
shared.config.set(
'bitmessagesettings', 'maxacceptablenoncetrialsperbyte', '0')
shared.config.set(
'bitmessagesettings', 'maxacceptablepayloadlengthextrabytes', '0')
shared.config.set('bitmessagesettings', 'settingsversion', '6')
# From now on, let us keep a 'version' embedded in the messages.dat
# file so that when we make changes to the database, the database
# version we are on can stay embedded in the messages.dat file. Let us
# check to see if the settings table exists yet.
item = '''SELECT name FROM sqlite_master WHERE type='table' AND name='settings';'''
parameters = ''
self.cur.execute(item, parameters)
if self.cur.fetchall() == []:
# The settings table doesn't exist. We need to make it.
logger.debug('In messages.dat database, creating new \'settings\' table.')
self.cur.execute(
'''CREATE TABLE settings (key text, value blob, UNIQUE(key) ON CONFLICT REPLACE)''' )
self.cur.execute( '''INSERT INTO settings VALUES('version','1')''')
self.cur.execute( '''INSERT INTO settings VALUES('lastvacuumtime',?)''', (
int(time.time()),))
logger.debug('In messages.dat database, removing an obsolete field from the pubkeys table.')
self.cur.execute(
'''CREATE TEMPORARY TABLE pubkeys_backup(hash blob, transmitdata blob, time int, usedpersonally text, UNIQUE(hash) ON CONFLICT REPLACE);''')
self.cur.execute(
'''INSERT INTO pubkeys_backup SELECT hash, transmitdata, time, usedpersonally FROM pubkeys;''')
self.cur.execute( '''DROP TABLE pubkeys''')
self.cur.execute(
'''CREATE TABLE pubkeys (hash blob, transmitdata blob, time int, usedpersonally text, UNIQUE(hash) ON CONFLICT REPLACE)''' )
self.cur.execute(
'''INSERT INTO pubkeys SELECT hash, transmitdata, time, usedpersonally FROM pubkeys_backup;''')
self.cur.execute( '''DROP TABLE pubkeys_backup;''')
logger.debug('Deleting all pubkeys from inventory. They will be redownloaded and then saved with the correct times.')
self.cur.execute(
'''delete from inventory where objecttype = 'pubkey';''')
logger.debug('replacing Bitmessage announcements mailing list with a new one.')
self.cur.execute(
'''delete from subscriptions where address='BM-BbkPSZbzPwpVcYZpU4yHwf9ZPEapN5Zx' ''')
self.cur.execute(
'''INSERT INTO subscriptions VALUES('Bitmessage new releases/announcements','BM-GtovgYdgs7qXPkoYaRgrLFuFKz1SFpsw',1)''')
logger.debug('Commiting.')
self.conn.commit()
logger.debug('Vacuuming message.dat. You might notice that the file size gets much smaller.')
self.cur.execute( ''' VACUUM ''')
# After code refactoring, the possible status values for sent messages
# have changed.
self.cur.execute(
'''update sent set status='doingmsgpow' where status='doingpow' ''')
self.cur.execute(
'''update sent set status='msgsent' where status='sentmessage' ''')
self.cur.execute(
'''update sent set status='doingpubkeypow' where status='findingpubkey' ''')
self.cur.execute(
'''update sent set status='broadcastqueued' where status='broadcastpending' ''')
self.conn.commit()
if not shared.config.has_option('bitmessagesettings', 'sockslisten'):
shared.config.set('bitmessagesettings', 'sockslisten', 'false')
ensureNamecoinOptions()
"""# Add a new column to the inventory table to store the first 20 bytes of encrypted messages to support Android app
item = '''SELECT value FROM settings WHERE key='version';'''
parameters = ''
self.cur.execute(item, parameters)
if int(self.cur.fetchall()[0][0]) == 1:
print 'upgrading database'
item = '''ALTER TABLE inventory ADD first20bytesofencryptedmessage blob DEFAULT '' '''
parameters = ''
self.cur.execute(item, parameters)
item = '''update settings set value=? WHERE key='version';'''
parameters = (2,)
self.cur.execute(item, parameters)"""
# Let's get rid of the first20bytesofencryptedmessage field in the inventory table.
item = '''SELECT value FROM settings WHERE key='version';'''
parameters = ''
self.cur.execute(item, parameters)
if int(self.cur.fetchall()[0][0]) == 2:
logger.debug('In messages.dat database, removing an obsolete field from the inventory table.')
self.cur.execute(
'''CREATE TEMPORARY TABLE inventory_backup(hash blob, objecttype text, streamnumber int, payload blob, receivedtime integer, UNIQUE(hash) ON CONFLICT REPLACE);''')
self.cur.execute(
'''INSERT INTO inventory_backup SELECT hash, objecttype, streamnumber, payload, receivedtime FROM inventory;''')
self.cur.execute( '''DROP TABLE inventory''')
self.cur.execute(
'''CREATE TABLE inventory (hash blob, objecttype text, streamnumber int, payload blob, receivedtime integer, UNIQUE(hash) ON CONFLICT REPLACE)''' )
self.cur.execute(
'''INSERT INTO inventory SELECT hash, objecttype, streamnumber, payload, receivedtime FROM inventory_backup;''')
self.cur.execute( '''DROP TABLE inventory_backup;''')
item = '''update settings set value=? WHERE key='version';'''
parameters = (3,)
self.cur.execute(item, parameters)
# Add a new column to the inventory table to store tags.
item = '''SELECT value FROM settings WHERE key='version';'''
parameters = ''
self.cur.execute(item, parameters)
currentVersion = int(self.cur.fetchall()[0][0])
if currentVersion == 1 or currentVersion == 3:
logger.debug('In messages.dat database, adding tag field to the inventory table.')
item = '''ALTER TABLE inventory ADD tag blob DEFAULT '' '''
parameters = ''
self.cur.execute(item, parameters)
item = '''update settings set value=? WHERE key='version';'''
parameters = (4,)
self.cur.execute(item, parameters)
if not shared.config.has_option('bitmessagesettings', 'userlocale'):
shared.config.set('bitmessagesettings', 'userlocale', 'system')
if not shared.config.has_option('bitmessagesettings', 'sendoutgoingconnections'):
shared.config.set('bitmessagesettings', 'sendoutgoingconnections', 'True')
# Raise the default required difficulty from 1 to 2
# With the change to protocol v3, this is obsolete.
if shared.config.getint('bitmessagesettings', 'settingsversion') == 6:
"""if int(shared.config.get('bitmessagesettings','defaultnoncetrialsperbyte')) == shared.networkDefaultProofOfWorkNonceTrialsPerByte:
shared.config.set('bitmessagesettings','defaultnoncetrialsperbyte', str(shared.networkDefaultProofOfWorkNonceTrialsPerByte * 2))
"""
shared.config.set('bitmessagesettings', 'settingsversion', '7')
# Add a new column to the pubkeys table to store the address version.
# We're going to trash all of our pubkeys and let them be redownloaded.
item = '''SELECT value FROM settings WHERE key='version';'''
parameters = ''
self.cur.execute(item, parameters)
currentVersion = int(self.cur.fetchall()[0][0])
if currentVersion == 4:
self.cur.execute( '''DROP TABLE pubkeys''')
self.cur.execute(
'''CREATE TABLE pubkeys (hash blob, addressversion int, transmitdata blob, time int, usedpersonally text, UNIQUE(hash, addressversion) ON CONFLICT REPLACE)''' )
self.cur.execute(
'''delete from inventory where objecttype = 'pubkey';''')
item = '''update settings set value=? WHERE key='version';'''
parameters = (5,)
self.cur.execute(item, parameters)
if not shared.config.has_option('bitmessagesettings', 'useidenticons'):
shared.config.set('bitmessagesettings', 'useidenticons', 'True')
if not shared.config.has_option('bitmessagesettings', 'identiconsuffix'): # acts as a salt
shared.config.set('bitmessagesettings', 'identiconsuffix', ''.join(random.choice("123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz") for x in range(12))) # a twelve character pseudo-password to salt the identicons
#Add settings to support no longer resending messages after a certain period of time even if we never get an ack
if shared.config.getint('bitmessagesettings', 'settingsversion') == 7:
shared.config.set(
'bitmessagesettings', 'stopresendingafterxdays', '')
shared.config.set(
'bitmessagesettings', 'stopresendingafterxmonths', '')
shared.config.set('bitmessagesettings', 'settingsversion', '8')
# Add a new table: objectprocessorqueue with which to hold objects
# that have yet to be processed if the user shuts down Bitmessage.
item = '''SELECT value FROM settings WHERE key='version';'''
parameters = ''
self.cur.execute(item, parameters)
currentVersion = int(self.cur.fetchall()[0][0])
if currentVersion == 5:
self.cur.execute( '''DROP TABLE knownnodes''')
self.cur.execute(
'''CREATE TABLE objectprocessorqueue (objecttype text, data blob, UNIQUE(objecttype, data) ON CONFLICT REPLACE)''' )
item = '''update settings set value=? WHERE key='version';'''
parameters = (6,)
self.cur.execute(item, parameters)
# changes related to protocol v3
# In table inventory and objectprocessorqueue, objecttype is now an integer (it was a human-friendly string previously)
item = '''SELECT value FROM settings WHERE key='version';'''
parameters = ''
self.cur.execute(item, parameters)
currentVersion = int(self.cur.fetchall()[0][0])
if currentVersion == 6:
logger.debug('In messages.dat database, dropping and recreating the inventory table.')
self.cur.execute( '''DROP TABLE inventory''')
self.cur.execute( '''CREATE TABLE inventory (hash blob, objecttype int, streamnumber int, payload blob, expirestime integer, tag blob, UNIQUE(hash) ON CONFLICT REPLACE)''' )
self.cur.execute( '''DROP TABLE objectprocessorqueue''')
self.cur.execute( '''CREATE TABLE objectprocessorqueue (objecttype int, data blob, UNIQUE(objecttype, data) ON CONFLICT REPLACE)''' )
item = '''update settings set value=? WHERE key='version';'''
parameters = (7,)
self.cur.execute(item, parameters)
logger.debug('Finished dropping and recreating the inventory table.')
# With the change to protocol version 3, reset the user-settable difficulties to 1
if shared.config.getint('bitmessagesettings', 'settingsversion') == 8:
shared.config.set('bitmessagesettings','defaultnoncetrialsperbyte', str(shared.networkDefaultProofOfWorkNonceTrialsPerByte))
shared.config.set('bitmessagesettings','defaultpayloadlengthextrabytes', str(shared.networkDefaultPayloadLengthExtraBytes))
previousTotalDifficulty = int(shared.config.getint('bitmessagesettings', 'maxacceptablenoncetrialsperbyte')) / 320
previousSmallMessageDifficulty = int(shared.config.getint('bitmessagesettings', 'maxacceptablepayloadlengthextrabytes')) / 14000
shared.config.set('bitmessagesettings','maxacceptablenoncetrialsperbyte', str(previousTotalDifficulty * 1000))
shared.config.set('bitmessagesettings','maxacceptablepayloadlengthextrabytes', str(previousSmallMessageDifficulty * 1000))
shared.config.set('bitmessagesettings', 'settingsversion', '9')
# Adjust the required POW values for each of this user's addresses to conform to protocol v3 norms.
if shared.config.getint('bitmessagesettings', 'settingsversion') == 9:
for addressInKeysFile in shared.config.sections():
try:
previousTotalDifficulty = float(shared.config.getint(addressInKeysFile, 'noncetrialsperbyte')) / 320
previousSmallMessageDifficulty = float(shared.config.getint(addressInKeysFile, 'payloadlengthextrabytes')) / 14000
if previousTotalDifficulty <= 2:
previousTotalDifficulty = 1
if previousSmallMessageDifficulty < 1:
previousSmallMessageDifficulty = 1
shared.config.set(addressInKeysFile,'noncetrialsperbyte', str(int(previousTotalDifficulty * 1000)))
shared.config.set(addressInKeysFile,'payloadlengthextrabytes', str(int(previousSmallMessageDifficulty * 1000)))
except:
continue
shared.config.set('bitmessagesettings', 'maxdownloadrate', '0')
shared.config.set('bitmessagesettings', 'maxuploadrate', '0')
shared.config.set('bitmessagesettings', 'settingsversion', '10')
shared.writeKeysFile()
# sanity check
if shared.config.getint('bitmessagesettings', 'maxacceptablenoncetrialsperbyte') == 0:
shared.config.set('bitmessagesettings','maxacceptablenoncetrialsperbyte', str(shared.ridiculousDifficulty * shared.networkDefaultProofOfWorkNonceTrialsPerByte))
if shared.config.getint('bitmessagesettings', 'maxacceptablepayloadlengthextrabytes') == 0:
shared.config.set('bitmessagesettings','maxacceptablepayloadlengthextrabytes', str(shared.ridiculousDifficulty * shared.networkDefaultPayloadLengthExtraBytes))
# The format of data stored in the pubkeys table has changed. Let's
# clear it, and the pubkeys from inventory, so that they'll be re-downloaded.
item = '''SELECT value FROM settings WHERE key='version';'''
parameters = ''
self.cur.execute(item, parameters)
currentVersion = int(self.cur.fetchall()[0][0])
if currentVersion == 7:
logger.debug('In messages.dat database, clearing pubkeys table because the data format has been updated.')
self.cur.execute(
'''delete from inventory where objecttype = 1;''')
self.cur.execute(
'''delete from pubkeys;''')
# Any sending messages for which we *thought* that we had the pubkey must
# be rechecked.
self.cur.execute(
'''UPDATE sent SET status='msgqueued' WHERE status='doingmsgpow' or status='badkey';''')
query = '''update settings set value=? WHERE key='version';'''
parameters = (8,)
self.cur.execute(query, parameters)
logger.debug('Finished clearing currently held pubkeys.')
# Add a new column to the inbox table to store the hash of the message signature.
# We'll use this as temporary message UUID in order to detect duplicates.
item = '''SELECT value FROM settings WHERE key='version';'''
parameters = ''
self.cur.execute(item, parameters)
currentVersion = int(self.cur.fetchall()[0][0])
if currentVersion == 8:
logger.debug('In messages.dat database, adding sighash field to the inbox table.')
item = '''ALTER TABLE inbox ADD sighash blob DEFAULT '' '''
parameters = ''
self.cur.execute(item, parameters)
item = '''update settings set value=? WHERE key='version';'''
parameters = (9,)
self.cur.execute(item, parameters)
# TTL is now user-specifiable. Let's add an option to save whatever the user selects.
if not shared.config.has_option('bitmessagesettings', 'ttl'):
shared.config.set('bitmessagesettings', 'ttl', '367200')
# We'll also need a `sleeptill` field and a `ttl` field. Also we can combine
# the pubkeyretrynumber and msgretrynumber into one.
item = '''SELECT value FROM settings WHERE key='version';'''
parameters = ''
self.cur.execute(item, parameters)
currentVersion = int(self.cur.fetchall()[0][0])
if currentVersion == 9:
logger.info('In messages.dat database, making TTL-related changes: combining the pubkeyretrynumber and msgretrynumber fields into the retrynumber field and adding the sleeptill and ttl fields...')
self.cur.execute(
'''CREATE TEMPORARY TABLE sent_backup (msgid blob, toaddress text, toripe blob, fromaddress text, subject text, message text, ackdata blob, lastactiontime integer, status text, retrynumber integer, folder text, encodingtype int)''' )
self.cur.execute(
'''INSERT INTO sent_backup SELECT msgid, toaddress, toripe, fromaddress, subject, message, ackdata, lastactiontime, status, 0, folder, encodingtype FROM sent;''')
self.cur.execute( '''DROP TABLE sent''')
self.cur.execute(
'''CREATE TABLE sent (msgid blob, toaddress text, toripe blob, fromaddress text, subject text, message text, ackdata blob, senttime integer, lastactiontime integer, sleeptill int, status text, retrynumber integer, folder text, encodingtype int, ttl int)''' )
self.cur.execute(
'''INSERT INTO sent SELECT msgid, toaddress, toripe, fromaddress, subject, message, ackdata, lastactiontime, lastactiontime, 0, status, 0, folder, encodingtype, 216000 FROM sent_backup;''')
self.cur.execute( '''DROP TABLE sent_backup''')
logger.info('In messages.dat database, finished making TTL-related changes.')
logger.debug('In messages.dat database, adding address field to the pubkeys table.')
# We're going to have to calculate the address for each row in the pubkeys
# table. Then we can take out the hash field.
self.cur.execute('''ALTER TABLE pubkeys ADD address text DEFAULT '' ''')
self.cur.execute('''SELECT hash, addressversion FROM pubkeys''')
queryResult = self.cur.fetchall()
from addresses import encodeAddress
for row in queryResult:
hash, addressVersion = row
address = encodeAddress(addressVersion, 1, hash)
item = '''UPDATE pubkeys SET address=? WHERE hash=?;'''
parameters = (address, hash)
self.cur.execute(item, parameters)
# Now we can remove the hash field from the pubkeys table.
self.cur.execute(
'''CREATE TEMPORARY TABLE pubkeys_backup (address text, addressversion int, transmitdata blob, time int, usedpersonally text, UNIQUE(address) ON CONFLICT REPLACE)''' )
self.cur.execute(
'''INSERT INTO pubkeys_backup SELECT address, addressversion, transmitdata, time, usedpersonally FROM pubkeys;''')
self.cur.execute( '''DROP TABLE pubkeys''')
self.cur.execute(
'''CREATE TABLE pubkeys (address text, addressversion int, transmitdata blob, time int, usedpersonally text, UNIQUE(address) ON CONFLICT REPLACE)''' )
self.cur.execute(
'''INSERT INTO pubkeys SELECT address, addressversion, transmitdata, time, usedpersonally FROM pubkeys_backup;''')
self.cur.execute( '''DROP TABLE pubkeys_backup''')
logger.debug('In messages.dat database, done adding address field to the pubkeys table and removing the hash field.')
self.cur.execute('''update settings set value=10 WHERE key='version';''')
# Are you hoping to add a new option to the keys.dat file of existing
# Bitmessage users or modify the SQLite database? Add it right above this line!
try:
testpayload = '\x00\x00'
t = ('1234', 1, testpayload, '12345678', 'no')
self.cur.execute( '''INSERT INTO pubkeys VALUES(?,?,?,?,?)''', t)
self.conn.commit()
self.cur.execute(
'''SELECT transmitdata FROM pubkeys WHERE address='1234' ''')
queryreturn = self.cur.fetchall()
for row in queryreturn:
transmitdata, = row
self.cur.execute('''DELETE FROM pubkeys WHERE address='1234' ''')
self.conn.commit()
if transmitdata == '':
logger.fatal('Problem: The version of SQLite you have cannot store Null values. Please download and install the latest revision of your version of Python (for example, the latest Python 2.7 revision) and try again.\n')
logger.fatal('PyBitmessage will now exit very abruptly. You may now see threading errors related to this abrupt exit but the problem you need to solve is related to SQLite.\n\n')
os._exit(0)
except Exception as err:
if str(err) == 'database or disk is full':
logger.fatal('(While null value test) Alert: Your disk or data storage volume is full. sqlThread will now exit.')
shared.UISignalQueue.put(('alert', (tr._translate("MainWindow", "Disk full"), tr._translate("MainWindow", 'Alert: Your disk or data storage volume is full. Bitmessage will now exit.'), True)))
if shared.daemon:
os._exit(0)
else:
return
else:
logger.error(err)
# Let us check to see the last time we vaccumed the messages.dat file.
# If it has been more than a month let's do it now.
item = '''SELECT value FROM settings WHERE key='lastvacuumtime';'''
parameters = ''
self.cur.execute(item, parameters)
queryreturn = self.cur.fetchall()
for row in queryreturn:
value, = row
if int(value) < int(time.time()) - 86400:
logger.info('It has been a long time since the messages.dat file has been vacuumed. Vacuuming now...')
try:
self.cur.execute( ''' VACUUM ''')
except Exception as err:
if str(err) == 'database or disk is full':
logger.fatal('(While VACUUM) Alert: Your disk or data storage volume is full. sqlThread will now exit.')
shared.UISignalQueue.put(('alert', (tr._translate("MainWindow", "Disk full"), tr._translate("MainWindow", 'Alert: Your disk or data storage volume is full. Bitmessage will now exit.'), True)))
if shared.daemon:
os._exit(0)
else:
return
item = '''update settings set value=? WHERE key='lastvacuumtime';'''
parameters = (int(time.time()),)
self.cur.execute(item, parameters)
while True:
item = shared.sqlSubmitQueue.get()
if item == 'commit':
try:
self.conn.commit()
except Exception as err:
if str(err) == 'database or disk is full':
logger.fatal('(While committing) Alert: Your disk or data storage volume is full. sqlThread will now exit.')
shared.UISignalQueue.put(('alert', (tr._translate("MainWindow", "Disk full"), tr._translate("MainWindow", 'Alert: Your disk or data storage volume is full. Bitmessage will now exit.'), True)))
if shared.daemon:
os._exit(0)
else:
return
elif item == 'exit':
self.conn.close()
logger.info('sqlThread exiting gracefully.')
return
elif item == 'movemessagstoprog':
logger.debug('the sqlThread is moving the messages.dat file to the local program directory.')
try:
self.conn.commit()
except Exception as err:
if str(err) == 'database or disk is full':
logger.fatal('(while movemessagstoprog) Alert: Your disk or data storage volume is full. sqlThread will now exit.')
shared.UISignalQueue.put(('alert', (tr._translate("MainWindow", "Disk full"), tr._translate("MainWindow", 'Alert: Your disk or data storage volume is full. Bitmessage will now exit.'), True)))
if shared.daemon:
os._exit(0)
else:
return
self.conn.close()
shutil.move(
shared.lookupAppdataFolder() + 'messages.dat', shared.lookupExeFolder() + 'messages.dat')
self.conn = sqlite3.connect(shared.lookupExeFolder() + 'messages.dat')
self.conn.text_factory = str
self.cur = self.conn.cursor()
elif item == 'movemessagstoappdata':
logger.debug('the sqlThread is moving the messages.dat file to the Appdata folder.')
try:
self.conn.commit()
except Exception as err:
if str(err) == 'database or disk is full':
logger.fatal('(while movemessagstoappdata) Alert: Your disk or data storage volume is full. sqlThread will now exit.')
shared.UISignalQueue.put(('alert', (tr._translate("MainWindow", "Disk full"), tr._translate("MainWindow", 'Alert: Your disk or data storage volume is full. Bitmessage will now exit.'), True)))
if shared.daemon:
os._exit(0)
else:
return
self.conn.close()
shutil.move(
shared.lookupExeFolder() + 'messages.dat', shared.lookupAppdataFolder() + 'messages.dat')
self.conn = sqlite3.connect(shared.lookupAppdataFolder() + 'messages.dat')
self.conn.text_factory = str
self.cur = self.conn.cursor()
elif item == 'deleteandvacuume':
self.cur.execute('''delete from inbox where folder='trash' ''')
self.cur.execute('''delete from sent where folder='trash' ''')
self.conn.commit()
try:
self.cur.execute( ''' VACUUM ''')
except Exception as err:
if str(err) == 'database or disk is full':
logger.fatal('(while deleteandvacuume) Alert: Your disk or data storage volume is full. sqlThread will now exit.')
shared.UISignalQueue.put(('alert', (tr._translate("MainWindow", "Disk full"), tr._translate("MainWindow", 'Alert: Your disk or data storage volume is full. Bitmessage will now exit.'), True)))
if shared.daemon:
os._exit(0)
else:
return
else:
parameters = shared.sqlSubmitQueue.get()
rowcount = 0
# print 'item', item
# print 'parameters', parameters
try:
self.cur.execute(item, parameters)
rowcount = self.cur.rowcount
except Exception as err:
if str(err) == 'database or disk is full':
logger.fatal('(while cur.execute) Alert: Your disk or data storage volume is full. sqlThread will now exit.')
shared.UISignalQueue.put(('alert', (tr._translate("MainWindow", "Disk full"), tr._translate("MainWindow", 'Alert: Your disk or data storage volume is full. Bitmessage will now exit.'), True)))
if shared.daemon:
os._exit(0)
else:
return
else:
logger.fatal('Major error occurred when trying to execute a SQL statement within the sqlThread. Please tell Atheros about this error message or post it in the forum! Error occurred while trying to execute statement: "%s" Here are the parameters; you might want to censor this data with asterisks (***) as it can contain private information: %s. Here is the actual error message thrown by the sqlThread: %s', str(item), str(repr(parameters)), str(err))
logger.fatal('This program shall now abruptly exit!')
os._exit(0)
shared.sqlReturnQueue.put((self.cur.fetchall(), rowcount))
def run(self):
timeWeLastClearedInventoryAndPubkeysTables = 0
while True:
shared.UISignalQueue.put(
('updateStatusBar',
'Doing housekeeping (Flushing inventory in memory to disk...)'
))
with shared.inventoryLock: # If you use both the inventoryLock and the sqlLock, always use the inventoryLock OUTSIDE of the sqlLock.
with SqlBulkExecute() as sql:
for hash, storedValue in shared.inventory.items():
objectType, streamNumber, payload, receivedTime, tag = storedValue
if int(time.time()) - 3600 > receivedTime:
sql.execute(
'''INSERT INTO inventory VALUES (?,?,?,?,?,?)''',
hash, objectType, streamNumber, payload,
receivedTime, tag)
del shared.inventory[hash]
shared.UISignalQueue.put(('updateStatusBar', ''))
shared.broadcastToSendDataQueues(
(0, 'pong', 'no data')
) # commands the sendData threads to send out a pong message if they haven't sent anything else in the last five minutes. The socket timeout-time is 10 minutes.
# If we are running as a daemon then we are going to fill up the UI
# queue which will never be handled by a UI. We should clear it to
# save memory.
if shared.safeConfigGetBoolean('bitmessagesettings', 'daemon'):
shared.UISignalQueue.queue.clear()
if timeWeLastClearedInventoryAndPubkeysTables < int(
time.time()) - 7380:
timeWeLastClearedInventoryAndPubkeysTables = int(time.time())
# inventory (moves data from the inventory data structure to
# the on-disk sql database)
# inventory (clears pubkeys after 28 days and everything else
# after 2 days and 12 hours)
sqlExecute(
'''DELETE FROM inventory WHERE (receivedtime<? AND objecttype<>'pubkey') OR (receivedtime<? AND objecttype='pubkey') ''',
int(time.time()) -
shared.lengthOfTimeToLeaveObjectsInInventory,
int(time.time()) - shared.lengthOfTimeToHoldOnToAllPubkeys)
# pubkeys
sqlExecute(
'''DELETE FROM pubkeys WHERE time<? AND usedpersonally='no' ''',
int(time.time()) - shared.lengthOfTimeToHoldOnToAllPubkeys)
queryreturn = sqlQuery(
'''select toaddress, toripe, fromaddress, subject, message, ackdata, lastactiontime, status, pubkeyretrynumber, msgretrynumber FROM sent WHERE ((status='awaitingpubkey' OR status='msgsent') AND folder='sent') '''
) # If the message's folder='trash' then we'll ignore it.
for row in queryreturn:
if len(row) < 5:
with shared.printLock:
sys.stderr.write(
'Something went wrong in the singleCleaner thread: a query did not return the requested fields. '
+ repr(row))
time.sleep(3)
break
toaddress, toripe, fromaddress, subject, message, ackdata, lastactiontime, status, pubkeyretrynumber, msgretrynumber = row
if status == 'awaitingpubkey':
if int(time.time()) - lastactiontime > (
shared.
maximumAgeOfAnObjectThatIAmWillingToAccept *
(2**(pubkeyretrynumber))):
print 'It has been a long time and we haven\'t heard a response to our getpubkey request. Sending again.'
try:
del shared.neededPubkeys[
toripe] # We need to take this entry out of the shared.neededPubkeys structure because the shared.workerQueue checks to see whether the entry is already present and will not do the POW and send the message because it assumes that it has already done it recently.
except:
pass
shared.UISignalQueue.put((
'updateStatusBar',
'Doing work necessary to again attempt to request a public key...'
))
t = ()
sqlExecute(
'''UPDATE sent SET lastactiontime=?, pubkeyretrynumber=?, status='msgqueued' WHERE toripe=?''',
int(time.time()), pubkeyretrynumber + 1,
toripe)
shared.workerQueue.put(('sendmessage', ''))
else: # status == msgsent
if int(time.time()) - lastactiontime > (
shared.
maximumAgeOfAnObjectThatIAmWillingToAccept *
(2**(msgretrynumber))):
print 'It has been a long time and we haven\'t heard an acknowledgement to our msg. Sending again.'
sqlExecute(
'''UPDATE sent SET lastactiontime=?, msgretrynumber=?, status=? WHERE ackdata=?''',
int(time.time()), msgretrynumber + 1,
'msgqueued', ackdata)
shared.workerQueue.put(('sendmessage', ''))
shared.UISignalQueue.put((
'updateStatusBar',
'Doing work necessary to again attempt to deliver a message...'
))
# Let's also clear and reload shared.inventorySets to keep it from
# taking up an unnecessary amount of memory.
for streamNumber in shared.inventorySets:
shared.inventorySets[streamNumber] = set()
queryData = sqlQuery(
'''SELECT hash FROM inventory WHERE streamnumber=?''',
streamNumber)
for row in queryData:
shared.inventorySets[streamNumber].add(row[0])
with shared.inventoryLock:
for hash, storedValue in shared.inventory.items():
objectType, streamNumber, payload, receivedTime, tag = storedValue
if streamNumber in shared.inventorySets:
shared.inventorySets[streamNumber].add(hash)
# Let us write out the knowNodes to disk if there is anything new to write out.
if shared.needToWriteKnownNodesToDisk:
shared.knownNodesLock.acquire()
output = open(shared.appdata + 'knownnodes.dat', 'wb')
try:
pickle.dump(shared.knownNodes, output)
output.close()
except Exception as err:
if "Errno 28" in str(err):
logger.fatal(
'(while receiveDataThread shared.needToWriteKnownNodesToDisk) Alert: Your disk or data storage volume is full. '
)
shared.UISignalQueue.put(('alert', (
tr.translateText("MainWindow", "Disk full"),
tr.translateText(
"MainWindow",
'Alert: Your disk or data storage volume is full. Bitmessage will now exit.'
), True)))
if shared.daemon:
os._exit(0)
shared.knownNodesLock.release()
shared.needToWriteKnownNodesToDisk = False
time.sleep(300)
def run(self):
timeWeLastClearedInventoryAndPubkeysTables = 0
try:
shared.maximumLengthOfTimeToBotherResendingMessages = (float(shared.config.get('bitmessagesettings', 'stopresendingafterxdays')) * 24 * 60 * 60) + (float(shared.config.get('bitmessagesettings', 'stopresendingafterxmonths')) * (60 * 60 * 24 *365)/12)
except:
# Either the user hasn't set stopresendingafterxdays and stopresendingafterxmonths yet or the options are missing from the config file.
shared.maximumLengthOfTimeToBotherResendingMessages = float('inf')
while shared.shutdown == 0:
shared.UISignalQueue.put((
'updateStatusBar', 'Doing housekeeping (Flushing inventory in memory to disk...)'))
shared.inventory.flush()
shared.UISignalQueue.put(('updateStatusBar', ''))
shared.broadcastToSendDataQueues((
0, 'pong', 'no data')) # commands the sendData threads to send out a pong message if they haven't sent anything else in the last five minutes. The socket timeout-time is 10 minutes.
# If we are running as a daemon then we are going to fill up the UI
# queue which will never be handled by a UI. We should clear it to
# save memory.
if shared.safeConfigGetBoolean('bitmessagesettings', 'daemon'):
shared.UISignalQueue.queue.clear()
if timeWeLastClearedInventoryAndPubkeysTables < int(time.time()) - 7380:
timeWeLastClearedInventoryAndPubkeysTables = int(time.time())
shared.inventory.clean()
# pubkeys
sqlExecute(
'''DELETE FROM pubkeys WHERE time<? AND usedpersonally='no' ''',
int(time.time()) - shared.lengthOfTimeToHoldOnToAllPubkeys)
# Let us resend getpubkey objects if we have not yet heard a pubkey, and also msg objects if we have not yet heard an acknowledgement
queryreturn = sqlQuery(
'''select toaddress, ackdata, status FROM sent WHERE ((status='awaitingpubkey' OR status='msgsent') AND folder='sent' AND sleeptill<? AND senttime>?) ''',
int(time.time()),
int(time.time()) - shared.maximumLengthOfTimeToBotherResendingMessages)
for row in queryreturn:
if len(row) < 2:
logger.error('Something went wrong in the singleCleaner thread: a query did not return the requested fields. ' + repr(row))
self.stop.wait(3)
break
toAddress, ackData, status = row
if status == 'awaitingpubkey':
resendPubkeyRequest(toAddress)
elif status == 'msgsent':
resendMsg(ackData)
# cleanup old nodes
now = int(time.time())
toDelete = []
shared.knownNodesLock.acquire()
for stream in shared.knownNodes:
for node in shared.knownNodes[stream].keys():
if now - shared.knownNodes[stream][node] > 2419200: # 28 days
shared.needToWriteKownNodesToDisk = True
del shared.knownNodes[stream][node]
shared.knownNodesLock.release()
# Let us write out the knowNodes to disk if there is anything new to write out.
if shared.needToWriteKnownNodesToDisk:
shared.knownNodesLock.acquire()
output = open(shared.appdata + 'knownnodes.dat', 'wb')
try:
pickle.dump(shared.knownNodes, output)
output.close()
except Exception as err:
if "Errno 28" in str(err):
logger.fatal('(while receiveDataThread shared.needToWriteKnownNodesToDisk) Alert: Your disk or data storage volume is full. ')
shared.UISignalQueue.put(('alert', (tr._translate("MainWindow", "Disk full"), tr._translate("MainWindow", 'Alert: Your disk or data storage volume is full. Bitmessage will now exit.'), True)))
if shared.daemon:
os._exit(0)
shared.knownNodesLock.release()
shared.needToWriteKnownNodesToDisk = False
self.stop.wait(300)
def run(self):
self.conn = sqlite3.connect(state.appdata + 'messages.dat')
self.conn.text_factory = str
self.cur = self.conn.cursor()
self.cur.execute('PRAGMA secure_delete = true')
try:
self.cur.execute(
'''CREATE TABLE inbox (msgid blob, toaddress text, fromaddress text, subject text, received text, message text, folder text, encodingtype int, read bool, sighash blob, UNIQUE(msgid) ON CONFLICT REPLACE)'''
)
self.cur.execute(
'''CREATE TABLE sent (msgid blob, toaddress text, toripe blob, fromaddress text, subject text, message text, ackdata blob, senttime integer, lastactiontime integer, sleeptill integer, status text, retrynumber integer, folder text, encodingtype int, ttl int)'''
)
self.cur.execute(
'''CREATE TABLE subscriptions (label text, address text, enabled bool)'''
)
self.cur.execute(
'''CREATE TABLE addressbook (label text, address text)''')
self.cur.execute(
'''CREATE TABLE blacklist (label text, address text, enabled bool)'''
)
self.cur.execute(
'''CREATE TABLE whitelist (label text, address text, enabled bool)'''
)
self.cur.execute(
'''CREATE TABLE pubkeys (address text, addressversion int, transmitdata blob, time int, usedpersonally text, UNIQUE(address) ON CONFLICT REPLACE)'''
)
self.cur.execute(
'''CREATE TABLE inventory (hash blob, objecttype int, streamnumber int, payload blob, expirestime integer, tag blob, UNIQUE(hash) ON CONFLICT REPLACE)'''
)
self.cur.execute(
'''INSERT INTO subscriptions VALUES('Bitmessage new releases/announcements','BM-GtovgYdgs7qXPkoYaRgrLFuFKz1SFpsw',1)'''
)
self.cur.execute(
'''CREATE TABLE settings (key blob, value blob, UNIQUE(key) ON CONFLICT REPLACE)'''
)
self.cur.execute('''INSERT INTO settings VALUES('version','10')''')
self.cur.execute(
'''INSERT INTO settings VALUES('lastvacuumtime',?)''',
(int(time.time()), ))
self.cur.execute(
'''CREATE TABLE objectprocessorqueue (objecttype int, data blob, UNIQUE(objecttype, data) ON CONFLICT REPLACE)'''
)
self.conn.commit()
logger.info('Created messages database file')
except Exception as err:
if str(err) == 'table inbox already exists':
logger.debug('Database file already exists.')
else:
sys.stderr.write(
'ERROR trying to create database file (message.dat). Error message: %s\n'
% str(err))
os._exit(0)
if BMConfigParser().getint('bitmessagesettings',
'settingsversion') == 1:
BMConfigParser().set('bitmessagesettings', 'settingsversion', '2')
# If the settings version is equal to 2 or 3 then the
# sqlThread will modify the pubkeys table and change
# the settings version to 4.
BMConfigParser().set('bitmessagesettings', 'socksproxytype',
'none')
BMConfigParser().set('bitmessagesettings', 'sockshostname',
'localhost')
BMConfigParser().set('bitmessagesettings', 'socksport', '9050')
BMConfigParser().set('bitmessagesettings', 'socksauthentication',
'false')
BMConfigParser().set('bitmessagesettings', 'socksusername', '')
BMConfigParser().set('bitmessagesettings', 'sockspassword', '')
BMConfigParser().set('bitmessagesettings', 'sockslisten', 'false')
BMConfigParser().set('bitmessagesettings', 'keysencrypted',
'false')
BMConfigParser().set('bitmessagesettings', 'messagesencrypted',
'false')
# People running earlier versions of PyBitmessage do not have the
# usedpersonally field in their pubkeys table. Let's add it.
if BMConfigParser().getint('bitmessagesettings',
'settingsversion') == 2:
item = '''ALTER TABLE pubkeys ADD usedpersonally text DEFAULT 'no' '''
parameters = ''
self.cur.execute(item, parameters)
self.conn.commit()
BMConfigParser().set('bitmessagesettings', 'settingsversion', '3')
# People running earlier versions of PyBitmessage do not have the
# encodingtype field in their inbox and sent tables or the read field
# in the inbox table. Let's add them.
if BMConfigParser().getint('bitmessagesettings',
'settingsversion') == 3:
item = '''ALTER TABLE inbox ADD encodingtype int DEFAULT '2' '''
parameters = ''
self.cur.execute(item, parameters)
item = '''ALTER TABLE inbox ADD read bool DEFAULT '1' '''
parameters = ''
self.cur.execute(item, parameters)
item = '''ALTER TABLE sent ADD encodingtype int DEFAULT '2' '''
parameters = ''
self.cur.execute(item, parameters)
self.conn.commit()
BMConfigParser().set('bitmessagesettings', 'settingsversion', '4')
if BMConfigParser().getint('bitmessagesettings',
'settingsversion') == 4:
BMConfigParser().set(
'bitmessagesettings', 'defaultnoncetrialsperbyte',
str(defaults.networkDefaultProofOfWorkNonceTrialsPerByte))
BMConfigParser().set(
'bitmessagesettings', 'defaultpayloadlengthextrabytes',
str(defaults.networkDefaultPayloadLengthExtraBytes))
BMConfigParser().set('bitmessagesettings', 'settingsversion', '5')
if BMConfigParser().getint('bitmessagesettings',
'settingsversion') == 5:
BMConfigParser().set('bitmessagesettings',
'maxacceptablenoncetrialsperbyte', '0')
BMConfigParser().set('bitmessagesettings',
'maxacceptablepayloadlengthextrabytes', '0')
BMConfigParser().set('bitmessagesettings', 'settingsversion', '6')
# From now on, let us keep a 'version' embedded in the messages.dat
# file so that when we make changes to the database, the database
# version we are on can stay embedded in the messages.dat file. Let us
# check to see if the settings table exists yet.
item = '''SELECT name FROM sqlite_master WHERE type='table' AND name='settings';'''
parameters = ''
self.cur.execute(item, parameters)
if self.cur.fetchall() == []:
# The settings table doesn't exist. We need to make it.
logger.debug(
'In messages.dat database, creating new \'settings\' table.')
self.cur.execute(
'''CREATE TABLE settings (key text, value blob, UNIQUE(key) ON CONFLICT REPLACE)'''
)
self.cur.execute('''INSERT INTO settings VALUES('version','1')''')
self.cur.execute(
'''INSERT INTO settings VALUES('lastvacuumtime',?)''',
(int(time.time()), ))
logger.debug(
'In messages.dat database, removing an obsolete field from the pubkeys table.'
)
self.cur.execute(
'''CREATE TEMPORARY TABLE pubkeys_backup(hash blob, transmitdata blob, time int, usedpersonally text, UNIQUE(hash) ON CONFLICT REPLACE);'''
)
self.cur.execute(
'''INSERT INTO pubkeys_backup SELECT hash, transmitdata, time, usedpersonally FROM pubkeys;'''
)
self.cur.execute('''DROP TABLE pubkeys''')
self.cur.execute(
'''CREATE TABLE pubkeys (hash blob, transmitdata blob, time int, usedpersonally text, UNIQUE(hash) ON CONFLICT REPLACE)'''
)
self.cur.execute(
'''INSERT INTO pubkeys SELECT hash, transmitdata, time, usedpersonally FROM pubkeys_backup;'''
)
self.cur.execute('''DROP TABLE pubkeys_backup;''')
logger.debug(
'Deleting all pubkeys from inventory. They will be redownloaded and then saved with the correct times.'
)
self.cur.execute(
'''delete from inventory where objecttype = 'pubkey';''')
logger.debug(
'replacing Bitmessage announcements mailing list with a new one.'
)
self.cur.execute(
'''delete from subscriptions where address='BM-BbkPSZbzPwpVcYZpU4yHwf9ZPEapN5Zx' '''
)
self.cur.execute(
'''INSERT INTO subscriptions VALUES('Bitmessage new releases/announcements','BM-GtovgYdgs7qXPkoYaRgrLFuFKz1SFpsw',1)'''
)
logger.debug('Commiting.')
self.conn.commit()
logger.debug(
'Vacuuming message.dat. You might notice that the file size gets much smaller.'
)
self.cur.execute(''' VACUUM ''')
# After code refactoring, the possible status values for sent messages
# have changed.
self.cur.execute(
'''update sent set status='doingmsgpow' where status='doingpow' '''
)
self.cur.execute(
'''update sent set status='msgsent' where status='sentmessage' '''
)
self.cur.execute(
'''update sent set status='doingpubkeypow' where status='findingpubkey' '''
)
self.cur.execute(
'''update sent set status='broadcastqueued' where status='broadcastpending' '''
)
self.conn.commit()
if not BMConfigParser().has_option('bitmessagesettings',
'sockslisten'):
BMConfigParser().set('bitmessagesettings', 'sockslisten', 'false')
ensureNamecoinOptions()
# Let's get rid of the first20bytesofencryptedmessage field in the inventory table.
item = '''SELECT value FROM settings WHERE key='version';'''
parameters = ''
self.cur.execute(item, parameters)
if int(self.cur.fetchall()[0][0]) == 2:
logger.debug(
'In messages.dat database, removing an obsolete field from the inventory table.'
)
self.cur.execute(
'''CREATE TEMPORARY TABLE inventory_backup(hash blob, objecttype text, streamnumber int, payload blob, receivedtime integer, UNIQUE(hash) ON CONFLICT REPLACE);'''
)
self.cur.execute(
'''INSERT INTO inventory_backup SELECT hash, objecttype, streamnumber, payload, receivedtime FROM inventory;'''
)
self.cur.execute('''DROP TABLE inventory''')
self.cur.execute(
'''CREATE TABLE inventory (hash blob, objecttype text, streamnumber int, payload blob, receivedtime integer, UNIQUE(hash) ON CONFLICT REPLACE)'''
)
self.cur.execute(
'''INSERT INTO inventory SELECT hash, objecttype, streamnumber, payload, receivedtime FROM inventory_backup;'''
)
self.cur.execute('''DROP TABLE inventory_backup;''')
item = '''update settings set value=? WHERE key='version';'''
parameters = (3, )
self.cur.execute(item, parameters)
# Add a new column to the inventory table to store tags.
item = '''SELECT value FROM settings WHERE key='version';'''
parameters = ''
self.cur.execute(item, parameters)
currentVersion = int(self.cur.fetchall()[0][0])
if currentVersion == 1 or currentVersion == 3:
logger.debug(
'In messages.dat database, adding tag field to the inventory table.'
)
item = '''ALTER TABLE inventory ADD tag blob DEFAULT '' '''
parameters = ''
self.cur.execute(item, parameters)
item = '''update settings set value=? WHERE key='version';'''
parameters = (4, )
self.cur.execute(item, parameters)
if not BMConfigParser().has_option('bitmessagesettings', 'userlocale'):
BMConfigParser().set('bitmessagesettings', 'userlocale', 'system')
if not BMConfigParser().has_option('bitmessagesettings',
'sendoutgoingconnections'):
BMConfigParser().set('bitmessagesettings',
'sendoutgoingconnections', 'True')
# Raise the default required difficulty from 1 to 2
# With the change to protocol v3, this is obsolete.
if BMConfigParser().getint('bitmessagesettings',
'settingsversion') == 6:
BMConfigParser().set('bitmessagesettings', 'settingsversion', '7')
# Add a new column to the pubkeys table to store the address version.
# We're going to trash all of our pubkeys and let them be redownloaded.
item = '''SELECT value FROM settings WHERE key='version';'''
parameters = ''
self.cur.execute(item, parameters)
currentVersion = int(self.cur.fetchall()[0][0])
if currentVersion == 4:
self.cur.execute('''DROP TABLE pubkeys''')
self.cur.execute(
'''CREATE TABLE pubkeys (hash blob, addressversion int, transmitdata blob, time int, usedpersonally text, UNIQUE(hash, addressversion) ON CONFLICT REPLACE)'''
)
self.cur.execute(
'''delete from inventory where objecttype = 'pubkey';''')
item = '''update settings set value=? WHERE key='version';'''
parameters = (5, )
self.cur.execute(item, parameters)
if not BMConfigParser().has_option('bitmessagesettings',
'useidenticons'):
BMConfigParser().set('bitmessagesettings', 'useidenticons', 'True')
if not BMConfigParser().has_option(
'bitmessagesettings', 'identiconsuffix'): # acts as a salt
BMConfigParser(
).set('bitmessagesettings', 'identiconsuffix', ''.join(
helper_random.randomchoice(
"123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz"
) for x in range(12)
)) # a twelve character pseudo-password to salt the identicons
#Add settings to support no longer resending messages after a certain period of time even if we never get an ack
if BMConfigParser().getint('bitmessagesettings',
'settingsversion') == 7:
BMConfigParser().set('bitmessagesettings',
'stopresendingafterxdays', '')
BMConfigParser().set('bitmessagesettings',
'stopresendingafterxmonths', '')
BMConfigParser().set('bitmessagesettings', 'settingsversion', '8')
# Add a new table: objectprocessorqueue with which to hold objects
# that have yet to be processed if the user shuts down Bitmessage.
item = '''SELECT value FROM settings WHERE key='version';'''
parameters = ''
self.cur.execute(item, parameters)
currentVersion = int(self.cur.fetchall()[0][0])
if currentVersion == 5:
self.cur.execute('''DROP TABLE knownnodes''')
self.cur.execute(
'''CREATE TABLE objectprocessorqueue (objecttype text, data blob, UNIQUE(objecttype, data) ON CONFLICT REPLACE)'''
)
item = '''update settings set value=? WHERE key='version';'''
parameters = (6, )
self.cur.execute(item, parameters)
# changes related to protocol v3
# In table inventory and objectprocessorqueue, objecttype is now an integer (it was a human-friendly string previously)
item = '''SELECT value FROM settings WHERE key='version';'''
parameters = ''
self.cur.execute(item, parameters)
currentVersion = int(self.cur.fetchall()[0][0])
if currentVersion == 6:
logger.debug(
'In messages.dat database, dropping and recreating the inventory table.'
)
self.cur.execute('''DROP TABLE inventory''')
self.cur.execute(
'''CREATE TABLE inventory (hash blob, objecttype int, streamnumber int, payload blob, expirestime integer, tag blob, UNIQUE(hash) ON CONFLICT REPLACE)'''
)
self.cur.execute('''DROP TABLE objectprocessorqueue''')
self.cur.execute(
'''CREATE TABLE objectprocessorqueue (objecttype int, data blob, UNIQUE(objecttype, data) ON CONFLICT REPLACE)'''
)
item = '''update settings set value=? WHERE key='version';'''
parameters = (7, )
self.cur.execute(item, parameters)
logger.debug(
'Finished dropping and recreating the inventory table.')
# With the change to protocol version 3, reset the user-settable difficulties to 1
if BMConfigParser().getint('bitmessagesettings',
'settingsversion') == 8:
BMConfigParser().set(
'bitmessagesettings', 'defaultnoncetrialsperbyte',
str(defaults.networkDefaultProofOfWorkNonceTrialsPerByte))
BMConfigParser().set(
'bitmessagesettings', 'defaultpayloadlengthextrabytes',
str(defaults.networkDefaultPayloadLengthExtraBytes))
previousTotalDifficulty = int(BMConfigParser().getint(
'bitmessagesettings', 'maxacceptablenoncetrialsperbyte')) / 320
previousSmallMessageDifficulty = int(BMConfigParser().getint(
'bitmessagesettings',
'maxacceptablepayloadlengthextrabytes')) / 14000
BMConfigParser().set('bitmessagesettings',
'maxacceptablenoncetrialsperbyte',
str(previousTotalDifficulty * 1000))
BMConfigParser().set('bitmessagesettings',
'maxacceptablepayloadlengthextrabytes',
str(previousSmallMessageDifficulty * 1000))
BMConfigParser().set('bitmessagesettings', 'settingsversion', '9')
# Adjust the required POW values for each of this user's addresses to conform to protocol v3 norms.
if BMConfigParser().getint('bitmessagesettings',
'settingsversion') == 9:
for addressInKeysFile in BMConfigParser().addressses():
try:
previousTotalDifficulty = float(BMConfigParser().getint(
addressInKeysFile, 'noncetrialsperbyte')) / 320
previousSmallMessageDifficulty = float(
BMConfigParser().getint(
addressInKeysFile,
'payloadlengthextrabytes')) / 14000
if previousTotalDifficulty <= 2:
previousTotalDifficulty = 1
if previousSmallMessageDifficulty < 1:
previousSmallMessageDifficulty = 1
BMConfigParser().set(
addressInKeysFile, 'noncetrialsperbyte',
str(int(previousTotalDifficulty * 1000)))
BMConfigParser().set(
addressInKeysFile, 'payloadlengthextrabytes',
str(int(previousSmallMessageDifficulty * 1000)))
except:
continue
BMConfigParser().set('bitmessagesettings', 'maxdownloadrate', '0')
BMConfigParser().set('bitmessagesettings', 'maxuploadrate', '0')
BMConfigParser().set('bitmessagesettings', 'settingsversion', '10')
BMConfigParser().save()
# sanity check
if BMConfigParser().getint('bitmessagesettings',
'maxacceptablenoncetrialsperbyte') == 0:
BMConfigParser().set(
'bitmessagesettings', 'maxacceptablenoncetrialsperbyte',
str(defaults.ridiculousDifficulty *
defaults.networkDefaultProofOfWorkNonceTrialsPerByte))
if BMConfigParser().getint(
'bitmessagesettings',
'maxacceptablepayloadlengthextrabytes') == 0:
BMConfigParser().set(
'bitmessagesettings', 'maxacceptablepayloadlengthextrabytes',
str(defaults.ridiculousDifficulty *
defaults.networkDefaultPayloadLengthExtraBytes))
# The format of data stored in the pubkeys table has changed. Let's
# clear it, and the pubkeys from inventory, so that they'll be re-downloaded.
item = '''SELECT value FROM settings WHERE key='version';'''
parameters = ''
self.cur.execute(item, parameters)
currentVersion = int(self.cur.fetchall()[0][0])
if currentVersion == 7:
logger.debug(
'In messages.dat database, clearing pubkeys table because the data format has been updated.'
)
self.cur.execute('''delete from inventory where objecttype = 1;''')
self.cur.execute('''delete from pubkeys;''')
# Any sending messages for which we *thought* that we had the pubkey must
# be rechecked.
self.cur.execute(
'''UPDATE sent SET status='msgqueued' WHERE status='doingmsgpow' or status='badkey';'''
)
query = '''update settings set value=? WHERE key='version';'''
parameters = (8, )
self.cur.execute(query, parameters)
logger.debug('Finished clearing currently held pubkeys.')
# Add a new column to the inbox table to store the hash of the message signature.
# We'll use this as temporary message UUID in order to detect duplicates.
item = '''SELECT value FROM settings WHERE key='version';'''
parameters = ''
self.cur.execute(item, parameters)
currentVersion = int(self.cur.fetchall()[0][0])
if currentVersion == 8:
logger.debug(
'In messages.dat database, adding sighash field to the inbox table.'
)
item = '''ALTER TABLE inbox ADD sighash blob DEFAULT '' '''
parameters = ''
self.cur.execute(item, parameters)
item = '''update settings set value=? WHERE key='version';'''
parameters = (9, )
self.cur.execute(item, parameters)
# TTL is now user-specifiable. Let's add an option to save whatever the user selects.
if not BMConfigParser().has_option('bitmessagesettings', 'ttl'):
BMConfigParser().set('bitmessagesettings', 'ttl', '367200')
# We'll also need a `sleeptill` field and a `ttl` field. Also we can combine
# the pubkeyretrynumber and msgretrynumber into one.
item = '''SELECT value FROM settings WHERE key='version';'''
parameters = ''
self.cur.execute(item, parameters)
currentVersion = int(self.cur.fetchall()[0][0])
if currentVersion == 9:
logger.info(
'In messages.dat database, making TTL-related changes: combining the pubkeyretrynumber and msgretrynumber fields into the retrynumber field and adding the sleeptill and ttl fields...'
)
self.cur.execute(
'''CREATE TEMPORARY TABLE sent_backup (msgid blob, toaddress text, toripe blob, fromaddress text, subject text, message text, ackdata blob, lastactiontime integer, status text, retrynumber integer, folder text, encodingtype int)'''
)
self.cur.execute(
'''INSERT INTO sent_backup SELECT msgid, toaddress, toripe, fromaddress, subject, message, ackdata, lastactiontime, status, 0, folder, encodingtype FROM sent;'''
)
self.cur.execute('''DROP TABLE sent''')
self.cur.execute(
'''CREATE TABLE sent (msgid blob, toaddress text, toripe blob, fromaddress text, subject text, message text, ackdata blob, senttime integer, lastactiontime integer, sleeptill int, status text, retrynumber integer, folder text, encodingtype int, ttl int)'''
)
self.cur.execute(
'''INSERT INTO sent SELECT msgid, toaddress, toripe, fromaddress, subject, message, ackdata, lastactiontime, lastactiontime, 0, status, 0, folder, encodingtype, 216000 FROM sent_backup;'''
)
self.cur.execute('''DROP TABLE sent_backup''')
logger.info(
'In messages.dat database, finished making TTL-related changes.'
)
logger.debug(
'In messages.dat database, adding address field to the pubkeys table.'
)
# We're going to have to calculate the address for each row in the pubkeys
# table. Then we can take out the hash field.
self.cur.execute(
'''ALTER TABLE pubkeys ADD address text DEFAULT '' ''')
self.cur.execute('''SELECT hash, addressversion FROM pubkeys''')
queryResult = self.cur.fetchall()
from addresses import encodeAddress
for row in queryResult:
addressHash, addressVersion = row
address = encodeAddress(addressVersion, 1, hash)
item = '''UPDATE pubkeys SET address=? WHERE hash=?;'''
parameters = (address, addressHash)
self.cur.execute(item, parameters)
# Now we can remove the hash field from the pubkeys table.
self.cur.execute(
'''CREATE TEMPORARY TABLE pubkeys_backup (address text, addressversion int, transmitdata blob, time int, usedpersonally text, UNIQUE(address) ON CONFLICT REPLACE)'''
)
self.cur.execute(
'''INSERT INTO pubkeys_backup SELECT address, addressversion, transmitdata, time, usedpersonally FROM pubkeys;'''
)
self.cur.execute('''DROP TABLE pubkeys''')
self.cur.execute(
'''CREATE TABLE pubkeys (address text, addressversion int, transmitdata blob, time int, usedpersonally text, UNIQUE(address) ON CONFLICT REPLACE)'''
)
self.cur.execute(
'''INSERT INTO pubkeys SELECT address, addressversion, transmitdata, time, usedpersonally FROM pubkeys_backup;'''
)
self.cur.execute('''DROP TABLE pubkeys_backup''')
logger.debug(
'In messages.dat database, done adding address field to the pubkeys table and removing the hash field.'
)
self.cur.execute(
'''update settings set value=10 WHERE key='version';''')
if not BMConfigParser().has_option('bitmessagesettings',
'onionhostname'):
BMConfigParser().set('bitmessagesettings', 'onionhostname', '')
if not BMConfigParser().has_option('bitmessagesettings', 'onionport'):
BMConfigParser().set('bitmessagesettings', 'onionport', '8444')
if not BMConfigParser().has_option('bitmessagesettings',
'onionbindip'):
BMConfigParser().set('bitmessagesettings', 'onionbindip',
'127.0.0.1')
if not BMConfigParser().has_option('bitmessagesettings',
'smtpdeliver'):
BMConfigParser().set('bitmessagesettings', 'smtpdeliver', '')
if not BMConfigParser().has_option('bitmessagesettings',
'hidetrayconnectionnotifications'):
BMConfigParser().set('bitmessagesettings',
'hidetrayconnectionnotifications', 'false')
if BMConfigParser().has_option('bitmessagesettings',
'maxoutboundconnections'):
try:
if BMConfigParser().getint('bitmessagesettings',
'maxoutboundconnections') < 1:
raise ValueError
except ValueError as err:
BMConfigParser().remove_option('bitmessagesettings',
'maxoutboundconnections')
logger.error(
'Your maximum outbound connections must be a number.')
if not BMConfigParser().has_option('bitmessagesettings',
'maxoutboundconnections'):
logger.info('Setting maximum outbound connections to 8.')
BMConfigParser().set('bitmessagesettings',
'maxoutboundconnections', '8')
BMConfigParser().save()
# Are you hoping to add a new option to the keys.dat file of existing
# Bitmessage users or modify the SQLite database? Add it right above this line!
try:
testpayload = '\x00\x00'
t = ('1234', 1, testpayload, '12345678', 'no')
self.cur.execute('''INSERT INTO pubkeys VALUES(?,?,?,?,?)''', t)
self.conn.commit()
self.cur.execute(
'''SELECT transmitdata FROM pubkeys WHERE address='1234' ''')
queryreturn = self.cur.fetchall()
for row in queryreturn:
transmitdata, = row
self.cur.execute('''DELETE FROM pubkeys WHERE address='1234' ''')
self.conn.commit()
if transmitdata == '':
logger.fatal(
'Problem: The version of SQLite you have cannot store Null values. Please download and install the latest revision of your version of Python (for example, the latest Python 2.7 revision) and try again.\n'
)
logger.fatal(
'PyBitmessage will now exit very abruptly. You may now see threading errors related to this abrupt exit but the problem you need to solve is related to SQLite.\n\n'
)
os._exit(0)
except Exception as err:
if str(err) == 'database or disk is full':
logger.fatal(
'(While null value test) Alert: Your disk or data storage volume is full. sqlThread will now exit.'
)
queues.UISignalQueue.put(('alert', (
tr._translate("MainWindow", "Disk full"),
tr._translate(
"MainWindow",
'Alert: Your disk or data storage volume is full. Bitmessage will now exit.'
), True)))
os._exit(0)
else:
logger.error(err)
# Let us check to see the last time we vaccumed the messages.dat file.
# If it has been more than a month let's do it now.
item = '''SELECT value FROM settings WHERE key='lastvacuumtime';'''
parameters = ''
self.cur.execute(item, parameters)
queryreturn = self.cur.fetchall()
for row in queryreturn:
value, = row
if int(value) < int(time.time()) - 86400:
logger.info(
'It has been a long time since the messages.dat file has been vacuumed. Vacuuming now...'
)
try:
self.cur.execute(''' VACUUM ''')
except Exception as err:
if str(err) == 'database or disk is full':
logger.fatal(
'(While VACUUM) Alert: Your disk or data storage volume is full. sqlThread will now exit.'
)
queues.UISignalQueue.put(('alert', (
tr._translate("MainWindow", "Disk full"),
tr._translate(
"MainWindow",
'Alert: Your disk or data storage volume is full. Bitmessage will now exit.'
), True)))
os._exit(0)
item = '''update settings set value=? WHERE key='lastvacuumtime';'''
parameters = (int(time.time()), )
self.cur.execute(item, parameters)
state.sqlReady = True
while True:
item = helper_sql.sqlSubmitQueue.get()
if item == 'commit':
try:
self.conn.commit()
except Exception as err:
if str(err) == 'database or disk is full':
logger.fatal(
'(While committing) Alert: Your disk or data storage volume is full. sqlThread will now exit.'
)
queues.UISignalQueue.put(('alert', (
tr._translate("MainWindow", "Disk full"),
tr._translate(
"MainWindow",
'Alert: Your disk or data storage volume is full. Bitmessage will now exit.'
), True)))
os._exit(0)
elif item == 'exit':
self.conn.close()
logger.info('sqlThread exiting gracefully.')
return
elif item == 'movemessagstoprog':
logger.debug(
'the sqlThread is moving the messages.dat file to the local program directory.'
)
try:
self.conn.commit()
except Exception as err:
if str(err) == 'database or disk is full':
logger.fatal(
'(while movemessagstoprog) Alert: Your disk or data storage volume is full. sqlThread will now exit.'
)
queues.UISignalQueue.put(('alert', (
tr._translate("MainWindow", "Disk full"),
tr._translate(
"MainWindow",
'Alert: Your disk or data storage volume is full. Bitmessage will now exit.'
), True)))
os._exit(0)
self.conn.close()
shutil.move(paths.lookupAppdataFolder() + 'messages.dat',
paths.lookupExeFolder() + 'messages.dat')
self.conn = sqlite3.connect(paths.lookupExeFolder() +
'messages.dat')
self.conn.text_factory = str
self.cur = self.conn.cursor()
elif item == 'movemessagstoappdata':
logger.debug(
'the sqlThread is moving the messages.dat file to the Appdata folder.'
)
try:
self.conn.commit()
except Exception as err:
if str(err) == 'database or disk is full':
logger.fatal(
'(while movemessagstoappdata) Alert: Your disk or data storage volume is full. sqlThread will now exit.'
)
queues.UISignalQueue.put(('alert', (
tr._translate("MainWindow", "Disk full"),
tr._translate(
"MainWindow",
'Alert: Your disk or data storage volume is full. Bitmessage will now exit.'
), True)))
os._exit(0)
self.conn.close()
shutil.move(paths.lookupExeFolder() + 'messages.dat',
paths.lookupAppdataFolder() + 'messages.dat')
self.conn = sqlite3.connect(paths.lookupAppdataFolder() +
'messages.dat')
self.conn.text_factory = str
self.cur = self.conn.cursor()
elif item == 'deleteandvacuume':
self.cur.execute('''delete from inbox where folder='trash' ''')
self.cur.execute('''delete from sent where folder='trash' ''')
self.conn.commit()
try:
self.cur.execute(''' VACUUM ''')
except Exception as err:
if str(err) == 'database or disk is full':
logger.fatal(
'(while deleteandvacuume) Alert: Your disk or data storage volume is full. sqlThread will now exit.'
)
queues.UISignalQueue.put(('alert', (
tr._translate("MainWindow", "Disk full"),
tr._translate(
"MainWindow",
'Alert: Your disk or data storage volume is full. Bitmessage will now exit.'
), True)))
os._exit(0)
else:
parameters = helper_sql.sqlSubmitQueue.get()
rowcount = 0
# print 'item', item
# print 'parameters', parameters
try:
self.cur.execute(item, parameters)
rowcount = self.cur.rowcount
except Exception as err:
if str(err) == 'database or disk is full':
logger.fatal(
'(while cur.execute) Alert: Your disk or data storage volume is full. sqlThread will now exit.'
)
queues.UISignalQueue.put(('alert', (
tr._translate("MainWindow", "Disk full"),
tr._translate(
"MainWindow",
'Alert: Your disk or data storage volume is full. Bitmessage will now exit.'
), True)))
os._exit(0)
else:
logger.fatal(
'Major error occurred when trying to execute a SQL statement within the sqlThread. Please tell Atheros about this error message or post it in the forum! Error occurred while trying to execute statement: "%s" Here are the parameters; you might want to censor this data with asterisks (***) as it can contain private information: %s. Here is the actual error message thrown by the sqlThread: %s',
str(item), str(repr(parameters)), str(err))
logger.fatal('This program shall now abruptly exit!')
os._exit(0)
helper_sql.sqlReturnQueue.put((self.cur.fetchall(), rowcount))
def run(self):
self.conn = sqlite3.connect(shared.appdata + "messages.dat")
self.conn.text_factory = str
self.cur = self.conn.cursor()
try:
self.cur.execute(
"""CREATE TABLE inbox (msgid blob, toaddress text, fromaddress text, subject text, received text, message text, folder text, encodingtype int, read bool, UNIQUE(msgid) ON CONFLICT REPLACE)"""
)
self.cur.execute(
"""CREATE TABLE sent (msgid blob, toaddress text, toripe blob, fromaddress text, subject text, message text, ackdata blob, lastactiontime integer, status text, pubkeyretrynumber integer, msgretrynumber integer, folder text, encodingtype int)"""
)
self.cur.execute("""CREATE TABLE subscriptions (label text, address text, enabled bool)""")
self.cur.execute("""CREATE TABLE addressbook (label text, address text)""")
self.cur.execute("""CREATE TABLE blacklist (label text, address text, enabled bool)""")
self.cur.execute("""CREATE TABLE whitelist (label text, address text, enabled bool)""")
"""
Explanation of what is in the pubkeys table:
The hash is the RIPEMD160 hash that is encoded in the Bitmessage address.
transmitdata /was/ literally the data that was included in the Bitmessage pubkey message when it arrived,
except for the 24 byte protocol header- ie, it started with the POW nonce. Since protocol v3, to maintain
backwards compability, the data format of the data on disk is staying the same even though the wire format has changed.
time is the time that the pubkey was broadcast on the network same as with every other type of Bitmessage object.
usedpersonally is set to "yes" if we have used the key personally. This keeps us from deleting it because we may want to
reply to a message in the future. This field is not a bool because we may need more flexability in the future and it doesn't
take up much more space anyway.
"""
self.cur.execute(
"""CREATE TABLE pubkeys (hash blob, addressversion int, transmitdata blob, time int, usedpersonally text, UNIQUE(hash, addressversion) ON CONFLICT REPLACE)"""
)
self.cur.execute(
"""CREATE TABLE inventory (hash blob, objecttype int, streamnumber int, payload blob, expirestime integer, tag blob, UNIQUE(hash) ON CONFLICT REPLACE)"""
)
self.cur.execute(
"""INSERT INTO subscriptions VALUES('Bitmessage new releases/announcements','BM-GtovgYdgs7qXPkoYaRgrLFuFKz1SFpsw',1)"""
)
self.cur.execute("""CREATE TABLE settings (key blob, value blob, UNIQUE(key) ON CONFLICT REPLACE)""")
self.cur.execute("""INSERT INTO settings VALUES('version','7')""")
self.cur.execute("""INSERT INTO settings VALUES('lastvacuumtime',?)""", (int(time.time()),))
self.cur.execute(
"""CREATE TABLE objectprocessorqueue (objecttype int, data blob, UNIQUE(objecttype, data) ON CONFLICT REPLACE)"""
)
self.conn.commit()
logger.info("Created messages database file")
except Exception as err:
if str(err) == "table inbox already exists":
logger.debug("Database file already exists.")
else:
sys.stderr.write("ERROR trying to create database file (message.dat). Error message: %s\n" % str(err))
os._exit(0)
try:
self.cur.execute(
"""CREATE TABLE consensus (id INTEGER PRIMARY KEY AUTOINCREMENT, chanaddress text, hash text,
data BLOB, settings TEXT, UNIQUE(chanaddress))"""
)
self.cur.execute(
"""CREATE TABLE consensus_messages (consensus_id INTEGER, local_time BIGINT, message BLOB,
message_hash BLOB, state INTEGER)"""
)
self.conn.commit()
except Exception as err:
if str(err) != "table consensus already exists":
sys.stderr.write("ERROR trying to create consensus table. Error message: %s\n" % str(err))
os._exit(0)
if shared.config.getint("bitmessagesettings", "settingsversion") == 1:
shared.config.set("bitmessagesettings", "settingsversion", "2")
# If the settings version is equal to 2 or 3 then the
# sqlThread will modify the pubkeys table and change
# the settings version to 4.
shared.config.set("bitmessagesettings", "socksproxytype", "none")
shared.config.set("bitmessagesettings", "sockshostname", "localhost")
shared.config.set("bitmessagesettings", "socksport", "9050")
shared.config.set("bitmessagesettings", "socksauthentication", "false")
shared.config.set("bitmessagesettings", "socksusername", "")
shared.config.set("bitmessagesettings", "sockspassword", "")
shared.config.set("bitmessagesettings", "sockslisten", "false")
shared.config.set("bitmessagesettings", "keysencrypted", "false")
shared.config.set("bitmessagesettings", "messagesencrypted", "false")
with open(shared.appdata + "keys.dat", "wb") as configfile:
shared.config.write(configfile)
# People running earlier versions of PyBitmessage do not have the
# usedpersonally field in their pubkeys table. Let's add it.
if shared.config.getint("bitmessagesettings", "settingsversion") == 2:
item = """ALTER TABLE pubkeys ADD usedpersonally text DEFAULT 'no' """
parameters = ""
self.cur.execute(item, parameters)
self.conn.commit()
shared.config.set("bitmessagesettings", "settingsversion", "3")
with open(shared.appdata + "keys.dat", "wb") as configfile:
shared.config.write(configfile)
# People running earlier versions of PyBitmessage do not have the
# encodingtype field in their inbox and sent tables or the read field
# in the inbox table. Let's add them.
if shared.config.getint("bitmessagesettings", "settingsversion") == 3:
item = """ALTER TABLE inbox ADD encodingtype int DEFAULT '2' """
parameters = ""
self.cur.execute(item, parameters)
item = """ALTER TABLE inbox ADD read bool DEFAULT '1' """
parameters = ""
self.cur.execute(item, parameters)
item = """ALTER TABLE sent ADD encodingtype int DEFAULT '2' """
parameters = ""
self.cur.execute(item, parameters)
self.conn.commit()
shared.config.set("bitmessagesettings", "settingsversion", "4")
if shared.config.getint("bitmessagesettings", "settingsversion") == 4:
shared.config.set(
"bitmessagesettings",
"defaultnoncetrialsperbyte",
str(shared.networkDefaultProofOfWorkNonceTrialsPerByte),
)
shared.config.set(
"bitmessagesettings",
"defaultpayloadlengthextrabytes",
str(shared.networkDefaultPayloadLengthExtraBytes),
)
shared.config.set("bitmessagesettings", "settingsversion", "5")
if shared.config.getint("bitmessagesettings", "settingsversion") == 5:
shared.config.set("bitmessagesettings", "maxacceptablenoncetrialsperbyte", "0")
shared.config.set("bitmessagesettings", "maxacceptablepayloadlengthextrabytes", "0")
shared.config.set("bitmessagesettings", "settingsversion", "6")
# From now on, let us keep a 'version' embedded in the messages.dat
# file so that when we make changes to the database, the database
# version we are on can stay embedded in the messages.dat file. Let us
# check to see if the settings table exists yet.
item = """SELECT name FROM sqlite_master WHERE type='table' AND name='settings';"""
parameters = ""
self.cur.execute(item, parameters)
if self.cur.fetchall() == []:
# The settings table doesn't exist. We need to make it.
logger.debug("In messages.dat database, creating new 'settings' table.")
self.cur.execute("""CREATE TABLE settings (key text, value blob, UNIQUE(key) ON CONFLICT REPLACE)""")
self.cur.execute("""INSERT INTO settings VALUES('version','1')""")
self.cur.execute("""INSERT INTO settings VALUES('lastvacuumtime',?)""", (int(time.time()),))
logger.debug("In messages.dat database, removing an obsolete field from the pubkeys table.")
self.cur.execute(
"""CREATE TEMPORARY TABLE pubkeys_backup(hash blob, transmitdata blob, time int, usedpersonally text, UNIQUE(hash) ON CONFLICT REPLACE);"""
)
self.cur.execute(
"""INSERT INTO pubkeys_backup SELECT hash, transmitdata, time, usedpersonally FROM pubkeys;"""
)
self.cur.execute("""DROP TABLE pubkeys""")
self.cur.execute(
"""CREATE TABLE pubkeys (hash blob, transmitdata blob, time int, usedpersonally text, UNIQUE(hash) ON CONFLICT REPLACE)"""
)
self.cur.execute(
"""INSERT INTO pubkeys SELECT hash, transmitdata, time, usedpersonally FROM pubkeys_backup;"""
)
self.cur.execute("""DROP TABLE pubkeys_backup;""")
logger.debug(
"Deleting all pubkeys from inventory. They will be redownloaded and then saved with the correct times."
)
self.cur.execute("""delete from inventory where objecttype = 'pubkey';""")
logger.debug("replacing Bitmessage announcements mailing list with a new one.")
self.cur.execute("""delete from subscriptions where address='BM-BbkPSZbzPwpVcYZpU4yHwf9ZPEapN5Zx' """)
self.cur.execute(
"""INSERT INTO subscriptions VALUES('Bitmessage new releases/announcements','BM-GtovgYdgs7qXPkoYaRgrLFuFKz1SFpsw',1)"""
)
logger.debug("Commiting.")
self.conn.commit()
logger.debug("Vacuuming message.dat. You might notice that the file size gets much smaller.")
self.cur.execute(""" VACUUM """)
# After code refactoring, the possible status values for sent messages
# have changed.
self.cur.execute("""update sent set status='doingmsgpow' where status='doingpow' """)
self.cur.execute("""update sent set status='msgsent' where status='sentmessage' """)
self.cur.execute("""update sent set status='doingpubkeypow' where status='findingpubkey' """)
self.cur.execute("""update sent set status='broadcastqueued' where status='broadcastpending' """)
self.conn.commit()
if not shared.config.has_option("bitmessagesettings", "sockslisten"):
shared.config.set("bitmessagesettings", "sockslisten", "false")
ensureNamecoinOptions()
"""# Add a new column to the inventory table to store the first 20 bytes of encrypted messages to support Android app
item = '''SELECT value FROM settings WHERE key='version';'''
parameters = ''
self.cur.execute(item, parameters)
if int(self.cur.fetchall()[0][0]) == 1:
print 'upgrading database'
item = '''ALTER TABLE inventory ADD first20bytesofencryptedmessage blob DEFAULT '' '''
parameters = ''
self.cur.execute(item, parameters)
item = '''update settings set value=? WHERE key='version';'''
parameters = (2,)
self.cur.execute(item, parameters)"""
# Let's get rid of the first20bytesofencryptedmessage field in the inventory table.
item = """SELECT value FROM settings WHERE key='version';"""
parameters = ""
self.cur.execute(item, parameters)
if int(self.cur.fetchall()[0][0]) == 2:
logger.debug("In messages.dat database, removing an obsolete field from the inventory table.")
self.cur.execute(
"""CREATE TEMPORARY TABLE inventory_backup(hash blob, objecttype text, streamnumber int, payload blob, receivedtime integer, UNIQUE(hash) ON CONFLICT REPLACE);"""
)
self.cur.execute(
"""INSERT INTO inventory_backup SELECT hash, objecttype, streamnumber, payload, receivedtime FROM inventory;"""
)
self.cur.execute("""DROP TABLE inventory""")
self.cur.execute(
"""CREATE TABLE inventory (hash blob, objecttype text, streamnumber int, payload blob, receivedtime integer, UNIQUE(hash) ON CONFLICT REPLACE)"""
)
self.cur.execute(
"""INSERT INTO inventory SELECT hash, objecttype, streamnumber, payload, receivedtime FROM inventory_backup;"""
)
self.cur.execute("""DROP TABLE inventory_backup;""")
item = """update settings set value=? WHERE key='version';"""
parameters = (3,)
self.cur.execute(item, parameters)
# Add a new column to the inventory table to store tags.
item = """SELECT value FROM settings WHERE key='version';"""
parameters = ""
self.cur.execute(item, parameters)
currentVersion = int(self.cur.fetchall()[0][0])
if currentVersion == 1 or currentVersion == 3:
logger.debug("In messages.dat database, adding tag field to the inventory table.")
item = """ALTER TABLE inventory ADD tag blob DEFAULT '' """
parameters = ""
self.cur.execute(item, parameters)
item = """update settings set value=? WHERE key='version';"""
parameters = (4,)
self.cur.execute(item, parameters)
if not shared.config.has_option("bitmessagesettings", "userlocale"):
shared.config.set("bitmessagesettings", "userlocale", "system")
if not shared.config.has_option("bitmessagesettings", "sendoutgoingconnections"):
shared.config.set("bitmessagesettings", "sendoutgoingconnections", "True")
# Raise the default required difficulty from 1 to 2
# With the change to protocol v3, this is obsolete.
if shared.config.getint("bitmessagesettings", "settingsversion") == 6:
"""if int(shared.config.get('bitmessagesettings','defaultnoncetrialsperbyte')) == shared.networkDefaultProofOfWorkNonceTrialsPerByte:
shared.config.set('bitmessagesettings','defaultnoncetrialsperbyte', str(shared.networkDefaultProofOfWorkNonceTrialsPerByte * 2))
"""
shared.config.set("bitmessagesettings", "settingsversion", "7")
# Add a new column to the pubkeys table to store the address version.
# We're going to trash all of our pubkeys and let them be redownloaded.
item = """SELECT value FROM settings WHERE key='version';"""
parameters = ""
self.cur.execute(item, parameters)
currentVersion = int(self.cur.fetchall()[0][0])
if currentVersion == 4:
self.cur.execute("""DROP TABLE pubkeys""")
self.cur.execute(
"""CREATE TABLE pubkeys (hash blob, addressversion int, transmitdata blob, time int, usedpersonally text, UNIQUE(hash, addressversion) ON CONFLICT REPLACE)"""
)
self.cur.execute("""delete from inventory where objecttype = 'pubkey';""")
item = """update settings set value=? WHERE key='version';"""
parameters = (5,)
self.cur.execute(item, parameters)
if not shared.config.has_option("bitmessagesettings", "useidenticons"):
shared.config.set("bitmessagesettings", "useidenticons", "True")
if not shared.config.has_option("bitmessagesettings", "identiconsuffix"): # acts as a salt
shared.config.set(
"bitmessagesettings",
"identiconsuffix",
"".join(random.choice("123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz") for x in range(12)),
) # a twelve character pseudo-password to salt the identicons
# Add settings to support no longer resending messages after a certain period of time even if we never get an ack
if shared.config.getint("bitmessagesettings", "settingsversion") == 7:
shared.config.set("bitmessagesettings", "stopresendingafterxdays", "")
shared.config.set("bitmessagesettings", "stopresendingafterxmonths", "")
# shared.config.set(
shared.config.set("bitmessagesettings", "settingsversion", "8")
with open(shared.appdata + "keys.dat", "wb") as configfile:
shared.config.write(configfile)
# Add a new table: objectprocessorqueue with which to hold objects
# that have yet to be processed if the user shuts down Bitmessage.
item = """SELECT value FROM settings WHERE key='version';"""
parameters = ""
self.cur.execute(item, parameters)
currentVersion = int(self.cur.fetchall()[0][0])
if currentVersion == 5:
self.cur.execute("""DROP TABLE knownnodes""")
self.cur.execute(
"""CREATE TABLE objectprocessorqueue (objecttype text, data blob, UNIQUE(objecttype, data) ON CONFLICT REPLACE)"""
)
item = """update settings set value=? WHERE key='version';"""
parameters = (6,)
self.cur.execute(item, parameters)
# changes related to protocol v3
# In table inventory and objectprocessorqueue, objecttype is now an integer (it was a human-friendly string previously)
item = """SELECT value FROM settings WHERE key='version';"""
parameters = ""
self.cur.execute(item, parameters)
currentVersion = int(self.cur.fetchall()[0][0])
if currentVersion == 6:
logger.debug("In messages.dat database, dropping and recreating the inventory table.")
self.cur.execute("""DROP TABLE inventory""")
self.cur.execute(
"""CREATE TABLE inventory (hash blob, objecttype int, streamnumber int, payload blob, expirestime integer, tag blob, UNIQUE(hash) ON CONFLICT REPLACE)"""
)
self.cur.execute("""DROP TABLE objectprocessorqueue""")
self.cur.execute(
"""CREATE TABLE objectprocessorqueue (objecttype int, data blob, UNIQUE(objecttype, data) ON CONFLICT REPLACE)"""
)
item = """update settings set value=? WHERE key='version';"""
parameters = (7,)
self.cur.execute(item, parameters)
logger.debug("Finished dropping and recreating the inventory table.")
# With the change to protocol version 3, reset the user-settable difficulties to 1
if shared.config.getint("bitmessagesettings", "settingsversion") == 8:
shared.config.set(
"bitmessagesettings",
"defaultnoncetrialsperbyte",
str(shared.networkDefaultProofOfWorkNonceTrialsPerByte),
)
shared.config.set(
"bitmessagesettings",
"defaultpayloadlengthextrabytes",
str(shared.networkDefaultPayloadLengthExtraBytes),
)
previousTotalDifficulty = (
int(shared.config.getint("bitmessagesettings", "maxacceptablenoncetrialsperbyte")) / 320
)
previousSmallMessageDifficulty = (
int(shared.config.getint("bitmessagesettings", "maxacceptablepayloadlengthextrabytes")) / 14000
)
shared.config.set(
"bitmessagesettings", "maxacceptablenoncetrialsperbyte", str(previousTotalDifficulty * 1000)
)
shared.config.set(
"bitmessagesettings", "maxacceptablepayloadlengthextrabytes", str(previousSmallMessageDifficulty * 1000)
)
shared.config.set("bitmessagesettings", "settingsversion", "9")
# Adjust the required POW values for each of this user's addresses to conform to protocol v3 norms.
if shared.config.getint("bitmessagesettings", "settingsversion") == 9:
for addressInKeysFile in shared.config.sections():
try:
previousTotalDifficulty = float(shared.config.getint(addressInKeysFile, "noncetrialsperbyte")) / 320
previousSmallMessageDifficulty = (
float(shared.config.getint(addressInKeysFile, "payloadlengthextrabytes")) / 14000
)
if previousTotalDifficulty <= 2:
previousTotalDifficulty = 1
if previousSmallMessageDifficulty < 1:
previousSmallMessageDifficulty = 1
shared.config.set(addressInKeysFile, "noncetrialsperbyte", str(int(previousTotalDifficulty * 1000)))
shared.config.set(
addressInKeysFile, "payloadlengthextrabytes", str(int(previousSmallMessageDifficulty * 1000))
)
except:
continue
shared.config.set("bitmessagesettings", "maxdownloadrate", "0")
shared.config.set("bitmessagesettings", "maxuploadrate", "0")
shared.config.set("bitmessagesettings", "settingsversion", "10")
with open(shared.appdata + "keys.dat", "wb") as configfile:
shared.config.write(configfile)
# Are you hoping to add a new option to the keys.dat file of existing
# Bitmessage users or modify the SQLite database? Add it right above this line!
try:
testpayload = "\x00\x00"
t = ("1234", 1, testpayload, "12345678", "no")
self.cur.execute("""INSERT INTO pubkeys VALUES(?,?,?,?,?)""", t)
self.conn.commit()
self.cur.execute("""SELECT transmitdata FROM pubkeys WHERE hash='1234' """)
queryreturn = self.cur.fetchall()
for row in queryreturn:
transmitdata, = row
self.cur.execute("""DELETE FROM pubkeys WHERE hash='1234' """)
self.conn.commit()
if transmitdata == "":
logger.fatal(
"Problem: The version of SQLite you have cannot store Null values. Please download and install the latest revision of your version of Python (for example, the latest Python 2.7 revision) and try again.\n"
)
logger.fatal(
"PyBitmessage will now exit very abruptly. You may now see threading errors related to this abrupt exit but the problem you need to solve is related to SQLite.\n\n"
)
os._exit(0)
except Exception as err:
if str(err) == "database or disk is full":
logger.fatal(
"(While null value test) Alert: Your disk or data storage volume is full. sqlThread will now exit."
)
shared.UISignalQueue.put(
(
"alert",
(
tr.translateText("MainWindow", "Disk full"),
tr.translateText(
"MainWindow",
"Alert: Your disk or data storage volume is full. Bitmessage will now exit.",
),
True,
),
)
)
if shared.daemon:
os._exit(0)
else:
return
else:
logger.error(err)
# Let us check to see the last time we vaccumed the messages.dat file.
# If it has been more than a month let's do it now.
item = """SELECT value FROM settings WHERE key='lastvacuumtime';"""
parameters = ""
self.cur.execute(item, parameters)
queryreturn = self.cur.fetchall()
for row in queryreturn:
value, = row
if int(value) < int(time.time()) - 2592000:
logger.info("It has been a long time since the messages.dat file has been vacuumed. Vacuuming now...")
try:
self.cur.execute(""" VACUUM """)
except Exception as err:
if str(err) == "database or disk is full":
logger.fatal(
"(While VACUUM) Alert: Your disk or data storage volume is full. sqlThread will now exit."
)
shared.UISignalQueue.put(
(
"alert",
(
tr.translateText("MainWindow", "Disk full"),
tr.translateText(
"MainWindow",
"Alert: Your disk or data storage volume is full. Bitmessage will now exit.",
),
True,
),
)
)
if shared.daemon:
os._exit(0)
else:
return
item = """update settings set value=? WHERE key='lastvacuumtime';"""
parameters = (int(time.time()),)
self.cur.execute(item, parameters)
while True:
item = shared.sqlSubmitQueue.get()
if item == "commit":
try:
self.conn.commit()
except Exception as err:
if str(err) == "database or disk is full":
logger.fatal(
"(While committing) Alert: Your disk or data storage volume is full. sqlThread will now exit."
)
shared.UISignalQueue.put(
(
"alert",
(
tr.translateText("MainWindow", "Disk full"),
tr.translateText(
"MainWindow",
"Alert: Your disk or data storage volume is full. Bitmessage will now exit.",
),
True,
),
)
)
if shared.daemon:
os._exit(0)
else:
return
elif item == "exit":
self.conn.close()
logger.info("sqlThread exiting gracefully.")
return
elif item == "movemessagstoprog":
logger.debug("the sqlThread is moving the messages.dat file to the local program directory.")
try:
self.conn.commit()
except Exception as err:
if str(err) == "database or disk is full":
logger.fatal(
"(while movemessagstoprog) Alert: Your disk or data storage volume is full. sqlThread will now exit."
)
shared.UISignalQueue.put(
(
"alert",
(
tr.translateText("MainWindow", "Disk full"),
tr.translateText(
"MainWindow",
"Alert: Your disk or data storage volume is full. Bitmessage will now exit.",
),
True,
),
)
)
if shared.daemon:
os._exit(0)
else:
return
self.conn.close()
shutil.move(shared.lookupAppdataFolder() + "messages.dat", "messages.dat")
self.conn = sqlite3.connect("messages.dat")
self.conn.text_factory = str
self.cur = self.conn.cursor()
elif item == "movemessagstoappdata":
logger.debug("the sqlThread is moving the messages.dat file to the Appdata folder.")
try:
self.conn.commit()
except Exception as err:
if str(err) == "database or disk is full":
logger.fatal(
"(while movemessagstoappdata) Alert: Your disk or data storage volume is full. sqlThread will now exit."
)
shared.UISignalQueue.put(
(
"alert",
(
tr.translateText("MainWindow", "Disk full"),
tr.translateText(
"MainWindow",
"Alert: Your disk or data storage volume is full. Bitmessage will now exit.",
),
True,
),
)
)
if shared.daemon:
os._exit(0)
else:
return
self.conn.close()
shutil.move("messages.dat", shared.lookupAppdataFolder() + "messages.dat")
self.conn = sqlite3.connect(shared.appdata + "messages.dat")
self.conn.text_factory = str
self.cur = self.conn.cursor()
elif item == "deleteandvacuume":
self.cur.execute("""delete from inbox where folder='trash' """)
self.cur.execute("""delete from sent where folder='trash' """)
self.conn.commit()
try:
self.cur.execute(""" VACUUM """)
except Exception as err:
if str(err) == "database or disk is full":
logger.fatal(
"(while deleteandvacuume) Alert: Your disk or data storage volume is full. sqlThread will now exit."
)
shared.UISignalQueue.put(
(
"alert",
(
tr.translateText("MainWindow", "Disk full"),
tr.translateText(
"MainWindow",
"Alert: Your disk or data storage volume is full. Bitmessage will now exit.",
),
True,
),
)
)
if shared.daemon:
os._exit(0)
else:
return
else:
parameters = shared.sqlSubmitQueue.get()
# print 'item', item
# print 'parameters', parameters
try:
self.cur.execute(item, parameters)
except Exception as err:
if str(err) == "database or disk is full":
logger.fatal(
"(while cur.execute) Alert: Your disk or data storage volume is full. sqlThread will now exit."
)
shared.UISignalQueue.put(
(
"alert",
(
tr.translateText("MainWindow", "Disk full"),
tr.translateText(
"MainWindow",
"Alert: Your disk or data storage volume is full. Bitmessage will now exit.",
),
True,
),
)
)
if shared.daemon:
os._exit(0)
else:
return
else:
logger.fatal(
'Major error occurred when trying to execute a SQL statement within the sqlThread. Please tell Atheros about this error message or post it in the forum! Error occurred while trying to execute statement: "%s" Here are the parameters; you might want to censor this data with asterisks (***) as it can contain private information: %s. Here is the actual error message thrown by the sqlThread: %s',
str(item),
str(repr(parameters)),
str(err),
)
logger.fatal("This program shall now abruptly exit!")
os._exit(0)
shared.sqlReturnQueue.put(self.cur.fetchall())
def run(self):
timeWeLastClearedInventoryAndPubkeysTables = 0
while True:
shared.UISignalQueue.put(
("updateStatusBar", "Doing housekeeping (Flushing inventory in memory to disk...)")
)
with shared.inventoryLock: # If you use both the inventoryLock and the sqlLock, always use the inventoryLock OUTSIDE of the sqlLock.
with SqlBulkExecute() as sql:
for hash, storedValue in shared.inventory.items():
objectType, streamNumber, payload, receivedTime, tag = storedValue
if int(time.time()) - 3600 > receivedTime:
sql.execute(
"""INSERT INTO inventory VALUES (?,?,?,?,?,?)""",
hash,
objectType,
streamNumber,
payload,
receivedTime,
tag,
)
del shared.inventory[hash]
shared.UISignalQueue.put(("updateStatusBar", ""))
shared.broadcastToSendDataQueues(
(0, "pong", "no data")
) # commands the sendData threads to send out a pong message if they haven't sent anything else in the last five minutes. The socket timeout-time is 10 minutes.
# If we are running as a daemon then we are going to fill up the UI
# queue which will never be handled by a UI. We should clear it to
# save memory.
if shared.safeConfigGetBoolean("bitmessagesettings", "daemon"):
shared.UISignalQueue.queue.clear()
if timeWeLastClearedInventoryAndPubkeysTables < int(time.time()) - 7380:
timeWeLastClearedInventoryAndPubkeysTables = int(time.time())
# inventory (moves data from the inventory data structure to
# the on-disk sql database)
# inventory (clears pubkeys after 28 days and everything else
# after 2 days and 12 hours)
sqlExecute(
"""DELETE FROM inventory WHERE (receivedtime<? AND objecttype<>'pubkey') OR (receivedtime<? AND objecttype='pubkey') """,
int(time.time()) - shared.lengthOfTimeToLeaveObjectsInInventory,
int(time.time()) - shared.lengthOfTimeToHoldOnToAllPubkeys,
)
# pubkeys
sqlExecute(
"""DELETE FROM pubkeys WHERE time<? AND usedpersonally='no' """,
int(time.time()) - shared.lengthOfTimeToHoldOnToAllPubkeys,
)
queryreturn = sqlQuery(
"""select toaddress, toripe, fromaddress, subject, message, ackdata, lastactiontime, status, pubkeyretrynumber, msgretrynumber FROM sent WHERE ((status='awaitingpubkey' OR status='msgsent') AND folder='sent') """
) # If the message's folder='trash' then we'll ignore it.
for row in queryreturn:
if len(row) < 5:
with shared.printLock:
sys.stderr.write(
"Something went wrong in the singleCleaner thread: a query did not return the requested fields. "
+ repr(row)
)
time.sleep(3)
break
toaddress, toripe, fromaddress, subject, message, ackdata, lastactiontime, status, pubkeyretrynumber, msgretrynumber = (
row
)
if status == "awaitingpubkey":
if int(time.time()) - lastactiontime > (
shared.maximumAgeOfAnObjectThatIAmWillingToAccept * (2 ** (pubkeyretrynumber))
):
print "It has been a long time and we haven't heard a response to our getpubkey request. Sending again."
try:
del shared.neededPubkeys[
toripe
] # We need to take this entry out of the shared.neededPubkeys structure because the shared.workerQueue checks to see whether the entry is already present and will not do the POW and send the message because it assumes that it has already done it recently.
except:
pass
shared.UISignalQueue.put(
("updateStatusBar", "Doing work necessary to again attempt to request a public key...")
)
t = ()
sqlExecute(
"""UPDATE sent SET lastactiontime=?, pubkeyretrynumber=?, status='msgqueued' WHERE toripe=?""",
int(time.time()),
pubkeyretrynumber + 1,
toripe,
)
shared.workerQueue.put(("sendmessage", ""))
else: # status == msgsent
if int(time.time()) - lastactiontime > (
shared.maximumAgeOfAnObjectThatIAmWillingToAccept * (2 ** (msgretrynumber))
):
print "It has been a long time and we haven't heard an acknowledgement to our msg. Sending again."
sqlExecute(
"""UPDATE sent SET lastactiontime=?, msgretrynumber=?, status=? WHERE ackdata=?""",
int(time.time()),
msgretrynumber + 1,
"msgqueued",
ackdata,
)
shared.workerQueue.put(("sendmessage", ""))
shared.UISignalQueue.put(
("updateStatusBar", "Doing work necessary to again attempt to deliver a message...")
)
# Let's also clear and reload shared.inventorySets to keep it from
# taking up an unnecessary amount of memory.
for streamNumber in shared.inventorySets:
shared.inventorySets[streamNumber] = set()
queryData = sqlQuery("""SELECT hash FROM inventory WHERE streamnumber=?""", streamNumber)
for row in queryData:
shared.inventorySets[streamNumber].add(row[0])
with shared.inventoryLock:
for hash, storedValue in shared.inventory.items():
objectType, streamNumber, payload, receivedTime, tag = storedValue
if streamNumber in shared.inventorySets:
shared.inventorySets[streamNumber].add(hash)
# Let us write out the knowNodes to disk if there is anything new to write out.
if shared.needToWriteKnownNodesToDisk:
shared.knownNodesLock.acquire()
output = open(shared.appdata + "knownnodes.dat", "wb")
try:
pickle.dump(shared.knownNodes, output)
output.close()
except Exception as err:
if "Errno 28" in str(err):
logger.fatal(
"(while receiveDataThread shared.needToWriteKnownNodesToDisk) Alert: Your disk or data storage volume is full. "
)
shared.UISignalQueue.put(
(
"alert",
(
tr.translateText("MainWindow", "Disk full"),
tr.translateText(
"MainWindow",
"Alert: Your disk or data storage volume is full. Bitmessage will now exit.",
),
True,
),
)
)
if shared.daemon:
os._exit(0)
shared.knownNodesLock.release()
shared.needToWriteKnownNodesToDisk = False
time.sleep(300)
