def createTables(db: DB):
commands = []
# Create rent table
commands.append("""
CREATE TABLE IF NOT EXISTS addresses (
address VARCHAR NOT NULL,
name VARCHAR NOT NULL,
created_at TIMESTAMP NOT NULL DEFAULT NOW()
);
""")
db.execute(commands, False)
def run(self, limit, to=0):
ids = DB.fetchall("SELECT id from bs_movie")
for id in ids:
self.idList.append(str(id[0]))
print('正在抓取....')
# self.makedir()
if to == 0:
nums = range(limit)
else:
nums = range(limit, to + 1)
threads = []
for i in nums:
url = self.domain + '/html/gndy/dyzz/list_23_%s.html' % (i + 1)
th = threading.Thread(target=self.get_page, args=(url, ))
th.start()
threads.append(th)
while len(threads) >= self.count_async / 2:
for j, _th in enumerate(threads):
if not _th.isAlive():
threads.pop(j).join()
while len(threads) > 0:
for i, _th in enumerate(threads):
if not _th.isAlive():
threads.pop(i).join()
threads = []
num = len(self.movies)
print('共%s' % num)
for link in self.movies:
th = threading.Thread(target=self.get_movie, args=(link, ))
th.start()
threads.append(th)
while len(threads) >= self.count_async / 2:
for j, _th in enumerate(threads):
if not _th.isAlive():
threads.pop(j).join()
num = num - 1
print('剩%s' % num)
while len(threads) > 0:
for i, _th in enumerate(threads):
if not _th.isAlive():
threads.pop(i).join()
num = num - 1
print('剩%s' % num)
# print('正在入库....')
# if len(self.sqlList) > 0:
# DB.doTrans(self.sqlList)
DB.execute(
"DELETE from bs_movie where id in (select a.id from (SELECT min(id) as id FROM bs_movie GROUP BY name HAVING count(name) > 1) a)"
)
def createTables(db: DB):
commands = []
# Create rent table
commands.append("""
CREATE TABLE transactions (
receiver VARCHAR NOT NULL,
sender VARCHAR NOT NULL,
amount DOUBLE precision NOT NULL,
created_at TIMESTAMP NOT NULL DEFAULT NOW()
);
""")
db.execute(commands, False)
class Tester():
def __init__(self, test_folder, options):
self.test_folder = os.path.join(test_folder, "tester")
self.options = options
self.config = Config.get_config()
self.store_folder = os.path.join(self.test_folder, "stores")
self.files_folder = os.path.join(self.test_folder, "files")
self.restore_folder = os.path.join(self.test_folder, "restore")
self.db = DB()
self.max_fs_id = self.db.query("select max(fs_id) from fs", ())[0][0]
if self.max_fs_id is None:
self.max_fs_id = 0
self.max_version_id = self.db.query("select max(version_id) from versions", ())[0][0]
if self.max_version_id is None:
self.max_version_id = 0
self.max_run_id = self.db.query("select max(run_id) from runs", ())[0][0]
if self.max_run_id is None:
self.max_run_id = 0
self.max_message_id = self.db.query("select max(message_id) from messages", ())[0][0]
if self.max_message_id is None:
self.max_message_id = 0
log.debug("MAX IDs", self.max_fs_id, self.max_version_id, self.max_run_id, self.max_message_id)
self.teststring1 = os.urandom(204800)
self.teststring2 = os.urandom(204800)
def run(self):
try:
self.simpleCycleTest()
self.restoreTest()
except:
pass
if self.options.message:
try:
from lib.dlg import Notify
Notify(const.AppTitle, "Test run is complete")
except:
log.debug("Unable to notify. No-one logged in?")
if self.options.email:
self.send_email(True)
if self.options.shutdown:
os.system("shutdown -P +2")
def simpleCycleTest(self):
try:
# build the test files/folders
self.make_folders()
# fill the files folder with the initial set of files
self.fill_files()
# Build a backup and store
self.build_config()
# self.run_test()
for i in xrange(int(self.options.cycles)):
log.info("Cycle", i)
# Run a full and one incremental
self.run_cycle_test()
except Exception as e:
log.error("Test run failed: %s" % str(e))
finally:
self.cleanup()
log.info("********Success")
def restoreTest(self):
try:
# build the test files/folders
self.make_folders()
# fill the files folder with the initial set of files
self.fill_files()
except:
pass
# def build_config(self):
# log.trace("build_config")
# try:
# if self.options.store:
# store = self.config.storage[self.options.store]
# else:
# # Make the store about 3x the options size
# s, dummy, dummy = utils.from_readable_form(self.options.size)
# store_size = utils.readable_form(s * 3)
# store = FolderStore("teststore1", store_size, True, os.path.join(self.store_folder, "teststore1"))
# self.config.storage[store.name] = store
#
# backup = Backup("testbackup1")
# backup.include_folders = [self.files_folder]
# backup.store = store.name
# backup.notify_msg = False
# self.config.backups[backup.name] = backup
#
#
# # Run full backup
# # change some files
# # Run incremental backup
# # restore as at full date
# # restore as at incr date
#
#
# except Exception as e:
# log.error("Test run failed: %s" % str(e))
# finally:
# self.cleanup()
log.info("********Success")
def send_email(self, result, error_message=None):
'''
Send a message to the appropriate users.
If result is False (failure) then error message will contain the reason.
@param result:
@param error_message:
'''
log.debug("send_email: ", result, error_message)
if result:
message_text = "Test to folder %s completed." % (self.test_folder)
subject = "Test Completed"
else:
message_text = "Test FAILED\n\nERROR: %s" % (error_message)
subject = "Test FAILED"
log.debug("Starting mail send")
try:
sendemail.sendemail(subject, message_text)
except Exception as e:
self.db.save_message("Unable to email results: %s" % str(e))
log.trace("send_email completed")
def run_cycle_test(self):
options = BlankClass()
options.dry_run = False
options.message = False
options.email = False
options.shutdown = False
options.norecurse = False
# Run a full backup
b = Run("testbackup1", const.FullBackup, options)
b.run()
# Run a full backup
b = Run("testbackup2", const.FullBackup, options)
b.run()
# Now restore two files, one that will be on each store.
restore_file1 = os.path.join(self.files_folder, "dir1", "f2.mp3")
dest_file1 = os.path.join(self.restore_folder, restore_file1[1:])
restore_file2 = os.path.join(self.files_folder, "dir2", "f3.exe")
dest_file2 = os.path.join(self.restore_folder, restore_file2[1:])
restore_file3 = os.path.join(self.files_folder, "dir3", "f4.txt")
dest_file3 = os.path.join(self.restore_folder, restore_file3[1:])
r = Restore(self.restore_folder, [restore_file1, restore_file2, restore_file3],
datetime.now(), options)
r.run()
for path in [dest_file1, dest_file2, dest_file3]:
if not os.path.exists(path):
raise Exception("File %s was not restored" % path)
if open(path).read() != self.teststring1:
raise Exception("Restored file contents incorrect %s" % path)
os.remove(path)
# Make sure the store is the right size
for name in self.config.storage:
store = self.config.storage[name].copy()
size, used, avail = store.current_usage()
log.debug("Store", store.name, "size", size, "used", used, "avail", avail)
if store.auto_manage and used > size:
raise Exception("Store %s has grown too large" % store.name)
######################PART 2
#wait a little
time.sleep(1.1)
for path in [restore_file1, restore_file2, restore_file3]:
# Make a change
with open(path, "w") as f:
f.write(self.teststring2)
#wait a little
time.sleep(1.1)
# Run an incremental backup
b = Run("testbackup1", const.IncrBackup, options)
b.run()
# Run an incremental backup
b = Run("testbackup2", const.IncrBackup, options)
b.run()
time.sleep(1.1)
r = Restore(self.restore_folder, [restore_file1, restore_file2, restore_file3],
datetime.now(), options)
r.run()
for path in [dest_file1, dest_file2, dest_file3]:
if not os.path.exists(path):
raise Exception("File %s was not restored after INCR %s" % path)
if open(path).read() != self.teststring2:
raise Exception("Restored file contents incorrect after INCR %s" % path)
# raise Exception("Test Failure")
# Make sure the store is the right size
for name in self.config.storage:
store = self.config.storage[name].copy()
size, used, avail = store.current_usage()
log.debug("Store", store.name, "size", size, "used", used)
if store.auto_manage and used > size:
raise Exception("Store %s has grown too large" % store.name)
time.sleep(1.1)
# change it back
for path in [restore_file1, restore_file2, restore_file3]:
with open(path, "w") as f:
f.write(self.teststring1)
def build_config(self):
log.trace("build_config")
#store1 = FTPStore("teststore1", "4MB", True, "localhost", "store1", "ftpuser", "ftpuserX9", False)
#store2 = FTPStore("teststore2", "4MB", True, "localhost", "store2", "ftpuser", "ftpuserX9", False)
if self.options.store:
store1 = self.config.storage[self.options.store].copy()
store2 = store1
else:
# Make the store about 3x the options size
s, dummy, dummy = utils.from_readable_form(self.options.size)
store_size = utils.readable_form(s * 3)
store1 = FolderStore("teststore1", store_size, True, os.path.join(self.store_folder, "teststore1"))
store2 = FolderStore("teststore2", store_size, True, os.path.join(self.store_folder, "teststore2"))
self.config.storage[store1.name] = store1
self.config.storage[store2.name] = store2
backup1 = Backup("testbackup1")
backup1.include_folders = [self.files_folder]
backup1.include_packages = True
backup1.exclude_types = ["Music"]
backup1.exclude_patterns = []
backup1.store = store1.name
backup1.notify_msg = False
self.config.backups[backup1.name] = backup1
backup2 = Backup("testbackup2")
backup2.include_folders = [self.files_folder]
backup2.include_packages = True
backup2.exclude_patterns = []
backup1.exclude_types = ["Videos", "Programs"]
backup2.store = store2.name
backup2.notify_msg = False
self.config.backups[backup2.name] = backup2
def fill_files(self, remaining=None, root=None):
log.trace("fill_files")
if not remaining:
# First time in...
remaining, dummy, dummy = utils.from_readable_form(self.options.size)
root = self.files_folder
list = [root]
done = False
while not done:
newlist = []
for folder in list:
for dname in ["dir1", "dir2", "dir3"]:
path = os.path.join(folder, dname)
utils.makedirs(path)
newlist.append(path)
for fname in ["f1.avi", "f2.mp3", "f3.exe", "f4.txt"]:
path = os.path.join(folder, fname)
with open(path, "w") as f:
f.write(self.teststring1)
remaining -= len(self.teststring1)
if remaining < 0:
done = True
break
list = newlist
return
def make_folders(self):
log.trace("make_folders")
if not os.path.isdir(self.test_folder):
os.makedirs(self.test_folder)
# if not os.path.isdir(self.store_folder):
# os.makedirs(self.store_folder)
if not os.path.isdir(self.files_folder):
os.makedirs(self.files_folder)
if not os.path.isdir(self.restore_folder):
os.makedirs(self.restore_folder)
def cleanup(self):
log.trace("Cleanup")
self.db.execute("delete from messages where message_id > ?", (self.max_message_id,))
self.db.execute("delete from versions where version_id > ?", (self.max_version_id,))
self.db.execute("delete from fs where fs_id > ?", (self.max_fs_id,))
self.db.execute("delete from runs where run_id > ?", (self.max_run_id,))
if self.options.store:
stores = self.options.store
else:
stores = ["teststore1", "teststore2"]
for name in stores:
log.info("Cleaning up", name)
store = self.config.storage[name].copy()
store.connect()
store.delete_store_data()
store.disconnect()
if not self.options.store:
del self.config.storage["teststore1"]
del self.config.storage["teststore2"]
del self.config.backups["testbackup1"]
del self.config.backups["testbackup2"]
shutil.rmtree(self.test_folder)
class LongRunTestCase(unittest.TestCase):
def setUp(self):
self.config = Config.get_config()
self.db = DB()
self.db.check_upgrade()
self.mark_db_ids()
self.test_folder = tempfile.mkdtemp()
self.files_folder = os.path.join(self.test_folder, "files")
self.store_folder = os.path.join(self.test_folder, "store")
self.restore_folder = os.path.join(self.test_folder, "restore")
utils.makedirs(self.files_folder)
utils.makedirs(self.store_folder)
utils.makedirs(self.restore_folder)
# Build the base set of files
with open(os.path.join(self.files_folder, "base"), "w") as f:
f.write("base")
with open(os.path.join(self.files_folder, "incr"), "w") as f:
f.write("0")
config_file = os.path.expanduser("~/.vault")
if not os.path.exists(config_file):
raise Exception("Vault test configuration file (~/.vault) does not exist")
self.store_config = ConfigParser.RawConfigParser()
self.store_config.read(config_file)
# FOLDER STORE
self.store = FolderStore("teststore", "50MB", True, self.store_folder)
# DROPBOX STORE
# self.login = self.store_config.get("DropBox", "login")
# self.password = self.store_config.get("DropBox", "password")
# self.folder = self.store_config.get("DropBox", "folder")
# self.app_key = self.store_config.get("DropBox", "app_key")
# self.app_secret_key = self.store_config.get("DropBox", "app_secret_key")
# self.store = DropBoxStore("teststore", 0, False, self.folder, self.login, self.password,
# self.app_key, self.app_secret_key)
# S3 STORE
# self.key = self.store_config.get("Amazon", "aws_access_key_id")
# self.secret_key = self.store_config.get("Amazon", "aws_secret_access_key")
# self.bucket = self.store_config.get("Amazon", "bucket")
# self.store = S3Store("teststore", 0, False, bucket=self.bucket, key=self.key, secret_key=self.secret_key)
# Now record the existance of this store
self.config.storage[self.store.name] = self.store
# Build the backup object (dont save config)
self.backup = Backup("testbackup")
self.backup.include_folders = [self.files_folder]
self.backup.store = self.store.name
self.backup.notify_msg = False
self.old_pass = self.config.data_passphrase
self.config.data_passphrase = "goofy"
self.backup.encrypt = True
self.config.backups[self.backup.name] = self.backup
# build an options object for use with the backup
self.options = BlankClass()
self.options.dry_run = False
self.options.message = False
self.options.email = False
self.options.shutdown = False
self.options.norecurse = False
# How many cycles?
self.cycles = 20
def tearDown(self):
self.config.data_passphrase = self.old_pass
# Remove all DB records created during this test
self.clean_db()
shutil.rmtree(self.test_folder)
self.assertFalse(os.path.isdir(self.test_folder))
def testLongRun(self):
# Run a full backup
b = Run(self.backup.name, const.FullBackup, self.options)
b.run()
for cycle in xrange(self.cycles):
print(str(cycle)+"\r")
time.sleep(1)
# Change some files
with open(os.path.join(self.files_folder, "incr"), "w") as f:
f.write(os.urandom(100))
with open(os.path.join(self.files_folder, str(cycle)), "w") as f:
f.write(os.urandom(100))
# Run an incr backup
b = Run(self.backup.name, const.IncrBackup, self.options)
b.run()
# Attempt to restore every file
r = Restore(self.restore_folder, [self.files_folder],
datetime.now(), self.options)
r.run()
# Lets break it
# os.remove(os.path.join(self.restore_folder, self.files_folder[1:], "1"))
# with open(os.path.join(self.files_folder, "incr"), "w") as f:
# f.write("-1")
# with open(os.path.join(self.restore_folder, self.files_folder[1:], "8"), "w") as f:
# f.write("-1")
# Check that the restored folder and original folder are identical
left = unicode(self.files_folder)
right = unicode(os.path.join(self.restore_folder, self.files_folder[1:]))
d = utils.dircmp(left, right)
self.assertEqual(d.left_only, set())
self.assertEqual(d.right_only, set())
self.assertEqual(d.diff_files, set())
self.assertTrue(len(d.same_files) > 0)
# Check that all files are in the DB
for folder, _, local_files in os.walk(self.files_folder):
for file in local_files:
path = os.path.join(file, folder)
# This will raise an exception if it does not exist
self.db.select_path(path, build=False)
############################################################################
#
# Utility Routines
#
############################################################################
def mark_db_ids(self):
self.max_fs_id = self.db.query("select max(fs_id) from fs", ())[0][0]
if self.max_fs_id is None:
self.max_fs_id = 0
self.max_version_id = self.db.query("select max(version_id) from versions", ())[0][0]
if self.max_version_id is None:
self.max_version_id = 0
self.max_run_id = self.db.query("select max(run_id) from runs", ())[0][0]
if self.max_run_id is None:
self.max_run_id = 0
self.max_message_id = self.db.query("select max(message_id) from messages", ())[0][0]
if self.max_message_id is None:
self.max_message_id = 0
def clean_db(self):
self.db.execute("delete from messages where message_id > ?", (self.max_message_id,))
self.db.execute("delete from versions where version_id > ?", (self.max_version_id,))
self.db.execute("delete from fs where fs_id > ?", (self.max_fs_id,))
self.db.execute("delete from runs where run_id > ?", (self.max_run_id,))
def get_movie(self, url):
try:
sqlList = []
info = {
'id': '',
'name': '',
'date': None,
'score': '0',
'introduction': '',
'download': '',
}
pattern = re.compile(r'.*/(?P<id>\d*)\.html')
info['id'] = re.search(pattern, url).group('id')
if info['id'] in self.idList:
return
html = Grab.get_content(url).replace(
'xmlns="http://www.w3.org/1999/xhtml" /',
'').replace('xmlns="http://www.w3.org/1999/xhtml"',
'').decode('gb2312', 'ignore')
doc = pq(html)
info['download'] = doc('#Zoom table td:first-child').eq(0).text()
name = doc('.bd3r .co_area2 .title_all h1').text()
info['name'] = re.search(r'.*《(.*)》.*', name).group(1)
# name = re.search(r'.*阳光电影www\.ygdy8\.com\.(\W*\d?)\..*', info['download']).group(1).strip()
content = doc('#Zoom p:first-child')
if not content:
return
content = content.remove('br').html().replace(' ', '')
pattern = re.compile(
r'.*◎年代.*(?:.*/)?(?P<date>\d{4}[-年]\d{2}[-月]\d{2}日?)\(.*')
res = re.match(pattern, content)
if res:
info['date'] = res.group('date').strip().replace(
'年', '-').replace('月', '-').replace('日', '-')
pattern = re.compile(r'.*◎豆瓣评分(?P<score>.*)/10 from .* users.*')
res = re.match(pattern, content)
if res:
info['score'] = res.group('score').strip().replace(',', '.')
if not info['score']:
info['score'] = '0'
else:
pattern = re.compile(
r'.*◎IMDb评分(?P<score>.*)/10 from .* users.*', re.I)
res = re.match(pattern, content)
if res:
info['score'] = res.group('score').strip().replace(
',', '.')
if not info['score']:
info['score'] = '0'
pattern = re.compile(r'.*◎简介(?P<introduction>.*)◎获奖情况.*<img.*')
res = re.match(pattern, content)
if res:
info['introduction'] = res.group(
'introduction').strip().replace('\'', '\\\'')
else:
pattern = re.compile(r'.*◎简介(?P<introduction>.*).*<img.*')
res = re.match(pattern, content)
if res:
info['introduction'] = res.group(
'introduction').strip().replace('\'', '\\\'')
if len(info['introduction']) >= 1024:
info['introduction'] = ''
# print(info['introduction'], url)
# return
sql = "select * from bs_movie where name='%s' or id='%s'" % (
info['name'], info['id'])
res = DB.fetchone(sql)
if res:
if int(info['id']) > res[0]:
sql = "delete from bs_movie where id='%s'" % res[0]
sqlList.append(sql)
# self.sqlList.append(sql)
else:
return
if info['date']:
sql = "insert into bs_movie (id,name,date,score,introduction,url,download) values ('%s','%s','%s','%s','%s','%s','%s')" % (
info['id'], info['name'], info['date'], info['score'],
info['introduction'], url, info['download'])
else:
sql = "insert into bs_movie (id,name,date,score,introduction,url,download) values ('%s','%s',NULL,'%s','%s','%s','%s')" % (
info['id'], info['name'], info['score'],
info['introduction'], url, info['download'])
# print(sql)
sqlList.append(sql)
if len(sqlList) == 1:
DB.execute(sqlList[0])
else:
DB.doTrans(sqlList)
# self.sqlList.append(sql)
except Exception as ex:
print(str(ex), url)
class VerifyTestCase(unittest.TestCase):
def setUp(self):
self.config = Config.get_config()
self.db = DB()
self.db.check_upgrade()
self.mark_db_ids()
self.test_folder = tempfile.mkdtemp()
self.files_folder = os.path.join(self.test_folder, "files")
self.store_folder = os.path.join(self.test_folder, "store")
self.restore_folder = os.path.join(self.test_folder, "restore")
utils.makedirs(self.files_folder)
utils.makedirs(self.store_folder)
utils.makedirs(self.restore_folder)
utils.build_file_structure(self.files_folder, 50 * const.Kilobyte, 500 * const.Kilobyte)
# Build a store object (dont save config)
# Note the careful size selection - we want backups to overflow the FolderStore.
self.store = FolderStore("teststore", "2MB", True, self.store_folder)
self.config.storage[self.store.name] = self.store
# Build the backup object (dont save config)
self.backup = Backup("testbackup")
self.backup.include_folders = [self.files_folder]
self.backup.store = self.store.name
self.backup.notify_msg = False
self.include_packages = True
self.config.backups[self.backup.name] = self.backup
# build an options object for use with the backup
self.options = BlankClass()
self.options.dry_run = False
self.options.message = False
self.options.email = False
self.options.shutdown = False
self.options.norecurse = False
self.old_pass = self.config.data_passphrase
self.config.data_passphrase = "banana"
def tearDown(self):
self.config.data_passphrase = self.old_pass
# Remove all DB records created during this test
self.clean_db()
shutil.rmtree(self.test_folder)
self.assertFalse(os.path.isdir(self.test_folder))
def testVerify(self):
# Run a full backup
b = Run("testbackup", const.FullBackup, self.options)
b.run()
# Get the times
runs = self.db.runs("testbackup")
run = runs[0]
v = Verify("testbackup", run.start_time)
self.assertTrue(v.run())
def testBadVerify(self):
# Run a full backup
b = Run("testbackup", const.FullBackup, self.options)
b.run()
# Get the times
runs = self.db.runs("testbackup")
run = runs[0]
# Get the location of the data file from the streamer
streamer = StreamOut(None, self.store, b.backup_folder)
datafile = os.path.join(self.store.root, streamer.get_path(0))
size = os.path.getsize(datafile)
# Now corrupt the data file a little
with open(datafile, "r+b") as f:
f.seek(size // 2, 0)
f.write("X")
v = Verify("testbackup", run.start_time)
self.assertRaises(Exception, v.run)
def testBadConfig(self):
# Run a full backup
b = Run("testbackup", const.FullBackup, self.options)
b.run()
# Get the times
runs = self.db.runs("testbackup")
run = runs[0]
# Delete The Config File
configfile = os.path.join(run.folder, const.ConfigName)
self.store.remove_file(configfile)
v = Verify("testbackup", run.start_time)
self.assertRaises(Exception, v.run)
def testBadVerifyEncrypted(self):
backup = self.config.backups[self.backup.name]
backup.encrypt = True
self.config.backups[backup.name] = backup
# Run a full backup
b = Run("testbackup", const.FullBackup, self.options)
b.run()
# Get the times
runs = self.db.runs("testbackup")
run = runs[0]
# Get the location of the data file from the streamer
streamer = StreamOut(None, self.store, b.backup_folder)
datafile = os.path.join(self.store.root, streamer.get_path(0))
# Now corrupt the data file a little
size = os.path.getsize(datafile)
with open(datafile, "r+b") as f:
f.seek(size // 2, 0)
f.write("X")
v = Verify("testbackup", run.start_time)
self.assertRaises(Exception, v.run)
############################################################################
#
# Utility Routines
#
############################################################################
def mark_db_ids(self):
self.max_fs_id = self.db.query("select max(fs_id) from fs", ())[0][0]
if self.max_fs_id is None:
self.max_fs_id = 0
self.max_version_id = self.db.query("select max(version_id) from versions", ())[0][0]
if self.max_version_id is None:
self.max_version_id = 0
self.max_run_id = self.db.query("select max(run_id) from runs", ())[0][0]
if self.max_run_id is None:
self.max_run_id = 0
self.max_message_id = self.db.query("select max(message_id) from messages", ())[0][0]
if self.max_message_id is None:
self.max_message_id = 0
def clean_db(self):
self.db.execute("delete from messages where message_id > ?", (self.max_message_id,))
self.db.execute("delete from versions where version_id > ?", (self.max_version_id,))
self.db.execute("delete from fs where fs_id > ?", (self.max_fs_id,))
self.db.execute("delete from runs where run_id > ?", (self.max_run_id,))
# Load configuration file and connect to database
with open('config.json') as json_file:
config = json.load(json_file)
db = DB(config['DB'])
# createTables(db)
# Load transaction json file
with open('data.json') as json_file:
transactions = json.load(json_file)['data']
query = """
INSERT INTO TRANSACTIONS (receiver, sender, amount) VALUES ('{0}', '{1}', {2});
"""
for transaction in transactions:
receiver1=transaction['to'][18:][:-24]
sender1=transaction['from'][18:][:-24]
amount1=transaction['value'][:-4]
# db.execute([query.format(str(receiver1),str(sender1),str(amount1))],False);
results = db.execute(["SELECT receiver, sender from transactions"]);
ds = DisjointSet()
for result in results:
ds.add(result[0], result[1])
pprint(ds.group)
class ServerTestCase(unittest.TestCase):
def setUp(self):
self.config = Config.get_config()
self.db = DB()
self.db.check_upgrade()
self.mark_db_ids()
self.test_folder = tempfile.mkdtemp()
self.files_folder = os.path.join(self.test_folder, "files")
self.store_folder = os.path.join(self.test_folder, "store")
self.restore_folder = os.path.join(self.test_folder, "restore")
utils.makedirs(self.files_folder)
utils.makedirs(self.store_folder)
utils.makedirs(self.restore_folder)
utils.build_file_structure(self.files_folder, 50 * const.Kilobyte, 500 * const.Kilobyte)
# Build a store object (dont save the config)
# Note the careful size selection - we want backups to overflow the FolderStore.
self.store = FolderStore("teststore", "2MB", True, self.store_folder)
self.config.storage[self.store.name] = self.store
# Build the backup object (dont save config)
self.backup = Backup("testbackup")
self.backup.include_folders = [self.files_folder]
self.backup.store = self.store.name
self.backup.notify_msg = False
self.old_pass = self.config.data_passphrase
self.config.data_passphrase = "goofy"
self.backup.encrypt = True
self.config.backups[self.backup.name] = self.backup
# build an options object for use with the backup
self.options = BlankClass()
self.options.dry_run = False
self.options.message = False
self.options.email = False
self.options.shutdown = False
self.options.norecurse = False
def tearDown(self):
self.config.data_passphrase = self.old_pass
# Remove all DB records created during this test
self.clean_db()
shutil.rmtree(self.test_folder)
self.assertFalse(os.path.isdir(self.test_folder))
def testBackupRestore(self):
self.backup_restore_compare()
def testCheckFiles(self):
self.backup.include_packages = True
b = Run("testbackup", const.FullBackup, self.options)
b.run()
# Check that all the right files are there.
runs = self.db.runs(self.backup.name)
self.assertEqual(len(runs), 1)
run = runs[0]
folder = run.folder
self.assertTrue(self.store.exists(os.path.join(folder, const.PackageFile + const.EncryptionSuffix)))
self.assertTrue(self.store.exists(os.path.join(folder, const.LOFFile + const.EncryptionSuffix)))
self.assertTrue(self.store.exists(os.path.join(folder, const.ConfigName + const.EncryptionSuffix)))
self.assertTrue(self.store.exists)
def testAutoManagementOfStore1(self):
# Run a set of backups that will overload the store.
# The automanaged store should continue to archive old backups as required.
# Store space reclaimation happens across all backups (i.e. any run).
# We should see older runs from the first backup disappear.
max_size, dummy, dummy = self.store.limit_details()
filesize = utils.du(self.backup.include_folders[0])
# Lets make sure we are going to do enough backups that
# the older ones will be removed.
RunCount = (max_size // filesize) + 2
last_start = None
for cycle in xrange(RunCount):
if last_start:
# Make sure we have ticked to another second since the start of the last backup.
while datetime.now() - last_start < timedelta(seconds=1):
time.sleep(0.01)
backup = Backup(self.backup.name + str(cycle))
backup.include_folders = self.backup.include_folders
backup.store = self.backup.store
backup.notify_msg = False
self.config.backups[backup.name] = backup
# Run a full backup
b = Run(backup.name, const.FullBackup, self.options)
b.run()
last_start = b.start_time
# Assert that the store is still of an appropriate size
size, used, avail = self.store.current_usage()
self.assertTrue(avail >= 0)
self.assertTrue(used <= max_size)
# Confirm that's true on disk
disksize = utils.du(self.store.root)
self.assertTrue(disksize <= max_size)
# Check that some runs have actually been deleted
runs = self.db.runs(self.backup.name + "0")
self.assertTrue(len(runs) == 0)
runs = self.db.runs(self.backup.name + "1")
self.assertTrue(len(runs) == 0)
def testAutoManagementOfStore2(self):
# Run one backup multiple times to overload a store
max_size, dummy, dummy = self.store.limit_details()
filesize = utils.du(self.backup.include_folders[0])
# Lets make sure we are going to do enough backups that
# the older ones will be removed.
RunCount = (max_size // filesize) + 2
last_start = None
for cycle in xrange(RunCount):
if last_start:
# Make sure we have ticked to another second since the start of the last backup.
while datetime.now() - last_start < timedelta(seconds=1):
time.sleep(0.01)
# Run a full backup
b = Run(self.backup.name, const.FullBackup, self.options)
b.run()
last_start = b.start_time
# Assert that the store is still of an appropriate size
size, used, avail = self.store.current_usage()
self.assertTrue(avail >= 0)
self.assertTrue(used <= max_size)
# Confirm that's true on disk
disksize = utils.du(self.store.root)
self.assertTrue(disksize <= max_size)
# Check that some runs have actually been deleted
runs = self.db.runs(self.backup.name)
self.assertTrue(len(runs) < RunCount)
def testChanges(self):
pass
# Full Backup
# change a file
# Incremental backup
# Restore most recent. ensure you get latest file
# Restore to just prior to incremental, ensure you get earlier file
# Run a full backup
file = os.path.join(self.files_folder, "changer")
restore_file = os.path.join(self.restore_folder, file[1:])
# t=0 - file does not exist
b = Run("testbackup", const.FullBackup, self.options)
b.run()
# Make sure we have ticked to another second since the start of the last backup.
while datetime.now() - b.start_time < timedelta(seconds=1):
time.sleep(0.01)
# t=1 - file exists
with open(file, "w") as f:
f.write("1")
b = Run("testbackup", const.IncrBackup, self.options)
b.run()
# Make sure we have ticked to another second since the start of the last backup.
while datetime.now() - b.start_time < timedelta(seconds=1):
time.sleep(0.01)
# t=2 - file changed
with open(file, "w") as f:
f.write("2")
b = Run("testbackup", const.IncrBackup, self.options)
b.run()
# Get the times
runs = self.db.runs("testbackup")
t0 = runs[0].start_time
t1 = runs[1].start_time
t2 = runs[2].start_time
for t, exists, contents in [(t0, False, None), (t1, True, "1"), (t2, True, "2"), (None, True, "2")]:
# Attempt to restore most recent of ALL files
# This tests the default restore.
r = Restore(self.restore_folder, [self.files_folder], t, self.options)
r.run()
if exists:
with open(restore_file, "r") as f:
self.assertEqual(f.read(), contents)
else:
self.assertFalse(os.path.exists(restore_file))
# clean
shutil.rmtree(self.restore_folder)
utils.makedirs(self.restore_folder)
def test7bitFilenames(self):
# Make some 7 bit filenames
strange_folder = os.path.join(self.files_folder, "strange")
utils.makedirs(strange_folder)
for i in xrange(1, 117, 10):
name = "".join([chr(j) for j in xrange(i, i + 10) if chr(j) != "/"])
path = os.path.join(strange_folder, name)
with open(path, "w") as f:
f.write(os.urandom(100))
self.backup_restore_compare()
def testUnicodeFilenames(self):
# Make some unicode bit filenames
# Clean out the ordinary files
shutil.rmtree(self.files_folder)
utils.makedirs(self.files_folder)
unicode_folder = os.path.join(unicode(self.files_folder), u"unicode")
utils.makedirs(unicode_folder)
for i in xrange(1000, 1200, 10):
name = u"".join([unichr(j) for j in xrange(i, i + 10) if unichr(j) != u"/"])
path = os.path.join(unicode_folder, name)
with open(path, "w") as f:
f.write(os.urandom(10))
self.backup_restore_compare()
def backup_restore_compare(self):
# Run a full backup
b = Run("testbackup", const.FullBackup, self.options)
b.run()
# Make sure we have ticked to another second since the start of the last backup.
while datetime.now() - b.start_time < timedelta(seconds=1):
time.sleep(0.01)
# Attempt to restore every file
r = Restore(self.restore_folder, [self.files_folder],
datetime.now(), self.options)
r.run()
# Check that the restored folder and original folder are identical
left = unicode(self.files_folder)
right = unicode(os.path.join(self.restore_folder, self.files_folder[1:]))
d = utils.dircmp(left, right)
self.assertEqual(d.left_only, set())
self.assertEqual(d.right_only, set())
self.assertEqual(d.diff_files, set())
self.assertTrue(len(d.same_files) > 0)
# Check that all files are in the DB
for folder, _, local_files in os.walk(self.files_folder):
for file in local_files:
path = os.path.join(file, folder)
# This will raise an exception if it does not exist
self.db.select_path(path, build=False)
############################################################################
#
# Utility Routines
#
############################################################################
def mark_db_ids(self):
self.max_fs_id = self.db.query("select max(fs_id) from fs", ())[0][0]
if self.max_fs_id is None:
self.max_fs_id = 0
self.max_version_id = self.db.query("select max(version_id) from versions", ())[0][0]
if self.max_version_id is None:
self.max_version_id = 0
self.max_run_id = self.db.query("select max(run_id) from runs", ())[0][0]
if self.max_run_id is None:
self.max_run_id = 0
self.max_message_id = self.db.query("select max(message_id) from messages", ())[0][0]
if self.max_message_id is None:
self.max_message_id = 0
def clean_db(self):
self.db.execute("delete from messages where message_id > ?", (self.max_message_id,))
self.db.execute("delete from versions where version_id > ?", (self.max_version_id,))
self.db.execute("delete from fs where fs_id > ?", (self.max_fs_id,))
self.db.execute("delete from runs where run_id > ?", (self.max_run_id,))
def update_past_issues_processed(self):
"""Update processed issues with current batch name and list of issues processed"""
sql = "INSERT INTO issues_processed (`issue_key`, `batch_name`) VALUES (%s, %s)"
for issue in self.issue_queue:
DB.execute(sql, (issue.key(), self.full_name))
class Monitor(object):
"""
Allows us to actively monitor certain coins on exchanges & updates values to database
"""
def __init__(self, metadata):
self.db = DB()
self.metadata = metadata
#exchanges dict has {exchange_name: exchange_object}
self.exchanges_dict = {}
#init all exchanges we need
for exchange_name, markets in metadata.items():
#init exchange & record
print("Initializing exchange: " + str(exchange_name))
exchange = init_exchange(exchange_name)
self.exchanges_dict[exchange_name] = exchange
#create tables for exchange
self.db.execute("CREATE TABLE IF NOT EXISTS " + exchange_name + "(datestamp TIMESTAMP, ask REAL, bid REAL, market VARCHAR(14), market_sym VARCHAR(14))") #hard code what columns we want to record
#stage changes to DB; commit occurs in update_data
def update_data_exchanges(self, exch_name, exch_obj, exch_markets):
# for each market
temp_store = []
for market in exch_markets:
base_coin = market[0]
quote_coin = market[1]
curr_tries = 1
while (curr_tries <= max_tries):
try:
curr_data = exch_obj.grab_data(base_coin, quote_coin)
break
except Exception as e:
curr_tries += 1
print("Failed to grab data for " + str(exch_name) + " " + str(market) + ". Error: " + str(e))
print(traceback.format_exc())
print("Attempt #: " + str(curr_tries))
temp_store.append((exch_name, curr_data))
return temp_store
def post_data(self, temp_store):
now = datetime.now()
print(len(temp_store))
print(temp_store)
for exch_name, curr_data in temp_store:
insert_query = "INSERT INTO " + exch_name + " values ('{0}', {1}, {2}, '{3}', '{4}')".format(*curr_data)
self.db.execute(insert_query)
self.db.commit()
end = datetime.now()
print("Updated DB in " + str((end-now).total_seconds()) + " s." )
#update database for each exchange & market
def update_data(self):
total_markets = 0
# for each exchange
# create methods dict to run in parallel
methods_arr = []
for exch_name, exch_obj in self.exchanges_dict.items():
curr_markets = self.metadata[exch_name]
methods_arr.append([self.update_data_exchanges, exch_name, exch_obj, curr_markets])
total_markets += len(curr_markets)
#run all exchanges in parrallel
temp_store = run_methods_parallel(methods_arr)
#push data in temp store to database
self.post_data(temp_store)
return total_markets
class Run():
def __init__(self, name, type, options):
'''
Prepare to run a backup event
@param name: name of the backup
@param type: type (Full/Incr)
@param type: dry_run
If dry_run is True, then we will print the files we *would have* backed
up to stdout.
'''
self.type = type
self.dry_run = options.dry_run
self.options = options
self.config = Config.get_config()
try:
self.backup = self.config.backups[name]
except:
raise Exception(_("Backup is missing or corrupt. Please reconfigure backup."))
try:
# Get a fresh store (a copy of the config version
self.store = self.config.storage[self.backup.store].copy()
except:
raise Exception(_("Storage definition is missing. Please reconfigure backup."))
self.db = DB()
self.start_time = None
self.nfiles = None
self.nfolders = None
self.bytes = None
self.run_id = None
self.backup_folder = None
# Make sure there are no other backups running of this name
self.lock = locking.InterProcessLock(name="Vault-%s" % self.backup.name)
# Build a quick file exclusion list, to speed up exclusion checking
self.excl_ext = self.build_excl_exts()
log.debug("Exclusion List:", ",".join(self.excl_ext))
def run(self):
'''
Execute the backup
'''
try:
self.lock.acquire()
except:
msg = _("Backup '%s' is already running. New backup run cannot start") \
% (self.backup.name)
if not self.dry_run:
# Since this is a real backup, we create the run, write to the log and fail immediately.
self.db.start_run(self.backup.name, self.backup.store, self.type, datetime.now())
self.db.save_message(msg)
self.db.update_run_status(const.StatusFailed)
else:
# We dont need to do anything for a dry run. The message will
# be returned to the user.
pass
raise Exception(msg)
# We have the lock now...
try:
self.orig_type = self.type
self.check_old_backups()
self.do_backup()
finally:
self.lock.release()
def check_old_backups(self):
'''
We have got the lock, but if there was a crash, there may be a "running"
state backup left behind. Note that we *know* its not running because
the lock is gone.
Check for it and fail it if there is.
'''
log.debug("Checking for dead backups")
runs = self.db.runs(self.backup.name)
runs = [run for run in runs if run.status == const.StatusRunning]
# It looks like there is a run that is still running.
for run in runs:
log.warn("A prior run crashed. Cleaning up %s/%s" % (run.name, run.start_time_str))
# Get the store
log.debug("Attempting to delete remote run data")
try:
self.store.delete_run_data(run)
except:
pass
# Delete the entries in the database (but not the failed run itself)
# This means the messages will persist, so we can see the usage.
log.debug("Attempting to delete DB run data")
self.db.delete_run_versions(self.run_id)
# Update the status
log.debug("Setting status to failed and logging")
self.db.execute("update runs set status = ? where run_id = ?", (const.StatusFailed, run.run_id))
self.db.execute("insert into messages (run_id, message, time) values (?, ?, ?)",
(run.run_id, _("Backup run crashed. It was cleaned up."),
datetime.now().strftime(const.DateTimeFormat)))
# If there are *no* full backups in the history, then this run MUST be a full,
# even if an incremental is requested. Actually a request for incremental will
# grab all files anyway, but still... lets make the name match the contents.
runs = self.db.runs(self.backup.name)
full_count = len([run for run in runs
if run.type == const.FullBackup and
run.status == const.StatusSuccess])
log.debug("Full backups: %d" % full_count)
if full_count == 0 and self.type != const.FullBackup:
log.debug("Resetting type to Full")
self.type = const.FullBackup
def do_backup(self) :
self.start_time = datetime.now()
self.nfiles = 0
self.nfolders = 0
self.bytes = 0
success = False
message = ""
self.backup_folder = os.path.join(self.backup.name, self.start_time.strftime(const.DateTimeFormat) + " " + self.type)
if not self.dry_run:
self.run_id = self.db.start_run(self.backup.name, self.backup.store, self.type, self.start_time)
msg = _("Backup {server}/{backup}/{type} beginning").format(
server=utils.get_hostname(),
backup=self.backup.name,
type=self.type)
if self.dry_run:
msg += _(" (Dry Run)")
log.info(msg)
self.db.save_message(msg)
if self.orig_type != self.type:
# The backup type was switched
self.db.save_message(_("NOTE: Backup type switched to {newtype} from {oldtype}").format(
newtype=self.type, oldtype=self.orig_type))
# After here we have a run set up in the database, and can begin logging errors.
try:
# Check that if ENCRYPTION is enabled, that there is a password defined.
if self.backup.encrypt and not self.config.data_passphrase:
raise Exception("Backup encryption required, but no passphrase has been configured. Backup cancelled.")
self.prepare_store()
# Prepare output/destinations/encryption
self.prepare_output()
try:
# Now we actually DO the backup, for each listed folder
for folder in self.backup.include_folders:
self.recursive_backup_folder(folder)
log.debug("Committing saved fs entries...")
self.db.fs_saved_commit()
log.debug("Closing...")
self.close_output(success=True)
#raise Exception("Test Exception")
except Exception as e:
log.warn("Exception during backup:", str(e))
# We are going to fail. But we need to try and close
# whatever we can. Closing may fail, but in this case
# we ignore that error.
try:
self.close_output(success=False)
except:
pass
raise e
if self.backup.verify and not self.dry_run:
log.info("Starting verify phase")
msg = _("Backup {server}/{backup}/{type} verification starting").format(
server=utils.get_hostname(),
backup=self.backup.name,
type=self.type)
self.db.save_message(msg)
v = Verify(self.backup.name, self.start_time)
v.run()
msg = _("Backup {server}/{backup}/{type} verification succeeded").format(
server=utils.get_hostname(),
backup=self.backup.name,
type=self.type)
self.db.save_message(msg)
# self.do_verify()
# Messaging...
# If its a dry run, the command line specifies messaging.
# Otherwise both the command line AND backup spec do.
if not self.dry_run:
self.db.update_run_status(const.StatusSuccess)
message = _("Backup {server}/{backup}/{type} completed").format(
server=utils.get_hostname(),
backup=self.backup.name,
type=self.type)
if self.dry_run:
message += " " + _("(Dry Run)")
success = True
if not self.dry_run:
self.db.save_message(message)
except Exception as e:
log.error("Exception in backup. Recording. ", e)
message = _("Backup {server}/{backup}/{type} failed. {error}").format(
server=utils.get_hostname(),
backup=self.backup.name,
type=self.type,
error=str(e))
success = False
if not self.dry_run:
self.db.update_run_status(const.StatusFailed)
# After a failed backup - we must remove the backup data because it
# cannot be trusted.
run = self.db.run_details(self.run_id)
# Delete the remote data
log.debug("Attempting to delete remote run data")
self.store.delete_run_data(run)
# Delete the entries in the database (but not the failed run itself)
# This means the messages will persist, so we can see the usage.
log.debug("Attempting to delete DB run data")
self.db.delete_run_versions(self.run_id)
self.db.save_message(message)
if self.options.message or (self.backup.notify_msg and not self.dry_run):
try:
from lib.dlg import Notify
Notify(const.AppTitle, message)
except Exception as e:
# This one is not really an error... there is probably no-one logged in.
msg = _("Unable to send notification message (no-one logged in)")
if not self.dry_run:
self.db.save_message(message)
log.info(msg)
if self.options.email or (self.backup.notify_email and not self.dry_run):
try:
self.send_email(success, message)
except Exception as e:
msg = _("Unable to email notification message: {error}").format(
error=str(e))
if not self.dry_run:
self.db.save_message(message)
log.error(msg)
if self.options.shutdown or (self.backup.shutdown_after and not self.dry_run):
try:
cmd = ["zenity", "--question",
"--ok-label", _("Shutdown Now"),
"--cancel-label", _("Cancel Shutdown"),
"--text",
_("Backup {backup} complete. Computer will shut down in 2 minutes").format(backup=self.backup.name),
"--timeout", "120"]
status = subprocess.call(cmd)
log.debug("Shutdown query. status=%d" % status)
if status == 0 or status == 5:
print("Running shutdown")
subprocess.Popen(["shutdown", "-P", "now"])
print("Done running shutdown")
except Exception as e:
msg = _("Unable to shutdown PC: {error}").format(
error=str(e))
if not self.dry_run:
self.db.save_message(message)
log.error(msg)
def send_email(self, result, head):
'''
Send a message to the appropriate users.
If result is False (failure) then error message will contain the reason.
@param result:
@param error_message:
'''
log.debug("send_email: ", result, head)
if result:
message_text = head + \
_("\n\nStatistics:\n {files} files backed up.\n "
"{folders} folders backed up.\n {size} copied.\n"
).format(files=self.nfiles, folders=self.nfolders, size=utils.readable_form(self.bytes))
subject = _("Backup {server}/{backup}/{type} completed").format(server=utils.get_hostname(), backup=self.backup.name, type=self.type)
else:
message_text = head
subject = _("Backup {server}/{backup}/{type} failed").format(server=utils.get_hostname(), backup=self.backup.name, type=self.type)
if not self.options.dry_run:
messages = " " + "\n ".join([message.time + " " + message.message for message in self.db.run_messages(self.run_id)])
message_text += _("\nBackup messages:\n") + messages
else:
message_text = "\n"
log.debug("Starting mail send")
try:
sendemail.sendemail(subject, message_text)
except Exception as e:
msg = _("Unable to email results. {error}").format(error=str(e))
if not self.dry_run:
self.db.save_message(msg)
else:
print(msg)
log.trace("send_email completed")
def backup_packages(self):
'''
Build the package list. Then send to the backup server
'''
log.info("Backing up packages")
package_list = utils.get_packages()
f = tempfile.NamedTemporaryFile(delete=False)
f.write("\n".join(package_list))
f.close()
self.copy_file(f.name, const.PackageFile)
os.remove(f.name)
def copy_file(self, path, name=None):
log.debug("CopyFile: ", path, name)
if not name:
name = os.path.basename(path)
if self.dry_run:
print(utils.escape(name))
sys.stdout.flush()
else:
if self.backup.encrypt:
if name:
name = name + ".enc" # Otherwise left as None
enc_path = path + ".enc"
cryptor.encrypt_file(self.config.data_passphrase, path, enc_path)
self.store.send(enc_path, os.path.join(self.backup_folder, name))
os.remove(enc_path)
else:
self.store.send(path, os.path.join(self.backup_folder, name))
def build_excl_exts(self):
list = []
for name in self.backup.exclude_types:
try:
for ext in self.config.file_types[name]:
list.append(ext)
except:
self.db.save_message(_("Exclusion Type %s is no longer recongnised. Ignored.") % name)
return list
def list_to_unicode(self, l):
return [utils.path_to_unicode(p) for p in l]
def recursive_backup_folder(self, root):
'''
Backup a folder and all its sub-folders.
This routine REQUIRES an absolute path.
@param folder:
'''
log.trace("recursive_backup_folder", root)
# Before we interact with the FS - convert to utf-8
root = root.encode('utf-8')
if len(root) == 0:
raise Exception(_("Backup_folder called on empty folder name"))
if root[0] != "/":
raise Exception(_("Backup_folder requires absolute paths"))
for folder, local_folders, local_files in os.walk(root):
# Lets get everything to unicode
# local_folders = self.list_to_unicode(local_folders)
# local_files = self.list_to_unicode(local_files)
log.debug("os.walk", folder, local_folders, local_files)
# First: Check if this is specifically excluded
if self.check_exclusion(folder):
log.info("Excluding Dir:", folder)
continue
log.info("Backing up folder: %s" % folder)
# local_files.sort()
# local_folders.sort()
# Get the data on this folder from the db
db_files = self.db.list_dir(folder)
log.debug("Backing up folder", folder)
log.debug("local files:", local_files)
log.debug("local folders:", local_folders)
log.debug("DB files:", db_files)
for local_file in local_files:
try:
local_path = os.path.join(folder, local_file)
if self.check_backup(local_path, local_file, db_files):
self.do_backup_file(folder, local_file)
except StoreFullException as e:
log.error(str(e))
raise e
except Exception as e:
log.warn("Skipping file %s: %s" % (local_file, str(e)))
# Convert to unicode for checks below...
local_folders = self.list_to_unicode(local_folders)
local_files = self.list_to_unicode(local_files)
# Have backed up all the local files. Now look for DB files
# that exist, but are not local (i.e. they have been deleted)
# Make sure we are only looking for 'F' and 'D' (ignore 'X')
for db_file in db_files.itervalues():
try:
uname = utils.path_to_unicode(db_file.name)
if db_file.type in ['D', 'F'] and not uname in local_files and not uname in local_folders:
self.do_backup_deleted(folder, db_file.name)
except Exception as e:
log.warn("Ignoring exception logging deleted file %s: %s" % (db_file.name, e))
for local_folder in local_folders:
try:
local_path = os.path.join(folder, local_folder)
if self.check_backup(local_path, local_folder, db_files):
self.do_backup_folder(folder, local_folder)
except Exception as e:
log.warn("Ignoring exception backing up folder %s: %s" % (local_path, e))
# # At the completion of a folder - we update the DB storage usage
if not self.dry_run:
self.bytes, self.hash = self.store_thread.get_hash()
self.db.update_run_stats(self.bytes, self.nfiles, self.nfolders, self.backup.include_packages, self.hash)
def lof_record(self, folder, name, type, mod_time=None, size=None):
# Save the entry in the LOF
log.trace("lof_record", folder, name)
if folder != self.lof_folder:
self.lof.write("\n%s\n" % utils.escape(folder))
self.lof_folder = folder
self.lof.write("%s,%s" % (type, utils.escape(name)))
if mod_time:
self.lof.write(',%s,%d' % (mod_time, size))
self.lof.write("\n")
def check_exclusion(self, path):
_, ext = os.path.splitext(path)
ext = ext[1:].lower() # Remove the '.'
# Is this file excluded by type
if ext in self.excl_ext:
return True
# Is this file excluded by filename/folder/glob
ancestors = utils.ancestor_paths(path)
#log.debug("Ancestor Pathlist:", ",".join(ancestors))
for patt in self.backup.exclude_patterns:
for path in ancestors:
if fnmatch.fnmatch(path, patt):
return True
return False
def check_backup(self, path, basename, db_files):
log.trace("check_backup", path, basename)
# Check for exclusions
if self.check_exclusion(path):
log.debug("Excluding", path)
return False
# If this is a full backup, then its included
if self.type == const.FullBackup:
log.debug("Include", path, "because full backup")
return True
# Its an incremental - check timestamps.
# Find it in the database...
if not basename in db_files:
log.debug("Include", path, "because new file")
return True
db_file = db_files[basename]
# Was the last entry a delete? If so, its back and we back it up
if db_file.type == 'X':
log.debug("Include", path, "because file was deleted")
return True
if db_file.mod_time == None:
# The last version record for this file must have been removed.
# This can happen when the storage is too small for a complete
# cycle of FULL and INCREMENTALS. The FULL has been deleted and
# so there are FS entries, but no version entries.
return True
# Check the timestamp
#local_modtime = self.get_file_time_str(path)
local_modtime = datetime.fromtimestamp(os.path.getmtime(path))
# The file datetime is to the nearest microsecond. The DB time is not.
local_modtime -= timedelta(microseconds=local_modtime.microsecond)
log.debug("MOD TIME CHECK: db: %s local: %s" % (str(db_file.mod_time), str(local_modtime)))
if local_modtime > db_file.mod_time:
log.debug("Include", path, "because changed")
return True
# Check if the type has changed
log.debug("TYPE CHECK: db type = %s isfile=%d" % (db_file.type, os.path.isfile(path)))
if db_file.type == 'F' and not os.path.isfile(path):
log.debug("Include", path, "because changed type")
return True
if db_file.type == 'D' and not os.path.isdir(path):
log.debug("Include", path, "because changed type")
return True
# One extra check - just in case. Only for files
if db_file.type == 'F':
size = os.path.getsize(path)
log.debug("SIZE CHECK: db: %s local: %s" % (db_file.size, size))
if size != db_file.size:
log.debug("Include", path, "because changed size")
return True
return False
def get_file_time_str(self, path):
return datetime.fromtimestamp(os.path.getmtime(path)).strftime(const.DateTimeFormat)
def do_backup_file(self, folder, name):
log.trace("do_backup_file", folder, name)
path = os.path.join(folder, name)
if self.dry_run:
print("F - %s" % utils.escape(path))
sys.stdout.flush()
return
# Add it to the tar
try:
# What's the encoding on the file?
upath = utils.path_to_unicode(path)
uname = utils.path_to_unicode(name)
ufolder = utils.path_to_unicode(folder)
# Due to issues with encoding... I'll open the file myself and pass to tar
with open(path, "rb") as f:
info = self.tarfile.gettarinfo(arcname=upath, fileobj=f)
self.tarfile.addfile(info, f)
#self.tarfile.addfile(name=path, recursive=False)
mod_time = self.get_file_time_str(path)
size = os.path.getsize(path)
type = 'F'
self.db.fs_saved(upath, mod_time, size, type)
self.nfiles += 1
# Save the entry in the LOF
self.lof_record(ufolder, uname, "F", mod_time, size)
except Exception as e:
# If the exception is in the store - we crash and burn
# as we cannot save
if self.store_thread.error:
raise self.store_thread.error
# Otherwise log it and keep going...
msg = "Unable to backup %s: %s" % (path, str(e))
self.db.save_message(msg)
log.warn(msg)
def do_backup_folder(self, folder, name):
log.trace("do_backup_folder", folder, name)
path = os.path.join(folder, name)
if self.dry_run:
print("D - %s" % utils.escape(path))
sys.stdout.flush()
return
# We dont need to add it to the
try:
self.tarfile.add(name=path, recursive=False)
mod_time = self.get_file_time_str(path)
size = 0
type = 'D'
self.db.fs_saved(path, mod_time, size, type)
self.nfolders += 1
# Save the entry in the LOF
self.lof_record(folder, name, "D", mod_time, size)
except Exception as e:
log.warn("Exception backing up folder %s: %s" % (path, str(e)))
# If the exception is in the store - we crash and burn
# as we cannot save
if self.store_thread.error:
raise self.store_thread.error
# Otherwise log it and keep going...
msg = "Unable to backup %s: %s" % (path, str(e))
self.db.save_message(msg)
log.warn(msg)
def do_backup_deleted(self, folder, name):
path = os.path.join(folder, name)
if self.dry_run:
print("X - %s" % utils.escape(path))
sys.stdout.flush()
return
try:
log.debug("FILE/FOLDER DELETED:", name)
self.db.fs_deleted(path)
# We keep track of deletions
self.lof_record(folder, name, "X")
except Exception as e:
# If the exception is in the store - we crash and burn
# as we cannot save
if self.store_thread.error:
raise self.store_thread.error
# Otherwise log it and keep going...
msg = "Unable to backup %s: %s" % (path, str(e))
self.db.save_message(msg)
log.warn(msg)
def prepare_store(self):
# Test that the storage is available (this will connect and disconnect)
self.store.test()
# Connect to the store
self.store.connect()
try:
log.info("Preparing store")
# Ensure the store is properly marked and configured.
self.store.setup_store()
if self.backup.include_packages:
self.backup_packages()
# Backup the configuration
self.copy_file(const.ConfigFile, const.ConfigName)
finally:
self.store.disconnect()
def prepare_output(self):
'''
Open the tar file.
Connect the output of the tar to either:
a) the storage handler
b) to encryption (openssl), THEN the storage handler
'''
log.trace("Setting up output processes")
# If we are using an external save command, we do nothing here
if self.dry_run:
log.info("No output processes: Dry run")
return
# Set up the encryptor (use TEE for now)
self.crypt_proc = None
if self.backup.encrypt:
log.debug("Creating crypto stream")
self.crypto = cryptor.EncryptStream(self.config.data_passphrase)
else:
self.crypto = cryptor.Buffer()
# Set up the storage handler
log.debug("Starting storage thread")
self.store_thread = StreamOut(self.crypto, self.store, self.backup_folder)
self.store_thread.start()
# Now set up the tar file which will feed all this
log.debug("Connecting tar object")
self.tarfile = tarfile.open(mode="w|gz", fileobj=self.crypto, format=tarfile.PAX_FORMAT, encoding="utf-8",
dereference=False, bufsize=const.BufferSize)
# Now set up a zipped temp file to record the list of files/folders saved
tmp = tempfile.NamedTemporaryFile(delete=False)
self.lof = gzip.GzipFile(tmp.name, mode="wb", compresslevel=9)
tmp.close()
# Used to keep track of the current folder
self.lof_folder = ""
log.trace("Completed output preparation")
def close_output(self, success):
'''
Close as much as we can. This means we may need to ignore some
errors as we go to ensure everything gets closed.
If success is True, then we have been successful up to this point.
'''
log.trace("Closing output managers")
# If we are using an external save command, we do nothing here
if self.dry_run:
log.info("No output managers: Dry Run")
return
# First exception will be returned.
error = None
try:
self.tarfile.close()
# Tar object tries to write after delete. Seems that in an error state,
# we get more data after the crypto pipe is closed.
# Its an ignorable error, that only occurs when output write fails.
# del self.tarfile
except Exception as e:
if not error:
error = e
try:
# Must manually close this (tarfile wont
# close a file_obj
self.crypto.close()
except Exception as e:
if not error:
error = e
try:
# Now we are ready to wait for the storage.
self.store_thread.join()
if self.store_thread.error:
msg = _("Error saving backup: %s") % str(self.store_thread.error)
log.error(msg)
self.db.save_message(msg)
raise self.store_thread.error
except Exception as e:
# This one we will propogate
error = e
try:
self.lof.close()
if not error and success:
# Send the lof only on success
self.copy_file(self.lof.name, const.LOFFile)
os.remove(self.lof.name)
except Exception as e:
if not error:
error = e
try:
self.store.disconnect()
# Update run data, ONLY if this was successful.
if not error and success:
self.bytes, self.hash = self.store_thread.get_hash()
self.db.update_run_stats(self.bytes, self.nfiles, self.nfolders, self.backup.include_packages, self.hash)
except Exception as e:
if not error:
error = e
log.debug("All output closed. ")
if error:
raise error