Esempio n. 1
0
def createTables(db: DB):

    commands = []

    # Create rent table
    commands.append("""
        CREATE TABLE IF NOT EXISTS addresses  (
        address VARCHAR NOT NULL,
        name VARCHAR NOT NULL,
        created_at TIMESTAMP NOT NULL DEFAULT NOW()
        );
    """)

    db.execute(commands, False)
Esempio n. 2
0
 def run(self, limit, to=0):
     ids = DB.fetchall("SELECT id from bs_movie")
     for id in ids:
         self.idList.append(str(id[0]))
     print('正在抓取....')
     # self.makedir()
     if to == 0:
         nums = range(limit)
     else:
         nums = range(limit, to + 1)
     threads = []
     for i in nums:
         url = self.domain + '/html/gndy/dyzz/list_23_%s.html' % (i + 1)
         th = threading.Thread(target=self.get_page, args=(url, ))
         th.start()
         threads.append(th)
         while len(threads) >= self.count_async / 2:
             for j, _th in enumerate(threads):
                 if not _th.isAlive():
                     threads.pop(j).join()
     while len(threads) > 0:
         for i, _th in enumerate(threads):
             if not _th.isAlive():
                 threads.pop(i).join()
     threads = []
     num = len(self.movies)
     print('共%s' % num)
     for link in self.movies:
         th = threading.Thread(target=self.get_movie, args=(link, ))
         th.start()
         threads.append(th)
         while len(threads) >= self.count_async / 2:
             for j, _th in enumerate(threads):
                 if not _th.isAlive():
                     threads.pop(j).join()
                     num = num - 1
                     print('剩%s' % num)
     while len(threads) > 0:
         for i, _th in enumerate(threads):
             if not _th.isAlive():
                 threads.pop(i).join()
                 num = num - 1
                 print('剩%s' % num)
     # print('正在入库....')
     # if len(self.sqlList) > 0:
     #     DB.doTrans(self.sqlList)
     DB.execute(
         "DELETE from bs_movie where id in (select a.id from (SELECT min(id) as id FROM bs_movie GROUP BY name HAVING count(name) > 1) a)"
     )
Esempio n. 3
0
def createTables(db: DB):

    commands = []

    # Create rent table
    commands.append("""
        CREATE TABLE transactions (
        receiver VARCHAR NOT NULL,
        sender VARCHAR NOT NULL,
        amount DOUBLE precision NOT NULL,
        created_at TIMESTAMP NOT NULL DEFAULT NOW()
        );
    """)

    db.execute(commands, False)
Esempio n. 4
0
class Tester():
    def __init__(self, test_folder, options):

        self.test_folder = os.path.join(test_folder, "tester")
        self.options = options
        self.config = Config.get_config()

        self.store_folder = os.path.join(self.test_folder, "stores")
        self.files_folder = os.path.join(self.test_folder, "files")
        self.restore_folder = os.path.join(self.test_folder, "restore")

        self.db = DB()

        self.max_fs_id = self.db.query("select max(fs_id) from fs", ())[0][0]
        if self.max_fs_id is None:
            self.max_fs_id = 0
        self.max_version_id = self.db.query("select max(version_id) from versions", ())[0][0]
        if self.max_version_id is None:
            self.max_version_id = 0
        self.max_run_id = self.db.query("select max(run_id) from runs", ())[0][0]
        if self.max_run_id is None:
            self.max_run_id = 0
        self.max_message_id = self.db.query("select max(message_id) from messages", ())[0][0]
        if self.max_message_id is None:
            self.max_message_id = 0
        log.debug("MAX IDs", self.max_fs_id, self.max_version_id, self.max_run_id, self.max_message_id)

        self.teststring1 = os.urandom(204800)
        self.teststring2 = os.urandom(204800)


    def run(self):
        try:
            self.simpleCycleTest()
            self.restoreTest()
        except:
            pass
            
        if self.options.message:
            try:
                from lib.dlg import Notify
                Notify(const.AppTitle, "Test run is complete")
            except:
                log.debug("Unable to notify. No-one logged in?")
        if self.options.email:
            self.send_email(True)
        if self.options.shutdown:
            os.system("shutdown -P +2")
        
        
    def simpleCycleTest(self):
        try:
            #   build the test files/folders
            self.make_folders()
            #    fill the files folder with the initial set of files
            self.fill_files()


            #    Build a backup and store
            self.build_config()

#            self.run_test()

            for i in xrange(int(self.options.cycles)):
                log.info("Cycle", i)
                #    Run a full and one incremental 
                self.run_cycle_test()

        except Exception as e:
            log.error("Test run failed: %s" % str(e))
        finally:
            self.cleanup()

        log.info("********Success")

    def restoreTest(self):
        try:
            #   build the test files/folders
            self.make_folders()
            #    fill the files folder with the initial set of files
            self.fill_files()
        except:
            pass

#    def build_config(self):
#        log.trace("build_config")
#        try:
#            if self.options.store:
#                store = self.config.storage[self.options.store]
#            else:
#                #    Make the store about 3x the options size
#                s, dummy, dummy = utils.from_readable_form(self.options.size)
#                store_size = utils.readable_form(s * 3)
#                store = FolderStore("teststore1", store_size, True, os.path.join(self.store_folder, "teststore1"))
#                self.config.storage[store.name] = store
#    
#            backup = Backup("testbackup1")
#            backup.include_folders = [self.files_folder]
#            backup.store = store.name
#            backup.notify_msg = False
#            self.config.backups[backup.name] = backup
#
#
#            #    Run full backup
#            #    change some files
#            #    Run incremental backup
#            #    restore as at full date
#            #    restore as at incr date
#
#
#        except Exception as e:
#            log.error("Test run failed: %s" % str(e))
#        finally:
#            self.cleanup()

        log.info("********Success")
        

    def send_email(self, result, error_message=None):
        '''
        Send a message to the appropriate users.
        If result is False (failure) then error message will contain the reason.
        
        @param result:
        @param error_message:
        '''
        log.debug("send_email: ", result, error_message)
        if result:
            message_text = "Test to folder %s completed." % (self.test_folder)

            subject = "Test Completed"
        else:
            message_text = "Test FAILED\n\nERROR: %s" % (error_message)
            subject = "Test FAILED"

        log.debug("Starting mail send")
        try:
            sendemail.sendemail(subject, message_text)
        except Exception as e:
            self.db.save_message("Unable to email results: %s" % str(e))

        log.trace("send_email completed")

    def run_cycle_test(self):
        options = BlankClass()
        options.dry_run = False
        options.message = False
        options.email = False
        options.shutdown = False
        options.norecurse = False
        #    Run a full backup
        b = Run("testbackup1", const.FullBackup, options)
        b.run()
        #    Run a full backup
        b = Run("testbackup2", const.FullBackup, options)
        b.run()

        #    Now restore two files, one that will be on each store.
        restore_file1 = os.path.join(self.files_folder, "dir1", "f2.mp3")
        dest_file1 = os.path.join(self.restore_folder, restore_file1[1:])
        restore_file2 = os.path.join(self.files_folder, "dir2", "f3.exe")
        dest_file2 = os.path.join(self.restore_folder, restore_file2[1:])
        restore_file3 = os.path.join(self.files_folder, "dir3", "f4.txt")
        dest_file3 = os.path.join(self.restore_folder, restore_file3[1:])
        r = Restore(self.restore_folder, [restore_file1, restore_file2, restore_file3],
                                          datetime.now(), options)
        r.run()

        for path in [dest_file1, dest_file2, dest_file3]:
            if not os.path.exists(path):
                raise Exception("File %s was not restored" % path)
            if open(path).read() != self.teststring1:
                raise Exception("Restored file contents incorrect %s" % path)
            os.remove(path)

        #    Make sure the store is the right size
        for name in self.config.storage:
            store = self.config.storage[name].copy()
            size, used, avail = store.current_usage()
            log.debug("Store", store.name, "size", size, "used", used, "avail", avail)
            if store.auto_manage and used > size:
                raise Exception("Store %s has grown too large" % store.name)

        ######################PART 2

        #wait a little
        time.sleep(1.1)

        for path in [restore_file1, restore_file2, restore_file3]:
            #    Make a change
            with open(path, "w") as f:
                f.write(self.teststring2)

        #wait a little
        time.sleep(1.1)


        #    Run an incremental backup
        b = Run("testbackup1", const.IncrBackup, options)
        b.run()
        #    Run an incremental backup
        b = Run("testbackup2", const.IncrBackup, options)
        b.run()
        
        time.sleep(1.1)

        r = Restore(self.restore_folder, [restore_file1, restore_file2, restore_file3],
                                          datetime.now(), options)
        r.run()

        for path in [dest_file1, dest_file2, dest_file3]:
            if not os.path.exists(path):
                raise Exception("File %s was not restored after INCR %s" % path)
            if open(path).read() != self.teststring2:
                raise Exception("Restored file contents incorrect after INCR %s" % path)

#        raise Exception("Test Failure")

        #    Make sure the store is the right size
        for name in self.config.storage:
            store = self.config.storage[name].copy()
            size, used, avail = store.current_usage()
            log.debug("Store", store.name, "size", size, "used", used)
            if store.auto_manage and used > size:
                raise Exception("Store %s has grown too large" % store.name)

        time.sleep(1.1)


        #    change it back
        for path in [restore_file1, restore_file2, restore_file3]:
            with open(path, "w") as f:
                f.write(self.teststring1)

    def build_config(self):
        log.trace("build_config")
        #store1 = FTPStore("teststore1", "4MB", True, "localhost", "store1", "ftpuser", "ftpuserX9", False)
        #store2 = FTPStore("teststore2", "4MB", True, "localhost", "store2", "ftpuser", "ftpuserX9", False)
        if self.options.store:
            store1 = self.config.storage[self.options.store].copy()
            store2 = store1
        else:
            #    Make the store about 3x the options size
            s, dummy, dummy = utils.from_readable_form(self.options.size)
            store_size = utils.readable_form(s * 3)
            store1 = FolderStore("teststore1", store_size, True, os.path.join(self.store_folder, "teststore1"))
            store2 = FolderStore("teststore2", store_size, True, os.path.join(self.store_folder, "teststore2"))
            self.config.storage[store1.name] = store1
            self.config.storage[store2.name] = store2

        backup1 = Backup("testbackup1")
        backup1.include_folders = [self.files_folder]
        backup1.include_packages = True
        backup1.exclude_types = ["Music"]
        backup1.exclude_patterns = []
        backup1.store = store1.name
        backup1.notify_msg = False
        self.config.backups[backup1.name] = backup1

        backup2 = Backup("testbackup2")
        backup2.include_folders = [self.files_folder]
        backup2.include_packages = True
        backup2.exclude_patterns = []
        backup1.exclude_types = ["Videos", "Programs"]
        backup2.store = store2.name
        backup2.notify_msg = False
        self.config.backups[backup2.name] = backup2

    def fill_files(self, remaining=None, root=None):
        log.trace("fill_files")
        if not remaining:
            #    First time in...
            remaining, dummy, dummy = utils.from_readable_form(self.options.size)
            root = self.files_folder
            
        list = [root]
        done = False
        while not done:
            newlist = []
            for folder in list:
                for dname in ["dir1", "dir2", "dir3"]:
                    path = os.path.join(folder, dname)
                    utils.makedirs(path)
                    newlist.append(path)
                for fname in ["f1.avi", "f2.mp3", "f3.exe", "f4.txt"]:
                    path = os.path.join(folder, fname)
                    with open(path, "w") as f:
                        f.write(self.teststring1)
                    remaining -= len(self.teststring1)
                if remaining < 0:
                    done = True
                    break
            list = newlist
            
        return
                
            


    def make_folders(self):
        log.trace("make_folders")
        if not os.path.isdir(self.test_folder):
            os.makedirs(self.test_folder)

#        if not os.path.isdir(self.store_folder):
#            os.makedirs(self.store_folder)
        if not os.path.isdir(self.files_folder):
            os.makedirs(self.files_folder)
        if not os.path.isdir(self.restore_folder):
            os.makedirs(self.restore_folder)


    def cleanup(self):
        log.trace("Cleanup")
        self.db.execute("delete from messages where message_id > ?", (self.max_message_id,))
        self.db.execute("delete from versions where version_id > ?", (self.max_version_id,))
        self.db.execute("delete from fs where fs_id > ?", (self.max_fs_id,))
        self.db.execute("delete from runs where run_id > ?", (self.max_run_id,))


        if self.options.store:
            stores = self.options.store
        else:
            stores = ["teststore1", "teststore2"]
        for name in stores:
            log.info("Cleaning up", name)
            store = self.config.storage[name].copy()
            store.connect()
            store.delete_store_data()
            store.disconnect()

        if not self.options.store:
            del self.config.storage["teststore1"]
            del self.config.storage["teststore2"]
        del self.config.backups["testbackup1"]
        del self.config.backups["testbackup2"]

        shutil.rmtree(self.test_folder)
Esempio n. 5
0
class LongRunTestCase(unittest.TestCase):

    def setUp(self):
        self.config = Config.get_config()
        self.db = DB()
        self.db.check_upgrade()
        self.mark_db_ids()

        self.test_folder = tempfile.mkdtemp()
        self.files_folder = os.path.join(self.test_folder, "files")
        self.store_folder = os.path.join(self.test_folder, "store")
        self.restore_folder = os.path.join(self.test_folder, "restore")
        utils.makedirs(self.files_folder)
        utils.makedirs(self.store_folder)
        utils.makedirs(self.restore_folder)
        
        #    Build the base set of files
        with open(os.path.join(self.files_folder, "base"), "w") as f:
            f.write("base")

        with open(os.path.join(self.files_folder, "incr"), "w") as f:
            f.write("0")
            
        config_file = os.path.expanduser("~/.vault")
        if not os.path.exists(config_file):
            raise Exception("Vault test configuration file (~/.vault) does not exist")
        self.store_config = ConfigParser.RawConfigParser()
        self.store_config.read(config_file)

    #    FOLDER STORE
        self.store = FolderStore("teststore", "50MB", True, self.store_folder)
    #    DROPBOX STORE
#        self.login = self.store_config.get("DropBox", "login")
#        self.password = self.store_config.get("DropBox", "password")
#        self.folder = self.store_config.get("DropBox", "folder")
#        self.app_key = self.store_config.get("DropBox", "app_key")
#        self.app_secret_key = self.store_config.get("DropBox", "app_secret_key")
#        self.store = DropBoxStore("teststore", 0, False, self.folder, self.login, self.password,
#                                  self.app_key, self.app_secret_key)
    #    S3 STORE
#        self.key = self.store_config.get("Amazon", "aws_access_key_id")
#        self.secret_key = self.store_config.get("Amazon", "aws_secret_access_key")
#        self.bucket = self.store_config.get("Amazon", "bucket")
#        self.store = S3Store("teststore", 0, False, bucket=self.bucket, key=self.key, secret_key=self.secret_key)

        #    Now record the existance of this store
        self.config.storage[self.store.name] = self.store


        #    Build the backup object (dont save config)
        self.backup = Backup("testbackup")
        self.backup.include_folders = [self.files_folder]
        self.backup.store = self.store.name
        self.backup.notify_msg = False
        self.old_pass = self.config.data_passphrase
        self.config.data_passphrase = "goofy"
        self.backup.encrypt = True
        self.config.backups[self.backup.name] = self.backup

        #    build an options object for use with the backup
        self.options = BlankClass()
        self.options.dry_run = False
        self.options.message = False
        self.options.email = False
        self.options.shutdown = False
        self.options.norecurse = False
        
        #    How many cycles?
        self.cycles = 20

    def tearDown(self):
        self.config.data_passphrase = self.old_pass
        #    Remove all DB records created during this test
        self.clean_db()
        shutil.rmtree(self.test_folder)
        self.assertFalse(os.path.isdir(self.test_folder))


    def testLongRun(self):

        #    Run a full backup
        b = Run(self.backup.name, const.FullBackup, self.options)
        b.run()

        for cycle in xrange(self.cycles):
            print(str(cycle)+"\r")
            time.sleep(1)
            
            #    Change some files
            with open(os.path.join(self.files_folder, "incr"), "w") as f:
                f.write(os.urandom(100))

            with open(os.path.join(self.files_folder, str(cycle)), "w") as f:
                f.write(os.urandom(100))
            

            #    Run an incr backup
            b = Run(self.backup.name, const.IncrBackup, self.options)
            b.run()

        
        #    Attempt to restore every file
        r = Restore(self.restore_folder, [self.files_folder],
                                          datetime.now(), self.options)
        r.run()
        
        #    Lets break it
#        os.remove(os.path.join(self.restore_folder, self.files_folder[1:], "1"))
#        with open(os.path.join(self.files_folder, "incr"), "w") as f:
#            f.write("-1")       
#        with open(os.path.join(self.restore_folder, self.files_folder[1:], "8"), "w") as f:
#            f.write("-1")       

        #    Check that the restored folder and original folder are identical
        left = unicode(self.files_folder)
        right = unicode(os.path.join(self.restore_folder, self.files_folder[1:]))
        d = utils.dircmp(left, right)

        self.assertEqual(d.left_only, set())
        self.assertEqual(d.right_only, set())
        self.assertEqual(d.diff_files, set())
        self.assertTrue(len(d.same_files) > 0)

        #    Check that all files are in the DB
        for folder, _, local_files in os.walk(self.files_folder):
            for file in local_files:
                path = os.path.join(file, folder)
                #    This will raise an exception if it does not exist
                self.db.select_path(path, build=False)



############################################################################
#
#    Utility Routines
#
############################################################################

    def mark_db_ids(self):
        self.max_fs_id = self.db.query("select max(fs_id) from fs", ())[0][0]
        if self.max_fs_id is None:
            self.max_fs_id = 0
        self.max_version_id = self.db.query("select max(version_id) from versions", ())[0][0]
        if self.max_version_id is None:
            self.max_version_id = 0
        self.max_run_id = self.db.query("select max(run_id) from runs", ())[0][0]
        if self.max_run_id is None:
            self.max_run_id = 0
        self.max_message_id = self.db.query("select max(message_id) from messages", ())[0][0]
        if self.max_message_id is None:
            self.max_message_id = 0

    def clean_db(self):
        self.db.execute("delete from messages where message_id > ?", (self.max_message_id,))
        self.db.execute("delete from versions where version_id > ?", (self.max_version_id,))
        self.db.execute("delete from fs where fs_id > ?", (self.max_fs_id,))
        self.db.execute("delete from runs where run_id > ?", (self.max_run_id,))
Esempio n. 6
0
 def get_movie(self, url):
     try:
         sqlList = []
         info = {
             'id': '',
             'name': '',
             'date': None,
             'score': '0',
             'introduction': '',
             'download': '',
         }
         pattern = re.compile(r'.*/(?P<id>\d*)\.html')
         info['id'] = re.search(pattern, url).group('id')
         if info['id'] in self.idList:
             return
         html = Grab.get_content(url).replace(
             'xmlns="http://www.w3.org/1999/xhtml" /',
             '').replace('xmlns="http://www.w3.org/1999/xhtml"',
                         '').decode('gb2312', 'ignore')
         doc = pq(html)
         info['download'] = doc('#Zoom table td:first-child').eq(0).text()
         name = doc('.bd3r .co_area2 .title_all h1').text()
         info['name'] = re.search(r'.*《(.*)》.*', name).group(1)
         # name = re.search(r'.*阳光电影www\.ygdy8\.com\.(\W*\d?)\..*', info['download']).group(1).strip()
         content = doc('#Zoom p:first-child')
         if not content:
             return
         content = content.remove('br').html().replace(' ', '')
         pattern = re.compile(
             r'.*◎年代.*(?:.*/)?(?P<date>\d{4}[-年]\d{2}[-月]\d{2}日?)\(.*')
         res = re.match(pattern, content)
         if res:
             info['date'] = res.group('date').strip().replace(
                 '年', '-').replace('月', '-').replace('日', '-')
         pattern = re.compile(r'.*◎豆瓣评分(?P<score>.*)/10 from .* users.*')
         res = re.match(pattern, content)
         if res:
             info['score'] = res.group('score').strip().replace(',', '.')
             if not info['score']:
                 info['score'] = '0'
         else:
             pattern = re.compile(
                 r'.*◎IMDb评分(?P<score>.*)/10 from .* users.*', re.I)
             res = re.match(pattern, content)
             if res:
                 info['score'] = res.group('score').strip().replace(
                     ',', '.')
                 if not info['score']:
                     info['score'] = '0'
         pattern = re.compile(r'.*◎简介(?P<introduction>.*)◎获奖情况.*<img.*')
         res = re.match(pattern, content)
         if res:
             info['introduction'] = res.group(
                 'introduction').strip().replace('\'', '\\\'')
         else:
             pattern = re.compile(r'.*◎简介(?P<introduction>.*).*<img.*')
             res = re.match(pattern, content)
             if res:
                 info['introduction'] = res.group(
                     'introduction').strip().replace('\'', '\\\'')
         if len(info['introduction']) >= 1024:
             info['introduction'] = ''
         # print(info['introduction'], url)
         # return
         sql = "select * from bs_movie where name='%s' or id='%s'" % (
             info['name'], info['id'])
         res = DB.fetchone(sql)
         if res:
             if int(info['id']) > res[0]:
                 sql = "delete from bs_movie where id='%s'" % res[0]
                 sqlList.append(sql)
                 # self.sqlList.append(sql)
             else:
                 return
         if info['date']:
             sql = "insert into bs_movie (id,name,date,score,introduction,url,download) values ('%s','%s','%s','%s','%s','%s','%s')" % (
                 info['id'], info['name'], info['date'], info['score'],
                 info['introduction'], url, info['download'])
         else:
             sql = "insert into bs_movie (id,name,date,score,introduction,url,download) values ('%s','%s',NULL,'%s','%s','%s','%s')" % (
                 info['id'], info['name'], info['score'],
                 info['introduction'], url, info['download'])
         # print(sql)
         sqlList.append(sql)
         if len(sqlList) == 1:
             DB.execute(sqlList[0])
         else:
             DB.doTrans(sqlList)
         # self.sqlList.append(sql)
     except Exception as ex:
         print(str(ex), url)
Esempio n. 7
0
class VerifyTestCase(unittest.TestCase):

    def setUp(self):
        self.config = Config.get_config()

        self.db = DB()
        self.db.check_upgrade()
        self.mark_db_ids()

        self.test_folder = tempfile.mkdtemp()
        self.files_folder = os.path.join(self.test_folder, "files")
        self.store_folder = os.path.join(self.test_folder, "store")
        self.restore_folder = os.path.join(self.test_folder, "restore")
        utils.makedirs(self.files_folder)
        utils.makedirs(self.store_folder)
        utils.makedirs(self.restore_folder)

        utils.build_file_structure(self.files_folder, 50 * const.Kilobyte, 500 * const.Kilobyte)

        #    Build a store object (dont save config)
        #    Note the careful size selection - we want backups to overflow the FolderStore.
        self.store = FolderStore("teststore", "2MB", True, self.store_folder)
        self.config.storage[self.store.name] = self.store

        #    Build the backup object (dont save config)
        self.backup = Backup("testbackup")
        self.backup.include_folders = [self.files_folder]
        self.backup.store = self.store.name
        self.backup.notify_msg = False
        self.include_packages = True
        self.config.backups[self.backup.name] = self.backup

        #    build an options object for use with the backup
        self.options = BlankClass()
        self.options.dry_run = False
        self.options.message = False
        self.options.email = False
        self.options.shutdown = False
        self.options.norecurse = False

        self.old_pass = self.config.data_passphrase
        self.config.data_passphrase = "banana"


    def tearDown(self):
        self.config.data_passphrase = self.old_pass
        #    Remove all DB records created during this test
        self.clean_db()
        shutil.rmtree(self.test_folder)
        self.assertFalse(os.path.isdir(self.test_folder))


    def testVerify(self):
        #    Run a full backup
        b = Run("testbackup", const.FullBackup, self.options)
        b.run()

        #    Get the times
        runs = self.db.runs("testbackup")
        run = runs[0]


        v = Verify("testbackup", run.start_time)
        self.assertTrue(v.run())


    def testBadVerify(self):
        #    Run a full backup
        b = Run("testbackup", const.FullBackup, self.options)
        b.run()

        #    Get the times
        runs = self.db.runs("testbackup")
        run = runs[0]

        #    Get the location of the data file from the streamer
        streamer = StreamOut(None, self.store, b.backup_folder)
        datafile = os.path.join(self.store.root, streamer.get_path(0))
        size = os.path.getsize(datafile)
        #    Now corrupt the data file a little
        with open(datafile, "r+b") as f:
            f.seek(size // 2, 0)
            f.write("X")


        v = Verify("testbackup", run.start_time)
        self.assertRaises(Exception, v.run)


    def testBadConfig(self):
        #    Run a full backup
        b = Run("testbackup", const.FullBackup, self.options)
        b.run()

        #    Get the times
        runs = self.db.runs("testbackup")
        run = runs[0]

        #    Delete The Config File
        configfile = os.path.join(run.folder, const.ConfigName)
        self.store.remove_file(configfile)


        v = Verify("testbackup", run.start_time)
        self.assertRaises(Exception, v.run)

    def testBadVerifyEncrypted(self):
        backup = self.config.backups[self.backup.name]
        backup.encrypt = True
        self.config.backups[backup.name] = backup

        #    Run a full backup
        b = Run("testbackup", const.FullBackup, self.options)
        b.run()

        #    Get the times
        runs = self.db.runs("testbackup")
        run = runs[0]

        #    Get the location of the data file from the streamer
        streamer = StreamOut(None, self.store, b.backup_folder)
        datafile = os.path.join(self.store.root, streamer.get_path(0))
        #    Now corrupt the data file a little
        size = os.path.getsize(datafile)
        with open(datafile, "r+b") as f:
            f.seek(size // 2, 0)
            f.write("X")


        v = Verify("testbackup", run.start_time)
        self.assertRaises(Exception, v.run)



############################################################################
#
#    Utility Routines
#
############################################################################

    def mark_db_ids(self):
        self.max_fs_id = self.db.query("select max(fs_id) from fs", ())[0][0]
        if self.max_fs_id is None:
            self.max_fs_id = 0
        self.max_version_id = self.db.query("select max(version_id) from versions", ())[0][0]
        if self.max_version_id is None:
            self.max_version_id = 0
        self.max_run_id = self.db.query("select max(run_id) from runs", ())[0][0]
        if self.max_run_id is None:
            self.max_run_id = 0
        self.max_message_id = self.db.query("select max(message_id) from messages", ())[0][0]
        if self.max_message_id is None:
            self.max_message_id = 0

    def clean_db(self):
        self.db.execute("delete from messages where message_id > ?", (self.max_message_id,))
        self.db.execute("delete from versions where version_id > ?", (self.max_version_id,))
        self.db.execute("delete from fs where fs_id > ?", (self.max_fs_id,))
        self.db.execute("delete from runs where run_id > ?", (self.max_run_id,))
Esempio n. 8
0
# Load configuration file and connect to database
with open('config.json') as json_file:
    config = json.load(json_file)

db = DB(config['DB'])

# createTables(db)

# Load transaction json file
with open('data.json') as json_file:
    transactions = json.load(json_file)['data']

query = """
        INSERT INTO TRANSACTIONS (receiver, sender, amount) VALUES ('{0}', '{1}', {2});
    """

for transaction in transactions:
    receiver1=transaction['to'][18:][:-24]
    sender1=transaction['from'][18:][:-24]
    amount1=transaction['value'][:-4]

    # db.execute([query.format(str(receiver1),str(sender1),str(amount1))],False);

results = db.execute(["SELECT receiver, sender from transactions"]);

ds = DisjointSet()

for result in results:
    ds.add(result[0], result[1])

pprint(ds.group)
Esempio n. 9
0
class ServerTestCase(unittest.TestCase):

    def setUp(self):
        self.config = Config.get_config()
        self.db = DB()
        self.db.check_upgrade()
        self.mark_db_ids()

        self.test_folder = tempfile.mkdtemp()
        self.files_folder = os.path.join(self.test_folder, "files")
        self.store_folder = os.path.join(self.test_folder, "store")
        self.restore_folder = os.path.join(self.test_folder, "restore")
        utils.makedirs(self.files_folder)
        utils.makedirs(self.store_folder)
        utils.makedirs(self.restore_folder)

        utils.build_file_structure(self.files_folder, 50 * const.Kilobyte, 500 * const.Kilobyte)

        #    Build a store object (dont save the config)
        #    Note the careful size selection - we want backups to overflow the FolderStore.
        self.store = FolderStore("teststore", "2MB", True, self.store_folder)
        self.config.storage[self.store.name] = self.store

        #    Build the backup object (dont save config)
        self.backup = Backup("testbackup")
        self.backup.include_folders = [self.files_folder]
        self.backup.store = self.store.name
        self.backup.notify_msg = False
        self.old_pass = self.config.data_passphrase
        self.config.data_passphrase = "goofy"
        self.backup.encrypt = True
        self.config.backups[self.backup.name] = self.backup

        #    build an options object for use with the backup
        self.options = BlankClass()
        self.options.dry_run = False
        self.options.message = False
        self.options.email = False
        self.options.shutdown = False
        self.options.norecurse = False


    def tearDown(self):
        self.config.data_passphrase = self.old_pass
        #    Remove all DB records created during this test
        self.clean_db()
        shutil.rmtree(self.test_folder)
        self.assertFalse(os.path.isdir(self.test_folder))


    def testBackupRestore(self):
        self.backup_restore_compare()

    def testCheckFiles(self):
        self.backup.include_packages = True
        b = Run("testbackup", const.FullBackup, self.options)
        b.run()
        #    Check that all the right files are there.
        runs = self.db.runs(self.backup.name)
        self.assertEqual(len(runs), 1)
        run = runs[0]
        folder = run.folder
        self.assertTrue(self.store.exists(os.path.join(folder, const.PackageFile + const.EncryptionSuffix)))
        self.assertTrue(self.store.exists(os.path.join(folder, const.LOFFile + const.EncryptionSuffix)))
        self.assertTrue(self.store.exists(os.path.join(folder, const.ConfigName + const.EncryptionSuffix)))

        self.assertTrue(self.store.exists)

    def testAutoManagementOfStore1(self):
        #    Run a set of backups that will overload the store. 
        #    The automanaged store should continue to archive old backups as required.
        #    Store space reclaimation happens across all backups (i.e. any run).
        #    We should see older runs from the first backup disappear.
        max_size, dummy, dummy = self.store.limit_details()

        filesize = utils.du(self.backup.include_folders[0])

        #    Lets make sure we are going to do enough backups that
        #    the older ones will be removed.
        RunCount = (max_size // filesize) + 2


        last_start = None
        for cycle in xrange(RunCount):
            if last_start:
                #    Make sure we have ticked to another second since the start of the last backup.
                while datetime.now() - last_start < timedelta(seconds=1):
                    time.sleep(0.01)

            backup = Backup(self.backup.name + str(cycle))
            backup.include_folders = self.backup.include_folders
            backup.store = self.backup.store
            backup.notify_msg = False
            self.config.backups[backup.name] = backup

            #    Run a full backup
            b = Run(backup.name, const.FullBackup, self.options)
            b.run()
            last_start = b.start_time

            #    Assert that the store is still of an appropriate size
            size, used, avail = self.store.current_usage()
            self.assertTrue(avail >= 0)
            self.assertTrue(used <= max_size)

            #    Confirm that's true on disk
            disksize = utils.du(self.store.root)
            self.assertTrue(disksize <= max_size)


        #    Check that some runs have actually been deleted
        runs = self.db.runs(self.backup.name + "0")
        self.assertTrue(len(runs) == 0)
        runs = self.db.runs(self.backup.name + "1")
        self.assertTrue(len(runs) == 0)

    def testAutoManagementOfStore2(self):
        #    Run one backup multiple times to overload a store
        max_size, dummy, dummy = self.store.limit_details()

        filesize = utils.du(self.backup.include_folders[0])

        #    Lets make sure we are going to do enough backups that
        #    the older ones will be removed.
        RunCount = (max_size // filesize) + 2

        last_start = None
        for cycle in xrange(RunCount):

            if last_start:
                #    Make sure we have ticked to another second since the start of the last backup.
                while datetime.now() - last_start < timedelta(seconds=1):
                    time.sleep(0.01)

            #    Run a full backup
            b = Run(self.backup.name, const.FullBackup, self.options)
            b.run()

            last_start = b.start_time

            #    Assert that the store is still of an appropriate size
            size, used, avail = self.store.current_usage()
            self.assertTrue(avail >= 0)
            self.assertTrue(used <= max_size)

            #    Confirm that's true on disk
            disksize = utils.du(self.store.root)
            self.assertTrue(disksize <= max_size)


        #    Check that some runs have actually been deleted
        runs = self.db.runs(self.backup.name)
        self.assertTrue(len(runs) < RunCount)

    def testChanges(self):
        pass
        #    Full Backup
        #    change a file
        #    Incremental backup
        #    Restore most recent. ensure you get latest file
        #    Restore to just prior to incremental, ensure you get earlier file
        #    Run a full backup
        file = os.path.join(self.files_folder, "changer")
        restore_file = os.path.join(self.restore_folder, file[1:])

        #    t=0 - file does not exist
        b = Run("testbackup", const.FullBackup, self.options)
        b.run()

        #    Make sure we have ticked to another second since the start of the last backup.
        while datetime.now() - b.start_time < timedelta(seconds=1):
            time.sleep(0.01)

        #    t=1 - file exists
        with open(file, "w") as f:
            f.write("1")
        b = Run("testbackup", const.IncrBackup, self.options)
        b.run()

        #    Make sure we have ticked to another second since the start of the last backup.
        while datetime.now() - b.start_time < timedelta(seconds=1):
            time.sleep(0.01)

        #    t=2 - file changed
        with open(file, "w") as f:
            f.write("2")
        b = Run("testbackup", const.IncrBackup, self.options)
        b.run()

        #    Get the times
        runs = self.db.runs("testbackup")
        t0 = runs[0].start_time
        t1 = runs[1].start_time
        t2 = runs[2].start_time

        for t, exists, contents in [(t0, False, None), (t1, True, "1"), (t2, True, "2"), (None, True, "2")]:
            #    Attempt to restore most recent of ALL files
            #    This tests the default restore.
            r = Restore(self.restore_folder, [self.files_folder], t, self.options)
            r.run()
            if exists:
                with open(restore_file, "r") as f:
                    self.assertEqual(f.read(), contents)
            else:
                self.assertFalse(os.path.exists(restore_file))
            #    clean
            shutil.rmtree(self.restore_folder)
            utils.makedirs(self.restore_folder)


    def test7bitFilenames(self):
        #    Make some 7 bit filenames
        strange_folder = os.path.join(self.files_folder, "strange")
        utils.makedirs(strange_folder)
        for i in xrange(1, 117, 10):
            name = "".join([chr(j) for j in xrange(i, i + 10) if chr(j) != "/"])
            path = os.path.join(strange_folder, name)
            with open(path, "w") as f:
                f.write(os.urandom(100))

        self.backup_restore_compare()

    def testUnicodeFilenames(self):
        #    Make some unicode bit filenames
        #    Clean out the ordinary files
        shutil.rmtree(self.files_folder)
        utils.makedirs(self.files_folder)
        unicode_folder = os.path.join(unicode(self.files_folder), u"unicode")
        utils.makedirs(unicode_folder)
        for i in xrange(1000, 1200, 10):
            name = u"".join([unichr(j) for j in xrange(i, i + 10) if unichr(j) != u"/"])
            path = os.path.join(unicode_folder, name)
            with open(path, "w") as f:
                f.write(os.urandom(10))

        self.backup_restore_compare()

    def backup_restore_compare(self):
        #    Run a full backup
        b = Run("testbackup", const.FullBackup, self.options)
        b.run()

        #    Make sure we have ticked to another second since the start of the last backup.
        while datetime.now() - b.start_time < timedelta(seconds=1):
            time.sleep(0.01)


        #    Attempt to restore every file
        r = Restore(self.restore_folder, [self.files_folder],
                                          datetime.now(), self.options)
        r.run()

        #    Check that the restored folder and original folder are identical
        left = unicode(self.files_folder)
        right = unicode(os.path.join(self.restore_folder, self.files_folder[1:]))
        d = utils.dircmp(left, right)

        self.assertEqual(d.left_only, set())
        self.assertEqual(d.right_only, set())
        self.assertEqual(d.diff_files, set())
        self.assertTrue(len(d.same_files) > 0)

        #    Check that all files are in the DB
        for folder, _, local_files in os.walk(self.files_folder):
            for file in local_files:
                path = os.path.join(file, folder)
                #    This will raise an exception if it does not exist
                self.db.select_path(path, build=False)

############################################################################
#
#    Utility Routines
#
############################################################################

    def mark_db_ids(self):
        self.max_fs_id = self.db.query("select max(fs_id) from fs", ())[0][0]
        if self.max_fs_id is None:
            self.max_fs_id = 0
        self.max_version_id = self.db.query("select max(version_id) from versions", ())[0][0]
        if self.max_version_id is None:
            self.max_version_id = 0
        self.max_run_id = self.db.query("select max(run_id) from runs", ())[0][0]
        if self.max_run_id is None:
            self.max_run_id = 0
        self.max_message_id = self.db.query("select max(message_id) from messages", ())[0][0]
        if self.max_message_id is None:
            self.max_message_id = 0

    def clean_db(self):
        self.db.execute("delete from messages where message_id > ?", (self.max_message_id,))
        self.db.execute("delete from versions where version_id > ?", (self.max_version_id,))
        self.db.execute("delete from fs where fs_id > ?", (self.max_fs_id,))
        self.db.execute("delete from runs where run_id > ?", (self.max_run_id,))
 def update_past_issues_processed(self):
   """Update processed issues with current batch name and list of issues processed"""
   sql = "INSERT INTO issues_processed (`issue_key`, `batch_name`) VALUES (%s, %s)"
   for issue in self.issue_queue:
     DB.execute(sql, (issue.key(), self.full_name))
Esempio n. 11
0
class Monitor(object):
    """
    Allows us to actively monitor certain coins on exchanges & updates values to database
    """

    def __init__(self, metadata):
        self.db = DB()
        self.metadata = metadata
        #exchanges dict has {exchange_name: exchange_object}
        self.exchanges_dict = {}

        #init all exchanges we need
        for exchange_name, markets in metadata.items():
            #init exchange & record
            print("Initializing exchange: " + str(exchange_name))
            exchange = init_exchange(exchange_name)
            self.exchanges_dict[exchange_name] = exchange

            #create tables for exchange
            self.db.execute("CREATE TABLE IF NOT EXISTS " + exchange_name + "(datestamp TIMESTAMP, ask REAL, bid REAL, market VARCHAR(14), market_sym VARCHAR(14))") #hard code what columns we want to record

    #stage changes to DB; commit occurs in update_data
    def update_data_exchanges(self, exch_name, exch_obj, exch_markets):
        # for each market
        temp_store = []
        for market in exch_markets:
            base_coin = market[0]
            quote_coin = market[1]
            curr_tries = 1
            while (curr_tries <= max_tries):
                try:
                    curr_data = exch_obj.grab_data(base_coin, quote_coin)
                    break
                except Exception as e:
                    curr_tries += 1
                    print("Failed to grab data for " + str(exch_name) + " " + str(market) + ". Error: " + str(e))
                    print(traceback.format_exc())
                    print("Attempt #: " + str(curr_tries))
            temp_store.append((exch_name, curr_data))
        return temp_store

    def post_data(self, temp_store):
        now = datetime.now()
        print(len(temp_store))
        print(temp_store)
        for exch_name, curr_data in temp_store:
            insert_query = "INSERT INTO " + exch_name + " values ('{0}', {1}, {2}, '{3}', '{4}')".format(*curr_data)
            self.db.execute(insert_query)
        self.db.commit()
        end = datetime.now()
        print("Updated DB in " + str((end-now).total_seconds()) + " s." )


    #update database for each exchange & market 
    def update_data(self):
        total_markets = 0
        # for each exchange
        # create methods dict to run in parallel
        methods_arr = []
        for exch_name, exch_obj in self.exchanges_dict.items():
            curr_markets = self.metadata[exch_name]
            methods_arr.append([self.update_data_exchanges, exch_name, exch_obj, curr_markets])
            total_markets += len(curr_markets)
        #run all exchanges in parrallel
        temp_store = run_methods_parallel(methods_arr)
        #push data in temp store to database
        self.post_data(temp_store)
        return total_markets
Esempio n. 12
0
File: run.py Progetto: tussock/Vault
class Run():
    def __init__(self, name, type, options):
        '''
        Prepare to run a backup event
        
        @param name: name of the backup
        @param type: type (Full/Incr)
        @param type: dry_run
        
        If dry_run is True, then we will print the files we *would have* backed
        up to stdout.
        
        '''
        self.type = type
        self.dry_run = options.dry_run
        self.options = options
        self.config = Config.get_config()

        try:
            self.backup = self.config.backups[name]
        except:
            raise Exception(_("Backup is missing or corrupt. Please reconfigure backup."))

        try:
            #    Get a fresh store (a copy of the config version
            self.store = self.config.storage[self.backup.store].copy()
        except:
            raise Exception(_("Storage definition is missing. Please reconfigure backup."))

        self.db = DB()
        self.start_time = None
        self.nfiles = None
        self.nfolders = None
        self.bytes = None
        self.run_id = None
        self.backup_folder = None

        #    Make sure there are no other backups running of this name
        self.lock = locking.InterProcessLock(name="Vault-%s" % self.backup.name)

        #    Build a quick file exclusion list, to speed up exclusion checking
        self.excl_ext = self.build_excl_exts()
        log.debug("Exclusion List:", ",".join(self.excl_ext))


    def run(self):
        '''
        Execute the backup
        '''
        try:
            self.lock.acquire()
        except:
            msg = _("Backup '%s' is already running. New backup run cannot start") \
                    % (self.backup.name)

            if not self.dry_run:
                #    Since this is a real backup, we create the run, write to the log and fail immediately.
                self.db.start_run(self.backup.name, self.backup.store, self.type, datetime.now())
                self.db.save_message(msg)
                self.db.update_run_status(const.StatusFailed)
            else:
                #    We dont need to do anything for a dry run. The message will
                #    be returned to the user.
                pass
            raise Exception(msg)

        #    We have the lock now...
        try:
            self.orig_type = self.type
            self.check_old_backups()
            self.do_backup()
        finally:
            self.lock.release()

    def check_old_backups(self):
        '''
        We have got the lock, but if there was a crash, there may be a "running"
        state backup left behind. Note that we *know* its not running because
        the lock is gone.
        
        Check for it and fail it if there is.
        '''
        log.debug("Checking for dead backups")
        runs = self.db.runs(self.backup.name)
        runs = [run for run in runs if run.status == const.StatusRunning]

        #    It looks like there is a run that is still running.
        for run in runs:
            log.warn("A prior run crashed. Cleaning up %s/%s" % (run.name, run.start_time_str))
            #    Get the store
            log.debug("Attempting to delete remote run data")
            try:
                self.store.delete_run_data(run)
            except:
                pass
            #    Delete the entries in the database (but not the failed run itself)
            #    This means the messages will persist, so we can see the usage.
            log.debug("Attempting to delete DB run data")
            self.db.delete_run_versions(self.run_id)
            #    Update the status
            log.debug("Setting status to failed and logging")
            self.db.execute("update runs set status = ? where run_id = ?", (const.StatusFailed, run.run_id))
            self.db.execute("insert into messages (run_id, message, time) values (?, ?, ?)",
                            (run.run_id, _("Backup run crashed. It was cleaned up."),
                             datetime.now().strftime(const.DateTimeFormat)))


        #        If there are *no* full backups in the history, then this run MUST be a full,
        #        even if an incremental is requested. Actually a request for incremental will
        #        grab all files anyway, but still... lets make the name match the contents.
        runs = self.db.runs(self.backup.name)
        full_count = len([run for run in runs
                            if run.type == const.FullBackup and
                            run.status == const.StatusSuccess])
        log.debug("Full backups: %d" % full_count)
        if full_count == 0 and self.type != const.FullBackup:
            log.debug("Resetting type to Full")
            self.type = const.FullBackup


    def do_backup(self)      :
        self.start_time = datetime.now()
        self.nfiles = 0
        self.nfolders = 0
        self.bytes = 0
        success = False
        message = ""

        self.backup_folder = os.path.join(self.backup.name, self.start_time.strftime(const.DateTimeFormat) + " " + self.type)
        if not self.dry_run:
            self.run_id = self.db.start_run(self.backup.name, self.backup.store, self.type, self.start_time)
            msg = _("Backup {server}/{backup}/{type} beginning").format(
                                                            server=utils.get_hostname(), 
                                                            backup=self.backup.name, 
                                                            type=self.type)
            if self.dry_run:
                msg += _(" (Dry Run)")
            log.info(msg)
            self.db.save_message(msg)
            if self.orig_type != self.type:
                #    The backup type was switched
                self.db.save_message(_("NOTE: Backup type switched to {newtype} from {oldtype}").format(
                                                        newtype=self.type, oldtype=self.orig_type))

        #    After here we have a run set up in the database, and can begin logging errors.
        try:
            #    Check that if ENCRYPTION is enabled, that there is a password defined.
            if self.backup.encrypt and not self.config.data_passphrase:
                raise Exception("Backup encryption required, but no passphrase has been configured. Backup cancelled.")
            
            self.prepare_store()

            #    Prepare output/destinations/encryption
            self.prepare_output()
            try:
                #    Now we actually DO the backup, for each listed folder
                for folder in self.backup.include_folders:
                    self.recursive_backup_folder(folder)

                log.debug("Committing saved fs entries...")
                self.db.fs_saved_commit()

                log.debug("Closing...")
                self.close_output(success=True)
            #raise Exception("Test Exception")
            except Exception as e:
                log.warn("Exception during backup:", str(e))
                #    We are going to fail. But we need to try and close
                #    whatever we can. Closing may fail, but in this case
                #    we ignore that error.
                try:
                    self.close_output(success=False)
                except:
                    pass
                raise e

            if self.backup.verify and not self.dry_run:
                log.info("Starting verify phase")
                msg = _("Backup {server}/{backup}/{type} verification starting").format(
                                                            server=utils.get_hostname(), 
                                                            backup=self.backup.name, 
                                                            type=self.type)
                
                self.db.save_message(msg)
                v = Verify(self.backup.name, self.start_time)
                v.run()
                msg = _("Backup {server}/{backup}/{type} verification succeeded").format(
                                                            server=utils.get_hostname(), 
                                                            backup=self.backup.name, 
                                                            type=self.type)
                self.db.save_message(msg)
#                self.do_verify()

            #    Messaging...
            #    If its a dry run, the command line specifies messaging.
            #    Otherwise both the command line AND backup spec do.
            if not self.dry_run:
                self.db.update_run_status(const.StatusSuccess)
            message = _("Backup {server}/{backup}/{type} completed").format(
                                                            server=utils.get_hostname(), 
                                                            backup=self.backup.name, 
                                                            type=self.type)
            if self.dry_run:
                message += " " + _("(Dry Run)")
            success = True
            if not self.dry_run:
                self.db.save_message(message)

        except Exception as e:
            log.error("Exception in backup. Recording. ", e)
            message = _("Backup {server}/{backup}/{type} failed. {error}").format(
                                                            server=utils.get_hostname(), 
                                                            backup=self.backup.name, 
                                                            type=self.type, 
                                                            error=str(e))
            success = False
            if not self.dry_run:
                self.db.update_run_status(const.StatusFailed)

                #    After a failed backup - we must remove the backup data because it 
                #    cannot be trusted.
                run = self.db.run_details(self.run_id)
                #    Delete the remote data
                log.debug("Attempting to delete remote run data")
                self.store.delete_run_data(run)
                #    Delete the entries in the database (but not the failed run itself)
                #    This means the messages will persist, so we can see the usage.
                log.debug("Attempting to delete DB run data")
                self.db.delete_run_versions(self.run_id)

                self.db.save_message(message)

        if self.options.message or (self.backup.notify_msg and not self.dry_run):
            try:
                from lib.dlg import Notify
                Notify(const.AppTitle, message)
            except Exception as e:
                #    This one is not really an error... there is probably no-one logged in.
                msg = _("Unable to send notification message (no-one logged in)")
                if not self.dry_run:
                    self.db.save_message(message)
                log.info(msg)

        if self.options.email or (self.backup.notify_email and not self.dry_run):
            try:
                self.send_email(success, message)
            except Exception as e:
                msg = _("Unable to email notification message: {error}").format(
                                                error=str(e))
                if not self.dry_run:
                    self.db.save_message(message)
                log.error(msg)
        if self.options.shutdown or (self.backup.shutdown_after and not self.dry_run):
            try:
                cmd = ["zenity", "--question",
                       "--ok-label", _("Shutdown Now"),
                       "--cancel-label", _("Cancel Shutdown"),
                       "--text",
                       _("Backup {backup} complete. Computer will shut down in 2 minutes").format(backup=self.backup.name),
                       "--timeout", "120"]
                status = subprocess.call(cmd)
                log.debug("Shutdown query. status=%d" % status)
                if status == 0 or status == 5:
                    print("Running shutdown")
                    subprocess.Popen(["shutdown", "-P", "now"])
                    print("Done running shutdown")
            except Exception as e:
                msg = _("Unable to shutdown PC: {error}").format(
                                                error=str(e))
                if not self.dry_run:
                    self.db.save_message(message)
                log.error(msg)

    def send_email(self, result, head):

        '''
        Send a message to the appropriate users.
        If result is False (failure) then error message will contain the reason.
        
        @param result:
        @param error_message:
        '''
        log.debug("send_email: ", result, head)
        if result:
            message_text = head + \
                    _("\n\nStatistics:\n    {files} files backed up.\n    "
                      "{folders} folders backed up.\n    {size} copied.\n"
                      ).format(files=self.nfiles, folders=self.nfolders, size=utils.readable_form(self.bytes))

            subject = _("Backup {server}/{backup}/{type} completed").format(server=utils.get_hostname(), backup=self.backup.name, type=self.type)
        else:
            message_text = head
            subject = _("Backup {server}/{backup}/{type} failed").format(server=utils.get_hostname(), backup=self.backup.name, type=self.type)

        if not self.options.dry_run:
            messages = "    " + "\n    ".join([message.time + " " + message.message for message in self.db.run_messages(self.run_id)])
            message_text += _("\nBackup messages:\n") + messages
        else:
            message_text = "\n"

        log.debug("Starting mail send")
        try:
            sendemail.sendemail(subject, message_text)
        except Exception as e:
            msg = _("Unable to email results. {error}").format(error=str(e))
            if not self.dry_run:
                self.db.save_message(msg)
            else:
                print(msg)

        log.trace("send_email completed")



    def backup_packages(self):
        '''
        Build the package list. Then send to the backup server
        '''
        log.info("Backing up packages")
        package_list = utils.get_packages()
        f = tempfile.NamedTemporaryFile(delete=False)
        f.write("\n".join(package_list))
        f.close()

        self.copy_file(f.name, const.PackageFile)
        os.remove(f.name)


    def copy_file(self, path, name=None):
        log.debug("CopyFile: ", path, name)
        if not name:
            name = os.path.basename(path)
        if self.dry_run:
            print(utils.escape(name))
            sys.stdout.flush()
        else:
            if self.backup.encrypt:
                if name:
                    name = name + ".enc"    #    Otherwise left as None
                enc_path = path + ".enc"
                cryptor.encrypt_file(self.config.data_passphrase, path, enc_path)
                self.store.send(enc_path, os.path.join(self.backup_folder, name))
                os.remove(enc_path)
            else:
                self.store.send(path, os.path.join(self.backup_folder, name))


    def build_excl_exts(self):
        list = []
        for name in self.backup.exclude_types:
            try:
                for ext in self.config.file_types[name]:
                    list.append(ext)
            except:
                self.db.save_message(_("Exclusion Type %s is no longer recongnised. Ignored.") % name)
        return list

    def list_to_unicode(self, l):
        return [utils.path_to_unicode(p) for p in l]




    def recursive_backup_folder(self, root):
        '''
        Backup a folder and all its sub-folders.
        This routine REQUIRES an absolute path.
        
        @param folder:
        '''
        log.trace("recursive_backup_folder", root)
        #    Before we interact with the FS - convert to utf-8
        root = root.encode('utf-8')
        if len(root) == 0:
            raise Exception(_("Backup_folder called on empty folder name"))
        if root[0] != "/":
            raise Exception(_("Backup_folder requires absolute paths"))
        for folder, local_folders, local_files in os.walk(root):
            #    Lets get everything to unicode
#            local_folders = self.list_to_unicode(local_folders)
#            local_files = self.list_to_unicode(local_files)
            log.debug("os.walk", folder, local_folders, local_files)
            #    First: Check if this is specifically excluded
            if self.check_exclusion(folder):
                log.info("Excluding Dir:", folder)
                continue

            log.info("Backing up folder: %s" % folder)

#            local_files.sort()
#            local_folders.sort()

            #    Get the data on this folder from the db
            db_files = self.db.list_dir(folder)
            log.debug("Backing up folder", folder)
            log.debug("local files:", local_files)
            log.debug("local folders:", local_folders)
            log.debug("DB files:", db_files)

            for local_file in local_files:
                try:
                    local_path = os.path.join(folder, local_file)
                    if self.check_backup(local_path, local_file, db_files):
                        self.do_backup_file(folder, local_file)
                except StoreFullException as e:
                    log.error(str(e))
                    raise e
                except Exception as e:
                    log.warn("Skipping file %s: %s" % (local_file, str(e)))

            #    Convert to unicode for checks below...
            local_folders = self.list_to_unicode(local_folders)
            local_files = self.list_to_unicode(local_files)
            #    Have backed up all the local files. Now look for DB files
            #    that exist, but are not local (i.e. they have been deleted)
            #    Make sure we are only looking for 'F' and 'D' (ignore 'X')
            for db_file in db_files.itervalues():
                try:
                    uname = utils.path_to_unicode(db_file.name)
                    if db_file.type in ['D', 'F'] and not uname in local_files and not uname in local_folders:
                        self.do_backup_deleted(folder, db_file.name)
                except Exception as e:
                    log.warn("Ignoring exception logging deleted file %s: %s" % (db_file.name, e))

            for local_folder in local_folders:
                try:
                    local_path = os.path.join(folder, local_folder)
                    if self.check_backup(local_path, local_folder, db_files):
                        self.do_backup_folder(folder, local_folder)
                except Exception as e:
                    log.warn("Ignoring exception backing up folder %s: %s" % (local_path, e))

#            #    At the completion of a folder - we update the DB storage usage
            if not self.dry_run:
                self.bytes, self.hash = self.store_thread.get_hash()
                self.db.update_run_stats(self.bytes, self.nfiles, self.nfolders, self.backup.include_packages, self.hash)

    def lof_record(self, folder, name, type, mod_time=None, size=None):
        #    Save the entry in the LOF
        log.trace("lof_record", folder, name)
        if folder != self.lof_folder:
            self.lof.write("\n%s\n" % utils.escape(folder))
            self.lof_folder = folder
        self.lof.write("%s,%s" % (type, utils.escape(name)))
        if mod_time:
            self.lof.write(',%s,%d' % (mod_time, size))
        self.lof.write("\n")


    def check_exclusion(self, path):
        _, ext = os.path.splitext(path)
        ext = ext[1:].lower()           #    Remove the '.'
        #    Is this file excluded by type
        if ext in self.excl_ext:
            return True

        #    Is this file excluded by filename/folder/glob
        ancestors = utils.ancestor_paths(path)
        #log.debug("Ancestor Pathlist:", ",".join(ancestors))
        for patt in self.backup.exclude_patterns:
            for path in ancestors:
                if fnmatch.fnmatch(path, patt):
                    return True

        return False



    def check_backup(self, path, basename, db_files):
        log.trace("check_backup", path, basename)
        #    Check for exclusions
        if self.check_exclusion(path):
            log.debug("Excluding", path)
            return False

        #    If this is a full backup, then its included
        if self.type == const.FullBackup:
            log.debug("Include", path, "because full backup")
            return True

        #    Its an incremental - check timestamps.
        #    Find it in the database...
        if not basename in db_files:
            log.debug("Include", path, "because new file")
            return True

        db_file = db_files[basename]
        #    Was the last entry a delete? If so, its back and we back it up
        if db_file.type == 'X':
            log.debug("Include", path, "because file was deleted")
            return True

        if db_file.mod_time == None:
            #    The last version record for this file must have been removed.
            #    This can happen when the storage is too small for a complete
            #    cycle of FULL and INCREMENTALS. The FULL has been deleted and
            #    so there are FS entries, but no version entries.
            return True

        #    Check the timestamp
        #local_modtime = self.get_file_time_str(path)
        local_modtime = datetime.fromtimestamp(os.path.getmtime(path))
        #    The file datetime is to the nearest microsecond. The DB time is not.
        local_modtime -= timedelta(microseconds=local_modtime.microsecond)
        log.debug("MOD TIME CHECK: db: %s local: %s" % (str(db_file.mod_time), str(local_modtime)))
        if local_modtime > db_file.mod_time:
            log.debug("Include", path, "because changed")
            return True

        #    Check if the type has changed
        log.debug("TYPE CHECK: db type = %s isfile=%d" % (db_file.type, os.path.isfile(path)))
        if db_file.type == 'F' and not os.path.isfile(path):
            log.debug("Include", path, "because changed type")
            return True
        if db_file.type == 'D' and not os.path.isdir(path):
            log.debug("Include", path, "because changed type")
            return True

        #    One extra check - just in case. Only for files
        if db_file.type == 'F':
            size = os.path.getsize(path)
            log.debug("SIZE CHECK: db: %s local: %s" % (db_file.size, size))
            if size != db_file.size:
                log.debug("Include", path, "because changed size")
                return True

        return False

    def get_file_time_str(self, path):
        return datetime.fromtimestamp(os.path.getmtime(path)).strftime(const.DateTimeFormat)

    def do_backup_file(self, folder, name):
        log.trace("do_backup_file", folder, name)
        path = os.path.join(folder, name)
        if self.dry_run:
            print("F - %s" % utils.escape(path))
            sys.stdout.flush()
            return

        #    Add it to the tar
        try:
            #    What's the encoding on the file?
            upath = utils.path_to_unicode(path)
            uname = utils.path_to_unicode(name)
            ufolder = utils.path_to_unicode(folder)
            #    Due to issues with encoding... I'll open the file myself and pass to tar
            with open(path, "rb") as f:
                info = self.tarfile.gettarinfo(arcname=upath, fileobj=f)
                self.tarfile.addfile(info, f)
            #self.tarfile.addfile(name=path, recursive=False)

            mod_time = self.get_file_time_str(path)
            size = os.path.getsize(path)
            type = 'F'
            self.db.fs_saved(upath, mod_time, size, type)
            self.nfiles += 1

            #    Save the entry in the LOF
            self.lof_record(ufolder, uname, "F", mod_time, size)

        except Exception as e:
            #    If the exception is in the store - we crash and burn
            #    as we cannot save
            if self.store_thread.error:
                raise self.store_thread.error

            #    Otherwise log it and keep going...
            msg = "Unable to backup %s: %s" % (path, str(e))
            self.db.save_message(msg)
            log.warn(msg)

    def do_backup_folder(self, folder, name):
        log.trace("do_backup_folder", folder, name)
        path = os.path.join(folder, name)
        if self.dry_run:
            print("D - %s" % utils.escape(path))
            sys.stdout.flush()
            return

        #    We dont need to add it to the 
        try:
            self.tarfile.add(name=path, recursive=False)
            mod_time = self.get_file_time_str(path)
            size = 0
            type = 'D'
            self.db.fs_saved(path, mod_time, size, type)
            self.nfolders += 1

            #    Save the entry in the LOF
            self.lof_record(folder, name, "D", mod_time, size)
        except Exception as e:
            log.warn("Exception backing up folder %s: %s" % (path, str(e)))
            #    If the exception is in the store - we crash and burn
            #    as we cannot save
            if self.store_thread.error:
                raise self.store_thread.error

            #    Otherwise log it and keep going...
            msg = "Unable to backup %s: %s" % (path, str(e))
            self.db.save_message(msg)
            log.warn(msg)

    def do_backup_deleted(self, folder, name):
        path = os.path.join(folder, name)
        if self.dry_run:
            print("X - %s" % utils.escape(path))
            sys.stdout.flush()
            return
        try:
            log.debug("FILE/FOLDER DELETED:", name)
            self.db.fs_deleted(path)
            #    We keep track of deletions
            self.lof_record(folder, name, "X")
        except Exception as e:
            #    If the exception is in the store - we crash and burn
            #    as we cannot save
            if self.store_thread.error:
                raise self.store_thread.error

            #    Otherwise log it and keep going...
            msg = "Unable to backup %s: %s" % (path, str(e))
            self.db.save_message(msg)
            log.warn(msg)


    def prepare_store(self):
        #    Test that the storage is available (this will connect and disconnect)
        self.store.test()

        #    Connect to the store
        self.store.connect()

        try:
            log.info("Preparing store")
            #    Ensure the store is properly marked and configured.
            self.store.setup_store()

            if self.backup.include_packages:
                self.backup_packages()

            #    Backup the configuration
            self.copy_file(const.ConfigFile, const.ConfigName)
        finally:
            self.store.disconnect()


    def prepare_output(self):
        '''
        Open the tar file.
        Connect the output of the tar to either:
        a) the storage handler
        b) to encryption (openssl), THEN the storage handler
        
        '''
        log.trace("Setting up output processes")

        #    If we are using an external save command, we do nothing here
        if self.dry_run:
            log.info("No output processes: Dry run")
            return

        #    Set up the encryptor (use TEE for now)
        self.crypt_proc = None
        if self.backup.encrypt:
            log.debug("Creating crypto stream")

            self.crypto = cryptor.EncryptStream(self.config.data_passphrase)

        else:
            self.crypto = cryptor.Buffer()

        #    Set up the storage handler
        log.debug("Starting storage thread")

        self.store_thread = StreamOut(self.crypto, self.store, self.backup_folder)
        self.store_thread.start()

        #    Now set up the tar file which will feed all this
        log.debug("Connecting tar object")
        self.tarfile = tarfile.open(mode="w|gz", fileobj=self.crypto, format=tarfile.PAX_FORMAT, encoding="utf-8",
                                    dereference=False, bufsize=const.BufferSize)

        #    Now set up a zipped temp file to record the list of files/folders saved
        tmp = tempfile.NamedTemporaryFile(delete=False)
        self.lof = gzip.GzipFile(tmp.name, mode="wb", compresslevel=9)
        tmp.close()
        #    Used to keep track of the current folder
        self.lof_folder = ""

        log.trace("Completed output preparation")

    def close_output(self, success):
        '''
        Close as much as we can. This means we may need to ignore some
        errors as we go to ensure everything gets closed.
        
        If success is True, then we have been successful up to this point.
        '''
        log.trace("Closing output managers")
        #    If we are using an external save command, we do nothing here
        if self.dry_run:
            log.info("No output managers: Dry Run")
            return
        #    First exception will be returned.
        error = None
        try:
            self.tarfile.close()
            #    Tar object tries to write after delete. Seems that in an error state,
            #    we get more data after the crypto pipe is closed.
            #    Its an ignorable error, that only occurs when output write fails.
#            del self.tarfile            
        except Exception as e:
            if not error:
                error = e
        try:
            #    Must manually close this (tarfile wont
            #    close a file_obj
            self.crypto.close()
        except Exception as e:
            if not error:
                error = e
        try:
            #    Now we are ready to wait for the storage.
            self.store_thread.join()
            if self.store_thread.error:
                msg = _("Error saving backup: %s") % str(self.store_thread.error)
                log.error(msg)
                self.db.save_message(msg)
                raise self.store_thread.error
        except Exception as e:
            #    This one we will propogate
            error = e
        try:
            self.lof.close()
            if not error and success:
                #    Send the lof only on success
                self.copy_file(self.lof.name, const.LOFFile)
            os.remove(self.lof.name)
        except Exception as e:
            if not error:
                error = e
        try:

            self.store.disconnect()

            #    Update run data, ONLY if this was successful.
            if not error and success:
                self.bytes, self.hash = self.store_thread.get_hash()
                self.db.update_run_stats(self.bytes, self.nfiles, self.nfolders, self.backup.include_packages, self.hash)
        except Exception as e:
            if not error:
                error = e

        log.debug("All output closed. ")
        if error:
            raise error