def ztest_restore_on_gmail(self): """ clean db disk sync with gmail for few emails restore them on gmail test """ db_dir = '/tmp/gmail_bk' #clean db dir delete_db_dir(db_dir) credential = { 'type' : 'passwd', 'value': self.passwd} gs_credential = { 'type' : 'passwd', 'value': self.gmvault_passwd} search_req = { 'type' : 'imap', 'req': "Since 1-Nov-2011 Before 3-Nov-2011"} syncer = gmvault.GMVaulter(db_dir, 'imap.gmail.com', 993, self.login, credential, read_only_access = False, use_encryption = True) #syncer.sync(imap_req = "Since 1-Nov-2011 Before 4-Nov-2011") # Nov-2007 BigDataset syncer.sync(imap_req = search_req) restorer = gmvault.GMVaulter(db_dir, 'imap.gmail.com', 993, self.gmvault_login, gs_credential, read_only_access = False) restorer.restore() print("Done \n")
def ztest_difference(self): """ """ gsync_credential = {'type': 'passwd', 'value': self.gsync_passwd} gmvault_test_credential = { 'type': 'passwd', 'value': self.gmvault_test_passwd } ba_credential = {'type': 'passwd', 'value': self.ba_passwd} gmv_dir_a = "/tmp/a-db" gmv_dir_b = "/tmp/b-db" gmv_a = gmvault.GMVaulter(gmv_dir_a, 'imap.gmail.com', 993, self.gsync_login, gsync_credential, read_only_access=True) #gmv_a = gmvault.GMVaulter(gmv_dir_a, 'imap.gmail.com', 993, self.gmvault_test_login, gmvault_test_credential, read_only_access = False) #gmv_b = gmvault.GMVaulter(gmv_dir_b, 'imap.gmail.com', 993, self.gmvault_test_login, gmvault_test_credential, read_only_access = False) #gmv_b = gmvault.GMVaulter(gmv_dir_b, 'imap.gmail.com', 993, self.ba_login, ba_credential, read_only_access = True) gmv_b = gmvault.GMVaulter(gmv_dir_b, 'imap.gmail.com', 993, self.ga_login, self.ga_cred, read_only_access=True) test_utils.diff_online_mailboxes(gmv_a, gmv_b)
def test_backup_and_restore(self): """ Backup from gmvault_test and restore """ gsync_credential = {'type': 'passwd', 'value': self.gsync_passwd} gmvault_test_credential = { 'type': 'passwd', 'value': self.gmvault_test_passwd } test_utils.clean_mailbox(self.gsync_login, gsync_credential) gmvault_test_db_dir = "/tmp/backup-restore" backuper = gmvault.GMVaulter(gmvault_test_db_dir, 'imap.gmail.com', 993, \ self.gmvault_test_login, gmvault_test_credential, \ read_only_access = False) backuper.sync({'mode': 'full', 'type': 'imap', 'req': 'ALL'}) #check that we have x emails in the database restorer = gmvault.GMVaulter(gmvault_test_db_dir, 'imap.gmail.com', 993, \ self.gsync_login, gsync_credential, \ read_only_access = False) restorer.restore() #restore all emails from this essential-db test_utils.check_remote_mailbox_identical_to_local(self, restorer) test_utils.diff_online_mailboxes(backuper, restorer) gmvault_utils.delete_all_under(gmvault_test_db_dir, delete_top_dir=True)
def ztest_few_days_syncer(self): """ Test with the Syncer object """ syncer = gmvault.GMVaulter('/tmp/gmail_bk', 'imap.gmail.com', 993, self.login, self.passwd) syncer.sync(imap_req="Since 1-Nov-2011 Before 4-Nov-2011") storage_dir = "%s/%s" % ('/tmp/gmail_bk/db', '2011-11') gstorer = gmvault.GmailStorer('/tmp/gmail_bk') metadata = gmvault.GMVaulter.check_email_on_disk( gstorer, 1384313269332005293) self.assertEquals(metadata['gm_id'], 1384313269332005293) metadata = gmvault.GMVaulter.check_email_on_disk( gstorer, 1384403887202624608) self.assertEquals(metadata['gm_id'], 1384403887202624608) metadata = gmvault.GMVaulter.check_email_on_disk( gstorer, 1384486067720566818) self.assertEquals(metadata['gm_id'], 1384486067720566818)
def ztest_few_days_syncer_with_deletion(self): #pylint:disable-msg=C0103 """ check that there was a deletion """ db_dir = '/tmp/gmail_bk' #clean db dir delete_db_dir(db_dir) #copy test email in dest dir storage_dir = "%s/db/%s" % (db_dir, '2011-11') gmvault_utils.makedirs(storage_dir) shutil.copyfile( '../etc/tests/test_few_days_syncer/2384403887202624608.eml.gz', '%s/2384403887202624608.eml.gz' % (storage_dir)) shutil.copyfile( '../etc/tests/test_few_days_syncer/2384403887202624608.meta', '%s/2384403887202624608.meta' % (storage_dir)) syncer = gmvault.GMVaulter('/tmp/gmail_bk', 'imap.gmail.com', 993, self.login, self.passwd) syncer.sync(imap_req="Since 1-Nov-2011 Before 2-Nov-2011", db_cleaning=True) self.assertFalse( os.path.exists('%s/2384403887202624608.eml.gz' % (storage_dir))) self.assertFalse( os.path.exists('%s/2384403887202624608.meta' % (storage_dir))) self.assertTrue( os.path.exists('%s/1384313269332005293.meta' % (storage_dir))) self.assertTrue( os.path.exists('%s/1384313269332005293.eml.gz' % (storage_dir)))
def ztest_double_login(self): """ double login """ # now read the password sys.argv = [ 'gmvault.py', 'sync', '--db-dir', '/tmp/new-db-1', self.login ] gmvault_launcher = gmv_cmd.GMVaultLauncher() args = gmvault_launcher.parse_args() credential = credential_utils.CredentialHelper.get_credential(args) syncer = gmvault.GMVaulter(args['db-dir'], args['host'], args['port'], \ args['email'], credential) print("First connection \n") syncer.src.connect() import time time.sleep(60 * 10) print("Connection 10 min later") syncer.src.connect()
def test_help_msg_spawned_by_def(self): """ spawn python gmv_runner account > help_msg_spawned.txt check that res is 0 or 1 """ credential = {'type': 'passwd', 'value': self.test_passwd} test_db_dir = "/tmp/gmvault-tests" restorer = gmvault.GMVaulter(test_db_dir, 'imap.gmail.com', 993, self.test_login, credential, \ read_only_access = False) restorer.restore() #restore all emails from this essential-db #need to check that all labels are there for emails in essential gmail_ids = restorer.gstorer.get_all_existing_gmail_ids() for gm_id in gmail_ids: #get disk_metadata disk_metadata = restorer.gstorer.unbury_metadata(gm_id) # get online_metadata online_metadata = restorer.src.fetch( gm_id, imap_utils.GIMAPFetcher.GET_ALL_BUT_DATA) #compare metadata for key in disk_metadata: self.assertEquals(disk_metadata[key], online_metadata[key])
def test_oauth2_reconnect(self): """ oauth2 login test """ # now read the password sys.argv = [ 'gmvault.py', 'sync', '--db-dir', '/tmp/new-db-1', self.login ] gmvault_launcher = gmv_cmd.GMVaultLauncher() args = gmvault_launcher.parse_args() credential = credential_utils.CredentialHelper.get_credential(args) print("CREDENTIALS:%s" % (credential)) syncer = gmvault.GMVaulter(args['db-dir'], args['host'], args['port'], \ args['email'], credential) print("First connection \n") syncer.src.connect() print("Sleep 1 sec and connect again") import time time.sleep(1) syncer.src.connect() print("Sleep 1 sec and reconnect again") time.sleep(1) syncer.src.reconnect()
def ztest_fix_bug(self): """ bug with uid 142221L => empty email returned by gmail """ db_dir = '/tmp/gmail_bk' credential = {'type': 'passwd', 'value': self.passwd} syncer = gmvault.GMVaulter(db_dir, 'imap.gmail.com', 993, self.login, credential, 'verySecRetKeY') syncer._create_update_sync([142221L], compress=True)
def _check_db(cls, args, credential): """ Check DB """ LOG.critical("Connect to Gmail server.\n") # handle credential in all levels checker = gmvault.GMVaulter(args['db-dir'], args['host'], args['port'], \ args['email'], credential, read_only_access = True) checker.check_clean_db(db_cleaning=True)
def ztest_find_identicals(self): """ """ gsync_credential = {'type': 'passwd', 'value': self.gsync_passwd} gmv_dir_a = "/tmp/a-db" gmv_a = gmvault.GMVaulter(gmv_dir_a, 'imap.gmail.com', 993, self.gsync_login, gsync_credential, read_only_access=True) test_utils.find_identical_emails(gmv_a)
def ztest_encrypt_restore_on_gmail(self): """ Doesn't work to be fixed clean db disk sync with gmail for few emails restore them on gmail test """ db_dir = '/tmp/gmail_bk' #clean db dir delete_db_dir(db_dir) credential = {'type': 'passwd', 'value': self.passwd} search_req = { 'type': 'imap', 'req': "Since 1-Nov-2011 Before 3-Nov-2011" } use_encryption = True syncer = gmvault.GMVaulter(db_dir, 'imap.gmail.com', 993, self.login, credential, read_only_access=True, use_encryption=use_encryption) syncer.sync(imap_req=search_req) # check that the email can be read gstorer = gmvault.GmailStorer('/tmp/gmail_bk', use_encryption) metadata = gmvault.GMVaulter.check_email_on_disk( gstorer, 1384313269332005293) self.assertEquals(metadata['gm_id'], 1384313269332005293) email_meta, email_data = gstorer.unbury_email(1384313269332005293) self.assertTrue( email_data.startswith("Delivered-To: [email protected]")) #print("Email Data = \n%s\n" % (email_data)) print("Done \n")
def ztest_fix_bug_search_broken_gm_id_and_quarantine(self): """ Search with a gm_id and quarantine it """ db_dir = '/tmp/gmail_bk' #clean db dir delete_db_dir(db_dir) credential = {'type': 'passwd', 'value': self.passwd} gs_credential = {'type': 'passwd', 'value': self.gmvault_passwd} gstorer = gmvault.GmailStorer(db_dir) gimap = imap_utils.GIMAPFetcher('imap.gmail.com', 993, self.login, credential) gimap.connect() criteria = { 'type': 'imap', 'req': ['X-GM-MSGID 1254269417797093924'] } #broken one #criteria = ['X-GM-MSGID 1254267782370534098'] #criteria = ['ALL'] ids = gimap.search(criteria) for the_id in ids: res = gimap.fetch(the_id, gimap.GET_ALL_INFO) gm_id = gstorer.bury_email(res[the_id], compress=True) syncer = gmvault.GMVaulter(db_dir, 'imap.gmail.com', 993, self.gmvault_login, gs_credential) syncer.restore() #check that the file has been quarantine quarantine_dir = '%s/quarantine' % (db_dir) self.assertTrue( os.path.exists('%s/1254269417797093924.eml.gz' % (quarantine_dir))) self.assertTrue( os.path.exists('%s/1254269417797093924.meta' % (quarantine_dir)))
def _restore(cls, args, credential): """ Execute All restore operations """ LOG.critical("Connect to Gmail server.\n") # Create a gmvault releaving read_only_access restorer = gmvault.GMVaulter(args['db-dir'], args['host'], args['port'], \ args['email'], credential, read_only_access = False) #full sync is the first one if args.get('type', '') == 'full': #call restore labels = [args['apply_label']] if args['apply_label'] else [] restorer.restore(extra_labels = labels, restart = args['restart'], \ emails_only = args['emails_only'], chats_only = args['chats_only']) elif args.get('type', '') == 'quick': #take the last two to 3 months depending on the current date # today - 2 months today = datetime.date.today() begin = today - datetime.timedelta( gmvault_utils.get_conf_defaults().getint( "Restore", "quick_days", 8)) starting_dir = gmvault_utils.get_ym_from_datetime(begin) #call restore labels = [args['apply_label']] if args['apply_label'] else [] restorer.restore(pivot_dir = starting_dir, extra_labels = labels, restart = args['restart'], \ emails_only = args['emails_only'], chats_only = args['chats_only']) else: raise ValueError( "Unknown synchronisation mode %s. Please use full (default), quick." ) #print error report LOG.critical(restorer.get_operation_report())
def ztest_restore_labels(self): """ test all kind of labels that can be restored """ db_dir = '/tmp/gmail_bk' #clean db dir delete_db_dir(db_dir) syncer = gmvault.GMVaulter(db_dir, 'imap.gmail.com', 993, self.login, self.passwd) #syncer.sync(imap_req = "Since 1-Nov-2011 Before 4-Nov-2011") syncer.sync(imap_req="Since 1-Nov-2011 Before 3-Nov-2011") syncer.sync_with_gmail_acc( 'imap.gmail.com', 993, self.gmvault_login, self.gmvault_passwd, [ "The Beginning", "EUMETSAT", "Very Important", "\\Important", "\\Starred", "The End" ])
def test_restore_tricky_emails(self): """ Test_restore_tricky_emails. Restore emails with some specificities (japanese characters) in the a mailbox """ gsync_credential = {'type': 'passwd', 'value': self.gsync_passwd} extra_labels = [u"My-Extra-Label"] test_utils.clean_mailbox(self.gsync_login, gsync_credential) # test restore test_db_dir = "/homespace/gaubert/gmvault-dbs/essential-dbs" #test_db_dir = "/home/gmv/Dev/projects/gmvault-develop/src/test-db" #test_db_dir = "/Users/gaubert/Dev/projects/gmvault-develop/src/test-db" restorer = gmvault.GMVaulter(test_db_dir, 'imap.gmail.com', 993, \ self.gsync_login, gsync_credential, \ read_only_access = False) restorer.restore(extra_labels=extra_labels ) #restore all emails from this essential-db test_utils.check_remote_mailbox_identical_to_local( self, restorer, extra_labels)
def _sync(cls, args, credential): """ Execute All synchronisation operations """ LOG.critical("Connect to Gmail server.\n") # handle credential in all levels syncer = gmvault.GMVaulter(args['db-dir'], args['host'], args['port'], \ args['email'], credential, read_only_access = True, use_encryption = args['encrypt']) #full sync is the first one if args.get('type', '') == 'full': #choose full sync. Ignore the request syncer.sync({ 'mode': 'full', 'type': 'imap', 'req': 'ALL' } , compress_on_disk = args['compression'], \ db_cleaning = args['db-cleaning'], ownership_checking = args['ownership_control'],\ restart = args['restart'], emails_only = args['emails_only'], chats_only = args['chats_only']) elif args.get('type', '') == 'auto': #choose auto sync. imap request = ALL and restart = True syncer.sync({ 'mode': 'auto', 'type': 'imap', 'req': 'ALL' } , compress_on_disk = args['compression'], \ db_cleaning = args['db-cleaning'], ownership_checking = args['ownership_control'],\ restart = True, emails_only = args['emails_only'], chats_only = args['chats_only']) elif args.get('type', '') == 'quick': #sync only the last x days (taken in defaults) in order to be quick #(cleaning is import here because recent days might move again # today - 2 months today = datetime.date.today() begin = today - datetime.timedelta( gmvault_utils.get_conf_defaults().getint( "Sync", "quick_days", 8)) LOG.critical("Quick sync mode. Check for new emails since %s." % (begin.strftime('%d-%b-%Y'))) # today + 1 day end = today + datetime.timedelta(1) req = { 'type' : 'imap', \ 'req' : syncer.get_imap_request_btw_2_dates(begin, end), \ 'mode' : 'quick'} syncer.sync( req, \ compress_on_disk = args['compression'], \ db_cleaning = args['db-cleaning'], \ ownership_checking = args['ownership_control'], restart = args['restart'], \ emails_only = args['emails_only'], chats_only = args['chats_only']) elif args.get('type', '') == 'custom': #convert args to unicode args['request']['req'] = gmvault_utils.convert_to_unicode( args['request']['req']) args['request']['charset'] = 'utf-8' #for the moment always utf-8 args['request']['mode'] = 'custom' # pass an imap request. Assume that the user know what to do here LOG.critical("Perform custom synchronisation with %s request: %s.\n" \ % (args['request']['type'], args['request']['req'])) syncer.sync(args['request'], compress_on_disk = args['compression'], db_cleaning = args['db-cleaning'], \ ownership_checking = args['ownership_control'], restart = args['restart'], \ emails_only = args['emails_only'], chats_only = args['chats_only']) else: raise ValueError( "Unknown synchronisation mode %s. Please use full (default), quick or custom." ) #print error report LOG.critical(syncer.get_operation_report())