def ztest_restore_on_gmail(self):
"""
clean db disk
sync with gmail for few emails
restore them on gmail test
"""
db_dir = '/tmp/gmail_bk'
#clean db dir
delete_db_dir(db_dir)
credential = { 'type' : 'passwd', 'value': self.passwd}
gs_credential = { 'type' : 'passwd', 'value': self.gmvault_passwd}
search_req = { 'type' : 'imap', 'req': "Since 1-Nov-2011 Before 3-Nov-2011"}
syncer = gmvault.GMVaulter(db_dir, 'imap.gmail.com', 993, self.login, credential, read_only_access = False, use_encryption = True)
#syncer.sync(imap_req = "Since 1-Nov-2011 Before 4-Nov-2011")
# Nov-2007 BigDataset
syncer.sync(imap_req = search_req)
restorer = gmvault.GMVaulter(db_dir, 'imap.gmail.com', 993, self.gmvault_login, gs_credential, read_only_access = False)
restorer.restore()
print("Done \n")
def ztest_difference(self):
"""
"""
gsync_credential = {'type': 'passwd', 'value': self.gsync_passwd}
gmvault_test_credential = {
'type': 'passwd',
'value': self.gmvault_test_passwd
}
ba_credential = {'type': 'passwd', 'value': self.ba_passwd}
gmv_dir_a = "/tmp/a-db"
gmv_dir_b = "/tmp/b-db"
gmv_a = gmvault.GMVaulter(gmv_dir_a,
'imap.gmail.com',
993,
self.gsync_login,
gsync_credential,
read_only_access=True)
#gmv_a = gmvault.GMVaulter(gmv_dir_a, 'imap.gmail.com', 993, self.gmvault_test_login, gmvault_test_credential, read_only_access = False)
#gmv_b = gmvault.GMVaulter(gmv_dir_b, 'imap.gmail.com', 993, self.gmvault_test_login, gmvault_test_credential, read_only_access = False)
#gmv_b = gmvault.GMVaulter(gmv_dir_b, 'imap.gmail.com', 993, self.ba_login, ba_credential, read_only_access = True)
gmv_b = gmvault.GMVaulter(gmv_dir_b,
'imap.gmail.com',
993,
self.ga_login,
self.ga_cred,
read_only_access=True)
test_utils.diff_online_mailboxes(gmv_a, gmv_b)
def test_backup_and_restore(self):
""" Backup from gmvault_test and restore """
gsync_credential = {'type': 'passwd', 'value': self.gsync_passwd}
gmvault_test_credential = {
'type': 'passwd',
'value': self.gmvault_test_passwd
}
test_utils.clean_mailbox(self.gsync_login, gsync_credential)
gmvault_test_db_dir = "/tmp/backup-restore"
backuper = gmvault.GMVaulter(gmvault_test_db_dir, 'imap.gmail.com', 993, \
self.gmvault_test_login, gmvault_test_credential, \
read_only_access = False)
backuper.sync({'mode': 'full', 'type': 'imap', 'req': 'ALL'})
#check that we have x emails in the database
restorer = gmvault.GMVaulter(gmvault_test_db_dir, 'imap.gmail.com', 993, \
self.gsync_login, gsync_credential, \
read_only_access = False)
restorer.restore() #restore all emails from this essential-db
test_utils.check_remote_mailbox_identical_to_local(self, restorer)
test_utils.diff_online_mailboxes(backuper, restorer)
gmvault_utils.delete_all_under(gmvault_test_db_dir,
delete_top_dir=True)
def ztest_few_days_syncer(self):
"""
Test with the Syncer object
"""
syncer = gmvault.GMVaulter('/tmp/gmail_bk', 'imap.gmail.com', 993,
self.login, self.passwd)
syncer.sync(imap_req="Since 1-Nov-2011 Before 4-Nov-2011")
storage_dir = "%s/%s" % ('/tmp/gmail_bk/db', '2011-11')
gstorer = gmvault.GmailStorer('/tmp/gmail_bk')
metadata = gmvault.GMVaulter.check_email_on_disk(
gstorer, 1384313269332005293)
self.assertEquals(metadata['gm_id'], 1384313269332005293)
metadata = gmvault.GMVaulter.check_email_on_disk(
gstorer, 1384403887202624608)
self.assertEquals(metadata['gm_id'], 1384403887202624608)
metadata = gmvault.GMVaulter.check_email_on_disk(
gstorer, 1384486067720566818)
self.assertEquals(metadata['gm_id'], 1384486067720566818)
def ztest_few_days_syncer_with_deletion(self): #pylint:disable-msg=C0103
"""
check that there was a deletion
"""
db_dir = '/tmp/gmail_bk'
#clean db dir
delete_db_dir(db_dir)
#copy test email in dest dir
storage_dir = "%s/db/%s" % (db_dir, '2011-11')
gmvault_utils.makedirs(storage_dir)
shutil.copyfile(
'../etc/tests/test_few_days_syncer/2384403887202624608.eml.gz',
'%s/2384403887202624608.eml.gz' % (storage_dir))
shutil.copyfile(
'../etc/tests/test_few_days_syncer/2384403887202624608.meta',
'%s/2384403887202624608.meta' % (storage_dir))
syncer = gmvault.GMVaulter('/tmp/gmail_bk', 'imap.gmail.com', 993,
self.login, self.passwd)
syncer.sync(imap_req="Since 1-Nov-2011 Before 2-Nov-2011",
db_cleaning=True)
self.assertFalse(
os.path.exists('%s/2384403887202624608.eml.gz' % (storage_dir)))
self.assertFalse(
os.path.exists('%s/2384403887202624608.meta' % (storage_dir)))
self.assertTrue(
os.path.exists('%s/1384313269332005293.meta' % (storage_dir)))
self.assertTrue(
os.path.exists('%s/1384313269332005293.eml.gz' % (storage_dir)))
def ztest_double_login(self):
"""
double login
"""
# now read the password
sys.argv = [
'gmvault.py', 'sync', '--db-dir', '/tmp/new-db-1', self.login
]
gmvault_launcher = gmv_cmd.GMVaultLauncher()
args = gmvault_launcher.parse_args()
credential = credential_utils.CredentialHelper.get_credential(args)
syncer = gmvault.GMVaulter(args['db-dir'], args['host'], args['port'], \
args['email'], credential)
print("First connection \n")
syncer.src.connect()
import time
time.sleep(60 * 10)
print("Connection 10 min later")
syncer.src.connect()
def test_help_msg_spawned_by_def(self):
"""
spawn python gmv_runner account > help_msg_spawned.txt
check that res is 0 or 1
"""
credential = {'type': 'passwd', 'value': self.test_passwd}
test_db_dir = "/tmp/gmvault-tests"
restorer = gmvault.GMVaulter(test_db_dir, 'imap.gmail.com', 993, self.test_login, credential, \
read_only_access = False)
restorer.restore() #restore all emails from this essential-db
#need to check that all labels are there for emails in essential
gmail_ids = restorer.gstorer.get_all_existing_gmail_ids()
for gm_id in gmail_ids:
#get disk_metadata
disk_metadata = restorer.gstorer.unbury_metadata(gm_id)
# get online_metadata
online_metadata = restorer.src.fetch(
gm_id, imap_utils.GIMAPFetcher.GET_ALL_BUT_DATA)
#compare metadata
for key in disk_metadata:
self.assertEquals(disk_metadata[key], online_metadata[key])
def test_oauth2_reconnect(self):
"""
oauth2 login test
"""
# now read the password
sys.argv = [
'gmvault.py', 'sync', '--db-dir', '/tmp/new-db-1', self.login
]
gmvault_launcher = gmv_cmd.GMVaultLauncher()
args = gmvault_launcher.parse_args()
credential = credential_utils.CredentialHelper.get_credential(args)
print("CREDENTIALS:%s" % (credential))
syncer = gmvault.GMVaulter(args['db-dir'], args['host'], args['port'], \
args['email'], credential)
print("First connection \n")
syncer.src.connect()
print("Sleep 1 sec and connect again")
import time
time.sleep(1)
syncer.src.connect()
print("Sleep 1 sec and reconnect again")
time.sleep(1)
syncer.src.reconnect()
def ztest_fix_bug(self):
"""
bug with uid 142221L => empty email returned by gmail
"""
db_dir = '/tmp/gmail_bk'
credential = {'type': 'passwd', 'value': self.passwd}
syncer = gmvault.GMVaulter(db_dir, 'imap.gmail.com', 993, self.login,
credential, 'verySecRetKeY')
syncer._create_update_sync([142221L], compress=True)
def _check_db(cls, args, credential):
"""
Check DB
"""
LOG.critical("Connect to Gmail server.\n")
# handle credential in all levels
checker = gmvault.GMVaulter(args['db-dir'], args['host'], args['port'], \
args['email'], credential, read_only_access = True)
checker.check_clean_db(db_cleaning=True)
def ztest_find_identicals(self):
"""
"""
gsync_credential = {'type': 'passwd', 'value': self.gsync_passwd}
gmv_dir_a = "/tmp/a-db"
gmv_a = gmvault.GMVaulter(gmv_dir_a,
'imap.gmail.com',
993,
self.gsync_login,
gsync_credential,
read_only_access=True)
test_utils.find_identical_emails(gmv_a)
def ztest_encrypt_restore_on_gmail(self):
"""
Doesn't work to be fixed
clean db disk
sync with gmail for few emails
restore them on gmail test
"""
db_dir = '/tmp/gmail_bk'
#clean db dir
delete_db_dir(db_dir)
credential = {'type': 'passwd', 'value': self.passwd}
search_req = {
'type': 'imap',
'req': "Since 1-Nov-2011 Before 3-Nov-2011"
}
use_encryption = True
syncer = gmvault.GMVaulter(db_dir,
'imap.gmail.com',
993,
self.login,
credential,
read_only_access=True,
use_encryption=use_encryption)
syncer.sync(imap_req=search_req)
# check that the email can be read
gstorer = gmvault.GmailStorer('/tmp/gmail_bk', use_encryption)
metadata = gmvault.GMVaulter.check_email_on_disk(
gstorer, 1384313269332005293)
self.assertEquals(metadata['gm_id'], 1384313269332005293)
email_meta, email_data = gstorer.unbury_email(1384313269332005293)
self.assertTrue(
email_data.startswith("Delivered-To: [email protected]"))
#print("Email Data = \n%s\n" % (email_data))
print("Done \n")
def ztest_fix_bug_search_broken_gm_id_and_quarantine(self):
"""
Search with a gm_id and quarantine it
"""
db_dir = '/tmp/gmail_bk'
#clean db dir
delete_db_dir(db_dir)
credential = {'type': 'passwd', 'value': self.passwd}
gs_credential = {'type': 'passwd', 'value': self.gmvault_passwd}
gstorer = gmvault.GmailStorer(db_dir)
gimap = imap_utils.GIMAPFetcher('imap.gmail.com', 993, self.login,
credential)
gimap.connect()
criteria = {
'type': 'imap',
'req': ['X-GM-MSGID 1254269417797093924']
} #broken one
#criteria = ['X-GM-MSGID 1254267782370534098']
#criteria = ['ALL']
ids = gimap.search(criteria)
for the_id in ids:
res = gimap.fetch(the_id, gimap.GET_ALL_INFO)
gm_id = gstorer.bury_email(res[the_id], compress=True)
syncer = gmvault.GMVaulter(db_dir, 'imap.gmail.com', 993,
self.gmvault_login, gs_credential)
syncer.restore()
#check that the file has been quarantine
quarantine_dir = '%s/quarantine' % (db_dir)
self.assertTrue(
os.path.exists('%s/1254269417797093924.eml.gz' % (quarantine_dir)))
self.assertTrue(
os.path.exists('%s/1254269417797093924.meta' % (quarantine_dir)))
def _restore(cls, args, credential):
"""
Execute All restore operations
"""
LOG.critical("Connect to Gmail server.\n")
# Create a gmvault releaving read_only_access
restorer = gmvault.GMVaulter(args['db-dir'], args['host'], args['port'], \
args['email'], credential, read_only_access = False)
#full sync is the first one
if args.get('type', '') == 'full':
#call restore
labels = [args['apply_label']] if args['apply_label'] else []
restorer.restore(extra_labels = labels, restart = args['restart'], \
emails_only = args['emails_only'], chats_only = args['chats_only'])
elif args.get('type', '') == 'quick':
#take the last two to 3 months depending on the current date
# today - 2 months
today = datetime.date.today()
begin = today - datetime.timedelta(
gmvault_utils.get_conf_defaults().getint(
"Restore", "quick_days", 8))
starting_dir = gmvault_utils.get_ym_from_datetime(begin)
#call restore
labels = [args['apply_label']] if args['apply_label'] else []
restorer.restore(pivot_dir = starting_dir, extra_labels = labels, restart = args['restart'], \
emails_only = args['emails_only'], chats_only = args['chats_only'])
else:
raise ValueError(
"Unknown synchronisation mode %s. Please use full (default), quick."
)
#print error report
LOG.critical(restorer.get_operation_report())
def ztest_restore_labels(self):
"""
test all kind of labels that can be restored
"""
db_dir = '/tmp/gmail_bk'
#clean db dir
delete_db_dir(db_dir)
syncer = gmvault.GMVaulter(db_dir, 'imap.gmail.com', 993, self.login,
self.passwd)
#syncer.sync(imap_req = "Since 1-Nov-2011 Before 4-Nov-2011")
syncer.sync(imap_req="Since 1-Nov-2011 Before 3-Nov-2011")
syncer.sync_with_gmail_acc(
'imap.gmail.com', 993, self.gmvault_login, self.gmvault_passwd, [
"The Beginning", "EUMETSAT", "Very Important", "\\Important",
"\\Starred", "The End"
])
def test_restore_tricky_emails(self):
""" Test_restore_tricky_emails. Restore emails with some specificities (japanese characters) in the a mailbox """
gsync_credential = {'type': 'passwd', 'value': self.gsync_passwd}
extra_labels = [u"My-Extra-Label"]
test_utils.clean_mailbox(self.gsync_login, gsync_credential)
# test restore
test_db_dir = "/homespace/gaubert/gmvault-dbs/essential-dbs"
#test_db_dir = "/home/gmv/Dev/projects/gmvault-develop/src/test-db"
#test_db_dir = "/Users/gaubert/Dev/projects/gmvault-develop/src/test-db"
restorer = gmvault.GMVaulter(test_db_dir, 'imap.gmail.com', 993, \
self.gsync_login, gsync_credential, \
read_only_access = False)
restorer.restore(extra_labels=extra_labels
) #restore all emails from this essential-db
test_utils.check_remote_mailbox_identical_to_local(
self, restorer, extra_labels)
def _sync(cls, args, credential):
"""
Execute All synchronisation operations
"""
LOG.critical("Connect to Gmail server.\n")
# handle credential in all levels
syncer = gmvault.GMVaulter(args['db-dir'], args['host'], args['port'], \
args['email'], credential, read_only_access = True, use_encryption = args['encrypt'])
#full sync is the first one
if args.get('type', '') == 'full':
#choose full sync. Ignore the request
syncer.sync({ 'mode': 'full', 'type': 'imap', 'req': 'ALL' } , compress_on_disk = args['compression'], \
db_cleaning = args['db-cleaning'], ownership_checking = args['ownership_control'],\
restart = args['restart'], emails_only = args['emails_only'], chats_only = args['chats_only'])
elif args.get('type', '') == 'auto':
#choose auto sync. imap request = ALL and restart = True
syncer.sync({ 'mode': 'auto', 'type': 'imap', 'req': 'ALL' } , compress_on_disk = args['compression'], \
db_cleaning = args['db-cleaning'], ownership_checking = args['ownership_control'],\
restart = True, emails_only = args['emails_only'], chats_only = args['chats_only'])
elif args.get('type', '') == 'quick':
#sync only the last x days (taken in defaults) in order to be quick
#(cleaning is import here because recent days might move again
# today - 2 months
today = datetime.date.today()
begin = today - datetime.timedelta(
gmvault_utils.get_conf_defaults().getint(
"Sync", "quick_days", 8))
LOG.critical("Quick sync mode. Check for new emails since %s." %
(begin.strftime('%d-%b-%Y')))
# today + 1 day
end = today + datetime.timedelta(1)
req = { 'type' : 'imap', \
'req' : syncer.get_imap_request_btw_2_dates(begin, end), \
'mode' : 'quick'}
syncer.sync( req, \
compress_on_disk = args['compression'], \
db_cleaning = args['db-cleaning'], \
ownership_checking = args['ownership_control'], restart = args['restart'], \
emails_only = args['emails_only'], chats_only = args['chats_only'])
elif args.get('type', '') == 'custom':
#convert args to unicode
args['request']['req'] = gmvault_utils.convert_to_unicode(
args['request']['req'])
args['request']['charset'] = 'utf-8' #for the moment always utf-8
args['request']['mode'] = 'custom'
# pass an imap request. Assume that the user know what to do here
LOG.critical("Perform custom synchronisation with %s request: %s.\n" \
% (args['request']['type'], args['request']['req']))
syncer.sync(args['request'], compress_on_disk = args['compression'], db_cleaning = args['db-cleaning'], \
ownership_checking = args['ownership_control'], restart = args['restart'], \
emails_only = args['emails_only'], chats_only = args['chats_only'])
else:
raise ValueError(
"Unknown synchronisation mode %s. Please use full (default), quick or custom."
)
#print error report
LOG.critical(syncer.get_operation_report())
