def bury_email(self, email_info, local_dir=None, compress=False,
extra_labels=()):
"""
store all email info in 2 files (.meta and .eml files)
Arguments:
email_info: the email content
local_dir : intermediary dir (month dir)
compress : if compress is True, use gzip compression
"""
if local_dir:
the_dir = '%s/%s' % (self._db_dir, local_dir)
gmvault_utils.makedirs(the_dir)
else:
the_dir = self._db_dir
data_path = self.DATA_FNAME % (
the_dir, email_info[imap_utils.GIMAPFetcher.GMAIL_ID])
# TODO: First compress then encrypt
# create a compressed CIOString and encrypt it
#if compress:
# data_path = '%s.gz' % data_path
# data_desc = StringIO.StringIO()
#else:
# data_desc = open(data_path, 'wb')
#if self._encrypt_data:
# data_path = '%s.crypt2' % data_path
#TODO create a wrapper fileobj that compress in io string
#then chunk write
#then compress
#then encrypt if it is required
# if the data has to be encrypted
if self._encrypt_data:
data_path = '%s.crypt' % data_path
if compress:
data_path = '%s.gz' % data_path
data_desc = gzip.open(data_path, 'wb')
else:
data_desc = open(data_path, 'wb')
try:
if self._encrypt_data:
# need to be done for every encryption
cipher = self.get_encryption_cipher()
cipher.initCTR()
data = cipher.encryptCTR(email_info[imap_utils.GIMAPFetcher.EMAIL_BODY])
LOG.debug("Encrypt data.")
#write encrypted data without encoding
data_desc.write(data)
#no encryption then utf-8 encode and write
else:
#convert email content to unicode
data = gmvault_utils.convert_to_unicode(email_info[imap_utils.GIMAPFetcher.EMAIL_BODY])
# write in chunks of one 1 MB
for chunk in gmvault_utils.chunker(data, 1048576):
data_desc.write(chunk.encode('utf-8'))
#store metadata info
self.bury_metadata(email_info, local_dir, extra_labels)
data_desc.flush()
finally:
data_desc.close()
return email_info[imap_utils.GIMAPFetcher.GMAIL_ID]
def _sync(cls, args, credential):
"""
Execute All synchronisation operations
"""
LOG.critical("Connect to Gmail server.\n")
# handle credential in all levels
syncer = gmvault.GMVaulter(args['db-dir'], args['host'], args['port'], \
args['email'], credential, read_only_access = True, use_encryption = args['encrypt'])
#full sync is the first one
if args.get('type', '') == 'full':
#choose full sync. Ignore the request
syncer.sync({ 'mode': 'full', 'type': 'imap', 'req': 'ALL' } , compress_on_disk = args['compression'], \
db_cleaning = args['db-cleaning'], ownership_checking = args['ownership_control'],\
restart = args['restart'], emails_only = args['emails_only'], chats_only = args['chats_only'])
elif args.get('type', '') == 'auto':
#choose auto sync. imap request = ALL and restart = True
syncer.sync({ 'mode': 'auto', 'type': 'imap', 'req': 'ALL' } , compress_on_disk = args['compression'], \
db_cleaning = args['db-cleaning'], ownership_checking = args['ownership_control'],\
restart = True, emails_only = args['emails_only'], chats_only = args['chats_only'])
elif args.get('type', '') == 'quick':
#sync only the last x days (taken in defaults) in order to be quick
#(cleaning is import here because recent days might move again
# today - 2 months
today = datetime.date.today()
begin = today - datetime.timedelta(gmvault_utils.get_conf_defaults().getint("Sync", "quick_days", 8))
LOG.critical("Quick sync mode. Check for new emails since %s." % (begin.strftime('%d-%b-%Y')))
# today + 1 day
end = today + datetime.timedelta(1)
req = { 'type' : 'imap', \
'req' : syncer.get_imap_request_btw_2_dates(begin, end), \
'mode' : 'quick'}
syncer.sync( req, \
compress_on_disk = args['compression'], \
db_cleaning = args['db-cleaning'], \
ownership_checking = args['ownership_control'], restart = args['restart'], \
emails_only = args['emails_only'], chats_only = args['chats_only'])
elif args.get('type', '') == 'custom':
#convert args to unicode
args['request']['req'] = gmvault_utils.convert_to_unicode(args['request']['req'])
args['request']['charset'] = 'utf-8' #for the moment always utf-8
args['request']['mode'] = 'custom'
# pass an imap request. Assume that the user know what to do here
LOG.critical("Perform custom synchronisation with %s request: %s.\n" \
% (args['request']['type'], args['request']['req']))
syncer.sync(args['request'], compress_on_disk = args['compression'], db_cleaning = args['db-cleaning'], \
ownership_checking = args['ownership_control'], restart = args['restart'], \
emails_only = args['emails_only'], chats_only = args['chats_only'])
else:
raise ValueError("Unknown synchronisation mode %s. Please use full (default), quick or custom.")
#print error report
LOG.critical(syncer.get_operation_report())
def _sync(cls, args, credential):
"""
Execute All synchronisation operations
"""
LOG.critical("Connect to Gmail server.\n")
# handle credential in all levels
syncer = gmvault.GMVaulter(args['db-dir'], args['host'], args['port'], \
args['email'], credential, read_only_access = True, use_encryption = args['encrypt'])
#full sync is the first one
if args.get('type', '') == 'full':
#choose full sync. Ignore the request
syncer.sync({ 'mode': 'full', 'type': 'imap', 'req': 'ALL' } , compress_on_disk = args['compression'], \
db_cleaning = args['db-cleaning'], ownership_checking = args['ownership_control'],\
restart = args['restart'], emails_only = args['emails_only'], chats_only = args['chats_only'])
elif args.get('type', '') == 'auto':
#choose auto sync. imap request = ALL and restart = True
syncer.sync({ 'mode': 'auto', 'type': 'imap', 'req': 'ALL' } , compress_on_disk = args['compression'], \
db_cleaning = args['db-cleaning'], ownership_checking = args['ownership_control'],\
restart = True, emails_only = args['emails_only'], chats_only = args['chats_only'])
elif args.get('type', '') == 'quick':
#sync only the last x days (taken in defaults) in order to be quick
#(cleaning is import here because recent days might move again
# today - 2 months
today = datetime.date.today()
begin = today - datetime.timedelta(
gmvault_utils.get_conf_defaults().getint(
"Sync", "quick_days", 8))
LOG.critical("Quick sync mode. Check for new emails since %s." %
(begin.strftime('%d-%b-%Y')))
# today + 1 day
end = today + datetime.timedelta(1)
req = { 'type' : 'imap', \
'req' : syncer.get_imap_request_btw_2_dates(begin, end), \
'mode' : 'quick'}
syncer.sync( req, \
compress_on_disk = args['compression'], \
db_cleaning = args['db-cleaning'], \
ownership_checking = args['ownership_control'], restart = args['restart'], \
emails_only = args['emails_only'], chats_only = args['chats_only'])
elif args.get('type', '') == 'custom':
#convert args to unicode
args['request']['req'] = gmvault_utils.convert_to_unicode(
args['request']['req'])
args['request']['charset'] = 'utf-8' #for the moment always utf-8
args['request']['mode'] = 'custom'
# pass an imap request. Assume that the user know what to do here
LOG.critical("Perform custom synchronisation with %s request: %s.\n" \
% (args['request']['type'], args['request']['req']))
syncer.sync(args['request'], compress_on_disk = args['compression'], db_cleaning = args['db-cleaning'], \
ownership_checking = args['ownership_control'], restart = args['restart'], \
emails_only = args['emails_only'], chats_only = args['chats_only'])
else:
raise ValueError(
"Unknown synchronisation mode %s. Please use full (default), quick or custom."
)
#print error report
LOG.critical(syncer.get_operation_report())
def bury_email(self, email_info, local_dir=None, compress=False,
extra_labels=()):
"""
store all email info in 2 files (.meta and .eml files)
Arguments:
email_info: the email content
local_dir : intermediary dir (month dir)
compress : if compress is True, use gzip compression
"""
if local_dir:
the_dir = '%s/%s' % (self._db_dir, local_dir)
gmvault_utils.makedirs(the_dir)
else:
the_dir = self._db_dir
data_path = self.DATA_FNAME % (
the_dir, email_info[imap_utils.GIMAPFetcher.GMAIL_ID])
# TODO: First compress then encrypt
# create a compressed CIOString and encrypt it
#if compress:
# data_path = '%s.gz' % data_path
# data_desc = StringIO.StringIO()
#else:
# data_desc = open(data_path, 'wb')
#if self._encrypt_data:
# data_path = '%s.crypt2' % data_path
#TODO create a wrapper fileobj that compress in io string
#then chunk write
#then compress
#then encrypt if it is required
# if the data has to be encrypted
if self._encrypt_data:
data_path = '%s.crypt' % data_path
if compress:
data_path = '%s.gz' % data_path
data_desc = gzip.open(data_path, 'wb')
else:
data_desc = open(data_path, 'wb')
try:
if self._encrypt_data:
# need to be done for every encryption
cipher = self.get_encryption_cipher()
cipher.initCTR()
data = cipher.encryptCTR(email_info[imap_utils.GIMAPFetcher.EMAIL_BODY])
LOG.debug("Encrypt data.")
#write encrypted data without encoding
data_desc.write(data)
#no encryption then utf-8 encode and write
else:
#convert email content to unicode
data = gmvault_utils.convert_to_unicode(email_info[imap_utils.GIMAPFetcher.EMAIL_BODY])
# write in chunks of one 1 MB
for chunk in gmvault_utils.chunker(data, 1048576):
data_desc.write(chunk.encode('utf-8'))
#store metadata info
self.bury_metadata(email_info, local_dir, extra_labels)
data_desc.flush()
finally:
data_desc.close()
return email_info[imap_utils.GIMAPFetcher.GMAIL_ID]