def bury_metadata(self, email_info, local_dir=None):
"""
Store metadata info in .meta file
Arguments:
email_info: metadata info
local_dir : intermdiary dir (month dir)
"""
if local_dir:
the_dir = '%s/%s' % (self._db_dir, local_dir)
gmvault_utils.makedirs(the_dir)
else:
the_dir = self._db_dir
meta_path = self.METADATA_FNAME % (
the_dir, email_info[imap_utils.GIMAPFetcher.GMAIL_ID])
meta_desc = open(meta_path, 'w')
# parse header fields to extract subject and msgid
subject, msgid = self.parse_header_fields(
email_info[imap_utils.GIMAPFetcher.IMAP_HEADER_FIELDS])
#create json structure for metadata
meta_obj = {
self.ID_K:
email_info[imap_utils.GIMAPFetcher.GMAIL_ID],
self.LABELS_K:
email_info[imap_utils.GIMAPFetcher.GMAIL_LABELS],
self.FLAGS_K:
email_info[imap_utils.GIMAPFetcher.IMAP_FLAGS],
self.THREAD_IDS_K:
email_info[imap_utils.GIMAPFetcher.GMAIL_THREAD_ID],
self.INT_DATE_K:
gmvault_utils.datetime2e(
email_info[imap_utils.GIMAPFetcher.IMAP_INTERNALDATE]),
self.FLAGS_K:
email_info[imap_utils.GIMAPFetcher.IMAP_FLAGS],
self.SUBJECT_K:
subject,
self.MSGID_K:
msgid
}
json.dump(meta_obj, meta_desc, ensure_ascii=False)
meta_desc.flush()
meta_desc.close()
return email_info[imap_utils.GIMAPFetcher.GMAIL_ID]
def bury_metadata(self, email_info, local_dir = None):
"""
Store metadata info in .meta file
Arguments:
email_info: metadata info
local_dir : intermdiary dir (month dir)
"""
if local_dir:
the_dir = '%s/%s' % (self._db_dir, local_dir)
gmvault_utils.makedirs(the_dir)
else:
the_dir = self._db_dir
meta_path = self.METADATA_FNAME % (the_dir, email_info[imap_utils.GIMAPFetcher.GMAIL_ID])
meta_desc = open(meta_path, 'w')
# parse header fields to extract subject and msgid
subject, msgid = self.parse_header_fields(email_info[imap_utils.GIMAPFetcher.IMAP_HEADER_FIELDS])
# need to convert labels that are number as string
# come from imap_lib when label is a number
labels = [ str(elem) for elem in email_info[imap_utils.GIMAPFetcher.GMAIL_LABELS] ]
#create json structure for metadata
meta_obj = {
self.ID_K : email_info[imap_utils.GIMAPFetcher.GMAIL_ID],
self.LABELS_K : labels,
self.FLAGS_K : email_info[imap_utils.GIMAPFetcher.IMAP_FLAGS],
self.THREAD_IDS_K : email_info[imap_utils.GIMAPFetcher.GMAIL_THREAD_ID],
self.INT_DATE_K : gmvault_utils.datetime2e(email_info[imap_utils.GIMAPFetcher.IMAP_INTERNALDATE]),
self.FLAGS_K : email_info[imap_utils.GIMAPFetcher.IMAP_FLAGS],
self.SUBJECT_K : subject,
self.MSGID_K : msgid
}
json.dump(meta_obj, meta_desc, ensure_ascii = False)
meta_desc.flush()
meta_desc.close()
return email_info[imap_utils.GIMAPFetcher.GMAIL_ID]
def bury_email(self, email_info, local_dir = None, compress = False):
"""
store all email info in 2 files (.meta and .eml files)
Arguments:
email_info: info
local_dir : intermdiary dir (month dir)
compress : If compress is True, use gzip compression
"""
if local_dir:
the_dir = '%s/%s' % (self._db_dir, local_dir)
gmvault_utils.makedirs(the_dir)
else:
the_dir = self._db_dir
meta_path = self.METADATA_FNAME % (the_dir, email_info[imap_utils.GIMAPFetcher.GMAIL_ID])
data_path = self.DATA_FNAME % (the_dir, email_info[imap_utils.GIMAPFetcher.GMAIL_ID])
# if the data has to be encrypted
if self._encrypt_data:
data_path = '%s.crypt' % (data_path)
if compress:
data_path = '%s.gz' % (data_path)
data_desc = gzip.open(data_path, 'wb')
else:
data_desc = open(data_path, 'wb')
meta_desc = open(meta_path, 'w')
if self._encrypt_data:
# need to be done for every encryption
cipher = self.get_encryption_cipher()
cipher.initCTR()
data_desc.write(cipher.encryptCTR(email_info[imap_utils.GIMAPFetcher.EMAIL_BODY]))
else:
data_desc.write(email_info[imap_utils.GIMAPFetcher.EMAIL_BODY])
# parse header fields to extract subject and msgid
subject, msgid = self.parse_header_fields(email_info[imap_utils.GIMAPFetcher.IMAP_HEADER_FIELDS])
#create json structure for metadata
meta_obj = {
self.ID_K : email_info[imap_utils.GIMAPFetcher.GMAIL_ID],
self.LABELS_K : email_info[imap_utils.GIMAPFetcher.GMAIL_LABELS],
self.FLAGS_K : email_info[imap_utils.GIMAPFetcher.IMAP_FLAGS],
self.THREAD_IDS_K : email_info[imap_utils.GIMAPFetcher.GMAIL_THREAD_ID],
self.INT_DATE_K : gmvault_utils.datetime2e(email_info[imap_utils.GIMAPFetcher.IMAP_INTERNALDATE]),
self.FLAGS_K : email_info[imap_utils.GIMAPFetcher.IMAP_FLAGS],
self.SUBJECT_K : subject,
self.MSGID_K : msgid
}
json.dump(meta_obj, meta_desc, ensure_ascii = False)
meta_desc.flush()
meta_desc.close()
data_desc.flush()
data_desc.close()
return email_info[imap_utils.GIMAPFetcher.GMAIL_ID]
def bury_email(self, email_info, local_dir=None, compress=False):
"""
store all email info in 2 files (.meta and .eml files)
Arguments:
email_info: info
local_dir : intermdiary dir (month dir)
compress : If compress is True, use gzip compression
"""
if local_dir:
the_dir = '%s/%s' % (self._db_dir, local_dir)
gmvault_utils.makedirs(the_dir)
else:
the_dir = self._db_dir
meta_path = self.METADATA_FNAME % (
the_dir, email_info[imap_utils.GIMAPFetcher.GMAIL_ID])
data_path = self.DATA_FNAME % (
the_dir, email_info[imap_utils.GIMAPFetcher.GMAIL_ID])
# if the data has to be encrypted
if self._encrypt_data:
data_path = '%s.crypt' % (data_path)
if compress:
data_path = '%s.gz' % (data_path)
data_desc = gzip.open(data_path, 'wb')
else:
data_desc = open(data_path, 'wb')
meta_desc = open(meta_path, 'w')
if self._encrypt_data:
# need to be done for every encryption
cipher = self.get_encryption_cipher()
cipher.initCTR()
data_desc.write(
cipher.encryptCTR(
email_info[imap_utils.GIMAPFetcher.EMAIL_BODY]))
else:
data_desc.write(email_info[imap_utils.GIMAPFetcher.EMAIL_BODY])
# parse header fields to extract subject and msgid
subject, msgid = self.parse_header_fields(
email_info[imap_utils.GIMAPFetcher.IMAP_HEADER_FIELDS])
#create json structure for metadata
meta_obj = {
self.ID_K:
email_info[imap_utils.GIMAPFetcher.GMAIL_ID],
self.LABELS_K:
email_info[imap_utils.GIMAPFetcher.GMAIL_LABELS],
self.FLAGS_K:
email_info[imap_utils.GIMAPFetcher.IMAP_FLAGS],
self.THREAD_IDS_K:
email_info[imap_utils.GIMAPFetcher.GMAIL_THREAD_ID],
self.INT_DATE_K:
gmvault_utils.datetime2e(
email_info[imap_utils.GIMAPFetcher.IMAP_INTERNALDATE]),
self.FLAGS_K:
email_info[imap_utils.GIMAPFetcher.IMAP_FLAGS],
self.SUBJECT_K:
subject,
self.MSGID_K:
msgid
}
json.dump(meta_obj, meta_desc, ensure_ascii=False)
meta_desc.flush()
meta_desc.close()
data_desc.flush()
data_desc.close()
return email_info[imap_utils.GIMAPFetcher.GMAIL_ID]
