def __init__(self, db_root_dir, host, port, login, credential, read_only_access = True, use_encryption = False): #pylint:disable-msg=R0913
"""
constructor
"""
self.db_root_dir = db_root_dir
#create dir if it doesn't exist
gmvault_utils.makedirs(self.db_root_dir)
#keep track of login email
self.login = login
# create source and try to connect
self.src = imap_utils.GIMAPFetcher(host, port, login, credential, readonly_folder = read_only_access)
self.src.connect()
# enable compression if possible
self.src.enable_compression()
self.use_encryption = use_encryption
#to report gmail imap problems
self.error_report = { 'empty' : [] ,
'cannot_be_fetched' : [],
'emails_in_quarantine' : [],
'reconnections' : 0}
def bury_metadata(self, email_info, local_dir=None):
"""
Store metadata info in .meta file
Arguments:
email_info: metadata info
local_dir : intermdiary dir (month dir)
"""
if local_dir:
the_dir = '%s/%s' % (self._db_dir, local_dir)
gmvault_utils.makedirs(the_dir)
else:
the_dir = self._db_dir
meta_path = self.METADATA_FNAME % (
the_dir, email_info[imap_utils.GIMAPFetcher.GMAIL_ID])
meta_desc = open(meta_path, 'w')
# parse header fields to extract subject and msgid
subject, msgid = self.parse_header_fields(
email_info[imap_utils.GIMAPFetcher.IMAP_HEADER_FIELDS])
#create json structure for metadata
meta_obj = {
self.ID_K:
email_info[imap_utils.GIMAPFetcher.GMAIL_ID],
self.LABELS_K:
email_info[imap_utils.GIMAPFetcher.GMAIL_LABELS],
self.FLAGS_K:
email_info[imap_utils.GIMAPFetcher.IMAP_FLAGS],
self.THREAD_IDS_K:
email_info[imap_utils.GIMAPFetcher.GMAIL_THREAD_ID],
self.INT_DATE_K:
gmvault_utils.datetime2e(
email_info[imap_utils.GIMAPFetcher.IMAP_INTERNALDATE]),
self.FLAGS_K:
email_info[imap_utils.GIMAPFetcher.IMAP_FLAGS],
self.SUBJECT_K:
subject,
self.MSGID_K:
msgid
}
json.dump(meta_obj, meta_desc, ensure_ascii=False)
meta_desc.flush()
meta_desc.close()
return email_info[imap_utils.GIMAPFetcher.GMAIL_ID]
def get_home_dir_path(cls):
"""
Get the Home dir
"""
gmvault_dir = os.getenv(cls.GMVAULT_DIR, None)
# check by default in user[HOME]
if not gmvault_dir:
LOG.info("no ENV variable $GMVAULT_DIR defined. Set by default $GMVAULT_DIR to $HOME/.gmvault")
gmvault_dir = "%s/.gmvault" % (os.getenv("HOME", "."))
#create dir if not there
gmvault_utils.makedirs(gmvault_dir)
return gmvault_dir
def get_home_dir_path(cls):
"""
Get the Home dir
"""
gmvault_dir = os.getenv(cls.GMVAULT_DIR, None)
# check by default in user[HOME]
if not gmvault_dir:
LOG.info(
"no ENV variable $GMVAULT_DIR defined. Set by default $GMVAULT_DIR to $HOME/.gmvault"
)
gmvault_dir = "%s/.gmvault" % (os.getenv("HOME", "."))
#create dir if not there
gmvault_utils.makedirs(gmvault_dir)
return gmvault_dir
def bury_metadata(self, email_info, local_dir = None):
"""
Store metadata info in .meta file
Arguments:
email_info: metadata info
local_dir : intermdiary dir (month dir)
"""
if local_dir:
the_dir = '%s/%s' % (self._db_dir, local_dir)
gmvault_utils.makedirs(the_dir)
else:
the_dir = self._db_dir
meta_path = self.METADATA_FNAME % (the_dir, email_info[imap_utils.GIMAPFetcher.GMAIL_ID])
meta_desc = open(meta_path, 'w')
# parse header fields to extract subject and msgid
subject, msgid = self.parse_header_fields(email_info[imap_utils.GIMAPFetcher.IMAP_HEADER_FIELDS])
# need to convert labels that are number as string
# come from imap_lib when label is a number
labels = [ str(elem) for elem in email_info[imap_utils.GIMAPFetcher.GMAIL_LABELS] ]
#create json structure for metadata
meta_obj = {
self.ID_K : email_info[imap_utils.GIMAPFetcher.GMAIL_ID],
self.LABELS_K : labels,
self.FLAGS_K : email_info[imap_utils.GIMAPFetcher.IMAP_FLAGS],
self.THREAD_IDS_K : email_info[imap_utils.GIMAPFetcher.GMAIL_THREAD_ID],
self.INT_DATE_K : gmvault_utils.datetime2e(email_info[imap_utils.GIMAPFetcher.IMAP_INTERNALDATE]),
self.FLAGS_K : email_info[imap_utils.GIMAPFetcher.IMAP_FLAGS],
self.SUBJECT_K : subject,
self.MSGID_K : msgid
}
json.dump(meta_obj, meta_desc, ensure_ascii = False)
meta_desc.flush()
meta_desc.close()
return email_info[imap_utils.GIMAPFetcher.GMAIL_ID]
def __init__(self, a_storage_dir, encrypt_data = False):
"""
Store on disks
args:
a_storage_dir: Storage directory
a_use_encryption: Encryption key. If there then encrypt
"""
self._top_dir = a_storage_dir
self._db_dir = '%s/%s' % (a_storage_dir, GmailStorer.DB_AREA)
self._quarantine_dir = '%s/%s' % (a_storage_dir, GmailStorer.QUARANTINE_AREA)
self._info_dir = '%s/%s' % (a_storage_dir, GmailStorer.INFO_AREA)
#make dirs
if not os.path.exists(self._db_dir):
LOG.critical("No Storage DB in %s. Create it.\n" % (a_storage_dir))
gmvault_utils.makedirs(self._db_dir)
gmvault_utils.makedirs(self._quarantine_dir)
gmvault_utils.makedirs(self._info_dir)
self.fsystem_info_cache = {}
self._encrypt_data = encrypt_data
self._encryption_key = None
self._cipher = None
def __init__(self, a_storage_dir, encrypt_data=False):
"""
Store on disks
args:
a_storage_dir: Storage directory
a_use_encryption: Encryption key. If there then encrypt
"""
self._top_dir = a_storage_dir
self._db_dir = '%s/%s' % (a_storage_dir, GmailStorer.DB_AREA)
self._quarantine_dir = '%s/%s' % (a_storage_dir,
GmailStorer.QUARANTINE_AREA)
self._info_dir = '%s/%s' % (a_storage_dir, GmailStorer.INFO_AREA)
#make dirs
if not os.path.exists(self._db_dir):
LOG.critical("No Storage DB in %s. Create it.\n" % (a_storage_dir))
gmvault_utils.makedirs(self._db_dir)
gmvault_utils.makedirs(self._quarantine_dir)
gmvault_utils.makedirs(self._info_dir)
self.fsystem_info_cache = {}
self._encrypt_data = encrypt_data
self._encryption_key = None
self._cipher = None
def __init__(self,
db_root_dir,
host,
port,
login,
credential,
read_only_access=True,
use_encryption=False): #pylint:disable-msg=R0913
"""
constructor
"""
self.db_root_dir = db_root_dir
#create dir if it doesn't exist
gmvault_utils.makedirs(self.db_root_dir)
#keep track of login email
self.login = login
# create source and try to connect
self.src = imap_utils.GIMAPFetcher(host,
port,
login,
credential,
readonly_folder=read_only_access)
self.src.connect()
# enable compression if possible
self.src.enable_compression()
self.use_encryption = use_encryption
#to report gmail imap problems
self.error_report = {
'empty': [],
'cannot_be_fetched': [],
'emails_in_quarantine': [],
'reconnections': 0
}
def bury_email(self, email_info, local_dir = None, compress = False):
"""
store all email info in 2 files (.meta and .eml files)
Arguments:
email_info: info
local_dir : intermdiary dir (month dir)
compress : If compress is True, use gzip compression
"""
if local_dir:
the_dir = '%s/%s' % (self._db_dir, local_dir)
gmvault_utils.makedirs(the_dir)
else:
the_dir = self._db_dir
meta_path = self.METADATA_FNAME % (the_dir, email_info[imap_utils.GIMAPFetcher.GMAIL_ID])
data_path = self.DATA_FNAME % (the_dir, email_info[imap_utils.GIMAPFetcher.GMAIL_ID])
# if the data has to be encrypted
if self._encrypt_data:
data_path = '%s.crypt' % (data_path)
if compress:
data_path = '%s.gz' % (data_path)
data_desc = gzip.open(data_path, 'wb')
else:
data_desc = open(data_path, 'wb')
meta_desc = open(meta_path, 'w')
if self._encrypt_data:
# need to be done for every encryption
cipher = self.get_encryption_cipher()
cipher.initCTR()
data_desc.write(cipher.encryptCTR(email_info[imap_utils.GIMAPFetcher.EMAIL_BODY]))
else:
data_desc.write(email_info[imap_utils.GIMAPFetcher.EMAIL_BODY])
# parse header fields to extract subject and msgid
subject, msgid = self.parse_header_fields(email_info[imap_utils.GIMAPFetcher.IMAP_HEADER_FIELDS])
#create json structure for metadata
meta_obj = {
self.ID_K : email_info[imap_utils.GIMAPFetcher.GMAIL_ID],
self.LABELS_K : email_info[imap_utils.GIMAPFetcher.GMAIL_LABELS],
self.FLAGS_K : email_info[imap_utils.GIMAPFetcher.IMAP_FLAGS],
self.THREAD_IDS_K : email_info[imap_utils.GIMAPFetcher.GMAIL_THREAD_ID],
self.INT_DATE_K : gmvault_utils.datetime2e(email_info[imap_utils.GIMAPFetcher.IMAP_INTERNALDATE]),
self.FLAGS_K : email_info[imap_utils.GIMAPFetcher.IMAP_FLAGS],
self.SUBJECT_K : subject,
self.MSGID_K : msgid
}
json.dump(meta_obj, meta_desc, ensure_ascii = False)
meta_desc.flush()
meta_desc.close()
data_desc.flush()
data_desc.close()
return email_info[imap_utils.GIMAPFetcher.GMAIL_ID]
def bury_email(self, email_info, local_dir=None, compress=False):
"""
store all email info in 2 files (.meta and .eml files)
Arguments:
email_info: info
local_dir : intermdiary dir (month dir)
compress : If compress is True, use gzip compression
"""
if local_dir:
the_dir = '%s/%s' % (self._db_dir, local_dir)
gmvault_utils.makedirs(the_dir)
else:
the_dir = self._db_dir
meta_path = self.METADATA_FNAME % (
the_dir, email_info[imap_utils.GIMAPFetcher.GMAIL_ID])
data_path = self.DATA_FNAME % (
the_dir, email_info[imap_utils.GIMAPFetcher.GMAIL_ID])
# if the data has to be encrypted
if self._encrypt_data:
data_path = '%s.crypt' % (data_path)
if compress:
data_path = '%s.gz' % (data_path)
data_desc = gzip.open(data_path, 'wb')
else:
data_desc = open(data_path, 'wb')
meta_desc = open(meta_path, 'w')
if self._encrypt_data:
# need to be done for every encryption
cipher = self.get_encryption_cipher()
cipher.initCTR()
data_desc.write(
cipher.encryptCTR(
email_info[imap_utils.GIMAPFetcher.EMAIL_BODY]))
else:
data_desc.write(email_info[imap_utils.GIMAPFetcher.EMAIL_BODY])
# parse header fields to extract subject and msgid
subject, msgid = self.parse_header_fields(
email_info[imap_utils.GIMAPFetcher.IMAP_HEADER_FIELDS])
#create json structure for metadata
meta_obj = {
self.ID_K:
email_info[imap_utils.GIMAPFetcher.GMAIL_ID],
self.LABELS_K:
email_info[imap_utils.GIMAPFetcher.GMAIL_LABELS],
self.FLAGS_K:
email_info[imap_utils.GIMAPFetcher.IMAP_FLAGS],
self.THREAD_IDS_K:
email_info[imap_utils.GIMAPFetcher.GMAIL_THREAD_ID],
self.INT_DATE_K:
gmvault_utils.datetime2e(
email_info[imap_utils.GIMAPFetcher.IMAP_INTERNALDATE]),
self.FLAGS_K:
email_info[imap_utils.GIMAPFetcher.IMAP_FLAGS],
self.SUBJECT_K:
subject,
self.MSGID_K:
msgid
}
json.dump(meta_obj, meta_desc, ensure_ascii=False)
meta_desc.flush()
meta_desc.close()
data_desc.flush()
data_desc.close()
return email_info[imap_utils.GIMAPFetcher.GMAIL_ID]
