def __init__(self, db_root_dir, host, port, login, credential, read_only_access = True, use_encryption = False): #pylint:disable-msg=R0913 """ constructor """ self.db_root_dir = db_root_dir #create dir if it doesn't exist gmvault_utils.makedirs(self.db_root_dir) #keep track of login email self.login = login # create source and try to connect self.src = imap_utils.GIMAPFetcher(host, port, login, credential, readonly_folder = read_only_access) self.src.connect() # enable compression if possible self.src.enable_compression() self.use_encryption = use_encryption #to report gmail imap problems self.error_report = { 'empty' : [] , 'cannot_be_fetched' : [], 'emails_in_quarantine' : [], 'reconnections' : 0}
def bury_metadata(self, email_info, local_dir=None): """ Store metadata info in .meta file Arguments: email_info: metadata info local_dir : intermdiary dir (month dir) """ if local_dir: the_dir = '%s/%s' % (self._db_dir, local_dir) gmvault_utils.makedirs(the_dir) else: the_dir = self._db_dir meta_path = self.METADATA_FNAME % ( the_dir, email_info[imap_utils.GIMAPFetcher.GMAIL_ID]) meta_desc = open(meta_path, 'w') # parse header fields to extract subject and msgid subject, msgid = self.parse_header_fields( email_info[imap_utils.GIMAPFetcher.IMAP_HEADER_FIELDS]) #create json structure for metadata meta_obj = { self.ID_K: email_info[imap_utils.GIMAPFetcher.GMAIL_ID], self.LABELS_K: email_info[imap_utils.GIMAPFetcher.GMAIL_LABELS], self.FLAGS_K: email_info[imap_utils.GIMAPFetcher.IMAP_FLAGS], self.THREAD_IDS_K: email_info[imap_utils.GIMAPFetcher.GMAIL_THREAD_ID], self.INT_DATE_K: gmvault_utils.datetime2e( email_info[imap_utils.GIMAPFetcher.IMAP_INTERNALDATE]), self.FLAGS_K: email_info[imap_utils.GIMAPFetcher.IMAP_FLAGS], self.SUBJECT_K: subject, self.MSGID_K: msgid } json.dump(meta_obj, meta_desc, ensure_ascii=False) meta_desc.flush() meta_desc.close() return email_info[imap_utils.GIMAPFetcher.GMAIL_ID]
def get_home_dir_path(cls): """ Get the Home dir """ gmvault_dir = os.getenv(cls.GMVAULT_DIR, None) # check by default in user[HOME] if not gmvault_dir: LOG.info("no ENV variable $GMVAULT_DIR defined. Set by default $GMVAULT_DIR to $HOME/.gmvault") gmvault_dir = "%s/.gmvault" % (os.getenv("HOME", ".")) #create dir if not there gmvault_utils.makedirs(gmvault_dir) return gmvault_dir
def get_home_dir_path(cls): """ Get the Home dir """ gmvault_dir = os.getenv(cls.GMVAULT_DIR, None) # check by default in user[HOME] if not gmvault_dir: LOG.info( "no ENV variable $GMVAULT_DIR defined. Set by default $GMVAULT_DIR to $HOME/.gmvault" ) gmvault_dir = "%s/.gmvault" % (os.getenv("HOME", ".")) #create dir if not there gmvault_utils.makedirs(gmvault_dir) return gmvault_dir
def bury_metadata(self, email_info, local_dir = None): """ Store metadata info in .meta file Arguments: email_info: metadata info local_dir : intermdiary dir (month dir) """ if local_dir: the_dir = '%s/%s' % (self._db_dir, local_dir) gmvault_utils.makedirs(the_dir) else: the_dir = self._db_dir meta_path = self.METADATA_FNAME % (the_dir, email_info[imap_utils.GIMAPFetcher.GMAIL_ID]) meta_desc = open(meta_path, 'w') # parse header fields to extract subject and msgid subject, msgid = self.parse_header_fields(email_info[imap_utils.GIMAPFetcher.IMAP_HEADER_FIELDS]) # need to convert labels that are number as string # come from imap_lib when label is a number labels = [ str(elem) for elem in email_info[imap_utils.GIMAPFetcher.GMAIL_LABELS] ] #create json structure for metadata meta_obj = { self.ID_K : email_info[imap_utils.GIMAPFetcher.GMAIL_ID], self.LABELS_K : labels, self.FLAGS_K : email_info[imap_utils.GIMAPFetcher.IMAP_FLAGS], self.THREAD_IDS_K : email_info[imap_utils.GIMAPFetcher.GMAIL_THREAD_ID], self.INT_DATE_K : gmvault_utils.datetime2e(email_info[imap_utils.GIMAPFetcher.IMAP_INTERNALDATE]), self.FLAGS_K : email_info[imap_utils.GIMAPFetcher.IMAP_FLAGS], self.SUBJECT_K : subject, self.MSGID_K : msgid } json.dump(meta_obj, meta_desc, ensure_ascii = False) meta_desc.flush() meta_desc.close() return email_info[imap_utils.GIMAPFetcher.GMAIL_ID]
def __init__(self, a_storage_dir, encrypt_data = False): """ Store on disks args: a_storage_dir: Storage directory a_use_encryption: Encryption key. If there then encrypt """ self._top_dir = a_storage_dir self._db_dir = '%s/%s' % (a_storage_dir, GmailStorer.DB_AREA) self._quarantine_dir = '%s/%s' % (a_storage_dir, GmailStorer.QUARANTINE_AREA) self._info_dir = '%s/%s' % (a_storage_dir, GmailStorer.INFO_AREA) #make dirs if not os.path.exists(self._db_dir): LOG.critical("No Storage DB in %s. Create it.\n" % (a_storage_dir)) gmvault_utils.makedirs(self._db_dir) gmvault_utils.makedirs(self._quarantine_dir) gmvault_utils.makedirs(self._info_dir) self.fsystem_info_cache = {} self._encrypt_data = encrypt_data self._encryption_key = None self._cipher = None
def __init__(self, a_storage_dir, encrypt_data=False): """ Store on disks args: a_storage_dir: Storage directory a_use_encryption: Encryption key. If there then encrypt """ self._top_dir = a_storage_dir self._db_dir = '%s/%s' % (a_storage_dir, GmailStorer.DB_AREA) self._quarantine_dir = '%s/%s' % (a_storage_dir, GmailStorer.QUARANTINE_AREA) self._info_dir = '%s/%s' % (a_storage_dir, GmailStorer.INFO_AREA) #make dirs if not os.path.exists(self._db_dir): LOG.critical("No Storage DB in %s. Create it.\n" % (a_storage_dir)) gmvault_utils.makedirs(self._db_dir) gmvault_utils.makedirs(self._quarantine_dir) gmvault_utils.makedirs(self._info_dir) self.fsystem_info_cache = {} self._encrypt_data = encrypt_data self._encryption_key = None self._cipher = None
def __init__(self, db_root_dir, host, port, login, credential, read_only_access=True, use_encryption=False): #pylint:disable-msg=R0913 """ constructor """ self.db_root_dir = db_root_dir #create dir if it doesn't exist gmvault_utils.makedirs(self.db_root_dir) #keep track of login email self.login = login # create source and try to connect self.src = imap_utils.GIMAPFetcher(host, port, login, credential, readonly_folder=read_only_access) self.src.connect() # enable compression if possible self.src.enable_compression() self.use_encryption = use_encryption #to report gmail imap problems self.error_report = { 'empty': [], 'cannot_be_fetched': [], 'emails_in_quarantine': [], 'reconnections': 0 }
def bury_email(self, email_info, local_dir = None, compress = False): """ store all email info in 2 files (.meta and .eml files) Arguments: email_info: info local_dir : intermdiary dir (month dir) compress : If compress is True, use gzip compression """ if local_dir: the_dir = '%s/%s' % (self._db_dir, local_dir) gmvault_utils.makedirs(the_dir) else: the_dir = self._db_dir meta_path = self.METADATA_FNAME % (the_dir, email_info[imap_utils.GIMAPFetcher.GMAIL_ID]) data_path = self.DATA_FNAME % (the_dir, email_info[imap_utils.GIMAPFetcher.GMAIL_ID]) # if the data has to be encrypted if self._encrypt_data: data_path = '%s.crypt' % (data_path) if compress: data_path = '%s.gz' % (data_path) data_desc = gzip.open(data_path, 'wb') else: data_desc = open(data_path, 'wb') meta_desc = open(meta_path, 'w') if self._encrypt_data: # need to be done for every encryption cipher = self.get_encryption_cipher() cipher.initCTR() data_desc.write(cipher.encryptCTR(email_info[imap_utils.GIMAPFetcher.EMAIL_BODY])) else: data_desc.write(email_info[imap_utils.GIMAPFetcher.EMAIL_BODY]) # parse header fields to extract subject and msgid subject, msgid = self.parse_header_fields(email_info[imap_utils.GIMAPFetcher.IMAP_HEADER_FIELDS]) #create json structure for metadata meta_obj = { self.ID_K : email_info[imap_utils.GIMAPFetcher.GMAIL_ID], self.LABELS_K : email_info[imap_utils.GIMAPFetcher.GMAIL_LABELS], self.FLAGS_K : email_info[imap_utils.GIMAPFetcher.IMAP_FLAGS], self.THREAD_IDS_K : email_info[imap_utils.GIMAPFetcher.GMAIL_THREAD_ID], self.INT_DATE_K : gmvault_utils.datetime2e(email_info[imap_utils.GIMAPFetcher.IMAP_INTERNALDATE]), self.FLAGS_K : email_info[imap_utils.GIMAPFetcher.IMAP_FLAGS], self.SUBJECT_K : subject, self.MSGID_K : msgid } json.dump(meta_obj, meta_desc, ensure_ascii = False) meta_desc.flush() meta_desc.close() data_desc.flush() data_desc.close() return email_info[imap_utils.GIMAPFetcher.GMAIL_ID]
def bury_email(self, email_info, local_dir=None, compress=False): """ store all email info in 2 files (.meta and .eml files) Arguments: email_info: info local_dir : intermdiary dir (month dir) compress : If compress is True, use gzip compression """ if local_dir: the_dir = '%s/%s' % (self._db_dir, local_dir) gmvault_utils.makedirs(the_dir) else: the_dir = self._db_dir meta_path = self.METADATA_FNAME % ( the_dir, email_info[imap_utils.GIMAPFetcher.GMAIL_ID]) data_path = self.DATA_FNAME % ( the_dir, email_info[imap_utils.GIMAPFetcher.GMAIL_ID]) # if the data has to be encrypted if self._encrypt_data: data_path = '%s.crypt' % (data_path) if compress: data_path = '%s.gz' % (data_path) data_desc = gzip.open(data_path, 'wb') else: data_desc = open(data_path, 'wb') meta_desc = open(meta_path, 'w') if self._encrypt_data: # need to be done for every encryption cipher = self.get_encryption_cipher() cipher.initCTR() data_desc.write( cipher.encryptCTR( email_info[imap_utils.GIMAPFetcher.EMAIL_BODY])) else: data_desc.write(email_info[imap_utils.GIMAPFetcher.EMAIL_BODY]) # parse header fields to extract subject and msgid subject, msgid = self.parse_header_fields( email_info[imap_utils.GIMAPFetcher.IMAP_HEADER_FIELDS]) #create json structure for metadata meta_obj = { self.ID_K: email_info[imap_utils.GIMAPFetcher.GMAIL_ID], self.LABELS_K: email_info[imap_utils.GIMAPFetcher.GMAIL_LABELS], self.FLAGS_K: email_info[imap_utils.GIMAPFetcher.IMAP_FLAGS], self.THREAD_IDS_K: email_info[imap_utils.GIMAPFetcher.GMAIL_THREAD_ID], self.INT_DATE_K: gmvault_utils.datetime2e( email_info[imap_utils.GIMAPFetcher.IMAP_INTERNALDATE]), self.FLAGS_K: email_info[imap_utils.GIMAPFetcher.IMAP_FLAGS], self.SUBJECT_K: subject, self.MSGID_K: msgid } json.dump(meta_obj, meta_desc, ensure_ascii=False) meta_desc.flush() meta_desc.close() data_desc.flush() data_desc.close() return email_info[imap_utils.GIMAPFetcher.GMAIL_ID]