def wrapper(*args, **kwargs): #pylint:disable-msg=C0111
nb_tries = [0] # make it mutable in reconnect
m_sleep_time = [a_sleep_time] #make it mutable in reconnect
while True:
try:
return the_func(*args, **kwargs)
except PushEmailError, p_err:
LOG.debug("error message = %s. traceback:%s" %
(p_err, gmvault_utils.get_exception_traceback()))
LOG.critical(
"Cannot reach the gmail server (see logs). Wait %s seconds and retrying."
% (m_sleep_time[0]))
reconnect(args[0], nb_tries, p_err, m_sleep_time)
except imaplib.IMAP4.abort, err: #abort is recoverable and error is not
LOG.debug("error message = %s. traceback:%s" %
(err, gmvault_utils.get_exception_traceback()))
LOG.critical(
"Cannot reach the gmail server (see logs). Wait 1 seconds and retrying"
)
# problem with this email, put it in quarantine
reconnect(args[0], nb_tries, err, m_sleep_time)
def inner_retry(the_func): #pylint:disable-msg=C0111
def wrapper(*args, **kwargs): #pylint:disable-msg=C0111
nb_tries = [0] # make it mutable in reconnect
m_sleep_time = [a_sleep_time] #make it mutable in reconnect
while True:
try:
return the_func(*args, **kwargs)
except PushEmailError, p_err:
LOG.debug("error message = %s. traceback:%s" % (p_err, gmvault_utils.get_exception_traceback()))
LOG.critical("Cannot reach the gmail server (see logs). Wait %s seconds and retrying." % (m_sleep_time[0]))
reconnect(args[0], nb_tries, p_err, m_sleep_time)
except imaplib.IMAP4.abort, err: #abort is recoverable and error is not
LOG.debug("error message = %s. traceback:%s" % (err, gmvault_utils.get_exception_traceback()))
LOG.critical("Cannot reach the gmail server (see logs). Wait 1 seconds and retrying")
# problem with this email, put it in quarantine
reconnect(args[0], nb_tries, err, m_sleep_time)
except socket.error, sock_err:
LOG.debug("error message = %s. traceback:%s" % (sock_err, gmvault_utils.get_exception_traceback()))
LOG.critical("Cannot reach the gmail server (see logs). Wait 1 seconds and retrying")
reconnect(args[0], nb_tries, sock_err, m_sleep_time)
def wrapper(*args, **kwargs): #pylint:disable-msg=C0111
nb_tries = [0] # make it mutable in reconnect
m_sleep_time = [a_sleep_time] #make it mutable in reconnect
while True:
try:
return the_func(*args, **kwargs)
except PushEmailError, p_err:
LOG.debug("error message = %s. traceback:%s" % (p_err, gmvault_utils.get_exception_traceback()))
if nb_tries[0] < a_nb_tries:
LOG.critical("Cannot reach the Gmail server. Wait %s seconds and retrying." % (m_sleep_time[0]))
else:
LOG.critical("Stop retrying, tried too many times ...")
reconnect(args[0], nb_tries, a_nb_tries, p_err, m_sleep_time)
except imaplib.IMAP4.abort, err: #abort is recoverable and error is not
LOG.debug("IMAP (abort) error message = %s. traceback:%s" % (err, gmvault_utils.get_exception_traceback()))
if nb_tries[0] < a_nb_tries:
LOG.critical("Received an IMAP abort error. Wait %s seconds and retrying." % (m_sleep_time[0]))
else:
LOG.critical("Stop retrying, tried too many times ...")
# problem with this email, put it in quarantine
reconnect(args[0], nb_tries, a_nb_tries, err, m_sleep_time)
def get_oauth_tok_sec(email, use_webbrowser = False, debug=False):
'''
Generate token and secret
'''
scopes = ['https://mail.google.com/', # IMAP/SMTP client access
'https://www.googleapis.com/auth/userinfo#email'] # Email address access (verify token authorized by correct account
gdata_serv = gdata.service.GDataService()
gdata_serv.debug = debug
gdata_serv.source = 'gmvault '
gdata_serv.SetOAuthInputParameters(gdata.auth.OAuthSignatureMethod.HMAC_SHA1, \
consumer_key = 'anonymous', consumer_secret = 'anonymous')
params = {'xoauth_displayname':'gmvault - Backup your Gmail account'}
try:
request_token = gdata_serv.FetchOAuthRequestToken(scopes=scopes, extra_parameters = params)
except gdata.service.FetchingOAuthRequestTokenFailed, err:
if str(err).find('Timestamp') != -1:
LOG.critical('Is your system clock up to date? See the FAQ http://code.google.com/p/googlecl/wiki/FAQ'\
'#Timestamp_too_far_from_current_time\n')
LOG.critical("Received Error: %s.\n" % (err) )
LOG.critical("=== Exception traceback ===")
LOG.critical(gmvault_utils.get_exception_traceback())
LOG.critical("=== End of Exception traceback ===\n")
return (None, None)
def read_oauth_tok_sec(cls, email):
"""
Read oauth token secret credential
Look by default to ~/.gmvault
Look for file ~/.gmvault/email.oauth
"""
gmv_dir = gmvault_utils.get_home_dir_path()
#look for email.passwed in GMV_DIR
user_oauth_file_path = "%s/%s.oauth" % (gmv_dir, email)
token = None
secret = None
if os.path.exists(user_oauth_file_path):
LOG.critical("Get XOAuth credential from %s.\n" % (user_oauth_file_path))
oauth_file = open(user_oauth_file_path)
try:
oauth_result = oauth_file.read()
if oauth_result:
oauth_result = oauth_result.split('::')
if len(oauth_result) == 2:
token = oauth_result[0]
secret = oauth_result[1]
except Exception, _: #pylint: disable-msg=W0703
LOG.critical("Cannot read oauth credentials from %s. Force oauth credentials renewal." % (user_oauth_file_path))
LOG.critical("=== Exception traceback ===")
LOG.critical(gmvault_utils.get_exception_traceback())
LOG.critical("=== End of Exception traceback ===\n")
def create_gmail_labels(self, labels, existing_folders):
"""
Create folders and subfolders on Gmail in order
to recreate the label hierarchy before to upload emails
Note that adding labels with +X-GM-LABELS create only nested labels
but not nested ones. This is why this trick must be used to
recreate the label hierarchy
labels: list of labels to create
"""
#1.5-beta moved that out of the loop to minimize the number of calls
#to that method. (Could go further and memoize it)
#get existing directories (or label parts)
# get in lower case because Gmail labels are case insensitive
#listed_folders = set([ directory.lower() for (_, _, directory) in self.server.list_folders() ])
listed_folders = set([
directory.lower() for (_, _, directory) in self.get_all_folders()
])
existing_folders = listed_folders.union(existing_folders)
for lab in labels:
#split all labels
labs = self._get_dir_from_labels(lab)
for directory in labs:
low_directory = directory.lower(
) #get lower case directory but store original label
if (low_directory not in existing_folders) and (
low_directory not in self.GMAIL_SPECIAL_DIRS_LOWER):
try:
if self.server.create_folder(directory) != 'Success':
raise Exception(
"Cannot create label %s: the directory %s cannot be created."
% (lab, directory))
else:
LOG.debug(
"============== ####### Created Labels %s." %
(directory))
except imaplib.IMAP4.error, error:
#log error in log file if it exists
LOG.debug(gmvault_utils.get_exception_traceback())
if str(error).startswith(
"create failed: '[ALREADYEXISTS] Duplicate folder"
):
LOG.critical(
"Warning: label %s already exists on Gmail and Gmvault tried to create it. Ignore this issue."
% (directory))
else:
raise error
#add created folder in folders
existing_folders.add(low_directory)
def inner_retry(the_func): #pylint:disable-msg=C0111
def wrapper(*args, **kwargs): #pylint:disable-msg=C0111
nb_tries = [0] # make it mutable in reconnect
m_sleep_time = [a_sleep_time] #make it mutable in reconnect
while True:
try:
return the_func(*args, **kwargs)
except PushEmailError, p_err:
LOG.debug("error message = %s. traceback:%s" %
(p_err, gmvault_utils.get_exception_traceback()))
if nb_tries[0] < a_nb_tries:
LOG.critical(
"Cannot reach the Gmail server. Wait %s seconds and retrying."
% (m_sleep_time[0]))
else:
LOG.critical("Stop retrying, tried too many times ...")
reconnect(args[0], nb_tries, a_nb_tries, p_err,
m_sleep_time)
except imaplib.IMAP4.abort, err: #abort is recoverable and error is not
LOG.debug("IMAP (abort) error message = %s. traceback:%s" %
(err, gmvault_utils.get_exception_traceback()))
if nb_tries[0] < a_nb_tries:
LOG.critical(
"Received an IMAP abort error. Wait %s seconds and retrying."
% (m_sleep_time[0]))
else:
LOG.critical("Stop retrying, tried too many times ...")
# problem with this email, put it in quarantine
reconnect(args[0], nb_tries, a_nb_tries, err, m_sleep_time)
except socket.error, sock_err:
LOG.debug(
"error message = %s. traceback:%s" %
(sock_err, gmvault_utils.get_exception_traceback()))
LOG.critical(
"Cannot reach the Gmail server. Wait %s seconds and retrying."
% (m_sleep_time[0]))
reconnect(args[0], nb_tries, a_nb_tries, sock_err,
m_sleep_time)
def create_gmail_labels(self, labels, existing_folders):
"""
Create folders and subfolders on Gmail in order
to recreate the label hierarchy before to upload emails
Note that adding labels with +X-GM-LABELS create only nested labels
but not nested ones. This is why this trick must be used to
recreate the label hierarchy
labels: list of labels to create
"""
#1.5-beta moved that out of the loop to minimize the number of calls
#to that method. (Could go further and memoize it)
#get existing directories (or label parts)
# get in lower case because Gmail labels are case insensitive
#listed_folders = set([ directory.lower() for (_, _, directory) in self.server.list_folders() ])
listed_folders = set([ directory.lower() for (_, _, directory) in self.get_all_folders() ])
existing_folders = listed_folders.union(existing_folders)
for lab in labels:
#split all labels
labs = self._get_dir_from_labels(lab)
for directory in labs:
low_directory = directory.lower() #get lower case directory but store original label
if (low_directory not in existing_folders) and (low_directory not in self.GMAIL_SPECIAL_DIRS_LOWER):
try:
if self.server.create_folder(directory) != 'Success':
raise Exception("Cannot create label %s: the directory %s cannot be created." % (lab, directory))
else:
LOG.debug("============== ####### Created Labels %s." % (directory))
except imaplib.IMAP4.error, error:
#log error in log file if it exists
LOG.debug(gmvault_utils.get_exception_traceback())
if str(error).startswith("create failed: '[ALREADYEXISTS] Duplicate folder"):
LOG.critical("Warning: label %s already exists on Gmail and Gmvault tried to create it. Ignore this issue." % (directory) )
else:
raise error
#add created folder in folders
existing_folders.add(low_directory)
def delete_gmail_labels(self, labels, force_delete = False):
"""
Delete passed labels. Beware experimental and labels must be ordered
"""
for label in reversed(labels):
labs = self._get_dir_from_labels(label)
for directory in reversed(labs):
#listed_folders = set([ repertoire.lower() for (flag, delimiter, repertoire) in self.server.xlist_folders() ])
#print("Existing folders on server side = %s\n" % (listed_folders))
if force_delete or ( (directory.lower() not in self.GMAIL_SPECIAL_DIRS_LOWER) and self.server.folder_exists(directory) ): #call server exists each time
try:
self.server.delete_folder(directory)
except imaplib.IMAP4.error, _:
LOG.debug(gmvault_utils.get_exception_traceback())
def delete_gmail_labels(self, labels, force_delete=False):
"""
Delete passed labels. Beware experimental and labels must be ordered
"""
for label in reversed(labels):
labs = self._get_dir_from_labels(label)
for directory in reversed(labs):
#listed_folders = set([ repertoire.lower() for (flag, delimiter, repertoire) in self.server.xlist_folders() ])
#print("Existing folders on server side = %s\n" % (listed_folders))
if force_delete or (
(directory.lower() not in self.GMAIL_SPECIAL_DIRS_LOWER)
and self.server.folder_exists(directory)
): #call server exists each time
try:
self.server.delete_folder(directory)
except imaplib.IMAP4.error, _:
LOG.debug(gmvault_utils.get_exception_traceback())
LOG.critical("Cannot reach the gmail server (see logs). Wait 1 seconds and retrying")
# problem with this email, put it in quarantine
reconnect(args[0], nb_tries, err, m_sleep_time)
except socket.error, sock_err:
LOG.debug("error message = %s. traceback:%s" % (sock_err, gmvault_utils.get_exception_traceback()))
LOG.critical("Cannot reach the gmail server (see logs). Wait 1 seconds and retrying")
reconnect(args[0], nb_tries, sock_err, m_sleep_time)
except imaplib.IMAP4.error, err:
#just trace it back for the moment
LOG.debug("IMAP (normal) error message = %s. traceback:%s" % (err, gmvault_utils.get_exception_traceback()))
raise err
return functools.wraps(the_func)(wrapper)
#return wrapper
return inner_retry
class GIMAPFetcher(object): #pylint:disable-msg=R0902
'''
IMAP Class reading the information
'''
GMAIL_EXTENSION = 'X-GM-EXT-1' # GMAIL capability
GMAIL_ALL = '[Gmail]/All Mail' #GMAIL All Mail mailbox
GOOGLE_MAIL_ALL = '[Google Mail]/All Mail' #Google Mail All Mail mailbox for Germany
GMAIL_ID = 'X-GM-MSGID' #GMAIL ID attribute
self._restore(args, credential)
elif args.get('command', '') == 'config':
LOG.critical("Configure something. TBD.\n")
on_error = False
except KeyboardInterrupt, _:
LOG.critical("\nCRTL^C. Stop all operations.\n")
on_error = False
except socket.error:
LOG.critical("Error: Network problem. Please check your gmail server hostname, the internet connection or your network setup.\n")
LOG.critical("=== Exception traceback ===")
LOG.critical(gmvault_utils.get_exception_traceback())
LOG.critical("=== End of Exception traceback ===\n")
die_with_usage = False
except imaplib.IMAP4.error, imap_err:
#bad login or password
if str(imap_err) in ['[AUTHENTICATIONFAILED] Invalid credentials (Failure)', \
'[ALERT] Web login required: http://support.google.com/mail/bin/answer.py?answer=78754 (Failure)', \
'[ALERT] Invalid credentials (Failure)'] :
LOG.critical("ERROR: Invalid credentials, cannot login to the gmail server. Please check your login and password or xoauth token.\n")
die_with_usage = False
else:
LOG.critical("Error: %s.\n" % (imap_err) )
LOG.critical("=== Exception traceback ===")
LOG.critical(gmvault_utils.get_exception_traceback())
LOG.critical("=== End of Exception traceback ===\n")
except Exception, err:
LOG.debug(
"error message = %s. traceback:%s" %
(sock_err, gmvault_utils.get_exception_traceback()))
LOG.critical(
"Cannot reach the gmail server (see logs). Wait 1 seconds and retrying"
)
reconnect(args[0], nb_tries, sock_err, m_sleep_time)
except imaplib.IMAP4.error, err:
#just trace it back for the moment
LOG.debug(
"IMAP (normal) error message = %s. traceback:%s" %
(err, gmvault_utils.get_exception_traceback()))
raise err
return functools.wraps(the_func)(wrapper)
#return wrapper
return inner_retry
class GIMAPFetcher(object): #pylint:disable-msg=R0902
'''
IMAP Class reading the information
'''
GMAIL_EXTENSION = 'X-GM-EXT-1' # GMAIL capability
GMAIL_ALL = '[Gmail]/All Mail' #GMAIL All Mail mailbox
LOG.critical("\nCRTL^C. Stop all operations.\n")
on_error = False
except socket.error:
LOG.critical("ERROR: Network problem. Please check your gmail server hostname, the internet connection or your network setup.")
LOG.critical("For more information see log file.\n")
die_with_usage = False
except imaplib.IMAP4.error, imap_err:
#bad login or password
if str(imap_err) in ['[AUTHENTICATIONFAILED] Invalid credentials (Failure)', \
'[ALERT] Web login required: http://support.google.com/mail/bin/answer.py?answer=78754 (Failure)', \
'[ALERT] Invalid credentials (Failure)'] :
LOG.critical("ERROR: Invalid credentials, cannot login to the gmail server. Please check your login and password or xoauth token.\n")
die_with_usage = False
else:
LOG.critical("Error %s. For more information see log file\n" % (imap_err) )
LOG.exception(gmvault_utils.get_exception_traceback())
except Exception, err:
LOG.critical("Error %s. For more information see log file\n" % (err) )
LOG.exception(gmvault_utils.get_exception_traceback())
finally:
if on_error and die_with_usage:
args['parser'].die_with_usage()
def init_logging():
"""
init logging infrastructure
"""
#setup application logs: one handler for stdout and one for a log file
log_utils.LoggerFactory.setup_cli_app_handler(activate_log_file=False, file_path="./gmvault.log")
def activate_debug_mode():
def _create_update_sync(self, imap_ids, compress, ownership_control = True ):
"""
First part of the double pass strategy:
create and update emails in db
"""
gstorer = GmailStorer(self.db_root_dir, self.use_encryption)
#check ownership
self._check_email_db_ownership(gstorer, ownership_control)
#save db_owner for next time
gstorer.store_db_owner(self.login)
total_nb_emails_to_process = len(imap_ids) # total number of emails to get
LOG.critical("%d emails to be fetched." % (total_nb_emails_to_process))
nb_emails_processed = 0
timer = gmvault_utils.Timer() # needed for enhancing the user information
timer.start()
for the_id in imap_ids:
try:
gid = None
LOG.debug("\nProcess imap id %s" % ( the_id ))
#if the_id == 12:
# print("we have to break")
#get everything but data
new_data = self.src.fetch(the_id, imap_utils.GIMAPFetcher.GET_ALL_BUT_DATA )
#print("data = %s" %(new_data[the_id]))
#if 0 in new_data[the_id][imap_utils.GIMAPFetcher.GMAIL_LABELS]:
# print("we have to break")
if new_data.get(the_id, None):
gid = new_data[the_id][imap_utils.GIMAPFetcher.GMAIL_ID]
the_dir = gmvault_utils.get_ym_from_datetime(new_data[the_id][imap_utils.GIMAPFetcher.IMAP_INTERNALDATE])
LOG.critical("Process email num %d (imap_id:%s) from %s." % (nb_emails_processed, the_id, the_dir))
#pass the dir and the ID
curr_metadata = GMVaulter.check_email_on_disk( gstorer , \
new_data[the_id][imap_utils.GIMAPFetcher.GMAIL_ID], \
the_dir)
#if on disk check that the data is not different
if curr_metadata:
LOG.debug("metadata for %s already exists. Check if different." % (gid))
if self._metadata_needs_update(curr_metadata, new_data[the_id]):
#restore everything at the moment
gid = gstorer.bury_metadata(new_data[the_id], local_dir = the_dir)
LOG.debug("update email with imap id %s and gmail id %s." % (the_id, gid))
#update local index id gid => index per directory to be thought out
else:
#get the data
email_data = self.src.fetch(the_id, imap_utils.GIMAPFetcher.GET_DATA_ONLY )
new_data[the_id][imap_utils.GIMAPFetcher.EMAIL_BODY] = email_data[the_id][imap_utils.GIMAPFetcher.EMAIL_BODY]
# store data on disk within year month dir
gid = gstorer.bury_email(new_data[the_id], local_dir = the_dir, compress = compress)
#update local index id gid => index per directory to be thought out
LOG.debug("Create and store email with imap id %s, gmail id %s." % (the_id, gid))
else:
# case when gmail IMAP server returns OK without any data whatsoever
# eg. imap uid 142221L ignore it
self.error_report['empty'].append((the_id, None))
nb_emails_processed += 1
#indicate every 50 messages the number of messages left to process
left_emails = (total_nb_emails_to_process - nb_emails_processed)
if (nb_emails_processed % 50) == 0 and (left_emails > 0):
elapsed = timer.elapsed() #elapsed time in seconds
LOG.critical("\n== Processed %d emails in %s. %d left to be stored (time estimate %s).==\n" % \
(nb_emails_processed, timer.seconds_to_human_time(elapsed), left_emails, timer.estimate_time_left(nb_emails_processed, elapsed, left_emails)))
# save id every 20 restored emails
if (nb_emails_processed % 20) == 0:
if gid:
self.save_lastid(self.OP_SYNC, gid)
except imaplib.IMAP4.abort, _:
# imap abort error
# ignore it
# will have to do something with these ignored messages
LOG.critical("Error while fetching message with imap id %s." % (the_id))
LOG.critical("\n=== Exception traceback ===\n")
LOG.critical(gmvault_utils.get_exception_traceback())
LOG.critical("=== End of Exception traceback ===\n")
try:
#try to get the gmail_id
raise Exception("Error")
curr = self.src.fetch(the_id, imap_utils.GIMAPFetcher.GET_GMAIL_ID)
except Exception, _: #pylint:disable-msg=W0703
curr = None
LOG.critical("Error when trying to get gmail id for message with imap id %s." % (the_id))
LOG.critical("Disconnect, wait for 20 sec then reconnect.")
self.src.disconnect()
#could not fetch the gm_id so disconnect and sleep
#sleep 20 sec
time.sleep(20)
LOG.critical("Reconnecting ...")
self.src.connect()
if curr:
gmail_id = curr[the_id][imap_utils.GIMAPFetcher.GMAIL_ID]
else:
gmail_id = None
#add ignored id
self.error_report['cannot_be_fetched'].append((the_id, gmail_id))
LOG.critical("Forced to ignore message with imap id %s, (gmail id %s)." % (the_id, (gmail_id if gmail_id else "cannot be read")))
self._check_db(args, credential)
elif args.get('command', '') == 'config':
LOG.critical("Configure something. TBD.\n")
on_error = False
except KeyboardInterrupt, _:
LOG.critical("\nCRTL^C. Stop all operations.\n")
on_error = False
except socket.error:
LOG.critical("Error: Network problem. Please check your gmail server hostname, the internet connection or your network setup.\n")
LOG.critical("=== Exception traceback ===")
LOG.critical(gmvault_utils.get_exception_traceback())
LOG.critical("=== End of Exception traceback ===\n")
die_with_usage = False
except imaplib.IMAP4.error, imap_err:
#bad login or password
if str(imap_err) in ['[AUTHENTICATIONFAILED] Invalid credentials (Failure)', \
'[ALERT] Web login required: http://support.google.com/mail/bin/answer.py?answer=78754 (Failure)', \
'[ALERT] Invalid credentials (Failure)'] :
LOG.critical("ERROR: Invalid credentials, cannot login to the gmail server. Please check your login and password or xoauth token.\n")
die_with_usage = False
else:
LOG.critical("Error: %s.\n" % (imap_err) )
LOG.critical("=== Exception traceback ===")
LOG.critical(gmvault_utils.get_exception_traceback())
LOG.critical("=== End of Exception traceback ===\n")
except Exception, err:
