def wrapper(*args, **kwargs): #pylint:disable=C0111,R0912 nb_tries = [0] # make it mutable in reconnect m_sleep_time = [a_sleep_time] #make it mutable in reconnect while True: try: return the_func(*args, **kwargs) except PushEmailError, p_err: LOG.debug("error message = %s. traceback:%s" % (p_err, gmvault_utils.get_exception_traceback())) if nb_tries[0] < a_nb_tries: LOG.critical( "Cannot reach the Gmail server. Wait %s second(s) and retrying." % (m_sleep_time[0])) else: LOG.critical("Stop retrying, tried too many times ...") reconnect(args[0], nb_tries, a_nb_tries, p_err, m_sleep_time) except imaplib.IMAP4.abort, err: #abort is recoverable and error is not LOG.debug("IMAP (abort) error message = %s. traceback:%s" % (err, gmvault_utils.get_exception_traceback())) if nb_tries[0] < a_nb_tries: LOG.critical( "Received an IMAP abort error. Wait %s second(s) and retrying." % (m_sleep_time[0])) else: LOG.critical("Stop retrying, tried too many times ...") # problem with this email, put it in quarantine reconnect(args[0], nb_tries, a_nb_tries, err, m_sleep_time)
def get_oauth_cred(email, cred_path): """ Read oauth token secret credential Look by default to ~/.gmvault Look for file ~/.gmvault/email.oauth """ user_oauth_file_path = cred_path token = None secret = None if os.path.exists(user_oauth_file_path): print("Get XOAuth credential from %s.\n" % (user_oauth_file_path)) oauth_file = open(user_oauth_file_path) try: oauth_result = oauth_file.read() if oauth_result: oauth_result = oauth_result.split('::') if len(oauth_result) == 2: token = oauth_result[0] secret = oauth_result[1] except Exception, _: #pylint: disable-msg=W0703 print( "Cannot read oauth credentials from %s. Force oauth credentials renewal." % (user_oauth_file_path)) print("=== Exception traceback ===") print(gmvault_utils.get_exception_traceback()) print("=== End of Exception traceback ===\n") if token: token = token.strip() #pylint: disable-msg=C0321 if secret: secret = secret.strip() #pylint: disable-msg=C0321
def _get_oauth2_tokens(cls, email, use_webbrowser = False, debug=False): ''' Handle the OAUTH2 workflow sequence with either a new request or based on a refresh token ''' #create permission url permission_url = generate_permission_url() #message to indicate that a browser will be opened raw_input('gmvault will now open a web browser page in order for you to grant gmvault access to your Gmail.\n'\ 'Please make sure you\'re logged into the correct Gmail account (%s) before granting access.\n'\ 'Press ENTER to open the browser.' % (email)) # run web browser otherwise print message with url if use_webbrowser: try: webbrowser.open(str(permission_url)) except Exception, err: #pylint: disable-msg=W0703 LOG.critical("Error: %s.\n" % (err) ) LOG.critical("=== Exception traceback ===") LOG.critical(gmvault_utils.get_exception_traceback()) LOG.critical("=== End of Exception traceback ===\n") verification_code = raw_input("You should now see the web page on your browser now.\n"\ "If you don\'t, you can manually open:\n\n%s\n\nOnce you've granted"\ " gmvault access, enter the verification code and press enter:\n" % (permission_url))
def get_oauth_cred(email, cred_path): """ Read oauth token secret credential Look by default to ~/.gmvault Look for file ~/.gmvault/email.oauth """ user_oauth_file_path = cred_path token = None secret = None if os.path.exists(user_oauth_file_path): print("Get XOAuth credential from %s.\n" % (user_oauth_file_path)) oauth_file = open(user_oauth_file_path) try: oauth_result = oauth_file.read() if oauth_result: oauth_result = oauth_result.split('::') if len(oauth_result) == 2: token = oauth_result[0] secret = oauth_result[1] except Exception, _: #pylint: disable-msg=W0703 print("Cannot read oauth credentials from %s. Force oauth credentials renewal." % (user_oauth_file_path)) print("=== Exception traceback ===") print(gmvault_utils.get_exception_traceback()) print("=== End of Exception traceback ===\n") if token: token = token.strip() #pylint: disable-msg=C0321 if secret: secret = secret.strip() #pylint: disable-msg=C0321
def read_oauth2_tok_sec(cls, email): """ Read oauth2 refresh token secret Look by default to ~/.gmvault Look for file ~/.gmvault/email.oauth2 """ gmv_dir = gmvault_utils.get_home_dir_path() #look for email.passwed in GMV_DIR user_oauth_file_path = "%s/%s.oauth2" % (gmv_dir, email) oauth_result = None if os.path.exists(user_oauth_file_path): LOG.critical("Get OAuth2 credential from %s.\n" % user_oauth_file_path) try: with open(user_oauth_file_path) as oauth_file: oauth_result = json.load(oauth_file) except Exception, _: #pylint: disable-msg=W0703 LOG.critical( "Cannot read oauth credentials from %s. Force oauth credentials renewal." % user_oauth_file_path) LOG.critical("=== Exception traceback ===") LOG.critical(gmvault_utils.get_exception_traceback()) LOG.critical("=== End of Exception traceback ===\n")
def wrapper(*args, **kwargs): #pylint:disable=C0111,R0912 nb_tries = [0] # make it mutable in reconnect m_sleep_time = [a_sleep_time] #make it mutable in reconnect while True: try: return the_func(*args, **kwargs) except PushEmailError, p_err: LOG.debug("error message = %s. traceback:%s" % (p_err, gmvault_utils.get_exception_traceback())) if nb_tries[0] < a_nb_tries: LOG.critical("Cannot reach the Gmail server. Wait %s seconds and retrying." % (m_sleep_time[0])) else: LOG.critical("Stop retrying, tried too many times ...") reconnect(args[0], nb_tries, a_nb_tries, p_err, m_sleep_time) except imaplib.IMAP4.abort, err: #abort is recoverable and error is not LOG.debug("IMAP (abort) error message = %s. traceback:%s" % (err, gmvault_utils.get_exception_traceback())) if nb_tries[0] < a_nb_tries: LOG.critical("Received an IMAP abort error. Wait %s seconds and retrying." % (m_sleep_time[0])) else: LOG.critical("Stop retrying, tried too many times ...") # problem with this email, put it in quarantine reconnect(args[0], nb_tries, a_nb_tries, err, m_sleep_time)
def get_oauth_tok_sec(email, use_webbrowser = False, debug=False): ''' Generate token and secret ''' scopes = ['https://mail.google.com/', # IMAP/SMTP client access 'https://www.googleapis.com/auth/userinfo#email'] # Email address access (verify token authorized by correct account gdata_serv = gdata.service.GDataService() gdata_serv.debug = debug gdata_serv.source = 'gmvault ' gdata_serv.SetOAuthInputParameters(gdata.auth.OAuthSignatureMethod.HMAC_SHA1, \ consumer_key = 'anonymous', consumer_secret = 'anonymous') params = {'xoauth_displayname':'Gmvault - Backup your Gmail account'} try: request_token = gdata_serv.FetchOAuthRequestToken(scopes=scopes, extra_parameters = params) except gdata.service.FetchingOAuthRequestTokenFailed, err: if str(err).find('Timestamp') != -1: LOG.critical('Is your system clock up to date? See the FAQ http://code.google.com/p/googlecl/wiki/FAQ'\ '#Timestamp_too_far_from_current_time\n') LOG.critical("Received Error: %s.\n" % (err) ) LOG.critical("=== Exception traceback ===") LOG.critical(gmvault_utils.get_exception_traceback()) LOG.critical("=== End of Exception traceback ===\n") return (None, None)
def _get_oauth2_tokens(cls, email, use_webbrowser=False, debug=False): ''' Handle the OAUTH2 workflow sequence with either a new request or based on a refresh token ''' #create permission url permission_url = generate_permission_url() #message to indicate that a browser will be opened raw_input('gmvault will now open a web browser page in order for you to grant gmvault access to your Gmail.\n'\ 'Please make sure you\'re logged into the correct Gmail account (%s) before granting access.\n'\ 'Press ENTER to open the browser.' % (email)) # run web browser otherwise print message with url if use_webbrowser: try: webbrowser.open(str(permission_url)) except Exception, err: #pylint: disable-msg=W0703 LOG.critical("Error: %s.\n" % (err)) LOG.critical("=== Exception traceback ===") LOG.critical(gmvault_utils.get_exception_traceback()) LOG.critical("=== End of Exception traceback ===\n") verification_code = raw_input("You should now see the web page on your browser now.\n"\ "If you don\'t, you can manually open:\n\n%s\n\nOnce you've granted"\ " gmvault access, enter the verification code and press enter:\n" % (permission_url))
def create_gmail_labels(self, labels, existing_folders): """ Create folders and subfolders on Gmail in order to recreate the label hierarchy before to upload emails Note that adding labels with +X-GM-LABELS create only nested labels but not nested ones. This is why this trick must be used to recreate the label hierarchy labels: list of labels to create """ #1.5-beta moved that out of the loop to minimize the number of calls #to that method. (Could go further and memoize it) #get existing directories (or label parts) # get in lower case because Gmail labels are case insensitive listed_folders = set([ directory.lower() for (_, _, directory) in self.list_all_folders() ]) existing_folders = listed_folders.union(existing_folders) reserved_labels_map = gmvault_utils.get_conf_defaults().get_dict("Restore", "reserved_labels_map", \ { u'migrated' : u'gmv-migrated', u'\muted' : u'gmv-muted' }) LOG.debug("Labels to create: [%s]" % (labels)) for lab in labels: #LOG.info("Reserved labels = %s\n" % (reserved_labels)) #LOG.info("lab.lower = %s\n" % (lab.lower())) if lab.lower() in reserved_labels_map.keys(): #exclude creation of migrated label n_lab = reserved_labels_map.get(lab.lower(), "gmv-default-label") LOG.info("Warning ! label '%s' (lower or uppercase) is reserved by Gmail and cannot be used."\ "Use %s instead" % (lab, n_lab)) lab = n_lab LOG.info("translated lab = %s\n" % (lab)) #split all labels labs = self._get_dir_from_labels(lab) for directory in labs: low_directory = directory.lower() #get lower case directory but store original label if (low_directory not in existing_folders) and (low_directory not in self.GMAIL_SPECIAL_DIRS_LOWER): try: if self.server.create_folder(directory) != 'Success': raise Exception("Cannot create label %s: the directory %s cannot be created." % (lab, directory)) else: LOG.debug("============== ####### Created Labels (%s)." % (directory)) except imaplib.IMAP4.error, error: #log error in log file if it exists LOG.debug(gmvault_utils.get_exception_traceback()) if str(error).startswith("create failed: '[ALREADYEXISTS] Duplicate folder"): LOG.critical("Warning: label %s already exists on Gmail and Gmvault tried to create it."\ " Ignore this issue." % (directory) ) else: raise error #add created folder in folders existing_folders.add(low_directory)
def delete_gmail_labels(self, labels, force_delete = False): """ Delete passed labels. Beware experimental and labels must be ordered """ for label in reversed(labels): labs = self._get_dir_from_labels(label) for directory in reversed(labs): if force_delete or ( (directory.lower() not in self.GMAIL_SPECIAL_DIRS_LOWER) \ and self.server.folder_exists(directory) ): #call server exists each time try: self.server.delete_folder(directory) except imaplib.IMAP4.error, _: LOG.debug(gmvault_utils.get_exception_traceback())
def delete_gmail_labels(self, labels, force_delete=False): """ Delete passed labels. Beware experimental and labels must be ordered """ for label in reversed(labels): labs = self._get_dir_from_labels(label) for directory in reversed(labs): if force_delete or ( (directory.lower() not in self.GMAIL_SPECIAL_DIRS_LOWER) \ and self.server.folder_exists(directory) ): #call server exists each time try: self.server.delete_folder(directory) except imaplib.IMAP4.error, _: LOG.debug(gmvault_utils.get_exception_traceback())
def apply_labels_to(self, imap_ids, labels): """ apply one labels to x emails """ # go to All Mail folder LOG.debug("Applying labels %s" % (labels)) the_timer = gmvault_utils.Timer() the_timer.start() #utf7 the labels as they should be labels = [utf7_encode(label) for label in labels] labels_str = self._build_labels_str(labels) # create labels str if labels_str: #has labels so update email the_timer.start() LOG.debug("Before to store labels %s" % (labels_str)) id_list = ",".join(map(str, imap_ids)) #+X-GM-LABELS.SILENT to have not returned data try: ret_code, data = self.server._imap.uid('STORE', id_list, '+X-GM-LABELS.SILENT', labels_str) #pylint: disable=W0212 except imaplib.IMAP4.error, original_err: LOG.info("Error in apply_labels_to. See exception traceback") LOG.debug(gmvault_utils.get_exception_traceback()) # try to add labels to each individual ids faulty_ids = [] for the_id in imap_ids: try: ret_code, data = self.server._imap.uid( 'STORE', the_id, '+X-GM-LABELS.SILENT', labels_str) #pylint: disable=W0212 except imaplib.IMAP4.error, store_err: LOG.debug( "Error when trying to apply labels %s to emails with imap_id %s. Error:%s" % (labels_str, the_id, store_err)) faulty_ids.append(the_id) #raise an error to ignore faulty emails raise LabelError( "Cannot add Labels %s to emails with uids %s. Error:%s" % (labels_str, faulty_ids, original_err), ignore=True)
def apply_labels_to(self, imap_ids, labels): """ apply one labels to x emails """ # go to All Mail folder LOG.debug("Applying labels %s" % (labels)) the_timer = gmvault_utils.Timer() the_timer.start() #utf7 the labels as they should be labels = [ utf7_encode(label) for label in labels ] labels_str = self._build_labels_str(labels) # create labels str if labels_str: #has labels so update email the_timer.start() LOG.debug("Before to store labels %s" % (labels_str)) id_list = ",".join(map(str, imap_ids)) #+X-GM-LABELS.SILENT to have not returned data try: ret_code, data = self.server._imap.uid('STORE', id_list, '+X-GM-LABELS.SILENT', labels_str) #pylint: disable=W0212 except imaplib.IMAP4.error, original_err: LOG.info("Error in apply_labels_to. See exception traceback") LOG.debug(gmvault_utils.get_exception_traceback()) # try to add labels to each individual ids faulty_ids = [] for the_id in imap_ids: try: ret_code, data = self.server._imap.uid('STORE', the_id, '+X-GM-LABELS.SILENT', labels_str) #pylint: disable=W0212 except imaplib.IMAP4.error, store_err: LOG.debug("Error when trying to apply labels %s to emails with imap_id %s. Error:%s" % (labels_str, the_id, store_err)) faulty_ids.append(the_id) #raise an error to ignore faulty emails raise LabelError("Cannot add Labels %s to emails with uids %s. Error:%s" % (labels_str, faulty_ids, original_err), ignore = True)
def read_oauth2_tok_sec(cls, email): """ Read oauth2 refresh token secret Look by default to ~/.gmvault Look for file ~/.gmvault/email.oauth2 """ gmv_dir = gmvault_utils.get_home_dir_path() #look for email.passwed in GMV_DIR user_oauth_file_path = "%s/%s.oauth2" % (gmv_dir, email) oauth_result = None if os.path.exists(user_oauth_file_path): LOG.critical("Get OAuth2 credential from %s.\n" % user_oauth_file_path) try: with open(user_oauth_file_path) as oauth_file: oauth_result = json.load(oauth_file) except Exception, _: #pylint: disable-msg=W0703 LOG.critical("Cannot read oauth credentials from %s. Force oauth credentials renewal." % user_oauth_file_path) LOG.critical("=== Exception traceback ===") LOG.critical(gmvault_utils.get_exception_traceback()) LOG.critical("=== End of Exception traceback ===\n")
def read_oauth_tok_sec(cls, email): """ Read oauth token secret credential Look by default to ~/.gmvault Look for file ~/.gmvault/email.oauth """ gmv_dir = gmvault_utils.get_home_dir_path() #look for email.passwed in GMV_DIR user_oauth_file_path = "%s/%s.oauth" % (gmv_dir, email) token = None secret = None type = None if os.path.exists(user_oauth_file_path): LOG.critical("Get XOAuth credential from %s.\n" % (user_oauth_file_path)) oauth_file = open(user_oauth_file_path) try: oauth_result = oauth_file.read() if oauth_result: oauth_result = oauth_result.split('::') if len(oauth_result) == 2: token = oauth_result[0] secret = oauth_result[1] type = "normal" elif len(oauth_result) == 3: token = oauth_result[0] secret = oauth_result[1] type = oauth_result[2] except Exception, _: #pylint: disable-msg=W0703 LOG.critical("Cannot read oauth credentials from %s. Force oauth credentials renewal." % (user_oauth_file_path)) LOG.critical("=== Exception traceback ===") LOG.critical(gmvault_utils.get_exception_traceback()) LOG.critical("=== End of Exception traceback ===\n")
# problem with this email, put it in quarantine reconnect(args[0], nb_tries, a_nb_tries, err, m_sleep_time) except socket.error, sock_err: LOG.debug("error message = %s. traceback:%s" % (sock_err, gmvault_utils.get_exception_traceback())) if nb_tries[0] < a_nb_tries: LOG.critical("Cannot reach the Gmail server. Wait %s second(s) and retrying." % (m_sleep_time[0])) else: LOG.critical("Stop retrying, tried too many times ...") reconnect(args[0], nb_tries, a_nb_tries, sock_err, m_sleep_time) except ssl.SSLError, ssl_err: LOG.debug("error message = %s. traceback:%s" % (ssl_err, gmvault_utils.get_exception_traceback())) if nb_tries[0] < a_nb_tries: LOG.critical("Cannot reach the Gmail server. Wait %s second(s) and retrying." % (m_sleep_time[0])) else: LOG.critical("Stop retrying, tried too many times ...") reconnect(args[0], nb_tries, a_nb_tries, sock_err, m_sleep_time) except imaplib.IMAP4.error, err: #just trace it back for the moment LOG.debug("IMAP (normal) error message = %s. traceback:%s" % (err, gmvault_utils.get_exception_traceback())) if nb_tries[0] < a_nb_tries: LOG.critical("Error when reaching Gmail server. Wait %s second(s) and retry up to 2 times." \
(sock_err, gmvault_utils.get_exception_traceback())) if nb_tries[0] < a_nb_tries: LOG.critical( "Cannot reach the Gmail server. Wait %s second(s) and retrying." % (m_sleep_time[0])) else: LOG.critical("Stop retrying, tried too many times ...") reconnect(args[0], nb_tries, a_nb_tries, sock_err, m_sleep_time) except ssl.SSLError, ssl_err: LOG.debug( "error message = %s. traceback:%s" % (ssl_err, gmvault_utils.get_exception_traceback())) if nb_tries[0] < a_nb_tries: LOG.critical( "Cannot reach the Gmail server. Wait %s second(s) and retrying." % (m_sleep_time[0])) else: LOG.critical("Stop retrying, tried too many times ...") reconnect(args[0], nb_tries, a_nb_tries, sock_err, m_sleep_time) except imaplib.IMAP4.error, err: #just trace it back for the moment LOG.debug(
# problem with this email, put it in quarantine reconnect(args[0], nb_tries, a_nb_tries, err, m_sleep_time) except socket.error, sock_err: LOG.debug("error message = %s. traceback:%s" % (sock_err, gmvault_utils.get_exception_traceback())) if nb_tries[0] < a_nb_tries: LOG.critical("Cannot reach the Gmail server. Wait %s seconds and retrying." % (m_sleep_time[0])) else: LOG.critical("Stop retrying, tried too many times ...") reconnect(args[0], nb_tries, a_nb_tries, sock_err, m_sleep_time) except ssl.SSLError, ssl_err: LOG.debug("error message = %s. traceback:%s" % (ssl_err, gmvault_utils.get_exception_traceback())) if nb_tries[0] < a_nb_tries: LOG.critical("Cannot reach the Gmail server. Wait %s seconds and retrying." % (m_sleep_time[0])) else: LOG.critical("Stop retrying, tried too many times ...") reconnect(args[0], nb_tries, a_nb_tries, sock_err, m_sleep_time) except imaplib.IMAP4.error, err: #just trace it back for the moment LOG.debug("IMAP (normal) error message = %s. traceback:%s" % (err, gmvault_utils.get_exception_traceback())) if nb_tries[0] < a_nb_tries: LOG.critical("Error when reaching Gmail server. Wait %s seconds and retry up to 2 times." \
self._export(args) elif args.get('command', '') == 'config': LOG.critical("Configure something. TBD.\n") on_error = False except KeyboardInterrupt, _: LOG.critical("\nCTRL-C. Stop all operations.\n") on_error = False except socket.error: LOG.critical("Error: Network problem. Please check your gmail server hostname,"\ " the internet connection or your network setup.\n") LOG.critical("=== Exception traceback ===") LOG.critical(gmvault_utils.get_exception_traceback()) LOG.critical("=== End of Exception traceback ===\n") die_with_usage = False except imaplib.IMAP4.error, imap_err: #bad login or password if str(imap_err) in ['[AUTHENTICATIONFAILED] Invalid credentials (Failure)', \ '[ALERT] Web login required: http://support.google.com/'\ 'mail/bin/answer.py?answer=78754 (Failure)', \ '[ALERT] Invalid credentials (Failure)'] : LOG.critical("ERROR: Invalid credentials, cannot login to the gmail server."\ " Please check your login and password or xoauth token.\n") die_with_usage = False else: LOG.critical("Error: %s. \n" % (imap_err)) LOG.critical("=== Exception traceback ===") LOG.critical(gmvault_utils.get_exception_traceback())
self._export(args) elif args.get('command', '') == 'config': LOG.critical("Configure something. TBD.\n") on_error = False except KeyboardInterrupt, _: LOG.critical("\nCRTL^C. Stop all operations.\n") on_error = False except socket.error: LOG.critical("Error: Network problem. Please check your gmail server hostname,"\ " the internet connection or your network setup.\n") LOG.critical("=== Exception traceback ===") LOG.critical(gmvault_utils.get_exception_traceback()) LOG.critical("=== End of Exception traceback ===\n") die_with_usage = False except imaplib.IMAP4.error, imap_err: #bad login or password if str(imap_err) in ['[AUTHENTICATIONFAILED] Invalid credentials (Failure)', \ '[ALERT] Web login required: http://support.google.com/'\ 'mail/bin/answer.py?answer=78754 (Failure)', \ '[ALERT] Invalid credentials (Failure)'] : LOG.critical("ERROR: Invalid credentials, cannot login to the gmail server."\ " Please check your login and password or xoauth token.\n") die_with_usage = False else: LOG.critical("Error: %s. \n" % (imap_err) ) LOG.critical("=== Exception traceback ===") LOG.critical(gmvault_utils.get_exception_traceback())