def _get_authorization_tokens(cls, authorization_code): """Obtains OAuth access token and refresh token. This uses the application portion of the "OAuth2 for Installed Applications" flow at https://developers.google.com/accounts/docs/OAuth2InstalledApp#handlingtheresponse Args: client_id: Client ID obtained by registering your app. client_secret: Client secret obtained by registering your app. authorization_code: code generated by Google Accounts after user grants permission. Returns: The decoded response from the Google Accounts server, as a dict. Expected fields include 'access_token', 'expires_in', and 'refresh_token'. """ params = {} params['client_id'] = gmvault_utils.get_conf_defaults().get("GoogleOauth2", "gmvault_client_id", "1070918343777-0eecradokiu8i77qfo8e3stbi0mkrtog.apps.googleusercontent.com") params['client_secret'] = gmvault_utils.get_conf_defaults().get("GoogleOauth2", "gmvault_client_secret", "IVkl_pglv5cXzugpmnRNqtT7") params['code'] = authorization_code params['redirect_uri'] = gmvault_utils.get_conf_defaults().get("GoogleOauth2", "redirect_uri", 'urn:ietf:wg:oauth:2.0:oob') params['grant_type'] = 'authorization_code' account_base_url = gmvault_utils.get_conf_defaults().get("GoogleOauth2", "google_accounts_base_url", 'https://accounts.google.com') request_url = '%s/%s' % (account_base_url, 'o/oauth2/token') try: response = urllib2.urlopen(request_url, urllib.urlencode(params)).read() except Exception, err: #pylint: disable-msg=W0703 LOG.critical("Error: Problems when trying to connect to Google oauth2 endpoint: %s." % (request_url)) raise err
def _get_oauth2_acc_tok_from_ref_tok(cls, refresh_token): """Obtains a new token given a refresh token. See https://developers.google.com/accounts/docs/OAuth2InstalledApp#refresh Args: client_id: Client ID obtained by registering your app. client_secret: Client secret obtained by registering your app. refresh_token: A previously-obtained refresh token. Returns: The decoded response from the Google Accounts server, as a dict. Expected fields include 'access_token', 'expires_in', and 'refresh_token'. """ params = {} params['client_id'] = gmvault_utils.get_conf_defaults().get("GoogleOauth2", "gmvault_client_id", "1070918343777-0eecradokiu8i77qfo8e3stbi0mkrtog.apps.googleusercontent.com") params['client_secret'] = gmvault_utils.get_conf_defaults().get("GoogleOauth2", "gmvault_client_secret", "IVkl_pglv5cXzugpmnRNqtT7") params['refresh_token'] = refresh_token params['grant_type'] = 'refresh_token' account_base_url = gmvault_utils.get_conf_defaults().get("GoogleOauth2", "google_accounts_base_url", 'https://accounts.google.com') request_url = '%s/%s' % (account_base_url, 'o/oauth2/token') try: response = urllib2.urlopen(request_url, urllib.urlencode(params)).read() except Exception, err: #pylint: disable-msg=W0703 LOG.critical("Error: Problems when trying to connect to Google oauth2 endpoint: %s.\n" % (request_url)) raise err
def generate_permission_url(): """Generates the URL for authorizing access. This uses the "OAuth2 for Installed Applications" flow described at https://developers.google.com/accounts/docs/OAuth2InstalledApp Args: client_id: Client ID obtained by registering your app. scope: scope for access token, e.g. 'https://mail.google.com' Returns: A URL that the user should visit in their browser. """ params = {} params['client_id'] = gmvault_utils.get_conf_defaults().get( "GoogleOauth2", "gmvault_client_id", "") params['redirect_uri'] = gmvault_utils.get_conf_defaults().get( "GoogleOauth2", "redirect_uri", '') params['scope'] = gmvault_utils.get_conf_defaults().get( "GoogleOauth2", "scope", 'https://mail.google.com/') params['response_type'] = 'code' account_base_url = gmvault_utils.get_conf_defaults().get( "GoogleOauth2", "google_accounts_base_url", 'https://accounts.google.com') return '%s/%s?%s' % (account_base_url, 'o/oauth2/auth', gmvault_utils.format_url_params(params))
def _get_oauth2_acc_tok_from_ref_tok(cls, refresh_token): """Obtains a new token given a refresh token. See https://developers.google.com/accounts/docs/OAuth2InstalledApp#refresh Args: client_id: Client ID obtained by registering your app. client_secret: Client secret obtained by registering your app. refresh_token: A previously-obtained refresh token. Returns: The decoded response from the Google Accounts server, as a dict. Expected fields include 'access_token', 'expires_in', and 'refresh_token'. """ params = {} params['client_id'] = gmvault_utils.get_conf_defaults().get( "GoogleOauth2", "gmvault_client_id", "") params['client_secret'] = gmvault_utils.get_conf_defaults().get( "GoogleOauth2", "gmvault_client_secret", "") params['refresh_token'] = refresh_token params['grant_type'] = 'refresh_token' account_base_url = gmvault_utils.get_conf_defaults().get( "GoogleOauth2", "google_accounts_base_url", 'https://accounts.google.com') request_url = '%s/%s' % (account_base_url, 'o/oauth2/token') try: response = urllib2.urlopen(request_url, urllib.urlencode(params)).read() except Exception, err: #pylint: disable-msg=W0703 LOG.critical( "Error: Problems when trying to connect to Google oauth2 endpoint: %s.\nMaybe you changed your password? Rename/remove the auth token file that is named after the email address, and try again to refresh the token." % (request_url)) raise err
def connect(self, go_to_current_folder = False): """ connect to the IMAP server """ # check if it is needed to disbale cert verification (leave that option at the last resort) # if the cert verification doesn't work. disable_cert_verification = gmvault_utils.get_conf_defaults().getboolean("GoogleOauth2", "disable_cacert_verification", default=False) context = None if disable_cert_verification: LOG.critical("Beware disabling CA Cert verification for the IMAP connection to Gmail.") context = imapclient.create_default_context() # don't check if certificate hostname doesn't match target hostname context.check_hostname = False # don't check if the certificate is trusted by a certificate authority context.verify_mode = ssl.CERT_NONE else: # create a custom ssl context to handle CA cert verification in all cases # In some plateform the CA certs are not available so use the ones provided by Gmvault # CA certs are coming from https://curl.haxx.se/ca/cacert.pem context = imapclient.create_default_context(cafile= gmvault_utils.get_ca_certs_filepath()) # create imap object self.server = mimap.MonkeyIMAPClient(self.host, port = self.port, use_uid = self.use_uid, need_ssl = self.ssl, ssl_context = context) # connect with password or xoauth if self.credential['type'] == 'passwd': self.server.login(self.login, self.credential['value']) elif self.credential['type'] == 'oauth2': #connect with oauth2 if self.once_connected: self.credential = credential_utils.CredentialHelper.get_oauth2_credential(self.login, renew_cred = False) LOG.debug("credential['value'] = %s" % (self.credential['value'])) #try to login self.server.oauth2_login(self.credential['value']) else: raise Exception("Unknown authentication method %s. Please use xoauth or passwd authentication " \ % (self.credential['type'])) #set connected to True to handle reconnection in case of failure self.once_connected = True # check gmailness self.check_gmailness() # find allmail chats and drafts folders self.find_folder_names() if go_to_current_folder and self.current_folder: self.server.select_folder(self.current_folder, readonly = self.readonly_folder) #enable compression if gmvault_utils.get_conf_defaults().get_boolean('General', 'enable_imap_compression', True): self.enable_compression() LOG.debug("After Enabling compression.") else: LOG.debug("Do not enable imap compression.")
def __init__(self, host, port, login, credential, readonly_folder = True): #pylint:disable=R0913 ''' Constructor ''' self.host = host self.port = port self.login = login self.once_connected = False self.credential = credential self.ssl = True self.use_uid = True self.readonly_folder = readonly_folder self.localized_folders = { 'ALLMAIL': { 'loc_dir' : None, 'friendly_name' : 'allmail'}, 'CHATS' : { 'loc_dir' : None, 'friendly_name' : 'chats'}, 'DRAFTS' : { 'loc_dir' : None, 'friendly_name' : 'drafts'} } # memoize the current folder (All Mail or Chats) for reconnection management self.current_folder = None self.server = None self.go_to_all_folder = True self.total_nb_reconns = 0 # True when CHATS or other folder error msg has been already printed self.printed_folder_error_msg = { 'ALLMAIL' : False, 'CHATS': False , 'DRAFTS':False } #update GENERIC_GMAIL_CHATS. Should be done at the class level self.GENERIC_GMAIL_CHATS.extend(gmvault_utils.get_conf_defaults().get_list('Localisation', 'chat_folder', []))
def __init__(self, host, port, login, credential, readonly_folder = True): #pylint:disable=R0913 ''' Constructor ''' self.host = host self.port = port self.login = login self.once_connected = False self.credential = credential self.ssl = True self.use_uid = True self.readonly_folder = readonly_folder self.localized_folders = { 'ALLMAIL': { 'loc_dir' : None, 'friendly_name' : 'allmail'}, 'CHATS' : { 'loc_dir' : None, 'friendly_name' : 'chats'}, 'DRAFTS' :{ 'loc_dir' : None, 'friendly_name' : 'drafts'} } # memoize the current folder (All Mail or Chats) for reconnection management self.current_folder = None self.server = None self.go_to_all_folder = True self.total_nb_reconns = 0 # True when CHATS or other folder error msg has been already printed self.printed_folder_error_msg = { 'ALLMAIL' : False, 'CHATS': False , 'DRAFTS':False } #update GENERIC_GMAIL_CHATS. Should be done at the class level self.GENERIC_GMAIL_CHATS.extend(gmvault_utils.get_conf_defaults().get_list('Localisation', 'chat_folder', []))
def find_chats_folder(self): """ depending on your account the chats folder can be named [GMAIL]/Chats or [GoogleMail]/Chats, [GMAIL]/tous les chats ... Find and set the right one Npte: Cannot use the flags as Chats is not a system label. Thanks Google """ #use xlist because of localized dir names folders = self.server.xlist_folders() LOG.debug("Folders = %s\n" % (folders)) the_dir = None for (_, _, the_dir) in folders: #look for GMAIL Chats if the_dir in GIMAPFetcher.GENERIC_GMAIL_CHATS: #it could be a localized Dir name self.localized_folders['CHATS']['loc_dir'] = the_dir return the_dir #Error did not find Chats dir if gmvault_utils.get_conf_defaults().getboolean( "General", "errors_if_chat_not_visible", False): raise Exception("Cannot find global 'Chats' folder ! Check whether 'Show in IMAP for 'Chats' "\ "is enabled in Gmail (Go to Settings->Labels->All Mail)") return None
def find_folder_names(self): """ depending on your account the all mail folder can be named [GMAIL]/ALL Mail or [GoogleMail]/All Mail. Find and set the right one """ #use xlist because of localized dir names folders = self.server.xlist_folders() the_dir = None for (flags, _, the_dir) in folders: #non localised GMAIL_ALL if GIMAPFetcher.GENERIC_GMAIL_ALL in flags: #it could be a localized Dir name self.localized_folders['ALLMAIL']['loc_dir'] = the_dir elif the_dir in GIMAPFetcher.GENERIC_GMAIL_CHATS: #it could be a localized Dir name self.localized_folders['CHATS']['loc_dir'] = the_dir elif GIMAPFetcher.GENERIC_DRAFTS in flags: self.localized_folders['DRAFTS']['loc_dir'] = the_dir if not self.localized_folders['ALLMAIL']['loc_dir']: # all mail error raise Exception("Cannot find global 'All Mail' folder (maybe localized and translated into your language) ! "\ "Check whether 'Show in IMAP for 'All Mail' is enabled in Gmail (Go to Settings->Labels->All Mail)") elif not self.localized_folders['CHATS']['loc_dir'] and \ gmvault_utils.get_conf_defaults().getboolean("General","errors_if_chat_not_visible", False): raise Exception("Cannot find global 'Chats' folder ! Check whether 'Show in IMAP for 'Chats' "\ "is enabled in Gmail (Go to Settings->Labels->All Mail)") elif not self.localized_folders['DRAFTS']['loc_dir']: raise Exception("Cannot find global 'Drafts' folder.")
def find_folder_names(self): """ depending on your account the all mail folder can be named [GMAIL]/ALL Mail or [GoogleMail]/All Mail. Find and set the right one """ #use xlist because of localized dir names folders = self.server.xlist_folders() the_dir = None for (flags, _, the_dir) in folders: #non localised GMAIL_ALL if GIMAPFetcher.GENERIC_GMAIL_ALL in flags: #it could be a localized Dir name self.localized_folders['ALLMAIL']['loc_dir'] = the_dir elif the_dir in GIMAPFetcher.GENERIC_GMAIL_CHATS : #it could be a localized Dir name self.localized_folders['CHATS']['loc_dir'] = the_dir elif GIMAPFetcher.GENERIC_DRAFTS in flags: self.localized_folders['DRAFTS']['loc_dir'] = the_dir if not self.localized_folders['ALLMAIL']['loc_dir']: # all mail error raise Exception("Cannot find global 'All Mail' folder (maybe localized and translated into your language) ! "\ "Check whether 'Show in IMAP for 'All Mail' is enabled in Gmail (Go to Settings->Labels->All Mail)") elif not self.localized_folders['CHATS']['loc_dir'] and \ gmvault_utils.get_conf_defaults().getboolean("General","errors_if_chat_not_visible", False): raise Exception("Cannot find global 'Chats' folder ! Check whether 'Show in IMAP for 'Chats' "\ "is enabled in Gmail (Go to Settings->Labels->All Mail)") elif not self.localized_folders['DRAFTS']['loc_dir']: raise Exception("Cannot find global 'Drafts' folder.")
def find_chats_folder(self): """ depending on your account the chats folder can be named [GMAIL]/Chats or [GoogleMail]/Chats, [GMAIL]/tous les chats ... Find and set the right one Npte: Cannot use the flags as Chats is not a system label. Thanks Google """ #use xlist because of localized dir names folders = self.server.xlist_folders() LOG.debug("Folders = %s\n" % (folders)) the_dir = None for (_, _, the_dir) in folders: #look for GMAIL Chats if the_dir in GIMAPFetcher.GENERIC_GMAIL_CHATS : #it could be a localized Dir name self.localized_folders['CHATS']['loc_dir'] = the_dir return the_dir #Error did not find Chats dir if gmvault_utils.get_conf_defaults().getboolean("General", "errors_if_chat_not_visible", False): raise Exception("Cannot find global 'Chats' folder ! Check whether 'Show in IMAP for 'Chats' "\ "is enabled in Gmail (Go to Settings->Labels->All Mail)") return None
def create_gmail_labels(self, labels, existing_folders): """ Create folders and subfolders on Gmail in order to recreate the label hierarchy before to upload emails Note that adding labels with +X-GM-LABELS create only nested labels but not nested ones. This is why this trick must be used to recreate the label hierarchy labels: list of labels to create """ #1.5-beta moved that out of the loop to minimize the number of calls #to that method. (Could go further and memoize it) #get existing directories (or label parts) # get in lower case because Gmail labels are case insensitive listed_folders = set([ directory.lower() for (_, _, directory) in self.list_all_folders() ]) existing_folders = listed_folders.union(existing_folders) reserved_labels_map = gmvault_utils.get_conf_defaults().get_dict("Restore", "reserved_labels_map", \ { u'migrated' : u'gmv-migrated', u'\muted' : u'gmv-muted' }) LOG.debug("Labels to create: [%s]" % (labels)) for lab in labels: #LOG.info("Reserved labels = %s\n" % (reserved_labels)) #LOG.info("lab.lower = %s\n" % (lab.lower())) if lab.lower() in reserved_labels_map.keys(): #exclude creation of migrated label n_lab = reserved_labels_map.get(lab.lower(), "gmv-default-label") LOG.info("Warning ! label '%s' (lower or uppercase) is reserved by Gmail and cannot be used."\ "Use %s instead" % (lab, n_lab)) lab = n_lab LOG.info("translated lab = %s\n" % (lab)) #split all labels labs = self._get_dir_from_labels(lab) for directory in labs: low_directory = directory.lower() #get lower case directory but store original label if (low_directory not in existing_folders) and (low_directory not in self.GMAIL_SPECIAL_DIRS_LOWER): try: if self.server.create_folder(directory) != 'Success': raise Exception("Cannot create label %s: the directory %s cannot be created." % (lab, directory)) else: LOG.debug("============== ####### Created Labels (%s)." % (directory)) except imaplib.IMAP4.error, error: #log error in log file if it exists LOG.debug(gmvault_utils.get_exception_traceback()) if str(error).startswith("create failed: '[ALREADYEXISTS] Duplicate folder"): LOG.critical("Warning: label %s already exists on Gmail and Gmvault tried to create it."\ " Ignore this issue." % (directory) ) else: raise error #add created folder in folders existing_folders.add(low_directory)
def delete_emails(self, emails_info, msg_type): """ Delete all emails and metadata with ids """ if msg_type == 'email': db_dir = self._db_dir else: db_dir = self._chats_dir move_to_bin = gmvault_utils.get_conf_defaults().get_boolean( "General", "keep_in_bin" , False) if move_to_bin: LOG.critical("Move emails to the bin:%s" % self._bin_dir) for (a_id, date_dir) in emails_info: the_dir = '%s/%s' % (db_dir, date_dir) data_p = self.DATA_FNAME % (the_dir, a_id) comp_data_p = '%s.gz' % data_p cryp_comp_data_p = '%s.crypt.gz' % data_p metadata_p = self.METADATA_FNAME % (the_dir, a_id) if move_to_bin: #move files to the bin gmvault_utils.makedirs(self._bin_dir) # create bin filenames bin_p = self.DATA_FNAME % (self._bin_dir, a_id) metadata_bin_p = self.METADATA_FNAME % (self._bin_dir, a_id) if os.path.exists(data_p): os.rename(data_p, bin_p) elif os.path.exists(comp_data_p): os.rename(comp_data_p, '%s.gz' % bin_p) elif os.path.exists(cryp_comp_data_p): os.rename(cryp_comp_data_p, '%s.crypt.gz' % bin_p) if os.path.exists(metadata_p): os.rename(metadata_p, metadata_bin_p) else: #delete files if they exists if os.path.exists(data_p): os.remove(data_p) elif os.path.exists(comp_data_p): os.remove(comp_data_p) elif os.path.exists(cryp_comp_data_p): os.remove(cryp_comp_data_p) if os.path.exists(metadata_p): os.remove(metadata_p) gmsql.GMSQL.delete_email(a_id)
def delete_emails(self, emails_info, msg_type): """ Delete all emails and metadata with ids """ if msg_type == 'email': db_dir = self._db_dir else: db_dir = self._chats_dir move_to_bin = gmvault_utils.get_conf_defaults().get_boolean( "General", "keep_in_bin" , False) if move_to_bin: LOG.critical("Move emails to the bin:%s" % self._bin_dir) for (a_id, date_dir) in emails_info: the_dir = '%s/%s' % (db_dir, date_dir) data_p = self.DATA_FNAME % (the_dir, a_id) comp_data_p = '%s.gz' % data_p cryp_comp_data_p = '%s.crypt.gz' % data_p metadata_p = self.METADATA_FNAME % (the_dir, a_id) if move_to_bin: #move files to the bin gmvault_utils.makedirs(self._bin_dir) # create bin filenames bin_p = self.DATA_FNAME % (self._bin_dir, a_id) metadata_bin_p = self.METADATA_FNAME % (self._bin_dir, a_id) if os.path.exists(data_p): os.rename(data_p, bin_p) elif os.path.exists(comp_data_p): os.rename(comp_data_p, '%s.gz' % bin_p) elif os.path.exists(cryp_comp_data_p): os.rename(cryp_comp_data_p, '%s.crypt.gz' % bin_p) if os.path.exists(metadata_p): os.rename(metadata_p, metadata_bin_p) else: #delete files if they exists if os.path.exists(data_p): os.remove(data_p) elif os.path.exists(comp_data_p): os.remove(comp_data_p) elif os.path.exists(cryp_comp_data_p): os.remove(cryp_comp_data_p) if os.path.exists(metadata_p): os.remove(metadata_p)
def generate_permission_url(): """Generates the URL for authorizing access. This uses the "OAuth2 for Installed Applications" flow described at https://developers.google.com/accounts/docs/OAuth2InstalledApp Args: client_id: Client ID obtained by registering your app. scope: scope for access token, e.g. 'https://mail.google.com' Returns: A URL that the user should visit in their browser. """ params = {} params['client_id'] = gmvault_utils.get_conf_defaults().get("GoogleOauth2", "gmvault_client_id", "1070918343777-0eecradokiu8i77qfo8e3stbi0mkrtog.apps.googleusercontent.com") params['redirect_uri'] = gmvault_utils.get_conf_defaults().get("GoogleOauth2", "redirect_uri", 'urn:ietf:wg:oauth:2.0:oob') params['scope'] = gmvault_utils.get_conf_defaults().get("GoogleOauth2","scope",'https://mail.google.com/') params['response_type'] = 'code' account_base_url = gmvault_utils.get_conf_defaults().get("GoogleOauth2", "google_accounts_base_url", 'https://accounts.google.com') return '%s/%s?%s' % (account_base_url, 'o/oauth2/auth', gmvault_utils.format_url_params(params))
def bootstrap_run(): """ temporary bootstrap """ init_logging() #force argv[0] to gmvault sys.argv[0] = "gmvault" LOG.critical("") gmvlt = GMVaultLauncher() args = gmvlt.parse_args() #activate debug if enabled if args['debug']: LOG.critical("Activate debugging information.") activate_debug_mode() # force instanciation of conf to load the defaults gmvault_utils.get_conf_defaults() gmvlt.run(args)
def check_to_disable_certificate_verification(): """ Check from the configuration if certificate verification needs to be disabled. Have to use monkey patching strategy while version older than 2.7.9 are still supported. """ #solve ssl context issue for some users if hasattr('ssl', 'SSLContext') == False: disable_cert_verification = gmvault_utils.get_conf_defaults().getboolean("GoogleOauth2", "disable_cacert_verification", default=False, fail_if_missing = True) if disable_cert_verification == True: #loc_context = ssl.SSLContext(ssl.PROTOCOL_TLSv1) #for the moment monkey patch later on will use the line above LOG.critical("Beware as asked Gmvault is going to disable the SSL Certificate Verification.") ssl._create_default_https_context = ssl._create_unverified_context else: LOG.critical("Use Default certificate context.") else: LOG.debug("Ignore check_to_disable_certificate_verification. No SSLContext Object in ssl, probably using a python version < to 2.7.9")
def connect(self, go_to_current_folder=False): """ connect to the IMAP server """ # create imap object self.server = mimap.MonkeyIMAPClient(self.host, port=self.port, use_uid=self.use_uid, need_ssl=self.ssl) # connect with password or xoauth if self.credential['type'] == 'passwd': self.server.login(self.login, self.credential['value']) elif self.credential['type'] == 'oauth2': #connect with oauth2 if self.once_connected: self.credential = credential_utils.CredentialHelper.get_oauth2_credential( self.login, renew_cred=False) LOG.debug("credential['value'] = %s" % (self.credential['value'])) #try to login self.server.oauth2_login(self.credential['value']) else: raise Exception("Unknown authentication method %s. Please use xoauth or passwd authentication " \ % (self.credential['type'])) #set connected to True to handle reconnection in case of failure self.once_connected = True # check gmailness self.check_gmailness() # find allmail chats and drafts folders self.find_folder_names() if go_to_current_folder and self.current_folder: self.server.select_folder(self.current_folder, readonly=self.readonly_folder) #enable compression if gmvault_utils.get_conf_defaults().get_boolean( 'General', 'enable_imap_compression', True): self.enable_compression() LOG.debug("After Enabling compression.") else: LOG.debug("Do not enable imap compression.")
def _restore(cls, args, credential): """ Execute All restore operations """ LOG.critical("Connect to Gmail server.\n") # Create a gmvault releaving read_only_access restorer = gmvault.GMVaulter(args['db-dir'], args['host'], args['port'], \ args['email'], credential, read_only_access = False) #full sync is the first one if args.get('type', '') == 'full': #call restore labels = [args['apply_label']] if args['apply_label'] else [] restorer.restore(extra_labels = labels, restart = args['restart'], \ emails_only = args['emails_only'], chats_only = args['chats_only']) elif args.get('type', '') == 'quick': #take the last two to 3 months depending on the current date # today - 2 months today = datetime.date.today() begin = today - datetime.timedelta( gmvault_utils.get_conf_defaults().getint( "Restore", "quick_days", 8)) starting_dir = gmvault_utils.get_ym_from_datetime(begin) #call restore labels = [args['apply_label']] if args['apply_label'] else [] restorer.restore(pivot_dir = starting_dir, extra_labels = labels, restart = args['restart'], \ emails_only = args['emails_only'], chats_only = args['chats_only']) else: raise ValueError( "Unknown synchronisation mode %s. Please use full (default), quick." ) #print error report LOG.critical(restorer.get_operation_report())
def __init__(self, a_storage_dir, encrypt_data=False): """ Store on disks args: a_storage_dir: Storage directory a_use_encryption: Encryption key. If there then encrypt """ self._top_dir = a_storage_dir self._db_dir = '%s/%s' % (a_storage_dir, GmailStorer.DB_AREA) self._quarantine_dir = '%s/%s' % (a_storage_dir, GmailStorer.QUARANTINE_AREA) self._info_dir = '%s/%s' % (a_storage_dir, GmailStorer.INFO_AREA) self._chats_dir = '%s/%s' % (self._db_dir, GmailStorer.CHATS_AREA) self._bin_dir = '%s/%s' % (a_storage_dir, GmailStorer.BIN_AREA) gmsql.GMSQL.connect('%s/meta.db' % (a_storage_dir)) self._sub_chats_dir = None self._sub_chats_inc = -1 self._sub_chats_nb = -1 self._limit_per_chat_dir = gmvault_utils.get_conf_defaults().getint( "General", "limit_per_chat_dir", 1500) #make dirs if not os.path.exists(self._db_dir): LOG.critical("No Storage DB in %s. Create it.\n" % a_storage_dir) gmvault_utils.makedirs(self._db_dir) gmvault_utils.makedirs(self._chats_dir) gmvault_utils.makedirs(self._quarantine_dir) gmvault_utils.makedirs(self._info_dir) self.fsystem_info_cache = {} self._encrypt_data = encrypt_data self._encryption_key = None self._cipher = None #add version if it is needed to migrate gmvault-db in the future self._create_gmvault_db_version()
def __init__(self, a_storage_dir, encrypt_data=False): """ Store on disks args: a_storage_dir: Storage directory a_use_encryption: Encryption key. If there then encrypt """ self._top_dir = a_storage_dir self._db_dir = '%s/%s' % (a_storage_dir, GmailStorer.DB_AREA) self._quarantine_dir = '%s/%s' % (a_storage_dir, GmailStorer.QUARANTINE_AREA) self._info_dir = '%s/%s' % (a_storage_dir, GmailStorer.INFO_AREA) self._chats_dir = '%s/%s' % (self._db_dir, GmailStorer.CHATS_AREA) self._bin_dir = '%s/%s' % (a_storage_dir, GmailStorer.BIN_AREA) self._sub_chats_dir = None self._sub_chats_inc = -1 self._sub_chats_nb = -1 self._limit_per_chat_dir = gmvault_utils.get_conf_defaults().getint( "General", "limit_per_chat_dir", 1500) #make dirs if not os.path.exists(self._db_dir): LOG.critical("No Storage DB in %s. Create it.\n" % a_storage_dir) gmvault_utils.makedirs(self._db_dir) gmvault_utils.makedirs(self._chats_dir) gmvault_utils.makedirs(self._quarantine_dir) gmvault_utils.makedirs(self._info_dir) self.fsystem_info_cache = {} self._encrypt_data = encrypt_data self._encryption_key = None self._cipher = None #add version if it is needed to migrate gmvault-db in the future self._create_gmvault_db_version()
def connect(self, go_to_current_folder = False): """ connect to the IMAP server """ # create imap object self.server = mimap.MonkeyIMAPClient(self.host, port = self.port, use_uid= self.use_uid, need_ssl= self.ssl) # connect with password or xoauth if self.credential['type'] == 'passwd': self.server.login(self.login, self.credential['value']) elif self.credential['type'] == 'oauth2': #connect with oauth2 if self.once_connected: self.credential = credential_utils.CredentialHelper.get_oauth2_credential(self.login, renew_cred = False) LOG.debug("credential['value'] = %s" % (self.credential['value'])) #try to login self.server.oauth2_login(self.credential['value']) else: raise Exception("Unknown authentication method %s. Please use xoauth or passwd authentication " \ % (self.credential['type'])) #set connected to True to handle reconnection in case of failure self.once_connected = True # check gmailness self.check_gmailness() # find allmail chats and drafts folders self.find_folder_names() if go_to_current_folder and self.current_folder: self.server.select_folder(self.current_folder, readonly = self.readonly_folder) #enable compression if gmvault_utils.get_conf_defaults().get_boolean('General', 'enable_imap_compression', True): self.enable_compression() LOG.debug("After Enabling compression.") else: LOG.debug("Do not enable imap compression.")
def _restore(cls, args, credential): """ Execute All restore operations """ LOG.critical("Connect to Gmail server.\n") # Create a gmvault releaving read_only_access restorer = gmvault.GMVaulter(args['db-dir'], args['host'], args['port'], \ args['email'], credential, read_only_access = False) #full sync is the first one if args.get('type', '') == 'full': #call restore labels = [args['apply_label']] if args['apply_label'] else [] restorer.restore(extra_labels = labels, restart = args['restart'], \ emails_only = args['emails_only'], chats_only = args['chats_only']) elif args.get('type', '') == 'quick': #take the last two to 3 months depending on the current date # today - 2 months today = datetime.date.today() begin = today - datetime.timedelta(gmvault_utils.get_conf_defaults().getint("Restore", "quick_days", 8)) starting_dir = gmvault_utils.get_ym_from_datetime(begin) #call restore labels = [args['apply_label']] if args['apply_label'] else [] restorer.restore(pivot_dir = starting_dir, extra_labels = labels, restart = args['restart'], \ emails_only = args['emails_only'], chats_only = args['chats_only']) else: raise ValueError("Unknown synchronisation mode %s. Please use full (default), quick.") #print error report LOG.critical(restorer.get_operation_report())
def _sync(cls, args, credential): """ Execute All synchronisation operations """ LOG.critical("Connect to Gmail server.\n") # handle credential in all levels syncer = gmvault.GMVaulter(args['db-dir'], args['host'], args['port'], \ args['email'], credential, read_only_access = True, use_encryption = args['encrypt']) #full sync is the first one if args.get('type', '') == 'full': #choose full sync. Ignore the request syncer.sync({ 'mode': 'full', 'type': 'imap', 'req': 'ALL' } , compress_on_disk = args['compression'], \ db_cleaning = args['db-cleaning'], ownership_checking = args['ownership_control'],\ restart = args['restart'], emails_only = args['emails_only'], chats_only = args['chats_only']) elif args.get('type', '') == 'auto': #choose auto sync. imap request = ALL and restart = True syncer.sync({ 'mode': 'auto', 'type': 'imap', 'req': 'ALL' } , compress_on_disk = args['compression'], \ db_cleaning = args['db-cleaning'], ownership_checking = args['ownership_control'],\ restart = True, emails_only = args['emails_only'], chats_only = args['chats_only']) elif args.get('type', '') == 'quick': #sync only the last x days (taken in defaults) in order to be quick #(cleaning is import here because recent days might move again # today - 2 months today = datetime.date.today() begin = today - datetime.timedelta(gmvault_utils.get_conf_defaults().getint("Sync", "quick_days", 8)) LOG.critical("Quick sync mode. Check for new emails since %s." % (begin.strftime('%d-%b-%Y'))) # today + 1 day end = today + datetime.timedelta(1) req = { 'type' : 'imap', \ 'req' : syncer.get_imap_request_btw_2_dates(begin, end), \ 'mode' : 'quick'} syncer.sync( req, \ compress_on_disk = args['compression'], \ db_cleaning = args['db-cleaning'], \ ownership_checking = args['ownership_control'], restart = args['restart'], \ emails_only = args['emails_only'], chats_only = args['chats_only']) elif args.get('type', '') == 'custom': #convert args to unicode args['request']['req'] = gmvault_utils.convert_to_unicode(args['request']['req']) args['request']['charset'] = 'utf-8' #for the moment always utf-8 args['request']['mode'] = 'custom' # pass an imap request. Assume that the user know what to do here LOG.critical("Perform custom synchronisation with %s request: %s.\n" \ % (args['request']['type'], args['request']['req'])) syncer.sync(args['request'], compress_on_disk = args['compression'], db_cleaning = args['db-cleaning'], \ ownership_checking = args['ownership_control'], restart = args['restart'], \ emails_only = args['emails_only'], chats_only = args['chats_only']) else: raise ValueError("Unknown synchronisation mode %s. Please use full (default), quick or custom.") #print error report LOG.critical(syncer.get_operation_report())
def setup_default_conf(): """ set the environment GMVAULT_CONF_FILE which is necessary for Conf object """ gmvault_utils.get_conf_defaults( ) # force instanciation of conf to load the defaults
def _sync(cls, args, credential): """ Execute All synchronisation operations """ LOG.critical("Connect to Gmail server.\n") # handle credential in all levels syncer = gmvault.GMVaulter(args['db-dir'], args['host'], args['port'], \ args['email'], credential, read_only_access = True, use_encryption = args['encrypt']) #full sync is the first one if args.get('type', '') == 'full': #choose full sync. Ignore the request syncer.sync({ 'mode': 'full', 'type': 'imap', 'req': 'ALL' } , compress_on_disk = args['compression'], \ db_cleaning = args['db-cleaning'], ownership_checking = args['ownership_control'],\ restart = args['restart'], emails_only = args['emails_only'], chats_only = args['chats_only']) elif args.get('type', '') == 'auto': #choose auto sync. imap request = ALL and restart = True syncer.sync({ 'mode': 'auto', 'type': 'imap', 'req': 'ALL' } , compress_on_disk = args['compression'], \ db_cleaning = args['db-cleaning'], ownership_checking = args['ownership_control'],\ restart = True, emails_only = args['emails_only'], chats_only = args['chats_only']) elif args.get('type', '') == 'quick': #sync only the last x days (taken in defaults) in order to be quick #(cleaning is import here because recent days might move again # today - 2 months today = datetime.date.today() begin = today - datetime.timedelta( gmvault_utils.get_conf_defaults().getint( "Sync", "quick_days", 8)) LOG.critical("Quick sync mode. Check for new emails since %s." % (begin.strftime('%d-%b-%Y'))) # today + 1 day end = today + datetime.timedelta(1) req = { 'type' : 'imap', \ 'req' : syncer.get_imap_request_btw_2_dates(begin, end), \ 'mode' : 'quick'} syncer.sync( req, \ compress_on_disk = args['compression'], \ db_cleaning = args['db-cleaning'], \ ownership_checking = args['ownership_control'], restart = args['restart'], \ emails_only = args['emails_only'], chats_only = args['chats_only']) elif args.get('type', '') == 'custom': #convert args to unicode args['request']['req'] = gmvault_utils.convert_to_unicode( args['request']['req']) args['request']['charset'] = 'utf-8' #for the moment always utf-8 args['request']['mode'] = 'custom' # pass an imap request. Assume that the user know what to do here LOG.critical("Perform custom synchronisation with %s request: %s.\n" \ % (args['request']['type'], args['request']['req'])) syncer.sync(args['request'], compress_on_disk = args['compression'], db_cleaning = args['db-cleaning'], \ ownership_checking = args['ownership_control'], restart = args['restart'], \ emails_only = args['emails_only'], chats_only = args['chats_only']) else: raise ValueError( "Unknown synchronisation mode %s. Please use full (default), quick or custom." ) #print error report LOG.critical(syncer.get_operation_report())
def setup_default_conf(): """ set the environment GMVAULT_CONF_FILE which is necessary for Conf object """ gmvault_utils.get_conf_defaults() # force instanciation of conf to load the defaults