def _get_authorization_tokens(cls, authorization_code):
        """Obtains OAuth access token and refresh token.

        This uses the application portion of the "OAuth2 for Installed Applications"
        flow at https://developers.google.com/accounts/docs/OAuth2InstalledApp#handlingtheresponse

        Args:
        client_id: Client ID obtained by registering your app.
        client_secret: Client secret obtained by registering your app.
        authorization_code: code generated by Google Accounts after user grants
            permission.
        Returns:
        The decoded response from the Google Accounts server, as a dict. Expected
        fields include 'access_token', 'expires_in', and 'refresh_token'.
        """
        params = {}
        params['client_id'] = gmvault_utils.get_conf_defaults().get("GoogleOauth2", "gmvault_client_id", "1070918343777-0eecradokiu8i77qfo8e3stbi0mkrtog.apps.googleusercontent.com")
        params['client_secret'] = gmvault_utils.get_conf_defaults().get("GoogleOauth2", "gmvault_client_secret", "IVkl_pglv5cXzugpmnRNqtT7")
        params['code'] = authorization_code
        params['redirect_uri'] = gmvault_utils.get_conf_defaults().get("GoogleOauth2", "redirect_uri", 'urn:ietf:wg:oauth:2.0:oob')
        params['grant_type'] = 'authorization_code'

        account_base_url = gmvault_utils.get_conf_defaults().get("GoogleOauth2", "google_accounts_base_url", 'https://accounts.google.com')

        request_url = '%s/%s' % (account_base_url, 'o/oauth2/token')

        try:
            response = urllib2.urlopen(request_url, urllib.urlencode(params)).read()
        except Exception, err: #pylint: disable-msg=W0703
            LOG.critical("Error: Problems when trying to connect to Google oauth2 endpoint: %s." % (request_url))
            raise err
    def _get_oauth2_acc_tok_from_ref_tok(cls, refresh_token):
      """Obtains a new token given a refresh token.

      See https://developers.google.com/accounts/docs/OAuth2InstalledApp#refresh

      Args:
        client_id: Client ID obtained by registering your app.
        client_secret: Client secret obtained by registering your app.
        refresh_token: A previously-obtained refresh token.
      Returns:
        The decoded response from the Google Accounts server, as a dict. Expected
        fields include 'access_token', 'expires_in', and 'refresh_token'.
      """
      params = {}
      params['client_id'] = gmvault_utils.get_conf_defaults().get("GoogleOauth2", "gmvault_client_id", "1070918343777-0eecradokiu8i77qfo8e3stbi0mkrtog.apps.googleusercontent.com")
      params['client_secret'] = gmvault_utils.get_conf_defaults().get("GoogleOauth2", "gmvault_client_secret", "IVkl_pglv5cXzugpmnRNqtT7")
      params['refresh_token'] = refresh_token
      params['grant_type'] = 'refresh_token'

      account_base_url = gmvault_utils.get_conf_defaults().get("GoogleOauth2", "google_accounts_base_url", 'https://accounts.google.com')

      request_url = '%s/%s' % (account_base_url, 'o/oauth2/token')

      try:
        response = urllib2.urlopen(request_url, urllib.urlencode(params)).read()
      except Exception, err: #pylint: disable-msg=W0703
        LOG.critical("Error: Problems when trying to connect to Google oauth2 endpoint: %s.\n" % (request_url))
        raise err
Exemplo n.º 3
0
def generate_permission_url():
    """Generates the URL for authorizing access.

  This uses the "OAuth2 for Installed Applications" flow described at
  https://developers.google.com/accounts/docs/OAuth2InstalledApp

  Args:
    client_id: Client ID obtained by registering your app.
    scope: scope for access token, e.g. 'https://mail.google.com'
  Returns:
    A URL that the user should visit in their browser.
  """
    params = {}
    params['client_id'] = gmvault_utils.get_conf_defaults().get(
        "GoogleOauth2", "gmvault_client_id", "")
    params['redirect_uri'] = gmvault_utils.get_conf_defaults().get(
        "GoogleOauth2", "redirect_uri", '')
    params['scope'] = gmvault_utils.get_conf_defaults().get(
        "GoogleOauth2", "scope", 'https://mail.google.com/')
    params['response_type'] = 'code'

    account_base_url = gmvault_utils.get_conf_defaults().get(
        "GoogleOauth2", "google_accounts_base_url",
        'https://accounts.google.com')

    return '%s/%s?%s' % (account_base_url, 'o/oauth2/auth',
                         gmvault_utils.format_url_params(params))
Exemplo n.º 4
0
    def _get_oauth2_acc_tok_from_ref_tok(cls, refresh_token):
        """Obtains a new token given a refresh token.

      See https://developers.google.com/accounts/docs/OAuth2InstalledApp#refresh

      Args:
        client_id: Client ID obtained by registering your app.
        client_secret: Client secret obtained by registering your app.
        refresh_token: A previously-obtained refresh token.
      Returns:
        The decoded response from the Google Accounts server, as a dict. Expected
        fields include 'access_token', 'expires_in', and 'refresh_token'.
      """
        params = {}
        params['client_id'] = gmvault_utils.get_conf_defaults().get(
            "GoogleOauth2", "gmvault_client_id", "")
        params['client_secret'] = gmvault_utils.get_conf_defaults().get(
            "GoogleOauth2", "gmvault_client_secret", "")
        params['refresh_token'] = refresh_token
        params['grant_type'] = 'refresh_token'

        account_base_url = gmvault_utils.get_conf_defaults().get(
            "GoogleOauth2", "google_accounts_base_url",
            'https://accounts.google.com')

        request_url = '%s/%s' % (account_base_url, 'o/oauth2/token')

        try:
            response = urllib2.urlopen(request_url,
                                       urllib.urlencode(params)).read()
        except Exception, err:  #pylint: disable-msg=W0703
            LOG.critical(
                "Error: Problems when trying to connect to Google oauth2 endpoint: %s.\nMaybe you changed your password? Rename/remove the auth token file that is named after the email address, and try again to refresh the token."
                % (request_url))
            raise err
Exemplo n.º 5
0
    def connect(self, go_to_current_folder = False):
        """
           connect to the IMAP server
        """

        # check if it is needed to disbale cert verification (leave that option at the last resort)
        # if the cert verification doesn't work.
        disable_cert_verification = gmvault_utils.get_conf_defaults().getboolean("GoogleOauth2", "disable_cacert_verification", default=False)
        context = None
        if disable_cert_verification:
            LOG.critical("Beware disabling CA Cert verification for the IMAP connection to Gmail.")
            context = imapclient.create_default_context()
            # don't check if certificate hostname doesn't match target hostname
            context.check_hostname = False
            # don't check if the certificate is trusted by a certificate authority
            context.verify_mode = ssl.CERT_NONE
        else:
            # create a custom ssl context to handle CA cert verification in all cases
            # In some plateform the CA certs are not available so use the ones provided by Gmvault
            # CA certs are coming from https://curl.haxx.se/ca/cacert.pem
            context = imapclient.create_default_context(cafile= gmvault_utils.get_ca_certs_filepath())

        # create imap object
        self.server = mimap.MonkeyIMAPClient(self.host, port = self.port, use_uid = self.use_uid, need_ssl = self.ssl, ssl_context = context)
        # connect with password or xoauth
        if self.credential['type'] == 'passwd':
            self.server.login(self.login, self.credential['value'])
        elif self.credential['type'] == 'oauth2':
            #connect with oauth2
            if self.once_connected:
                self.credential = credential_utils.CredentialHelper.get_oauth2_credential(self.login, renew_cred = False)

            LOG.debug("credential['value'] = %s" % (self.credential['value']))
            #try to login
            self.server.oauth2_login(self.credential['value'])
        else:
            raise Exception("Unknown authentication method %s. Please use xoauth or passwd authentication " \
                            % (self.credential['type']))
            
        #set connected to True to handle reconnection in case of failure
        self.once_connected = True
        
        # check gmailness
        self.check_gmailness()
         
        # find allmail chats and drafts folders
        self.find_folder_names()

        if go_to_current_folder and self.current_folder:
            self.server.select_folder(self.current_folder, readonly = self.readonly_folder)
            
        #enable compression
        if gmvault_utils.get_conf_defaults().get_boolean('General', 'enable_imap_compression', True):
            self.enable_compression()
            LOG.debug("After Enabling compression.")
        else:
            LOG.debug("Do not enable imap compression.") 
Exemplo n.º 6
0
 def __init__(self, host, port, login, credential, readonly_folder = True): #pylint:disable=R0913
     '''
         Constructor
     '''
     self.host                   = host
     self.port                   = port
     self.login                  = login
     self.once_connected         = False
     self.credential             = credential
     self.ssl                    = True
     self.use_uid                = True
     self.readonly_folder        = readonly_folder
     
     self.localized_folders      = { 'ALLMAIL': { 'loc_dir' : None, 'friendly_name' : 'allmail'}, 
                                     'CHATS'  : { 'loc_dir' : None, 'friendly_name' : 'chats'}, 
                                     'DRAFTS' : { 'loc_dir' : None, 'friendly_name' : 'drafts'} }
     
     # memoize the current folder (All Mail or Chats) for reconnection management
     self.current_folder        = None
     
     self.server                 = None
     self.go_to_all_folder       = True
     self.total_nb_reconns       = 0
     # True when CHATS or other folder error msg has been already printed
     self.printed_folder_error_msg = { 'ALLMAIL' : False, 'CHATS': False , 'DRAFTS':False }
     
     #update GENERIC_GMAIL_CHATS. Should be done at the class level
     self.GENERIC_GMAIL_CHATS.extend(gmvault_utils.get_conf_defaults().get_list('Localisation', 'chat_folder', []))
Exemplo n.º 7
0
 def __init__(self, host, port, login, credential, readonly_folder = True): #pylint:disable=R0913
     '''
         Constructor
     '''
     self.host                   = host
     self.port                   = port
     self.login                  = login
     self.once_connected         = False
     self.credential             = credential
     self.ssl                    = True
     self.use_uid                = True
     self.readonly_folder        = readonly_folder
     
     self.localized_folders      = { 'ALLMAIL': { 'loc_dir' : None, 'friendly_name' : 'allmail'}, 
                                     'CHATS'  : { 'loc_dir' : None, 'friendly_name' : 'chats'}, 
                                     'DRAFTS'  :{ 'loc_dir' : None, 'friendly_name' : 'drafts'} }
     
     # memoize the current folder (All Mail or Chats) for reconnection management
     self.current_folder        = None
     
     self.server                 = None
     self.go_to_all_folder       = True
     self.total_nb_reconns       = 0
     # True when CHATS or other folder error msg has been already printed
     self.printed_folder_error_msg = { 'ALLMAIL' : False, 'CHATS': False , 'DRAFTS':False }
     
     #update GENERIC_GMAIL_CHATS. Should be done at the class level
     self.GENERIC_GMAIL_CHATS.extend(gmvault_utils.get_conf_defaults().get_list('Localisation', 'chat_folder', []))
    def find_chats_folder(self):
        """
           depending on your account the chats folder can be named 
           [GMAIL]/Chats or [GoogleMail]/Chats, [GMAIL]/tous les chats ...
           Find and set the right one
           Npte: Cannot use the flags as Chats is not a system label. Thanks Google
        """
        #use xlist because of localized dir names
        folders = self.server.xlist_folders()

        LOG.debug("Folders = %s\n" % (folders))

        the_dir = None
        for (_, _, the_dir) in folders:
            #look for GMAIL Chats
            if the_dir in GIMAPFetcher.GENERIC_GMAIL_CHATS:
                #it could be a localized Dir name
                self.localized_folders['CHATS']['loc_dir'] = the_dir
                return the_dir

        #Error did not find Chats dir
        if gmvault_utils.get_conf_defaults().getboolean(
                "General", "errors_if_chat_not_visible", False):
            raise Exception("Cannot find global 'Chats' folder ! Check whether 'Show in IMAP for 'Chats' "\
                            "is enabled in Gmail (Go to Settings->Labels->All Mail)")

        return None
    def find_folder_names(self):
        """
           depending on your account the all mail folder can be named 
           [GMAIL]/ALL Mail or [GoogleMail]/All Mail.
           Find and set the right one
        """
        #use xlist because of localized dir names
        folders = self.server.xlist_folders()

        the_dir = None
        for (flags, _, the_dir) in folders:
            #non localised GMAIL_ALL
            if GIMAPFetcher.GENERIC_GMAIL_ALL in flags:
                #it could be a localized Dir name
                self.localized_folders['ALLMAIL']['loc_dir'] = the_dir
            elif the_dir in GIMAPFetcher.GENERIC_GMAIL_CHATS:
                #it could be a localized Dir name
                self.localized_folders['CHATS']['loc_dir'] = the_dir
            elif GIMAPFetcher.GENERIC_DRAFTS in flags:
                self.localized_folders['DRAFTS']['loc_dir'] = the_dir

        if not self.localized_folders['ALLMAIL']['loc_dir']:  # all mail error
            raise Exception("Cannot find global 'All Mail' folder (maybe localized and translated into your language) ! "\
                            "Check whether 'Show in IMAP for 'All Mail' is enabled in Gmail (Go to Settings->Labels->All Mail)")
        elif not self.localized_folders['CHATS']['loc_dir'] and \
                 gmvault_utils.get_conf_defaults().getboolean("General","errors_if_chat_not_visible", False):
            raise Exception("Cannot find global 'Chats' folder ! Check whether 'Show in IMAP for 'Chats' "\
                            "is enabled in Gmail (Go to Settings->Labels->All Mail)")
        elif not self.localized_folders['DRAFTS']['loc_dir']:
            raise Exception("Cannot find global 'Drafts' folder.")
Exemplo n.º 10
0
 def find_folder_names(self):
     """
        depending on your account the all mail folder can be named 
        [GMAIL]/ALL Mail or [GoogleMail]/All Mail.
        Find and set the right one
     """      
     #use xlist because of localized dir names
     folders = self.server.xlist_folders()
     
     the_dir = None
     for (flags, _, the_dir) in folders:
         #non localised GMAIL_ALL
         if GIMAPFetcher.GENERIC_GMAIL_ALL in flags:
             #it could be a localized Dir name
             self.localized_folders['ALLMAIL']['loc_dir'] = the_dir
         elif the_dir in GIMAPFetcher.GENERIC_GMAIL_CHATS :
             #it could be a localized Dir name
             self.localized_folders['CHATS']['loc_dir'] = the_dir
         elif GIMAPFetcher.GENERIC_DRAFTS in flags:
             self.localized_folders['DRAFTS']['loc_dir'] = the_dir
             
     if not self.localized_folders['ALLMAIL']['loc_dir']: # all mail error
         raise Exception("Cannot find global 'All Mail' folder (maybe localized and translated into your language) ! "\
                         "Check whether 'Show in IMAP for 'All Mail' is enabled in Gmail (Go to Settings->Labels->All Mail)")
     elif not self.localized_folders['CHATS']['loc_dir'] and \
              gmvault_utils.get_conf_defaults().getboolean("General","errors_if_chat_not_visible", False):
         raise Exception("Cannot find global 'Chats' folder ! Check whether 'Show in IMAP for 'Chats' "\
                         "is enabled in Gmail (Go to Settings->Labels->All Mail)") 
     elif not self.localized_folders['DRAFTS']['loc_dir']:
         raise Exception("Cannot find global 'Drafts' folder.")
Exemplo n.º 11
0
 def find_chats_folder(self):
     """
        depending on your account the chats folder can be named 
        [GMAIL]/Chats or [GoogleMail]/Chats, [GMAIL]/tous les chats ...
        Find and set the right one
        Npte: Cannot use the flags as Chats is not a system label. Thanks Google
     """
     #use xlist because of localized dir names
     folders = self.server.xlist_folders()
     
     LOG.debug("Folders = %s\n" % (folders))
     
     the_dir = None
     for (_, _, the_dir) in folders:
         #look for GMAIL Chats
         if the_dir in GIMAPFetcher.GENERIC_GMAIL_CHATS :
             #it could be a localized Dir name
             self.localized_folders['CHATS']['loc_dir'] = the_dir
             return the_dir
     
     #Error did not find Chats dir 
     if gmvault_utils.get_conf_defaults().getboolean("General", "errors_if_chat_not_visible", False):
         raise Exception("Cannot find global 'Chats' folder ! Check whether 'Show in IMAP for 'Chats' "\
                         "is enabled in Gmail (Go to Settings->Labels->All Mail)") 
    
     return None
Exemplo n.º 12
0
    def create_gmail_labels(self, labels, existing_folders):
        """
           Create folders and subfolders on Gmail in order
           to recreate the label hierarchy before to upload emails
           Note that adding labels with +X-GM-LABELS create only nested labels
           but not nested ones. This is why this trick must be used to 
           recreate the label hierarchy
           
           labels: list of labels to create
           
        """
        
        #1.5-beta moved that out of the loop to minimize the number of calls
        #to that method. (Could go further and memoize it)
        
        #get existing directories (or label parts)
        # get in lower case because Gmail labels are case insensitive
        listed_folders   = set([ directory.lower() for (_, _, directory) in self.list_all_folders() ])
        existing_folders = listed_folders.union(existing_folders)
        reserved_labels_map = gmvault_utils.get_conf_defaults().get_dict("Restore", "reserved_labels_map", \
                              { u'migrated' : u'gmv-migrated', u'\muted' : u'gmv-muted' })
        
        

        LOG.debug("Labels to create: [%s]" % (labels))
            
        for lab in labels:
            #LOG.info("Reserved labels = %s\n" % (reserved_labels))
            #LOG.info("lab.lower = %s\n" % (lab.lower()))
            if lab.lower() in reserved_labels_map.keys(): #exclude creation of migrated label
                n_lab = reserved_labels_map.get(lab.lower(), "gmv-default-label")
                LOG.info("Warning ! label '%s' (lower or uppercase) is reserved by Gmail and cannot be used."\
                         "Use %s instead" % (lab, n_lab)) 
                lab = n_lab
                LOG.info("translated lab = %s\n" % (lab))
           
            #split all labels
            labs = self._get_dir_from_labels(lab) 
            
            for directory in labs:
                low_directory = directory.lower() #get lower case directory but store original label
                if (low_directory not in existing_folders) and (low_directory not in self.GMAIL_SPECIAL_DIRS_LOWER):
                    try:
                        if self.server.create_folder(directory) != 'Success':
                            raise Exception("Cannot create label %s: the directory %s cannot be created." % (lab, directory))
                        else:
                            LOG.debug("============== ####### Created Labels (%s)." % (directory))
                    except imaplib.IMAP4.error, error:
                        #log error in log file if it exists
                        LOG.debug(gmvault_utils.get_exception_traceback())
                        if str(error).startswith("create failed: '[ALREADYEXISTS] Duplicate folder"):
                            LOG.critical("Warning: label %s already exists on Gmail and Gmvault tried to create it."\
                                         " Ignore this issue." % (directory) )
                        else:
                            raise error
                    
                    #add created folder in folders
                    existing_folders.add(low_directory)
Exemplo n.º 13
0
    def create_gmail_labels(self, labels, existing_folders):
        """
           Create folders and subfolders on Gmail in order
           to recreate the label hierarchy before to upload emails
           Note that adding labels with +X-GM-LABELS create only nested labels
           but not nested ones. This is why this trick must be used to 
           recreate the label hierarchy
           
           labels: list of labels to create
           
        """
        
        #1.5-beta moved that out of the loop to minimize the number of calls
        #to that method. (Could go further and memoize it)
        
        #get existing directories (or label parts)
        # get in lower case because Gmail labels are case insensitive
        listed_folders   = set([ directory.lower() for (_, _, directory) in self.list_all_folders() ])
        existing_folders = listed_folders.union(existing_folders)
        reserved_labels_map = gmvault_utils.get_conf_defaults().get_dict("Restore", "reserved_labels_map", \
                              { u'migrated' : u'gmv-migrated', u'\muted' : u'gmv-muted' })
        
        

        LOG.debug("Labels to create: [%s]" % (labels))
            
        for lab in labels:
            #LOG.info("Reserved labels = %s\n" % (reserved_labels))
            #LOG.info("lab.lower = %s\n" % (lab.lower()))
            if lab.lower() in reserved_labels_map.keys(): #exclude creation of migrated label
                n_lab = reserved_labels_map.get(lab.lower(), "gmv-default-label")
                LOG.info("Warning ! label '%s' (lower or uppercase) is reserved by Gmail and cannot be used."\
                         "Use %s instead" % (lab, n_lab)) 
                lab = n_lab
                LOG.info("translated lab = %s\n" % (lab))
           
            #split all labels
            labs = self._get_dir_from_labels(lab) 
            
            for directory in labs:
                low_directory = directory.lower() #get lower case directory but store original label
                if (low_directory not in existing_folders) and (low_directory not in self.GMAIL_SPECIAL_DIRS_LOWER):
                    try:
                        if self.server.create_folder(directory) != 'Success':
                            raise Exception("Cannot create label %s: the directory %s cannot be created." % (lab, directory))
                        else:
                            LOG.debug("============== ####### Created Labels (%s)." % (directory))
                    except imaplib.IMAP4.error, error:
                        #log error in log file if it exists
                        LOG.debug(gmvault_utils.get_exception_traceback())
                        if str(error).startswith("create failed: '[ALREADYEXISTS] Duplicate folder"):
                            LOG.critical("Warning: label %s already exists on Gmail and Gmvault tried to create it."\
                                         " Ignore this issue." % (directory) )
                        else:
                            raise error
                    
                    #add created folder in folders
                    existing_folders.add(low_directory)
Exemplo n.º 14
0
    def delete_emails(self, emails_info, msg_type):
        """
           Delete all emails and metadata with ids
        """
        if msg_type == 'email':
            db_dir = self._db_dir
        else:
            db_dir = self._chats_dir

        move_to_bin = gmvault_utils.get_conf_defaults().get_boolean(
            "General", "keep_in_bin" , False)

        if move_to_bin:
            LOG.critical("Move emails to the bin:%s" % self._bin_dir)

        for (a_id, date_dir) in emails_info:

            the_dir = '%s/%s' % (db_dir, date_dir)

            data_p      = self.DATA_FNAME % (the_dir, a_id)
            comp_data_p = '%s.gz' % data_p
            cryp_comp_data_p = '%s.crypt.gz' % data_p

            metadata_p  = self.METADATA_FNAME % (the_dir, a_id)

            if move_to_bin:
                #move files to the bin
                gmvault_utils.makedirs(self._bin_dir)

                # create bin filenames
                bin_p          = self.DATA_FNAME % (self._bin_dir, a_id)
                metadata_bin_p = self.METADATA_FNAME % (self._bin_dir, a_id)

                if os.path.exists(data_p):
                    os.rename(data_p, bin_p)
                elif os.path.exists(comp_data_p):
                    os.rename(comp_data_p, '%s.gz' % bin_p)
                elif os.path.exists(cryp_comp_data_p):
                    os.rename(cryp_comp_data_p, '%s.crypt.gz' % bin_p)   
                
                if os.path.exists(metadata_p):
                    os.rename(metadata_p, metadata_bin_p)
            else:
                #delete files if they exists
                if os.path.exists(data_p):
                    os.remove(data_p)
                elif os.path.exists(comp_data_p):
                    os.remove(comp_data_p)
                elif os.path.exists(cryp_comp_data_p):
                    os.remove(cryp_comp_data_p)   

                if os.path.exists(metadata_p):
                    os.remove(metadata_p)
                    
            gmsql.GMSQL.delete_email(a_id)
Exemplo n.º 15
0
    def delete_emails(self, emails_info, msg_type):
        """
           Delete all emails and metadata with ids
        """
        if msg_type == 'email':
            db_dir = self._db_dir
        else:
            db_dir = self._chats_dir

        move_to_bin = gmvault_utils.get_conf_defaults().get_boolean(
            "General", "keep_in_bin" , False)

        if move_to_bin:
            LOG.critical("Move emails to the bin:%s" % self._bin_dir)

        for (a_id, date_dir) in emails_info:

            the_dir = '%s/%s' % (db_dir, date_dir)

            data_p      = self.DATA_FNAME % (the_dir, a_id)
            comp_data_p = '%s.gz' % data_p
            cryp_comp_data_p = '%s.crypt.gz' % data_p

            metadata_p  = self.METADATA_FNAME % (the_dir, a_id)

            if move_to_bin:
                #move files to the bin
                gmvault_utils.makedirs(self._bin_dir)

                # create bin filenames
                bin_p          = self.DATA_FNAME % (self._bin_dir, a_id)
                metadata_bin_p = self.METADATA_FNAME % (self._bin_dir, a_id)

                if os.path.exists(data_p):
                    os.rename(data_p, bin_p)
                elif os.path.exists(comp_data_p):
                    os.rename(comp_data_p, '%s.gz' % bin_p)
                elif os.path.exists(cryp_comp_data_p):
                    os.rename(cryp_comp_data_p, '%s.crypt.gz' % bin_p)   
                
                if os.path.exists(metadata_p):
                    os.rename(metadata_p, metadata_bin_p)
            else:
                #delete files if they exists
                if os.path.exists(data_p):
                    os.remove(data_p)
                elif os.path.exists(comp_data_p):
                    os.remove(comp_data_p)
                elif os.path.exists(cryp_comp_data_p):
                    os.remove(cryp_comp_data_p)   

                if os.path.exists(metadata_p):
                    os.remove(metadata_p)
Exemplo n.º 16
0
def generate_permission_url():
  """Generates the URL for authorizing access.

  This uses the "OAuth2 for Installed Applications" flow described at
  https://developers.google.com/accounts/docs/OAuth2InstalledApp

  Args:
    client_id: Client ID obtained by registering your app.
    scope: scope for access token, e.g. 'https://mail.google.com'
  Returns:
    A URL that the user should visit in their browser.
  """
  params = {}
  params['client_id']     = gmvault_utils.get_conf_defaults().get("GoogleOauth2", "gmvault_client_id", "1070918343777-0eecradokiu8i77qfo8e3stbi0mkrtog.apps.googleusercontent.com")
  params['redirect_uri']  = gmvault_utils.get_conf_defaults().get("GoogleOauth2", "redirect_uri", 'urn:ietf:wg:oauth:2.0:oob')
  params['scope']         = gmvault_utils.get_conf_defaults().get("GoogleOauth2","scope",'https://mail.google.com/')
  params['response_type'] = 'code'

  account_base_url = gmvault_utils.get_conf_defaults().get("GoogleOauth2", "google_accounts_base_url", 'https://accounts.google.com')

  return '%s/%s?%s' % (account_base_url, 'o/oauth2/auth', gmvault_utils.format_url_params(params))
Exemplo n.º 17
0
def bootstrap_run():
    """ temporary bootstrap """

    init_logging()

    #force argv[0] to gmvault
    sys.argv[0] = "gmvault"

    LOG.critical("")

    gmvlt = GMVaultLauncher()

    args = gmvlt.parse_args()

    #activate debug if enabled
    if args['debug']:
        LOG.critical("Activate debugging information.")
        activate_debug_mode()

    # force instanciation of conf to load the defaults
    gmvault_utils.get_conf_defaults()

    gmvlt.run(args)
Exemplo n.º 18
0
def bootstrap_run():
    """ temporary bootstrap """
    
    init_logging()
    
    #force argv[0] to gmvault
    sys.argv[0] = "gmvault"
    
    LOG.critical("")
    
    gmvlt = GMVaultLauncher()
    
    args = gmvlt.parse_args()
    
    #activate debug if enabled
    if args['debug']:
        LOG.critical("Activate debugging information.")
        activate_debug_mode()
    
    # force instanciation of conf to load the defaults
    gmvault_utils.get_conf_defaults() 
    
    gmvlt.run(args)
Exemplo n.º 19
0
def check_to_disable_certificate_verification():
  """
     Check from the configuration if certificate verification needs to be disabled.
     Have to use monkey patching strategy while version older than 2.7.9 are still supported.
  """
  #solve ssl context issue for some users
  if hasattr('ssl', 'SSLContext') == False:
      disable_cert_verification = gmvault_utils.get_conf_defaults().getboolean("GoogleOauth2", "disable_cacert_verification", default=False, fail_if_missing = True)
      if disable_cert_verification == True:
         #loc_context = ssl.SSLContext(ssl.PROTOCOL_TLSv1)
         #for the moment monkey patch later on will use the line above
         LOG.critical("Beware as asked Gmvault is going to disable the SSL Certificate Verification.")
         ssl._create_default_https_context = ssl._create_unverified_context
      else:
         LOG.critical("Use Default certificate context.")
  else:
     LOG.debug("Ignore check_to_disable_certificate_verification. No SSLContext Object in ssl, probably using a python version < to 2.7.9")
    def connect(self, go_to_current_folder=False):
        """
           connect to the IMAP server
        """
        # create imap object
        self.server = mimap.MonkeyIMAPClient(self.host,
                                             port=self.port,
                                             use_uid=self.use_uid,
                                             need_ssl=self.ssl)
        # connect with password or xoauth
        if self.credential['type'] == 'passwd':
            self.server.login(self.login, self.credential['value'])
        elif self.credential['type'] == 'oauth2':
            #connect with oauth2
            if self.once_connected:
                self.credential = credential_utils.CredentialHelper.get_oauth2_credential(
                    self.login, renew_cred=False)

            LOG.debug("credential['value'] = %s" % (self.credential['value']))
            #try to login
            self.server.oauth2_login(self.credential['value'])
        else:
            raise Exception("Unknown authentication method %s. Please use xoauth or passwd authentication " \
                            % (self.credential['type']))

        #set connected to True to handle reconnection in case of failure
        self.once_connected = True

        # check gmailness
        self.check_gmailness()

        # find allmail chats and drafts folders
        self.find_folder_names()

        if go_to_current_folder and self.current_folder:
            self.server.select_folder(self.current_folder,
                                      readonly=self.readonly_folder)

        #enable compression
        if gmvault_utils.get_conf_defaults().get_boolean(
                'General', 'enable_imap_compression', True):
            self.enable_compression()
            LOG.debug("After Enabling compression.")
        else:
            LOG.debug("Do not enable imap compression.")
Exemplo n.º 21
0
    def _restore(cls, args, credential):
        """
           Execute All restore operations
        """
        LOG.critical("Connect to Gmail server.\n")
        # Create a gmvault releaving read_only_access
        restorer = gmvault.GMVaulter(args['db-dir'], args['host'], args['port'], \
                                       args['email'], credential, read_only_access = False)

        #full sync is the first one
        if args.get('type', '') == 'full':

            #call restore
            labels = [args['apply_label']] if args['apply_label'] else []
            restorer.restore(extra_labels = labels, restart = args['restart'], \
                             emails_only = args['emails_only'], chats_only = args['chats_only'])

        elif args.get('type', '') == 'quick':

            #take the last two to 3 months depending on the current date

            # today - 2 months
            today = datetime.date.today()
            begin = today - datetime.timedelta(
                gmvault_utils.get_conf_defaults().getint(
                    "Restore", "quick_days", 8))

            starting_dir = gmvault_utils.get_ym_from_datetime(begin)

            #call restore
            labels = [args['apply_label']] if args['apply_label'] else []
            restorer.restore(pivot_dir = starting_dir, extra_labels = labels, restart = args['restart'], \
                             emails_only = args['emails_only'], chats_only = args['chats_only'])

        else:
            raise ValueError(
                "Unknown synchronisation mode %s. Please use full (default), quick."
            )

        #print error report
        LOG.critical(restorer.get_operation_report())
Exemplo n.º 22
0
    def __init__(self, a_storage_dir, encrypt_data=False):
        """
           Store on disks
           args:
              a_storage_dir: Storage directory
              a_use_encryption: Encryption key. If there then encrypt
        """
        self._top_dir = a_storage_dir

        self._db_dir          = '%s/%s' % (a_storage_dir, GmailStorer.DB_AREA)
        self._quarantine_dir  = '%s/%s' % (a_storage_dir, GmailStorer.QUARANTINE_AREA)
        self._info_dir        = '%s/%s' % (a_storage_dir, GmailStorer.INFO_AREA)
        self._chats_dir       = '%s/%s' % (self._db_dir, GmailStorer.CHATS_AREA)
        self._bin_dir         = '%s/%s' % (a_storage_dir, GmailStorer.BIN_AREA)
        
        gmsql.GMSQL.connect('%s/meta.db' % (a_storage_dir))

        self._sub_chats_dir   = None
        self._sub_chats_inc   = -1
        self._sub_chats_nb    = -1

        self._limit_per_chat_dir = gmvault_utils.get_conf_defaults().getint(
            "General", "limit_per_chat_dir", 1500)

        #make dirs
        if not os.path.exists(self._db_dir):
            LOG.critical("No Storage DB in %s. Create it.\n" % a_storage_dir)

        gmvault_utils.makedirs(self._db_dir)
        gmvault_utils.makedirs(self._chats_dir)
        gmvault_utils.makedirs(self._quarantine_dir)
        gmvault_utils.makedirs(self._info_dir)

        self.fsystem_info_cache = {}

        self._encrypt_data   = encrypt_data
        self._encryption_key = None
        self._cipher         = None

        #add version if it is needed to migrate gmvault-db in the future
        self._create_gmvault_db_version()
Exemplo n.º 23
0
    def __init__(self, a_storage_dir, encrypt_data=False):
        """
           Store on disks
           args:
              a_storage_dir: Storage directory
              a_use_encryption: Encryption key. If there then encrypt
        """
        self._top_dir = a_storage_dir

        self._db_dir          = '%s/%s' % (a_storage_dir, GmailStorer.DB_AREA)
        self._quarantine_dir  = '%s/%s' % (a_storage_dir, GmailStorer.QUARANTINE_AREA)
        self._info_dir        = '%s/%s' % (a_storage_dir, GmailStorer.INFO_AREA)
        self._chats_dir       = '%s/%s' % (self._db_dir, GmailStorer.CHATS_AREA)
        self._bin_dir         = '%s/%s' % (a_storage_dir, GmailStorer.BIN_AREA)

        self._sub_chats_dir   = None
        self._sub_chats_inc   = -1
        self._sub_chats_nb    = -1

        self._limit_per_chat_dir = gmvault_utils.get_conf_defaults().getint(
            "General", "limit_per_chat_dir", 1500)

        #make dirs
        if not os.path.exists(self._db_dir):
            LOG.critical("No Storage DB in %s. Create it.\n" % a_storage_dir)

        gmvault_utils.makedirs(self._db_dir)
        gmvault_utils.makedirs(self._chats_dir)
        gmvault_utils.makedirs(self._quarantine_dir)
        gmvault_utils.makedirs(self._info_dir)

        self.fsystem_info_cache = {}

        self._encrypt_data   = encrypt_data
        self._encryption_key = None
        self._cipher         = None

        #add version if it is needed to migrate gmvault-db in the future
        self._create_gmvault_db_version()
Exemplo n.º 24
0
    def connect(self, go_to_current_folder = False):
        """
           connect to the IMAP server
        """
        # create imap object
        self.server = mimap.MonkeyIMAPClient(self.host, port = self.port, use_uid= self.use_uid, need_ssl= self.ssl)
        # connect with password or xoauth
        if self.credential['type'] == 'passwd':
            self.server.login(self.login, self.credential['value'])
        elif self.credential['type'] == 'oauth2':
            #connect with oauth2
            if self.once_connected:
                self.credential = credential_utils.CredentialHelper.get_oauth2_credential(self.login, renew_cred = False)

            LOG.debug("credential['value'] = %s" % (self.credential['value']))
            #try to login
            self.server.oauth2_login(self.credential['value'])
        else:
            raise Exception("Unknown authentication method %s. Please use xoauth or passwd authentication " \
                            % (self.credential['type']))
            
        #set connected to True to handle reconnection in case of failure
        self.once_connected = True
        
        # check gmailness
        self.check_gmailness()
         
        # find allmail chats and drafts folders
        self.find_folder_names()

        if go_to_current_folder and self.current_folder:
            self.server.select_folder(self.current_folder, readonly = self.readonly_folder)
            
        #enable compression
        if gmvault_utils.get_conf_defaults().get_boolean('General', 'enable_imap_compression', True):
            self.enable_compression()
            LOG.debug("After Enabling compression.")
        else:
            LOG.debug("Do not enable imap compression.") 
Exemplo n.º 25
0
 def _restore(cls, args, credential):
     """
        Execute All restore operations
     """
     LOG.critical("Connect to Gmail server.\n")
     # Create a gmvault releaving read_only_access
     restorer = gmvault.GMVaulter(args['db-dir'], args['host'], args['port'], \
                                    args['email'], credential, read_only_access = False)
     
     #full sync is the first one
     if args.get('type', '') == 'full':
         
         #call restore
         labels = [args['apply_label']] if args['apply_label'] else []
         restorer.restore(extra_labels = labels, restart = args['restart'], \
                          emails_only = args['emails_only'], chats_only = args['chats_only'])
         
     elif args.get('type', '') == 'quick':
         
         #take the last two to 3 months depending on the current date
         
         # today - 2 months
         today = datetime.date.today()
         begin = today - datetime.timedelta(gmvault_utils.get_conf_defaults().getint("Restore", "quick_days", 8))
         
         starting_dir = gmvault_utils.get_ym_from_datetime(begin)
         
         #call restore
         labels = [args['apply_label']] if args['apply_label'] else []
         restorer.restore(pivot_dir = starting_dir, extra_labels = labels, restart = args['restart'], \
                          emails_only = args['emails_only'], chats_only = args['chats_only'])
     
     else:
         raise ValueError("Unknown synchronisation mode %s. Please use full (default), quick.")
     
     #print error report
     LOG.critical(restorer.get_operation_report()) 
Exemplo n.º 26
0
    def _sync(cls, args, credential):
        """
           Execute All synchronisation operations
        """
        LOG.critical("Connect to Gmail server.\n")
        
        # handle credential in all levels
        syncer = gmvault.GMVaulter(args['db-dir'], args['host'], args['port'], \
                                       args['email'], credential, read_only_access = True, use_encryption = args['encrypt'])
        
        
        
        #full sync is the first one
        if args.get('type', '') == 'full':
        
            #choose full sync. Ignore the request
            syncer.sync({ 'mode': 'full', 'type': 'imap', 'req': 'ALL' } , compress_on_disk = args['compression'], \
                        db_cleaning = args['db-cleaning'], ownership_checking = args['ownership_control'],\
                        restart = args['restart'], emails_only = args['emails_only'], chats_only = args['chats_only'])
        
        elif args.get('type', '') == 'auto':
        
            #choose auto sync. imap request = ALL and restart = True
            syncer.sync({ 'mode': 'auto', 'type': 'imap', 'req': 'ALL' } , compress_on_disk = args['compression'], \
                        db_cleaning = args['db-cleaning'], ownership_checking = args['ownership_control'],\
                        restart = True, emails_only = args['emails_only'], chats_only = args['chats_only'])
              
        elif args.get('type', '') == 'quick':
            
            #sync only the last x days (taken in defaults) in order to be quick 
            #(cleaning is import here because recent days might move again
            
            # today - 2 months
            today = datetime.date.today()
            begin = today - datetime.timedelta(gmvault_utils.get_conf_defaults().getint("Sync", "quick_days", 8))
            
            LOG.critical("Quick sync mode. Check for new emails since %s." % (begin.strftime('%d-%b-%Y')))
            
            # today + 1 day
            end   = today + datetime.timedelta(1)
            
            req   = { 'type' : 'imap', \
                      'req'  : syncer.get_imap_request_btw_2_dates(begin, end), \
                      'mode' : 'quick'}
            
            syncer.sync( req, \
                         compress_on_disk = args['compression'], \
                         db_cleaning = args['db-cleaning'], \
                         ownership_checking = args['ownership_control'], restart = args['restart'], \
                         emails_only = args['emails_only'], chats_only = args['chats_only'])
            
        elif args.get('type', '') == 'custom':
            
            #convert args to unicode
            args['request']['req']     = gmvault_utils.convert_to_unicode(args['request']['req'])
            args['request']['charset'] = 'utf-8' #for the moment always utf-8
            args['request']['mode']    = 'custom'

            # pass an imap request. Assume that the user know what to do here
            LOG.critical("Perform custom synchronisation with %s request: %s.\n" \
                         % (args['request']['type'], args['request']['req']))
            
            syncer.sync(args['request'], compress_on_disk = args['compression'], db_cleaning = args['db-cleaning'], \
                        ownership_checking = args['ownership_control'], restart = args['restart'], \
                        emails_only = args['emails_only'], chats_only = args['chats_only'])
        else:
            raise ValueError("Unknown synchronisation mode %s. Please use full (default), quick or custom.")
        
        
        #print error report
        LOG.critical(syncer.get_operation_report())
Exemplo n.º 27
0
def setup_default_conf():
    """
       set the environment GMVAULT_CONF_FILE which is necessary for Conf object
    """
    gmvault_utils.get_conf_defaults(
    )  # force instanciation of conf to load the defaults
Exemplo n.º 28
0
    def _sync(cls, args, credential):
        """
           Execute All synchronisation operations
        """
        LOG.critical("Connect to Gmail server.\n")

        # handle credential in all levels
        syncer = gmvault.GMVaulter(args['db-dir'], args['host'], args['port'], \
                                       args['email'], credential, read_only_access = True, use_encryption = args['encrypt'])

        #full sync is the first one
        if args.get('type', '') == 'full':

            #choose full sync. Ignore the request
            syncer.sync({ 'mode': 'full', 'type': 'imap', 'req': 'ALL' } , compress_on_disk = args['compression'], \
                        db_cleaning = args['db-cleaning'], ownership_checking = args['ownership_control'],\
                        restart = args['restart'], emails_only = args['emails_only'], chats_only = args['chats_only'])

        elif args.get('type', '') == 'auto':

            #choose auto sync. imap request = ALL and restart = True
            syncer.sync({ 'mode': 'auto', 'type': 'imap', 'req': 'ALL' } , compress_on_disk = args['compression'], \
                        db_cleaning = args['db-cleaning'], ownership_checking = args['ownership_control'],\
                        restart = True, emails_only = args['emails_only'], chats_only = args['chats_only'])

        elif args.get('type', '') == 'quick':

            #sync only the last x days (taken in defaults) in order to be quick
            #(cleaning is import here because recent days might move again

            # today - 2 months
            today = datetime.date.today()
            begin = today - datetime.timedelta(
                gmvault_utils.get_conf_defaults().getint(
                    "Sync", "quick_days", 8))

            LOG.critical("Quick sync mode. Check for new emails since %s." %
                         (begin.strftime('%d-%b-%Y')))

            # today + 1 day
            end = today + datetime.timedelta(1)

            req   = { 'type' : 'imap', \
                      'req'  : syncer.get_imap_request_btw_2_dates(begin, end), \
                      'mode' : 'quick'}

            syncer.sync( req, \
                         compress_on_disk = args['compression'], \
                         db_cleaning = args['db-cleaning'], \
                         ownership_checking = args['ownership_control'], restart = args['restart'], \
                         emails_only = args['emails_only'], chats_only = args['chats_only'])

        elif args.get('type', '') == 'custom':

            #convert args to unicode
            args['request']['req'] = gmvault_utils.convert_to_unicode(
                args['request']['req'])
            args['request']['charset'] = 'utf-8'  #for the moment always utf-8
            args['request']['mode'] = 'custom'

            # pass an imap request. Assume that the user know what to do here
            LOG.critical("Perform custom synchronisation with %s request: %s.\n" \
                         % (args['request']['type'], args['request']['req']))

            syncer.sync(args['request'], compress_on_disk = args['compression'], db_cleaning = args['db-cleaning'], \
                        ownership_checking = args['ownership_control'], restart = args['restart'], \
                        emails_only = args['emails_only'], chats_only = args['chats_only'])
        else:
            raise ValueError(
                "Unknown synchronisation mode %s. Please use full (default), quick or custom."
            )

        #print error report
        LOG.critical(syncer.get_operation_report())
Exemplo n.º 29
0
def setup_default_conf():
    """
       set the environment GMVAULT_CONF_FILE which is necessary for Conf object
    """
    gmvault_utils.get_conf_defaults() # force instanciation of conf to load the defaults