def _get_authorization_tokens(cls, authorization_code):
"""Obtains OAuth access token and refresh token.
This uses the application portion of the "OAuth2 for Installed Applications"
flow at https://developers.google.com/accounts/docs/OAuth2InstalledApp#handlingtheresponse
Args:
client_id: Client ID obtained by registering your app.
client_secret: Client secret obtained by registering your app.
authorization_code: code generated by Google Accounts after user grants
permission.
Returns:
The decoded response from the Google Accounts server, as a dict. Expected
fields include 'access_token', 'expires_in', and 'refresh_token'.
"""
params = {}
params['client_id'] = gmvault_utils.get_conf_defaults().get("GoogleOauth2", "gmvault_client_id", "1070918343777-0eecradokiu8i77qfo8e3stbi0mkrtog.apps.googleusercontent.com")
params['client_secret'] = gmvault_utils.get_conf_defaults().get("GoogleOauth2", "gmvault_client_secret", "IVkl_pglv5cXzugpmnRNqtT7")
params['code'] = authorization_code
params['redirect_uri'] = gmvault_utils.get_conf_defaults().get("GoogleOauth2", "redirect_uri", 'urn:ietf:wg:oauth:2.0:oob')
params['grant_type'] = 'authorization_code'
account_base_url = gmvault_utils.get_conf_defaults().get("GoogleOauth2", "google_accounts_base_url", 'https://accounts.google.com')
request_url = '%s/%s' % (account_base_url, 'o/oauth2/token')
try:
response = urllib2.urlopen(request_url, urllib.urlencode(params)).read()
except Exception, err: #pylint: disable-msg=W0703
LOG.critical("Error: Problems when trying to connect to Google oauth2 endpoint: %s." % (request_url))
raise err
def _get_oauth2_acc_tok_from_ref_tok(cls, refresh_token):
"""Obtains a new token given a refresh token.
See https://developers.google.com/accounts/docs/OAuth2InstalledApp#refresh
Args:
client_id: Client ID obtained by registering your app.
client_secret: Client secret obtained by registering your app.
refresh_token: A previously-obtained refresh token.
Returns:
The decoded response from the Google Accounts server, as a dict. Expected
fields include 'access_token', 'expires_in', and 'refresh_token'.
"""
params = {}
params['client_id'] = gmvault_utils.get_conf_defaults().get("GoogleOauth2", "gmvault_client_id", "1070918343777-0eecradokiu8i77qfo8e3stbi0mkrtog.apps.googleusercontent.com")
params['client_secret'] = gmvault_utils.get_conf_defaults().get("GoogleOauth2", "gmvault_client_secret", "IVkl_pglv5cXzugpmnRNqtT7")
params['refresh_token'] = refresh_token
params['grant_type'] = 'refresh_token'
account_base_url = gmvault_utils.get_conf_defaults().get("GoogleOauth2", "google_accounts_base_url", 'https://accounts.google.com')
request_url = '%s/%s' % (account_base_url, 'o/oauth2/token')
try:
response = urllib2.urlopen(request_url, urllib.urlencode(params)).read()
except Exception, err: #pylint: disable-msg=W0703
LOG.critical("Error: Problems when trying to connect to Google oauth2 endpoint: %s.\n" % (request_url))
raise err
def generate_permission_url():
"""Generates the URL for authorizing access.
This uses the "OAuth2 for Installed Applications" flow described at
https://developers.google.com/accounts/docs/OAuth2InstalledApp
Args:
client_id: Client ID obtained by registering your app.
scope: scope for access token, e.g. 'https://mail.google.com'
Returns:
A URL that the user should visit in their browser.
"""
params = {}
params['client_id'] = gmvault_utils.get_conf_defaults().get(
"GoogleOauth2", "gmvault_client_id", "")
params['redirect_uri'] = gmvault_utils.get_conf_defaults().get(
"GoogleOauth2", "redirect_uri", '')
params['scope'] = gmvault_utils.get_conf_defaults().get(
"GoogleOauth2", "scope", 'https://mail.google.com/')
params['response_type'] = 'code'
account_base_url = gmvault_utils.get_conf_defaults().get(
"GoogleOauth2", "google_accounts_base_url",
'https://accounts.google.com')
return '%s/%s?%s' % (account_base_url, 'o/oauth2/auth',
gmvault_utils.format_url_params(params))
def _get_oauth2_acc_tok_from_ref_tok(cls, refresh_token):
"""Obtains a new token given a refresh token.
See https://developers.google.com/accounts/docs/OAuth2InstalledApp#refresh
Args:
client_id: Client ID obtained by registering your app.
client_secret: Client secret obtained by registering your app.
refresh_token: A previously-obtained refresh token.
Returns:
The decoded response from the Google Accounts server, as a dict. Expected
fields include 'access_token', 'expires_in', and 'refresh_token'.
"""
params = {}
params['client_id'] = gmvault_utils.get_conf_defaults().get(
"GoogleOauth2", "gmvault_client_id", "")
params['client_secret'] = gmvault_utils.get_conf_defaults().get(
"GoogleOauth2", "gmvault_client_secret", "")
params['refresh_token'] = refresh_token
params['grant_type'] = 'refresh_token'
account_base_url = gmvault_utils.get_conf_defaults().get(
"GoogleOauth2", "google_accounts_base_url",
'https://accounts.google.com')
request_url = '%s/%s' % (account_base_url, 'o/oauth2/token')
try:
response = urllib2.urlopen(request_url,
urllib.urlencode(params)).read()
except Exception, err: #pylint: disable-msg=W0703
LOG.critical(
"Error: Problems when trying to connect to Google oauth2 endpoint: %s.\nMaybe you changed your password? Rename/remove the auth token file that is named after the email address, and try again to refresh the token."
% (request_url))
raise err
def connect(self, go_to_current_folder = False):
"""
connect to the IMAP server
"""
# check if it is needed to disbale cert verification (leave that option at the last resort)
# if the cert verification doesn't work.
disable_cert_verification = gmvault_utils.get_conf_defaults().getboolean("GoogleOauth2", "disable_cacert_verification", default=False)
context = None
if disable_cert_verification:
LOG.critical("Beware disabling CA Cert verification for the IMAP connection to Gmail.")
context = imapclient.create_default_context()
# don't check if certificate hostname doesn't match target hostname
context.check_hostname = False
# don't check if the certificate is trusted by a certificate authority
context.verify_mode = ssl.CERT_NONE
else:
# create a custom ssl context to handle CA cert verification in all cases
# In some plateform the CA certs are not available so use the ones provided by Gmvault
# CA certs are coming from https://curl.haxx.se/ca/cacert.pem
context = imapclient.create_default_context(cafile= gmvault_utils.get_ca_certs_filepath())
# create imap object
self.server = mimap.MonkeyIMAPClient(self.host, port = self.port, use_uid = self.use_uid, need_ssl = self.ssl, ssl_context = context)
# connect with password or xoauth
if self.credential['type'] == 'passwd':
self.server.login(self.login, self.credential['value'])
elif self.credential['type'] == 'oauth2':
#connect with oauth2
if self.once_connected:
self.credential = credential_utils.CredentialHelper.get_oauth2_credential(self.login, renew_cred = False)
LOG.debug("credential['value'] = %s" % (self.credential['value']))
#try to login
self.server.oauth2_login(self.credential['value'])
else:
raise Exception("Unknown authentication method %s. Please use xoauth or passwd authentication " \
% (self.credential['type']))
#set connected to True to handle reconnection in case of failure
self.once_connected = True
# check gmailness
self.check_gmailness()
# find allmail chats and drafts folders
self.find_folder_names()
if go_to_current_folder and self.current_folder:
self.server.select_folder(self.current_folder, readonly = self.readonly_folder)
#enable compression
if gmvault_utils.get_conf_defaults().get_boolean('General', 'enable_imap_compression', True):
self.enable_compression()
LOG.debug("After Enabling compression.")
else:
LOG.debug("Do not enable imap compression.")
def __init__(self, host, port, login, credential, readonly_folder = True): #pylint:disable=R0913
'''
Constructor
'''
self.host = host
self.port = port
self.login = login
self.once_connected = False
self.credential = credential
self.ssl = True
self.use_uid = True
self.readonly_folder = readonly_folder
self.localized_folders = { 'ALLMAIL': { 'loc_dir' : None, 'friendly_name' : 'allmail'},
'CHATS' : { 'loc_dir' : None, 'friendly_name' : 'chats'},
'DRAFTS' : { 'loc_dir' : None, 'friendly_name' : 'drafts'} }
# memoize the current folder (All Mail or Chats) for reconnection management
self.current_folder = None
self.server = None
self.go_to_all_folder = True
self.total_nb_reconns = 0
# True when CHATS or other folder error msg has been already printed
self.printed_folder_error_msg = { 'ALLMAIL' : False, 'CHATS': False , 'DRAFTS':False }
#update GENERIC_GMAIL_CHATS. Should be done at the class level
self.GENERIC_GMAIL_CHATS.extend(gmvault_utils.get_conf_defaults().get_list('Localisation', 'chat_folder', []))
def __init__(self, host, port, login, credential, readonly_folder = True): #pylint:disable=R0913
'''
Constructor
'''
self.host = host
self.port = port
self.login = login
self.once_connected = False
self.credential = credential
self.ssl = True
self.use_uid = True
self.readonly_folder = readonly_folder
self.localized_folders = { 'ALLMAIL': { 'loc_dir' : None, 'friendly_name' : 'allmail'},
'CHATS' : { 'loc_dir' : None, 'friendly_name' : 'chats'},
'DRAFTS' :{ 'loc_dir' : None, 'friendly_name' : 'drafts'} }
# memoize the current folder (All Mail or Chats) for reconnection management
self.current_folder = None
self.server = None
self.go_to_all_folder = True
self.total_nb_reconns = 0
# True when CHATS or other folder error msg has been already printed
self.printed_folder_error_msg = { 'ALLMAIL' : False, 'CHATS': False , 'DRAFTS':False }
#update GENERIC_GMAIL_CHATS. Should be done at the class level
self.GENERIC_GMAIL_CHATS.extend(gmvault_utils.get_conf_defaults().get_list('Localisation', 'chat_folder', []))
def find_chats_folder(self):
"""
depending on your account the chats folder can be named
[GMAIL]/Chats or [GoogleMail]/Chats, [GMAIL]/tous les chats ...
Find and set the right one
Npte: Cannot use the flags as Chats is not a system label. Thanks Google
"""
#use xlist because of localized dir names
folders = self.server.xlist_folders()
LOG.debug("Folders = %s\n" % (folders))
the_dir = None
for (_, _, the_dir) in folders:
#look for GMAIL Chats
if the_dir in GIMAPFetcher.GENERIC_GMAIL_CHATS:
#it could be a localized Dir name
self.localized_folders['CHATS']['loc_dir'] = the_dir
return the_dir
#Error did not find Chats dir
if gmvault_utils.get_conf_defaults().getboolean(
"General", "errors_if_chat_not_visible", False):
raise Exception("Cannot find global 'Chats' folder ! Check whether 'Show in IMAP for 'Chats' "\
"is enabled in Gmail (Go to Settings->Labels->All Mail)")
return None
def find_folder_names(self):
"""
depending on your account the all mail folder can be named
[GMAIL]/ALL Mail or [GoogleMail]/All Mail.
Find and set the right one
"""
#use xlist because of localized dir names
folders = self.server.xlist_folders()
the_dir = None
for (flags, _, the_dir) in folders:
#non localised GMAIL_ALL
if GIMAPFetcher.GENERIC_GMAIL_ALL in flags:
#it could be a localized Dir name
self.localized_folders['ALLMAIL']['loc_dir'] = the_dir
elif the_dir in GIMAPFetcher.GENERIC_GMAIL_CHATS:
#it could be a localized Dir name
self.localized_folders['CHATS']['loc_dir'] = the_dir
elif GIMAPFetcher.GENERIC_DRAFTS in flags:
self.localized_folders['DRAFTS']['loc_dir'] = the_dir
if not self.localized_folders['ALLMAIL']['loc_dir']: # all mail error
raise Exception("Cannot find global 'All Mail' folder (maybe localized and translated into your language) ! "\
"Check whether 'Show in IMAP for 'All Mail' is enabled in Gmail (Go to Settings->Labels->All Mail)")
elif not self.localized_folders['CHATS']['loc_dir'] and \
gmvault_utils.get_conf_defaults().getboolean("General","errors_if_chat_not_visible", False):
raise Exception("Cannot find global 'Chats' folder ! Check whether 'Show in IMAP for 'Chats' "\
"is enabled in Gmail (Go to Settings->Labels->All Mail)")
elif not self.localized_folders['DRAFTS']['loc_dir']:
raise Exception("Cannot find global 'Drafts' folder.")
def find_folder_names(self):
"""
depending on your account the all mail folder can be named
[GMAIL]/ALL Mail or [GoogleMail]/All Mail.
Find and set the right one
"""
#use xlist because of localized dir names
folders = self.server.xlist_folders()
the_dir = None
for (flags, _, the_dir) in folders:
#non localised GMAIL_ALL
if GIMAPFetcher.GENERIC_GMAIL_ALL in flags:
#it could be a localized Dir name
self.localized_folders['ALLMAIL']['loc_dir'] = the_dir
elif the_dir in GIMAPFetcher.GENERIC_GMAIL_CHATS :
#it could be a localized Dir name
self.localized_folders['CHATS']['loc_dir'] = the_dir
elif GIMAPFetcher.GENERIC_DRAFTS in flags:
self.localized_folders['DRAFTS']['loc_dir'] = the_dir
if not self.localized_folders['ALLMAIL']['loc_dir']: # all mail error
raise Exception("Cannot find global 'All Mail' folder (maybe localized and translated into your language) ! "\
"Check whether 'Show in IMAP for 'All Mail' is enabled in Gmail (Go to Settings->Labels->All Mail)")
elif not self.localized_folders['CHATS']['loc_dir'] and \
gmvault_utils.get_conf_defaults().getboolean("General","errors_if_chat_not_visible", False):
raise Exception("Cannot find global 'Chats' folder ! Check whether 'Show in IMAP for 'Chats' "\
"is enabled in Gmail (Go to Settings->Labels->All Mail)")
elif not self.localized_folders['DRAFTS']['loc_dir']:
raise Exception("Cannot find global 'Drafts' folder.")
def find_chats_folder(self):
"""
depending on your account the chats folder can be named
[GMAIL]/Chats or [GoogleMail]/Chats, [GMAIL]/tous les chats ...
Find and set the right one
Npte: Cannot use the flags as Chats is not a system label. Thanks Google
"""
#use xlist because of localized dir names
folders = self.server.xlist_folders()
LOG.debug("Folders = %s\n" % (folders))
the_dir = None
for (_, _, the_dir) in folders:
#look for GMAIL Chats
if the_dir in GIMAPFetcher.GENERIC_GMAIL_CHATS :
#it could be a localized Dir name
self.localized_folders['CHATS']['loc_dir'] = the_dir
return the_dir
#Error did not find Chats dir
if gmvault_utils.get_conf_defaults().getboolean("General", "errors_if_chat_not_visible", False):
raise Exception("Cannot find global 'Chats' folder ! Check whether 'Show in IMAP for 'Chats' "\
"is enabled in Gmail (Go to Settings->Labels->All Mail)")
return None
def create_gmail_labels(self, labels, existing_folders):
"""
Create folders and subfolders on Gmail in order
to recreate the label hierarchy before to upload emails
Note that adding labels with +X-GM-LABELS create only nested labels
but not nested ones. This is why this trick must be used to
recreate the label hierarchy
labels: list of labels to create
"""
#1.5-beta moved that out of the loop to minimize the number of calls
#to that method. (Could go further and memoize it)
#get existing directories (or label parts)
# get in lower case because Gmail labels are case insensitive
listed_folders = set([ directory.lower() for (_, _, directory) in self.list_all_folders() ])
existing_folders = listed_folders.union(existing_folders)
reserved_labels_map = gmvault_utils.get_conf_defaults().get_dict("Restore", "reserved_labels_map", \
{ u'migrated' : u'gmv-migrated', u'\muted' : u'gmv-muted' })
LOG.debug("Labels to create: [%s]" % (labels))
for lab in labels:
#LOG.info("Reserved labels = %s\n" % (reserved_labels))
#LOG.info("lab.lower = %s\n" % (lab.lower()))
if lab.lower() in reserved_labels_map.keys(): #exclude creation of migrated label
n_lab = reserved_labels_map.get(lab.lower(), "gmv-default-label")
LOG.info("Warning ! label '%s' (lower or uppercase) is reserved by Gmail and cannot be used."\
"Use %s instead" % (lab, n_lab))
lab = n_lab
LOG.info("translated lab = %s\n" % (lab))
#split all labels
labs = self._get_dir_from_labels(lab)
for directory in labs:
low_directory = directory.lower() #get lower case directory but store original label
if (low_directory not in existing_folders) and (low_directory not in self.GMAIL_SPECIAL_DIRS_LOWER):
try:
if self.server.create_folder(directory) != 'Success':
raise Exception("Cannot create label %s: the directory %s cannot be created." % (lab, directory))
else:
LOG.debug("============== ####### Created Labels (%s)." % (directory))
except imaplib.IMAP4.error, error:
#log error in log file if it exists
LOG.debug(gmvault_utils.get_exception_traceback())
if str(error).startswith("create failed: '[ALREADYEXISTS] Duplicate folder"):
LOG.critical("Warning: label %s already exists on Gmail and Gmvault tried to create it."\
" Ignore this issue." % (directory) )
else:
raise error
#add created folder in folders
existing_folders.add(low_directory)
def create_gmail_labels(self, labels, existing_folders):
"""
Create folders and subfolders on Gmail in order
to recreate the label hierarchy before to upload emails
Note that adding labels with +X-GM-LABELS create only nested labels
but not nested ones. This is why this trick must be used to
recreate the label hierarchy
labels: list of labels to create
"""
#1.5-beta moved that out of the loop to minimize the number of calls
#to that method. (Could go further and memoize it)
#get existing directories (or label parts)
# get in lower case because Gmail labels are case insensitive
listed_folders = set([ directory.lower() for (_, _, directory) in self.list_all_folders() ])
existing_folders = listed_folders.union(existing_folders)
reserved_labels_map = gmvault_utils.get_conf_defaults().get_dict("Restore", "reserved_labels_map", \
{ u'migrated' : u'gmv-migrated', u'\muted' : u'gmv-muted' })
LOG.debug("Labels to create: [%s]" % (labels))
for lab in labels:
#LOG.info("Reserved labels = %s\n" % (reserved_labels))
#LOG.info("lab.lower = %s\n" % (lab.lower()))
if lab.lower() in reserved_labels_map.keys(): #exclude creation of migrated label
n_lab = reserved_labels_map.get(lab.lower(), "gmv-default-label")
LOG.info("Warning ! label '%s' (lower or uppercase) is reserved by Gmail and cannot be used."\
"Use %s instead" % (lab, n_lab))
lab = n_lab
LOG.info("translated lab = %s\n" % (lab))
#split all labels
labs = self._get_dir_from_labels(lab)
for directory in labs:
low_directory = directory.lower() #get lower case directory but store original label
if (low_directory not in existing_folders) and (low_directory not in self.GMAIL_SPECIAL_DIRS_LOWER):
try:
if self.server.create_folder(directory) != 'Success':
raise Exception("Cannot create label %s: the directory %s cannot be created." % (lab, directory))
else:
LOG.debug("============== ####### Created Labels (%s)." % (directory))
except imaplib.IMAP4.error, error:
#log error in log file if it exists
LOG.debug(gmvault_utils.get_exception_traceback())
if str(error).startswith("create failed: '[ALREADYEXISTS] Duplicate folder"):
LOG.critical("Warning: label %s already exists on Gmail and Gmvault tried to create it."\
" Ignore this issue." % (directory) )
else:
raise error
#add created folder in folders
existing_folders.add(low_directory)
def delete_emails(self, emails_info, msg_type):
"""
Delete all emails and metadata with ids
"""
if msg_type == 'email':
db_dir = self._db_dir
else:
db_dir = self._chats_dir
move_to_bin = gmvault_utils.get_conf_defaults().get_boolean(
"General", "keep_in_bin" , False)
if move_to_bin:
LOG.critical("Move emails to the bin:%s" % self._bin_dir)
for (a_id, date_dir) in emails_info:
the_dir = '%s/%s' % (db_dir, date_dir)
data_p = self.DATA_FNAME % (the_dir, a_id)
comp_data_p = '%s.gz' % data_p
cryp_comp_data_p = '%s.crypt.gz' % data_p
metadata_p = self.METADATA_FNAME % (the_dir, a_id)
if move_to_bin:
#move files to the bin
gmvault_utils.makedirs(self._bin_dir)
# create bin filenames
bin_p = self.DATA_FNAME % (self._bin_dir, a_id)
metadata_bin_p = self.METADATA_FNAME % (self._bin_dir, a_id)
if os.path.exists(data_p):
os.rename(data_p, bin_p)
elif os.path.exists(comp_data_p):
os.rename(comp_data_p, '%s.gz' % bin_p)
elif os.path.exists(cryp_comp_data_p):
os.rename(cryp_comp_data_p, '%s.crypt.gz' % bin_p)
if os.path.exists(metadata_p):
os.rename(metadata_p, metadata_bin_p)
else:
#delete files if they exists
if os.path.exists(data_p):
os.remove(data_p)
elif os.path.exists(comp_data_p):
os.remove(comp_data_p)
elif os.path.exists(cryp_comp_data_p):
os.remove(cryp_comp_data_p)
if os.path.exists(metadata_p):
os.remove(metadata_p)
gmsql.GMSQL.delete_email(a_id)
def delete_emails(self, emails_info, msg_type):
"""
Delete all emails and metadata with ids
"""
if msg_type == 'email':
db_dir = self._db_dir
else:
db_dir = self._chats_dir
move_to_bin = gmvault_utils.get_conf_defaults().get_boolean(
"General", "keep_in_bin" , False)
if move_to_bin:
LOG.critical("Move emails to the bin:%s" % self._bin_dir)
for (a_id, date_dir) in emails_info:
the_dir = '%s/%s' % (db_dir, date_dir)
data_p = self.DATA_FNAME % (the_dir, a_id)
comp_data_p = '%s.gz' % data_p
cryp_comp_data_p = '%s.crypt.gz' % data_p
metadata_p = self.METADATA_FNAME % (the_dir, a_id)
if move_to_bin:
#move files to the bin
gmvault_utils.makedirs(self._bin_dir)
# create bin filenames
bin_p = self.DATA_FNAME % (self._bin_dir, a_id)
metadata_bin_p = self.METADATA_FNAME % (self._bin_dir, a_id)
if os.path.exists(data_p):
os.rename(data_p, bin_p)
elif os.path.exists(comp_data_p):
os.rename(comp_data_p, '%s.gz' % bin_p)
elif os.path.exists(cryp_comp_data_p):
os.rename(cryp_comp_data_p, '%s.crypt.gz' % bin_p)
if os.path.exists(metadata_p):
os.rename(metadata_p, metadata_bin_p)
else:
#delete files if they exists
if os.path.exists(data_p):
os.remove(data_p)
elif os.path.exists(comp_data_p):
os.remove(comp_data_p)
elif os.path.exists(cryp_comp_data_p):
os.remove(cryp_comp_data_p)
if os.path.exists(metadata_p):
os.remove(metadata_p)
def generate_permission_url():
"""Generates the URL for authorizing access.
This uses the "OAuth2 for Installed Applications" flow described at
https://developers.google.com/accounts/docs/OAuth2InstalledApp
Args:
client_id: Client ID obtained by registering your app.
scope: scope for access token, e.g. 'https://mail.google.com'
Returns:
A URL that the user should visit in their browser.
"""
params = {}
params['client_id'] = gmvault_utils.get_conf_defaults().get("GoogleOauth2", "gmvault_client_id", "1070918343777-0eecradokiu8i77qfo8e3stbi0mkrtog.apps.googleusercontent.com")
params['redirect_uri'] = gmvault_utils.get_conf_defaults().get("GoogleOauth2", "redirect_uri", 'urn:ietf:wg:oauth:2.0:oob')
params['scope'] = gmvault_utils.get_conf_defaults().get("GoogleOauth2","scope",'https://mail.google.com/')
params['response_type'] = 'code'
account_base_url = gmvault_utils.get_conf_defaults().get("GoogleOauth2", "google_accounts_base_url", 'https://accounts.google.com')
return '%s/%s?%s' % (account_base_url, 'o/oauth2/auth', gmvault_utils.format_url_params(params))
def bootstrap_run():
""" temporary bootstrap """
init_logging()
#force argv[0] to gmvault
sys.argv[0] = "gmvault"
LOG.critical("")
gmvlt = GMVaultLauncher()
args = gmvlt.parse_args()
#activate debug if enabled
if args['debug']:
LOG.critical("Activate debugging information.")
activate_debug_mode()
# force instanciation of conf to load the defaults
gmvault_utils.get_conf_defaults()
gmvlt.run(args)
def bootstrap_run():
""" temporary bootstrap """
init_logging()
#force argv[0] to gmvault
sys.argv[0] = "gmvault"
LOG.critical("")
gmvlt = GMVaultLauncher()
args = gmvlt.parse_args()
#activate debug if enabled
if args['debug']:
LOG.critical("Activate debugging information.")
activate_debug_mode()
# force instanciation of conf to load the defaults
gmvault_utils.get_conf_defaults()
gmvlt.run(args)
def check_to_disable_certificate_verification():
"""
Check from the configuration if certificate verification needs to be disabled.
Have to use monkey patching strategy while version older than 2.7.9 are still supported.
"""
#solve ssl context issue for some users
if hasattr('ssl', 'SSLContext') == False:
disable_cert_verification = gmvault_utils.get_conf_defaults().getboolean("GoogleOauth2", "disable_cacert_verification", default=False, fail_if_missing = True)
if disable_cert_verification == True:
#loc_context = ssl.SSLContext(ssl.PROTOCOL_TLSv1)
#for the moment monkey patch later on will use the line above
LOG.critical("Beware as asked Gmvault is going to disable the SSL Certificate Verification.")
ssl._create_default_https_context = ssl._create_unverified_context
else:
LOG.critical("Use Default certificate context.")
else:
LOG.debug("Ignore check_to_disable_certificate_verification. No SSLContext Object in ssl, probably using a python version < to 2.7.9")
def connect(self, go_to_current_folder=False):
"""
connect to the IMAP server
"""
# create imap object
self.server = mimap.MonkeyIMAPClient(self.host,
port=self.port,
use_uid=self.use_uid,
need_ssl=self.ssl)
# connect with password or xoauth
if self.credential['type'] == 'passwd':
self.server.login(self.login, self.credential['value'])
elif self.credential['type'] == 'oauth2':
#connect with oauth2
if self.once_connected:
self.credential = credential_utils.CredentialHelper.get_oauth2_credential(
self.login, renew_cred=False)
LOG.debug("credential['value'] = %s" % (self.credential['value']))
#try to login
self.server.oauth2_login(self.credential['value'])
else:
raise Exception("Unknown authentication method %s. Please use xoauth or passwd authentication " \
% (self.credential['type']))
#set connected to True to handle reconnection in case of failure
self.once_connected = True
# check gmailness
self.check_gmailness()
# find allmail chats and drafts folders
self.find_folder_names()
if go_to_current_folder and self.current_folder:
self.server.select_folder(self.current_folder,
readonly=self.readonly_folder)
#enable compression
if gmvault_utils.get_conf_defaults().get_boolean(
'General', 'enable_imap_compression', True):
self.enable_compression()
LOG.debug("After Enabling compression.")
else:
LOG.debug("Do not enable imap compression.")
def _restore(cls, args, credential):
"""
Execute All restore operations
"""
LOG.critical("Connect to Gmail server.\n")
# Create a gmvault releaving read_only_access
restorer = gmvault.GMVaulter(args['db-dir'], args['host'], args['port'], \
args['email'], credential, read_only_access = False)
#full sync is the first one
if args.get('type', '') == 'full':
#call restore
labels = [args['apply_label']] if args['apply_label'] else []
restorer.restore(extra_labels = labels, restart = args['restart'], \
emails_only = args['emails_only'], chats_only = args['chats_only'])
elif args.get('type', '') == 'quick':
#take the last two to 3 months depending on the current date
# today - 2 months
today = datetime.date.today()
begin = today - datetime.timedelta(
gmvault_utils.get_conf_defaults().getint(
"Restore", "quick_days", 8))
starting_dir = gmvault_utils.get_ym_from_datetime(begin)
#call restore
labels = [args['apply_label']] if args['apply_label'] else []
restorer.restore(pivot_dir = starting_dir, extra_labels = labels, restart = args['restart'], \
emails_only = args['emails_only'], chats_only = args['chats_only'])
else:
raise ValueError(
"Unknown synchronisation mode %s. Please use full (default), quick."
)
#print error report
LOG.critical(restorer.get_operation_report())
def __init__(self, a_storage_dir, encrypt_data=False):
"""
Store on disks
args:
a_storage_dir: Storage directory
a_use_encryption: Encryption key. If there then encrypt
"""
self._top_dir = a_storage_dir
self._db_dir = '%s/%s' % (a_storage_dir, GmailStorer.DB_AREA)
self._quarantine_dir = '%s/%s' % (a_storage_dir, GmailStorer.QUARANTINE_AREA)
self._info_dir = '%s/%s' % (a_storage_dir, GmailStorer.INFO_AREA)
self._chats_dir = '%s/%s' % (self._db_dir, GmailStorer.CHATS_AREA)
self._bin_dir = '%s/%s' % (a_storage_dir, GmailStorer.BIN_AREA)
gmsql.GMSQL.connect('%s/meta.db' % (a_storage_dir))
self._sub_chats_dir = None
self._sub_chats_inc = -1
self._sub_chats_nb = -1
self._limit_per_chat_dir = gmvault_utils.get_conf_defaults().getint(
"General", "limit_per_chat_dir", 1500)
#make dirs
if not os.path.exists(self._db_dir):
LOG.critical("No Storage DB in %s. Create it.\n" % a_storage_dir)
gmvault_utils.makedirs(self._db_dir)
gmvault_utils.makedirs(self._chats_dir)
gmvault_utils.makedirs(self._quarantine_dir)
gmvault_utils.makedirs(self._info_dir)
self.fsystem_info_cache = {}
self._encrypt_data = encrypt_data
self._encryption_key = None
self._cipher = None
#add version if it is needed to migrate gmvault-db in the future
self._create_gmvault_db_version()
def __init__(self, a_storage_dir, encrypt_data=False):
"""
Store on disks
args:
a_storage_dir: Storage directory
a_use_encryption: Encryption key. If there then encrypt
"""
self._top_dir = a_storage_dir
self._db_dir = '%s/%s' % (a_storage_dir, GmailStorer.DB_AREA)
self._quarantine_dir = '%s/%s' % (a_storage_dir, GmailStorer.QUARANTINE_AREA)
self._info_dir = '%s/%s' % (a_storage_dir, GmailStorer.INFO_AREA)
self._chats_dir = '%s/%s' % (self._db_dir, GmailStorer.CHATS_AREA)
self._bin_dir = '%s/%s' % (a_storage_dir, GmailStorer.BIN_AREA)
self._sub_chats_dir = None
self._sub_chats_inc = -1
self._sub_chats_nb = -1
self._limit_per_chat_dir = gmvault_utils.get_conf_defaults().getint(
"General", "limit_per_chat_dir", 1500)
#make dirs
if not os.path.exists(self._db_dir):
LOG.critical("No Storage DB in %s. Create it.\n" % a_storage_dir)
gmvault_utils.makedirs(self._db_dir)
gmvault_utils.makedirs(self._chats_dir)
gmvault_utils.makedirs(self._quarantine_dir)
gmvault_utils.makedirs(self._info_dir)
self.fsystem_info_cache = {}
self._encrypt_data = encrypt_data
self._encryption_key = None
self._cipher = None
#add version if it is needed to migrate gmvault-db in the future
self._create_gmvault_db_version()
def connect(self, go_to_current_folder = False):
"""
connect to the IMAP server
"""
# create imap object
self.server = mimap.MonkeyIMAPClient(self.host, port = self.port, use_uid= self.use_uid, need_ssl= self.ssl)
# connect with password or xoauth
if self.credential['type'] == 'passwd':
self.server.login(self.login, self.credential['value'])
elif self.credential['type'] == 'oauth2':
#connect with oauth2
if self.once_connected:
self.credential = credential_utils.CredentialHelper.get_oauth2_credential(self.login, renew_cred = False)
LOG.debug("credential['value'] = %s" % (self.credential['value']))
#try to login
self.server.oauth2_login(self.credential['value'])
else:
raise Exception("Unknown authentication method %s. Please use xoauth or passwd authentication " \
% (self.credential['type']))
#set connected to True to handle reconnection in case of failure
self.once_connected = True
# check gmailness
self.check_gmailness()
# find allmail chats and drafts folders
self.find_folder_names()
if go_to_current_folder and self.current_folder:
self.server.select_folder(self.current_folder, readonly = self.readonly_folder)
#enable compression
if gmvault_utils.get_conf_defaults().get_boolean('General', 'enable_imap_compression', True):
self.enable_compression()
LOG.debug("After Enabling compression.")
else:
LOG.debug("Do not enable imap compression.")
def _restore(cls, args, credential):
"""
Execute All restore operations
"""
LOG.critical("Connect to Gmail server.\n")
# Create a gmvault releaving read_only_access
restorer = gmvault.GMVaulter(args['db-dir'], args['host'], args['port'], \
args['email'], credential, read_only_access = False)
#full sync is the first one
if args.get('type', '') == 'full':
#call restore
labels = [args['apply_label']] if args['apply_label'] else []
restorer.restore(extra_labels = labels, restart = args['restart'], \
emails_only = args['emails_only'], chats_only = args['chats_only'])
elif args.get('type', '') == 'quick':
#take the last two to 3 months depending on the current date
# today - 2 months
today = datetime.date.today()
begin = today - datetime.timedelta(gmvault_utils.get_conf_defaults().getint("Restore", "quick_days", 8))
starting_dir = gmvault_utils.get_ym_from_datetime(begin)
#call restore
labels = [args['apply_label']] if args['apply_label'] else []
restorer.restore(pivot_dir = starting_dir, extra_labels = labels, restart = args['restart'], \
emails_only = args['emails_only'], chats_only = args['chats_only'])
else:
raise ValueError("Unknown synchronisation mode %s. Please use full (default), quick.")
#print error report
LOG.critical(restorer.get_operation_report())
def _sync(cls, args, credential):
"""
Execute All synchronisation operations
"""
LOG.critical("Connect to Gmail server.\n")
# handle credential in all levels
syncer = gmvault.GMVaulter(args['db-dir'], args['host'], args['port'], \
args['email'], credential, read_only_access = True, use_encryption = args['encrypt'])
#full sync is the first one
if args.get('type', '') == 'full':
#choose full sync. Ignore the request
syncer.sync({ 'mode': 'full', 'type': 'imap', 'req': 'ALL' } , compress_on_disk = args['compression'], \
db_cleaning = args['db-cleaning'], ownership_checking = args['ownership_control'],\
restart = args['restart'], emails_only = args['emails_only'], chats_only = args['chats_only'])
elif args.get('type', '') == 'auto':
#choose auto sync. imap request = ALL and restart = True
syncer.sync({ 'mode': 'auto', 'type': 'imap', 'req': 'ALL' } , compress_on_disk = args['compression'], \
db_cleaning = args['db-cleaning'], ownership_checking = args['ownership_control'],\
restart = True, emails_only = args['emails_only'], chats_only = args['chats_only'])
elif args.get('type', '') == 'quick':
#sync only the last x days (taken in defaults) in order to be quick
#(cleaning is import here because recent days might move again
# today - 2 months
today = datetime.date.today()
begin = today - datetime.timedelta(gmvault_utils.get_conf_defaults().getint("Sync", "quick_days", 8))
LOG.critical("Quick sync mode. Check for new emails since %s." % (begin.strftime('%d-%b-%Y')))
# today + 1 day
end = today + datetime.timedelta(1)
req = { 'type' : 'imap', \
'req' : syncer.get_imap_request_btw_2_dates(begin, end), \
'mode' : 'quick'}
syncer.sync( req, \
compress_on_disk = args['compression'], \
db_cleaning = args['db-cleaning'], \
ownership_checking = args['ownership_control'], restart = args['restart'], \
emails_only = args['emails_only'], chats_only = args['chats_only'])
elif args.get('type', '') == 'custom':
#convert args to unicode
args['request']['req'] = gmvault_utils.convert_to_unicode(args['request']['req'])
args['request']['charset'] = 'utf-8' #for the moment always utf-8
args['request']['mode'] = 'custom'
# pass an imap request. Assume that the user know what to do here
LOG.critical("Perform custom synchronisation with %s request: %s.\n" \
% (args['request']['type'], args['request']['req']))
syncer.sync(args['request'], compress_on_disk = args['compression'], db_cleaning = args['db-cleaning'], \
ownership_checking = args['ownership_control'], restart = args['restart'], \
emails_only = args['emails_only'], chats_only = args['chats_only'])
else:
raise ValueError("Unknown synchronisation mode %s. Please use full (default), quick or custom.")
#print error report
LOG.critical(syncer.get_operation_report())
def setup_default_conf():
"""
set the environment GMVAULT_CONF_FILE which is necessary for Conf object
"""
gmvault_utils.get_conf_defaults(
) # force instanciation of conf to load the defaults
def _sync(cls, args, credential):
"""
Execute All synchronisation operations
"""
LOG.critical("Connect to Gmail server.\n")
# handle credential in all levels
syncer = gmvault.GMVaulter(args['db-dir'], args['host'], args['port'], \
args['email'], credential, read_only_access = True, use_encryption = args['encrypt'])
#full sync is the first one
if args.get('type', '') == 'full':
#choose full sync. Ignore the request
syncer.sync({ 'mode': 'full', 'type': 'imap', 'req': 'ALL' } , compress_on_disk = args['compression'], \
db_cleaning = args['db-cleaning'], ownership_checking = args['ownership_control'],\
restart = args['restart'], emails_only = args['emails_only'], chats_only = args['chats_only'])
elif args.get('type', '') == 'auto':
#choose auto sync. imap request = ALL and restart = True
syncer.sync({ 'mode': 'auto', 'type': 'imap', 'req': 'ALL' } , compress_on_disk = args['compression'], \
db_cleaning = args['db-cleaning'], ownership_checking = args['ownership_control'],\
restart = True, emails_only = args['emails_only'], chats_only = args['chats_only'])
elif args.get('type', '') == 'quick':
#sync only the last x days (taken in defaults) in order to be quick
#(cleaning is import here because recent days might move again
# today - 2 months
today = datetime.date.today()
begin = today - datetime.timedelta(
gmvault_utils.get_conf_defaults().getint(
"Sync", "quick_days", 8))
LOG.critical("Quick sync mode. Check for new emails since %s." %
(begin.strftime('%d-%b-%Y')))
# today + 1 day
end = today + datetime.timedelta(1)
req = { 'type' : 'imap', \
'req' : syncer.get_imap_request_btw_2_dates(begin, end), \
'mode' : 'quick'}
syncer.sync( req, \
compress_on_disk = args['compression'], \
db_cleaning = args['db-cleaning'], \
ownership_checking = args['ownership_control'], restart = args['restart'], \
emails_only = args['emails_only'], chats_only = args['chats_only'])
elif args.get('type', '') == 'custom':
#convert args to unicode
args['request']['req'] = gmvault_utils.convert_to_unicode(
args['request']['req'])
args['request']['charset'] = 'utf-8' #for the moment always utf-8
args['request']['mode'] = 'custom'
# pass an imap request. Assume that the user know what to do here
LOG.critical("Perform custom synchronisation with %s request: %s.\n" \
% (args['request']['type'], args['request']['req']))
syncer.sync(args['request'], compress_on_disk = args['compression'], db_cleaning = args['db-cleaning'], \
ownership_checking = args['ownership_control'], restart = args['restart'], \
emails_only = args['emails_only'], chats_only = args['chats_only'])
else:
raise ValueError(
"Unknown synchronisation mode %s. Please use full (default), quick or custom."
)
#print error report
LOG.critical(syncer.get_operation_report())
def setup_default_conf():
"""
set the environment GMVAULT_CONF_FILE which is necessary for Conf object
"""
gmvault_utils.get_conf_defaults() # force instanciation of conf to load the defaults