def GetAuthorizedHttp(flags):
"""Helper to create an http interface object and authorize it.
Made simple by oauth2client library. Because http object are NOT
thread-safe, create a new one every time (assumes being created by multiple
threads).
Args:
flags: argparse parsed flags object.
Returns:
Authorized httplib2 http interface object.
"""
credentials = GetCredentials(flags, _SCOPES)
try:
cse_tool_version = _TOOL_USER_AGENT % FILE_MANAGER.ReadAppVersion()
log_utils.LogDebug('user-agent: %s' % cse_tool_version)
http = httplib2.Http(timeout=_EXTENDED_SOCKET_TIMEOUT_S)
set_user_agent(http, cse_tool_version)
http = credentials.authorize(http)
except AccessTokenRefreshError:
log_utils.LogError('The credentials have been revoked or expired, '
'please re-run the application to re-authorize')
sys.exit(1)
return http
def GetAuthorizedHttp(flags):
"""Helper to create an http interface object and authorize it.
Made simple by oauth2client library. Because http object are NOT
thread-safe, create a new one every time (assumes being created by multiple
threads).
Args:
flags: argparse parsed flags object.
Returns:
Authorized httplib2 http interface object.
"""
credentials = GetCredentials(flags, _SCOPES)
try:
cse_tool_version = _TOOL_USER_AGENT % FILE_MANAGER.ReadAppVersion()
log_utils.LogDebug('user-agent: %s' % cse_tool_version)
http = httplib2.Http(timeout=_EXTENDED_SOCKET_TIMEOUT_S)
set_user_agent(http, cse_tool_version)
http = credentials.authorize(http)
except AccessTokenRefreshError:
log_utils.LogError('The credentials have been revoked or expired, '
'please re-run the application to re-authorize')
sys.exit(1)
return http
def test_set_user_agent_nested(self):
http = HttpMockSequence([({"status": "200"}, "echo_request_headers")])
http = set_user_agent(http, "my_app/5.5")
http = set_user_agent(http, "my_library/0.1")
resp, content = http.request("http://example.com")
self.assertEqual("my_app/5.5 my_library/0.1", content["user-agent"])
def test_set_user_agent_nested(self):
http = HttpMockSequence([
({'status': '200'}, 'echo_request_headers'),
])
http = set_user_agent(http, "my_app/5.5")
http = set_user_agent(http, "my_library/0.1")
resp, content = http.request("http://example.com")
self.assertEqual(content['user-agent'], 'my_app/5.5 my_library/0.1')
def __init__(self, context, db_driver=None):
self.check_gcs_options()
backup_bucket = CONF.backup_gcs_bucket
backup_credential = CONF.backup_gcs_credential_file
self.gcs_project_id = CONF.backup_gcs_project_id
chunk_size_bytes = CONF.backup_gcs_object_size
sha_block_size_bytes = CONF.backup_gcs_block_size
enable_progress_timer = CONF.backup_gcs_enable_progress_timer
super(GoogleBackupDriver, self).__init__(context, chunk_size_bytes,
sha_block_size_bytes,
backup_bucket,
enable_progress_timer,
db_driver)
credentials = client.GoogleCredentials.from_stream(backup_credential)
self.reader_chunk_size = CONF.backup_gcs_reader_chunk_size
self.writer_chunk_size = CONF.backup_gcs_writer_chunk_size
self.bucket_location = CONF.backup_gcs_bucket_location
self.storage_class = CONF.backup_gcs_storage_class
self.num_retries = CONF.backup_gcs_num_retries
http_user_agent = http.set_user_agent(httplib2.Http(),
CONF.backup_gcs_user_agent)
self.conn = discovery.build('storage',
'v1',
http=http_user_agent,
credentials=credentials)
self.resumable = self.writer_chunk_size != -1
def test_set_user_agent(self):
http = HttpMockSequence([
({'status': '200'}, 'echo_request_headers'),
])
http = set_user_agent(http, "my_app/5.5")
resp, content = http.request("http://example.com")
content = simplejson.loads(content.decode())
self.assertEqual('my_app/5.5', content['user-agent'])
def test_set_user_agent(self):
http = HttpMockSequence([
({
'status': '200'
}, 'echo_request_headers'),
])
http = set_user_agent(http, "my_app/5.5")
resp, content = http.request("http://example.com")
content = simplejson.loads(content.decode())
self.assertEqual('my_app/5.5', content['user-agent'])
def GetAuthorizedHttp(flags):
"""Helper to create an http interface object and authorize it.
Made simple by oauth2client library. Because http object are NOT
thread-safe, create a new one every time (assumes being created by multiple
threads).
Args:
flags: argparse parsed flags object.
Returns:
Authorized httplib2 http interface object.
"""
credentials = GetCredentials(flags)
try:
http = httplib2.Http()
set_user_agent(http, _TOOL_USER_AGENT)
http = credentials.authorize(http)
except AccessTokenRefreshError:
print ('The credentials have been revoked or expired, '
'please re-run the application to re-authorize')
sys.exit(1)
return http
def GetAuthorizedHttp(flags):
"""Helper to create an http interface object and authorize it.
Made simple by oauth2client library. Because http object are NOT
thread-safe, create a new one every time (assumes being created by multiple
threads).
Args:
flags: argparse parsed flags object.
Returns:
Authorized httplib2 http interface object.
"""
credentials = GetCredentials(flags)
try:
http = httplib2.Http()
set_user_agent(http, _TOOL_USER_AGENT)
http = credentials.authorize(http)
except AccessTokenRefreshError:
print(
'The credentials have been revoked or expired, '
'please re-run the application to re-authorize')
sys.exit(1)
return http
def main():
"""Import multiple users' mbox files to Gmail.
"""
httplib2.debuglevel = args.httplib2debuglevel
# Use args.logging_level if defined.
try:
logging_level = args.logging_level
except AttributeError:
logging_level = 'INFO'
# Default logging to standard output
logging.basicConfig(
level=logging_level,
format='%(asctime)s %(levelname)s %(funcName)s@%(filename)s %(message)s',
datefmt='%H:%M:%S')
# More detailed logging to file
file_handler = logging.handlers.RotatingFileHandler(args.log,
maxBytes=1024 * 1024 * 32,
backupCount=8)
file_formatter = logging.Formatter(
'%(asctime)s %(process)d %(levelname)s %(funcName)s '
'(%(filename)s:%(lineno)d) %(message)s')
file_formatter.datefmt = '%Y-%m-%dT%H:%M:%S (%z)'
file_handler.setFormatter(file_formatter)
logging.getLogger().addHandler(file_handler)
logging.info('*** Starting %s %s ***', APPLICATION_NAME, APPLICATION_VERSION)
logging.info('Arguments:')
for arg, value in sorted(vars(args).items()):
logging.info('\t%s: %r', arg, value)
number_of_labels_imported_without_error = 0
number_of_labels_imported_with_some_errors = 0
number_of_labels_failed = 0
number_of_messages_imported_without_error = 0
number_of_messages_failed = 0
number_of_users_imported_without_error = 0
number_of_users_imported_with_some_errors = 0
number_of_users_failed = 0
for username in next(os.walk(args.dir))[1]:
try:
logging.info('Processing user %s', username)
try:
credentials = get_credentials(username)
http = credentials.authorize(set_user_agent(
httplib2.Http(),
'%s-%s' % (APPLICATION_NAME, APPLICATION_VERSION)))
service = discovery.build('gmail', 'v1', http=http)
except Exception:
logging.error("Can't get access token for user %s", username)
raise
try:
results = service.users().labels().list(
userId=username,
fields='labels(id,name)').execute(num_retries=args.num_retries)
labels = results.get('labels', [])
except Exception:
logging.error("Can't get labels for user %s", username)
raise
try:
result = process_mbox_files(username, service, labels)
except Exception:
logging.error("Can't process mbox files for user %s", username)
raise
if result[2] == 0 and result[4] == 0:
number_of_users_imported_without_error += 1
elif result[0] > 0 or result[3] > 0:
number_of_users_imported_with_some_errors += 1
else:
number_of_users_failed += 1
number_of_labels_imported_without_error += result[0]
number_of_labels_imported_with_some_errors += result[1]
number_of_labels_failed += result[2]
number_of_messages_imported_without_error += result[3]
number_of_messages_failed += result[4]
logging.info('Done importing user %s. Labels: %d succeeded, %d with some '
'errors, %d failed. Messages: %d succeeded, %d failed.',
username,
result[0],
result[1],
result[2],
result[3],
result[4])
except Exception:
number_of_users_failed += 1
logging.exception("Can't process user %s", username)
logging.info("*** Done importing all users from directory '%s'", args.dir)
logging.info('*** Import summary:')
logging.info(' %d users imported with no failures',
number_of_users_imported_without_error)
logging.info(' %d users imported with some failures',
number_of_users_imported_with_some_errors)
logging.info(' %d users failed',
number_of_users_failed)
logging.info(' %d labels (mbox files) imported with no failures',
number_of_labels_imported_without_error)
logging.info(' %d labels (mbox files) imported with some failures',
number_of_labels_imported_with_some_errors)
logging.info(' %d labels (mbox files) failed',
number_of_labels_failed)
logging.info(' %d messages imported successfully',
number_of_messages_imported_without_error)
logging.info(' %d messages failed\n',
number_of_messages_failed)
if (number_of_messages_failed + number_of_labels_failed +
number_of_users_failed > 0):
logging.info('*** Check log file %s for detailed errors.', args.log)
logging.info('Finished.\n\n')
def main():
"""Import multiple users' mbox files to Gmail.
"""
httplib2.debuglevel = args.httplib2debuglevel
# Use args.logging_level if defined.
try:
logging_level = args.logging_level
except AttributeError:
logging_level = 'INFO'
# Default logging to standard output
logging.basicConfig(
level=logging_level,
format='%(asctime)s %(levelname)s %(funcName)s@%(filename)s %(message)s',
datefmt='%H:%M:%S')
# More detailed logging to file
file_handler = logging.handlers.RotatingFileHandler(args.log,
maxBytes=1024 * 1024 * 32,
backupCount=8)
file_formatter = logging.Formatter(
'%(asctime)s %(process)d %(levelname)s %(funcName)s '
'(%(filename)s:%(lineno)d) %(message)s')
file_formatter.datefmt = '%Y-%m-%dT%H:%M:%S (%z)'
file_handler.setFormatter(file_formatter)
logging.getLogger().addHandler(file_handler)
logging.info('*** Starting %s %s on Python %s ***',
APPLICATION_NAME,
APPLICATION_VERSION,
sys.version)
logging.info('Arguments:')
for arg, value in sorted(vars(args).items()):
logging.info('\t%s: %r', arg, value)
number_of_labels_imported_without_error = 0
number_of_labels_imported_with_some_errors = 0
number_of_labels_failed = 0
number_of_messages_imported_without_error = 0
number_of_messages_failed = 0
number_of_users_imported_without_error = 0
number_of_users_imported_with_some_errors = 0
number_of_users_failed = 0
for username in next(os.walk(args.dir))[1]:
try:
logging.info('Processing user %s', username)
try:
credentials = get_credentials(username)
http = credentials.authorize(set_user_agent(
httplib2.Http(),
'%s-%s' % (APPLICATION_NAME, APPLICATION_VERSION)))
service = discovery.build('gmail', 'v1', http=http)
except Exception:
logging.error("Can't get access token for user %s", username)
raise
try:
results = service.users().labels().list(
userId=username,
fields='labels(id,name)').execute(num_retries=args.num_retries)
labels = results.get('labels', [])
except Exception:
logging.error("Can't get labels for user %s", username)
raise
try:
result = process_mbox_files(username, service, labels)
except Exception:
logging.error("Can't process mbox files for user %s", username)
raise
if result[2] == 0 and result[4] == 0:
number_of_users_imported_without_error += 1
elif result[0] > 0 or result[3] > 0:
number_of_users_imported_with_some_errors += 1
else:
number_of_users_failed += 1
number_of_labels_imported_without_error += result[0]
number_of_labels_imported_with_some_errors += result[1]
number_of_labels_failed += result[2]
number_of_messages_imported_without_error += result[3]
number_of_messages_failed += result[4]
logging.info('Done importing user %s. Labels: %d succeeded, %d with some '
'errors, %d failed. Messages: %d succeeded, %d failed.',
username,
result[0],
result[1],
result[2],
result[3],
result[4])
except Exception:
number_of_users_failed += 1
logging.exception("Can't process user %s", username)
logging.info("*** Done importing all users from directory '%s'", args.dir)
logging.info('*** Import summary:')
logging.info(' %d users imported with no failures',
number_of_users_imported_without_error)
logging.info(' %d users imported with some failures',
number_of_users_imported_with_some_errors)
logging.info(' %d users failed',
number_of_users_failed)
logging.info(' %d labels (mbox files) imported with no failures',
number_of_labels_imported_without_error)
logging.info(' %d labels (mbox files) imported with some failures',
number_of_labels_imported_with_some_errors)
logging.info(' %d labels (mbox files) failed',
number_of_labels_failed)
logging.info(' %d messages imported successfully',
number_of_messages_imported_without_error)
logging.info(' %d messages failed\n',
number_of_messages_failed)
if (number_of_messages_failed + number_of_labels_failed +
number_of_users_failed > 0):
logging.info('*** Check log file %s for detailed errors.', args.log)
logging.info('Finished.\n\n')
def Http():
"""Returns an instance of `httplib2.Http` with User-agent set."""
http = httplib2.Http(memcache)
return apiclient_http.set_user_agent(http, USER_AGENT)
def main():
# Set the default encoding for logger
reload(sys)
sys.setdefaultencoding('utf8')
# Setup logging
httplib2.debuglevel = 0
# Increase log leven of file_cache error on oauth2client >= 4.0.0
# https://github.com/google/google-api-python-client/issues/299
logging.getLogger('googleapiclient.discovery_cache').setLevel(logging.ERROR)
logging.getLogger('googleapiclient').setLevel(logging.ERROR)
try:
logging_level = args.logging_level
except AttributeError:
logging_level = 'INFO'
logging.basicConfig(
level=logging_level,
format='%(asctime)s %(levelname)s %(message)s',
datefmt='%H:%M:%S')
# More detailed logging to file
file_handler = logging.handlers.RotatingFileHandler(args.log,
maxBytes=1024 * 1024 * 32,
backupCount=8)
file_formatter = logging.Formatter(
'%(asctime)s '
'(%(filename)s:%(lineno)d) %(message)s')
file_formatter.datefmt = '%Y-%m-%dT%H:%M:%S (%z)'
file_handler.setFormatter(file_formatter)
logging.getLogger().addHandler(file_handler)
for group_name in next(os.walk(args.dir))[1]:
logging.info('Processing group %s', group_name)
credentials = get_credentials(args.group_owner)
http = credentials.authorize(set_user_agent(
httplib2.Http(),
'%s-%s' % (APPLICATION_NAME, APPLICATION_VERSION)))
service_settings = discovery.build('groupssettings', 'v1', http=http)
try:
results = service_settings.groups().get(groupUniqueId=group_name, alt='json').execute()
logging.info('Successfully found group %s', group_name)
except Exception:
logging.error("Can't get group %s", group_name)
raise
try:
service_migration = discovery.build('groupsmigration', 'v1', http=http)
result = process_mbox_files(group_name, service_migration)
logging.info('Done importing user %s. Labels: %d succeeded, %d with some '
'errors, %d failed. Messages: %d succeeded, %d failed.',
group_name,
result[0],
result[1],
result[2],
result[3],
result[4])
except Exception:
logging.error("Can't process mbox files for group %s", group_name)
raise
logging.info('Finished.\n\n')
def user_worker(queue):
global number_of_labels_imported_without_error
global number_of_labels_imported_with_some_errors
global number_of_labels_failed
global number_of_messages_imported_without_error
global number_of_messages_failed
global number_of_users_imported_without_error
global number_of_users_imported_with_some_errors
global number_of_users_failed
while True:
username = queue.get(True)
if username == sentinel:
logging.info("Thread done")
break
## temp bypass
# time.sleep(5)
# queue.task_done()
# continue
try:
logging.info('Processing user %s', username)
try:
credentials = get_credentials(username)
http = credentials.authorize(set_user_agent(
httplib2.Http(),
'%s-%s' % (APPLICATION_NAME, APPLICATION_VERSION)))
service = discovery.build('gmail', 'v1', http=http)
except Exception:
logging.error("Can't get access token for user %s", username)
raise
try:
results = service.users().labels().list(
userId=username,
fields='labels(id,name)').execute(num_retries=args.num_retries)
labels = results.get('labels', [])
except Exception:
logging.error("Can't get labels for user %s", username)
raise
try:
result = process_mbox_files(username, service, labels)
except Exception:
logging.error("Can't process mbox files for user %s", username)
raise
if result[2] == 0 and result[4] == 0:
number_of_users_imported_without_error += 1
elif result[0] > 0 or result[3] > 0:
number_of_users_imported_with_some_errors += 1
else:
number_of_users_failed += 1
number_of_labels_imported_without_error += result[0]
number_of_labels_imported_with_some_errors += result[1]
number_of_labels_failed += result[2]
number_of_messages_imported_without_error += result[3]
number_of_messages_failed += result[4]
logging.info('Done importing user %s. Labels: %d succeeded, %d with some '
'errors, %d failed. Messages: %d succeeded, %d failed.',
username,
result[0],
result[1],
result[2],
result[3],
result[4])
queue.task_done()
except Exception:
number_of_users_failed += 1
logging.exception("Can't process user %s", username)
queue.task_done()
def worker(queue):
global number_of_successes_in_label
global number_of_failures_in_label
service = None
while True:
try:
data = queue.get(True)
# unpack json
message = data['message']
index = data['index']
labelname = data['labelname']
label_id = data['label_id']
username = data['username']
if service == None:
# Trying
credentials = get_credentials(username)
http = credentials.authorize(set_user_agent(
httplib2.Http(),
'%s-%s' % (APPLICATION_NAME, APPLICATION_VERSION)))
service = discovery.build('gmail', 'v1', http=http)
except Exception,e:
if 'EOFError' in str(e):
return
else:
logging.exception(e)
# Process the message
logging.info("%d : Processing message %d in label '%s'", thread.get_ident(), index, labelname)
try:
if (args.replace_quoted_printable and
'Content-Type' in message and
'text/quoted-printable' in message['Content-Type']):
message.replace_header(
'Content-Type', message['Content-Type'].replace(
'text/quoted-printable', 'text/plain'))
logging.info('Replaced text/quoted-printable with text/plain')
except Exception:
logging.exception(
'Failed to replace text/quoted-printable with text/plain '
'in Content-Type header')
try:
if args.fix_msgid and 'Message-ID' in message:
msgid = message['Message-ID']
if msgid[0] != '<':
msgid = '<' + msgid
logging.info('Added < to Message-ID: %s', msgid)
if msgid[-1] != '>':
msgid += '>'
logging.info('Added > to Message-ID: %s', msgid)
message.replace_header('Message-ID', msgid)
except Exception:
logging.exception('Failed to fix brackets in Message-ID header')
for n in range(0, args.num_retries):
try:
metadata_object = {'labelIds': [label_id]}
# Use media upload to allow messages more than 5mb.
# See https://developers.google.com/api-client-library/python/guide/media_upload
# and http://google-api-python-client.googlecode.com/hg/docs/epy/apiclient.http.MediaIoBaseUpload-class.html.
message_data = io.BytesIO(message.as_string().encode('utf-8'))
media = MediaIoBaseUpload(message_data,
mimetype='message/rfc822')
message_response = service.users().messages().insert(
userId=username,
internalDateSource='dateHeader',
body=metadata_object,
media_body=media).execute(num_retries=args.num_retries)
number_of_successes_in_label += 1
logging.debug("Imported mbox message '%s' to Gmail ID %s",
message.get_from(), message_response['id'])
break
except Exception,e:
if 'Too many concurrent' in str(e):
logging.info("Concurrency limit hit, waiting then retrying")
time.sleep((2 ** n) + random.random())
continue
else:
number_of_failures_in_label += 1
logging.exception('Failed to import mbox message')
