def get_label_list_info(self, messages_ids): """ Batch fetch label info given message_ids Args: message_ids (list): of message_ids Returns: dict with label info """ label_info_dict = {} # Callback function for every service request def get_label_info(request_id, response, exception): if response: label_info_dict[response['id']] = response if exception: # If 404, message no longer exists, otherwise raise error if exception.resp.status != 404: raise exception batch = BatchHttpRequest(callback=get_label_info) for message_id in messages_ids: # Temporary add snippet # TODO: remove snippet batch.add(self.service.users().messages().get( userId='me', id=message_id, fields='labelIds,id,threadId,snippet' )) self.execute_service_call(batch) return label_info_dict
def get_messages(contactId): contact = session['seen'][contactId] credentials = client.OAuth2Credentials.from_json(session['credentials']) http = credentials.authorize(httplib2.Http(cache=".cache")) # Build the Gmail service from discovery gmail_service = discovery.build('gmail', 'v1', http=http) def mailscallbackfunc(result, results, moreresults): # in old Python versions: # if seen.has_key(marker) # but in new ones: if 'UNREAD' in results['labelIds']: Unread = 'true' else: Unread = 'false' for header in results['payload']['headers']: if header['name'] == 'Date': Date = header['value'] if header['name'] == 'From': From = header['value'] if From == "Anders Damsgaard <*****@*****.**>": #SKAL ÆNDRES Sent = True else: Sent = False if header['name'] == 'Subject': Subject = header['value'] Contact = { 'messageId': results['id'], 'date': Date, 'subject': Subject, 'snippet': results['snippet'], 'unread': Unread, 'sent': Sent } session['mails'].append(Contact) #for msg_id in message_ids['messages']: # batchContacts.add(gmail_service.users().messages().get(userId='me', # id=msg_id['id'], format='metadata', # metadataHeaders=['from', 'date'])) #batchContacts.execute() query = "\"to:'" + contact + "' AND from:me \" OR from:'" + contact + "'" message_ids = gmail_service.users().messages().list(userId='me', maxResults=10, labelIds='INBOX', q=query).execute() batchMails = BatchHttpRequest(callback=mailscallbackfunc) for msg_id in message_ids['messages']: batchMails.add(gmail_service.users().messages().get(userId='me', id=msg_id['id'], format='metadata', metadataHeaders=['from', 'date', 'subject'])) batchMails.execute() response = {'messages': session['mails']} js = json.dumps(response) resp = Response(js, status=200, mimetype='application/json') return resp
def delete_bucket_content(self, bucket): '''Delete content of existing bucket. :param bucket: Name of the bucket in google storage to access :type bucket: str :retunrs: Response JSON str :rtype: json ''' objects = self.list_bucket_content(bucket) if objects: logger.info("Createing batch for objects deletion...") batch = BatchHttpRequest(callback=self.delete_object_cb) for o in objects: logger.debug( "Adding 'delete request' to batch: %s" % o['name'] ) batch.add(self.service.objects().delete( object=o['name'], bucket=bucket )) logger.info("Executing batch...") resp = batch.execute(http=self.http_auth) logger.info("Response content from batch: %s" % resp) return resp return None
def get_label_list_info(self, messages_ids): """ Batch fetch label info given message_ids Args: message_ids (list): of message_ids Returns: dict with label info """ label_info_dict = {} # Callback function for every service request def get_label_info(request_id, response, exception): if response: label_info_dict[response['id']] = response if exception: # If 404, message no longer exists, otherwise raise error if exception.resp.status != 404: raise exception batch = BatchHttpRequest(callback=get_label_info) for message_id in messages_ids: # Temporary add snippet # TODO: remove snippet batch.add(self.service.users().messages().get( userId='me', id=message_id, fields='labelIds,id,threadId,snippet')) self.execute_service_call(batch) return label_info_dict
def test_execute_initial_refresh_oauth2(self): batch = BatchHttpRequest() callbacks = Callbacks() cred = MockCredentials('Foo') # Pretend this is a OAuth2Credentials object cred.access_token = None http = HttpMockSequence([ ({'status': '200', 'content-type': 'multipart/mixed; boundary="batch_foobarbaz"'}, BATCH_SINGLE_RESPONSE), ]) cred.authorize(http) batch.add(self.request1, callback=callbacks.f) batch.execute(http=http) self.assertEqual({'foo': 42}, callbacks.responses['1']) self.assertIsNone(callbacks.exceptions['1']) self.assertEqual(1, cred._refreshed) self.assertEqual(1, cred._authorized) self.assertEqual(1, cred._applied)
def get_message_list_info(self, message_ids): """ Batch fetch message info given message_ids Args: message_ids (list): of message_ids Returns: dict with messages info """ messages_info = {} # Callback function for every service request def get_message_info(request_id, response, exception): if response: messages_info[response['id']] = response if exception: # If 404, message no longer exists, otherwise raise error if exception.resp.status != 404: raise exception else: logger.error('404 error: %s' % exception) # Setup batch batch = BatchHttpRequest(callback=get_message_info) for message_id in message_ids: batch.add(self.service.users().messages().get(userId='me', id=message_id)) self.execute_service_call(batch) return messages_info
def batched_messages_get(oauth_result, message_ids, params=dict()): email = oauth_result.email credentials = oauth_result.credentials results_list = list() # Google's API client throws away the callback's return value inside # BatchHttpRequest._callback, during BatchHttpRequest.execute. # The solution, in this case, is to define the callback in the scope that # gives it access to the data structure we'll be returning the results in. def process_batch_responses(request_id, response, exception): if exception is not None: results_list.append(exception) else: results_list.append(response) batch = BatchHttpRequest() service = build('gmail', 'v1', http=httplib2.Http()) users_resource = service.users() users_messages = users_resource.messages() for each_id in message_ids: api_call = users_messages.get(userId=email, id=each_id) api_call.uri = add_query_parameters(api_call.uri, params) batch.add(api_call, callback=process_batch_responses) execute_api_call(credentials, batch) return results_list
def test_deserialize_response(self): batch = BatchHttpRequest() resp, content = batch._deserialize_response(RESPONSE) self.assertEqual(200, resp.status) self.assertEqual('OK', resp.reason) self.assertEqual(11, resp.version) self.assertEqual('{"answer": 42}', content)
def __init__(self, service, **kwargs): if "batch_uri" not in kwargs: kwargs["batch_uri"] = service.new_batch_http_request()._batch_uri super(EnhancedBatchHttpRequest, self).__init__(**kwargs) self._counter = 0 self._kwargs = kwargs # Hidden batch for modtime. self._modtime_batch = BatchHttpRequest(**kwargs)
def test_add_fail_for_resumable(self): batch = BatchHttpRequest() upload = MediaFileUpload( datafile('small.png'), chunksize=500, resumable=True) self.request1.resumable = upload with self.assertRaises(BatchError) as batch_error: batch.add(self.request1, request_id='1') str(batch_error.exception)
def get_mailslist(message_ids, result, index, credentials): mails = [] def mailscallbackfunc(result, results, moreresults): # in old Python versions: # if seen.has_key(marker) # but in new ones: if "UNREAD" in results["labelIds"]: Unread = "true" else: Unread = "false" for header in results["payload"]["headers"]: if header["name"] == "Date": Date = header["value"] if header["name"] == "From": From = header["value"] if From == "Anders Damsgaard <*****@*****.**>": # SKAL ÆNDRES Sent = True else: Sent = False if header["name"] == "Subject": Subject = header["value"] Contact = { "id": result, "messageId": results["id"], "date": Date, "subject": Subject, "snippet": results["snippet"], "unread": Unread, "sent": Sent, } mails.append(Contact) # for msg_id in message_ids['messages']: # batchContacts.add(gmail_service.users().messages().get(userId='me', # id=msg_id['id'], format='metadata', # metadataHeaders=['from', 'date'])) # batchContacts.execute() credentials = client.OAuth2Credentials.from_json(credentials) http = credentials.authorize(httplib2.Http(cache=".cache")) # Build the Gmail service from discovery gmail_service = discovery.build("gmail", "v1", http=http) batchMails = BatchHttpRequest(callback=mailscallbackfunc) for msg_id in message_ids["messages"]: batchMails.add( gmail_service.users() .messages() .get(userId="me", id=msg_id["id"], format="metadata", metadataHeaders=["from", "date", "subject"]) ) batchMails.execute() results[index] = mails return True
def get_mailslist(message_ids, result, index, credentials): mails = [] def mailscallbackfunc(result, results, moreresults): # in old Python versions: # if seen.has_key(marker) # but in new ones: if 'UNREAD' in results['labelIds']: Unread = 'true' else: Unread = 'false' for header in results['payload']['headers']: if header['name'] == 'Date': Date = header['value'] if header['name'] == 'From': From = header['value'] if From == "Anders Damsgaard <*****@*****.**>": #SKAL ÆNDRES Sent = True else: Sent = False if header['name'] == 'Subject': Subject = header['value'] Contact = { 'id': result, 'messageId': results['id'], 'date': Date, 'subject': Subject, 'snippet': results['snippet'], 'unread': Unread, 'sent': Sent } mails.append(Contact) #for msg_id in message_ids['messages']: # batchContacts.add(gmail_service.users().messages().get(userId='me', # id=msg_id['id'], format='metadata', # metadataHeaders=['from', 'date'])) #batchContacts.execute() credentials = client.OAuth2Credentials.from_json(credentials) http = credentials.authorize(httplib2.Http(cache=".cache")) # Build the Gmail service from discovery gmail_service = discovery.build('gmail', 'v1', http=http) batchMails = BatchHttpRequest(callback=mailscallbackfunc) for msg_id in message_ids['messages']: batchMails.add(gmail_service.users().messages().get(userId='me', id=msg_id['id'], format='metadata', metadataHeaders=['from', 'date', 'subject'])) batchMails.execute() results[index] = mails return True
def test_serialize_get_request_no_body(self): batch = BatchHttpRequest() request = HttpRequest( None, None, 'https://www.googleapis.com/someapi/v1/collection/?foo=bar', method='GET', body=None, headers={'content-type': 'application/json'}, methodId=None, resumable=None) s = batch._serialize_request(request).splitlines() self.assertEqual(NO_BODY_EXPECTED_GET.splitlines(), s)
def test_serialize_request_no_body(self): batch = BatchHttpRequest() request = HttpRequest( None, None, 'https://www.googleapis.com/someapi/v1/collection/?foo=bar', method='POST', body='', headers={'content-type': 'application/json'}, methodId=None, resumable=None) s = batch._serialize_request(request).splitlines() self.assertEqual(NO_BODY_EXPECTED.splitlines(), s)
def test_add_fail_for_over_limit(self): from googleapiclient.http import MAX_BATCH_LIMIT batch = BatchHttpRequest() for i in range(0, MAX_BATCH_LIMIT): batch.add(HttpRequest( None, None, 'https://www.googleapis.com/someapi/v1/collection/?foo=bar', method='POST', body='{}', headers={'content-type': 'application/json'}) ) self.assertRaises(BatchError, batch.add, self.request1)
def test_add_fail_for_resumable(self): batch = BatchHttpRequest() upload = MediaFileUpload( datafile('small.png'), chunksize=500, resumable=True) self.request1.resumable = upload self.assertRaises(BatchError, batch.add, self.request1, request_id='1')
def list_email_ids_by_label(service, label): email_ids = [] max_results = 100 next_page_token = '' # create the batch request batch_requests = BatchHttpRequest() while True: response = service.users().messages().list( userId='me', labelIds=label, maxResults=max_results, pageToken=next_page_token).execute() # extract id for msg_id in response['messages']: email_ids.append(msg_id['id']) print(len(email_ids)) if 'nextPageToken' in response: print('next page token:', response['nextPageToken']) next_page_token = response['nextPageToken'] else: break # uncomment for testing cap 500 break return email_ids
def test_execute_batch_http_error(self): callbacks = Callbacks() batch = BatchHttpRequest(callback=callbacks.f) batch.add(self.request1) batch.add(self.request2) http = HttpMockSequence([ ({'status': '200', 'content-type': 'multipart/mixed; boundary="batch_foobarbaz"'}, BATCH_ERROR_RESPONSE), ]) batch.execute(http=http) self.assertEqual({'foo': 42}, callbacks.responses['1']) expected = ('<HttpError 403 when requesting ' 'https://www.googleapis.com/someapi/v1/collection/?foo=bar returned ' '"Access Not Configured">') self.assertEqual(expected, str(callbacks.exceptions['2']))
def test_serialize_request_media_body(self): batch = BatchHttpRequest() f = open(datafile('small.png'), 'rb') body = f.read() f.close() request = HttpRequest( None, None, 'https://www.googleapis.com/someapi/v1/collection/?foo=bar', method='POST', body=body, headers={'content-type': 'application/json'}, methodId=None, resumable=None) # Just testing it shouldn't raise an exception. s = batch._serialize_request(request).splitlines()
def test_serialize_request_media_body(self): batch = BatchHttpRequest() f = open(datafile('small.png')) body = f.read() f.close() request = HttpRequest( None, None, 'https://www.googleapis.com/someapi/v1/collection/?foo=bar', method='POST', body=body, headers={'content-type': 'application/json'}, methodId=None, resumable=None) # Just testing it shouldn't raise an exception. s = batch._serialize_request(request).splitlines()
def create_request_batch(list_msg_ids_batch, service): """ takes in a list of msg_id_batch, then package each batch into a http request batch :param list_msg_ids_batch: :param service: :return: """ request_batches = [] for batch in list_msg_ids_batch: batch_requests = BatchHttpRequest() for msg_id in batch: # print(msg_id) batch_requests.add( service.users().messages().get(userId='me', id=msg_id, format='metadata'), get_message) request_batches.append(batch_requests) return request_batches
def test_execute_refresh_and_retry_on_401(self): batch = BatchHttpRequest() callbacks = Callbacks() cred_1 = MockCredentials('Foo') cred_2 = MockCredentials('Bar') http = HttpMockSequence([ ({'status': '200', 'content-type': 'multipart/mixed; boundary="batch_foobarbaz"'}, BATCH_RESPONSE_WITH_401), ({'status': '200', 'content-type': 'multipart/mixed; boundary="batch_foobarbaz"'}, BATCH_SINGLE_RESPONSE), ]) creds_http_1 = HttpMockSequence([]) cred_1.authorize(creds_http_1) creds_http_2 = HttpMockSequence([]) cred_2.authorize(creds_http_2) self.request1.http = creds_http_1 self.request2.http = creds_http_2 batch.add(self.request1, callback=callbacks.f) batch.add(self.request2, callback=callbacks.f) batch.execute(http=http) self.assertEqual({'foo': 42}, callbacks.responses['1']) self.assertEqual(None, callbacks.exceptions['1']) self.assertEqual({'baz': 'qux'}, callbacks.responses['2']) self.assertEqual(None, callbacks.exceptions['2']) self.assertEqual(1, cred_1._refreshed) self.assertEqual(0, cred_2._refreshed) self.assertEqual(1, cred_1._authorized) self.assertEqual(1, cred_2._authorized) self.assertEqual(1, cred_2._applied) self.assertEqual(2, cred_1._applied)
def batchDeleteMails(service): if not to_delete_ids: return 1 print('Deleting e-mails...') try: # batch delete messages as 1000 msg limit for regular users batch_limit = 1000 iterations = math.ceil(len(to_delete_ids) / batch_limit) skip, take = 0, batch_limit print(f'Total Messages: {len(to_delete_ids)},\ Target Iterations: {iterations}') batch = BatchHttpRequest(callback=deleteCallback) for _ in range(iterations): payload = {'ids': []} payload['ids'].extend( [str(d['id']) for d in to_delete_ids[skip:take]]) batch.add(service.users().messages().batchDelete(userId=USER_ID, body=payload)) skip = take take = len(to_delete_ids) if (take + batch_limit >= len(to_delete_ids))\ else take + batch_limit batch.execute() except errors.HttpError as ex: print(f'Exception:{ex}') return 1
def test_execute_request_body_with_custom_long_request_ids(self): batch = BatchHttpRequest() batch.add(self.request1, request_id='abc'*20) batch.add(self.request2, request_id='def'*20) http = HttpMockSequence([ ({'status': '200', 'content-type': 'multipart/mixed; boundary="batch_foobarbaz"'}, 'echo_request_body'), ]) try: batch.execute(http=http) self.fail('Should raise exception') except BatchError as e: boundary, _ = e.content.split(None, 1) self.assertEqual('--', boundary[:2]) parts = e.content.split(boundary) self.assertEqual(4, len(parts)) self.assertEqual('', parts[0]) self.assertEqual('--', parts[3].rstrip()) for partindex, request_id in ((1, 'abc'*20), (2, 'def'*20)): lines = parts[partindex].splitlines() for n, line in enumerate(lines): if line.startswith('Content-ID:'): # assert correct header folding self.assertTrue(line.endswith('+'), line) header_continuation = lines[n+1] self.assertEqual( header_continuation, ' %s>' % request_id, header_continuation )
class EnhancedBatchHttpRequest(BatchHttpRequest): """Google batch request object that automatically calls execute on itself when too many calls are batched.""" CAP = 100 def __init__(self, service, **kwargs): if "batch_uri" not in kwargs: kwargs["batch_uri"] = service.new_batch_http_request()._batch_uri super(EnhancedBatchHttpRequest, self).__init__(**kwargs) self._counter = 0 self._kwargs = kwargs # Hidden batch for modtime. self._modtime_batch = BatchHttpRequest(**kwargs) def add(self, *args, **kwargs): super(EnhancedBatchHttpRequest, self).add(*args, **kwargs) self._counter += 1 if self._counter >= self.CAP: self.execute() def clear(self): super(EnhancedBatchHttpRequest, self).__init__(**self._kwargs) self._counter = 0 self._modtime_batch.__init__(**self._kwargs) def execute(self, *args, **kwargs): super(EnhancedBatchHttpRequest, self).execute(*args, **kwargs) self._modtime_batch.execute(*args, **kwargs) self.clear() def add_modtime(self, *args, **kwargs): self._modtime_batch.add(*args, **kwargs)
def batch_add_users(service, accountId, users): """ Batch add users Parameters: users (tuple): (user_email, list of requests) """ def handle_create_user(requestId, response, exception): if exception is not None: print(f"There was an error: {exception}") else: print(f"Request ID: {requestId}, Created user: {response}") batch = BatchHttpRequest(callback=handle_create_user) for user in users: user_email, requests = user for request in requests: webPropertyId, profileId, permissions = request batch.add(service.management().profileUserLinks().insert( accountId=accountId, webPropertyId=webPropertyId, profileId=profileId, body={ "permissions": { "local": permissions }, "userRef": { "email": user_email }, }, )) batch.execute()
def main(): """ GMail Dump * Get & Store credentials * Download all GMail messages into Maildir folder named mail """ credentials = get_credentials() http = credentials.authorize(httplib2.Http()) service = discovery.build('gmail', 'v1', http=http) results = service.users().labels().list(userId='me').execute() labels = results.get('labels', []) profile = service.users().getProfile(userId='me').execute() # example: {'historyId': '380957', 'emailAddress': 'user@domain', 'threadsTotal': 1135, 'messagesTotal': 1950} print("Downloading {messagesTotal} messages for {emailAddress}".format(**profile)) for new_dir in ['tmp','new','cur']: os.makedirs("mail/%s" % new_dir, mode=0o700, exist_ok=True) def process_message(request_id, response, exception): if exception is not None: print("ERROR: " + request_id) else: msg_bytes = base64.urlsafe_b64decode(response['raw'].encode('ASCII')) mime_msg = email.message_from_bytes(msg_bytes) maildir_message = mailbox.MaildirMessage(mime_msg) #box.add(maildir_message) message_id = response['id'] with open("mail/cur/%s" % message_id, "wb") as message_file: message_file.write(maildir_message.__bytes__()) try: message_count = 0 start = True while start or 'nextPageToken' in response: if start: page_token = None start = False else: page_token = response['nextPageToken'] response = service.users().messages().list(userId='me', pageToken=page_token).execute() if 'messages' in response: message_count += len(response['messages']) existing_message_count = 0 batch = BatchHttpRequest(callback=process_message) for message in response['messages']: message_id = message['id'] if os.path.exists('mail/cur/%s' % message_id): existing_message_count+=1 else: batch.add(service.users().messages().get(userId='me', format='raw', id=message_id)) batch.execute() info = "Downloaded %s messages" % message_count if existing_message_count: info += " (skipping %s messages already downloaded)" % existing_message_count print(info) except errors.HttpError as error: print('An HTTPError occurred: %s' % error)
def test_execute_initial_refresh_oauth2(self): batch = BatchHttpRequest() callbacks = Callbacks() cred = MockCredentials('Foo') # Pretend this is a OAuth2Credentials object cred.access_token = None http = HttpMockSequence([ ({ 'status': '200', 'content-type': 'multipart/mixed; boundary="batch_foobarbaz"' }, BATCH_SINGLE_RESPONSE), ]) cred.authorize(http) batch.add(self.request1, callback=callbacks.f) batch.execute(http=http) self.assertEqual({'foo': 42}, callbacks.responses['1']) self.assertIsNone(callbacks.exceptions['1']) self.assertEqual(1, cred._refreshed) self.assertEqual(1, cred._authorized) self.assertEqual(1, cred._applied)
def test_http_errors_passed_to_callback(self): batch = BatchHttpRequest() callbacks = Callbacks() cred_1 = MockCredentials('Foo') cred_2 = MockCredentials('Bar') http = HttpMockSequence([ ({'status': '200', 'content-type': 'multipart/mixed; boundary="batch_foobarbaz"'}, BATCH_RESPONSE_WITH_401), ({'status': '200', 'content-type': 'multipart/mixed; boundary="batch_foobarbaz"'}, BATCH_RESPONSE_WITH_401), ]) creds_http_1 = HttpMockSequence([]) cred_1.authorize(creds_http_1) creds_http_2 = HttpMockSequence([]) cred_2.authorize(creds_http_2) self.request1.http = creds_http_1 self.request2.http = creds_http_2 batch.add(self.request1, callback=callbacks.f) batch.add(self.request2, callback=callbacks.f) batch.execute(http=http) self.assertEqual(None, callbacks.responses['1']) self.assertEqual(401, callbacks.exceptions['1'].resp.status) self.assertEqual( 'Authorization Required', callbacks.exceptions['1'].resp.reason) self.assertEqual({u'baz': u'qux'}, callbacks.responses['2']) self.assertEqual(None, callbacks.exceptions['2'])
def test_execute_request_body(self): batch = BatchHttpRequest() batch.add(self.request1) batch.add(self.request2) http = HttpMockSequence([ ({'status': '200', 'content-type': 'multipart/mixed; boundary="batch_foobarbaz"'}, 'echo_request_body'), ]) try: batch.execute(http=http) self.fail('Should raise exception') except BatchError, e: boundary, _ = e.content.split(None, 1) self.assertEqual('--', boundary[:2]) parts = e.content.split(boundary) self.assertEqual(4, len(parts)) self.assertEqual('', parts[0]) self.assertEqual('--', parts[3].rstrip()) header = parts[1].splitlines()[1] self.assertEqual('Content-Type: application/http', header)
def summarize_mail(): service, http = authorize_me() batch = BatchHttpRequest() response = service.users().messages().list(userId=constants.USER_ID, q=constants.DEFAULT_QUERY).execute() messages = response.get('messages', []) if messages: # TODO think this only gets the first few, might need to page through results for message in messages: request = service.users().messages().get(userId=constants.USER_ID, id=message['id'], format=constants.RESPONSE_FORMAT, metadataHeaders=[constants.FROM_HEADER, constants.TO_HEADER]) batch.add(request, callback=parse_mail) batch.execute(http=http) print "There are %s unread messages in your inbox." % len(summary.messages) print "%s of them are marked as important and %s are addressed to you." % (summary.important, summary.addressed_to_me) print "%s of them are scary." % summary.flagged response = raw_input("\nYou can do the thing! Do you want to see a preview of your messages? ") if response.lower().startswith('y'): print "\n" for i, msg in enumerate(summary.messages): print "Message #%s" % i print "=" * 30 print "From: %s\nTo: %s\nSubject: %s" % (msg.sender, msg.to, msg.snippet) print "=" * 30 print "\n" else: print "No messages! You are free. :)"
def __init__(self, key_p12_path, service_account_mail): if not os.path.exists(key_p12_path): raise AttributeError('p12 key path is not vaild') f = file(key_p12_path, 'rb') key = f.read() f.close() credentials = SignedJwtAssertionCredentials( service_account_mail, key, scope=self._scope) http = httplib2.Http() self._http = credentials.authorize(http) self._service = build('calendar', 'v3', http=http) # batch is limited up to 1000 queries in one self._batch = BatchHttpRequest() self._calendar_id = None self._event_json_path = None self._event_name_to_id_dict = None self._service_mail = service_account_mail
def test_new_id(self): batch = BatchHttpRequest() id_ = batch._new_id() self.assertEqual('1', id_) id_ = batch._new_id() self.assertEqual('2', id_) batch.add(self.request1, request_id='3') id_ = batch._new_id() self.assertEqual('4', id_)
def main(): start_time = time.time() # building out the api service creds = get_creds() service = build('gmail', 'v1', credentials=creds) # label = 'CATEGORY_PERSONAL' # labels = list_user_labels(service) labels = ['INBOX'] pprint(labels) logging.basicConfig( level=args.loglevel, format='%(asctime)s %(levelname)s %(funcName)s:%(lineno)d %(message)s', stream=sys.stdout) # remove_bad_labels(labels) for label in labels: print('current label', label) # gets email ids per label global msg_ids msg_ids = list_email_ids_by_label(service, label) # print(len(msg_ids)) # run each batch while len(msg_ids) > 0: # break into batches msg_ids_batches = break_into_n_size_each_batch( msg_ids, max_per_batch) print('batches to process:', len(msg_ids_batches), ',per batch:', len(msg_ids_batches[0]), ',last batch:', len(msg_ids_batches[-1])) b_id = 0 for batch in msg_ids_batches: # print('batch ids', batch) b_id += 1 print('running batch', b_id) batch_request = BatchHttpRequest() for msg_id in batch: # print(msg_id) batch_request.add( service.users().messages().get( userId='me', id=msg_id, format='metadata', metadataHeaders=metadata), get_message) batch_request.execute() print(msg_df.shape) print(msg_df.head()) print('writing to', label + '.csv') msg_df.to_csv('new/' + label + '.csv', encoding='utf-8', index=False) print('program time:', time.time() - start_time)
def new_batch_http_request(callback=None): """Create a BatchHttpRequest object based on the discovery document. Args: callback: callable, A callback to be called for each response, of the form callback(id, response, exception). The first parameter is the request id, and the second is the deserialized response object. The third is an apiclient.errors.HttpError exception object if an HTTP error occurred while processing the request, or None if no error occurred. Returns: A BatchHttpRequest object based on the discovery document. """ return BatchHttpRequest(callback=callback, batch_uri=batch_uri)
def fetch(self, msg_ids): fetcher = MessageFetcher() batch = BatchHttpRequest() for msg_desc in msg_ids: batch.add(self.service.users().messages().get(userId='me', id=msg_desc['id'], format='raw'), callback=fetcher.fetch_message) batch.execute() return fetcher.messages
def query_email_api(access_token): """ this queries the Gmail Batch API for Gmail messages. param access_token: an oauth access token return: """ f = open("emaildata.json", "w+") f.close() def callback(response): """ generic callback function used to deal with the responses from the Batch API. """ with open('emaildata.json', 'a') as outfile: json.dump(response, outfile, indent=4, sort_keys=True) return credentials = google.oauth2.credentials.Credentials(access_token) GMAIL = build('gmail', 'v1', credentials=credentials) message_ids = GMAIL.users().messages().list(userId='me', ).execute() try: messages = [] count = 0 if 'messages' in message_ids: messages.extend(message_ids['messages']) while 'nextPageToken' in message_ids: page_token = message_ids['nextPageToken'] message_ids = GMAIL.users().messages().list( userId='me', pageToken=page_token).execute() messages.extend(message_ids['messages']) count += 1 message_estimate = count * 100 except errors.HttpError as error: print('An error occurred: %s' % error) choice = input( "%d Pages of Messages have been found on this account. It is estimated that" "would be %d Messages.\nWould you like to continue? Type Y to Proceed, otherwise type any key: " % (count, message_estimate)) if choice == "Y": message_ids_sorted = ([message_id['id'] for message_id in messages]) batch = BatchHttpRequest() for msg_id in message_ids_sorted: batch.add(GMAIL.users().messages().get(userId='me', id=msg_id, format='minimal'), callback=callback) batch.execute() return else: return
def getMailsByFilter(service, user_id, cleanup_list): try: batch = BatchHttpRequest(callback=getCallback) for lookup in cleanup_list: batch.add(service.users().messages().list(userId=user_id, q=lookup), request_id=lookup) batch.execute() return 0 except errors.HttpError as ex: print(f'Exception:{ex}') return 1
def GetEmails(user: str, message_ids: list, email_format: str = 'full', batch_size=100, batch_wait=0): """ Get's all of the emails given a list of message id's (typically from a list request). :param user: The user to get the emails from. :param message_ids: List of message id's to retrieve. :param email_format: The format that the email is returned as, default: "full". - "full": Returns the full email message data with body content parsed in the payload field; the raw field is not used. (default) - "metadata": Returns only email message ID, labels, and email headers. - "minimal": Returns only email message ID and labels; does not return the email headers, body, or payload. - "raw": Returns the full email message data with body content in the raw field as a base64url encoded string; the payload field is not used. :param batch_size: Size of the batch request (how many emails to download each iteration). Note: Gmail API's rate limit is 250 request units per second, but any batch_size over 100 tends to exceed that. :param batch_wait: How long to wait (seconds) between requests. :return: A list of dicts containing the emails and their metadata. """ service = GetService( email_address=user, scopes=['https://www.googleapis.com/auth/gmail.readonly']) print(f'Getting {len(message_ids)} Messages...') messages = [] good = 0 batches = [] for i in range(0, len(message_ids), batch_size): print(f"Messages: {i+1} - {min(i+batch_size, len(message_ids))}") batch = BatchHttpRequest() # start building a batch request for message in message_ids[i:( i + batch_size)]: # add a request for each id batch.add(service.users().messages().get(userId='me', id=message, format=email_format)) batch.execute() batches.append(batch) for header, body in batch._responses.values(): if header['status'] == '200': good += 1 message = json.loads(body) messages.append(message) time.sleep(batch_wait) print(f"Successfully retrieved {good}/{len(message_ids)} messages!") return messages, batches
def test_execute_global_callback(self): callbacks = Callbacks() batch = BatchHttpRequest(callback=callbacks.f) batch.add(self.request1) batch.add(self.request2) http = HttpMockSequence([ ({'status': '200', 'content-type': 'multipart/mixed; boundary="batch_foobarbaz"'}, BATCH_RESPONSE), ]) batch.execute(http=http) self.assertEqual({'foo': 42}, callbacks.responses['1']) self.assertEqual({'baz': 'qux'}, callbacks.responses['2'])
def get_messages_batch(msg_ids): # create the batch request batch_requests = BatchHttpRequest() print('in get_messages_batch') # print(msg_ids) for msg_id in msg_ids: print('msg_id', msg_id) batch_requests.add( service.users().messages().get(userId='me', id=str(msg_id), format='metadata', metadata=metadata), get_message) batch_requests.execute()
def handle_messages_details(message_list: dict, gmail_client: Resource, user_email: str) -> None: batch = BatchHttpRequest() for i in message_list["messages"]: callback = partial(find_and_save_flight_bookings, service=gmail_client, user_id=user_email) batch.add( gmail_client.users().messages().get(userId=user_email, id=i["id"]), callback=callback, ) batch.execute()
def test_add(self): batch = BatchHttpRequest() batch.add(self.request1, request_id='1') self.assertRaises(KeyError, batch.add, self.request1, request_id='1')
def get_listmessages(): def get_mailslist(message_ids, result, index, credentials): mails = [] def mailscallbackfunc(result, results, moreresults): # in old Python versions: # if seen.has_key(marker) # but in new ones: if "UNREAD" in results["labelIds"]: Unread = "true" else: Unread = "false" for header in results["payload"]["headers"]: if header["name"] == "Date": Date = header["value"] if header["name"] == "From": From = header["value"] if From == "Anders Damsgaard <*****@*****.**>": # SKAL ÆNDRES Sent = True else: Sent = False if header["name"] == "Subject": Subject = header["value"] Contact = { "id": result, "messageId": results["id"], "date": Date, "subject": Subject, "snippet": results["snippet"], "unread": Unread, "sent": Sent, } mails.append(Contact) # for msg_id in message_ids['messages']: # batchContacts.add(gmail_service.users().messages().get(userId='me', # id=msg_id['id'], format='metadata', # metadataHeaders=['from', 'date'])) # batchContacts.execute() credentials = client.OAuth2Credentials.from_json(credentials) http = credentials.authorize(httplib2.Http(cache=".cache")) # Build the Gmail service from discovery gmail_service = discovery.build("gmail", "v1", http=http) batchMails = BatchHttpRequest(callback=mailscallbackfunc) for msg_id in message_ids["messages"]: batchMails.add( gmail_service.users() .messages() .get(userId="me", id=msg_id["id"], format="metadata", metadataHeaders=["from", "date", "subject"]) ) batchMails.execute() results[index] = mails return True credentials = client.OAuth2Credentials.from_json(session["credentials"]) http = credentials.authorize(httplib2.Http(cache=".cache")) # Build the Gmail service from discovery gmail_service = discovery.build("gmail", "v1", http=http) def mailIdscallbackfunc(result, results, moreresults): session["messageIds"].append(results) batchMailIds = BatchHttpRequest(callback=mailIdscallbackfunc) for contact in session["seen"][0:10]: query = "\"to:'" + contact + "' AND from:me \" OR from:'" + contact + "'" batchMailIds.add(gmail_service.users().messages().list(userId="me", maxResults=10, labelIds="INBOX", q=query)) batchMailIds.execute() threads = [None] * 10 results = [None] * 10 for i in range(0, 10): threads[i] = Thread(target=get_mailslist, args=(session["messageIds"][i], results, i, session["credentials"])) threads[i].start() for i in range(0, 10): threads[i].join() js = json.dumps(results) resp = Response(js, status=200, mimetype="application/json") return resp
def test_id_to_from_content_id_header(self): batch = BatchHttpRequest() self.assertEquals('12', batch._header_to_id(batch._id_to_header('12')))
def get_messages(contactId): contact = session['seen'][contactId] contact_address = contact.split()[-1] contact_address = re.sub(r'[<>]', '', contact_address) contact_address = contact_address.strip() credentials = client.OAuth2Credentials.from_json(session['credentials']) http = credentials.authorize(httplib2.Http(cache='.cache')) # Build the Gmail service from discovery gmail_service = discovery.build('gmail', 'v1', http=http) def mailscallbackfunc(result, results, moreresults): if 'labelIds' in results: if 'UNREAD' in results['labelIds']: Unread = True else: Unread = False else: Unread = None for header in results['payload']['headers']: if header['name'].lower() == 'date': Date = header['value'] if header['name'].lower() == 'from': From = header['value'] from_address = From.split()[-1] from_address = re.sub(r'[<>]', '', from_address) from_address = from_address.strip() if from_address.lower() == session['user_address']: Sent = True else: Sent = False if header['name'].lower() == 'subject': Subject = header['value'] try: Subject # does a exist in the current namespace except NameError: Subject = 'Unknown Subject' # nope Contact = { 'messageId': results['id'], 'date': Date, 'subject': Subject, 'snippet': results['snippet'], 'unread': Unread, 'sent': Sent, } session['mails'].append(Contact) query = "(from:me to:" + contact_address + ") OR (from:" + contact + ")" print "#######query############" print query message_ids = gmail_service.users().messages().list(userId='me', maxResults=10, q=query).execute() #labelIds='INBOX', batchMails = BatchHttpRequest(callback=mailscallbackfunc) for msg_id in message_ids['messages']: batchMails.add(gmail_service.users().messages().get(userId='me' , id=msg_id['id'], format='metadata', metadataHeaders=['from', 'date', 'subject'])) batchMails.execute() response = {'messages': session['mails']} js = json.dumps(response) resp = Response(js, status=200, mimetype='application/json') return resp
class GoogleCalendar: _scope = 'https://www.googleapis.com/auth/calendar' def __init__(self, key_p12_path, service_account_mail): if not os.path.exists(key_p12_path): raise AttributeError('p12 key path is not vaild') f = file(key_p12_path, 'rb') key = f.read() f.close() credentials = SignedJwtAssertionCredentials( service_account_mail, key, scope=self._scope) http = httplib2.Http() self._http = credentials.authorize(http) self._service = build('calendar', 'v3', http=http) # batch is limited up to 1000 queries in one self._batch = BatchHttpRequest() self._calendar_id = None self._event_json_path = None self._event_name_to_id_dict = None self._service_mail = service_account_mail def pushChanges(self): self._batch.execute(http=self._http) def eventAdded(self, request_id, response, exception): if exception is None: self._event_name_to_id_dict[request_id] = response['id'] else: pass # TODO add log message def eventModified(self, request_id, response, exception): if exception is None: self._event_name_to_id_dict[request_id] = response['id'] else: pass # TODO add log message def _convertDateTime(self, date, time): return rfc3339.rfc3339(datetime.datetime(int(date[:4]), int(date[5:7]), int(date[8:10]), int(time[:2]), int(time[3:5]))) def _eventStartDateTime(self, event_details): return self._convertDateTime(event_details[u'Data rozpoczęcia'.encode('windows-1250')], event_details[u'Czas rozpoczęcia'.encode('windows-1250')]) def _eventEndDateTime(self, event_details): return self._convertDateTime(event_details[u'Data zakończenia'.encode('windows-1250')], event_details[u'Czas zakończenia'.encode('windows-1250')]) def _eventLocation(self, event_details): return event_details[u'Lokalizacja'.encode('windows-1250')] def addScheduleEvents(self, event_names, event_details): for ev_name in event_names: ev_det = event_details[ev_name] event = { 'summary': ev_name.decode('windows-1250'), 'start': { 'dateTime': self._eventStartDateTime(ev_det) }, 'end': { 'dateTime': self._eventEndDateTime(ev_det) }, } loc = self._eventLocation(ev_det) if loc is not None: event['location'] = loc if '(w)' in ev_name: col = '10' elif '(L)' in ev_name: col = '4' elif '(p)' in ev_name: col = '6' elif u'(ć)'.encode('windows-1250') in ev_name: col = '11' else: col = '1' event['colorId'] = col self._batch.add(self._service.events().insert(calendarId=self._calendar_id, body=event), callback=self.eventAdded, request_id=ev_name) def modifyScheduleEvents(self, event_names, new_event_details): for ev_name in event_names: ev_det = new_event_details[ev_name] patch = { 'start': { 'dateTime': self._eventStartDateTime(ev_det) }, 'end': { 'dateTime': self._eventEndDateTime(ev_det) }, } loc = self._eventLocation(ev_det) if loc is not None: patch['location'] = loc self._batch.add(self._service.events().patch(calendarId=self._calendar_id, eventId=self._event_name_to_id_dict[ev_name.decode('windows-1250')], body=patch), callback=self.eventModified, request_id=ev_name) def removeScheduleEvents(self, event_names): #TODO usuwanie z jsona lub dawanie gdzieś idziej informacji o usunięciu for ev_name in event_names: patch = { 'status': 'cancelled' } self._batch.add(self._service.events().patch(calendarId=self._calendar_id, eventId=self._event_name_to_id_dict[ev_name.decode('windows-1250')], body=patch), callback=self.eventModified, request_id=ev_name) def createCalendar(self, name): calendar = { 'summary': name, 'timeZone': 'Europe/Warsaw' } created_calendar = self._service.calendars().insert(body=calendar).execute() return created_calendar['id'] def setCalendar(self, calendar_id, event_dict_json_path): self._calendar_id = calendar_id self._event_json_path = event_dict_json_path self._event_name_to_id_dict = json.loads(open(event_dict_json_path).read(), encoding='windows-1250') if os.path.exists(event_dict_json_path) else {} def shareCalendarWithGroup(self, email): rule = { 'scope': { 'type': 'group', 'value': email, }, 'role': 'reader' } created_rule = self._service.acl().insert(calendarId=self._calendar_id, body=rule).execute() return created_rule['id'] def shareCalendarWithOwner(self, email): rule = { 'scope': { 'type': 'user', 'value': email, }, 'role': 'owner' } created_rule = self._service.acl().insert(calendarId=self._calendar_id, body=rule).execute() return created_rule['id'] def deletePrivilege(self, acl_id): self._service.acl().delete(calendarId=self._calendar_id, ruleId=acl_id).execute() def clearAllDataInCalendar(self): self._service.calendars().clear(calendarId=self._calendar_id).execute() def removeCalendar(self): self._service.calendarList().delete(calendarId=self._calendar_id).execute() def removeAllCalendars(self): page_token = None while True: calendar_list = self._service.calendarList().list(pageToken=page_token).execute() for calendar_list_entry in calendar_list['items']: self._service.calendars().delete(calendarId=calendar_list_entry['id']).execute() page_token = calendar_list.get('nextPageToken') if not page_token: break def end(self): json_file = open(self._event_json_path, mode='w') json_file.write(json.dumps(self._event_name_to_id_dict, encoding='windows-1250')) json_file.close()
def contacts(pagenr): # Authorize the httplib2.Http object with our credentials credentials = client.OAuth2Credentials.from_json(session["credentials"]) http = credentials.authorize(httplib2.Http(cache=".cache")) # Build the Gmail service from discovery gmail_service = discovery.build("gmail", "v1", http=http) while len(session["seen"]) + 1 < pagenr * 10: # Retrieve a page of threads # Print ID for each thread # if threads['threads']: # for thread in threads['threads']: # print 'Thread ID: %s' % (thread['id']) def contactscallbackfunc(result, results, moreresults): # in old Python versions: # if seen.has_key(marker) # but in new ones: if "UNREAD" in results["labelIds"]: Unread = "true" else: Unread = "false" for header in results["payload"]["headers"]: if header["name"] == "Date": Date = header["value"] if header["name"] == "From": From = header["value"] address = From.split()[-1] address = re.sub(r"[<>]", "", address) Address = address.strip() Name = From.rsplit(" ", 1)[0] From = From.replace('"', "") Name = Name.replace('"', "") if From not in session["seen"]: Contact = { "id": len(session["seen"]), "date": Date, "name": Name, "address": Address, "snippet": results["snippet"], "unread": Unread, } session["contacts"].append(Contact) session["seen"].append(From) message_ids = ( gmail_service.users() .messages() .list(userId="me", maxResults=50, labelIds="INBOX", pageToken=session["contactsNextPageToken"]) .execute() ) batchContacts = BatchHttpRequest(callback=contactscallbackfunc) # return message_ids['messages'][0]['id'] session["contactsNextPageToken"] = message_ids["nextPageToken"] for msg_id in message_ids["messages"]: batchContacts.add( gmail_service.users() .messages() .get(userId="me", id=msg_id["id"], format="metadata", metadataHeaders=["from", "date"]) ) batchContacts.execute() # return "test" # def mailscallbackfunc(result, results, moreresults): # if 'UNREAD' in results['labelIds']: # Unread = 'true' # else: # Unread = 'false' # # batchMails = BatchHttpRequest(callback=mailscallbackfunc) # for contact in session['seen']: # batchContacts.add(gmail_service.users().messages().get(userId='me', # id=msg_id['id'], format='metadata', # metadataHeaders=['from', 'date'])) # batchMails.execute() if pagenr - 1 == 0: start = 0 end = 9 else: start = (pagenr - 1) * 10 - 1 end = pagenr * 10 - 1 response = {"nextPageToken": session["contactsNextPageToken"], "contacts": session["contacts"][start:end]} js = json.dumps(response) resp = Response(js, status=200, mimetype="application/json") return resp
def contacts(pagenr): # Authorize the httplib2.Http object with our credentials credentials = client.OAuth2Credentials.from_json(session['credentials']) http = credentials.authorize(httplib2.Http(cache='.cache')) # Build the Gmail service from discovery gmail_service = discovery.build('gmail', 'v1', http=http) while len(session['seen']) + 1 < pagenr * 10: # Retrieve a page of threads # Print ID for each thread # if threads['threads']: # for thread in threads['threads']: # print 'Thread ID: %s' % (thread['id']) def contactscallbackfunc(result, results, moreresults): if 'UNREAD' in results['labelIds']: Unread = True else: Unread = False for header in results['payload']['headers']: if header['name'].lower() == 'date': Date = header['value'] if header['name'].lower() == 'from': From = header['value'] address = From.split()[-1] address = re.sub(r'[<>]', '', address) Address = address.strip() Domain = Address.split('@')[1] if Domain == 'facebookmail.com': Domain = 'facebook.com' Name = From.rsplit(' ', 1)[0] From = From.replace('"', '') Name = Name.replace('"', '') if From not in session['seen'] and Address != session['user_address']: Contact = { 'id': len(session['seen']), 'date': Date, 'name': Name, 'initial': Name[0].upper(), 'address': Address, 'domain': Domain, 'snippet': results['snippet'], 'unread': Unread, } session['contacts'].append(Contact) session['seen'].append(From) #query = '' #if not session['seen']: # pass #else: # query = 'from:(' # for From in session['seen']: # query = query + "-'" + From + "', " # query = query + ')' # query = "\"to:'" + contact + "' AND from:me \" OR from:'" + contact + "'" user_id = gmail_service.users().getProfile(userId='me').execute() session['user_address'] = user_id['emailAddress'] message_ids = gmail_service.users().messages().list(userId='me' , maxResults=50, pageToken=session['contactsNextPageToken']).execute() batchContacts = BatchHttpRequest(callback=contactscallbackfunc) session['contactsNextPageToken'] = message_ids['nextPageToken'] for msg_id in message_ids['messages']: batchContacts.add(gmail_service.users().messages().get(userId='me' , id=msg_id['id'], format='metadata', metadataHeaders=['from', 'date'])) batchContacts.execute() if pagenr - 1 == 0: start = 0 end = 10 else: start = (pagenr - 1) * 10 end = pagenr * 10 response = {'nextPageToken': session['contactsNextPageToken'], 'contacts': (session['contacts'])[start:end]} js = json.dumps(response) resp = Response(js, status=200, mimetype='application/json') return resp
def test_execute_empty_batch_no_http(self): batch = BatchHttpRequest() ret = batch.execute() self.assertEqual(None, ret)
def contacts(pagenr): # Authorize the httplib2.Http object with our credentials credentials = client.OAuth2Credentials.from_json(session['credentials']) http = credentials.authorize(httplib2.Http(cache=".cache")) # Build the Gmail service from discovery gmail_service = discovery.build('gmail', 'v1', http=http) while (len(session['seen'])+1 < pagenr*10): # Retrieve a page of threads # Print ID for each thread # if threads['threads']: # for thread in threads['threads']: # print 'Thread ID: %s' % (thread['id']) def contactscallbackfunc(result, results, moreresults): # in old Python versions: # if seen.has_key(marker) # but in new ones: if 'UNREAD' in results['labelIds']: Unread = 'true' else: Unread = 'false' for header in results['payload']['headers']: if header['name'] == 'Date': Date = header['value'] if header['name'] == 'From': From = header['value'] address = From.split()[-1] address = re.sub(r'[<>]', '', address) Address = address.strip() Name = From.rsplit(' ', 1)[0] From = From.replace('\"','') Name = Name.replace('\"','') if From not in session['seen']: Contact = { 'id': len(session['seen']), 'date': Date, 'name': Name, 'address': Address, 'snippet': results['snippet'], 'unread': Unread } session['contacts'].append(Contact) session['seen'].append(From) message_ids = gmail_service.users().messages().list(userId='me', maxResults=50, labelIds='INBOX', pageToken=session['contactsNextPageToken']).execute() batchContacts = BatchHttpRequest(callback=contactscallbackfunc) #return message_ids['messages'][0]['id'] session['contactsNextPageToken'] = message_ids['nextPageToken'] for msg_id in message_ids['messages']: batchContacts.add(gmail_service.users().messages().get(userId='me', id=msg_id['id'], format='metadata', metadataHeaders=['from', 'date'])) batchContacts.execute() #return "test" #def mailscallbackfunc(result, results, moreresults): # if 'UNREAD' in results['labelIds']: # Unread = 'true' # else: # Unread = 'false' # #batchMails = BatchHttpRequest(callback=mailscallbackfunc) #for contact in session['seen']: # batchContacts.add(gmail_service.users().messages().get(userId='me', # id=msg_id['id'], format='metadata', # metadataHeaders=['from', 'date'])) #batchMails.execute() if pagenr - 1 == 0: start = 0 end = 10 else: start = (pagenr - 1) * 10 -1 end = pagenr * 10 -1 response = {'nextPageToken': session['contactsNextPageToken'], 'contacts': session['contacts'][start:end]} js = json.dumps(response) resp = Response(js, status=200, mimetype='application/json') return resp