def _get_service(self, base_service: Resource,
service_type: ServiceType) -> Resource:
if service_type == ServiceType.LABELS:
return base_service.users().labels()
if service_type == ServiceType.MESSAGES:
return base_service.users().messages()
raise NotImplementedError(service_type)
def _gadmin_user_insert(service: Resource, message: Message, email: str,
fname: str, lname: str) -> None:
"""
指定したユーザーを追加する
:param service: Google API との接続
:param email: メールアドレス
:param fname: ユーザーのfirst name(insert時に使用)
:param fname: ユーザーのlast name(insert時に使用)
"""
# パスワードを生成する
password = _generate_password()
body = {
"primaryEmail": email,
"password": password,
"name": {
"givenName": fname.title(),
"familyName": lname.title(),
},
}
try:
service.users().insert(body=body).execute()
botsend(message, f"ユーザー `{email}` を追加しました")
# password をユーザーにDMで伝える
_send_password_on_dm(message, email, password)
except HttpError as e:
botsend(message, f"ユーザーの追加に失敗しました\n`{e}`")
def download_all_message_specs(service: Resource,
query) -> List[Dict[str, str]]:
users = service.users()
message_specs = []
response = users.messages().list(userId="me", q=query).execute()
if "messages" in response:
message_specs.extend(response["messages"])
while "nextPageToken" in response:
page_token = response["nextPageToken"]
response = (service.users().messages().list(userId="me",
pageToken=page_token,
q=query).execute())
message_specs.extend(response["messages"])
return message_specs
def _gadmin_alias_delete(service: Resource, message: Message, email: str,
alias: str) -> None:
"""
指定したユーザーからエイリアスを削除する
:param service: Google API との接続
:param email: 追加対象のメールアドレス
:param alias: エイリアスのメールアドレス
"""
try:
service.users().aliases().delete(userKey=email, alias=alias).execute()
botsend(message, f"`{email}` からエイリアス `{alias}` を削除しました")
except HttpError as e:
botsend(message, f"エイリアスの削除に失敗しました\n`{e}`")
def get_message_attachments(service: discovery.Resource,
message_id: str) -> Dict[str, List[str]]:
"""
Return a list of attachment-ids in emails referenced by message_id.
form of response of maybe_message in ideal case, just keys:
maybe_message.keys()
> dict_keys(['id', 'threadId', 'labelIds', 'snippet', 'payload', 'sizeEstimate', 'historyId', 'internalDate'])
maybe_message["payload"].keys()
> dict_keys(['partId', 'mimeType', 'filename', 'headers', 'body', 'parts'])
maybe_message["payload"]["parts"][-1].keys()
> dict_keys(['partId', 'mimeType', 'filename', 'headers', 'body'])
maybe_message["payload"]["parts"][-1]["body"].keys()
> dict_keys(['attachmentId', 'size'])
"""
maybe_message = service.users().messages().get(
userId="me",
id=message_id) \
.execute()
message_parts = maybe_message.get("payload", {}).get("parts", [])
return {
message_id: [
part["body"]["attachmentId"] for part in message_parts
if part.get("body", {}).get("attachmentId")
]
}
def __send_message(service: Resource, user_id: str, message: str) -> str:
try:
message = (service.users().messages().send(userId=user_id,
body=message).execute())
print('Message Id: %s' % message['id'])
return message
except mail_errors.HttpError as error:
print('An error occurred: %s' % error)
def _gadmin_alias_insert(service: Resource, message: Message, email: str,
alias: str) -> None:
"""
指定したユーザーにエイリアスを追加する
:param service: Google API との接続
:param email: 追加対象のメールアドレス
:param alias: エイリアスのメールアドレス
"""
body = {
"alias": alias,
}
try:
service.users().aliases().insert(userKey=email, body=body).execute()
botsend(message, f"`{email}` にエイリアス `{alias}` を追加しました")
except HttpError as e:
botsend(message, f"エイリアスの追加に失敗しました\n`{e}`")
def get_all_messages(service: Resource, next_page: str, q: str,
limit: int) -> list:
"""Looping through messages and nextPageToken until end of limit or batch of message is over"""
messages = []
while (next_page and len(messages) < limit):
results = service.users().messages().list(
userId='me', q=q, pageToken=next_page).execute()
messages.extend(results.get('messages', []))
next_page = results.get('nextPageToken')
return messages
def _gadmin_user_update(service: Resource, message: Message, email: str,
suspended: bool) -> None:
"""
ユーザーの情報を更新する
:param service: Google API との接続
:param email: メールアドレス
:param suspended: ユーザーの状態、True or False
"""
body = {
"suspended": suspended,
}
try:
service.users().update(userKey=email, body=body).execute()
if suspended:
botsend(message, f"ユーザー `{email}` を停止しました")
else:
botsend(message, f"ユーザー `{email}` を再開しました")
except HttpError as e:
botsend(message, f"ユーザー情報の更新に失敗しました\n`{e}`")
def get_all_users(admin: Resource) -> List[Dict]:
"""
Return list of Google Users in your organization
Returns empty list if we are unable to enumerate the groups for any reasons
https://developers.google.com/admin-sdk/directory/v1/guides/manage-users
:param admin: apiclient discovery resource object
see
:return: List of Google users in domain
see https://developers.google.com/admin-sdk/directory/v1/guides/manage-users#get_all_domain_users
"""
request = admin.users().list(customer='my_customer',
maxResults=500,
orderBy='email')
response_objects = []
while request is not None:
resp = request.execute(num_retries=GOOGLE_API_NUM_RETRIES)
response_objects.append(resp)
request = admin.users().list_next(request, resp)
return response_objects
def send_message(service: Resource, message: Message) -> Union[Any, None]:
raw_message = create_message(message)
try:
ret = (service.users().messages().send(userId=message["sender"],
body=raw_message).execute())
logging.info("Sent message to %s (Id: %s)" %
(message["to"], ret["id"]))
return ret
except Exception:
logging.exception("Unable to send message to %s" + message["to"])
return None
def _gadmin_user_password_reset(service: Resource, message: Message,
email: str) -> None:
"""
ユーザーのパスワードをリセットする
:param service: Google API との接続
:param email: メールアドレス
"""
# パスワードを生成する
password = _generate_password()
body = {
"password": password,
}
try:
service.users().update(userKey=email, body=body).execute()
botsend(message, f"ユーザー `{email}` のパスワードをリセットしました")
# password を実行ユーザーにDMで伝える
_send_password_on_dm(message, email, password)
except HttpError as e:
botsend(message, f"ユーザーパスワードのリセットに失敗しました\n`{e}`")
def send(gmail: Resource, recipients: str, text: str,
image_path: Path) -> None:
"""
Sends an email with text and an image.
Args:
gmail : The handle to the API
recipients : A string of space-separated emails
text : The text to attach in the email
image_path : The path to the image to attach to the email
"""
# Send this email in monospace font and using html encoding
html_text = "<font face='Courier New, Courier, monospace'><pre>" + text + "</pre></font>"
html_text = html_text.replace("\n", "<br>")
message = MIMEMultipart()
message["to"] = recipients
# @TODO: Move this hardcoded email out
message["from"] = "*****@*****.**"
message["subject"] = f"{str(datetime.datetime.now().date())} Covid Metrics"
message.attach(MIMEText(html_text, "html"))
# Attach the image
content_type, _ = mimetypes.guess_type(str(image_path))
if not content_type or content_type != "image/jpeg":
raise RuntimeError(f"{image_path} is the wrong file type")
with image_path.open("rb") as f:
msg = MIMEImage(f.read(), _sub_type="jpeg")
msg.add_header("Content-Disposition",
"attachment",
filename=str(image_path))
message.attach(msg)
# @TODO: See if the string can be sent directly
body: Dict[str, str] = {
"raw": base64.urlsafe_b64encode(message.as_bytes()).decode()
}
# Send the email
gmail.users().messages().send(userId="*****@*****.**",
body=body).execute()
def _gadmin_user_delete(service: Resource, message: Message,
email: str) -> None:
"""
指定したユーザーを削除する
:param service: Google API との接続
:param email: メールアドレス
"""
try:
# 停止中のユーザーのみ削除対象とする
result = service.users().get(userKey=email).execute()
if not result["suspended"]:
botsend(
message,
"ユーザーはアクティブなため削除できません\n"
f"`$gadmin user suspend {email}` でユーザーを停止してから削除してください",
)
else:
service.users().delete(userKey=email).execute()
botsend(message, f"ユーザー `{email}` を削除しました")
except HttpError as e:
botsend(message, f"ユーザーの削除に失敗しました\n`{e}`")
def handle_messages_details(message_list: dict, gmail_client: Resource,
user_email: str) -> None:
batch = BatchHttpRequest()
for i in message_list["messages"]:
callback = partial(find_and_save_flight_bookings,
service=gmail_client,
user_id=user_email)
batch.add(
gmail_client.users().messages().get(userId=user_email, id=i["id"]),
callback=callback,
)
batch.execute()
def send_message(service: Resource, message: Dict[str, str]) -> Dict[str, str]:
"""
Sends an email message.
:param service: The authenticated Gmail service object.
:param message: A base64url encoded email message object.
:return: The sent message.
"""
try:
message = service.users().messages().send(userId=ME,
body=message).execute()
return message
except HttpError as error:
print(f'An error occurred: {error}')
def get_messages(service: discovery.Resource, query: str, max_results: int = 400) -> List[Dict]:
"""Gets the email messages that match the query
Args
----
service: Authorized Gmail API service instance.
query: A query string
max_results: The maximum number of results to load
Returns
-------
A list of message dictionaries
"""
response = service.users().messages().list(
userId="me", q=query, maxResults=max_results).execute()
messages = response.get("messages")
while 'nextPageToken' in response and len(messages) < max_results:
page_token = response['nextPageToken']
response = service.users().messages().list(
userId="me", q=query, pageToken=page_token).execute()
messages.extend(response['messages'])
return messages
def get_emails(service: Resource, query=None) -> List[Email]:
"""Get all the emails from connectd Gmail account.
If you specify query, will filter by query.
See https://support.google.com/mail/answer/7190?hl=en for example queries.
"""
users = service.users()
message_specs = download_all_message_specs(service, query)
message_ids = [ms["id"] for ms in message_specs]
emails: List[Email] = []
user_id = "me"
for message_id in message_ids:
message = users.messages().get(id=message_id, userId=user_id).execute()
payload = message["payload"]
headers = payload["headers"]
from_address = get_from_headers(headers, "From")
to_address = get_from_headers(headers, "To")
raw_cc_addresses = get_from_headers(headers, "Cc")
if raw_cc_addresses:
cc_addresses = tuple(raw_cc_addresses.split(","))
else:
cc_addresses = ()
subject = get_from_headers(headers, "Subject")
try:
date_receieved = dateutil.parser.parse(
get_from_headers(headers, "Date"))
except:
date_receieved = None
attachments = get_attachments_from_message(message,
service,
user_id=user_id)
email = Email(
from_address=from_address,
to_address=to_address,
cc_addresses=cc_addresses,
subject=subject,
date_receieved=date_receieved,
snippet=html.unescape(message["snippet"]),
raw_message=message,
gmail_message_id=message["id"],
attachments=attachments,
)
emails.append(email)
return emails
def GetMimeMessage(service: discovery.Resource, msg_id: str) -> email.message.Message:
"""Get a Message and use it to create a MIME Message.
Args
----
service: Authorized Gmail API service instance.
msg_id: The ID of the Message required.
Returns
-------
A MIME Message, consisting of data from Message.
"""
message = service.users().messages().get(userId="me", id=msg_id,
format='raw').execute()
msg_str = base64.urlsafe_b64decode(
message['raw'].encode('ASCII')).decode("utf-8")
mime_msg = email.message_from_string(msg_str)
return mime_msg
def get_mail_ids_with_data(service: discovery.Resource,
query: str) -> List[str]:
"""
Get emails which satisfy the gmail query.
Refer to readme to get an example of query value.
Shape of maybe_emails in an ideal scenario, the `key` messages
is an empty array if there is nothing that matches the query.
{
"messages": [{
"id": str,
"threadId": str
}],
"resultSizeEstimate": int
}
"""
maybe_emails = service.users().messages().list(userId="me",
q=query).execute()
emails = maybe_emails.get("messages")
L.info("Fetched a list of emails with relevant query = %s.", query)
return [email["id"] for email in emails]
def find_and_save_flight_bookings(request: str, message: dict,
exception: Exception, service: Resource,
user_id: str) -> None:
storage = get_storage_client()
datastore = get_datastore_client()
bucket = storage.bucket(settings.BUCKET_NAME)
parts = [message["payload"]]
files = list()
while parts:
try:
part = parts.pop()
if part.get("parts"):
parts.extend(part["parts"])
if part.get("filename") and "pdf" in part["filename"]:
if "attachmentId" in part["body"]:
attachment = (service.users().messages().attachments().get(
userId=user_id,
messageId=message["id"],
id=part["body"]["attachmentId"],
).execute())
file_data = base64.urlsafe_b64decode(
attachment["data"].encode("UTF-8"))
blob = bucket.blob(f"{user_id}/{part['filename']}")
blob.upload_from_string(file_data,
content_type="application/pdf")
files.append(blob.public_url)
except Exception as e:
logger.error(f"Can't upload file from email#{user_id} box: {e}")
if files:
sender = get_sender(message)
with datastore.context():
EmailMessage.make_record(sender, message["id"], files)
def attachments_as_df(
service: discovery.Resource,
attachment_data: Dict[str, List[str]]) -> Tuple[List[DataFrame], bool]:
"""
Save attachment data (a b64 encoded string) to csv.
maybe_attachment.keys()
> dict_keys(['size', 'data'])
maybe_attachment["data"] is the downloadable content.
"""
data_frames = []
process = psutil.Process(os.getpgid())
missing_files_warn = False
L.info("Load all attachments as df into memory.")
for email_id, attachment_ids in tqdm(attachment_data.items()):
for attachment_id in attachment_ids:
maybe_attachment = service.users().messages().attachments().get(
userId="me", messageId=email_id, id=attachment_id).execute()
if "data" not in maybe_attachment:
continue
df = read_xlsx_as_df(maybe_attachment["data"])
L.info("%0.2f", process.memory_percent())
if process.memory_percent() > MEMORY_CAP_CRIT:
L.warning(
"[CRITICAL]: Memory capacity | Saving files on disk instead."
)
missing_files_warn = True
date_string = datetime.now().strftime('%d-%M-%Y_%HH-%MM')
file_name = f"{date_string}.csv"
df.to_csv(file_name), missing_files_warn
else:
data_frames.append(df)
L.info("Number of data_frames loaded into memory %d", len(data_frames))
return data_frames, missing_files_warn
def __init__(self, service: Resource):
self._service = service.users().messages()
