def _update(self, provider): config = provider.get('config', {}) server = config.get('server', '') port = int(config.get('port', 993)) try: imap = imaplib.IMAP4_SSL(host=server, port=port) try: imap.login(config.get('user', None), config.get('password', None)) except imaplib.IMAP4.error: raise IngestEmailError.emailLoginError(imaplib.IMAP4.error, provider) rv, data = imap.select(config.get('mailbox', None), readonly=False) if rv == 'OK': rv, data = imap.search(None, config.get('filter', '(UNSEEN)')) if rv == 'OK': new_items = [] for num in data[0].split(): rv, data = imap.fetch(num, '(RFC822)') if rv == 'OK': try: parser = self.get_feed_parser(provider, data) new_items.append(parser.parse(data, provider)) rv, data = imap.store(num, '+FLAGS', '\\Seen') except IngestEmailError: continue imap.close() imap.logout() except IngestEmailError: raise except Exception as ex: raise IngestEmailError.emailError(ex, provider) return new_items
def _update(self, provider, update, test=False): config = provider.get('config', {}) server = config.get('server', '') port = int(config.get('port', 993)) new_items = [] try: try: socket.setdefaulttimeout(app.config.get('EMAIL_TIMEOUT', 10)) imap = imaplib.IMAP4_SSL(host=server, port=port) except (socket.gaierror, OSError) as e: raise IngestEmailError.emailHostError(exception=e, provider=provider) try: imap.login(config.get('user', None), config.get('password', None)) except imaplib.IMAP4.error: raise IngestEmailError.emailLoginError(imaplib.IMAP4.error, provider) try: rv, data = imap.select(config.get('mailbox', None), readonly=False) if rv != 'OK': raise IngestEmailError.emailMailboxError() try: rv, data = imap.search(None, config.get('filter', '(UNSEEN)')) if rv != 'OK': raise IngestEmailError.emailFilterError() for num in data[0].split(): rv, data = imap.fetch(num, '(RFC822)') if rv == 'OK' and not test: try: parser = self.get_feed_parser(provider, data) new_items.append(parser.parse(data, provider)) rv, data = imap.store(num, '+FLAGS', '\\Seen') except IngestEmailError: continue finally: imap.close() finally: imap.logout() except IngestEmailError: raise except Exception as ex: raise IngestEmailError.emailError(ex, provider) return new_items
class EmailFeedingService(FeedingService): """ Feeding Service class which can read the article(s) from a configured mail box. """ NAME = 'email' ERRORS = [ IngestEmailError.emailError().get_error_description(), IngestEmailError.emailLoginError().get_error_description() ] label = 'Email' fields = [{ 'id': 'server', 'type': 'text', 'label': 'Email Server', 'placeholder': 'Email Server', 'required': True, 'errors': { 6003: 'Server not found.', 6002: 'Unexpected server response' } }, { 'id': 'port', 'type': 'text', 'label': 'Email Server Port', 'placeholder': 'Email Server Port', 'required': True, 'default': '993' }, { 'id': 'user', 'type': 'text', 'label': 'User', 'placeholder': 'User', 'required': True }, { 'id': 'password', 'type': 'password', 'label': 'Password', 'placeholder': 'Password', 'required': True, 'errors': { 6000: 'Authentication error.' } }, { 'id': 'mailbox', 'type': 'text', 'label': 'Mailbox', 'placeholder': 'Mailbox', 'required': True, 'errors': { 6004: 'Authentication error.' } }, { 'id': 'formatted', 'type': 'boolean', 'label': 'Formatted Email Parser', 'required': True }, { 'id': 'filter', 'type': 'text', 'label': 'Filter', 'placeholder': 'Filter', 'required': True }] parser_restricted_values = ['email_rfc822'] def _test(self, provider): self._update(provider, update=None, test=True) def _update(self, provider, update, test=False): config = provider.get('config', {}) server = config.get('server', '') port = int(config.get('port', 993)) new_items = [] try: try: socket.setdefaulttimeout(app.config.get('EMAIL_TIMEOUT', 10)) imap = imaplib.IMAP4_SSL(host=server, port=port) except (socket.gaierror, OSError) as e: raise IngestEmailError.emailHostError(exception=e) try: imap.login(config.get('user', None), config.get('password', None)) except imaplib.IMAP4.error: raise IngestEmailError.emailLoginError(imaplib.IMAP4.error, provider) try: rv, data = imap.select(config.get('mailbox', None), readonly=False) if rv != 'OK': raise IngestEmailError.emailMailboxError() try: rv, data = imap.search(None, config.get('filter', '(UNSEEN)')) if rv != 'OK': raise IngestEmailError.emailFilterError() for num in data[0].split(): rv, data = imap.fetch(num, '(RFC822)') if rv == 'OK' and not test: try: parser = self.get_feed_parser(provider, data) new_items.append(parser.parse(data, provider)) rv, data = imap.store(num, '+FLAGS', '\\Seen') except IngestEmailError: continue finally: imap.close() finally: imap.logout() except IngestEmailError: raise except Exception as ex: raise IngestEmailError.emailError(ex, provider) return new_items def prepare_href(self, href, mimetype=None): return url_for_media(href, mimetype)
class GMailFeedingService(EmailFeedingService): """ Feeding Service class which can read the article(s) from a configured mail box. """ NAME = "gmail" ERRORS = [ IngestEmailError.emailError().get_error_description(), IngestEmailError.emailLoginError().get_error_description(), ] label = "Gmail" fields = [ { "id": "email", "type": "text", "label": l_("email"), "readonly": True, "show_expression": "provider.config['email'] != null", }, { "id": "log_in_url", "type": "url_request", "label": l_("Log-in with GMail"), # provider._id != null provider has to be saved before trying to log in # provider.config['email'] == null do not display log-in button if logged-in already "show_expression": "provider._id != null && provider.config['email'] == null", }, { "id": "log_out_url", "type": "url_request", "label": l_("Log-out"), # provider.config['email'] != null only display log-out button if already logged in "show_expression": "provider.config['email'] != null", }, { "id": "mailbox", "type": "text", "label": l_("Mailbox"), "default_value": "INBOX", "placeholder": l_("Mailbox"), "required": True, "errors": { 6004: "Authentication error." }, }, { "id": "filter", "type": "text", "label": l_("Filter"), "placeholder": "Filter", "required": False }, ] @classmethod def init_app(cls, app): # we need to access config to set the URL, so we do it here field = next(f for f in cls.fields if f["id"] == "log_in_url") field["url"] = join(app.config["SERVER_URL"], "login", "google", "{PROVIDER_ID}") field = next(f for f in cls.fields if f["id"] == "log_out_url") field["url"] = join(app.config["SERVER_URL"], "logout", "google", "{PROVIDER_ID}") def _test(self, provider): self._update(provider, update=None, test=True) def authenticate(self, provider: dict, config: dict) -> imaplib.IMAP4_SSL: oauth2_token_service = superdesk.get_resource_service("oauth2_token") token = oauth2_token_service.find_one(req=None, _id=ObjectId(provider["_id"])) if token is None: raise IngestEmailError.notConfiguredError(ValueError( l_("You need to log in first")), provider=provider) imap = imaplib.IMAP4_SSL("imap.gmail.com") if token["expires_at"].timestamp() < time.time() + 600: logger.info("Refreshing token for {provider_name}".format( provider_name=provider["name"])) token = oauth.refresh_google_token(token["_id"]) auth_string = "user={email}\x01auth=Bearer {token}\x01\x01".format( email=token["email"], token=token["access_token"]) imap.authenticate("XOAUTH2", lambda __: auth_string.encode()) return imap def parse_extra(self, imap: imaplib.IMAP4_SSL, num: str, parsed_items: List[dict]) -> None: """Add GMail labels to parsed_items""" try: # we use GMail IMAP Extensions # https://developers.google.com/gmail/imap/imap-extensions#access_to_gmail_labels_x-gm-labels _, data = imap.fetch(num, "(X-GM-LABELS)") # it seems that there is nothing to help parsing in standard lib # thus we use some regex to get our labels data_bytes = data[0] if not isinstance(data_bytes, bytes): raise ValueError(f"Unexpected data type: {type(data_bytes)}") data_str = data_bytes.decode("utf-7") match_labels_str = RE_LABELS_STR.search(data_str) if match_labels_str is None: raise ValueError( f"Can't find the expected label string in data: {data_str:r}" ) labels_str = match_labels_str.group(1) labels = [(m.group("quoted") or m.group("unquoted")).replace('\\"', '"') for m in RE_LABEL.finditer(labels_str)] for parsed_item in parsed_items: subjects = parsed_item.setdefault("subject", []) for label in labels: subjects.append({ "name": label, "qcode": label, "scheme": "gmail_label" }) except Exception: logger.exception("Can't retrieve GMail labels")
class EmailFeedingService(FeedingService): """ Feeding Service class which can read the article(s) from a configured mail box. """ NAME = "email" ERRORS = [ IngestEmailError.emailError().get_error_description(), IngestEmailError.emailLoginError().get_error_description(), ] label = "Email" fields = [ { "id": "server", "type": "text", "label": l_("Email Server"), "placeholder": "Email Server", "required": True, "errors": { 6003: "Server not found.", 6002: "Unexpected server response" }, }, { "id": "port", "type": "text", "label": l_("Email Server Port"), "placeholder": "Email Server Port", "required": True, "default": "993", }, { "id": "user", "type": "text", "label": l_("User"), "placeholder": "User", "required": True }, { "id": "password", "type": "password", "label": l_("Password"), "placeholder": "Password", "required": True, "errors": { 6000: "Authentication error." }, }, { "id": "mailbox", "type": "text", "label": l_("Mailbox"), "placeholder": "Mailbox", "required": True, "errors": { 6004: "Authentication error." }, }, { "id": "formatted", "type": "boolean", "label": l_("Formatted Email Parser"), "required": True }, { "id": "filter", "type": "text", "label": l_("Filter"), "placeholder": "Filter", "required": False }, ] def _test(self, provider): self._update(provider, update=None, test=True) def authenticate(self, provider: dict, config: dict) -> imaplib.IMAP4_SSL: server = config.get("server", "") port = int(config.get("port", 993)) try: socket.setdefaulttimeout(app.config.get("EMAIL_TIMEOUT", 10)) imap = imaplib.IMAP4_SSL(host=server, port=port) except (socket.gaierror, OSError) as e: raise IngestEmailError.emailHostError(exception=e, provider=provider) try: imap.login(config.get("user", None), config.get("password", None)) except imaplib.IMAP4.error: raise IngestEmailError.emailLoginError(imaplib.IMAP4.error, provider) return imap def parse_extra(self, imap: imaplib.IMAP4_SSL, num: str, parsed_items: List[dict]) -> None: """Parse extra metadata This method is called after main parsing, and can be used by subclasses """ pass def _update(self, provider, update, test=False): config = provider.get("config", {}) new_items = [] try: imap = self.authenticate(provider, config) try: rv, data = imap.select(config.get("mailbox", None), readonly=False) if rv != "OK": raise IngestEmailError.emailMailboxError() try: # at least one criterion must be set # (see file:///usr/share/doc/python/html/library/imaplib.html#imaplib.IMAP4.search) rv, data = imap.search(None, config.get("filter") or "(UNSEEN)") if rv != "OK": raise IngestEmailError.emailFilterError() for num in data[0].split(): rv, data = imap.fetch(num, "(RFC822)") if rv == "OK" and not test: try: parser = self.get_feed_parser(provider, data) parsed_items = parser.parse(data, provider) self.parse_extra(imap, num, parsed_items) new_items.append(parsed_items) rv, data = imap.store(num, "+FLAGS", "\\Seen") except IngestEmailError: continue finally: imap.close() finally: imap.logout() except IngestEmailError: raise except Exception as ex: raise IngestEmailError.emailError(ex, provider) return new_items def prepare_href(self, href, mimetype=None): return url_for_media(href, mimetype)
class EmailFeedingService(FeedingService): """ Feeding Service class which can read the article(s) from a configured mail box. """ NAME = 'email' ERRORS = [IngestEmailError.emailError().get_error_description(), IngestEmailError.emailLoginError().get_error_description()] label = 'Email' def _test(self, provider): self._update(provider, update=None, test=True) def _update(self, provider, update, test=False): config = provider.get('config', {}) server = config.get('server', '') port = int(config.get('port', 993)) new_items = [] try: try: socket.setdefaulttimeout(app.config.get('EMAIL_TIMEOUT', 10)) imap = imaplib.IMAP4_SSL(host=server, port=port) except (socket.gaierror, OSError) as e: raise IngestEmailError.emailHostError(exception=e) try: imap.login(config.get('user', None), config.get('password', None)) except imaplib.IMAP4.error: raise IngestEmailError.emailLoginError(imaplib.IMAP4.error, provider) try: rv, data = imap.select(config.get('mailbox', None), readonly=False) if rv != 'OK': raise IngestEmailError.emailMailboxError() try: rv, data = imap.search(None, config.get('filter', '(UNSEEN)')) if rv != 'OK': raise IngestEmailError.emailFilterError() for num in data[0].split(): rv, data = imap.fetch(num, '(RFC822)') if rv == 'OK' and not test: try: parser = self.get_feed_parser(provider, data) new_items.append(parser.parse(data, provider)) rv, data = imap.store(num, '+FLAGS', '\\Seen') except IngestEmailError: continue finally: imap.close() finally: imap.logout() except IngestEmailError: raise except Exception as ex: raise IngestEmailError.emailError(ex, provider) return new_items def prepare_href(self, href, mimetype=None): return url_for_media(href, mimetype)
# # For the full copyright and license information, please see the # AUTHORS and LICENSE files distributed with this source code, or # at https://www.sourcefabric.org/superdesk/license import imaplib from .ingest_service import IngestService from superdesk.io import register_provider from superdesk.upload import url_for_media from superdesk.errors import IngestEmailError from superdesk.io.rfc822 import rfc822Parser PROVIDER = 'email' errors = [IngestEmailError.emailError().get_error_description(), IngestEmailError.emailLoginError().get_error_description()] class EmailReaderService(IngestService): def __init__(self): self.parser = rfc822Parser() def _update(self, provider): config = provider.get('config', {}) server = config.get('server', '') port = int(config.get('port', 993)) try: imap = imaplib.IMAP4_SSL(host=server, port=port) try:
class GMailFeedingService(EmailFeedingService): """ Feeding Service class which can read the article(s) from a configured mail box. """ NAME = "gmail" ERRORS = [ IngestEmailError.emailError().get_error_description(), IngestEmailError.emailLoginError().get_error_description(), ] label = "Gmail" fields = [ { "id": "email", "type": "text", "label": l_("email"), "readonly": True, "show_expression": "provider.config['email'] != null", }, { "id": "log_in_url", "type": "url_request", "label": l_("Log-in with GMail"), # provider._id != null provider has to be saved before trying to log in # provider.config['email'] == null do not display log-in button if logged-in already "show_expression": "provider._id != null && provider.config['email'] == null", }, { "id": "log_out_url", "type": "url_request", "label": l_("Log-out"), # provider.config['email'] != null only display log-out button if already logged in "show_expression": "provider.config['email'] != null", }, { "id": "mailbox", "type": "text", "label": l_("Mailbox"), "default_value": "INBOX", "placeholder": l_("Mailbox"), "required": True, "errors": { 6004: "Authentication error." }, }, { "id": "filter", "type": "text", "label": l_("Filter"), "placeholder": "Filter", "required": False }, ] @classmethod def init_app(cls, app): # we need to access config to set the URL, so we do it here field = next(f for f in cls.fields if f["id"] == "log_in_url") field["url"] = join(app.config["SERVER_URL"], "login", "google", "{PROVIDER_ID}") field = next(f for f in cls.fields if f["id"] == "log_out_url") field["url"] = join(app.config["SERVER_URL"], "logout", "google", "{PROVIDER_ID}") def _test(self, provider): self._update(provider, update=None, test=True) def authenticate(self, provider: dict, config: dict) -> imaplib.IMAP4_SSL: oauth2_token_service = superdesk.get_resource_service("oauth2_token") token = oauth2_token_service.find_one(req=None, _id=ObjectId(provider["_id"])) if token is None: raise IngestEmailError.notConfiguredError(ValueError( l_("You need to log in first")), provider=provider) imap = imaplib.IMAP4_SSL("imap.gmail.com") if token["expires_at"].timestamp() < time.time() + 600: logger.info("Refreshing token for {provider_name}".format( provider_name=provider["name"])) token = oauth.refresh_google_token(token["_id"]) auth_string = "user={email}\x01auth=Bearer {token}\x01\x01".format( email=token["email"], token=token["access_token"]) imap.authenticate("XOAUTH2", lambda __: auth_string.encode()) return imap