예제 #1
0
    def data(self):
        if self.size == 0:
            log.warning('Block size is 0')
            return ''
        elif hasattr(self, '_data'):
            # On initial download we temporarily store data in memory
            value = self._data
        else:
            value = get_from_blockstore(self.data_sha256)

        if value is None:
            log.warning(
                "Couldn't find data on S3 for block with hash {}".format(
                    self.data_sha256))

            from inbox.models.block import Block
            if isinstance(self, Block):
                if self.parts:
                    # This block is an attachment of a message that was
                    # accidentially deleted. We will attempt to fetch the raw
                    # message and parse out the needed attachment.

                    message = self.parts[0].message  # only grab one
                    raw_mime = get_from_blockstore(message.data_sha256)

                    if raw_mime is None:
                        log.error("Don't have raw message for hash {}".format(
                            message.data_sha256))
                        return None

                    parsed = mime.from_string(raw_mime)
                    if parsed is not None:
                        for mimepart in parsed.walk(
                                with_self=parsed.content_type.is_singlepart()):
                            if mimepart.content_type.is_multipart():
                                continue  # TODO should we store relations?

                            data = mimepart.body

                            if isinstance(data, unicode):
                                data = data.encode('utf-8', 'strict')

                            # Found it!
                            if sha256(data).hexdigest() == self.data_sha256:
                                log.info('Found subpart with hash {}'.format(
                                    self.data_sha256))
                                save_to_blockstore(self.data_sha256, data)
                                return data

            log.error('No data returned!')
            return value

        assert self.data_sha256 == sha256(value).hexdigest(), \
            "Returned data doesn't match stored hash!"
        return value
예제 #2
0
    def data(self):
        if self.size == 0:
            log.warning("Block size is 0")
            return ""
        elif hasattr(self, "_data"):
            # On initial download we temporarily store data in memory
            value = self._data
        else:
            value = get_from_blockstore(self.data_sha256)

        if value is None:
            log.warning("Couldn't find data on S3 for block with hash {}".format(self.data_sha256))

            from inbox.models.block import Block

            if isinstance(self, Block):
                if self.parts:
                    # This block is an attachment of a message that was
                    # accidentially deleted. We will attempt to fetch the raw
                    # message and parse out the needed attachment.

                    message = self.parts[0].message  # only grab one
                    raw_mime = get_from_blockstore(message.data_sha256)

                    if raw_mime is None:
                        log.error("Don't have raw message for hash {}".format(message.data_sha256))
                        return None

                    parsed = mime.from_string(raw_mime)
                    if parsed is not None:
                        for mimepart in parsed.walk(with_self=parsed.content_type.is_singlepart()):
                            if mimepart.content_type.is_multipart():
                                continue  # TODO should we store relations?

                            data = mimepart.body

                            if isinstance(data, unicode):
                                data = data.encode("utf-8", "strict")

                            # Found it!
                            if sha256(data).hexdigest() == self.data_sha256:
                                log.info("Found subpart with hash {}".format(self.data_sha256))
                                save_to_blockstore(self.data_sha256, data)
                                return data

            log.error("No data returned!")
            return value

        assert self.data_sha256 == sha256(value).hexdigest(), "Returned data doesn't match stored hash!"
        return value
예제 #3
0
def test_store_full_body_on_parse_error(default_account,
                                        mime_message_with_bad_date):
    received_date = None
    m = Message.create_from_synced(default_account, 139219, '[Gmail]/All Mail',
                                   received_date,
                                   mime_message_with_bad_date.to_string())
    assert get_from_blockstore(m.data_sha256)
예제 #4
0
def test_rfc822_format(stub_message_from_raw, api_client, mime_message):
    """ Test the API response to retreive raw message contents """
    full_path = '/messages/{}'.format(stub_message_from_raw.public_id)

    results = api_client.get_raw(full_path,
                                 headers={'Accept': 'message/rfc822'})
    assert results.data == get_from_blockstore(stub_message_from_raw.data_sha256)
예제 #5
0
def test_store_full_body_on_parse_error(
        default_account, mime_message_with_bad_date):
    received_date = None
    m = Message.create_from_synced(default_account, 139219, '[Gmail]/All Mail',
                                   received_date,
                                   mime_message_with_bad_date.to_string())
    assert get_from_blockstore(m.data_sha256)
예제 #6
0
def test_rfc822_format(stub_message_from_raw, api_client, mime_message):
    """ Test the API response to retreive raw message contents """
    full_path = '/messages/{}'.format(stub_message_from_raw.public_id)

    results = api_client.get_raw(full_path,
                                 headers={'Accept': 'message/rfc822'})
    assert results.data == get_from_blockstore(
        stub_message_from_raw.data_sha256)
예제 #7
0
    def send_raw(self, msg):
        recipient_emails = [email for name, email in itertools.chain(
            msg.bcc_addr, msg.cc_addr, msg.to_addr)]

        raw_message = get_from_blockstore(msg.data_sha256)
        mime_body = _substitute_bcc(raw_message)
        self._send(recipient_emails, mime_body)

        # Sent to all successfully
        sender_email = msg.from_addr[0][1]
        self.log.info('Sending successful', sender=sender_email,
                      recipients=recipient_emails)
예제 #8
0
    def send_raw(self, msg):
        recipient_emails = [email for name, email in itertools.chain(
            msg.bcc_addr, msg.cc_addr, msg.to_addr)]

        raw_message = get_from_blockstore(msg.data_sha256)
        mime_body = re.sub(r'Bcc: [^\r\n]*\r\n', '', raw_message)
        self._send(recipient_emails, mime_body)

        # Sent to all successfully
        sender_email = msg.from_addr[0][1]
        self.log.info('Sending successful', sender=sender_email,
                      recipients=recipient_emails)
예제 #9
0
    def data(self):
        if self.size == 0:
            log.warning('Block size is 0')
            return ''
        elif hasattr(self, '_data'):
            # On initial download we temporarily store data in memory
            value = self._data
        else:
            value = get_from_blockstore(self.data_sha256)

        if value is None:
            log.error('No data returned!')
            return value

        assert self.data_sha256 == sha256(value).hexdigest(), \
            "Returned data doesn't match stored hash!"
        return value
예제 #10
0
    def data(self):
        if self.size == 0:
            log.warning('Block size is 0')
            # Placeholder for "empty bytes". If this doesn't work as intended,
            # it will trigger the hash assertion later.
            value = ''
        elif hasattr(self, '_data'):
            # On initial download we temporarily store data in memory
            value = self._data
        else:
            value = get_from_blockstore(self.data_sha256)

        if value is None:
            log.error('No data returned!')
            return value

        assert self.data_sha256 == sha256(value).hexdigest(), \
            "Returned data doesn't match stored hash!"
        return value
예제 #11
0
파일: roles.py 프로젝트: nylas/sync-engine
    def data(self):
        if self.size == 0:
            log.warning('Block size is 0')
            return ''
        elif hasattr(self, '_data'):
            # On initial download we temporarily store data in memory
            value = self._data
        else:
            value = blockstore.get_from_blockstore(self.data_sha256)

        if value is None:
            log.warning("Couldn't find data on S3 for block",
                        sha_hash=self.data_sha256)

            from inbox.models.block import Block
            if isinstance(self, Block):
                if self.parts:
                    # This block is an attachment of a message that was
                    # deleted. We will attempt to fetch the raw
                    # message and parse out the needed attachment.

                    message = self.parts[0].message  # only grab one
                    account = message.namespace.account

                    statsd_string = 'api.direct_fetching.{}.{}'.format(
                        account.provider, account.id)

                    # Try to fetch the message from S3 first.
                    with statsd_client.timer('{}.blockstore_latency'.format(
                                             statsd_string)):
                        raw_mime = blockstore.get_from_blockstore(message.data_sha256)

                    # If it's not there, get it from the provider.
                    if raw_mime is None:
                        statsd_client.incr('{}.cache_misses'.format(statsd_string))

                        with statsd_client.timer('{}.provider_latency'.format(
                                                 statsd_string)):
                            raw_mime = get_raw_from_provider(message)

                        msg_sha256 = sha256(raw_mime).hexdigest()

                        # Cache the raw message in the blockstore so that
                        # we don't have to fetch it over and over.

                        with statsd_client.timer('{}.blockstore_save_latency'.format(
                                                 statsd_string)):
                            blockstore.save_to_blockstore(msg_sha256, raw_mime)
                    else:
                        # We found it in the blockstore --- report this.
                        statsd_client.incr('{}.cache_hits'.format(statsd_string))

                    # If we couldn't find it there, give up.
                    if raw_mime is None:
                        log.error("Don't have raw message for hash {}"
                                  .format(message.data_sha256))
                        return None

                    parsed = mime.from_string(raw_mime)
                    if parsed is not None:
                        for mimepart in parsed.walk(
                                with_self=parsed.content_type.is_singlepart()):
                            if mimepart.content_type.is_multipart():
                                continue  # TODO should we store relations?

                            data = mimepart.body

                            if isinstance(data, unicode):
                                data = data.encode('utf-8', 'strict')

                            if data is None:
                                continue

                            # Found it!
                            if sha256(data).hexdigest() == self.data_sha256:
                                log.info('Found subpart with hash {}'.format(
                                    self.data_sha256))

                                with statsd_client.timer('{}.blockstore_save_latency'.format(
                                                         statsd_string)):
                                    blockstore.save_to_blockstore(self.data_sha256, data)
                                    return data
                    log.error("Couldn't find the attachment in the raw message", message_id=message.id)

            log.error('No data returned!')
            return value

        assert self.data_sha256 == sha256(value).hexdigest(), \
            "Returned data doesn't match stored hash!"
        return value
예제 #12
0
    def data(self):
        if self.size == 0:
            log.warning('Block size is 0')
            return ''
        elif hasattr(self, '_data'):
            # On initial download we temporarily store data in memory
            value = self._data
        else:
            value = blockstore.get_from_blockstore(self.data_sha256)

        if value is None:
            log.warning("Couldn't find data on S3 for block",
                        sha_hash=self.data_sha256)

            from inbox.models.block import Block
            if isinstance(self, Block):
                if self.parts:
                    # This block is an attachment of a message that was
                    # deleted. We will attempt to fetch the raw
                    # message and parse out the needed attachment.

                    message = self.parts[0].message  # only grab one
                    account = message.namespace.account

                    statsd_string = 'api.direct_fetching.{}.{}'.format(
                        account.provider, account.id)

                    # Try to fetch the message from S3 first.
                    with statsd_client.timer(
                            '{}.blockstore_latency'.format(statsd_string)):
                        raw_mime = blockstore.get_from_blockstore(
                            message.data_sha256)

                    # If it's not there, get it from the provider.
                    if raw_mime is None:
                        statsd_client.incr(
                            '{}.cache_misses'.format(statsd_string))

                        with statsd_client.timer(
                                '{}.provider_latency'.format(statsd_string)):
                            raw_mime = get_raw_from_provider(message)

                        msg_sha256 = sha256(raw_mime).hexdigest()

                        # Cache the raw message in the blockstore so that
                        # we don't have to fetch it over and over.

                        with statsd_client.timer(
                                '{}.blockstore_save_latency'.format(
                                    statsd_string)):
                            blockstore.save_to_blockstore(msg_sha256, raw_mime)
                    else:
                        # We found it in the blockstore --- report this.
                        statsd_client.incr(
                            '{}.cache_hits'.format(statsd_string))

                    # If we couldn't find it there, give up.
                    if raw_mime is None:
                        log.error("Don't have raw message for hash {}".format(
                            message.data_sha256))
                        return None

                    parsed = mime.from_string(raw_mime)
                    if parsed is not None:
                        for mimepart in parsed.walk(
                                with_self=parsed.content_type.is_singlepart()):
                            if mimepart.content_type.is_multipart():
                                continue  # TODO should we store relations?

                            data = mimepart.body

                            if isinstance(data, unicode):
                                data = data.encode('utf-8', 'strict')

                            if data is None:
                                continue

                            # Found it!
                            if sha256(data).hexdigest() == self.data_sha256:
                                log.info('Found subpart with hash {}'.format(
                                    self.data_sha256))

                                with statsd_client.timer(
                                        '{}.blockstore_save_latency'.format(
                                            statsd_string)):
                                    blockstore.save_to_blockstore(
                                        self.data_sha256, data)
                                    return data
                    log.error(
                        "Couldn't find the attachment in the raw message",
                        message_id=message.id)

            log.error('No data returned!')
            return value

        assert self.data_sha256 == sha256(value).hexdigest(), \
            "Returned data doesn't match stored hash!"
        return value