def data(self): if self.size == 0: log.warning('Block size is 0') return '' elif hasattr(self, '_data'): # On initial download we temporarily store data in memory value = self._data else: value = get_from_blockstore(self.data_sha256) if value is None: log.warning( "Couldn't find data on S3 for block with hash {}".format( self.data_sha256)) from inbox.models.block import Block if isinstance(self, Block): if self.parts: # This block is an attachment of a message that was # accidentially deleted. We will attempt to fetch the raw # message and parse out the needed attachment. message = self.parts[0].message # only grab one raw_mime = get_from_blockstore(message.data_sha256) if raw_mime is None: log.error("Don't have raw message for hash {}".format( message.data_sha256)) return None parsed = mime.from_string(raw_mime) if parsed is not None: for mimepart in parsed.walk( with_self=parsed.content_type.is_singlepart()): if mimepart.content_type.is_multipart(): continue # TODO should we store relations? data = mimepart.body if isinstance(data, unicode): data = data.encode('utf-8', 'strict') # Found it! if sha256(data).hexdigest() == self.data_sha256: log.info('Found subpart with hash {}'.format( self.data_sha256)) save_to_blockstore(self.data_sha256, data) return data log.error('No data returned!') return value assert self.data_sha256 == sha256(value).hexdigest(), \ "Returned data doesn't match stored hash!" return value
def data(self): if self.size == 0: log.warning("Block size is 0") return "" elif hasattr(self, "_data"): # On initial download we temporarily store data in memory value = self._data else: value = get_from_blockstore(self.data_sha256) if value is None: log.warning("Couldn't find data on S3 for block with hash {}".format(self.data_sha256)) from inbox.models.block import Block if isinstance(self, Block): if self.parts: # This block is an attachment of a message that was # accidentially deleted. We will attempt to fetch the raw # message and parse out the needed attachment. message = self.parts[0].message # only grab one raw_mime = get_from_blockstore(message.data_sha256) if raw_mime is None: log.error("Don't have raw message for hash {}".format(message.data_sha256)) return None parsed = mime.from_string(raw_mime) if parsed is not None: for mimepart in parsed.walk(with_self=parsed.content_type.is_singlepart()): if mimepart.content_type.is_multipart(): continue # TODO should we store relations? data = mimepart.body if isinstance(data, unicode): data = data.encode("utf-8", "strict") # Found it! if sha256(data).hexdigest() == self.data_sha256: log.info("Found subpart with hash {}".format(self.data_sha256)) save_to_blockstore(self.data_sha256, data) return data log.error("No data returned!") return value assert self.data_sha256 == sha256(value).hexdigest(), "Returned data doesn't match stored hash!" return value
def test_store_full_body_on_parse_error(default_account, mime_message_with_bad_date): received_date = None m = Message.create_from_synced(default_account, 139219, '[Gmail]/All Mail', received_date, mime_message_with_bad_date.to_string()) assert get_from_blockstore(m.data_sha256)
def test_rfc822_format(stub_message_from_raw, api_client, mime_message): """ Test the API response to retreive raw message contents """ full_path = '/messages/{}'.format(stub_message_from_raw.public_id) results = api_client.get_raw(full_path, headers={'Accept': 'message/rfc822'}) assert results.data == get_from_blockstore(stub_message_from_raw.data_sha256)
def test_store_full_body_on_parse_error( default_account, mime_message_with_bad_date): received_date = None m = Message.create_from_synced(default_account, 139219, '[Gmail]/All Mail', received_date, mime_message_with_bad_date.to_string()) assert get_from_blockstore(m.data_sha256)
def test_rfc822_format(stub_message_from_raw, api_client, mime_message): """ Test the API response to retreive raw message contents """ full_path = '/messages/{}'.format(stub_message_from_raw.public_id) results = api_client.get_raw(full_path, headers={'Accept': 'message/rfc822'}) assert results.data == get_from_blockstore( stub_message_from_raw.data_sha256)
def send_raw(self, msg): recipient_emails = [email for name, email in itertools.chain( msg.bcc_addr, msg.cc_addr, msg.to_addr)] raw_message = get_from_blockstore(msg.data_sha256) mime_body = _substitute_bcc(raw_message) self._send(recipient_emails, mime_body) # Sent to all successfully sender_email = msg.from_addr[0][1] self.log.info('Sending successful', sender=sender_email, recipients=recipient_emails)
def send_raw(self, msg): recipient_emails = [email for name, email in itertools.chain( msg.bcc_addr, msg.cc_addr, msg.to_addr)] raw_message = get_from_blockstore(msg.data_sha256) mime_body = re.sub(r'Bcc: [^\r\n]*\r\n', '', raw_message) self._send(recipient_emails, mime_body) # Sent to all successfully sender_email = msg.from_addr[0][1] self.log.info('Sending successful', sender=sender_email, recipients=recipient_emails)
def data(self): if self.size == 0: log.warning('Block size is 0') return '' elif hasattr(self, '_data'): # On initial download we temporarily store data in memory value = self._data else: value = get_from_blockstore(self.data_sha256) if value is None: log.error('No data returned!') return value assert self.data_sha256 == sha256(value).hexdigest(), \ "Returned data doesn't match stored hash!" return value
def data(self): if self.size == 0: log.warning('Block size is 0') # Placeholder for "empty bytes". If this doesn't work as intended, # it will trigger the hash assertion later. value = '' elif hasattr(self, '_data'): # On initial download we temporarily store data in memory value = self._data else: value = get_from_blockstore(self.data_sha256) if value is None: log.error('No data returned!') return value assert self.data_sha256 == sha256(value).hexdigest(), \ "Returned data doesn't match stored hash!" return value
def data(self): if self.size == 0: log.warning('Block size is 0') return '' elif hasattr(self, '_data'): # On initial download we temporarily store data in memory value = self._data else: value = blockstore.get_from_blockstore(self.data_sha256) if value is None: log.warning("Couldn't find data on S3 for block", sha_hash=self.data_sha256) from inbox.models.block import Block if isinstance(self, Block): if self.parts: # This block is an attachment of a message that was # deleted. We will attempt to fetch the raw # message and parse out the needed attachment. message = self.parts[0].message # only grab one account = message.namespace.account statsd_string = 'api.direct_fetching.{}.{}'.format( account.provider, account.id) # Try to fetch the message from S3 first. with statsd_client.timer('{}.blockstore_latency'.format( statsd_string)): raw_mime = blockstore.get_from_blockstore(message.data_sha256) # If it's not there, get it from the provider. if raw_mime is None: statsd_client.incr('{}.cache_misses'.format(statsd_string)) with statsd_client.timer('{}.provider_latency'.format( statsd_string)): raw_mime = get_raw_from_provider(message) msg_sha256 = sha256(raw_mime).hexdigest() # Cache the raw message in the blockstore so that # we don't have to fetch it over and over. with statsd_client.timer('{}.blockstore_save_latency'.format( statsd_string)): blockstore.save_to_blockstore(msg_sha256, raw_mime) else: # We found it in the blockstore --- report this. statsd_client.incr('{}.cache_hits'.format(statsd_string)) # If we couldn't find it there, give up. if raw_mime is None: log.error("Don't have raw message for hash {}" .format(message.data_sha256)) return None parsed = mime.from_string(raw_mime) if parsed is not None: for mimepart in parsed.walk( with_self=parsed.content_type.is_singlepart()): if mimepart.content_type.is_multipart(): continue # TODO should we store relations? data = mimepart.body if isinstance(data, unicode): data = data.encode('utf-8', 'strict') if data is None: continue # Found it! if sha256(data).hexdigest() == self.data_sha256: log.info('Found subpart with hash {}'.format( self.data_sha256)) with statsd_client.timer('{}.blockstore_save_latency'.format( statsd_string)): blockstore.save_to_blockstore(self.data_sha256, data) return data log.error("Couldn't find the attachment in the raw message", message_id=message.id) log.error('No data returned!') return value assert self.data_sha256 == sha256(value).hexdigest(), \ "Returned data doesn't match stored hash!" return value
def data(self): if self.size == 0: log.warning('Block size is 0') return '' elif hasattr(self, '_data'): # On initial download we temporarily store data in memory value = self._data else: value = blockstore.get_from_blockstore(self.data_sha256) if value is None: log.warning("Couldn't find data on S3 for block", sha_hash=self.data_sha256) from inbox.models.block import Block if isinstance(self, Block): if self.parts: # This block is an attachment of a message that was # deleted. We will attempt to fetch the raw # message and parse out the needed attachment. message = self.parts[0].message # only grab one account = message.namespace.account statsd_string = 'api.direct_fetching.{}.{}'.format( account.provider, account.id) # Try to fetch the message from S3 first. with statsd_client.timer( '{}.blockstore_latency'.format(statsd_string)): raw_mime = blockstore.get_from_blockstore( message.data_sha256) # If it's not there, get it from the provider. if raw_mime is None: statsd_client.incr( '{}.cache_misses'.format(statsd_string)) with statsd_client.timer( '{}.provider_latency'.format(statsd_string)): raw_mime = get_raw_from_provider(message) msg_sha256 = sha256(raw_mime).hexdigest() # Cache the raw message in the blockstore so that # we don't have to fetch it over and over. with statsd_client.timer( '{}.blockstore_save_latency'.format( statsd_string)): blockstore.save_to_blockstore(msg_sha256, raw_mime) else: # We found it in the blockstore --- report this. statsd_client.incr( '{}.cache_hits'.format(statsd_string)) # If we couldn't find it there, give up. if raw_mime is None: log.error("Don't have raw message for hash {}".format( message.data_sha256)) return None parsed = mime.from_string(raw_mime) if parsed is not None: for mimepart in parsed.walk( with_self=parsed.content_type.is_singlepart()): if mimepart.content_type.is_multipart(): continue # TODO should we store relations? data = mimepart.body if isinstance(data, unicode): data = data.encode('utf-8', 'strict') if data is None: continue # Found it! if sha256(data).hexdigest() == self.data_sha256: log.info('Found subpart with hash {}'.format( self.data_sha256)) with statsd_client.timer( '{}.blockstore_save_latency'.format( statsd_string)): blockstore.save_to_blockstore( self.data_sha256, data) return data log.error( "Couldn't find the attachment in the raw message", message_id=message.id) log.error('No data returned!') return value assert self.data_sha256 == sha256(value).hexdigest(), \ "Returned data doesn't match stored hash!" return value