def data(self):
if self.size == 0:
log.warning('Block size is 0')
return ''
elif hasattr(self, '_data'):
# On initial download we temporarily store data in memory
value = self._data
else:
value = get_from_blockstore(self.data_sha256)
if value is None:
log.warning(
"Couldn't find data on S3 for block with hash {}".format(
self.data_sha256))
from inbox.models.block import Block
if isinstance(self, Block):
if self.parts:
# This block is an attachment of a message that was
# accidentially deleted. We will attempt to fetch the raw
# message and parse out the needed attachment.
message = self.parts[0].message # only grab one
raw_mime = get_from_blockstore(message.data_sha256)
if raw_mime is None:
log.error("Don't have raw message for hash {}".format(
message.data_sha256))
return None
parsed = mime.from_string(raw_mime)
if parsed is not None:
for mimepart in parsed.walk(
with_self=parsed.content_type.is_singlepart()):
if mimepart.content_type.is_multipart():
continue # TODO should we store relations?
data = mimepart.body
if isinstance(data, unicode):
data = data.encode('utf-8', 'strict')
# Found it!
if sha256(data).hexdigest() == self.data_sha256:
log.info('Found subpart with hash {}'.format(
self.data_sha256))
save_to_blockstore(self.data_sha256, data)
return data
log.error('No data returned!')
return value
assert self.data_sha256 == sha256(value).hexdigest(), \
"Returned data doesn't match stored hash!"
return value
def data(self):
if self.size == 0:
log.warning("Block size is 0")
return ""
elif hasattr(self, "_data"):
# On initial download we temporarily store data in memory
value = self._data
else:
value = get_from_blockstore(self.data_sha256)
if value is None:
log.warning("Couldn't find data on S3 for block with hash {}".format(self.data_sha256))
from inbox.models.block import Block
if isinstance(self, Block):
if self.parts:
# This block is an attachment of a message that was
# accidentially deleted. We will attempt to fetch the raw
# message and parse out the needed attachment.
message = self.parts[0].message # only grab one
raw_mime = get_from_blockstore(message.data_sha256)
if raw_mime is None:
log.error("Don't have raw message for hash {}".format(message.data_sha256))
return None
parsed = mime.from_string(raw_mime)
if parsed is not None:
for mimepart in parsed.walk(with_self=parsed.content_type.is_singlepart()):
if mimepart.content_type.is_multipart():
continue # TODO should we store relations?
data = mimepart.body
if isinstance(data, unicode):
data = data.encode("utf-8", "strict")
# Found it!
if sha256(data).hexdigest() == self.data_sha256:
log.info("Found subpart with hash {}".format(self.data_sha256))
save_to_blockstore(self.data_sha256, data)
return data
log.error("No data returned!")
return value
assert self.data_sha256 == sha256(value).hexdigest(), "Returned data doesn't match stored hash!"
return value
def test_store_full_body_on_parse_error(default_account,
mime_message_with_bad_date):
received_date = None
m = Message.create_from_synced(default_account, 139219, '[Gmail]/All Mail',
received_date,
mime_message_with_bad_date.to_string())
assert get_from_blockstore(m.data_sha256)
def test_rfc822_format(stub_message_from_raw, api_client, mime_message):
""" Test the API response to retreive raw message contents """
full_path = '/messages/{}'.format(stub_message_from_raw.public_id)
results = api_client.get_raw(full_path,
headers={'Accept': 'message/rfc822'})
assert results.data == get_from_blockstore(stub_message_from_raw.data_sha256)
def test_store_full_body_on_parse_error(
default_account, mime_message_with_bad_date):
received_date = None
m = Message.create_from_synced(default_account, 139219, '[Gmail]/All Mail',
received_date,
mime_message_with_bad_date.to_string())
assert get_from_blockstore(m.data_sha256)
def test_rfc822_format(stub_message_from_raw, api_client, mime_message):
""" Test the API response to retreive raw message contents """
full_path = '/messages/{}'.format(stub_message_from_raw.public_id)
results = api_client.get_raw(full_path,
headers={'Accept': 'message/rfc822'})
assert results.data == get_from_blockstore(
stub_message_from_raw.data_sha256)
def send_raw(self, msg):
recipient_emails = [email for name, email in itertools.chain(
msg.bcc_addr, msg.cc_addr, msg.to_addr)]
raw_message = get_from_blockstore(msg.data_sha256)
mime_body = _substitute_bcc(raw_message)
self._send(recipient_emails, mime_body)
# Sent to all successfully
sender_email = msg.from_addr[0][1]
self.log.info('Sending successful', sender=sender_email,
recipients=recipient_emails)
def send_raw(self, msg):
recipient_emails = [email for name, email in itertools.chain(
msg.bcc_addr, msg.cc_addr, msg.to_addr)]
raw_message = get_from_blockstore(msg.data_sha256)
mime_body = re.sub(r'Bcc: [^\r\n]*\r\n', '', raw_message)
self._send(recipient_emails, mime_body)
# Sent to all successfully
sender_email = msg.from_addr[0][1]
self.log.info('Sending successful', sender=sender_email,
recipients=recipient_emails)
def data(self):
if self.size == 0:
log.warning('Block size is 0')
return ''
elif hasattr(self, '_data'):
# On initial download we temporarily store data in memory
value = self._data
else:
value = get_from_blockstore(self.data_sha256)
if value is None:
log.error('No data returned!')
return value
assert self.data_sha256 == sha256(value).hexdigest(), \
"Returned data doesn't match stored hash!"
return value
def data(self):
if self.size == 0:
log.warning('Block size is 0')
# Placeholder for "empty bytes". If this doesn't work as intended,
# it will trigger the hash assertion later.
value = ''
elif hasattr(self, '_data'):
# On initial download we temporarily store data in memory
value = self._data
else:
value = get_from_blockstore(self.data_sha256)
if value is None:
log.error('No data returned!')
return value
assert self.data_sha256 == sha256(value).hexdigest(), \
"Returned data doesn't match stored hash!"
return value
def data(self):
if self.size == 0:
log.warning('Block size is 0')
return ''
elif hasattr(self, '_data'):
# On initial download we temporarily store data in memory
value = self._data
else:
value = blockstore.get_from_blockstore(self.data_sha256)
if value is None:
log.warning("Couldn't find data on S3 for block",
sha_hash=self.data_sha256)
from inbox.models.block import Block
if isinstance(self, Block):
if self.parts:
# This block is an attachment of a message that was
# deleted. We will attempt to fetch the raw
# message and parse out the needed attachment.
message = self.parts[0].message # only grab one
account = message.namespace.account
statsd_string = 'api.direct_fetching.{}.{}'.format(
account.provider, account.id)
# Try to fetch the message from S3 first.
with statsd_client.timer('{}.blockstore_latency'.format(
statsd_string)):
raw_mime = blockstore.get_from_blockstore(message.data_sha256)
# If it's not there, get it from the provider.
if raw_mime is None:
statsd_client.incr('{}.cache_misses'.format(statsd_string))
with statsd_client.timer('{}.provider_latency'.format(
statsd_string)):
raw_mime = get_raw_from_provider(message)
msg_sha256 = sha256(raw_mime).hexdigest()
# Cache the raw message in the blockstore so that
# we don't have to fetch it over and over.
with statsd_client.timer('{}.blockstore_save_latency'.format(
statsd_string)):
blockstore.save_to_blockstore(msg_sha256, raw_mime)
else:
# We found it in the blockstore --- report this.
statsd_client.incr('{}.cache_hits'.format(statsd_string))
# If we couldn't find it there, give up.
if raw_mime is None:
log.error("Don't have raw message for hash {}"
.format(message.data_sha256))
return None
parsed = mime.from_string(raw_mime)
if parsed is not None:
for mimepart in parsed.walk(
with_self=parsed.content_type.is_singlepart()):
if mimepart.content_type.is_multipart():
continue # TODO should we store relations?
data = mimepart.body
if isinstance(data, unicode):
data = data.encode('utf-8', 'strict')
if data is None:
continue
# Found it!
if sha256(data).hexdigest() == self.data_sha256:
log.info('Found subpart with hash {}'.format(
self.data_sha256))
with statsd_client.timer('{}.blockstore_save_latency'.format(
statsd_string)):
blockstore.save_to_blockstore(self.data_sha256, data)
return data
log.error("Couldn't find the attachment in the raw message", message_id=message.id)
log.error('No data returned!')
return value
assert self.data_sha256 == sha256(value).hexdigest(), \
"Returned data doesn't match stored hash!"
return value
def data(self):
if self.size == 0:
log.warning('Block size is 0')
return ''
elif hasattr(self, '_data'):
# On initial download we temporarily store data in memory
value = self._data
else:
value = blockstore.get_from_blockstore(self.data_sha256)
if value is None:
log.warning("Couldn't find data on S3 for block",
sha_hash=self.data_sha256)
from inbox.models.block import Block
if isinstance(self, Block):
if self.parts:
# This block is an attachment of a message that was
# deleted. We will attempt to fetch the raw
# message and parse out the needed attachment.
message = self.parts[0].message # only grab one
account = message.namespace.account
statsd_string = 'api.direct_fetching.{}.{}'.format(
account.provider, account.id)
# Try to fetch the message from S3 first.
with statsd_client.timer(
'{}.blockstore_latency'.format(statsd_string)):
raw_mime = blockstore.get_from_blockstore(
message.data_sha256)
# If it's not there, get it from the provider.
if raw_mime is None:
statsd_client.incr(
'{}.cache_misses'.format(statsd_string))
with statsd_client.timer(
'{}.provider_latency'.format(statsd_string)):
raw_mime = get_raw_from_provider(message)
msg_sha256 = sha256(raw_mime).hexdigest()
# Cache the raw message in the blockstore so that
# we don't have to fetch it over and over.
with statsd_client.timer(
'{}.blockstore_save_latency'.format(
statsd_string)):
blockstore.save_to_blockstore(msg_sha256, raw_mime)
else:
# We found it in the blockstore --- report this.
statsd_client.incr(
'{}.cache_hits'.format(statsd_string))
# If we couldn't find it there, give up.
if raw_mime is None:
log.error("Don't have raw message for hash {}".format(
message.data_sha256))
return None
parsed = mime.from_string(raw_mime)
if parsed is not None:
for mimepart in parsed.walk(
with_self=parsed.content_type.is_singlepart()):
if mimepart.content_type.is_multipart():
continue # TODO should we store relations?
data = mimepart.body
if isinstance(data, unicode):
data = data.encode('utf-8', 'strict')
if data is None:
continue
# Found it!
if sha256(data).hexdigest() == self.data_sha256:
log.info('Found subpart with hash {}'.format(
self.data_sha256))
with statsd_client.timer(
'{}.blockstore_save_latency'.format(
statsd_string)):
blockstore.save_to_blockstore(
self.data_sha256, data)
return data
log.error(
"Couldn't find the attachment in the raw message",
message_id=message.id)
log.error('No data returned!')
return value
assert self.data_sha256 == sha256(value).hexdigest(), \
"Returned data doesn't match stored hash!"
return value
