def test_immutable_list_pickle():
"""ImmutableLists are pickle-able"""
for protocol in xrange(pickle.HIGHEST_PROTOCOL + 1):
l = ImmutableList(range(100))
ul = pickle.loads(pickle.dumps(l, protocol))
assert l == ul
assert pickle.loads(pickle.dumps(l, protocol) \
.replace('werkzeug.datastructures', 'werkzeug')) == l
assert type(l) is type(ul)
def test_immutable_structures():
"""Test immutable structures"""
l = ImmutableList([1, 2, 3])
assert_raises(TypeError, l.__delitem__, 0)
assert_raises(TypeError, l.__delslice__, 0, 1)
assert_raises(TypeError, l.__iadd__, [1, 2])
assert_raises(TypeError, l.__setitem__, 0, 1)
assert_raises(TypeError, l.__setslice__, 0, 1, [2, 3])
assert_raises(TypeError, l.append, 42)
assert_raises(TypeError, l.insert, 0, 32)
assert_raises(TypeError, l.pop)
assert_raises(TypeError, l.extend, [2, 3])
assert_raises(TypeError, l.reverse)
assert_raises(TypeError, l.sort)
assert l == [1, 2, 3]
d = ImmutableDict(foo=23, bar=42)
assert_raises(TypeError, d.setdefault, 'baz')
assert_raises(TypeError, d.update, {2: 3})
assert_raises(TypeError, d.popitem)
assert_raises(TypeError, d.__delitem__, 'foo')
assert_raises(TypeError, d.clear)
assert d == dict(foo=23, bar=42)
d = ImmutableDict.fromkeys([1, 2])
assert d[1] == d[2] == None
d = ImmutableMultiDict(d)
assert_raises(TypeError, d.add, 'fuss', 44)
assert_raises(TypeError, d.popitemlist)
assert_raises(TypeError, d.poplist, 'foo')
assert_raises(TypeError, d.setlist, 'tadaa', [1, 2])
assert_raises(TypeError, d.setlistdefault, 'tadaa')
d = ImmutableMultiDict.fromkeys([1, 2])
assert d[1] == d[2] == None
d = EnvironHeaders({'HTTP_X_FOO': 'test'})
assert_raises(TypeError, d.__delitem__, 0)
assert_raises(TypeError, d.add, 42)
assert_raises(TypeError, d.pop, 'x-foo')
assert_raises(TypeError, d.popitem)
assert_raises(TypeError, d.setdefault, 'foo', 42)
assert dict(d.items()) == {'X-Foo': 'test'}
assert_raises(TypeError, d.copy)
def __init__(self):
super(AnonymousUser, self).__init__()
self.roles = ImmutableList()
def __init__(self):
self.roles = ImmutableList()
def __init__(self, inbound_request: LocalProxy):
inbound_request.get_data(as_text=True)
self._logger = EmeraldLogger(logging_module_name=type(self).__name__)
print('The type of request is ' + str(type(inbound_request)))
self._sendgrid_payload = inbound_request.form
print('the type of payload is ' + str(type(self._sendgrid_payload)))
# The type of request is <class 'werkzeug.local.LocalProxy'>
# the type of payload is <class 'werkzeug.datastructures.ImmutableMultiDict'>
# run through the tuples that represent key value pairs
email_data_dictionary = dict()
for kvcount, kvpair in enumerate(self.sendgrid_payload.items(),
start=1):
print('Pair #' + str(kvcount) + ': ' + str(kvpair))
email_data_dictionary[kvpair[0]] = kvpair[1]
# get the charsets
# ('charsets', '{"to":"UTF-8","html":"UTF-8","subject":"UTF-8","from":"UTF-8","text":"UTF-8"}')
try:
self._charsets = self.sendgrid_payload['charsets']
except KeyError:
raise EmeraldEmailParsingError(
'Unable to find charsets in key list for inbound email' +
os.linesep + 'Keys found: ' +
','.join([str(x) for x in email_data_dictionary.keys()]))
###############
# Email Container Element: METADATA
###############
# DET FIXME TODO - get DKIM and SPF
# ('dkim', '{[email protected] : pass}')
# ('SPF', 'pass')
#
# get the headers
try:
email_message_headers = self.sendgrid_payload['headers']
except KeyError:
raise EmeraldEmailParsingError(
'Unable to find email headers in key list for inbound email' +
os.linesep + 'Keys found: ' +
','.join([str(x) for x in email_data_dictionary.keys()]))
# get the sender ip
# ('sender_ip', '136.143.188.19')
try:
sender_ip = self.sendgrid_payload['sender_ip']
except KeyError:
raise EmeraldEmailParsingError(
'Unable to find sender_ip in key list for inbound email' +
os.linesep + 'Keys found: ' +
','.join([str(x) for x in email_data_dictionary.keys()]))
# get attachment count and then we can decide if attachment parsing needed
# ('attachments', '0')
try:
attachment_count = int(self.sendgrid_payload['attachments'])
except KeyError:
raise EmeraldEmailParsingError(
'Unable to find attachments (the count) in key list for inbound email'
+ os.linesep + 'Keys found: ' +
','.join([str(x) for x in email_data_dictionary.keys()]))
except (TypeError, ValueError):
raise EmeraldEmailParsingError(
'Unable to parse the attachment count as integer' +
os.linesep + 'Value sent is ' +
str(self.sendgrid_payload['attachments']))
#
# SPF is a single check but we get as a string "pass"
#
try:
email_spf_passed: Optional[bool] = True if self.sendgrid_payload[
'spf'].casefold() == 'pass' else False
except KeyError:
email_spf_passed: Optional[bool] = None
#
# DKIM arrives as a dictionary keyed by email address - set TRUE only if all say pass
#
email_dkim_passed: Optional[bool] = None
try:
email_dkim_passed_dict_string: str = self.sendgrid_payload['dkim']
# unfortunately it comes like this: "{ [email protected]: pass}" which because there are no quotes
# means you can't read with json.loads. We need JSON.stringify as in javascript
# Just split the string on commas, then colons
if not email_dkim_passed_dict_string.startswith('{'):
raise ValueError(
'Expected DKIM string to begin with { character')
if not email_dkim_passed_dict_string.endswith('}'):
raise ValueError(
'Expected DKIM string to end with } character')
# now strip braces and leading and trailing spaces
email_dkim_passed_dict_string = email_dkim_passed_dict_string.lstrip(
'{').rstrip('}').lstrip().rstrip()
email_dkim_entries = email_dkim_passed_dict_string.split(',')
for entry_counter, this_entry in enumerate(email_dkim_entries,
start=1):
# split on colon to get key value and throw error if not two elements
this_entry_kv_pair = this_entry.split(':')
if len(this_entry_kv_pair) != 2:
raise ValueError('DKIM entry #' + str(entry_counter) +
' is not valid - expected format ' +
'<key>:<value>' + os.linesep +
'\tValue provided = ' +
email_dkim_passed_dict_string)
dkim_status_this_entry_string = this_entry_kv_pair[1].rstrip(
).lstrip().casefold()
if dkim_status_this_entry_string != 'pass':
print('DKIM entry #' + str(entry_counter) +
' does not have pass value ' + '<key>:<value>' +
os.linesep + '\tValue provided = ' +
email_dkim_passed_dict_string + os.linesep +
'\tValue = ' + dkim_status_this_entry_string)
email_dkim_passed = False
break
email_dkim_passed: Optional[bool] = True
except KeyError:
# Not found
print('No DKIM status info passed for email')
pass
except ValueError as vex:
print(
'Unable to parse expected pseudo-JSON DKIM dictionary - value is "'
+ str(self.sendgrid_payload['dkim']) + '"' + os.linesep +
'Exception: ' + str(vex))
pass
email_container_metadata = EmailMessageMetadata(
router_source_tag='self',
routed_timestamp_iso8601=EmeraldLogger.get_iso8601_utc_now_string(
),
email_sender_ip=sender_ip,
attachment_count=attachment_count,
email_headers=email_message_headers,
email_spf_sender_passed=email_spf_passed,
email_dkim_sender_passed=email_dkim_passed)
###############
# Email Container Element: ENVELOPE
###############
# read the envelope and get the to and from data more directly (we don't care about text name)
# ('to', '"test1" <*****@*****.**>')
# ('from', 'David Thompson <*****@*****.**>')
# ('envelope', '{"to":["*****@*****.**"],"from":"*****@*****.**"}')
try:
envelope_json = self.sendgrid_payload['envelope']
except KeyError:
raise EmeraldEmailParsingError(
'Unable to find envelope in key list for inbound email' +
os.linesep + 'Keys found: ' +
','.join([str(x) for x in email_data_dictionary.keys()]))
#
# Subject is in text form
# ('subject', 'with at')
#
try:
email_subject = self.sendgrid_payload['subject']
except KeyError:
raise EmeraldEmailParsingError(
'Unable to parse subject in key list for inbound email' +
os.linesep + 'Keys found: ' +
','.join([str(x) for x in email_data_dictionary.keys()]))
# SendGrid envelop contains only the from and to - we add subject
try:
envelope = json.loads(envelope_json)
except json.JSONDecodeError as jdex:
raise EmeraldEmailParsingError(
'Unable to parse expected data from envelope json string' +
os.linesep + 'Value of text = ' + os.linesep +
str(envelope_json) + os.linesep + 'Exception info: ' +
str(jdex))
else:
#
# Now initialize the email envelop
#
email_container_envelope = EmailEnvelope(
address_from=envelope['from'],
address_to_collection=frozenset(envelope['to']),
message_subject=email_subject,
message_rx_timestamp_iso8601=EmeraldLogger.
get_iso8601_utc_now_string())
###############
# Email Container Element: BODY
###############
# read the text of message, recognizing it could be in HTML form or even RTF
# Use the charsets to know the encoding
#
# ('text', '')
try:
message_body_text = self.sendgrid_payload['text']
except KeyError:
raise EmeraldEmailParsingError(
'Unable to find text (message body) in key list for inbound email'
+ os.linesep + 'Keys found: ' +
','.join([str(x) for x in email_data_dictionary.keys()]))
# and look for html as contents are there
# Pair #4: ('html', '<!DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN">
# <html><head><meta content="text/html;charset=UTF-8" http-equiv="Content-Type"></head>
# <body >
# <div style="font-family: Verdana, Arial, Helvetica, sans-serif; font-size: 10pt;">
# <div>Hello this is <span class="highlight" style="background-color:#ff0000">html</span><br></div>
# <div><br></div><div class="align-left" style="text-align: left;">Because<u> I marked it up</u></div><
# /div><br></body></html>')
try:
message_body_html = self.sendgrid_payload['html']
except KeyError:
print('No HTML element in payload')
message_body_html = None
# DET TODO handle HTML and text encoding cases later
email_container_body = EmailBody(message_body_text=message_body_text,
message_body_html=message_body_html)
###############
# Email Container Element: ATTACHMENT COLLECTION
###############
# if attachment count > 0 we need attachment-info
attachments: List[EmailAttachment] = []
if attachment_count > 0:
print('Attachment count: ' + str(attachment_count))
try:
attachment_info_json = self.sendgrid_payload['attachment-info']
except KeyError:
raise EmeraldEmailParsingError(
'Unable to find attachment-info in key list for inbound email'
+ os.linesep + 'Attachment count shows as ' +
str(attachment_count) + os.linesep + 'Keys found: ' +
','.join([str(x) for x in email_data_dictionary.keys()]))
try:
attachment_info = json.loads(attachment_info_json)
except json.JSONDecodeError as jdex:
raise EmeraldEmailParsingError(
'Unable to parse expected data from attachment-info json string'
+ os.linesep + 'Value of text = ' + os.linesep +
str(attachment_info_json) + os.linesep)
self.logger.logger.info('Attachment info: ' + str(attachment_info))
print('Now get attachments - files type = ' +
str(type(request.files)))
# Now get attachments - files type = <class 'werkzeug.datastructures.ImmutableMultiDict'>
for _, filestorage in iteritems(request.files):
if filestorage.filename not in (None, 'fdopen', '<fdopen>'):
filename = secure_filename(filestorage.filename)
attachment = EmailAttachment(
filename=filename,
mimetype=filestorage.content_type,
contents_base64=base64.b64encode(filestorage.read()))
attachments.append(attachment)
else:
print('Found attachment filename as unsupported value "' +
str(filestorage.filename))
print('Total attachment count = ' + str(len(attachments)))
print('Total specified count = ' + str(attachment_count))
# Now build overall container
self._email_container = EmailContainer(
email_container_metadata=email_container_metadata,
email_envelope=email_container_envelope,
email_body=email_container_body,
email_attachment_collection=ImmutableList(attachments))
print('Info on the email: ' + os.linesep +
self._email_container.get_info())
return
def __init__(self):
self.roles = ImmutableList()
self.followed_posts = "followed_posts"
def devices_per_event_subject_month(self, options):
"""
Returns the number of devices per event, subject and month.
Subject is the series of the graph, and
:param options:
:return:
"""
pipeline = [
{
'$match': {
'@type': options['event'],
'_created': {'$gte': datetime.datetime(datetime.date.today().year, 1, 1)}
}
},
{
'$unwind': '$devices'
},
{
'$group': {
'_id': {
'month': {'$month': '$_created'},
'subject': '$' + options['subject']
},
'arrayOfDevices': {'$push': '$devices'},
}
},
{
'$project': {
'_id': False,
'month': '$_id.month',
'subject': '$_id.subject',
'countPerSubjectAndMonth': {'$size': '$arrayOfDevices'}
}
},
{
'$sort': {
'subject': DESCENDING,
'month': DESCENDING
}
},
{
'$group': {
'_id': {
'subject': '$subject'
},
'devices': {'$push': '$countPerSubjectAndMonth'},
'months': {'$push': '$month'}
}
},
{
'$project': {
'_id': False,
'subject': '$_id.subject',
'devices': True,
'months': True
}
}
]
if 'receiverType' in options and options['event'] == 'Receive':
pipeline[0]['$match']['type'] = options['receiverType']
res = {
'labels': list(calendar.month_name)[1:],
'series': [],
'data': []
}
a = self._aggregate(ImmutableList(pipeline))
for org in a:
res['series'].append('Others' if org['subject'] is None else org['subject'])
res['data'].append([0] * len(res['labels']))
i = 0
for pos in org['months']:
res['data'][-1][pos - 1] = org['devices'][i]
i += 1
return res
def tags(self):
# type: ()->List[ApiTag]
return ImmutableList(self._tags)