def from_bytes(cls, corre_b):
mail_parser = BytesFeedParser()
mail_parser.feed(corre_b)
correo = mail_parser.close()
txt_codec = correo.get_content_charset()
transfer_codec = correo['Content-Transfer-Encoding']
origen, codec = decode_header(correo['from'])[0]
origen = origen.decode(codec)
destino = es_correo.findall(' '.join([correo.get('Cc', ''), correo.get('To', '')]))
message_id = correo['Message-Id']
fecha = datetime.strptime(correo['date'], '%a, %d %b %Y %H:%M:%S %z')
fecha = datetime(fecha.year, fecha.month, fecha.day, fecha.hour, fecha.minute, fecha.second)
if transfer_codec is None:
asunto = correo['subject']
payload = correo.get_payload(0)
transfer_codec = payload['Content-Transfer-Encoding']
txt_codec = payload.get_content_charset()
mensaje = payload.get_payload(decode=transfer_codec).decode(txt_codec)
elif transfer_codec == '7bit':
asunto, codec = decode_header(correo['subject'])[0]
asunto = asunto.decode(codec)
mensaje = correo.get_payload()
elif transfer_codec == 'quoted-printable':
asunto = correo['subject']
mensaje = correo.get_payload(decode=transfer_codec).decode(txt_codec)
else:
raise NotImplementedError(mail_log.error('Este transfer_codec no está implememntado: {}', transfer_codec))
return cls(message_id, origen, destino, asunto, mensaje, fecha)
def create_msg_from_source(message_source, to=None):
message_source = message_source.encode('utf-8')
parser = BytesFeedParser()
parser.feed(message_source)
message = parser.close()
if to is not None:
message.replace_header('To', to)
return encode_message(message)
def _send_message(self, filename):
with open("./"+messages_dir+"/"+filename, "rb") as f:
byte_array = bytearray(f.read())
parser = BytesFeedParser()
parser.feed(byte_array)
msg = parser.close()
with self.sender as sndr:
sndr.send_message(msg)
os.remove("./"+messages_dir+"/"+filename)
def truncate(self, size=None):
if (size is None and self._written != 0) and size != 0:
raise NotImplementedError(
"The 'size' argument to truncate() must be 0 - partial "
"truncation is not supported")
if self._closed:
raise ValueError("File is closed")
self._parser = BytesFeedParser()
self._written = 0
def __init__(self, context):
self.context = context
self._mimeType = None
self._encoding = 'utf-8'
self._closed = False
self._name = None
self._written = 0
self._parser = BytesFeedParser()
self._message = None
def read_headers(fp):
''' Read headers from a binary file such as an HTTP stream, return the raw binary data and the corresponding Message object.
'''
def is_header_line(line):
return line.startswith(b' ') or line.startswith(b'\t') or line.rstrip()
header_lines = list(takewhile(is_header_line, fp))
parser = BytesFeedParser()
parser.feed(b''.join(header_lines))
return b''.join(header_lines), parser.close()
def _read_multipart_mixed(self):
parser = BytesFeedParser()
for k, v in self.request.headers.items():
parser.feed('{}: {}\n'.format(k, v).encode('utf-8'))
parser.feed(b'\n')
parser.feed(self.request.body)
msg = parser.close()
if not msg.is_multipart():
logger.error('Bad multipart request')
self.send_error(400)
return
main_part = msg.get_payload(0)
if main_part.get_content_type() != 'application/json':
logger.error('Mainpart is not JSON')
self.send_error(400)
return
payload = json.loads(main_part.get_payload(decode=True))
attachments = {}
for idx in range(1, len(msg.get_payload())):
part = msg.get_payload(idx)
if part.is_multipart():
logger.error('Nesting multipart is not supported')
self.send_error(400)
return
attachments[part.get_filename()] = part.get_content_type(
), part.get_payload(decode=True)
return payload, attachments
def handle_xop(self, client, operation, response):
"""Decode a XML-binary Optimized Packaging (XOP) response.
Spec: https://www.w3.org/TR/xop10/
"""
# example Content-Type: multipart/related; boundary="192ACTH5zu2kQtJcnRwnTXJzXb2hoxEPoAksrlRlihV2HO2NmRC6h5NdO4n44nSA2Q19"; type="application/xop+xml"; start="<soap:Envelope>"; start-info="application/soap+xml; charset=utf-8"
content_type = response.headers.get("content-type", "")
# don't break pre-Py33 code
if not content_type.strip().lower().startswith("multipart/related"):
return response.content
# parse the Content-Type header; is there an easy and robust pre-Py33 way?
import email.headerregistry # New in version 3.3: as a provisional module.
parsed_ct = email.headerregistry.HeaderRegistry()("content-type", content_type)
if parsed_ct.content_type.lower() != "multipart/related":
# this is not a XOP Package
return response.content
if parsed_ct.params["type"].lower() != "application/xop+xml":
# this is not a XOP Package (but it's not plain SOAP either...??)
return response.content
# OK, this is indeed a XOP Package, so now we need to break down the multipart/related MIME message
message_parser = BytesFeedParser()
# we will only feed the parser the actual body of the HTTP response, so we must first simulate at least the
# Content-Type header
message_parser.feed(("Content-Type: %s\r\n\r\n" % (content_type,)).encode()) # additional CRLF for end of headers
for chunk in response.iter_content(CHUNK_SIZE):
message_parser.feed(chunk)
message = message_parser.close()
message_parts = message.get_payload()
root = message_parts[0]
assert root.get_content_type() == "application/xop+xml"
# startwith (not equals) because it might be "application/soap+xml; charset=utf-8" or similar
assert root.get_param("type").startswith("application/soap+xml")
assert all(not message.is_multipart() for message in message_parts[1:])
assert all(message["Content-ID"] for message in message_parts[1:])
operation.xop_replaced_data_by_cid = {message["Content-ID"]: message.get_payload(decode=True) for message in message_parts[1:]}
return root.get_payload(decode=True)
def email_open(args):
""" Read the file descriptor and return a msg file """
parser = BytesFeedParser()
if args.stdin or args.infile is None:
with io.BufferedReader(sys.stdin.buffer) as input:
parser.feed(input.read())
else:
with open(args.infile, 'rb') as input:
parser.feed(input.read())
msg = parser.close()
# Check for invalid (0 bytes) message
if len(msg) == 0:
error(msg, "Invalid Message", out_streams)
else:
return msg
async def read_form(self):
if self._form is None:
self._form = CIMultiDict()
if (self.headers.content_type.type == 'application') and (self.headers.content_type.subtype == 'x-www-form-urlencoded'):
body = await self.read_body()
for parameter in body.split(b'&'):
name, value = parameter.split(b'=')
self._form.add(
unquote_to_bytes(name).decode('utf-8'),
unquote_to_bytes(value).decode('utf-8'))
elif (self.headers.content_type.type == 'multipart') and (self.headers.content_type.subtype == 'form-data'):
# TODO: replace with multifruits
parser = BytesFeedParser(policy=HTTP, _factory=message_factory)
parser.feed(b'Content-Type: %s\r\n\r\n' % self.raw_headers['Content-Type'])
async with self.open_body() as stream:
while True:
data = await stream.read()
if not data:
break
parser.feed(data)
message = parser.close()
for part in message.walk():
if part.get_content_type() != 'multipart/form-data':
params = dict(part.get_params(header='content-disposition'))
name = params.get('name')
if name:
payload = part.get_payload(decode=True)
if payload:
if part.get_content_type() == 'application/form-data':
self._form.add(name, payload.decode('utf-8'))
else:
self._form.add(
name,
RequestFilePart(params.get('filename'), part.items(), part.get_payload(decode=True)))
return self._form
def email_open(args):
""" Read the file descriptor and return a msg file """
parser = BytesFeedParser()
if args.stdin or args.infile is None:
with io.BufferedReader(sys.stdin.buffer) as input:
parser.feed(input.read())
else:
with open(args.infile, "rb") as input:
parser.feed(input.read())
msg = parser.close()
# Check for invalid (0 bytes) message
if len(msg) == 0:
error(msg, "Invalid Message", out_streams)
else:
return msg
def handle_xop(self, client, operation, response):
"""Decode a XML-binary Optimized Packaging (XOP) response.
Spec: https://www.w3.org/TR/xop10/
"""
# example Content-Type: multipart/related; boundary="192ACTH5zu2kQtJcnRwnTXJzXb2hoxEPoAksrlRlihV2HO2NmRC6h5NdO4n44nSA2Q19"; type="application/xop+xml"; start="<soap:Envelope>"; start-info="application/soap+xml; charset=utf-8"
content_type = response.headers.get("content-type", "")
# don't break pre-Py33 code
if not content_type.strip().lower().startswith("multipart/related"):
return response.content
# parse the Content-Type header; is there an easy and robust pre-Py33 way?
import email.headerregistry # New in version 3.3: as a provisional module.
parsed_ct = email.headerregistry.HeaderRegistry()("content-type",
content_type)
if parsed_ct.content_type.lower() != "multipart/related":
# this is not a XOP Package
return response.content
if parsed_ct.params["type"].lower() != "application/xop+xml":
# this is not a XOP Package (but it's not plain SOAP either...??)
return response.content
# OK, this is indeed a XOP Package, so now we need to break down the multipart/related MIME message
message_parser = BytesFeedParser()
# we will only feed the parser the actual body of the HTTP response, so we must first simulate at least the
# Content-Type header
message_parser.feed(
("Content-Type: %s\r\n\r\n" %
(content_type, )).encode()) # additional CRLF for end of headers
for chunk in response.iter_content(CHUNK_SIZE):
message_parser.feed(chunk)
message = message_parser.close()
message_parts = message.get_payload()
root = message_parts[0]
assert root.get_content_type() == "application/xop+xml"
# startwith (not equals) because it might be "application/soap+xml; charset=utf-8" or similar
assert root.get_param("type").startswith("application/soap+xml")
assert all(not message.is_multipart() for message in message_parts[1:])
assert all(message["Content-ID"] for message in message_parts[1:])
operation.xop_replaced_data_by_cid = {
message["Content-ID"]: message.get_payload(decode=True)
for message in message_parts[1:]
}
return root.get_payload(decode=True)
def parseform(self, limit=67108864, tostr=True, safename=True):
'''
Parse form-data with multipart/form-data or application/x-www-form-urlencoded
In Python3, the keys of form and files are unicode, but values are bytes
If the key ends with '[]', it is considered to be a list:
a=1&b=2&b=3 => {'a':1,'b':3}
a[]=1&b[]=2&b[]=3 => {'a':[1],'b':[2,3]}
:param limit: limit total input size, default to 64MB. None = no limit. Note that all the form
data is stored in memory (including upload files), so it is dangerous to accept a very large input.
:param tostr: convert values to str in Python3. Only apply to form, files data are always bytes
:param safename: if True, extra security checks are performed on filenames to reduce known security risks.
'''
if tostr:
def _str(s):
try:
if not isinstance(s, str):
return s.decode(self.encoding)
else:
return s
except:
raise HttpInputException(
'Invalid encoding in post data: ' + repr(s))
else:
def _str(s):
return s
try:
form = {}
files = {}
# If there is not a content-type header, maybe there is not a content.
if b'content-type' in self.headerdict and self.inputstream is not None:
contenttype = self.headerdict[b'content-type']
m = Message()
# Email library expects string, which is unicode in Python 3
try:
m.add_header('Content-Type',
str(contenttype.decode('ascii')))
except UnicodeDecodeError:
raise HttpInputException(
'Content-Type has non-ascii characters')
if m.get_content_type() == 'multipart/form-data':
fp = BytesFeedParser()
fp.feed(b'Content-Type: ' + contenttype + b'\r\n\r\n')
total_length = 0
while True:
try:
for m in self.inputstream.prepareRead(
self.container):
yield m
data = self.inputstream.readonce()
total_length += len(data)
if limit is not None and total_length > limit:
raise HttpInputException('Data is too large')
fp.feed(data)
except EOFError:
break
msg = fp.close()
if not msg.is_multipart() or msg.defects:
# Reject the data
raise HttpInputException(
'Not valid multipart/form-data format')
for part in msg.get_payload():
if part.is_multipart() or part.defects:
raise HttpInputException(
'Not valid multipart/form-data format')
disposition = part.get_params(
header='content-disposition')
if not disposition:
raise HttpInputException(
'Not valid multipart/form-data format')
disposition = dict(disposition)
if 'form-data' not in disposition or 'name' not in disposition:
raise HttpInputException(
'Not valid multipart/form-data format')
if 'filename' in disposition:
name = disposition['name']
filename = disposition['filename']
if safename:
filename = _safename(filename)
if name.endswith('[]'):
files.setdefault(name[:-2], []).append({
'filename':
filename,
'content':
part.get_payload(decode=True)
})
else:
files[name] = {
'filename': filename,
'content': part.get_payload(decode=True)
}
else:
name = disposition['name']
if name.endswith('[]'):
form.setdefault(name[:-2], []).append(
_str(part.get_payload(decode=True)))
else:
form[name] = _str(
part.get_payload(decode=True))
elif m.get_content_type() == 'application/x-www-form-urlencoded' or \
m.get_content_type() == 'application/x-url-encoded':
if limit is not None:
for m in self.inputstream.read(self.container,
limit + 1):
yield m
data = self.container.data
if len(data) > limit:
raise HttpInputException('Data is too large')
else:
for m in self.inputstream.read(self.container):
yield m
data = self.container.data
result = parse_qs(data, True)
def convert(k, v):
try:
k = str(k.decode('ascii'))
except:
raise HttpInputException(
'Form-data key must be ASCII')
if not k.endswith('[]'):
v = _str(v[-1])
else:
k = k[:-2]
v = [_str(i) for i in v]
return (k, v)
form = dict(convert(k, v) for k, v in result.items())
else:
# Other formats, treat like no data
pass
self.form = form
self.files = files
except Exception as exc:
raise HttpInputException('Failed to parse form-data: ' + str(exc))
class DefaultWriteFile(object):
"""IRawWriteFile file adapter for Dexterity objects.
Uses RFC822 marshaler.
"""
def __init__(self, context):
self.context = context
self._mimeType = None
self._encoding = 'utf-8'
self._closed = False
self._name = None
self._written = 0
self._parser = BytesFeedParser()
self._message = None
@property
def mimeType(self):
if self._message is None:
return self._mimeType
elif not self._message.is_multipart():
return 'text/plain'
else:
return 'message/rfc822'
@mimeType.setter
def mimeType(self, value):
self._mimeType = value
@property
def encoding(self):
if self._message is not None:
return self._message.get_charset() or self._encoding
return self._encoding
@encoding.setter
def encoding(self, value):
self._encoding = value
@property
def closed(self):
return self._closed
@property
def name(self):
if self._message is not None:
return self._message.get_filename(self._name)
return self._name
@name.setter
def name(self, value):
self._name = value
def seek(self, offset, whence=None):
raise NotImplementedError("Seeking is not supported")
def tell(self):
return self._written
def close(self):
self._message = self._parser.close()
self._closed = True
initializeObjectFromSchemata(self.context, iterSchemata(self.context),
self._message, self._encoding)
def write(self, data):
if self._closed:
raise ValueError("File is closed")
if isinstance(data, six.text_type):
data = data.encode()
self._written += len(data)
self._parser.feed(data)
def writelines(self, sequence):
for item in sequence:
self.write(item)
def truncate(self, size=None):
if (size is None and self._written != 0) and size != 0:
raise NotImplementedError(
"The 'size' argument to truncate() must be 0 - partial "
"truncation is not supported")
if self._closed:
raise ValueError("File is closed")
self._parser = BytesFeedParser()
self._written = 0
def flush(self):
pass
def get_email_requests(redis_client, IMAP_SERVER, IMAP_USERNAME,
IMAP_PASSWORD):
"""
This function connects to an email server and retrieves all new emails to
identify new VPN requests.
"""
email_requests = []
msg_type = "email"
try:
# Connect to email
mail = imaplib.IMAP4_SSL(IMAP_SERVER)
mail.login(IMAP_USERNAME, IMAP_PASSWORD)
logging.debug("Connected to account successful")
# Connect to Inbox. Readyonly option: False=marks msgs as read; True=keep messages as unread.
mail.select("Inbox", readonly=False)
# Search and return UIDS of all UNSEEN/UNREAD emails in Inbox
result, data = mail.uid('search', None, "UNSEEN")
# We receive a list of unread email UID
id_list = data[0].split() # data is a list.
logging.info("Found {} new requests to process".format(len(id_list)))
# Process the new unread emails. If zero, nothing returns
while len(id_list) > 0:
# In case another process checked emails simultaneously, we refresh the list of ids
result, data = mail.uid('search', None, "UNSEEN")
id_list = data[0].split() # data is a list.
# Get the first email ID to process
email_uid = id_list.pop(0)
logging.debug(f"Processing email UID {email_uid}")
# Fetch the email headers and body (RFC822) for the given email UID
result, data = mail.uid('fetch', email_uid, '(RFC822)')
# Parse email to extract header and body
email_parser = BytesFeedParser()
email_parser.feed(data[0][1])
msg = email_parser.close()
# Do not process answers to the emails we send
if msg['In-Reply-To'] is not None:
continue
# Parse email receiver
email_to = re.search(r'[\w\.-]+@[\w\.-]+', msg['to']).group(0)
logging.debug(f"Processing email receiver {email_to}")
# Do not process messages where we are not the receivers
if not email_to == IMAP_USERNAME:
continue
# Parse subject and find matches for keyword VPN
email_subject = ""
msg_request = ""
try:
email_subject = re.search(r'NOENCRYPTEDVPN', msg['subject'],
re.IGNORECASE).group(0)
msg_request = "novpn"
except:
try:
email_subject = re.search(r'NOTENCRYPTEDVPN',
msg['subject'],
re.IGNORECASE).group(0)
msg_request = "novpn"
except:
try:
email_subject = re.search(r'WIREGUARD', msg['subject'],
re.IGNORECASE).group(0)
msg_request = "wireguard"
except:
try:
email_subject = re.search(r'VPN', msg['subject'],
re.IGNORECASE).group(0)
msg_request = "openvpn"
except:
pass
logging.debug(
f"Extracted email subject: {email_subject} ({msg_request})")
# Parse email body and find matches for keyword VPN
try:
# Extract email body in rich email
email_body = msg.get_payload().pop().get_payload()
except:
# Extract email body in plain email
email_body = msg.get_payload()
try:
email_body = re.search(r'NOENCRYPTEDVPN', email_body,
re.IGNORECASE).group(0)
msg_request = "novpn"
except:
try:
email_body = re.search(r'NOTENCRYPTEDVPN', email_body,
re.IGNORECASE).group(0)
msg_request = "novpn"
except:
try:
email_body = re.search(r'WIREGUARD', email_body,
re.IGNORECASE).group(0)
msg_request = "wireguard"
except:
try:
email_body = re.search(r'VPN', email_body,
re.IGNORECASE).group(0)
msg_request = "openvpn"
except:
email_body = ""
logging.debug(
f"Extracted email body: {email_body} ({msg_request})")
# We only parse messages that contain VPN in subject or body
# These prints will be removed after we test everything is good
if msg_request != "":
logging.info(
f"Extracted email request: {email_subject}:{email_body}({msg_request})"
)
# Parse email date
email_date = msg['date']
# Parse email sender
email_from = re.search(r'[\w\.-]+@[\w\.-]+',
msg['from']).group(0)
# Write pending account to provision in REDIS
send_request_to_redis(int(email_uid), email_from, msg_type,
msg_request, logging, redis_client)
# Notify manager of new request
redis_client.publish('services_status',
'MOD_COMM_RECV:NEW_REQUEST')
logging.debug("This email matches the keywords")
logging.debug('{:8}: {}'.format("email id", int(email_uid)))
logging.debug('{:8}: {}'.format("date", email_date))
logging.debug('{:8}: {}'.format("to", email_to))
logging.debug('{:8}: {}'.format("from", email_from))
logging.debug('{:8}: {}'.format("reply_to",
msg['In-Reply-To']))
logging.debug('{:8}: {}'.format("subject", email_subject))
logging.debug('{:8}: {}'.format("body", email_body))
else:
logging.debug("This email does not match the keywords")
logging.debug('{:8}: {}'.format("Email ID", email_uid))
logging.debug('{:8}: {}'.format("Date", email_date))
logging.debug('{:8}: {}'.format("To", email_to))
logging.debug('{:8}: {}'.format("From", email_from))
logging.debug('{:8}: {}'.format("reply_to",
msg['In-Reply-To']))
logging.debug('{:8}: {}'.format("Subject", email_subject))
logging.debug('{:8}: {}'.format("Body", email_body))
# Close connection to server
mail.expunge()
mail.close()
mail.logout()
return True
except Exception as e:
print(e)
return False