def check_object(self, file_obj, key, update=False):
"""
download a file from s3 as a in-memory object and attempts to parse it
as an email / extract possible attachments
"""
#logging.info(" ==> checking email:", key)
try:
eml = parse_email(key, file_obj, forwarded=True)
except Exception as excp:
eml = Email(key)
eml.errors.append(fmt(excp))
return eml
try:
# edge cases
destinations = email_destinations(eml, self.rules)
is_local = False
if not eml.errors and update and self.client:
destination_dir = os.path.join(self.attachment_dest_dir, destinations[0])
self.place_attachments(eml, destination_dir, is_local)
self.move_email(key, "Processed", eml.date)
except Error as error:
eml.errors.append(fmt(error))
except Exception as excp:
eml.errors.append(fmt(excp))
return eml
def generate_frontend_catchall_entry(domain, mode):
if mode == 'http':
return fmt('use_backend b_catchall_' + mode + ' if { hdr_dom(host) -i ' + domain + ' }')
elif mode == 'https':
return fmt('use_backend b_catchall_' + mode + ' if { req_ssl_sni -i ' + domain + ' }')
return None
def generate_frontend_catchall_entry(domain, mode):
if mode == 'http':
return fmt('use_backend b_catchall_' + mode + ' if { hdr_dom(host) -i ' + domain + ' }')
elif mode == 'https':
return fmt('use_backend b_catchall_' + mode + ' if { req_ssl_sni -i ' + domain + ' }')
return None
def generate_backend_catchall_entry(domain, mode, port, server_options, override_domain=None):
result = None
if mode == 'http':
result = fmt('use-server ' + domain + ' if { hdr_dom(host) -i ' + domain + ' }')
if override_domain is None:
result += fmt('server ' + domain + ' ' + domain + ':' + str(port) + ' ' + server_options + os.linesep)
else:
result += fmt('server ' + domain + ' ' + override_domain + ':' + str(port) + ' ' + server_options + os.linesep)
elif mode == 'https':
result = fmt('use-server ' + domain + ' if { req_ssl_sni -i ' + domain + ' }')
result += fmt('server ' + domain + ' ' + domain + ':' + str(port) + ' ' + server_options + os.linesep)
return result
def generate_backend_catchall_entry(domain, mode, port, server_options, override_domain=None):
result = None
if mode == 'http':
result = fmt('use-server ' + domain + ' if { hdr_dom(host) -i ' + domain + ' }')
if override_domain is None:
result += fmt('server ' + domain + ' ' + domain + ':' + str(port) + ' ' + server_options + os.linesep)
else:
result += fmt('server ' + domain + ' ' + override_domain + ':' + str(port) + ' ' + server_options + os.linesep)
elif mode == 'https':
result = fmt('use-server ' + domain + ' if { req_ssl_sni -i ' + domain + ' }')
result += fmt('server ' + domain + ' ' + domain + ':' + str(port) + ' ' + server_options + os.linesep)
return result
def generate_global():
result = fmt('global', indent=None)
result += fmt('daemon')
result += fmt('maxconn 20000')
result += fmt('user haproxy')
result += fmt('group haproxy')
result += fmt('stats socket /var/run/haproxy.sock mode 0600 level admin')
result += fmt('log 127.0.0.1 local1 notice')
result += fmt('pidfile /var/run/haproxy.pid')
result += fmt('spread-checks 5')
result += os.linesep
return result
def generate_global():
result = fmt('global', indent=None)
result += fmt('daemon')
result += fmt('maxconn 20000')
result += fmt('user haproxy')
result += fmt('group haproxy')
result += fmt('stats socket /var/run/haproxy.sock mode 0600 level admin')
result += fmt('log 127.0.0.1 local1 notice')
result += fmt('pidfile /var/run/haproxy.pid')
result += fmt('spread-checks 5')
result += os.linesep
return result
def parse_email(key, file_obj, forwarded=False):
msg = email.message_from_file(open( file_obj.name, "r"))
original_date = email.utils.parsedate(msg.get("Date"))
if forwarded:
try:
tmp = extract_forwarded_email(msg)
msg = tmp
except Error as error:
logging.info("not a forwarded email?",error)
eml = Email(key, msg, original_date)
try:
body = [None, None]
for part in msg.get_payload():
if part.get_filename():
eml.attachments.append(part)
else:
for subpart in part.walk():
subcharset = subpart.get_content_charset() or "utf8"
if subpart.get_content_type() == "text/plain":
text = subpart.get_payload(decode=True)
eml.text = text.decode(subcharset)
elif subpart.get_content_type() == "text/html":
html = subpart.get_payload(decode=True)
eml.html = html.decode(subcharset)
if eml.html:
parser = LinkParser()
parser.feed(eml.html)
eml.links = parser.links
if eml.date:
timestamp = time.mktime(eml.date)
eml.date = datetime.fromtimestamp(timestamp)
except Exception as excp:
eml.errors.append(fmt(excp))
return eml
def info(self, include_links=True):
atch_names = [get_attachment_name(a) for a in self.attachments]
lines = [
fmt(self.key),
fmt("From :", self.fr),
fmt("To :", self.to),
fmt("CC :", self.cc),
fmt("Subject :", self.subject),
fmt("Date :", self.date),
fmt("Attachments :", atch_names),
]
if include_links:
lines.append(fmt("Links :", self.links))
if self.errors:
for error in self.errors:
lines.append(fmt("ERROR :", error))
else:
lines.append(fmt("OKAY :", dest=self.destinations[0]))
return "\n" + "\n".join(lines)
def validate(self, destinations):
self.destinations = destinations
# edge-case
if len(destinations) > 1:
self.errors.append(fmt("multiple destinations found:", destinations))
if not destinations:
self.errors.append("no destinations found")
if not self.date:
self.errors.append("unable to extract received date")
if not self.attachments:
self.errors.append("no attachments found")
def generate_stats(stats, bind_ip):
result = fmt('listen stats', indent=None)
result += fmt('bind ' + bind_ip + ':' + str(stats["port"]))
result += fmt('mode http')
result += fmt('stats enable')
result += fmt('stats realm Protected\\ Area')
result += fmt('stats uri /')
result += fmt('stats auth ' + stats["user"] + ':' + stats["password"])
result += os.linesep
return result
def generate_stats(stats, bind_ip):
result = fmt('listen stats', indent=None)
result += fmt('bind ' + bind_ip + ':' + str(stats["port"]))
result += fmt('mode http')
result += fmt('stats enable')
result += fmt('stats realm Protected\\ Area')
result += fmt('stats uri /')
result += fmt('stats auth ' + stats["user"] + ':' + stats["password"])
result += os.linesep
return result
def generate_deadend(mode):
result = fmt('backend b_deadend_' + mode, indent=None)
if mode == 'http':
result += fmt('mode http')
result += fmt('option httplog')
result += fmt('option accept-invalid-http-response')
result += fmt('option http-server-close')
elif mode == 'https':
result += fmt('mode tcp')
result += fmt('option tcplog')
result += os.linesep
return result
def generate_deadend(mode):
result = fmt('backend b_deadend_' + mode, indent=None)
if mode == 'http':
result += fmt('mode http')
result += fmt('option httplog')
result += fmt('option accept-invalid-http-response')
result += fmt('option http-server-close')
elif mode == 'https':
result += fmt('mode tcp')
result += fmt('option tcplog')
result += os.linesep
return result
def generate_backend(proxy_name, mode, domain, port, server_options, is_catchall):
result = fmt('backend b_' + proxy_name + '_' + mode, indent=None)
if mode == 'http':
result += fmt('mode http')
result += fmt('option httplog')
result += fmt('option accept-invalid-http-response')
elif mode == 'https':
result += fmt('mode tcp')
result += fmt('option tcplog')
if not is_catchall:
result += fmt('server ' + domain + ' ' + domain + ':' + str(port) + ' ' + server_options)
return result + os.linesep
def generate_backend(proxy_name, mode, domain, port, server_options, is_catchall):
result = fmt('backend b_' + proxy_name + '_' + mode, indent=None)
if mode == 'http':
result += fmt('mode http')
result += fmt('option httplog')
result += fmt('option accept-invalid-http-response')
elif mode == 'https':
result += fmt('mode tcp')
result += fmt('option tcplog')
if not is_catchall:
result += fmt('server ' + domain + ' ' + domain + ':' + str(port) + ' ' + server_options)
return result + os.linesep
def generate_defaults():
result = fmt('defaults', indent=None)
result += fmt('maxconn 19500')
result += fmt('log global')
result += fmt('mode http')
result += fmt('option httplog')
result += fmt('option abortonclose')
result += fmt('option http-server-close')
result += fmt('option persist')
result += fmt('timeout connect 20s')
result += fmt('timeout client 120s')
result += fmt('timeout server 120s')
result += fmt('timeout queue 120s')
result += fmt('timeout check 10s')
result += fmt('retries 3')
result += os.linesep
return result
def generate_defaults():
result = fmt('defaults', indent=None)
result += fmt('maxconn 19500')
result += fmt('log global')
result += fmt('mode http')
result += fmt('option httplog')
result += fmt('option abortonclose')
result += fmt('option http-server-close')
result += fmt('option persist')
result += fmt('timeout connect 20s')
result += fmt('timeout client 120s')
result += fmt('timeout server 120s')
result += fmt('timeout queue 120s')
result += fmt('timeout check 10s')
result += fmt('retries 3')
result += os.linesep
return result
def generate_frontend(proxy_name, mode, bind_ip, current_port, is_catchall):
result = fmt('frontend f_' + proxy_name + '_' + mode, indent=None)
result += fmt('bind ' + bind_ip + ':' + str(current_port))
if mode == 'http':
result += fmt('mode http')
result += fmt('option httplog')
result += fmt('capture request header Host len 50')
result += fmt('capture request header User-Agent len 150')
elif mode == 'https':
result += fmt('mode tcp')
result += fmt('option tcplog')
if is_catchall:
result += fmt('tcp-request inspect-delay 5s')
result += fmt(
'tcp-request content accept if { req_ssl_hello_type 1 }')
if is_catchall:
result += fmt('default_backend b_deadend_' + mode)
else:
result += fmt('default_backend b_' + proxy_name + '_' + mode)
result += os.linesep
return result
def generate_frontend(proxy_name, mode, bind_ip, current_port, is_catchall):
result = fmt('frontend f_' + proxy_name + '_' + mode, indent=None)
result += fmt('bind ' + bind_ip + ':' + str(current_port))
if mode == 'http':
result += fmt('mode http')
result += fmt('option httplog')
result += fmt('capture request header Host len 50')
result += fmt('capture request header User-Agent len 150')
elif mode == 'https':
result += fmt('mode tcp')
result += fmt('option tcplog')
if is_catchall:
result += fmt('tcp-request inspect-delay 5s')
result += fmt('tcp-request content accept if { req_ssl_hello_type 1 }')
if is_catchall:
result += fmt('default_backend b_deadend_' + mode)
else:
result += fmt('default_backend b_' + proxy_name + '_' + mode)
result += os.linesep
return result
