def process_archive(self, peer, sender, mail_options, recips, rcptopts, data):
"""Archives email meta data using a Backend"""
LOG(E_INFO, "%s: Sender is <%s> - Recipients (Envelope): %s" % (self.type, sender, ",".join(recips)))
size = len(data)
if size < MINSIZE:
return self.do_exit(550, "Invalid Mail")
if not data.endswith(NL):
data = data + NL
args = {}
aid = None
mid = None
stream = StringIO(data)
msg = Message(stream)
if sender == "":
LOG(E_INFO, "%s: Null return path mail, not archived" % (self.type))
return self.sendmail("<>", mail_options, recips, rcptopts, data, aid)
## Check if I have msgid in my cache
mid = msg.get("message-id", self.new_mid())
hash = hash_headers(msg.get)
if self.hashdb.has_key(hash):
LOG(E_TRACE, "%s: Message-id: %s" % (self.type, mid))
aid = self.hashdb[hash]
LOG(E_TRACE, "%s: Message already has year/pid pair, only adding header" % self.type)
return self.sendmail(sender, mail_options, recips, rcptopts, self.add_aid(data, msg, aid), aid, hash)
args["m_mid"] = mid
args["hash"] = hash
## Check for duplicate headers
dupe = dupe_check(msg.headers)
if dupe is not None:
LOG(E_ERR, "%s: Duplicate header %s" % (self.type, dupe))
return self.do_exit(552, "Duplicate header %s" % dupe)
## Extraction of From field
m_from = msg.getaddrlist("From")
if len(m_from) == 1:
m_from = safe_parseaddr(m_from[0][1])
else:
m_from = None
## Empty or invalid 'From' field, try to use sender
if m_from is None:
LOG(E_ERR, "%s: no From header in mail using sender" % self.type)
m_from = safe_parseaddr(sender)
## No luck
if m_from is None:
return self.do_exit(552, "Mail has not suitable From/Sender")
args["m_from"] = m_from
## Extract 'To' field
m_to = []
for h in msg.getaddrlist("To"):
rec = safe_parseaddr(h[1])
if rec is None:
continue
m_to.append(rec)
## Empty 'To' field use recipients
if len(m_to) == 0:
LOG(E_ERR, "%s: no To header in mail using recipients" % self.type)
for recipient in recips:
rec = safe_parseaddr(recipient)
if rec is None:
continue
m_to.append(rec)
if len(m_to) == 0:
return self.do_exit(552, "Mail has not suitable To/Recipient")
## Extract 'Cc' field
for h in msg.getaddrlist("Cc"):
rec = safe_parseaddr(h[1])
if rec is None:
continue
m_to.append(rec)
## Cleanup: remove duplicates
recs = []
for rec in m_to:
if rec not in recs:
recs.append(rec)
args["m_rec"] = recs
## Extract 'Subject' field
m_sub = mime_decode_header(msg.get("Subject", "No Subject"))
if subjpattern is not None and m_sub.find(subjpattern) != -1:
LOG(E_INFO, "%s: Subject pattern matched, not archived" % self.type)
return self.sendmail(sender, mail_options, recips, rcptopts, self.remove_aid(data, msg))
args["m_sub"] = m_sub
## Whitelist check: From, To and Sender (envelope)
checklist = [m_from] + m_to
ss = safe_parseaddr(sender)
if ss is not None:
checklist.append(ss)
for check in checklist:
if check.split("@", 1)[0] in whitelist:
LOG(E_INFO, "%s: Mail to: %s in whitelist, not archived" % (self.type, check))
return self.sendmail(sender, mail_options, recips, rcptopts, self.remove_aid(data, msg))
## Sender size limit check - in kb
if dbchecker is not None and dbchecker.quota_check(m_from, size >> 10):
return self.do_exit(422, "Sender quota execeded")
args["m_size"] = size
## Extract 'Date' field
m_date = None
if self.datefromemail:
m_date = msg.getdate("Date")
try:
mktime(m_date)
except:
m_date = None
if m_date is None:
m_date = localtime(time())
args["m_date"] = m_date
m_attach = []
if msg.maintype != "multipart":
m_parse = parse_message(msg)
if m_parse is not None:
m_attach.append(m_parse)
else:
filepart = MultiFile(stream)
filepart.push(msg.getparam("boundary"))
try:
while filepart.next():
submsg = Message(filepart)
subpart = parse_message(submsg)
if subpart is not None:
m_attach.append(subpart)
except:
LOG(E_ERR, "%s: Error in multipart splitting" % self.type)
args["m_attach"] = m_attach
if dbchecker is not None:
## Collect data for mb lookup
addrs = []
for addr in [m_from] + m_to:
addrs.append(addr)
args["m_mboxes"] = dbchecker.mblookup(addrs)
else:
args["m_mboxes"] = []
year, pid, error = self.backend.process(args)
if year == 0:
LOG(E_ERR, "%s: Backend Error: %s" % (self.type, error))
return self.do_exit(pid, error)
## Adding X-Archiver-ID: header
aid = "%d-%d" % (year, pid)
data = self.add_aid(data, msg, aid)
LOG(E_TRACE, "%s: inserting %s msg in hashdb" % (self.type, aid))
self.hashdb[hash] = aid
self.hashdb.sync()
## Next hop
LOG(E_TRACE, "%s: backend worked fine" % self.type)
LOG(E_TRACE, "%s: passing data to nexthop: %s:%s" % (self.type, self.output_address, self.output_port))
return self.sendmail(sender, mail_options, recips, rcptopts, data, aid, hash)
def process_archive(self, peer, sender, mail_options, recips, rcptopts,
data):
"""Archives email meta data using a Backend"""
LOG(
E_INFO, '%s: Sender is <%s> - Recipients (Envelope): %s' %
(self.type, sender, ','.join(recips)))
size = len(data)
if size < MINSIZE:
return self.do_exit(550, 'Invalid Mail')
if not data.endswith(NL):
data = data + NL
args = {}
aid = None
mid = None
stream = StringIO(data)
msg = Message(stream)
if sender == '':
LOG(E_INFO,
'%s: Null return path mail, not archived' % (self.type))
return self.sendmail('<>', mail_options, recips, rcptopts,
data, aid)
## Check if I have msgid in my cache
mid = msg.get('message-id', self.new_mid())
hash = hash_headers(msg.get)
if self.hashdb.has_key(hash):
LOG(E_TRACE, '%s: Message-id: %s' % (self.type, mid))
aid = self.hashdb[hash]
LOG(
E_TRACE,
'%s: Message already has year/pid pair, only adding header'
% self.type)
return self.sendmail(sender, mail_options, recips, rcptopts,
self.add_aid(data, msg, aid), aid, hash)
args['m_mid'] = mid
args['hash'] = hash
## Check for duplicate headers
dupe = dupe_check(msg.headers)
if dupe is not None:
LOG(E_ERR, '%s: Duplicate header %s' % (self.type, dupe))
return self.do_exit(552, 'Duplicate header %s' % dupe)
## Extraction of From field
m_from = msg.getaddrlist('From')
if len(m_from) == 1:
m_from = safe_parseaddr(m_from[0][1])
else:
m_from = None
## Empty or invalid 'From' field, try to use sender
if m_from is None:
LOG(E_ERR,
'%s: no From header in mail using sender' % self.type)
m_from = safe_parseaddr(sender)
## No luck
if m_from is None:
return self.do_exit(552, 'Mail has not suitable From/Sender')
args['m_from'] = m_from
## Extract 'To' field
m_to = []
for h in msg.getaddrlist('To'):
rec = safe_parseaddr(h[1])
if rec is None: continue
m_to.append(rec)
## Empty 'To' field use recipients
if len(m_to) == 0:
LOG(E_ERR,
'%s: no To header in mail using recipients' % self.type)
for recipient in recips:
rec = safe_parseaddr(recipient)
if rec is None:
continue
m_to.append(rec)
if len(m_to) == 0:
return self.do_exit(552,
'Mail has not suitable To/Recipient')
## Extract 'Cc' field
for h in msg.getaddrlist('Cc'):
rec = safe_parseaddr(h[1])
if rec is None: continue
m_to.append(rec)
## Cleanup: remove duplicates
recs = []
for rec in m_to:
if rec not in recs:
recs.append(rec)
args['m_rec'] = recs
## Extract 'Subject' field
m_sub = mime_decode_header(msg.get('Subject', 'No Subject'))
if subjpattern is not None and m_sub.find(subjpattern) != -1:
LOG(E_INFO,
'%s: Subject pattern matched, not archived' % self.type)
return self.sendmail(sender, mail_options, recips, rcptopts,
self.remove_aid(data, msg))
args['m_sub'] = m_sub
## Whitelist check: From, To and Sender (envelope)
checklist = [m_from] + m_to
ss = safe_parseaddr(sender)
if ss is not None:
checklist.append(ss)
for check in checklist:
if check.split('@', 1)[0] in whitelist:
LOG(
E_INFO, '%s: Mail to: %s in whitelist, not archived' %
(self.type, check))
return self.sendmail(sender, mail_options, recips,
rcptopts, self.remove_aid(data, msg))
## Sender size limit check - in kb
if dbchecker is not None and dbchecker.quota_check(
m_from, size >> 10):
return self.do_exit(422, 'Sender quota execeded')
args['m_size'] = size
## Extract 'Date' field
m_date = None
if self.datefromemail:
m_date = msg.getdate('Date')
try:
mktime(m_date)
except:
m_date = None
if m_date is None:
m_date = localtime(time())
args['m_date'] = m_date
m_attach = []
if msg.maintype != 'multipart':
m_parse = parse_message(msg)
if m_parse is not None:
m_attach.append(m_parse)
else:
filepart = MultiFile(stream)
filepart.push(msg.getparam('boundary'))
try:
while filepart.next():
submsg = Message(filepart)
subpart = parse_message(submsg)
if subpart is not None:
m_attach.append(subpart)
except:
LOG(E_ERR, '%s: Error in multipart splitting' % self.type)
args['m_attach'] = m_attach
if dbchecker is not None:
## Collect data for mb lookup
addrs = []
for addr in [m_from] + m_to:
addrs.append(addr)
args['m_mboxes'] = dbchecker.mblookup(addrs)
else:
args['m_mboxes'] = []
year, pid, error = self.backend.process(args)
if year == 0:
LOG(E_ERR, '%s: Backend Error: %s' % (self.type, error))
return self.do_exit(pid, error)
## Adding X-Archiver-ID: header
aid = '%d-%d' % (year, pid)
data = self.add_aid(data, msg, aid)
LOG(E_TRACE, '%s: inserting %s msg in hashdb' % (self.type, aid))
self.hashdb[hash] = aid
self.hashdb.sync()
## Next hop
LOG(E_TRACE, '%s: backend worked fine' % self.type)
LOG(
E_TRACE, '%s: passing data to nexthop: %s:%s' %
(self.type, self.output_address, self.output_port))
return self.sendmail(sender, mail_options, recips, rcptopts, data,
aid, hash)
def process_storage(self, peer, sender, mail_options, recips, rcptopts, data):
"""Stores the archived email using a Backend"""
size = len(data)
if size < MINSIZE:
return self.do_exit(550, "Invalid Mail")
if not data.endswith(NL):
data = data + NL
stream = StringIO(data)
msg = Message(stream)
aid = msg.get(AID, None)
## Check if I have msgid in my cache
mid = msg.get("message-id", self.new_mid())
LOG(E_TRACE, "%s: Message-id: %s" % (self.type, mid))
hash = hash_headers(msg.get)
if self.hashdb.has_key(hash):
aid = self.hashdb[hash]
LOG(E_ERR, "%s: Message already processed" % self.type)
return self.sendmail(sender, mail_options, recips, rcptopts, data, aid, hash)
## Date extraction
m_date = None
if self.datefromemail:
m_date = msg.getdate("Date")
try:
mktime(m_date)
except:
m_date = None
if m_date is None:
m_date = localtime(time())
del msg, stream
## Mail needs to be processed
if aid:
try:
year, pid = aid.split("-", 1)
year = int(year)
pid = int(pid)
except:
t, val, tb = exc_info()
del tb
LOG(E_ERR, "%s: Invalid X-Archiver-ID header [%s]" % (self.type, str(val)))
return self.do_exit(550, "Invalid X-Archiver-ID header")
args = dict(mail=data, year=year, pid=pid, date=m_date, mid=mid, hash=hash)
LOG(E_TRACE, "%s: year is %d - pid is %d (%s)" % (self.type, year, pid, mid))
status, code, msg = self.backend.process(args)
if status == 0:
LOG(E_ERR, "%s: process failed %s" % (self.type, msg))
return self.do_exit(code, msg)
## Inserting in hashdb
LOG(E_TRACE, "%s: inserting %s msg in hashdb" % (self.type, aid))
self.hashdb[hash] = aid
self.hashdb.sync()
LOG(E_TRACE, "%s: backend worked fine" % self.type)
else:
## Mail in whitelist - not processed
LOG(E_TRACE, "%s: X-Archiver-ID header not found in mail [whitelist]" % self.type)
## Next hop
LOG(E_TRACE, "%s: passing data to nexthop: %s:%s" % (self.type, self.output_address, self.output_port))
return self.sendmail(sender, mail_options, recips, rcptopts, data, aid, hash)
def process_storage(self, peer, sender, mail_options, recips, rcptopts,
data):
"""Stores the archived email using a Backend"""
size = len(data)
if size < MINSIZE:
return self.do_exit(550, 'Invalid Mail')
if not data.endswith(NL):
data = data + NL
stream = StringIO(data)
msg = Message(stream)
aid = msg.get(AID, None)
## Check if I have msgid in my cache
mid = msg.get('message-id', self.new_mid())
LOG(E_TRACE, '%s: Message-id: %s' % (self.type, mid))
hash = hash_headers(msg.get)
if self.hashdb.has_key(hash):
aid = self.hashdb[hash]
LOG(E_ERR, '%s: Message already processed' % self.type)
return self.sendmail(sender, mail_options, recips, rcptopts,
data, aid, hash)
## Date extraction
m_date = None
if self.datefromemail:
m_date = msg.getdate('Date')
try:
mktime(m_date)
except:
m_date = None
if m_date is None:
m_date = localtime(time())
del msg, stream
## Mail needs to be processed
if aid:
try:
year, pid = aid.split('-', 1)
year = int(year)
pid = int(pid)
except:
t, val, tb = exc_info()
del tb
LOG(
E_ERR, '%s: Invalid X-Archiver-ID header [%s]' %
(self.type, str(val)))
return self.do_exit(550, 'Invalid X-Archiver-ID header')
args = dict(mail=data,
year=year,
pid=pid,
date=m_date,
mid=mid,
hash=hash)
LOG(
E_TRACE, '%s: year is %d - pid is %d (%s)' %
(self.type, year, pid, mid))
status, code, msg = self.backend.process(args)
if status == 0:
LOG(E_ERR, '%s: process failed %s' % (self.type, msg))
return self.do_exit(code, msg)
## Inserting in hashdb
LOG(E_TRACE,
'%s: inserting %s msg in hashdb' % (self.type, aid))
self.hashdb[hash] = aid
self.hashdb.sync()
LOG(E_TRACE, '%s: backend worked fine' % self.type)
else:
## Mail in whitelist - not processed
LOG(
E_TRACE,
'%s: X-Archiver-ID header not found in mail [whitelist]' %
self.type)
## Next hop
LOG(
E_TRACE, '%s: passing data to nexthop: %s:%s' %
(self.type, self.output_address, self.output_port))
return self.sendmail(sender, mail_options, recips, rcptopts, data,
aid, hash)