def ti(mod=10000):
global counter
counter += 1
if not counter % mod:
message = '™%s· %d passed' % (' ' * local_indent, counter)
logger.debug(message)
print message
def proxyfunc(self, *args, **kw):
try:
client_ip = web.ctx.ip
except:
# No `ip` attr before starting http service.
return None
if not _is_allowed_client(client_ip):
logger.error(
'[{0}] Blocked request from disallowed client.'.format(
client_ip))
return api_render((False, 'NOT_AUTHORIZED_API_CLIENT'))
_auth_token = get_auth_token()
if not _auth_token:
return api_render((False, 'NO_API_AUTH_TOKEN'))
else:
logger.debug('[{0}] API AUTH TOKEN: {1:.8}...'.format(
client_ip, _auth_token))
if _auth_token not in settings.api_auth_tokens:
logger.error(
'[{0}] Blocked request with invalid auth token: {1}.'.format(
client_ip, _auth_token))
return api_render((False, 'INVALID_MLMMJADMIN_API_AUTH_TOKEN'))
return func(self, *args, **kw)
def ti(mod=10000):
global counter
counter += 1
if not counter % mod:
message = '™%s· %d passed' % (' ' * local_indent, counter)
logger.debug(message)
print message
def get_required_db_conns():
"""Establish SQL database connections."""
if settings.backend == 'ldap':
try:
conn_vmail = ldap.ldapobject.ReconnectLDAPObject(settings.ldap_uri)
logger.debug('LDAP connection initialied success.')
# Bind to ldap server.
try:
conn_vmail.bind_s(settings.ldap_binddn, settings.ldap_bindpw)
logger.debug('LDAP bind success.')
except ldap.INVALID_CREDENTIALS:
logger.error(
'LDAP bind failed: incorrect bind dn or password.')
except Exception as e:
logger.error("LDAP bind failed: {0}".format(repr(e)))
except Exception as e:
logger.error("Fail2ed to establish LDAP connection: {0}".format(
repr(e)))
conn_vmail = None
else:
# settings.backend in ['mysql', 'pgsql']
conn_vmail = get_db_conn('vmail')
conn_amavisd = get_db_conn('amavisd')
conn_iredapd = get_db_conn('iredapd')
return {
'conn_vmail': conn_vmail,
'conn_amavisd': conn_amavisd,
'conn_iredapd': conn_iredapd,
}
def found_terminator(self):
if self.buffer:
line = self.buffer.pop().decode()
logger.debug("{} input: {}".format(self.log_prefix, line))
if line.startswith('get '):
addr = line.strip().split(' ', 1)[-1]
if utils.is_email(addr):
domain = addr.split('@', 1)[-1]
if self.rewrite_address_type == 'sender':
reply = self.srs_forward(addr=addr, domain=domain)
logger.debug("{} {}".format(self.log_prefix, reply))
self.push(reply)
else:
reply = self.srs_reverse(addr=addr)
logger.debug("{} {}".format(self.log_prefix, reply))
self.push(reply)
else:
logger.debug("{} Not a valid email address, bypassed.".format(self.log_prefix))
self.push(TCP_REPLIES['not_exist'] + 'Not a valid email address, bypassed.')
else:
logger.debug("{} Unexpected input: {}".format(self.log_prefix, line))
self.push(TCP_REPLIES['not_exist'] + 'Unexpected input: {}'.format(line))
def get_access_policy(mail, account_type, conn):
"""Get access policy of (mlmmj) mailing list or mail alias account.
Returns access policy (string) or None if account doesn't exist."""
_policy = None
if account_type == 'alias':
table = 'alias'
elif account_type == 'maillist':
table = 'maillists'
else:
return _policy
sql = """SELECT accesspolicy
FROM %s
WHERE address=%s
LIMIT 1""" % (table, sqlquote(mail))
logger.debug("[SQL] query access policy: \n{}".format(sql))
qr = conn.execute(sql)
record = qr.fetchone()
logger.debug("[SQL] query result: {}".format(repr(record)))
if record:
_policy = str(record[0]).lower()
return _policy
def restriction(**kwargs):
rdns_name = kwargs['smtp_session_data']['reverse_client_name']
client_address = kwargs['smtp_session_data']['client_address']
# Bypass outgoing emails.
if kwargs['sasl_username']:
logger.debug('Found SASL username, bypass rDNS check for outbound.')
return SMTP_ACTIONS['default']
if rdns_name == 'unknown':
logger.debug('No reverse dns name, bypass.')
return SMTP_ACTIONS['default']
if is_trusted_client(client_address):
return SMTP_ACTIONS['default']
_policy_rdns_names = [rdns_name]
_splited = rdns_name.split('.')
for i in range(len(_splited)):
_name = '.' + '.'.join(_splited)
_policy_rdns_names.append(_name)
_splited.pop(0)
logger.debug('All policy rDNS names: %s' % repr(_policy_rdns_names))
conn = kwargs['conn_iredapd']
# Query whitelist
sql = """SELECT rdns
FROM wblist_rdns
WHERE rdns IN %s AND wb='W'
LIMIT 1""" % sqlquote(_policy_rdns_names)
logger.debug('[SQL] Query whitelisted rDNS names: \n%s' % sql)
qr = conn.execute(sql)
record = qr.fetchone()
if record:
rdns = str(record[0]).lower()
logger.info("[{}] Reverse client hostname is whitelisted: {}.".format(
client_address, rdns))
# better use 'DUNNO' instead of 'OK'
return SMTP_ACTIONS['default']
# Query blacklist
sql = """SELECT rdns
FROM wblist_rdns
WHERE rdns IN %s AND wb='B'
LIMIT 1""" % sqlquote(_policy_rdns_names)
logger.debug('[SQL] Query blacklisted rDNS names: \n%s' % sql)
qr = conn.execute(sql)
record = qr.fetchone()
if record:
rdns = str(record[0]).lower()
logger.info("[{}] Reverse client hostname is blacklisted: {}".format(
client_address, rdns))
return reject_action
return SMTP_ACTIONS['default']
def __add_subscribers_with_confirm(mail, subscribers, subscription='normal'):
"""
Add subscribers with confirm.
@mail -- mail address of mailing list
@subscribers -- a list/tuple/set of subscribers' mail addresses
@subscription -- subscription version (normal, digest, nomail)
"""
_dir = __get_ml_dir(mail)
# Get absolute path of command `mlmmj-sub`
_cmd_mlmmj_sub = settings.CMD_MLMMJ_SUB
if not _cmd_mlmmj_sub:
if os.path.exists('/usr/bin/mlmmj-sub'):
_cmd_mlmmj_sub = '/usr/bin/mlmmj-sub'
elif os.path.exists('/usr/local/bin/mlmmj-sub'):
_cmd_mlmmj_sub = '/usr/local/bin/mlmmj-sub'
else:
return (False, 'SUB_COMMAND_NOT_FOUND')
# mlmmj-sub arguments
#
# -L: Full path to list directory
# -a: Email address to subscribe
# -C: Request mail confirmation
# -d: Subscribe to `digest` version of the list
# -n: Subscribe to nomail version of the list
_cmd = [_cmd_mlmmj_sub, '-L', _dir, '-C']
if subscription == 'digest':
_cmd.append('-d')
elif subscription == 'nomail':
_cmd.append('-n')
# Directory used to store subscription confirm notifications
_subconf_dir = os.path.join(_dir, 'subconf')
_error = {}
for addr in subscribers:
try:
# Remove confirm file generated before this request
_old_conf_files = glob.glob(
os.path.join(_subconf_dir,
'????????????????-' + addr.replace('@', '=')))
for _f in _old_conf_files:
qr = __remove_file(path=_f)
if not qr[0]:
return qr
# Send new confirm
_new_cmd = _cmd[:] + ['-a', addr]
subprocess.Popen(_new_cmd, stdout=subprocess.PIPE)
logger.debug("[{0}] {1}, queued confirm mail for {2}.".format(
web.ctx.ip, mail, addr))
except Exception, e:
logger.error("[{0}] {1}, error while subscribing {2}: {3}".format(
web.ctx.ip, mail, addr, e))
_error[addr] = repr(e)
def __sendmail(conn,
user,
client_address,
throttle_tracking_id,
throttle_name,
throttle_value,
throttle_kind,
throttle_info,
throttle_value_unit=None):
"""Construct and send notification email."""
# conn: SQL connection cursor
# user: user email address
# client_address: client IP address
# throttle_tracking_id: value of sql column `throttle_tracking.id`
# throttle_name: name of throttle settings: msg_size, max_quota, max_msgs
# throttle_value: value throttle setting
# throttle_kind: one of throttle kinds: inbound, outbound
# throttle_info: detailed throttle setting
# throttle_value_unit: unit of throttle setting. e.g 'bytes' for max_quota
# and msg_size.
if not throttle_value_unit:
throttle_value_unit = ''
try:
_subject = 'Throttle quota exceeded: %s, %s=%d %s' % (
user, throttle_name, throttle_value, throttle_value_unit)
_body = '- User: '******'\n'
_body += '- Client IP address: ' + client_address + '\n'
_body += '- Throttle type: ' + throttle_kind + '\n'
_body += '- Throttle setting: ' + throttle_name + '\n'
_body += '- Limit: %d %s\n' % (throttle_value, throttle_value_unit)
_body += '- Detailed setting: ' + throttle_info + '\n'
utils.sendmail(subject=_subject, mail_body=_body)
logger.info(
'Sent notification email to admin(s) to report quota exceed: user=%s, %s=%d.'
% (user, throttle_name, throttle_value))
if throttle_tracking_id:
_now = int(time.time())
# Update last_notify_time.
_sql = """UPDATE throttle_tracking
SET last_notify_time=%d
WHERE id=%d;
""" % (_now, throttle_tracking_id)
try:
conn.execute(_sql)
logger.debug('Updated last notify time.')
except Exception as e:
logger.error(
'Error while updating last notify time of quota exceed: %s.'
% (repr(e)))
return (True, )
except Exception as e:
logger.error('Error while sending notification email: %s' % repr(e))
return (False, repr(e))
def tc(mod=10000):
global local_indent, start, name, counter
counter += 1
if not counter % mod:
delta = time.time() - start
passed = " (%d passed)" % counter if counter else ''
message = '™%s→ %s%s in %.3f' % (' ' * local_indent, name, passed, delta)
logger.debug(message)
print message
def ts(title='inline code'):
global local_indent, start, name, counter
counter = 0
name = title
message = '™%s↓ %s ' % (' ' * local_indent, name)
logger.debug(message)
print message
local_indent += 1
start = time.time()
def tp():
global local_indent, start, name, counter
finished = time.time()
delta = finished - start
local_indent -= 1
passed = "(%d passed)" % counter if counter else ''
message = '™%s↑ %s%s in %.3f' % (' ' * local_indent, name, passed, delta)
logger.debug(message)
print message
def ts(title='inline code'):
global local_indent, start, name, counter
counter = 0
name = title
message = '™%s↓ %s ' % (' ' * local_indent, name)
logger.debug(message)
print message
local_indent += 1
start = time.time()
def tp():
global local_indent, start, name, counter
finished = time.time()
delta = finished - start
local_indent -= 1
passed = "(%d passed)" % counter if counter else ''
message = '™%s↑ %s%s in %.3f' % (' ' * local_indent, name, passed, delta)
logger.debug(message)
print message
def tc(mod=10000):
global local_indent, start, name, counter
counter += 1
if not counter % mod:
delta = time.time() - start
passed = " (%d passed)" % counter if counter else ''
message = '™%s→ %s%s in %.3f' % (' ' * local_indent, name, passed,
delta)
logger.debug(message)
print message
def is_allowed_server_in_spf(sender_domain, ip):
"""
Check whether given IP address is listed in SPF DNS record of given
sender domain. Return True if exists, False if not.
"""
if (not sender_domain) or (not ip):
return False
qr = query_spf(domain=sender_domain, queried_domains=None)
_spf = qr['spf']
queried_domains = qr['queried_domains']
if not _spf:
logger.debug("[SPF] Domain {} does not have a valid SPF DNS record.".format(sender_domain))
return False
qr = parse_spf(domain=sender_domain,
spf=_spf,
queried_domains=queried_domains)
_ips = qr['ips']
if ip in _ips:
logger.debug("[SPF] IP {} is listed in SPF DNS record of sender domain {}.".format(ip, sender_domain))
return True
_ip_object = ipaddress.ip_address(ip)
_cidrs = []
# Get CIDR networks
if _ip_object.version == 4:
# if `ip=a.b.c.d`, ip prefix = `a.`
_ipv4_prefix = ip.split('.', 1)[0] + '.'
_cidrs = [i for i in _ips if (i.startswith(_ipv4_prefix) and '.0/' in i)]
elif _ip_object.version == 6:
_cidrs = [i for i in _ips if (':' in i and '/' in i)]
if _cidrs:
for _cidr in _cidrs:
try:
_network = ipaddress.ip_network(_cidr)
if _ip_object in _network:
logger.debug("[SPF] IP ({}) is listed in SPF DNS record "
"of sender domain {} "
"(network={}).".format(ip, sender_domain, _cidr))
return True
except Exception as e:
logger.debug("[SPF] Error while checking IP {} against network {}: {}".format(ip, _cidr, repr(e)))
logger.debug("[SPF] IP {} is NOT listed in SPF DNS record of domain {}.".format(ip, sender_domain))
return False
def apply_plugin(plugin, **kwargs):
action = SMTP_ACTIONS['default']
plugin_name = plugin.__name__
logger.debug("--> Apply plugin: {}".format(plugin_name))
try:
action = plugin.restriction(**kwargs)
logger.debug("<-- Result: {}".format(action))
except:
err_msg = get_traceback()
logger.error("<!> Error while applying plugin '{}': {}".format(plugin_name, err_msg))
return action
def apply_plugin(plugin, **kwargs):
action = SMTP_ACTIONS['default']
plugin_name = plugin.__name__
logger.debug(f"--> Apply plugin: {plugin_name}")
try:
action = plugin.restriction(**kwargs)
logger.debug(f"<-- Result: {action}")
except:
err_msg = get_traceback()
logger.error(
f"<!> Error while applying plugin '{plugin_name}': {err_msg}")
return action
def get_id_of_external_addresses(conn, addresses):
"""Return list of `mailaddr.id` of external addresses."""
ids = []
if not addresses:
logger.debug("No addresses, return empty list of ids.")
return ids
# Get `mailaddr.id` of external addresses, ordered by priority
sql = """SELECT id, email
FROM mailaddr
WHERE email IN %s
ORDER BY priority DESC""" % sqlquote(addresses)
logger.debug("[SQL] Query external addresses: \n{}".format(sql))
try:
qr = conn.execute(sql)
qr_addresses = qr.fetchall()
except Exception as e:
logger.error(
"Error while getting list of id of external addresses: {}, SQL: {}"
.format(repr(e), sql))
return ids
if qr_addresses:
ids = [int(r.id) for r in qr_addresses]
if not ids:
# don't waste time if we don't even have senders stored in sql db.
logger.debug("No record found in SQL database.")
return []
else:
logger.debug("Addresses (in `mailaddr`): {}".format(qr_addresses))
return ids
def apply_outbound_wblist(conn, sender_ids, recipient_ids):
# Return if no valid sender or recipient id.
if not (sender_ids and recipient_ids):
logger.debug("No valid sender id or recipient id.")
return SMTP_ACTIONS["default"]
# Bypass outgoing emails.
if settings.WBLIST_BYPASS_OUTGOING_EMAIL:
logger.debug(
"Bypass outgoing email as defined in WBLIST_BYPASS_OUTGOING_EMAIL."
)
return SMTP_ACTIONS["default"]
# Get wblist
sql = """SELECT rid, sid, wb
FROM outbound_wblist
WHERE sid IN %s
AND rid IN %s""" % (sqlquote(sender_ids),
sqlquote(recipient_ids))
logger.debug("[SQL] Query outbound wblist: \n{}".format(sql))
qr = conn.execute(sql)
wblists = qr.fetchall()
if not wblists:
# no wblist
logger.debug("No wblist found.")
return SMTP_ACTIONS["default"]
logger.debug("Found outbound wblist: {}".format(wblists))
# Check sender addresses
# rids/recipients are orded by priority
for sid in sender_ids:
for rid in recipient_ids:
if (rid, sid, "W") in wblists:
logger.info(
"Whitelisted: outbound_wblist=({}, {}, 'W')".format(
rid, sid))
return SMTP_ACTIONS[
"default"] + " outbound_wblist=({}, {}, 'W')".format(
rid, sid)
if (rid, sid, "B") in wblists:
logger.info(
"Blacklisted: outbound_wblist=({}, {}, 'B')".format(
rid, sid))
return reject_action
return SMTP_ACTIONS["default"]
def get_required_db_conns():
"""Establish SQL database connections."""
if settings.backend == 'ldap':
try:
conn_vmail = ldap.ldapobject.ReconnectLDAPObject(settings.ldap_uri)
logger.debug('LDAP connection initialied success.')
# Bind to ldap server.
try:
conn_vmail.bind_s(settings.ldap_binddn, settings.ldap_bindpw)
logger.debug('LDAP bind success.')
except ldap.INVALID_CREDENTIALS:
logger.error(
'LDAP bind failed: incorrect bind dn or password.')
except Exception as e:
logger.error("LDAP bind failed: {}".format(repr(e)))
except Exception as e:
logger.error("Fail2ed to establish LDAP connection: {}".format(
repr(e)))
conn_vmail = None
else:
# settings.backend in ['mysql', 'pgsql']
conn_vmail = get_db_conn('vmail')
conn_amavisd = get_db_conn('amavisd')
conn_iredapd = get_db_conn('iredapd')
# Create an in-memory SQLite database for global smtp session tracking.
# - gather data at RCPT state
# - used in END-OF-MESSAGE state
# - clean up after applied all enabled plugins
conn_tracking = sqlite3.connect(':memory:')
try:
with conn_tracking as c:
# Create SQL table used to store tracking data.
c.executescript("""
CREATE TABLE IF NOT EXISTS tracking (instance text, num_processed integer, init_time integer);
CREATE UNIQUE INDEX idx_tracking ON tracking (instance);
""")
except Exception as e:
logger.error("error while creating SQLite table and index: %s" %
repr(e))
return {
'conn_vmail': conn_vmail,
'conn_amavisd': conn_amavisd,
'conn_iredapd': conn_iredapd,
'conn_tracking': conn_tracking,
}
def is_available(domain):
"""Check if the domain name is registered"""
api = domain_check_api.format(domain)
result = request.urlopen(api.format(domain)).read().decode('utf-8')
logger.debug(result)
if '<original>210' in result:
# domain is available
return True
elif '<original>211' in result:
# domain is not available
return False
else:
logger.warning('api Forbidden')
return False
def post(self, request, *args, **kwargs):
logger.debug('¶')
steps = list()
for step in request.POST:
if step not in Steps.get_steps():
continue
steps.append(step)
steps = sorted(steps, key=lambda step: Steps.step_index(step))
app = Steps()
for step in steps:
print '#' * 80
print '#' * 1, step
print '#' * 80
getattr(app, step)()
return super(MainView, self).get(request, *args, **kwargs)
def call(*args, **kwargs):
global local_indent
name = f.__name__ if not title else title
message = '™%s↓ %s ' % (' ' * local_indent, name)
logger.debug(message)
print message
local_indent += 1
start = time.time()
res = f(*args, **kwargs)
finished = time.time()
delta = finished - start
local_indent -= 1
message = '™%s↑ %s in %.3f' % (' ' * local_indent, name, delta)
logger.debug(message)
print message
return res
def call(*args, **kwargs):
global local_indent
name = f.__name__ if not title else title
message = '™%s↓ %s ' % (' ' * local_indent, name)
logger.debug(message)
print message
local_indent += 1
start = time.time()
res = f(*args, **kwargs)
finished = time.time()
delta = finished - start
local_indent -= 1
message = '™%s↑ %s in %.3f' % (' ' * local_indent, name, delta)
logger.debug(message)
print message
return res
def is_allowed_alias_domain_user(sender, sender_username, sender_domain,
recipient_domain, rcpt_alias_domains,
restricted_members):
if sender_domain in rcpt_alias_domains:
policy_senders = [sender, sender_username + '@' + recipient_domain]
policy_senders += [
sender_username + '@' + d for d in rcpt_alias_domains
]
matched_senders = set(policy_senders) & set(restricted_members)
if matched_senders:
logger.debug('Matched alias domain user: %s' %
str(matched_senders))
return True
return False
def query_mx(domains, queried_domains=None, returned_ips=None):
"""
Return a list of IP addresses/networks defined in MX record of mail domain
names.
@domains - a list/tuple/set of mail domain names
@queried_domains - a set of mail domain names which already queried spf
@returned_ips - a set of IP addr/networks of queried mail domain names
"""
ips = set()
queried_domains = queried_domains or set()
returned_ips = returned_ips or set()
hostnames = set()
domains = [d for d in domains if d not in queried_domains]
for domain in domains:
try:
qr = resv.query(domain, 'MX')
if qr:
for r in qr:
hostname = str(r).split()[-1].rstrip('.')
logger.debug('[SPF][{0}] MX: {1}'.format(domain, hostname))
if utils.is_domain(hostname):
hostnames.add(hostname)
if hostnames:
qr = query_a(domains=hostnames,
queried_domains=queried_domains,
returned_ips=returned_ips)
ips_a = qr['ips']
queried_domains = qr['queried_domains']
returned_ips = qr['returned_ips']
ips.update(ips_a)
queried_domains.add('mx:' + domain)
except:
pass
return {
'ips': ips,
'queried_domains': queried_domains,
'returned_ips': returned_ips
}
def __remove_ml_sub_dir(mail, dirname):
if not dirname:
return (True, )
_ml_dir = __get_ml_dir(mail=mail)
_sub_dir = os.path.join(_ml_dir, dirname)
if os.path.exists(_sub_dir):
try:
shutil.rmtree(_sub_dir)
logger.debug("[{0}] {1}, removed sub-directory: {2}".format(
web.ctx.ip, mail, _sub_dir))
except Exception, e:
logger.error(
"[{0}] {1}, error while removing sub-directory: {2}".format(
web.ctx.ip, mail, _sub_dir))
return (False, repr(e))
def apply_outbound_wblist(conn, sender_ids, recipient_ids):
# Return if no valid sender or recipient id.
if not (sender_ids and recipient_ids):
logger.debug('No valid sender id or recipient id.')
return SMTP_ACTIONS['default']
# Bypass outgoing emails.
if settings.WBLIST_BYPASS_OUTGOING_EMAIL:
logger.debug(
'Bypass outgoing email as defined in WBLIST_BYPASS_OUTGOING_EMAIL.'
)
return SMTP_ACTIONS['default']
# Get wblist
sql = """SELECT rid, sid, wb
FROM outbound_wblist
WHERE sid IN {0} AND rid IN {1}""".format(
sqlquote(sender_ids), sqlquote(recipient_ids))
logger.debug('[SQL] Query outbound wblist: \n{0}'.format(sql))
qr = conn.execute(sql)
wblists = qr.fetchall()
if not wblists:
# no wblist
logger.debug('No wblist found.')
return SMTP_ACTIONS['default']
logger.debug(f'Found outbound wblist: {wblists}')
# Check sender addresses
# rids/recipients are orded by priority
for sid in sender_ids:
for rid in recipient_ids:
if (rid, sid, 'W') in wblists:
logger.info(
f"Whitelisted: outbound_wblist=({rid}, {sid}, 'W')")
return SMTP_ACTIONS[
'default'] + " outbound_wblist=({0}, {1}, 'W')".format(
rid, sid)
if (rid, sid, 'B') in wblists:
logger.info(
f"Blacklisted: outbound_wblist=({rid}, {sid}, 'B')")
return reject_action
return SMTP_ACTIONS['default']
def srs_forward(self, addr, domain):
# if domain is hostname, virtual mail domain or srs_domain, do not rewrite.
if domain == settings.srs_domain:
reply = TCP_REPLIES['not_exist'] + 'Domain is srs_domain, bypassed.'
return reply
elif domain == fqdn:
reply = TCP_REPLIES['not_exist'] + 'Domain is server hostname, bypassed.'
return reply
else:
_is_local_domain = False
try:
conn_vmail = self.db_conns['conn_vmail']
_is_local_domain = is_local_domain(conn=conn_vmail, domain=domain)
except Exception as e:
logger.error("{} Error while verifying domain: {}".format(self.log_prefix, repr(e)))
if _is_local_domain:
reply = TCP_REPLIES['not_exist'] + 'Domain is a local mail domain, bypassed.'
return reply
else:
possible_domains = []
_splited_parts = domain.split('.')
_length = len(_splited_parts)
for i in range(_length):
_part1 = '.'.join(_splited_parts[-i:])
_part2 = '.' + _part1
possible_domains += [_part1, _part2]
conn_iredapd = self.db_conns['conn_iredapd']
sql = """SELECT id FROM srs_exclude_domains WHERE domain IN %s LIMIT 1""" % sqlquote(list(possible_domains))
logger.debug("{} [SQL] Query srs_exclude_domains: {}".format(self.log_prefix, sql))
try:
qr = conn_iredapd.execute(sql)
sql_record = qr.fetchone()
logger.debug("{} [SQL] Query result: {}".format(self.log_prefix, sql_record))
except Exception as e:
logger.debug("{} Error while querying SQL: {}".format(self.log_prefix, repr(e)))
reply = TCP_REPLIES['not_exist']
return reply
if sql_record:
reply = TCP_REPLIES['not_exist'] + 'Domain is explicitly excluded, bypassed.'
return reply
else:
try:
new_addr = str(self.srslib_instance.forward(addr, settings.srs_domain))
logger.info("{} rewrote: {} -> {}".format(self.log_prefix, addr, new_addr))
reply = TCP_REPLIES['success'] + new_addr
return reply
except Exception as e:
logger.debug("{} Error while generating forward address: {}".format(self.log_prefix, repr(e)))
# Return original address.
reply = TCP_REPLIES['not_exist']
return reply
def restriction(**kwargs):
sender = kwargs['sender']
recipient = kwargs['recipient']
if recipient in settings.SPAM_TRAP_ACCOUNTS:
logger.debug('Spam trap recipient found: %s.' % recipient)
_block_sender(sender=sender)
return settings.SPAM_TRAP_SMTP_ACTION
for rcpt in settings.SPAM_TRAP_ACCOUNTS:
if rcpt.endswith('@'):
if recipient.startswith(rcpt):
logger.debug('Spam trap recipient found (matches: %s): %s.' %
(rcpt, recipient))
_block_sender(sender=sender)
return settings.SPAM_TRAP_SMTP_ACTION
return SMTP_ACTIONS['default']
def srs_reverse(self, addr):
# if address is not srs address, do not reverse.
_is_srs_address = self.srslib_instance.is_srs_address(addr, strict=True)
if _is_srs_address:
# Reverse
try:
new_addr = str(self.srslib_instance.reverse(addr))
logger.info("{} reversed: {} -> {}".format(self.log_prefix, addr, new_addr))
reply = TCP_REPLIES['success'] + new_addr
except Exception as e:
logger.debug("{} Error while generating reverse address: {}".format(self.log_prefix, repr(e)))
# Return original address.
reply = TCP_REPLIES['not_exist']
else:
reply = TCP_REPLIES['not_exist'] + 'Not a valid SRS address, bypassed.'
return reply
def query_spf(domain, queried_domains=None):
"""Return SPF record of given domain name."""
spf = None
queried_domains = queried_domains or set()
if 'spf:' + domain in queried_domains:
return {
'spf': None,
'queried_domains': queried_domains,
}
try:
# WARNING: DO NOT UPDATE queried_domains in this function
qr = resv.query(domain, 'TXT')
for r in qr:
# Remove heading/ending quotes
r = str(r).strip('"').strip("'")
# Some SPF records contains splited IP address like below:
# v=spf1 ... ip4:66.220.157" ".0/25 ...
# We should remove '"' and combine them.
_v = [v for v in r.split('"') if not v.startswith(' ')]
r = ''.join(_v)
if r.startswith('v=spf1'):
spf = r
break
except resolver.NoAnswer:
pass
except resolver.NXDOMAIN:
pass
except Exception as e:
logger.debug(
'[SPF] Error while querying DNS SPF record {0}: {1}'.format(
domain, e))
queried_domains.add('spf:' + domain)
return {
'spf': spf,
'queried_domains': queried_domains,
}
def get_moderators(conn, mail):
"""Return a list of moderators of mail alias account."""
_moderators = []
# Get access policy directly.
sql = """SELECT moderator
FROM moderators
WHERE address=%s""" % sqlquote(mail)
logger.debug('[SQL] query moderators: \n%s' % sql)
qr = conn.execute(sql)
records = qr.fetchall()
logger.debug('SQL query result: %s' % str(records))
if records:
for i in records:
_moderators.append(str(i[0]).lower())
return _moderators
def get_account_ldif(conn, account, query_filter=None, attrs=None):
logger.debug("[+] Getting LDIF data of account: {0}".format(account))
if not query_filter:
query_filter = '(&' + \
'(!(domainStatus=disabled))' + \
'(|(mail=%(account)s)(shadowAddress=%(account)s))' % {'account': account} + \
'(|' + \
'(objectClass=mailUser)' + \
'(objectClass=mailList)' + \
'(objectClass=mailAlias)' + \
'))'
logger.debug("search base dn: {0}\n"
"search scope: SUBTREE \n"
"search filter: {1}\n"
"search attributes: {2}".format(
settings.ldap_basedn,
query_filter,
attrs))
if not isinstance(attrs, list):
# Attribute list must be None (search all attributes) or non-empty list
attrs = None
try:
result = conn.search_s(settings.ldap_basedn,
ldap.SCOPE_SUBTREE,
query_filter,
attrs)
if result:
logger.debug("result: {0}".format(repr(result)))
# (dn, entry = result[0])
return result[0]
else:
logger.debug('No such account.')
return (None, None)
except Exception as e:
logger.debug("<!> ERROR: {0}".format(repr(e)))
return (None, None)
from libs.file import read_lines
from libs.logger import logger
from libs.timer import ts, tp
from libs.tools import dt
path = os.path.dirname(__file__)
os.chdir(path)
sys.path.append(path)
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "plagiat_clasterization.settings")
from main.steps import Steps
from main.models import News, NewsContent, NewsParagraph, NewsStemmed, \
NewsKeywords, ParagraphStemmed, ParagraphKeywords, NewsKeywordItem, \
ParagraphKeywordItem, TitleStemmed, TitleKeywords, TitleKeywordItem
logger.debug('¶')
# ## create News and NewsContent
# news_path = 'e:/news'
# # news_path = '/home/user/tmp/news/news'
# News.objects.load_from_folder(news_path)
#
# ## create NewsParagraph
# NewsContent.objects.create_paragraphs()
#
# ## create NewsStemmed and ParagraphStemmed
# News.objects.create_stems()
# NewsContent.objects.create_stems()
# NewsParagraph.objects.create_stems()
#
# stop_words = read_lines('.conf/stop_words.txt', 'cp1251')
def td(info):
message = '™%s· %s' % (' ' * local_indent, info)
logger.debug(message)
print message
