def __init__(self, conf):
"""Initzialize a new instance of out state-checker.
:param logger logger: The logger to use.
:param dict conf: Our StateCheckers configuration.
"""
self.db = Factory.get('Database')(client_encoding='UTF-8')
self.co = Factory.get('Constants')(self.db)
self.ac = Factory.get('Account')(self.db)
self.pe = Factory.get('Person')(self.db)
self.gr = Factory.get('Group')(self.db)
self.et = Factory.get('EmailTarget')(self.db)
self.eq = EmailQuota(self.db)
self.ea = EmailAddress(self.db)
self.ef = EmailForward(self.db)
self.cu = CerebrumUtils()
self.config = conf
self._ldap_page_size = 1000
self._cache_randzone_users = self._populate_randzone_cache(
self.config['randzone_unreserve_group'])
self._cache_accounts = self._populate_account_cache(
self.co.spread_exchange_account)
self._cache_addresses = self._populate_address_cache()
self._cache_local_delivery = self._populate_local_delivery_cache()
self._cache_forwards = self._populate_forward_cache()
self._cache_quotas = self._populate_quota_cache()
self._cache_targets = self._populate_target_cache()
self._cache_names = self._populate_name_cache()
self._cache_group_names = self._populate_group_name_cache()
self._cache_no_reservation = self._populate_no_reservation_cache()
self._cache_primary_accounts = self._populate_primary_account_cache()
def is_cnaddr_free_old(db, local_part, domain_part):
domain_id = get_domainid(db, domain_part)
ea = EmailAddress(db)
logger.debug("Considering %s, %s", local_part, domain_part)
try:
ea.find_by_local_part_and_domain(local_part, domain_id)
except Errors.NotFoundError:
# emailaddress is free.
logger.debug("Address %s@%s is free", local_part, domain_part)
else:
logger.warn("Address %s@%s is not free!", local_part, domain_part)
return False
return True
def __init__(self, conf):
"""Initzialize a new instance of out state-checker.
:param logger logger: The logger to use.
:param dict conf: Our StateCheckers configuration.
"""
self.db = Factory.get('Database')(client_encoding='UTF-8')
self.co = Factory.get('Constants')(self.db)
self.ac = Factory.get('Account')(self.db)
self.pe = Factory.get('Person')(self.db)
self.gr = Factory.get('Group')(self.db)
self.et = Factory.get('EmailTarget')(self.db)
self.eq = EmailQuota(self.db)
self.ea = EmailAddress(self.db)
self.ef = EmailForward(self.db)
self.cu = CerebrumUtils()
self.config = conf
self._ldap_page_size = 1000
self._cache_randzone_users = self._populate_randzone_cache(
self.config['randzone_unreserve_group'])
self._cache_accounts = self._populate_account_cache(
self.co.spread_exchange_account)
self._cache_addresses = self._populate_address_cache()
self._cache_local_delivery = self._populate_local_delivery_cache()
self._cache_forwards = self._populate_forward_cache()
self._cache_quotas = self._populate_quota_cache()
self._cache_targets = self._populate_target_cache()
self._cache_names = self._populate_name_cache()
self._cache_group_names = self._populate_group_name_cache()
self._cache_no_reservation = self._populate_no_reservation_cache()
self._cache_primary_accounts = self._populate_primary_account_cache()
def get_existing_emails(db):
"""
Return list of all email addresses with domain = 'post.uit.no' or 'uit.no',
format is: uit_addresse_in_use = ['localpart@domainpart']..
[localpart@domainpart]]
"""
ea = EmailAddress(db)
addresses_in_use = []
uit_no_list = ea.list_email_addresses_ext('uit.no')
# TODO: Use the post addresses as well.
# post_uit_no_list = ea.list_email_addresses_ext('post.uit.no')
for item in uit_no_list:
email_address = "{0}@{1}".format(item['local_part'], item['domain'])
addresses_in_use.append(email_address)
logger.debug("existing email: %s", email_address)
return addresses_in_use
def __init__(self, conf):
"""Initzialize a new instance of out state-checker.
:param logger logger: The logger to use.
:param dict conf: Our StateCheckers configuration.
"""
self.db = Factory.get('Database')(client_encoding='UTF-8')
self.co = Factory.get('Constants')(self.db)
self.dg = Factory.get('DistributionGroup')(self.db)
self.ac = Factory.get('Account')(self.db)
self.gr = Factory.get('Group')(self.db)
self.et = Factory.get('EmailTarget')(self.db)
self.ea = EmailAddress(self.db)
self.ut = CerebrumUtils()
self.config = conf
self._ldap_page_size = 1000
class StateChecker(object):
"""Wrapper class for state-checking functions.
The StateChecker class wraps all the functions we need in order to
verify and report deviances between Cerebrum and Exchange.
"""
# Connect params
LDAP_RETRY_DELAY = 60
LDAP_RETRY_MAX = 5
# Search and result params
LDAP_COM_DELAY = 30
LDAP_COM_MAX = 3
def __init__(self, conf):
"""Initzialize a new instance of out state-checker.
:param logger logger: The logger to use.
:param dict conf: Our StateCheckers configuration.
"""
self.db = Factory.get('Database')(client_encoding='UTF-8')
self.co = Factory.get('Constants')(self.db)
self.ac = Factory.get('Account')(self.db)
self.pe = Factory.get('Person')(self.db)
self.gr = Factory.get('Group')(self.db)
self.et = Factory.get('EmailTarget')(self.db)
self.eq = EmailQuota(self.db)
self.ea = EmailAddress(self.db)
self.ef = EmailForward(self.db)
self.cu = CerebrumUtils()
self.config = conf
self._ldap_page_size = 1000
self._cache_randzone_users = self._populate_randzone_cache(
self.config['randzone_unreserve_group'])
self._cache_accounts = self._populate_account_cache(
self.co.spread_exchange_account)
self._cache_addresses = self._populate_address_cache()
self._cache_local_delivery = self._populate_local_delivery_cache()
self._cache_forwards = self._populate_forward_cache()
self._cache_quotas = self._populate_quota_cache()
self._cache_targets = self._populate_target_cache()
self._cache_names = self._populate_name_cache()
self._cache_group_names = self._populate_group_name_cache()
self._cache_no_reservation = self._populate_no_reservation_cache()
self._cache_primary_accounts = self._populate_primary_account_cache()
def init_ldap(self):
"""Initzialize LDAP connection."""
self.ldap_srv = ldap.ldapobject.ReconnectLDAPObject(
'%s://%s/' %
(self.config['ldap_proto'], self.config['ldap_server']),
retry_max=self.LDAP_RETRY_MAX,
retry_delay=self.LDAP_RETRY_DELAY)
usr = self.config['ldap_user'].split('\\')[1]
self.ldap_srv.bind_s(self.config['ldap_user'],
read_password(usr, self.config['ldap_server']))
self.ldap_lc = ldap.controls.SimplePagedResultsControl(
True, self._ldap_page_size, '')
def _searcher(self, ou, scope, attrs, ctrls):
""" Perform ldap.search(), but retry in the event of an error.
This wraps the search with error handling, so that the search is
repeated with a delay between attempts.
"""
for attempt in itertools.count(1):
try:
return self.ldap_srv.search_ext(ou,
scope,
attrlist=attrs,
serverctrls=ctrls)
except ldap.LDAPError as e:
if attempt < self.LDAP_COM_MAX:
logger.debug('Caught %r in _searcher on attempt %d', e,
attempt)
time.sleep(self.LDAP_COM_DELAY)
continue
raise
def _recvr(self, msgid):
""" Perform ldap.result3(), but retry in the event of an error.
This wraps the result fetching with error handling, so that the fetch
is repeated with a delay between attempts.
It also decodes all attributes and attribute text values.
"""
for attempt in itertools.count(1):
try:
# return self.ldap_srv.result3(msgid)
rtype, rdata, rmsgid, sc = self.ldap_srv.result3(msgid)
return rtype, decode_attrs(rdata), rmsgid, sc
except ldap.LDAPError as e:
if attempt < self.LDAP_COM_MAX:
logger.debug('Caught %r in _recvr on attempt %d', e,
attempt)
time.sleep(self.LDAP_COM_DELAY)
continue
raise
def search(self, ou, attrs, scope=ldap.SCOPE_SUBTREE):
"""Wrapper for the search- and result-calls.
Implements paged searching.
:param str ou: The OU to search in.
:param list attrs: The attributes to fetch.
:param int scope: Our search scope, default is subtree.
"""
# Implementing paging, taken from
# http://www.novell.com/coolsolutions/tip/18274.html
msgid = self._searcher(ou, scope, attrs, [self.ldap_lc])
data = []
ctrltype = ldap.controls.SimplePagedResultsControl.controlType
while True:
time.sleep(1)
rtype, rdata, rmsgid, sc = self._recvr(msgid)
data.extend(rdata)
pctrls = [c for c in sc if c.controlType == ctrltype]
if pctrls:
cookie = pctrls[0].cookie
if cookie:
self.ldap_lc.cookie = cookie
time.sleep(1)
msgid = self._searcher(ou, scope, attrs, [self.ldap_lc])
else:
break
else:
logger.warn('Server ignores RFC 2696 control.')
break
return data[1:]
def close(self):
"""Close the LDAP connection."""
self.ldap_srv.unbind_s()
#
# Various cache-generating functions.
#
def _populate_randzone_cache(self, randzone):
self.gr.clear()
self.gr.find_by_name(randzone)
u = text_decoder(self.db.encoding)
return [
u(x['name']) for x in self.cu.get_group_members(self.gr.entity_id)
]
def _populate_account_cache(self, spread):
u = text_decoder(self.db.encoding)
def to_dict(row):
d = dict(row)
d['name'] = u(row['name'])
d['description'] = u(row['description'])
return d
return [to_dict(r) for r in self.ac.search(spread=spread)]
def _populate_address_cache(self):
u = text_decoder(self.db.encoding)
tmp = defaultdict(list)
# TODO: Implement fetchall?
for addr in self.ea.list_email_addresses_ext():
tmp[addr['target_id']].append(
u'%s@%s' % (u(addr['local_part']), u(addr['domain'])))
return dict(tmp)
def _populate_local_delivery_cache(self):
r = {}
for ld in self.ef.list_local_delivery():
r[ld['target_id']] = ld['local_delivery']
return r
def _populate_forward_cache(self):
u = text_decoder(self.db.encoding)
tmp = defaultdict(list)
for fwd in self.ef.list_email_forwards():
if fwd['enable'] == 'T':
tmp[fwd['target_id']].append(u(fwd['forward_to']))
return dict(tmp)
def _populate_quota_cache(self):
tmp = defaultdict(dict)
# TODO: Implement fetchall?
for quota in self.eq.list_email_quota_ext():
tmp[quota['target_id']]['soft'] = quota['quota_soft']
tmp[quota['target_id']]['hard'] = quota['quota_hard']
return dict(tmp)
def _populate_target_cache(self):
u = text_decoder(self.db.encoding)
tmp = defaultdict(dict)
for targ in self.et.list_email_target_primary_addresses(
target_type=self.co.email_target_account):
tmp[targ['target_entity_id']]['target_id'] = targ['target_id']
tmp[targ['target_entity_id']]['primary'] = \
u'%s@%s' % (u(targ['local_part']), u(targ['domain']))
return dict(tmp)
def _populate_name_cache(self):
u = text_decoder(self.db.encoding)
tmp = defaultdict(dict)
for name in self.pe.search_person_names(
name_variant=[
self.co.name_first, self.co.name_last, self.co.name_full
],
source_system=self.co.system_cached):
tmp[name['person_id']][name['name_variant']] = u(name['name'])
return dict(tmp)
def _populate_group_name_cache(self):
u = text_decoder(self.db.encoding)
tmp = {}
for eid, dom, name in self.gr.list_names(self.co.group_namespace):
tmp[eid] = u(name)
return tmp
def _populate_no_reservation_cache(self):
unreserved = []
for r in self.pe.list_traits(self.co.trait_public_reservation,
fetchall=True):
if r['numval'] == 0:
unreserved.append(r['entity_id'])
return unreserved
def _populate_primary_account_cache(self):
primary = []
for acc in self.ac.list_accounts_by_type(primary_only=True):
primary.append(acc['account_id'])
return primary
###
# Mailbox related state fetching & comparison
###
def collect_cerebrum_mail_info(self):
"""Collect E-mail related information from Cerebrum.
:rtype: dict
:return: A dict of users attributes. Uname is key.
"""
res = {}
for acc in self._cache_accounts:
tmp = {}
try:
tid = self._cache_targets[acc['account_id']]['target_id']
except KeyError:
logger.warn(
'Could not find account with id:%d in list '
'of targets, skipping..', acc['account_id'])
continue
# Fetch addresses
tmp[u'EmailAddresses'] = sorted(self._cache_addresses[tid])
# Fetch primary address
tmp[u'PrimaryAddress'] = \
self._cache_targets[acc['account_id']]['primary']
# Fetch names
if acc['owner_type'] == self.co.entity_person:
tmp[u'FirstName'] = \
self._cache_names[acc['owner_id']][int(self.co.name_first)]
tmp[u'LastName'] = \
self._cache_names[acc['owner_id']][int(self.co.name_last)]
tmp[u'DisplayName'] = \
self._cache_names[acc['owner_id']][int(self.co.name_full)]
else:
fn, ln, dn = self.cu.construct_group_names(
acc['name'],
self._cache_group_names.get(acc['owner_id'], None))
tmp[u'FirstName'] = fn
tmp[u'LastName'] = ln
tmp[u'DisplayName'] = dn
# Fetch quotas
hard = self._cache_quotas[tid]['hard'] * 1024
soft = self._cache_quotas[tid]['soft']
tmp[u'ProhibitSendQuota'] = str(hard)
tmp[u'ProhibitSendReceiveQuota'] = str(hard)
tmp[u'IssueWarningQuota'] = str(int(hard * soft / 100.))
# Randzone users will always be shown. This overrides everything
# else.
if acc['name'] in self._cache_randzone_users:
hide = False
elif acc['owner_id'] in self._cache_no_reservation and \
acc['account_id'] in self._cache_primary_accounts:
hide = False
else:
hide = True
tmp[u'HiddenFromAddressListsEnabled'] = hide
# Collect local delivery status
tmp[u'DeliverToMailboxAndForward'] = \
self._cache_local_delivery.get(tid, False)
# Collect forwarding address
# We do this by doing a difference operation on the forwards and
# the addresses, so we only end up with "external" addresses.
s_fwds = set(self._cache_forwards.get(tid, []))
s_addrs = set(self._cache_addresses.get(tid, []))
ext_fwds = list(s_fwds - s_addrs)
if ext_fwds:
tmp[u'ForwardingSmtpAddress'] = ext_fwds[0]
else:
tmp[u'ForwardingSmtpAddress'] = None
res[acc['name']] = tmp
return res
def collect_exchange_mail_info(self, mb_ou):
"""Collect mailbox-information from Exchange, via LDAP.
:param str mb_ou: The OrganizationalUnit to search for mailboxes.
:rtype: dict
:return: A dict with the mailboxes attributes. The key is the account
name.
"""
attrs = [
'proxyAddresses', 'displayName', 'givenName', 'sn',
'msExchHideFromAddressLists', 'extensionAttribute1',
'mDBUseDefaults', 'mDBOverQuotaLimit', 'mDBOverHardQuotaLimit',
'mDBStorageQuota', 'deliverAndRedirect',
'msExchGenericForwardingAddress'
]
r = self.search(mb_ou, attrs)
ret = {}
for cn, data in r:
if 'extensionAttribute1'in data and \
data['extensionAttribute1'] == ['not migrated'] or \
'ExchangeActiveSyncDevices' in cn:
continue
tmp = {}
name = cn[3:].split(',')[0]
for key in data:
if key == 'proxyAddresses':
addrs = []
for addr in data[key]:
if addr.startswith('SMTP:'):
tmp[u'PrimaryAddress'] = addr[5:]
addrs.append(addr[5:])
tmp[u'EmailAddresses'] = sorted(addrs)
elif key == 'displayName':
tmp[u'DisplayName'] = data[key][0]
elif key == 'givenName':
tmp[u'FirstName'] = data[key][0]
elif key == 'sn':
tmp[u'LastName'] = data[key][0]
elif key == 'mDBUseDefaults':
tmp[u'UseDatabaseQuotaDefaults'] = (True if data[key][0]
== 'TRUE' else False)
elif key == 'mDBOverQuotaLimit':
q = data[key][0]
tmp[u'ProhibitSendQuota'] = q
elif key == 'mDBOverHardQuotaLimit':
q = data[key][0]
tmp[u'ProhibitSendReceiveQuota'] = q
elif key == 'mDBStorageQuota':
q = data[key][0]
tmp[u'IssueWarningQuota'] = q
# Non-existent attribute means that the value is false. Fuckers.
# Collect status about if the mbox is hidden or not
tmp[u'HiddenFromAddressListsEnabled'] = False
if 'msExchHideFromAddressLists' in data:
val = (True if data['msExchHideFromAddressLists'][0] == 'TRUE'
else False)
tmp[u'HiddenFromAddressListsEnabled'] = val
# Collect local delivery status
tmp[u'DeliverToMailboxAndForward'] = False
if 'deliverAndRedirect' in data:
val = (True
if data['deliverAndRedirect'][0] == 'TRUE' else False)
tmp[u'DeliverToMailboxAndForward'] = val
# Collect forwarding address
tmp[u'ForwardingSmtpAddress'] = None
if 'msExchGenericForwardingAddress' in data:
val = data['msExchGenericForwardingAddress'][0]
# We split of smtp:, and store
tmp[u'ForwardingSmtpAddress'] = val.split(':')[1]
ret[name] = tmp
return ret
def compare_mailbox_state(self, ex_state, ce_state, state, config):
"""Compare the information fetched from Cerebrum and Exchange.
This method produces a dict with the state between the systems,
and a report that will be sent to the appropriate target system
administrators.
:param dict ex_state: The state in Exchange.
:param dict ce_state: The state in Cerebrum.
:param dict state: The previous state generated by this method.
:param dict config: Configuration of reporting delays for various
attributes.
:rtype: tuple
:return: A tuple consisting of the new difference-state and a
human-readable report of differences.
"""
s_ce_keys = set(ce_state.keys())
s_ex_keys = set(ex_state.keys())
diff_mb = {}
diff_stale = {}
diff_new = {}
##
# Populate some structures with information we need
# Mailboxes in Exchange, but not in Cerebrum
stale_keys = list(s_ex_keys - s_ce_keys)
for ident in stale_keys:
if state and ident in state['stale_mb']:
diff_stale[ident] = state['stale_mb'][ident]
else:
diff_stale[ident] = time.time()
# Mailboxes in Cerebrum, but not in Exchange
new_keys = list(s_ce_keys - s_ex_keys)
for ident in new_keys:
if state and ident in state['new_mb']:
diff_new[ident] = state['new_mb'][ident]
else:
diff_new[ident] = time.time()
# Check mailboxes that exists in both Cerebrum and Exchange for
# difference (& is union, in case you wondered). If an attribute is not
# in it's desired state in both this and the last run, save the
# timestamp from the last run. This is used for calculating when we nag
# to someone about stuff not beeing in sync.
for key in s_ex_keys & s_ce_keys:
for attr in ce_state[key]:
if state and key in state['mb'] and \
attr in state['mb'][key]:
t_0 = state['mb'][key][attr][u'Time']
else:
t_0 = time.time()
diff_mb.setdefault(key, {})
if attr not in ex_state[key]:
diff_mb[key][attr] = {
u'Exchange': None,
u'Cerebrum': ce_state[key][attr],
u'Time': t_0,
}
elif ce_state[key][attr] != ex_state[key][attr]:
# For quotas, we only want to report mismatches if the
# difference is between the quotas in Cerebrum and Exchange
# is greater than 1% on either side. Hope this is an
# appropriate value to use ;)
try:
if u'Quota' in attr:
exq = ex_state[key][attr]
ceq = ce_state[key][attr]
diff = abs(int(exq) - int(ceq))
avg = (int(exq) + int(ceq)) / 2
one_p = avg * 0.01
if avg + diff < avg + one_p and \
avg - diff > avg - one_p:
continue
except TypeError:
pass
diff_mb[key][attr] = {
u'Exchange': ex_state[key][attr],
u'Cerebrum': ce_state[key][attr],
u'Time': t_0,
}
ret = {'new_mb': diff_new, 'stale_mb': diff_stale, 'mb': diff_mb}
if not state:
return ret, []
now = time.time()
# By now, we have three different dicts. Loop trough them and check if
# we should report 'em
report = [u'# User Attribute Since Cerebrum_value:Exchange_value']
# Report attribute mismatches
for key in diff_mb:
for attr in diff_mb[key]:
delta = (config.get(attr) if attr in config else
config.get('UndefinedAttribute'))
if diff_mb[key][attr][u'Time'] < now - delta:
t = time.strftime(
u'%d%m%Y-%H:%M',
time.localtime(diff_mb[key][attr][u'Time']))
if attr == u'EmailAddresses':
# We report the difference for email addresses, for
# redability
s_ce_addr = set(diff_mb[key][attr][u'Cerebrum'])
s_ex_addr = set(diff_mb[key][attr][u'Exchange'])
new_addr = list(s_ce_addr - s_ex_addr)
stale_addr = list(s_ex_addr - s_ce_addr)
tmp = u'%-10s %-30s %s +%s:-%s' % (
key, attr, t, str(new_addr), str(stale_addr))
else:
tmp = (u'%-10s %-30s %s %s:%s' %
(key, attr, t,
repr(diff_mb[key][attr][u'Cerebrum']),
repr(diff_mb[key][attr][u'Exchange'])))
report += [tmp]
# Report uncreated mailboxes
report += [u'\n# Uncreated mailboxes (uname, time)']
delta = (config.get('UncreatedMailbox') if 'UncreatedMailbox' in config
else config.get('UndefinedAttribute'))
for key in diff_new:
if diff_new[key] < now - delta:
t = time.strftime(u'%d%m%Y-%H:%M',
time.localtime(diff_new[key]))
report += [u'%-10s uncreated_mb %s' % (key, t)]
# Report stale mailboxes
report += [u'\n# Stale mailboxes (uname, time)']
delta = (config.get('StaleMailbox') if 'StaleMailbox' in config else
config.get('UndefinedAttribute'))
for key in diff_stale:
t = time.strftime(u'%d%m%Y-%H:%M', time.localtime(diff_stale[key]))
if diff_stale[key] < now - delta:
report += [u'%-10s stale_mb %s' % (key, t)]
return ret, report
from Cerebrum.Utils import Factory
from Cerebrum.modules.Email import EmailDomain
from Cerebrum.modules.Email import EmailAddress
from Cerebrum.modules.Email import EmailPrimaryAddressTarget
db = Factory.get('Database')()
db.cl_init(change_program='create_dg_moderator')
ac = Factory.get('Account')(db)
co = Factory.get('Constants')(db)
dg = Factory.get('DistributionGroup')(db)
gr = Factory.get('Group')(db)
et = Factory.get('EmailTarget')(db)
epat = EmailPrimaryAddressTarget(db)
ed = EmailDomain(db)
ea = EmailAddress(db)
group_name = 'groupadmin'
if len(sys.argv) > 1:
group_domain = sys.argv[1]
else:
group_domain = 'groups.uio.no'
ac.clear()
ac.find_by_name('bootstrap_account')
gr.clear()
gr.populate(
creator_id=ac.entity_id,
def ea(self):
if not hasattr(self, '_ea'):
self._ea = EmailAddress(self._db)
return self._ea
class StateChecker(object):
"""Wrapper class for state-checking functions.
The StateChecker class wraps all the functions we need in order to
verify and report deviances between Cerebrum and Exchange.
"""
# Connect params
LDAP_RETRY_DELAY = 60
LDAP_RETRY_MAX = 5
# Search and result params
LDAP_COM_DELAY = 30
LDAP_COM_MAX = 3
def __init__(self, conf):
"""Initzialize a new instance of out state-checker.
:param logger logger: The logger to use.
:param dict conf: Our StateCheckers configuration.
"""
self.db = Factory.get('Database')(client_encoding='UTF-8')
self.co = Factory.get('Constants')(self.db)
self.ac = Factory.get('Account')(self.db)
self.pe = Factory.get('Person')(self.db)
self.gr = Factory.get('Group')(self.db)
self.et = Factory.get('EmailTarget')(self.db)
self.eq = EmailQuota(self.db)
self.ea = EmailAddress(self.db)
self.ef = EmailForward(self.db)
self.cu = CerebrumUtils()
self.config = conf
self._ldap_page_size = 1000
self._cache_randzone_users = self._populate_randzone_cache(
self.config['randzone_unreserve_group'])
self._cache_accounts = self._populate_account_cache(
self.co.spread_exchange_account)
self._cache_addresses = self._populate_address_cache()
self._cache_local_delivery = self._populate_local_delivery_cache()
self._cache_forwards = self._populate_forward_cache()
self._cache_quotas = self._populate_quota_cache()
self._cache_targets = self._populate_target_cache()
self._cache_names = self._populate_name_cache()
self._cache_group_names = self._populate_group_name_cache()
self._cache_no_reservation = self._populate_no_reservation_cache()
self._cache_primary_accounts = self._populate_primary_account_cache()
def init_ldap(self):
"""Initzialize LDAP connection."""
self.ldap_srv = ldap.ldapobject.ReconnectLDAPObject(
'%s://%s/' % (self.config['ldap_proto'],
self.config['ldap_server']),
retry_max=self.LDAP_RETRY_MAX,
retry_delay=self.LDAP_RETRY_DELAY)
usr = self.config['ldap_user'].split('\\')[1]
self.ldap_srv.bind_s(
self.config['ldap_user'],
read_password(usr, self.config['ldap_server']))
self.ldap_lc = ldap.controls.SimplePagedResultsControl(
True, self._ldap_page_size, '')
def _searcher(self, ou, scope, attrs, ctrls):
""" Perform ldap.search(), but retry in the event of an error.
This wraps the search with error handling, so that the search is
repeated with a delay between attempts.
"""
for attempt in itertools.count(1):
try:
return self.ldap_srv.search_ext(
ou, scope, attrlist=attrs, serverctrls=ctrls)
except ldap.LDAPError as e:
if attempt < self.LDAP_COM_MAX:
logger.debug('Caught %r in _searcher on attempt %d',
e, attempt)
time.sleep(self.LDAP_COM_DELAY)
continue
raise
def _recvr(self, msgid):
""" Perform ldap.result3(), but retry in the event of an error.
This wraps the result fetching with error handling, so that the fetch
is repeated with a delay between attempts.
It also decodes all attributes and attribute text values.
"""
for attempt in itertools.count(1):
try:
# return self.ldap_srv.result3(msgid)
rtype, rdata, rmsgid, sc = self.ldap_srv.result3(msgid)
return rtype, decode_attrs(rdata), rmsgid, sc
except ldap.LDAPError as e:
if attempt < self.LDAP_COM_MAX:
logger.debug('Caught %r in _recvr on attempt %d',
e, attempt)
time.sleep(self.LDAP_COM_DELAY)
continue
raise
def search(self, ou, attrs, scope=ldap.SCOPE_SUBTREE):
"""Wrapper for the search- and result-calls.
Implements paged searching.
:param str ou: The OU to search in.
:param list attrs: The attributes to fetch.
:param int scope: Our search scope, default is subtree.
"""
# Implementing paging, taken from
# http://www.novell.com/coolsolutions/tip/18274.html
msgid = self._searcher(ou, scope, attrs, [self.ldap_lc])
data = []
ctrltype = ldap.controls.SimplePagedResultsControl.controlType
while True:
time.sleep(1)
rtype, rdata, rmsgid, sc = self._recvr(msgid)
data.extend(rdata)
pctrls = [c for c in sc if c.controlType == ctrltype]
if pctrls:
cookie = pctrls[0].cookie
if cookie:
self.ldap_lc.cookie = cookie
time.sleep(1)
msgid = self._searcher(ou, scope, attrs, [self.ldap_lc])
else:
break
else:
logger.warn('Server ignores RFC 2696 control.')
break
return data[1:]
def close(self):
"""Close the LDAP connection."""
self.ldap_srv.unbind_s()
#
# Various cache-generating functions.
#
def _populate_randzone_cache(self, randzone):
self.gr.clear()
self.gr.find_by_name(randzone)
u = text_decoder(self.db.encoding)
return [u(x['name'])
for x in self.cu.get_group_members(self.gr.entity_id)]
def _populate_account_cache(self, spread):
u = text_decoder(self.db.encoding)
def to_dict(row):
d = dict(row)
d['name'] = u(row['name'])
d['description'] = u(row['description'])
return d
return [to_dict(r) for r in self.ac.search(spread=spread)]
def _populate_address_cache(self):
u = text_decoder(self.db.encoding)
tmp = defaultdict(list)
# TODO: Implement fetchall?
for addr in self.ea.list_email_addresses_ext():
tmp[addr['target_id']].append(
u'%s@%s' % (u(addr['local_part']), u(addr['domain'])))
return dict(tmp)
def _populate_local_delivery_cache(self):
r = {}
for ld in self.ef.list_local_delivery():
r[ld['target_id']] = ld['local_delivery']
return r
def _populate_forward_cache(self):
u = text_decoder(self.db.encoding)
tmp = defaultdict(list)
for fwd in self.ef.list_email_forwards():
if fwd['enable'] == 'T':
tmp[fwd['target_id']].append(u(fwd['forward_to']))
return dict(tmp)
def _populate_quota_cache(self):
tmp = defaultdict(dict)
# TODO: Implement fetchall?
for quota in self.eq.list_email_quota_ext():
tmp[quota['target_id']]['soft'] = quota['quota_soft']
tmp[quota['target_id']]['hard'] = quota['quota_hard']
return dict(tmp)
def _populate_target_cache(self):
u = text_decoder(self.db.encoding)
tmp = defaultdict(dict)
for targ in self.et.list_email_target_primary_addresses(
target_type=self.co.email_target_account):
tmp[targ['target_entity_id']]['target_id'] = targ['target_id']
tmp[targ['target_entity_id']]['primary'] = \
u'%s@%s' % (u(targ['local_part']), u(targ['domain']))
return dict(tmp)
def _populate_name_cache(self):
u = text_decoder(self.db.encoding)
tmp = defaultdict(dict)
for name in self.pe.search_person_names(
name_variant=[self.co.name_first,
self.co.name_last,
self.co.name_full],
source_system=self.co.system_cached):
tmp[name['person_id']][name['name_variant']] = u(name['name'])
return dict(tmp)
def _populate_group_name_cache(self):
u = text_decoder(self.db.encoding)
tmp = {}
for eid, dom, name in self.gr.list_names(self.co.group_namespace):
tmp[eid] = u(name)
return tmp
def _populate_no_reservation_cache(self):
unreserved = []
for r in self.pe.list_traits(self.co.trait_public_reservation,
fetchall=True):
if r['numval'] == 0:
unreserved.append(r['entity_id'])
return unreserved
def _populate_primary_account_cache(self):
primary = []
for acc in self.ac.list_accounts_by_type(primary_only=True):
primary.append(acc['account_id'])
return primary
###
# Mailbox related state fetching & comparison
###
def collect_cerebrum_mail_info(self):
"""Collect E-mail related information from Cerebrum.
:rtype: dict
:return: A dict of users attributes. Uname is key.
"""
res = {}
for acc in self._cache_accounts:
tmp = {}
try:
tid = self._cache_targets[acc['account_id']]['target_id']
except KeyError:
logger.warn('Could not find account with id:%d in list '
'of targets, skipping..', acc['account_id'])
continue
# Fetch addresses
tmp[u'EmailAddresses'] = sorted(self._cache_addresses[tid])
# Fetch primary address
tmp[u'PrimaryAddress'] = \
self._cache_targets[acc['account_id']]['primary']
# Fetch names
if acc['owner_type'] == self.co.entity_person:
tmp[u'FirstName'] = \
self._cache_names[acc['owner_id']][int(self.co.name_first)]
tmp[u'LastName'] = \
self._cache_names[acc['owner_id']][int(self.co.name_last)]
tmp[u'DisplayName'] = \
self._cache_names[acc['owner_id']][int(self.co.name_full)]
else:
fn, ln, dn = self.cu.construct_group_names(
acc['name'], self._cache_group_names.get(acc['owner_id'],
None))
tmp[u'FirstName'] = fn
tmp[u'LastName'] = ln
tmp[u'DisplayName'] = dn
# Fetch quotas
hard = self._cache_quotas[tid]['hard'] * 1024
soft = self._cache_quotas[tid]['soft']
tmp[u'ProhibitSendQuota'] = str(hard)
tmp[u'ProhibitSendReceiveQuota'] = str(hard)
tmp[u'IssueWarningQuota'] = str(int(hard * soft / 100.))
# Randzone users will always be shown. This overrides everything
# else.
if acc['name'] in self._cache_randzone_users:
hide = False
elif acc['owner_id'] in self._cache_no_reservation and \
acc['account_id'] in self._cache_primary_accounts:
hide = False
else:
hide = True
tmp[u'HiddenFromAddressListsEnabled'] = hide
# Collect local delivery status
tmp[u'DeliverToMailboxAndForward'] = \
self._cache_local_delivery.get(tid, False)
# Collect forwarding address
# We do this by doing a difference operation on the forwards and
# the addresses, so we only end up with "external" addresses.
s_fwds = set(self._cache_forwards.get(tid, []))
s_addrs = set(self._cache_addresses.get(tid, []))
ext_fwds = list(s_fwds - s_addrs)
if ext_fwds:
tmp[u'ForwardingSmtpAddress'] = ext_fwds[0]
else:
tmp[u'ForwardingSmtpAddress'] = None
res[acc['name']] = tmp
return res
def collect_exchange_mail_info(self, mb_ou):
"""Collect mailbox-information from Exchange, via LDAP.
:param str mb_ou: The OrganizationalUnit to search for mailboxes.
:rtype: dict
:return: A dict with the mailboxes attributes. The key is the account
name.
"""
attrs = ['proxyAddresses',
'displayName',
'givenName',
'sn',
'msExchHideFromAddressLists',
'extensionAttribute1',
'mDBUseDefaults',
'mDBOverQuotaLimit',
'mDBOverHardQuotaLimit',
'mDBStorageQuota',
'deliverAndRedirect',
'msExchGenericForwardingAddress']
r = self.search(mb_ou, attrs)
ret = {}
for cn, data in r:
if 'extensionAttribute1'in data and \
data['extensionAttribute1'] == ['not migrated'] or \
'ExchangeActiveSyncDevices' in cn:
continue
tmp = {}
name = cn[3:].split(',')[0]
for key in data:
if key == 'proxyAddresses':
addrs = []
for addr in data[key]:
if addr.startswith('SMTP:'):
tmp[u'PrimaryAddress'] = addr[5:]
addrs.append(addr[5:])
tmp[u'EmailAddresses'] = sorted(addrs)
elif key == 'displayName':
tmp[u'DisplayName'] = data[key][0]
elif key == 'givenName':
tmp[u'FirstName'] = data[key][0]
elif key == 'sn':
tmp[u'LastName'] = data[key][0]
elif key == 'mDBUseDefaults':
tmp[u'UseDatabaseQuotaDefaults'] = (
True if data[key][0] == 'TRUE' else False)
elif key == 'mDBOverQuotaLimit':
q = data[key][0]
tmp[u'ProhibitSendQuota'] = q
elif key == 'mDBOverHardQuotaLimit':
q = data[key][0]
tmp[u'ProhibitSendReceiveQuota'] = q
elif key == 'mDBStorageQuota':
q = data[key][0]
tmp[u'IssueWarningQuota'] = q
# Non-existent attribute means that the value is false. Fuckers.
# Collect status about if the mbox is hidden or not
tmp[u'HiddenFromAddressListsEnabled'] = False
if 'msExchHideFromAddressLists' in data:
val = (True if data['msExchHideFromAddressLists'][0] == 'TRUE'
else False)
tmp[u'HiddenFromAddressListsEnabled'] = val
# Collect local delivery status
tmp[u'DeliverToMailboxAndForward'] = False
if 'deliverAndRedirect' in data:
val = (True if data['deliverAndRedirect'][0] == 'TRUE'
else False)
tmp[u'DeliverToMailboxAndForward'] = val
# Collect forwarding address
tmp[u'ForwardingSmtpAddress'] = None
if 'msExchGenericForwardingAddress' in data:
val = data['msExchGenericForwardingAddress'][0]
# We split of smtp:, and store
tmp[u'ForwardingSmtpAddress'] = val.split(':')[1]
ret[name] = tmp
return ret
def compare_mailbox_state(self, ex_state, ce_state, state, config):
"""Compare the information fetched from Cerebrum and Exchange.
This method produces a dict with the state between the systems,
and a report that will be sent to the appropriate target system
administrators.
:param dict ex_state: The state in Exchange.
:param dict ce_state: The state in Cerebrum.
:param dict state: The previous state generated by this method.
:param dict config: Configuration of reporting delays for various
attributes.
:rtype: tuple
:return: A tuple consisting of the new difference-state and a
human-readable report of differences.
"""
s_ce_keys = set(ce_state.keys())
s_ex_keys = set(ex_state.keys())
diff_mb = {}
diff_stale = {}
diff_new = {}
##
# Populate some structures with information we need
# Mailboxes in Exchange, but not in Cerebrum
stale_keys = list(s_ex_keys - s_ce_keys)
for ident in stale_keys:
if state and ident in state['stale_mb']:
diff_stale[ident] = state['stale_mb'][ident]
else:
diff_stale[ident] = time.time()
# Mailboxes in Cerebrum, but not in Exchange
new_keys = list(s_ce_keys - s_ex_keys)
for ident in new_keys:
if state and ident in state['new_mb']:
diff_new[ident] = state['new_mb'][ident]
else:
diff_new[ident] = time.time()
# Check mailboxes that exists in both Cerebrum and Exchange for
# difference (& is union, in case you wondered). If an attribute is not
# in it's desired state in both this and the last run, save the
# timestamp from the last run. This is used for calculating when we nag
# to someone about stuff not beeing in sync.
for key in s_ex_keys & s_ce_keys:
for attr in ce_state[key]:
if state and key in state['mb'] and \
attr in state['mb'][key]:
t_0 = state['mb'][key][attr][u'Time']
else:
t_0 = time.time()
diff_mb.setdefault(key, {})
if attr not in ex_state[key]:
diff_mb[key][attr] = {
u'Exchange': None,
u'Cerebrum': ce_state[key][attr],
u'Time': t_0,
}
elif ce_state[key][attr] != ex_state[key][attr]:
# For quotas, we only want to report mismatches if the
# difference is between the quotas in Cerebrum and Exchange
# is greater than 1% on either side. Hope this is an
# appropriate value to use ;)
try:
if u'Quota' in attr:
exq = ex_state[key][attr]
ceq = ce_state[key][attr]
diff = abs(int(exq) - int(ceq))
avg = (int(exq) + int(ceq)) / 2
one_p = avg * 0.01
if avg + diff < avg + one_p and \
avg - diff > avg - one_p:
continue
except TypeError:
pass
diff_mb[key][attr] = {
u'Exchange': ex_state[key][attr],
u'Cerebrum': ce_state[key][attr],
u'Time': t_0,
}
ret = {'new_mb': diff_new, 'stale_mb': diff_stale, 'mb': diff_mb}
if not state:
return ret, []
now = time.time()
# By now, we have three different dicts. Loop trough them and check if
# we should report 'em
report = [u'# User Attribute Since Cerebrum_value:Exchange_value']
# Report attribute mismatches
for key in diff_mb:
for attr in diff_mb[key]:
delta = (config.get(attr) if attr in config else
config.get('UndefinedAttribute'))
if diff_mb[key][attr][u'Time'] < now - delta:
t = time.strftime(u'%d%m%Y-%H:%M', time.localtime(
diff_mb[key][attr][u'Time']))
if attr == u'EmailAddresses':
# We report the difference for email addresses, for
# redability
s_ce_addr = set(diff_mb[key][attr][u'Cerebrum'])
s_ex_addr = set(diff_mb[key][attr][u'Exchange'])
new_addr = list(s_ce_addr - s_ex_addr)
stale_addr = list(s_ex_addr - s_ce_addr)
tmp = u'%-10s %-30s %s +%s:-%s' % (key, attr, t,
str(new_addr),
str(stale_addr))
else:
tmp = (u'%-10s %-30s %s %s:%s' %
(key, attr, t,
repr(diff_mb[key][attr][u'Cerebrum']),
repr(diff_mb[key][attr][u'Exchange']))
)
report += [tmp]
# Report uncreated mailboxes
report += [u'\n# Uncreated mailboxes (uname, time)']
delta = (config.get('UncreatedMailbox') if 'UncreatedMailbox' in config
else config.get('UndefinedAttribute'))
for key in diff_new:
if diff_new[key] < now - delta:
t = time.strftime(u'%d%m%Y-%H:%M', time.localtime(
diff_new[key]))
report += [u'%-10s uncreated_mb %s' % (key, t)]
# Report stale mailboxes
report += [u'\n# Stale mailboxes (uname, time)']
delta = (config.get('StaleMailbox') if 'StaleMailbox' in config else
config.get('UndefinedAttribute'))
for key in diff_stale:
t = time.strftime(u'%d%m%Y-%H:%M', time.localtime(
diff_stale[key]))
if diff_stale[key] < now - delta:
report += [u'%-10s stale_mb %s' % (key, t)]
return ret, report
from Cerebrum.Utils import Factory
from Cerebrum.modules.Email import EmailDomain
from Cerebrum.modules.Email import EmailAddress
from Cerebrum.modules.Email import EmailPrimaryAddressTarget
db = Factory.get('Database')()
db.cl_init(change_program='create_dg_moderator')
ac = Factory.get('Account')(db)
co = Factory.get('Constants')(db)
dg = Factory.get('DistributionGroup')(db)
gr = Factory.get('Group')(db)
et = Factory.get('EmailTarget')(db)
epat = EmailPrimaryAddressTarget(db)
ed = EmailDomain(db)
ea = EmailAddress(db)
group_name = 'groupadmin'
if len(sys.argv) > 1:
group_domain = sys.argv[1]
else:
group_domain = 'groups.uio.no'
ac.clear()
ac.find_by_name('bootstrap_account')
gr.clear()
gr.populate(ac.entity_id,
co.group_visibility_all,