def __init__(self, conf):
"""Initzialize a new instance of out state-checker.
:param logger logger: The logger to use.
:param dict conf: Our StateCheckers configuration.
"""
self.db = Factory.get('Database')(client_encoding='UTF-8')
self.co = Factory.get('Constants')(self.db)
self.ac = Factory.get('Account')(self.db)
self.pe = Factory.get('Person')(self.db)
self.gr = Factory.get('Group')(self.db)
self.et = Factory.get('EmailTarget')(self.db)
self.eq = EmailQuota(self.db)
self.ea = EmailAddress(self.db)
self.ef = EmailForward(self.db)
self.cu = CerebrumUtils()
self.config = conf
self._ldap_page_size = 1000
self._cache_randzone_users = self._populate_randzone_cache(
self.config['randzone_unreserve_group'])
self._cache_accounts = self._populate_account_cache(
self.co.spread_exchange_account)
self._cache_addresses = self._populate_address_cache()
self._cache_local_delivery = self._populate_local_delivery_cache()
self._cache_forwards = self._populate_forward_cache()
self._cache_quotas = self._populate_quota_cache()
self._cache_targets = self._populate_target_cache()
self._cache_names = self._populate_name_cache()
self._cache_group_names = self._populate_group_name_cache()
self._cache_no_reservation = self._populate_no_reservation_cache()
self._cache_primary_accounts = self._populate_primary_account_cache()
def __init__(self, conf):
"""Initzialize a new instance of out state-checker.
:param logger logger: The logger to use.
:param dict conf: Our StateCheckers configuration.
"""
self.db = Factory.get('Database')(client_encoding='UTF-8')
self.co = Factory.get('Constants')(self.db)
self.ac = Factory.get('Account')(self.db)
self.pe = Factory.get('Person')(self.db)
self.gr = Factory.get('Group')(self.db)
self.et = Factory.get('EmailTarget')(self.db)
self.eq = EmailQuota(self.db)
self.ea = EmailAddress(self.db)
self.ef = EmailForward(self.db)
self.cu = CerebrumUtils()
self.config = conf
self._ldap_page_size = 1000
self._cache_randzone_users = self._populate_randzone_cache(
self.config['randzone_unreserve_group'])
self._cache_accounts = self._populate_account_cache(
self.co.spread_exchange_account)
self._cache_addresses = self._populate_address_cache()
self._cache_local_delivery = self._populate_local_delivery_cache()
self._cache_forwards = self._populate_forward_cache()
self._cache_quotas = self._populate_quota_cache()
self._cache_targets = self._populate_target_cache()
self._cache_names = self._populate_name_cache()
self._cache_group_names = self._populate_group_name_cache()
self._cache_no_reservation = self._populate_no_reservation_cache()
self._cache_primary_accounts = self._populate_primary_account_cache()
def __init__(self, conf):
"""Initzialize a new instance of out state-checker.
:param logger logger: The logger to use.
:param dict conf: Our StateCheckers configuration.
"""
self.db = Factory.get('Database')(client_encoding='UTF-8')
self.co = Factory.get('Constants')(self.db)
self.dg = Factory.get('DistributionGroup')(self.db)
self.ac = Factory.get('Account')(self.db)
self.gr = Factory.get('Group')(self.db)
self.et = Factory.get('EmailTarget')(self.db)
self.ea = EmailAddress(self.db)
self.ut = CerebrumUtils()
self.config = conf
self._ldap_page_size = 1000
def ut(self):
return CerebrumUtils()
class StateChecker(object):
"""Wrapper class for state-checking functions.
The StateChecker class wraps all the functions we need in order to
verify and report deviances between Cerebrum and Exchange.
"""
# Connect params
LDAP_RETRY_DELAY = 60
LDAP_RETRY_MAX = 5
# Search and result params
LDAP_COM_DELAY = 30
LDAP_COM_MAX = 3
def __init__(self, conf):
"""Initzialize a new instance of out state-checker.
:param logger logger: The logger to use.
:param dict conf: Our StateCheckers configuration.
"""
self.db = Factory.get('Database')(client_encoding='UTF-8')
self.co = Factory.get('Constants')(self.db)
self.ac = Factory.get('Account')(self.db)
self.pe = Factory.get('Person')(self.db)
self.gr = Factory.get('Group')(self.db)
self.et = Factory.get('EmailTarget')(self.db)
self.eq = EmailQuota(self.db)
self.ea = EmailAddress(self.db)
self.ef = EmailForward(self.db)
self.cu = CerebrumUtils()
self.config = conf
self._ldap_page_size = 1000
self._cache_randzone_users = self._populate_randzone_cache(
self.config['randzone_unreserve_group'])
self._cache_accounts = self._populate_account_cache(
self.co.spread_exchange_account)
self._cache_addresses = self._populate_address_cache()
self._cache_local_delivery = self._populate_local_delivery_cache()
self._cache_forwards = self._populate_forward_cache()
self._cache_quotas = self._populate_quota_cache()
self._cache_targets = self._populate_target_cache()
self._cache_names = self._populate_name_cache()
self._cache_group_names = self._populate_group_name_cache()
self._cache_no_reservation = self._populate_no_reservation_cache()
self._cache_primary_accounts = self._populate_primary_account_cache()
def init_ldap(self):
"""Initzialize LDAP connection."""
self.ldap_srv = ldap.ldapobject.ReconnectLDAPObject(
'%s://%s/' %
(self.config['ldap_proto'], self.config['ldap_server']),
retry_max=self.LDAP_RETRY_MAX,
retry_delay=self.LDAP_RETRY_DELAY)
usr = self.config['ldap_user'].split('\\')[1]
self.ldap_srv.bind_s(self.config['ldap_user'],
read_password(usr, self.config['ldap_server']))
self.ldap_lc = ldap.controls.SimplePagedResultsControl(
True, self._ldap_page_size, '')
def _searcher(self, ou, scope, attrs, ctrls):
""" Perform ldap.search(), but retry in the event of an error.
This wraps the search with error handling, so that the search is
repeated with a delay between attempts.
"""
for attempt in itertools.count(1):
try:
return self.ldap_srv.search_ext(ou,
scope,
attrlist=attrs,
serverctrls=ctrls)
except ldap.LDAPError as e:
if attempt < self.LDAP_COM_MAX:
logger.debug('Caught %r in _searcher on attempt %d', e,
attempt)
time.sleep(self.LDAP_COM_DELAY)
continue
raise
def _recvr(self, msgid):
""" Perform ldap.result3(), but retry in the event of an error.
This wraps the result fetching with error handling, so that the fetch
is repeated with a delay between attempts.
It also decodes all attributes and attribute text values.
"""
for attempt in itertools.count(1):
try:
# return self.ldap_srv.result3(msgid)
rtype, rdata, rmsgid, sc = self.ldap_srv.result3(msgid)
return rtype, decode_attrs(rdata), rmsgid, sc
except ldap.LDAPError as e:
if attempt < self.LDAP_COM_MAX:
logger.debug('Caught %r in _recvr on attempt %d', e,
attempt)
time.sleep(self.LDAP_COM_DELAY)
continue
raise
def search(self, ou, attrs, scope=ldap.SCOPE_SUBTREE):
"""Wrapper for the search- and result-calls.
Implements paged searching.
:param str ou: The OU to search in.
:param list attrs: The attributes to fetch.
:param int scope: Our search scope, default is subtree.
"""
# Implementing paging, taken from
# http://www.novell.com/coolsolutions/tip/18274.html
msgid = self._searcher(ou, scope, attrs, [self.ldap_lc])
data = []
ctrltype = ldap.controls.SimplePagedResultsControl.controlType
while True:
time.sleep(1)
rtype, rdata, rmsgid, sc = self._recvr(msgid)
data.extend(rdata)
pctrls = [c for c in sc if c.controlType == ctrltype]
if pctrls:
cookie = pctrls[0].cookie
if cookie:
self.ldap_lc.cookie = cookie
time.sleep(1)
msgid = self._searcher(ou, scope, attrs, [self.ldap_lc])
else:
break
else:
logger.warn('Server ignores RFC 2696 control.')
break
return data[1:]
def close(self):
"""Close the LDAP connection."""
self.ldap_srv.unbind_s()
#
# Various cache-generating functions.
#
def _populate_randzone_cache(self, randzone):
self.gr.clear()
self.gr.find_by_name(randzone)
u = text_decoder(self.db.encoding)
return [
u(x['name']) for x in self.cu.get_group_members(self.gr.entity_id)
]
def _populate_account_cache(self, spread):
u = text_decoder(self.db.encoding)
def to_dict(row):
d = dict(row)
d['name'] = u(row['name'])
d['description'] = u(row['description'])
return d
return [to_dict(r) for r in self.ac.search(spread=spread)]
def _populate_address_cache(self):
u = text_decoder(self.db.encoding)
tmp = defaultdict(list)
# TODO: Implement fetchall?
for addr in self.ea.list_email_addresses_ext():
tmp[addr['target_id']].append(
u'%s@%s' % (u(addr['local_part']), u(addr['domain'])))
return dict(tmp)
def _populate_local_delivery_cache(self):
r = {}
for ld in self.ef.list_local_delivery():
r[ld['target_id']] = ld['local_delivery']
return r
def _populate_forward_cache(self):
u = text_decoder(self.db.encoding)
tmp = defaultdict(list)
for fwd in self.ef.list_email_forwards():
if fwd['enable'] == 'T':
tmp[fwd['target_id']].append(u(fwd['forward_to']))
return dict(tmp)
def _populate_quota_cache(self):
tmp = defaultdict(dict)
# TODO: Implement fetchall?
for quota in self.eq.list_email_quota_ext():
tmp[quota['target_id']]['soft'] = quota['quota_soft']
tmp[quota['target_id']]['hard'] = quota['quota_hard']
return dict(tmp)
def _populate_target_cache(self):
u = text_decoder(self.db.encoding)
tmp = defaultdict(dict)
for targ in self.et.list_email_target_primary_addresses(
target_type=self.co.email_target_account):
tmp[targ['target_entity_id']]['target_id'] = targ['target_id']
tmp[targ['target_entity_id']]['primary'] = \
u'%s@%s' % (u(targ['local_part']), u(targ['domain']))
return dict(tmp)
def _populate_name_cache(self):
u = text_decoder(self.db.encoding)
tmp = defaultdict(dict)
for name in self.pe.search_person_names(
name_variant=[
self.co.name_first, self.co.name_last, self.co.name_full
],
source_system=self.co.system_cached):
tmp[name['person_id']][name['name_variant']] = u(name['name'])
return dict(tmp)
def _populate_group_name_cache(self):
u = text_decoder(self.db.encoding)
tmp = {}
for eid, dom, name in self.gr.list_names(self.co.group_namespace):
tmp[eid] = u(name)
return tmp
def _populate_no_reservation_cache(self):
unreserved = []
for r in self.pe.list_traits(self.co.trait_public_reservation,
fetchall=True):
if r['numval'] == 0:
unreserved.append(r['entity_id'])
return unreserved
def _populate_primary_account_cache(self):
primary = []
for acc in self.ac.list_accounts_by_type(primary_only=True):
primary.append(acc['account_id'])
return primary
###
# Mailbox related state fetching & comparison
###
def collect_cerebrum_mail_info(self):
"""Collect E-mail related information from Cerebrum.
:rtype: dict
:return: A dict of users attributes. Uname is key.
"""
res = {}
for acc in self._cache_accounts:
tmp = {}
try:
tid = self._cache_targets[acc['account_id']]['target_id']
except KeyError:
logger.warn(
'Could not find account with id:%d in list '
'of targets, skipping..', acc['account_id'])
continue
# Fetch addresses
tmp[u'EmailAddresses'] = sorted(self._cache_addresses[tid])
# Fetch primary address
tmp[u'PrimaryAddress'] = \
self._cache_targets[acc['account_id']]['primary']
# Fetch names
if acc['owner_type'] == self.co.entity_person:
tmp[u'FirstName'] = \
self._cache_names[acc['owner_id']][int(self.co.name_first)]
tmp[u'LastName'] = \
self._cache_names[acc['owner_id']][int(self.co.name_last)]
tmp[u'DisplayName'] = \
self._cache_names[acc['owner_id']][int(self.co.name_full)]
else:
fn, ln, dn = self.cu.construct_group_names(
acc['name'],
self._cache_group_names.get(acc['owner_id'], None))
tmp[u'FirstName'] = fn
tmp[u'LastName'] = ln
tmp[u'DisplayName'] = dn
# Fetch quotas
hard = self._cache_quotas[tid]['hard'] * 1024
soft = self._cache_quotas[tid]['soft']
tmp[u'ProhibitSendQuota'] = str(hard)
tmp[u'ProhibitSendReceiveQuota'] = str(hard)
tmp[u'IssueWarningQuota'] = str(int(hard * soft / 100.))
# Randzone users will always be shown. This overrides everything
# else.
if acc['name'] in self._cache_randzone_users:
hide = False
elif acc['owner_id'] in self._cache_no_reservation and \
acc['account_id'] in self._cache_primary_accounts:
hide = False
else:
hide = True
tmp[u'HiddenFromAddressListsEnabled'] = hide
# Collect local delivery status
tmp[u'DeliverToMailboxAndForward'] = \
self._cache_local_delivery.get(tid, False)
# Collect forwarding address
# We do this by doing a difference operation on the forwards and
# the addresses, so we only end up with "external" addresses.
s_fwds = set(self._cache_forwards.get(tid, []))
s_addrs = set(self._cache_addresses.get(tid, []))
ext_fwds = list(s_fwds - s_addrs)
if ext_fwds:
tmp[u'ForwardingSmtpAddress'] = ext_fwds[0]
else:
tmp[u'ForwardingSmtpAddress'] = None
res[acc['name']] = tmp
return res
def collect_exchange_mail_info(self, mb_ou):
"""Collect mailbox-information from Exchange, via LDAP.
:param str mb_ou: The OrganizationalUnit to search for mailboxes.
:rtype: dict
:return: A dict with the mailboxes attributes. The key is the account
name.
"""
attrs = [
'proxyAddresses', 'displayName', 'givenName', 'sn',
'msExchHideFromAddressLists', 'extensionAttribute1',
'mDBUseDefaults', 'mDBOverQuotaLimit', 'mDBOverHardQuotaLimit',
'mDBStorageQuota', 'deliverAndRedirect',
'msExchGenericForwardingAddress'
]
r = self.search(mb_ou, attrs)
ret = {}
for cn, data in r:
if 'extensionAttribute1'in data and \
data['extensionAttribute1'] == ['not migrated'] or \
'ExchangeActiveSyncDevices' in cn:
continue
tmp = {}
name = cn[3:].split(',')[0]
for key in data:
if key == 'proxyAddresses':
addrs = []
for addr in data[key]:
if addr.startswith('SMTP:'):
tmp[u'PrimaryAddress'] = addr[5:]
addrs.append(addr[5:])
tmp[u'EmailAddresses'] = sorted(addrs)
elif key == 'displayName':
tmp[u'DisplayName'] = data[key][0]
elif key == 'givenName':
tmp[u'FirstName'] = data[key][0]
elif key == 'sn':
tmp[u'LastName'] = data[key][0]
elif key == 'mDBUseDefaults':
tmp[u'UseDatabaseQuotaDefaults'] = (True if data[key][0]
== 'TRUE' else False)
elif key == 'mDBOverQuotaLimit':
q = data[key][0]
tmp[u'ProhibitSendQuota'] = q
elif key == 'mDBOverHardQuotaLimit':
q = data[key][0]
tmp[u'ProhibitSendReceiveQuota'] = q
elif key == 'mDBStorageQuota':
q = data[key][0]
tmp[u'IssueWarningQuota'] = q
# Non-existent attribute means that the value is false. Fuckers.
# Collect status about if the mbox is hidden or not
tmp[u'HiddenFromAddressListsEnabled'] = False
if 'msExchHideFromAddressLists' in data:
val = (True if data['msExchHideFromAddressLists'][0] == 'TRUE'
else False)
tmp[u'HiddenFromAddressListsEnabled'] = val
# Collect local delivery status
tmp[u'DeliverToMailboxAndForward'] = False
if 'deliverAndRedirect' in data:
val = (True
if data['deliverAndRedirect'][0] == 'TRUE' else False)
tmp[u'DeliverToMailboxAndForward'] = val
# Collect forwarding address
tmp[u'ForwardingSmtpAddress'] = None
if 'msExchGenericForwardingAddress' in data:
val = data['msExchGenericForwardingAddress'][0]
# We split of smtp:, and store
tmp[u'ForwardingSmtpAddress'] = val.split(':')[1]
ret[name] = tmp
return ret
def compare_mailbox_state(self, ex_state, ce_state, state, config):
"""Compare the information fetched from Cerebrum and Exchange.
This method produces a dict with the state between the systems,
and a report that will be sent to the appropriate target system
administrators.
:param dict ex_state: The state in Exchange.
:param dict ce_state: The state in Cerebrum.
:param dict state: The previous state generated by this method.
:param dict config: Configuration of reporting delays for various
attributes.
:rtype: tuple
:return: A tuple consisting of the new difference-state and a
human-readable report of differences.
"""
s_ce_keys = set(ce_state.keys())
s_ex_keys = set(ex_state.keys())
diff_mb = {}
diff_stale = {}
diff_new = {}
##
# Populate some structures with information we need
# Mailboxes in Exchange, but not in Cerebrum
stale_keys = list(s_ex_keys - s_ce_keys)
for ident in stale_keys:
if state and ident in state['stale_mb']:
diff_stale[ident] = state['stale_mb'][ident]
else:
diff_stale[ident] = time.time()
# Mailboxes in Cerebrum, but not in Exchange
new_keys = list(s_ce_keys - s_ex_keys)
for ident in new_keys:
if state and ident in state['new_mb']:
diff_new[ident] = state['new_mb'][ident]
else:
diff_new[ident] = time.time()
# Check mailboxes that exists in both Cerebrum and Exchange for
# difference (& is union, in case you wondered). If an attribute is not
# in it's desired state in both this and the last run, save the
# timestamp from the last run. This is used for calculating when we nag
# to someone about stuff not beeing in sync.
for key in s_ex_keys & s_ce_keys:
for attr in ce_state[key]:
if state and key in state['mb'] and \
attr in state['mb'][key]:
t_0 = state['mb'][key][attr][u'Time']
else:
t_0 = time.time()
diff_mb.setdefault(key, {})
if attr not in ex_state[key]:
diff_mb[key][attr] = {
u'Exchange': None,
u'Cerebrum': ce_state[key][attr],
u'Time': t_0,
}
elif ce_state[key][attr] != ex_state[key][attr]:
# For quotas, we only want to report mismatches if the
# difference is between the quotas in Cerebrum and Exchange
# is greater than 1% on either side. Hope this is an
# appropriate value to use ;)
try:
if u'Quota' in attr:
exq = ex_state[key][attr]
ceq = ce_state[key][attr]
diff = abs(int(exq) - int(ceq))
avg = (int(exq) + int(ceq)) / 2
one_p = avg * 0.01
if avg + diff < avg + one_p and \
avg - diff > avg - one_p:
continue
except TypeError:
pass
diff_mb[key][attr] = {
u'Exchange': ex_state[key][attr],
u'Cerebrum': ce_state[key][attr],
u'Time': t_0,
}
ret = {'new_mb': diff_new, 'stale_mb': diff_stale, 'mb': diff_mb}
if not state:
return ret, []
now = time.time()
# By now, we have three different dicts. Loop trough them and check if
# we should report 'em
report = [u'# User Attribute Since Cerebrum_value:Exchange_value']
# Report attribute mismatches
for key in diff_mb:
for attr in diff_mb[key]:
delta = (config.get(attr) if attr in config else
config.get('UndefinedAttribute'))
if diff_mb[key][attr][u'Time'] < now - delta:
t = time.strftime(
u'%d%m%Y-%H:%M',
time.localtime(diff_mb[key][attr][u'Time']))
if attr == u'EmailAddresses':
# We report the difference for email addresses, for
# redability
s_ce_addr = set(diff_mb[key][attr][u'Cerebrum'])
s_ex_addr = set(diff_mb[key][attr][u'Exchange'])
new_addr = list(s_ce_addr - s_ex_addr)
stale_addr = list(s_ex_addr - s_ce_addr)
tmp = u'%-10s %-30s %s +%s:-%s' % (
key, attr, t, str(new_addr), str(stale_addr))
else:
tmp = (u'%-10s %-30s %s %s:%s' %
(key, attr, t,
repr(diff_mb[key][attr][u'Cerebrum']),
repr(diff_mb[key][attr][u'Exchange'])))
report += [tmp]
# Report uncreated mailboxes
report += [u'\n# Uncreated mailboxes (uname, time)']
delta = (config.get('UncreatedMailbox') if 'UncreatedMailbox' in config
else config.get('UndefinedAttribute'))
for key in diff_new:
if diff_new[key] < now - delta:
t = time.strftime(u'%d%m%Y-%H:%M',
time.localtime(diff_new[key]))
report += [u'%-10s uncreated_mb %s' % (key, t)]
# Report stale mailboxes
report += [u'\n# Stale mailboxes (uname, time)']
delta = (config.get('StaleMailbox') if 'StaleMailbox' in config else
config.get('UndefinedAttribute'))
for key in diff_stale:
t = time.strftime(u'%d%m%Y-%H:%M', time.localtime(diff_stale[key]))
if diff_stale[key] < now - delta:
report += [u'%-10s stale_mb %s' % (key, t)]
return ret, report
def mangle(from_spread,
to_spread,
file,
commit=False,
log_event=False,
exclude_events=[],
disable_requests=True):
"""Remove and add spreads. Generate events."""
print('%s: Starting to migrate spreads' % str(now()))
db = Factory.get('Database')()
en = Factory.get('Entity')(db)
pe = Factory.get('Person')(db)
ac = Factory.get('Account')(db)
co = Factory.get('Constants')(db)
db.cl_init(change_program='migrate_spreads')
# Parse the types to be excluded
if exclude_events:
tmp = []
for x in exclude_events:
tmp += [int(co.ChangeType(*x.split(':')))]
exclude_events = tmp
# Monkeypatch in a function that avoids creating events for certain
# change types
log_it = db.log_change
def filter_out_types(*args, **kwargs):
if int(args[1]) in exclude_events or not log_event:
kwargs['skip_event'] = True
log_it(*args, **kwargs)
else:
log_it(*args, **kwargs)
db.log_change = filter_out_types
from_spread = co.Spread(from_spread)
to_spread = co.Spread(to_spread)
# TODO: You whom are reading this in the future should make this more
# generic! You can for example replace 'ac' with 'locals()[object]',
# where 'object' defines the object to act on. Then you should use
# __setattr__ to replace the appropriate bofhd_request-generating method.
# If you dare!
if disable_requests:
ac._UiO_order_cyrus_action = lambda *args, **kwargs: True
# Cache all auto groups ids.
exclude_gids = []
# Cache auto_group and meta_auto_group traits.
for trait in en.list_traits(
code=[co.trait_auto_group, co.trait_auto_meta_group]):
exclude_gids.append(trait['entity_id'])
# Import the Exchange-related utils.
from Cerebrum.modules.exchange.CerebrumUtils import CerebrumUtils
cu = CerebrumUtils()
f = open(file, 'r')
for line in f:
uname = line.strip()
if not uname:
continue
ac.clear()
try:
ac.find_by_name(uname)
except Errors.NotFoundError:
print('%s seems to be nuked, skipping..' % uname)
continue
# TODO: Is this _entirely_ correct?
if not ac.get_spread() and ac.expire_date:
print('%s seems to be deleted, skipping..' % uname)
continue
# TODO: We should report if the account does not have the spread
# we want to remove. We should really raise an error if we try to
# delete a spread that does not exist on an account...
# Use savepoints to handle the cases where a user occours two times
# in a file, or a user is remigrated. A savepoint is kind of like a
# nested transaction. We do it like this, because it is simpler to
# implement, than checking the input-file for duplicates and
# existing spreads...
try:
db.execute('SAVEPOINT migrate_spread')
ac.delete_spread(from_spread)
ac.add_spread(to_spread)
except db.IntegrityError:
db.execute('ROLLBACK TO SAVEPOINT migrate_spread')
print('%s allready migrated? skipping..' % uname)
continue
else:
db.execute('RELEASE SAVEPOINT migrate_spread')
ac.write_db()
# Figure out if we should add this user to an auto_group.
# We should only do this for primary accounts.
add_to_auto_group = False
try:
pe.clear()
pe.find(ac.owner_id)
except Errors.NotFoundError:
pass
else:
if ac.entity_id == pe.get_primary_account():
add_to_auto_group = True
# TODO: Factor out this and generalize it! This is a BAD hack
# We really need to generate some faux events. For example, when a user is
# migrated from cyrus to exchange, we must generate events for addition of
# the user into groups :S
###
# Fetch the accounts memberships
# Fetch the persons memberships IF the account is the primary account
# Combine the two above
# Generate events
for gname, gid in cu.get_account_group_memberships(
ac.account_name, co.spread_exchange_group):
# Skip all auto-groups if this is not the primary account.
if gid in exclude_gids and not add_to_auto_group:
continue
ct = co.ChangeType('e_group', 'add')
db.log_change(ac.entity_id,
int(ct),
gid,
change_params=None,
skip_change=True)
print('Creating e_group:add event for %s -> %s' %
(ac.account_name, gname))
###
print('%-9s: removed %s, added %s' %
(uname, str(from_spread), str(to_spread)))
f.close()
if commit:
db.commit()
print 'Committed all changes'
else:
db.rollback()
print 'Rolled back all changes'
print('%s: Finished migrating spreads' % str(now()))
class StateChecker(object):
"""Wrapper class for state-checking functions.
The StateChecker class wraps all the functions we need in order to
verify and report deviances between Cerebrum and Exchange.
"""
# Connect params
LDAP_RETRY_DELAY = 60
LDAP_RETRY_MAX = 5
# Search and result params
LDAP_COM_DELAY = 30
LDAP_COM_MAX = 3
def __init__(self, conf):
"""Initzialize a new instance of out state-checker.
:param logger logger: The logger to use.
:param dict conf: Our StateCheckers configuration.
"""
self.db = Factory.get('Database')(client_encoding='UTF-8')
self.co = Factory.get('Constants')(self.db)
self.dg = Factory.get('DistributionGroup')(self.db)
self.ac = Factory.get('Account')(self.db)
self.gr = Factory.get('Group')(self.db)
self.et = Factory.get('EmailTarget')(self.db)
self.ea = EmailAddress(self.db)
self.ut = CerebrumUtils()
self.config = conf
self._ldap_page_size = 1000
def u(self, db_value):
""" Decode bytestring from database. """
if isinstance(db_value, bytes):
return db_value.decode(self.db.encoding)
return text_type(db_value)
def init_ldap(self):
"""Initzialize LDAP connection."""
self.ldap_srv = ldap.ldapobject.ReconnectLDAPObject(
'%s://%s/' %
(self.config['ldap_proto'], self.config['ldap_server']),
retry_max=self.LDAP_RETRY_MAX,
retry_delay=self.LDAP_RETRY_DELAY)
usr = self.config['ldap_user'].split('\\')[1]
self.ldap_srv.bind_s(self.config['ldap_user'],
read_password(usr, self.config['ldap_server']))
self.ldap_lc = ldap.controls.SimplePagedResultsControl(
True, self._ldap_page_size, '')
def _searcher(self, ou, scope, attrs, ctrls):
""" Perform ldap.search(), but retry in the event of an error.
This wraps the search with error handling, so that the search is
repeated with a delay between attempts.
"""
for attempt in itertools.count(1):
try:
return self.ldap_srv.search_ext(ou,
scope,
attrlist=attrs,
serverctrls=ctrls)
except ldap.LDAPError as e:
if attempt < self.LDAP_COM_MAX:
logger.debug('Caught %r in _searcher on attempt %d', e,
attempt)
time.sleep(self.LDAP_COM_DELAY)
continue
raise
def _recvr(self, msgid):
""" Perform ldap.result3(), but retry in the event of an error.
This wraps the result fetching with error handling, so that the fetch
is repeated with a delay between attempts.
It also decodes all attributes and attribute text values.
"""
for attempt in itertools.count(1):
try:
# return self.ldap_srv.result3(msgid)
rtype, rdata, rmsgid, sc = self.ldap_srv.result3(msgid)
return rtype, decode_attrs(rdata), rmsgid, sc
except ldap.LDAPError as e:
if attempt < self.LDAP_COM_MAX:
logger.debug('Caught %r in _recvr on attempt %d', e,
attempt)
time.sleep(self.LDAP_COM_DELAY)
continue
raise
# This is a paging searcher, that should be used for large amounts of data
def search(self, ou, attrs, scope=ldap.SCOPE_SUBTREE):
"""Wrapper for the search- and result-calls.
Implements paged searching.
:param str ou: The OU to search in.
:param list attrs: The attributes to fetch.
:param int scope: Our search scope, default is subtree.
:rtype: list
:return: List of objects.
"""
# Implementing paging, taken from
# http://www.novell.com/coolsolutions/tip/18274.html
msgid = self._searcher(ou, scope, attrs, [self.ldap_lc])
data = []
ctrltype = ldap.controls.SimplePagedResultsControl.controlType
while True:
time.sleep(1)
rtype, rdata, rmsgid, sc = self._recvr(msgid)
data.extend(rdata)
pctrls = [c for c in sc if c.controlType == ctrltype]
if pctrls:
cookie = pctrls[0].cookie
if cookie:
self.ldap_lc.cookie = cookie
time.sleep(1)
msgid = self._searcher(ou, scope, attrs, [self.ldap_lc])
else:
break
else:
logger.warn('Server ignores RFC 2696 control.')
break
# Skip the OU itself, only return objects in the OU
return data[1:]
# This search wrapper should be used for fetching members
def member_searcher(self, dn, scope, attrs):
"""Utility method for searching for group members.
:param str dn: The groups distinguished name.
:param int scope: Which scope to search by, should be BASE.
:param list attrs: A list of attributes to fetch.
:rtype: tuple
:return: The return-type and the result.
"""
# Wrapping the search, try three times
for attempt in itertools.count(1):
try:
# Search
msgid = self.ldap_srv.search(dn, scope, attrlist=attrs)
# Fetch
rtype, r = self.ldap_srv.result(msgid)
return rtype, r
except ldap.LDAPError as e:
if attempt < self.LDAP_COM_MAX:
logger.debug('Caught %r in member_searcher on attempt %d',
e, attempt)
time.sleep(self.LDAP_COM_DELAY)
continue
raise
# We need to implement a special function to pull out all the members from
# a group, since the idiots at M$ forces us to select a range...
# F*****g asswipes will burn in hell.
def collect_members(self, dn):
"""Fetch a groups members.
This method picks out members in slices, since AD LDAP won't give us
more than 1500 users at a time. If the range-part of the attribute name
ends with a star, we know that we need to look for more members...
:param str dn: The groups distinguished name.
:rtype: list
:return: A list of the members.
"""
# We are searching trough a range. 0 is the start point.
low = str(0)
members = []
end = False
while not end:
# * means that we search for as many attributes as possible, from
# the start point defined by the low-param
attr = ['member;range=%s-*' % low]
# Search'n fetch
time.sleep(1) # Be polite
rtype, r = self.member_searcher(dn, ldap.SCOPE_BASE, attr)
# If this shit hits, no members exists. Break of.
if not r[0][1]:
end = True
break
# Dig out the data
r = r[0][1]
# Extract key
key = r.keys()[0]
# Store members
members.extend(r[key])
# If so, we have reached the end of the range
# (i.e. key is 'member;range=7500-*')
if '*' in key:
end = True
# Extract the new start point from the key
# (i.e. key is 'member;range=0-1499')
else:
low = str(int(key.split('-')[-1]) + 1)
return members
def close(self):
"""Close the connection to the LDAP server."""
self.ldap_srv.unbind_s()
###
# Group related fetching & comparison
###
def collect_exchange_group_info(self, group_ou):
"""Collect group-information from Exchange, via LDAP.
:param str group_ou: The OrganizationalUnit to search for groups.
:rtype: dict
:return: A dict with the group attributes. The key is the group name.
"""
attrs = [
'displayName', 'info', 'proxyAddresses',
'msExchHideFromAddressLists'
]
r = self.search(group_ou, attrs)
ret = {}
for cn, data in r:
tmp = {}
name = cn[3:].split(',')[0]
for key in data:
if key == 'info':
tmp[u'Description'] = data[key][0]
elif key == 'displayName':
tmp[u'DisplayName'] = data[key][0]
elif key == 'proxyAddresses':
addrs = []
for addr in data[key]:
if addr.startswith('SMTP:'):
tmp[u'Primary'] = addr[5:]
# TODO: Correct var?
if (cereconf.EXCHANGE_DEFAULT_ADDRESS_PLACEHOLDER
not in addr):
addrs.append(addr[5:])
tmp[u'Aliases'] = sorted(addrs)
elif key == 'managedBy':
tmp_man = data[key][0][3:].split(',')[0]
if tmp_man == 'Default group moderator':
tmp_man = u'groupadmin'
tmp[u'ManagedBy'] = [tmp_man]
# Skip reporting memberships for roomlists, since we don't manage
# those memberships.
# TODO: Generalize this
if name.startswith('rom-'):
tmp['Members'] = []
else:
# Pulling 'em out the logical way... S..
tmp['Members'] = sorted(
[m[3:].split(',')[0] for m in self.collect_members(cn)])
# Non-existent attribute means that the value is false. Fuckers.
if 'msExchHideFromAddressLists' in data:
tmp_key = 'msExchHideFromAddressLists'
tmp[u'HiddenFromAddressListsEnabled'] = (
True if data[tmp_key][0] == 'TRUE' else False)
else:
tmp[u'HiddenFromAddressListsEnabled'] = False
ret[name] = tmp
return ret
def collect_cerebrum_group_info(self, mb_spread, ad_spread):
"""Collect distgroup related information from Cerebrum.
:param int/str mb_spread: Spread of mailboxes in exchange.
:param int/str ad_spread: Spread of accounts in AD.
:rtype: dict
:return: A dict of users attributes. Uname is key.
"""
mb_spread = self.co.Spread(mb_spread)
ad_spread = self.co.Spread(ad_spread)
u = text_decoder(self.db.encoding)
def _true_or_false(val):
# Yes, we know...
if val == 'T':
return True
elif val == 'F':
return False
else:
return None
tmp = {}
for dg in self.dg.list_distribution_groups():
self.dg.clear()
self.dg.find(dg['group_id'])
roomlist = _true_or_false(self.dg.roomlist)
data = self.dg.get_distgroup_attributes_and_targetdata(
roomlist=roomlist)
tmp[u(self.dg.group_name)] = {
u'Description': u(self.dg.description),
u'DisplayName': u(data['displayname']),
}
if not roomlist:
# Split up the moderated by field, and resolve group members
# from groups if there are groups in the moderated by field!
tmp[u(self.dg.group_name)].update({
u'HiddenFromAddressListsEnabled':
_true_or_false(data['hidden']),
u'Primary':
u(data['primary']),
u'Aliases': [u(v) for v in sorted(data['aliases'])]
})
# Collect members
membs_unfiltered = self.ut.get_group_members(
self.dg.entity_id, spread=mb_spread, filter_spread=ad_spread)
members = [u(member['name']) for member in membs_unfiltered]
tmp[u(self.dg.group_name)].update({u'Members': sorted(members)})
return tmp
def compare_group_state(self, ex_group_info, cere_group_info, state,
config):
"""Compare the information fetched from Cerebrum and Exchange.
This method produces a dict with the state between the systems,
and a report that will be sent to the appropriate target system
administrators.
:param dict ex_state: The state in Exchange.
:param dict ce_state: The state in Cerebrum.
:param dict state: The previous state generated by this method.
:param dict config: Configuration of reporting delays for various
attributes.
:rtype: tuple
:return: A tuple consisting of the new difference-state and a
human-readable report of differences.
"""
s_ce_keys = set(cere_group_info.keys())
s_ex_keys = set(ex_group_info.keys())
diff_group = {}
diff_stale = {}
diff_new = {}
##
# Populate some structures with information we need
# Groups in Exchange, but not in Cerebrum
stale_keys = list(s_ex_keys - s_ce_keys)
for ident in stale_keys:
if state and ident in state['stale_group']:
diff_stale[ident] = state['stale_group'][ident]
else:
diff_stale[ident] = time.time()
# Groups in Cerebrum, but not in Exchange
new_keys = list(s_ce_keys - s_ex_keys)
for ident in new_keys:
if state and ident in state['new_group']:
diff_new[ident] = state['new_group'][ident]
else:
diff_new[ident] = time.time()
# Check groups that exists in both Cerebrum and Exchange for
# difference (& is union, in case you wondered). If an attribute is
# not in it's desired state in both this and the last run, save the
# timestamp from the last run. This is used for calculating when we
# nag to someone about stuff not beeing in sync.
for key in s_ex_keys & s_ce_keys:
for attr in cere_group_info[key]:
tmp = {}
if state and key in state['group'] and \
attr in state['group'][key]:
t_0 = state['group'][key][attr][u'Time']
else:
t_0 = time.time()
if attr not in ex_group_info[key]:
tmp = {
u'Exchange': None,
u'Cerebrum': cere_group_info[key][attr],
u'Time': t_0
}
elif cere_group_info[key][attr] != ex_group_info[key][attr]:
tmp = {
u'Exchange': ex_group_info[key][attr],
u'Cerebrum': cere_group_info[key][attr],
u'Time': t_0
}
if tmp:
diff_group.setdefault(key, {})[attr] = tmp
ret = {
'new_group': diff_new,
'stale_group': diff_stale,
'group': diff_group,
}
if not state:
return ret, []
now = time.time()
# By now, we have three different dicts. Loop trough them and check if
# we should report 'em
report = ['\n\n# Group Attribute Since Cerebrum_value:Exchange_value']
# Report attribute mismatches for groups
for key in diff_group:
for attr in diff_group[key]:
delta = (config.get(attr) if attr in config else
config.get('UndefinedAttribute'))
if diff_group[key][attr][u'Time'] < now - delta:
t = time.strftime(
u'%d%m%Y-%H:%M',
time.localtime(diff_group[key][attr][u'Time']))
if attr in (
u'Aliases',
u'Members',
):
# We report the difference for these types, for
# redability
s_ce_attr = set(diff_group[key][attr][u'Cerebrum'])
try:
s_ex_attr = set(diff_group[key][attr][u'Exchange'])
except TypeError:
s_ex_attr = set([])
new_attr = list(s_ce_attr - s_ex_attr)
stale_attr = list(s_ex_attr - s_ce_attr)
if new_attr == stale_attr:
continue
tmp = u'%-10s %-30s %s +%s:-%s' % (
key, attr, t, str(new_attr), str(stale_attr))
else:
tmp = u'%-10s %-30s %s %s:%s' % (
key, attr, t,
repr(diff_group[key][attr][u'Cerebrum']),
repr(diff_group[key][attr][u'Exchange']))
report += [tmp]
# Report uncreated groups
report += ['\n# Uncreated groups (uname, time)']
attr = 'UncreatedGroup'
delta = (config.get(attr)
if attr in config else config.get('UndefinedAttribute'))
for key in diff_new:
if diff_new[key] < now - delta:
t = time.strftime(u'%d%m%Y-%H:%M',
time.localtime(diff_new[key]))
report += [u'%-10s uncreated_group %s' % (key, t)]
# Report stale groups
report += ['\n# Stale groups (uname, time)']
attr = 'StaleGroup'
delta = (config.get(attr)
if attr in config else config.get('UndefinedAttribute'))
for key in diff_stale:
t = time.strftime(u'%d%m%Y-%H:%M', time.localtime(diff_stale[key]))
if diff_stale[key] < now - delta:
report += [u'%-10s stale_group %s' % (key, t)]
return ret, report
def mangle(from_spread, to_spread, file,
commit=False, log_event=False,
exclude_events=[], disable_requests=True):
"""Remove and add spreads. Generate events."""
print('%s: Starting to migrate spreads' % str(now()))
db = Factory.get('Database')()
en = Factory.get('Entity')(db)
pe = Factory.get('Person')(db)
ac = Factory.get('Account')(db)
co = Factory.get('Constants')(db)
db.cl_init(change_program='migrate_spreads')
# Parse the types to be excluded
if exclude_events:
tmp = []
for x in exclude_events:
tmp += [int(co.ChangeType(*x.split(':')))]
exclude_events = tmp
# Monkeypatch in a function that avoids creating events for certain
# change types
log_it = db.log_change
def filter_out_types(*args, **kwargs):
if int(args[1]) in exclude_events or not log_event:
kwargs['skip_event'] = True
log_it(*args, **kwargs)
else:
log_it(*args, **kwargs)
db.log_change = filter_out_types
from_spread = co.Spread(from_spread)
to_spread = co.Spread(to_spread)
# TODO: You whom are reading this in the future should make this more
# generic! You can for example replace 'ac' with 'locals()[object]',
# where 'object' defines the object to act on. Then you should use
# __setattr__ to replace the appropriate bofhd_request-generating method.
# If you dare!
if disable_requests:
ac._UiO_order_cyrus_action = lambda *args, **kwargs: True
# Cache all auto groups ids.
exclude_gids = []
# Cache auto_group and meta_auto_group traits.
for trait in en.list_traits(code=[co.trait_auto_group,
co.trait_auto_meta_group]):
exclude_gids.append(trait['entity_id'])
# Import the Exchange-related utils.
from Cerebrum.modules.exchange.CerebrumUtils import CerebrumUtils
cu = CerebrumUtils()
f = open(file, 'r')
for line in f:
uname = line.strip()
if not uname:
continue
ac.clear()
try:
ac.find_by_name(uname)
except Errors.NotFoundError:
print('%s seems to be nuked, skipping..' % uname)
continue
# TODO: Is this _entirely_ correct?
if not ac.get_spread() and ac.expire_date:
print('%s seems to be deleted, skipping..' % uname)
continue
# TODO: We should report if the account does not have the spread
# we want to remove. We should really raise an error if we try to
# delete a spread that does not exist on an account...
# Use savepoints to handle the cases where a user occours two times
# in a file, or a user is remigrated. A savepoint is kind of like a
# nested transaction. We do it like this, because it is simpler to
# implement, than checking the input-file for duplicates and
# existing spreads...
try:
db.execute('SAVEPOINT migrate_spread')
ac.delete_spread(from_spread)
ac.add_spread(to_spread)
except db.IntegrityError:
db.execute('ROLLBACK TO SAVEPOINT migrate_spread')
print('%s allready migrated? skipping..' % uname)
continue
else:
db.execute('RELEASE SAVEPOINT migrate_spread')
ac.write_db()
# Figure out if we should add this user to an auto_group.
# We should only do this for primary accounts.
add_to_auto_group = False
try:
pe.clear()
pe.find(ac.owner_id)
except Errors.NotFoundError:
pass
else:
if ac.entity_id == pe.get_primary_account():
add_to_auto_group = True
# TODO: Factor out this and generalize it! This is a BAD hack
# We really need to generate some faux events. For example, when a user is
# migrated from cyrus to exchange, we must generate events for addition of
# the user into groups :S
###
# Fetch the accounts memberships
# Fetch the persons memberships IF the account is the primary account
# Combine the two above
# Generate events
for gname, gid in cu.get_account_group_memberships(
ac.account_name, co.spread_exchange_group):
# Skip all auto-groups if this is not the primary account.
if gid in exclude_gids and not add_to_auto_group:
continue
ct = co.ChangeType('e_group', 'add')
db.log_change(ac.entity_id,
int(ct),
gid,
change_params=None,
skip_change=True)
print('Creating e_group:add event for %s -> %s' %
(ac.account_name, gname))
###
print('%-9s: removed %s, added %s' %
(uname, str(from_spread), str(to_spread)))
f.close()
if commit:
db.commit()
print 'Committed all changes'
else:
db.rollback()
print 'Rolled back all changes'
print('%s: Finished migrating spreads' % str(now()))
class StateChecker(object):
"""Wrapper class for state-checking functions.
The StateChecker class wraps all the functions we need in order to
verify and report deviances between Cerebrum and Exchange.
"""
# Connect params
LDAP_RETRY_DELAY = 60
LDAP_RETRY_MAX = 5
# Search and result params
LDAP_COM_DELAY = 30
LDAP_COM_MAX = 3
def __init__(self, conf):
"""Initzialize a new instance of out state-checker.
:param logger logger: The logger to use.
:param dict conf: Our StateCheckers configuration.
"""
self.db = Factory.get('Database')(client_encoding='UTF-8')
self.co = Factory.get('Constants')(self.db)
self.ac = Factory.get('Account')(self.db)
self.pe = Factory.get('Person')(self.db)
self.gr = Factory.get('Group')(self.db)
self.et = Factory.get('EmailTarget')(self.db)
self.eq = EmailQuota(self.db)
self.ea = EmailAddress(self.db)
self.ef = EmailForward(self.db)
self.cu = CerebrumUtils()
self.config = conf
self._ldap_page_size = 1000
self._cache_randzone_users = self._populate_randzone_cache(
self.config['randzone_unreserve_group'])
self._cache_accounts = self._populate_account_cache(
self.co.spread_exchange_account)
self._cache_addresses = self._populate_address_cache()
self._cache_local_delivery = self._populate_local_delivery_cache()
self._cache_forwards = self._populate_forward_cache()
self._cache_quotas = self._populate_quota_cache()
self._cache_targets = self._populate_target_cache()
self._cache_names = self._populate_name_cache()
self._cache_group_names = self._populate_group_name_cache()
self._cache_no_reservation = self._populate_no_reservation_cache()
self._cache_primary_accounts = self._populate_primary_account_cache()
def init_ldap(self):
"""Initzialize LDAP connection."""
self.ldap_srv = ldap.ldapobject.ReconnectLDAPObject(
'%s://%s/' % (self.config['ldap_proto'],
self.config['ldap_server']),
retry_max=self.LDAP_RETRY_MAX,
retry_delay=self.LDAP_RETRY_DELAY)
usr = self.config['ldap_user'].split('\\')[1]
self.ldap_srv.bind_s(
self.config['ldap_user'],
read_password(usr, self.config['ldap_server']))
self.ldap_lc = ldap.controls.SimplePagedResultsControl(
True, self._ldap_page_size, '')
def _searcher(self, ou, scope, attrs, ctrls):
""" Perform ldap.search(), but retry in the event of an error.
This wraps the search with error handling, so that the search is
repeated with a delay between attempts.
"""
for attempt in itertools.count(1):
try:
return self.ldap_srv.search_ext(
ou, scope, attrlist=attrs, serverctrls=ctrls)
except ldap.LDAPError as e:
if attempt < self.LDAP_COM_MAX:
logger.debug('Caught %r in _searcher on attempt %d',
e, attempt)
time.sleep(self.LDAP_COM_DELAY)
continue
raise
def _recvr(self, msgid):
""" Perform ldap.result3(), but retry in the event of an error.
This wraps the result fetching with error handling, so that the fetch
is repeated with a delay between attempts.
It also decodes all attributes and attribute text values.
"""
for attempt in itertools.count(1):
try:
# return self.ldap_srv.result3(msgid)
rtype, rdata, rmsgid, sc = self.ldap_srv.result3(msgid)
return rtype, decode_attrs(rdata), rmsgid, sc
except ldap.LDAPError as e:
if attempt < self.LDAP_COM_MAX:
logger.debug('Caught %r in _recvr on attempt %d',
e, attempt)
time.sleep(self.LDAP_COM_DELAY)
continue
raise
def search(self, ou, attrs, scope=ldap.SCOPE_SUBTREE):
"""Wrapper for the search- and result-calls.
Implements paged searching.
:param str ou: The OU to search in.
:param list attrs: The attributes to fetch.
:param int scope: Our search scope, default is subtree.
"""
# Implementing paging, taken from
# http://www.novell.com/coolsolutions/tip/18274.html
msgid = self._searcher(ou, scope, attrs, [self.ldap_lc])
data = []
ctrltype = ldap.controls.SimplePagedResultsControl.controlType
while True:
time.sleep(1)
rtype, rdata, rmsgid, sc = self._recvr(msgid)
data.extend(rdata)
pctrls = [c for c in sc if c.controlType == ctrltype]
if pctrls:
cookie = pctrls[0].cookie
if cookie:
self.ldap_lc.cookie = cookie
time.sleep(1)
msgid = self._searcher(ou, scope, attrs, [self.ldap_lc])
else:
break
else:
logger.warn('Server ignores RFC 2696 control.')
break
return data[1:]
def close(self):
"""Close the LDAP connection."""
self.ldap_srv.unbind_s()
#
# Various cache-generating functions.
#
def _populate_randzone_cache(self, randzone):
self.gr.clear()
self.gr.find_by_name(randzone)
u = text_decoder(self.db.encoding)
return [u(x['name'])
for x in self.cu.get_group_members(self.gr.entity_id)]
def _populate_account_cache(self, spread):
u = text_decoder(self.db.encoding)
def to_dict(row):
d = dict(row)
d['name'] = u(row['name'])
d['description'] = u(row['description'])
return d
return [to_dict(r) for r in self.ac.search(spread=spread)]
def _populate_address_cache(self):
u = text_decoder(self.db.encoding)
tmp = defaultdict(list)
# TODO: Implement fetchall?
for addr in self.ea.list_email_addresses_ext():
tmp[addr['target_id']].append(
u'%s@%s' % (u(addr['local_part']), u(addr['domain'])))
return dict(tmp)
def _populate_local_delivery_cache(self):
r = {}
for ld in self.ef.list_local_delivery():
r[ld['target_id']] = ld['local_delivery']
return r
def _populate_forward_cache(self):
u = text_decoder(self.db.encoding)
tmp = defaultdict(list)
for fwd in self.ef.list_email_forwards():
if fwd['enable'] == 'T':
tmp[fwd['target_id']].append(u(fwd['forward_to']))
return dict(tmp)
def _populate_quota_cache(self):
tmp = defaultdict(dict)
# TODO: Implement fetchall?
for quota in self.eq.list_email_quota_ext():
tmp[quota['target_id']]['soft'] = quota['quota_soft']
tmp[quota['target_id']]['hard'] = quota['quota_hard']
return dict(tmp)
def _populate_target_cache(self):
u = text_decoder(self.db.encoding)
tmp = defaultdict(dict)
for targ in self.et.list_email_target_primary_addresses(
target_type=self.co.email_target_account):
tmp[targ['target_entity_id']]['target_id'] = targ['target_id']
tmp[targ['target_entity_id']]['primary'] = \
u'%s@%s' % (u(targ['local_part']), u(targ['domain']))
return dict(tmp)
def _populate_name_cache(self):
u = text_decoder(self.db.encoding)
tmp = defaultdict(dict)
for name in self.pe.search_person_names(
name_variant=[self.co.name_first,
self.co.name_last,
self.co.name_full],
source_system=self.co.system_cached):
tmp[name['person_id']][name['name_variant']] = u(name['name'])
return dict(tmp)
def _populate_group_name_cache(self):
u = text_decoder(self.db.encoding)
tmp = {}
for eid, dom, name in self.gr.list_names(self.co.group_namespace):
tmp[eid] = u(name)
return tmp
def _populate_no_reservation_cache(self):
unreserved = []
for r in self.pe.list_traits(self.co.trait_public_reservation,
fetchall=True):
if r['numval'] == 0:
unreserved.append(r['entity_id'])
return unreserved
def _populate_primary_account_cache(self):
primary = []
for acc in self.ac.list_accounts_by_type(primary_only=True):
primary.append(acc['account_id'])
return primary
###
# Mailbox related state fetching & comparison
###
def collect_cerebrum_mail_info(self):
"""Collect E-mail related information from Cerebrum.
:rtype: dict
:return: A dict of users attributes. Uname is key.
"""
res = {}
for acc in self._cache_accounts:
tmp = {}
try:
tid = self._cache_targets[acc['account_id']]['target_id']
except KeyError:
logger.warn('Could not find account with id:%d in list '
'of targets, skipping..', acc['account_id'])
continue
# Fetch addresses
tmp[u'EmailAddresses'] = sorted(self._cache_addresses[tid])
# Fetch primary address
tmp[u'PrimaryAddress'] = \
self._cache_targets[acc['account_id']]['primary']
# Fetch names
if acc['owner_type'] == self.co.entity_person:
tmp[u'FirstName'] = \
self._cache_names[acc['owner_id']][int(self.co.name_first)]
tmp[u'LastName'] = \
self._cache_names[acc['owner_id']][int(self.co.name_last)]
tmp[u'DisplayName'] = \
self._cache_names[acc['owner_id']][int(self.co.name_full)]
else:
fn, ln, dn = self.cu.construct_group_names(
acc['name'], self._cache_group_names.get(acc['owner_id'],
None))
tmp[u'FirstName'] = fn
tmp[u'LastName'] = ln
tmp[u'DisplayName'] = dn
# Fetch quotas
hard = self._cache_quotas[tid]['hard'] * 1024
soft = self._cache_quotas[tid]['soft']
tmp[u'ProhibitSendQuota'] = str(hard)
tmp[u'ProhibitSendReceiveQuota'] = str(hard)
tmp[u'IssueWarningQuota'] = str(int(hard * soft / 100.))
# Randzone users will always be shown. This overrides everything
# else.
if acc['name'] in self._cache_randzone_users:
hide = False
elif acc['owner_id'] in self._cache_no_reservation and \
acc['account_id'] in self._cache_primary_accounts:
hide = False
else:
hide = True
tmp[u'HiddenFromAddressListsEnabled'] = hide
# Collect local delivery status
tmp[u'DeliverToMailboxAndForward'] = \
self._cache_local_delivery.get(tid, False)
# Collect forwarding address
# We do this by doing a difference operation on the forwards and
# the addresses, so we only end up with "external" addresses.
s_fwds = set(self._cache_forwards.get(tid, []))
s_addrs = set(self._cache_addresses.get(tid, []))
ext_fwds = list(s_fwds - s_addrs)
if ext_fwds:
tmp[u'ForwardingSmtpAddress'] = ext_fwds[0]
else:
tmp[u'ForwardingSmtpAddress'] = None
res[acc['name']] = tmp
return res
def collect_exchange_mail_info(self, mb_ou):
"""Collect mailbox-information from Exchange, via LDAP.
:param str mb_ou: The OrganizationalUnit to search for mailboxes.
:rtype: dict
:return: A dict with the mailboxes attributes. The key is the account
name.
"""
attrs = ['proxyAddresses',
'displayName',
'givenName',
'sn',
'msExchHideFromAddressLists',
'extensionAttribute1',
'mDBUseDefaults',
'mDBOverQuotaLimit',
'mDBOverHardQuotaLimit',
'mDBStorageQuota',
'deliverAndRedirect',
'msExchGenericForwardingAddress']
r = self.search(mb_ou, attrs)
ret = {}
for cn, data in r:
if 'extensionAttribute1'in data and \
data['extensionAttribute1'] == ['not migrated'] or \
'ExchangeActiveSyncDevices' in cn:
continue
tmp = {}
name = cn[3:].split(',')[0]
for key in data:
if key == 'proxyAddresses':
addrs = []
for addr in data[key]:
if addr.startswith('SMTP:'):
tmp[u'PrimaryAddress'] = addr[5:]
addrs.append(addr[5:])
tmp[u'EmailAddresses'] = sorted(addrs)
elif key == 'displayName':
tmp[u'DisplayName'] = data[key][0]
elif key == 'givenName':
tmp[u'FirstName'] = data[key][0]
elif key == 'sn':
tmp[u'LastName'] = data[key][0]
elif key == 'mDBUseDefaults':
tmp[u'UseDatabaseQuotaDefaults'] = (
True if data[key][0] == 'TRUE' else False)
elif key == 'mDBOverQuotaLimit':
q = data[key][0]
tmp[u'ProhibitSendQuota'] = q
elif key == 'mDBOverHardQuotaLimit':
q = data[key][0]
tmp[u'ProhibitSendReceiveQuota'] = q
elif key == 'mDBStorageQuota':
q = data[key][0]
tmp[u'IssueWarningQuota'] = q
# Non-existent attribute means that the value is false. Fuckers.
# Collect status about if the mbox is hidden or not
tmp[u'HiddenFromAddressListsEnabled'] = False
if 'msExchHideFromAddressLists' in data:
val = (True if data['msExchHideFromAddressLists'][0] == 'TRUE'
else False)
tmp[u'HiddenFromAddressListsEnabled'] = val
# Collect local delivery status
tmp[u'DeliverToMailboxAndForward'] = False
if 'deliverAndRedirect' in data:
val = (True if data['deliverAndRedirect'][0] == 'TRUE'
else False)
tmp[u'DeliverToMailboxAndForward'] = val
# Collect forwarding address
tmp[u'ForwardingSmtpAddress'] = None
if 'msExchGenericForwardingAddress' in data:
val = data['msExchGenericForwardingAddress'][0]
# We split of smtp:, and store
tmp[u'ForwardingSmtpAddress'] = val.split(':')[1]
ret[name] = tmp
return ret
def compare_mailbox_state(self, ex_state, ce_state, state, config):
"""Compare the information fetched from Cerebrum and Exchange.
This method produces a dict with the state between the systems,
and a report that will be sent to the appropriate target system
administrators.
:param dict ex_state: The state in Exchange.
:param dict ce_state: The state in Cerebrum.
:param dict state: The previous state generated by this method.
:param dict config: Configuration of reporting delays for various
attributes.
:rtype: tuple
:return: A tuple consisting of the new difference-state and a
human-readable report of differences.
"""
s_ce_keys = set(ce_state.keys())
s_ex_keys = set(ex_state.keys())
diff_mb = {}
diff_stale = {}
diff_new = {}
##
# Populate some structures with information we need
# Mailboxes in Exchange, but not in Cerebrum
stale_keys = list(s_ex_keys - s_ce_keys)
for ident in stale_keys:
if state and ident in state['stale_mb']:
diff_stale[ident] = state['stale_mb'][ident]
else:
diff_stale[ident] = time.time()
# Mailboxes in Cerebrum, but not in Exchange
new_keys = list(s_ce_keys - s_ex_keys)
for ident in new_keys:
if state and ident in state['new_mb']:
diff_new[ident] = state['new_mb'][ident]
else:
diff_new[ident] = time.time()
# Check mailboxes that exists in both Cerebrum and Exchange for
# difference (& is union, in case you wondered). If an attribute is not
# in it's desired state in both this and the last run, save the
# timestamp from the last run. This is used for calculating when we nag
# to someone about stuff not beeing in sync.
for key in s_ex_keys & s_ce_keys:
for attr in ce_state[key]:
if state and key in state['mb'] and \
attr in state['mb'][key]:
t_0 = state['mb'][key][attr][u'Time']
else:
t_0 = time.time()
diff_mb.setdefault(key, {})
if attr not in ex_state[key]:
diff_mb[key][attr] = {
u'Exchange': None,
u'Cerebrum': ce_state[key][attr],
u'Time': t_0,
}
elif ce_state[key][attr] != ex_state[key][attr]:
# For quotas, we only want to report mismatches if the
# difference is between the quotas in Cerebrum and Exchange
# is greater than 1% on either side. Hope this is an
# appropriate value to use ;)
try:
if u'Quota' in attr:
exq = ex_state[key][attr]
ceq = ce_state[key][attr]
diff = abs(int(exq) - int(ceq))
avg = (int(exq) + int(ceq)) / 2
one_p = avg * 0.01
if avg + diff < avg + one_p and \
avg - diff > avg - one_p:
continue
except TypeError:
pass
diff_mb[key][attr] = {
u'Exchange': ex_state[key][attr],
u'Cerebrum': ce_state[key][attr],
u'Time': t_0,
}
ret = {'new_mb': diff_new, 'stale_mb': diff_stale, 'mb': diff_mb}
if not state:
return ret, []
now = time.time()
# By now, we have three different dicts. Loop trough them and check if
# we should report 'em
report = [u'# User Attribute Since Cerebrum_value:Exchange_value']
# Report attribute mismatches
for key in diff_mb:
for attr in diff_mb[key]:
delta = (config.get(attr) if attr in config else
config.get('UndefinedAttribute'))
if diff_mb[key][attr][u'Time'] < now - delta:
t = time.strftime(u'%d%m%Y-%H:%M', time.localtime(
diff_mb[key][attr][u'Time']))
if attr == u'EmailAddresses':
# We report the difference for email addresses, for
# redability
s_ce_addr = set(diff_mb[key][attr][u'Cerebrum'])
s_ex_addr = set(diff_mb[key][attr][u'Exchange'])
new_addr = list(s_ce_addr - s_ex_addr)
stale_addr = list(s_ex_addr - s_ce_addr)
tmp = u'%-10s %-30s %s +%s:-%s' % (key, attr, t,
str(new_addr),
str(stale_addr))
else:
tmp = (u'%-10s %-30s %s %s:%s' %
(key, attr, t,
repr(diff_mb[key][attr][u'Cerebrum']),
repr(diff_mb[key][attr][u'Exchange']))
)
report += [tmp]
# Report uncreated mailboxes
report += [u'\n# Uncreated mailboxes (uname, time)']
delta = (config.get('UncreatedMailbox') if 'UncreatedMailbox' in config
else config.get('UndefinedAttribute'))
for key in diff_new:
if diff_new[key] < now - delta:
t = time.strftime(u'%d%m%Y-%H:%M', time.localtime(
diff_new[key]))
report += [u'%-10s uncreated_mb %s' % (key, t)]
# Report stale mailboxes
report += [u'\n# Stale mailboxes (uname, time)']
delta = (config.get('StaleMailbox') if 'StaleMailbox' in config else
config.get('UndefinedAttribute'))
for key in diff_stale:
t = time.strftime(u'%d%m%Y-%H:%M', time.localtime(
diff_stale[key]))
if diff_stale[key] < now - delta:
report += [u'%-10s stale_mb %s' % (key, t)]
return ret, report