def __init__(self, conf):
"""Initzialize a new instance of out state-checker.
:param logger logger: The logger to use.
:param dict conf: Our StateCheckers configuration.
"""
self.db = Factory.get('Database')(client_encoding='UTF-8')
self.co = Factory.get('Constants')(self.db)
self.ac = Factory.get('Account')(self.db)
self.pe = Factory.get('Person')(self.db)
self.gr = Factory.get('Group')(self.db)
self.et = Factory.get('EmailTarget')(self.db)
self.eq = EmailQuota(self.db)
self.ea = EmailAddress(self.db)
self.ef = EmailForward(self.db)
self.cu = CerebrumUtils()
self.config = conf
self._ldap_page_size = 1000
self._cache_randzone_users = self._populate_randzone_cache(
self.config['randzone_unreserve_group'])
self._cache_accounts = self._populate_account_cache(
self.co.spread_exchange_account)
self._cache_addresses = self._populate_address_cache()
self._cache_local_delivery = self._populate_local_delivery_cache()
self._cache_forwards = self._populate_forward_cache()
self._cache_quotas = self._populate_quota_cache()
self._cache_targets = self._populate_target_cache()
self._cache_names = self._populate_name_cache()
self._cache_group_names = self._populate_group_name_cache()
self._cache_no_reservation = self._populate_no_reservation_cache()
self._cache_primary_accounts = self._populate_primary_account_cache()
def fetch_forward_info(self):
"""
Fetch forward info for all users with both AD and exchange spread.
"""
# from Cerebrum.modules.Email import EmailDomain
from Cerebrum.modules.Email import EmailTarget
from Cerebrum.modules.Email import EmailForward
etarget = EmailTarget(self.db)
# rewrite = EmailDomain(self.db).rewrite_special_domains
eforward = EmailForward(self.db)
# We need a email target -> entity_id mapping
target_id2target_entity_id = {}
for row in etarget.list_email_targets_ext():
if row['target_entity_id']:
te_id = int(row['target_entity_id'])
target_id2target_entity_id[int(row['target_id'])] = te_id
# Check all email forwards
for row in eforward.list_email_forwards():
te_id = target_id2target_entity_id.get(int(row['target_id']))
acc = self.get_account(account_id=te_id)
# We're only interested in those with AD and exchange spread
if acc.to_exchange:
acc.add_forward(row['forward_to'])
def __init__(self):
"""Initialize the Utils."""
self.db = Factory.get('Database')(client_encoding='UTF-8')
self.en = Factory.get('Entity')(self.db)
self.ac = Factory.get('Account')(self.db)
self.pe = Factory.get('Person')(self.db)
self.gr = Factory.get('Group')(self.db)
self.co = Factory.get('Constants')(self.db)
self.ed = EmailDomain(self.db)
self.eq = EmailQuota(self.db)
self.ef = EmailForward(self.db)
self.et = Factory.get('EmailTarget')(self.db)
self.dg = DistributionGroup(self.db)
def __init__(self, conf):
"""Initzialize a new instance of out state-checker.
:param logger logger: The logger to use.
:param dict conf: Our StateCheckers configuration.
"""
self.db = Factory.get('Database')(client_encoding='UTF-8')
self.co = Factory.get('Constants')(self.db)
self.ac = Factory.get('Account')(self.db)
self.pe = Factory.get('Person')(self.db)
self.gr = Factory.get('Group')(self.db)
self.et = Factory.get('EmailTarget')(self.db)
self.eq = EmailQuota(self.db)
self.ea = EmailAddress(self.db)
self.ef = EmailForward(self.db)
self.cu = CerebrumUtils()
self.config = conf
self._ldap_page_size = 1000
self._cache_randzone_users = self._populate_randzone_cache(
self.config['randzone_unreserve_group'])
self._cache_accounts = self._populate_account_cache(
self.co.spread_exchange_account)
self._cache_addresses = self._populate_address_cache()
self._cache_local_delivery = self._populate_local_delivery_cache()
self._cache_forwards = self._populate_forward_cache()
self._cache_quotas = self._populate_quota_cache()
self._cache_targets = self._populate_target_cache()
self._cache_names = self._populate_name_cache()
self._cache_group_names = self._populate_group_name_cache()
self._cache_no_reservation = self._populate_no_reservation_cache()
self._cache_primary_accounts = self._populate_primary_account_cache()
def handle_person(database, source_system, affiliations, send_notifications,
email_config, data):
pe = Factory.get('Person')(database)
ac = Factory.get('Account')(database)
et = EmailTarget(database)
ef = EmailForward(database)
ed = EmailDomain(database)
if (data.get('resourceType') == 'persons' and
'affiliation' in data.get(
'urn:ietf:params:event:SCIM:modify', {}).get(
'attributes', [])):
ident = int(data.get('sub').split('/')[-1])
if not pe.list_affiliations(
person_id=ident,
source_system=source_system,
affiliation=affiliations):
return
pe.clear()
pe.find(ident)
removed_forwards = defaultdict(list)
for account_id in map(lambda x: x['account_id'],
pe.get_accounts(
filter_expired=False)):
try:
et.clear()
et.find_by_target_entity(account_id)
except Errors.NotFoundError:
continue
ef.clear()
ef.find(et.entity_id)
for forward in map(lambda x: x['forward_to'],
ef.get_forward()):
try:
ed.clear()
ed.find_by_domain(forward.split('@')[-1])
except Errors.NotFoundError:
ac.clear()
ac.find(account_id)
ef.delete_forward(forward)
removed_forwards[ac.get_primary_mailaddress()
].append(forward)
logger.info(
'Deleted forward {} from {}'.format(
forward, ac.account_name))
if send_notifications:
for k, v in removed_forwards.items():
sendmail(
toaddr=k,
fromaddr=email_config.sender,
subject=email_config.subject,
body=email_config.body_template.format('\n'.join(v)))
database.commit()
def handle_person(database, source_system, affiliations, send_notifications,
email_config, data):
pe = Factory.get('Person')(database)
ac = Factory.get('Account')(database)
et = EmailTarget(database)
ef = EmailForward(database)
ed = EmailDomain(database)
if (data.get('resourceType') == 'persons' and 'affiliation' in data.get(
'urn:ietf:params:event:SCIM:modify', {}).get('attributes', [])):
ident = int(data.get('sub').split('/')[-1])
if not pe.list_affiliations(person_id=ident,
source_system=source_system,
affiliation=affiliations):
return
pe.clear()
pe.find(ident)
removed_forwards = defaultdict(list)
for account_id in map(lambda x: x['account_id'],
pe.get_accounts(filter_expired=False)):
try:
et.clear()
et.find_by_target_entity(account_id)
except Errors.NotFoundError:
continue
ef.clear()
ef.find(et.entity_id)
for forward in map(lambda x: x['forward_to'], ef.get_forward()):
try:
ed.clear()
ed.find_by_domain(forward.split('@')[-1])
except Errors.NotFoundError:
ac.clear()
ac.find(account_id)
ef.delete_forward(forward)
removed_forwards[ac.get_primary_mailaddress()].append(
forward)
logger.info('Deleted forward {} from {}'.format(
forward, ac.account_name))
if send_notifications:
for k, v in removed_forwards.items():
sendmail(toaddr=k,
fromaddr=email_config.sender,
subject=email_config.subject,
body=email_config.body_template.format('\n'.join(v)))
database.commit()
def __init__(self):
"""Initialize the Utils."""
self.db = Factory.get('Database')(client_encoding='UTF-8')
self.en = Factory.get('Entity')(self.db)
self.ac = Factory.get('Account')(self.db)
self.pe = Factory.get('Person')(self.db)
self.gr = Factory.get('Group')(self.db)
self.co = Factory.get('Constants')(self.db)
self.ed = EmailDomain(self.db)
self.eq = EmailQuota(self.db)
self.ef = EmailForward(self.db)
self.et = Factory.get('EmailTarget')(self.db)
self.dg = DistributionGroup(self.db)
class StateChecker(object):
"""Wrapper class for state-checking functions.
The StateChecker class wraps all the functions we need in order to
verify and report deviances between Cerebrum and Exchange.
"""
# Connect params
LDAP_RETRY_DELAY = 60
LDAP_RETRY_MAX = 5
# Search and result params
LDAP_COM_DELAY = 30
LDAP_COM_MAX = 3
def __init__(self, conf):
"""Initzialize a new instance of out state-checker.
:param logger logger: The logger to use.
:param dict conf: Our StateCheckers configuration.
"""
self.db = Factory.get('Database')(client_encoding='UTF-8')
self.co = Factory.get('Constants')(self.db)
self.ac = Factory.get('Account')(self.db)
self.pe = Factory.get('Person')(self.db)
self.gr = Factory.get('Group')(self.db)
self.et = Factory.get('EmailTarget')(self.db)
self.eq = EmailQuota(self.db)
self.ea = EmailAddress(self.db)
self.ef = EmailForward(self.db)
self.cu = CerebrumUtils()
self.config = conf
self._ldap_page_size = 1000
self._cache_randzone_users = self._populate_randzone_cache(
self.config['randzone_unreserve_group'])
self._cache_accounts = self._populate_account_cache(
self.co.spread_exchange_account)
self._cache_addresses = self._populate_address_cache()
self._cache_local_delivery = self._populate_local_delivery_cache()
self._cache_forwards = self._populate_forward_cache()
self._cache_quotas = self._populate_quota_cache()
self._cache_targets = self._populate_target_cache()
self._cache_names = self._populate_name_cache()
self._cache_group_names = self._populate_group_name_cache()
self._cache_no_reservation = self._populate_no_reservation_cache()
self._cache_primary_accounts = self._populate_primary_account_cache()
def init_ldap(self):
"""Initzialize LDAP connection."""
self.ldap_srv = ldap.ldapobject.ReconnectLDAPObject(
'%s://%s/' %
(self.config['ldap_proto'], self.config['ldap_server']),
retry_max=self.LDAP_RETRY_MAX,
retry_delay=self.LDAP_RETRY_DELAY)
usr = self.config['ldap_user'].split('\\')[1]
self.ldap_srv.bind_s(self.config['ldap_user'],
read_password(usr, self.config['ldap_server']))
self.ldap_lc = ldap.controls.SimplePagedResultsControl(
True, self._ldap_page_size, '')
def _searcher(self, ou, scope, attrs, ctrls):
""" Perform ldap.search(), but retry in the event of an error.
This wraps the search with error handling, so that the search is
repeated with a delay between attempts.
"""
for attempt in itertools.count(1):
try:
return self.ldap_srv.search_ext(ou,
scope,
attrlist=attrs,
serverctrls=ctrls)
except ldap.LDAPError as e:
if attempt < self.LDAP_COM_MAX:
logger.debug('Caught %r in _searcher on attempt %d', e,
attempt)
time.sleep(self.LDAP_COM_DELAY)
continue
raise
def _recvr(self, msgid):
""" Perform ldap.result3(), but retry in the event of an error.
This wraps the result fetching with error handling, so that the fetch
is repeated with a delay between attempts.
It also decodes all attributes and attribute text values.
"""
for attempt in itertools.count(1):
try:
# return self.ldap_srv.result3(msgid)
rtype, rdata, rmsgid, sc = self.ldap_srv.result3(msgid)
return rtype, decode_attrs(rdata), rmsgid, sc
except ldap.LDAPError as e:
if attempt < self.LDAP_COM_MAX:
logger.debug('Caught %r in _recvr on attempt %d', e,
attempt)
time.sleep(self.LDAP_COM_DELAY)
continue
raise
def search(self, ou, attrs, scope=ldap.SCOPE_SUBTREE):
"""Wrapper for the search- and result-calls.
Implements paged searching.
:param str ou: The OU to search in.
:param list attrs: The attributes to fetch.
:param int scope: Our search scope, default is subtree.
"""
# Implementing paging, taken from
# http://www.novell.com/coolsolutions/tip/18274.html
msgid = self._searcher(ou, scope, attrs, [self.ldap_lc])
data = []
ctrltype = ldap.controls.SimplePagedResultsControl.controlType
while True:
time.sleep(1)
rtype, rdata, rmsgid, sc = self._recvr(msgid)
data.extend(rdata)
pctrls = [c for c in sc if c.controlType == ctrltype]
if pctrls:
cookie = pctrls[0].cookie
if cookie:
self.ldap_lc.cookie = cookie
time.sleep(1)
msgid = self._searcher(ou, scope, attrs, [self.ldap_lc])
else:
break
else:
logger.warn('Server ignores RFC 2696 control.')
break
return data[1:]
def close(self):
"""Close the LDAP connection."""
self.ldap_srv.unbind_s()
#
# Various cache-generating functions.
#
def _populate_randzone_cache(self, randzone):
self.gr.clear()
self.gr.find_by_name(randzone)
u = text_decoder(self.db.encoding)
return [
u(x['name']) for x in self.cu.get_group_members(self.gr.entity_id)
]
def _populate_account_cache(self, spread):
u = text_decoder(self.db.encoding)
def to_dict(row):
d = dict(row)
d['name'] = u(row['name'])
d['description'] = u(row['description'])
return d
return [to_dict(r) for r in self.ac.search(spread=spread)]
def _populate_address_cache(self):
u = text_decoder(self.db.encoding)
tmp = defaultdict(list)
# TODO: Implement fetchall?
for addr in self.ea.list_email_addresses_ext():
tmp[addr['target_id']].append(
u'%s@%s' % (u(addr['local_part']), u(addr['domain'])))
return dict(tmp)
def _populate_local_delivery_cache(self):
r = {}
for ld in self.ef.list_local_delivery():
r[ld['target_id']] = ld['local_delivery']
return r
def _populate_forward_cache(self):
u = text_decoder(self.db.encoding)
tmp = defaultdict(list)
for fwd in self.ef.list_email_forwards():
if fwd['enable'] == 'T':
tmp[fwd['target_id']].append(u(fwd['forward_to']))
return dict(tmp)
def _populate_quota_cache(self):
tmp = defaultdict(dict)
# TODO: Implement fetchall?
for quota in self.eq.list_email_quota_ext():
tmp[quota['target_id']]['soft'] = quota['quota_soft']
tmp[quota['target_id']]['hard'] = quota['quota_hard']
return dict(tmp)
def _populate_target_cache(self):
u = text_decoder(self.db.encoding)
tmp = defaultdict(dict)
for targ in self.et.list_email_target_primary_addresses(
target_type=self.co.email_target_account):
tmp[targ['target_entity_id']]['target_id'] = targ['target_id']
tmp[targ['target_entity_id']]['primary'] = \
u'%s@%s' % (u(targ['local_part']), u(targ['domain']))
return dict(tmp)
def _populate_name_cache(self):
u = text_decoder(self.db.encoding)
tmp = defaultdict(dict)
for name in self.pe.search_person_names(
name_variant=[
self.co.name_first, self.co.name_last, self.co.name_full
],
source_system=self.co.system_cached):
tmp[name['person_id']][name['name_variant']] = u(name['name'])
return dict(tmp)
def _populate_group_name_cache(self):
u = text_decoder(self.db.encoding)
tmp = {}
for eid, dom, name in self.gr.list_names(self.co.group_namespace):
tmp[eid] = u(name)
return tmp
def _populate_no_reservation_cache(self):
unreserved = []
for r in self.pe.list_traits(self.co.trait_public_reservation,
fetchall=True):
if r['numval'] == 0:
unreserved.append(r['entity_id'])
return unreserved
def _populate_primary_account_cache(self):
primary = []
for acc in self.ac.list_accounts_by_type(primary_only=True):
primary.append(acc['account_id'])
return primary
###
# Mailbox related state fetching & comparison
###
def collect_cerebrum_mail_info(self):
"""Collect E-mail related information from Cerebrum.
:rtype: dict
:return: A dict of users attributes. Uname is key.
"""
res = {}
for acc in self._cache_accounts:
tmp = {}
try:
tid = self._cache_targets[acc['account_id']]['target_id']
except KeyError:
logger.warn(
'Could not find account with id:%d in list '
'of targets, skipping..', acc['account_id'])
continue
# Fetch addresses
tmp[u'EmailAddresses'] = sorted(self._cache_addresses[tid])
# Fetch primary address
tmp[u'PrimaryAddress'] = \
self._cache_targets[acc['account_id']]['primary']
# Fetch names
if acc['owner_type'] == self.co.entity_person:
tmp[u'FirstName'] = \
self._cache_names[acc['owner_id']][int(self.co.name_first)]
tmp[u'LastName'] = \
self._cache_names[acc['owner_id']][int(self.co.name_last)]
tmp[u'DisplayName'] = \
self._cache_names[acc['owner_id']][int(self.co.name_full)]
else:
fn, ln, dn = self.cu.construct_group_names(
acc['name'],
self._cache_group_names.get(acc['owner_id'], None))
tmp[u'FirstName'] = fn
tmp[u'LastName'] = ln
tmp[u'DisplayName'] = dn
# Fetch quotas
hard = self._cache_quotas[tid]['hard'] * 1024
soft = self._cache_quotas[tid]['soft']
tmp[u'ProhibitSendQuota'] = str(hard)
tmp[u'ProhibitSendReceiveQuota'] = str(hard)
tmp[u'IssueWarningQuota'] = str(int(hard * soft / 100.))
# Randzone users will always be shown. This overrides everything
# else.
if acc['name'] in self._cache_randzone_users:
hide = False
elif acc['owner_id'] in self._cache_no_reservation and \
acc['account_id'] in self._cache_primary_accounts:
hide = False
else:
hide = True
tmp[u'HiddenFromAddressListsEnabled'] = hide
# Collect local delivery status
tmp[u'DeliverToMailboxAndForward'] = \
self._cache_local_delivery.get(tid, False)
# Collect forwarding address
# We do this by doing a difference operation on the forwards and
# the addresses, so we only end up with "external" addresses.
s_fwds = set(self._cache_forwards.get(tid, []))
s_addrs = set(self._cache_addresses.get(tid, []))
ext_fwds = list(s_fwds - s_addrs)
if ext_fwds:
tmp[u'ForwardingSmtpAddress'] = ext_fwds[0]
else:
tmp[u'ForwardingSmtpAddress'] = None
res[acc['name']] = tmp
return res
def collect_exchange_mail_info(self, mb_ou):
"""Collect mailbox-information from Exchange, via LDAP.
:param str mb_ou: The OrganizationalUnit to search for mailboxes.
:rtype: dict
:return: A dict with the mailboxes attributes. The key is the account
name.
"""
attrs = [
'proxyAddresses', 'displayName', 'givenName', 'sn',
'msExchHideFromAddressLists', 'extensionAttribute1',
'mDBUseDefaults', 'mDBOverQuotaLimit', 'mDBOverHardQuotaLimit',
'mDBStorageQuota', 'deliverAndRedirect',
'msExchGenericForwardingAddress'
]
r = self.search(mb_ou, attrs)
ret = {}
for cn, data in r:
if 'extensionAttribute1'in data and \
data['extensionAttribute1'] == ['not migrated'] or \
'ExchangeActiveSyncDevices' in cn:
continue
tmp = {}
name = cn[3:].split(',')[0]
for key in data:
if key == 'proxyAddresses':
addrs = []
for addr in data[key]:
if addr.startswith('SMTP:'):
tmp[u'PrimaryAddress'] = addr[5:]
addrs.append(addr[5:])
tmp[u'EmailAddresses'] = sorted(addrs)
elif key == 'displayName':
tmp[u'DisplayName'] = data[key][0]
elif key == 'givenName':
tmp[u'FirstName'] = data[key][0]
elif key == 'sn':
tmp[u'LastName'] = data[key][0]
elif key == 'mDBUseDefaults':
tmp[u'UseDatabaseQuotaDefaults'] = (True if data[key][0]
== 'TRUE' else False)
elif key == 'mDBOverQuotaLimit':
q = data[key][0]
tmp[u'ProhibitSendQuota'] = q
elif key == 'mDBOverHardQuotaLimit':
q = data[key][0]
tmp[u'ProhibitSendReceiveQuota'] = q
elif key == 'mDBStorageQuota':
q = data[key][0]
tmp[u'IssueWarningQuota'] = q
# Non-existent attribute means that the value is false. Fuckers.
# Collect status about if the mbox is hidden or not
tmp[u'HiddenFromAddressListsEnabled'] = False
if 'msExchHideFromAddressLists' in data:
val = (True if data['msExchHideFromAddressLists'][0] == 'TRUE'
else False)
tmp[u'HiddenFromAddressListsEnabled'] = val
# Collect local delivery status
tmp[u'DeliverToMailboxAndForward'] = False
if 'deliverAndRedirect' in data:
val = (True
if data['deliverAndRedirect'][0] == 'TRUE' else False)
tmp[u'DeliverToMailboxAndForward'] = val
# Collect forwarding address
tmp[u'ForwardingSmtpAddress'] = None
if 'msExchGenericForwardingAddress' in data:
val = data['msExchGenericForwardingAddress'][0]
# We split of smtp:, and store
tmp[u'ForwardingSmtpAddress'] = val.split(':')[1]
ret[name] = tmp
return ret
def compare_mailbox_state(self, ex_state, ce_state, state, config):
"""Compare the information fetched from Cerebrum and Exchange.
This method produces a dict with the state between the systems,
and a report that will be sent to the appropriate target system
administrators.
:param dict ex_state: The state in Exchange.
:param dict ce_state: The state in Cerebrum.
:param dict state: The previous state generated by this method.
:param dict config: Configuration of reporting delays for various
attributes.
:rtype: tuple
:return: A tuple consisting of the new difference-state and a
human-readable report of differences.
"""
s_ce_keys = set(ce_state.keys())
s_ex_keys = set(ex_state.keys())
diff_mb = {}
diff_stale = {}
diff_new = {}
##
# Populate some structures with information we need
# Mailboxes in Exchange, but not in Cerebrum
stale_keys = list(s_ex_keys - s_ce_keys)
for ident in stale_keys:
if state and ident in state['stale_mb']:
diff_stale[ident] = state['stale_mb'][ident]
else:
diff_stale[ident] = time.time()
# Mailboxes in Cerebrum, but not in Exchange
new_keys = list(s_ce_keys - s_ex_keys)
for ident in new_keys:
if state and ident in state['new_mb']:
diff_new[ident] = state['new_mb'][ident]
else:
diff_new[ident] = time.time()
# Check mailboxes that exists in both Cerebrum and Exchange for
# difference (& is union, in case you wondered). If an attribute is not
# in it's desired state in both this and the last run, save the
# timestamp from the last run. This is used for calculating when we nag
# to someone about stuff not beeing in sync.
for key in s_ex_keys & s_ce_keys:
for attr in ce_state[key]:
if state and key in state['mb'] and \
attr in state['mb'][key]:
t_0 = state['mb'][key][attr][u'Time']
else:
t_0 = time.time()
diff_mb.setdefault(key, {})
if attr not in ex_state[key]:
diff_mb[key][attr] = {
u'Exchange': None,
u'Cerebrum': ce_state[key][attr],
u'Time': t_0,
}
elif ce_state[key][attr] != ex_state[key][attr]:
# For quotas, we only want to report mismatches if the
# difference is between the quotas in Cerebrum and Exchange
# is greater than 1% on either side. Hope this is an
# appropriate value to use ;)
try:
if u'Quota' in attr:
exq = ex_state[key][attr]
ceq = ce_state[key][attr]
diff = abs(int(exq) - int(ceq))
avg = (int(exq) + int(ceq)) / 2
one_p = avg * 0.01
if avg + diff < avg + one_p and \
avg - diff > avg - one_p:
continue
except TypeError:
pass
diff_mb[key][attr] = {
u'Exchange': ex_state[key][attr],
u'Cerebrum': ce_state[key][attr],
u'Time': t_0,
}
ret = {'new_mb': diff_new, 'stale_mb': diff_stale, 'mb': diff_mb}
if not state:
return ret, []
now = time.time()
# By now, we have three different dicts. Loop trough them and check if
# we should report 'em
report = [u'# User Attribute Since Cerebrum_value:Exchange_value']
# Report attribute mismatches
for key in diff_mb:
for attr in diff_mb[key]:
delta = (config.get(attr) if attr in config else
config.get('UndefinedAttribute'))
if diff_mb[key][attr][u'Time'] < now - delta:
t = time.strftime(
u'%d%m%Y-%H:%M',
time.localtime(diff_mb[key][attr][u'Time']))
if attr == u'EmailAddresses':
# We report the difference for email addresses, for
# redability
s_ce_addr = set(diff_mb[key][attr][u'Cerebrum'])
s_ex_addr = set(diff_mb[key][attr][u'Exchange'])
new_addr = list(s_ce_addr - s_ex_addr)
stale_addr = list(s_ex_addr - s_ce_addr)
tmp = u'%-10s %-30s %s +%s:-%s' % (
key, attr, t, str(new_addr), str(stale_addr))
else:
tmp = (u'%-10s %-30s %s %s:%s' %
(key, attr, t,
repr(diff_mb[key][attr][u'Cerebrum']),
repr(diff_mb[key][attr][u'Exchange'])))
report += [tmp]
# Report uncreated mailboxes
report += [u'\n# Uncreated mailboxes (uname, time)']
delta = (config.get('UncreatedMailbox') if 'UncreatedMailbox' in config
else config.get('UndefinedAttribute'))
for key in diff_new:
if diff_new[key] < now - delta:
t = time.strftime(u'%d%m%Y-%H:%M',
time.localtime(diff_new[key]))
report += [u'%-10s uncreated_mb %s' % (key, t)]
# Report stale mailboxes
report += [u'\n# Stale mailboxes (uname, time)']
delta = (config.get('StaleMailbox') if 'StaleMailbox' in config else
config.get('UndefinedAttribute'))
for key in diff_stale:
t = time.strftime(u'%d%m%Y-%H:%M', time.localtime(diff_stale[key]))
if diff_stale[key] < now - delta:
report += [u'%-10s stale_mb %s' % (key, t)]
return ret, report
class CerebrumUtils(object):
"""Utility-class containing often used functions for Exchange."""
def __init__(self):
"""Initialize the Utils."""
self.db = Factory.get('Database')(client_encoding='UTF-8')
self.en = Factory.get('Entity')(self.db)
self.ac = Factory.get('Account')(self.db)
self.pe = Factory.get('Person')(self.db)
self.gr = Factory.get('Group')(self.db)
self.co = Factory.get('Constants')(self.db)
self.clconst = Factory.get('CLConstants')(self.db)
self.ed = EmailDomain(self.db)
self.eq = EmailQuota(self.db)
self.ef = EmailForward(self.db)
self.et = Factory.get('EmailTarget')(self.db)
self.dg = DistributionGroup(self.db)
####
# Person related methods
####
def get_person_accounts(self, person_id, spread=None):
"""Return a list of account information.
:type person_id: int
:param person_id: The person id to look up by.
:rtype: list
:return: A list of (account_id, username) tuples."""
ret = [(x['account_id'], x['name']) for x in
self.ac.search(owner_id=person_id,
owner_type=self.co.entity_person,
spread=self.co.spread_exchange_account)]
self.db.rollback()
return ret
def get_person_names(self, account_id=None, person_id=None):
"""Return a persons names.
:type account_id: int
:param account_id: The user to look up names by.
:type person_id: int
:param person_id: The person to look up names by.
:rtype: tuple
:return: (first_name, last_name, full name).
"""
# TODO: search_name_with_language?
if account_id:
self.ac.clear()
self.ac.find(account_id)
self.pe.clear()
self.pe.find(self.ac.owner_id)
elif person_id:
self.pe.clear()
self.pe.find(person_id)
else:
raise AssertionError('Called w/o person or account id')
ret = (self.pe.get_name(self.co.system_cached, self.co.name_first),
self.pe.get_name(self.co.system_cached, self.co.name_last),
self.pe.get_name(self.co.system_cached, self.co.name_full,))
self.db.rollback()
return ret
def get_person_membership_groupnames(self, person_id):
"""List all the groups the person is a member of.
:type person_id: int
:param person_id: The persons entity_id.
:rtype: list
:return: list(string) of groupnames."""
ret = [x['group_name'] for x in self.gr.search_members(
member_id=person_id, indirect_members=True)]
self.db.rollback()
return ret
def is_electronic_reserved(self, person_id=None, account_id=None):
"""Check if a person has reserved themself from listing.
:type person_id: int
:param person_id: The persons entity_id.
:type account_id: int
:param account_id: The accounts entity_id.
:rtype: bool
:return: True if reserved, False otherwise."""
if person_id:
self.pe.clear()
self.pe.find(person_id)
ret = self.pe.has_e_reservation()
elif account_id:
self.ac.clear()
self.ac.find(account_id)
ret = self.ac.owner_has_ereservation()
# TODO: This is sane?
else:
ret = True
self.db.rollback()
return ret
####
# Account related methods
####
def get_account_mailaddrs(self, account_id):
"""Collect mailaddresses of an account.
:type account_id: int
:param account_id: The account_id representing the object.
:rtype: list
:return: A list of addresses."""
self.et.clear()
self.et.find_by_target_entity(account_id)
addrs = ['%s@%s' % (x['local_part'], x['domain'])
for x in self.et.get_addresses()]
self.db.rollback()
return addrs
def get_account_forwards(self, account_id):
"""Collect an accounts forward addresses.
:param int account_id: The account_id representing the object.
:return: A list of forward addresses."""
self.ef.clear()
self.ef.find_by_target_entity(account_id)
r = []
for fwd in self.ef.get_forward():
# Need to do keys() for now, db_row is stupid.
if 'enable' in fwd.keys() and fwd['enable'] == 'T':
r.append(fwd['forward_to'])
self.db.rollback()
return r
def get_account_local_delivery(self, account_id):
"""Check if this account should have local delivery.
:param int account_id: The account_id representing the object.
:rtype: bool
:return: Local delivery on or off."""
self.ef.clear()
self.ef.find_by_target_entity(account_id)
return self.ef.local_delivery
def get_account_name(self, account_id):
"""Return information about the account.
:type account_id: int
:param acccount_id: The accounts entity id.
:rtype: string
:return: The accounts name."""
self.ac.clear()
self.ac.find(account_id)
ret = self.ac.account_name
self.db.rollback()
return ret
def get_account_owner_info(self, account_id):
"""Return the type and id of the accounts owner.
:type account_id: int
:param acccount_id: The accounts entity id.
:rtype: tuple
:return: (entity_type, entity_id)."""
self.ac.clear()
self.ac.find(account_id)
ret = (self.ac.owner_type, self.ac.owner_id)
self.db.rollback()
return ret
def get_account_spreads(self, account_id):
"""Return the accounts spread codes.
:type account_id: int
:param acccount_id: The accounts entity id.
:rtype: list
:return: A list of the accounts spread codes."""
self.ac.clear()
self.ac.find(account_id)
ret = [x['spread'] for x in self.ac.get_spread()]
self.db.rollback()
return ret
def get_account_primary_email(self, account_id):
"""Return the accounts primary address.
:type account_id: int
:param acccount_id: The accounts entity id.
:rtype: str
:return: The accounts primary email-address."""
self.ac.clear()
self.ac.find(account_id)
ret = self.ac.get_primary_mailaddress()
self.db.rollback()
return ret
def get_account_id(self, uname):
"""Get an accounts entity id.
:type uname: str
:param uname: The users name.
:rtype: int
:return: The entity id of the account."""
self.ac.clear()
self.ac.find_by_name(uname)
ret = self.ac.entity_id
self.db.rollback()
return ret
def get_primary_account(self, person_id):
"""Get a persons primary account.
:type person_id: int
:param person_id: The persons entity id.
:rtype: int
:return: The primary accounts entity_id or None if no primary."""
self.pe.clear()
self.pe.find(person_id)
ret = self.pe.get_primary_account()
self.db.rollback()
return ret
def get_account_group_memberships(self, uname, group_spread):
"""Get a list of group names, which the user (and users owner, if
person), is a member of.
:type uname: string
:param uname: The accounts name.
:type group_spread: _SpreadCode
:param group_spread: Spread to filter groups by.
:rtype: list(tuple)
:return: A list of tuples contining group-name and -id."""
self.ac.clear()
self.ac.find_by_name(uname)
groups = []
# Fetch the groups the person is a member of
if self.ac.owner_type == self.co.entity_person:
for group in self.gr.search(member_id=self.ac.owner_id,
spread=group_spread,
indirect_members=True):
groups.append((group['name'], group['group_id']))
# Fetch the groups the account is a member of
for group in self.gr.search(member_id=self.ac.entity_id,
spread=group_spread,
indirect_members=True):
groups.append((group['name'], group['group_id']))
self.db.rollback()
return groups
####
# Group related methods
####
def construct_group_names(self, uname, gname):
"""Construct Exchange related group names.
:param str uname: The users username.
:param str gname: The owning groups groupname.
:rtype: tuple(str)
:return: A tuple consisting of FirstName, LastName and DisplayName."""
fn = uname
ln = '(owner: %s)' % gname
dn = '%s (owner: %s)' % (uname, gname)
return (fn, ln, dn)
def get_group_information(self, group_id):
"""Get a groups name and description.
:type group_id: int
:param group_id: The groups entity id.
:rtype: tuple
:return: The groups name and description."""
# This function is kinda generic. We don't care
# (yet) if it's a distribution or security group.
r = self.gr.search(group_id=group_id)[0]
ret = (r['name'], r['description'])
self.db.rollback()
return ret
def get_group_id(self, group_name):
"""Get a groups entity_id.
:type group_name: str
:param group_name: The groups name.
:rtype: int
:return: The groups entity_id."""
self.gr.clear()
self.gr.find_by_name(group_name)
ret = self.gr.entity_id
self.db.rollback()
return ret
def get_group_spreads(self, group_id):
"""Return the groupss spread codes.
:type group_id: int
:param group_id: The accounts entity id.
:rtype: list
:return: A list of the accounts spread codes."""
self.gr.clear()
self.gr.find(group_id)
ret = [x['spread'] for x in self.gr.get_spread()]
self.db.rollback()
return ret
def get_group_members(self, group_id, spread=None, filter_spread=None):
"""Collect a list of the usernames of all users in a group.
:type group_id: int
:param group_id: The groups entity_id.
:type spread: _SpreadCode
:param spread: The spread to filter by.
:type filter_spread: _SpreadCode
:param filter_spread: A spread that the user must also have.
:rtype: list
:return: A list of usernames who are members of group_id."""
found_accounts = []
r = []
for x in self.gr.search_members(group_id=group_id,
indirect_members=True,
member_spread=spread,
member_type=self.co.entity_account):
if x['member_id'] not in found_accounts:
if filter_spread in self.get_account_spreads(x['member_id']):
r.append({'name': self.get_account_name(x['member_id']),
'account_id': x['member_id']})
found_accounts.append(x['member_id'])
spreads = []
if spread:
spreads += [int(spread)]
if filter_spread:
spreads += [int(filter_spread)]
for x in self.gr.search_members(group_id=group_id,
indirect_members=True,
member_type=self.co.entity_person):
self.pe.clear()
self.pe.find(x['member_id'])
aid = self.pe.get_primary_account()
if aid:
self.ac.clear()
self.ac.find(aid)
if self.ac.entity_id not in found_accounts:
# If/elif used to allow usage without filter and
# filter_spread params
if spreads and \
set(spreads).issubset(
set([x['spread'] for x in self.ac.get_spread()])):
r.append({'name': self.ac.account_name,
'account_id': self.ac.entity_id})
found_accounts.append(self.ac.entity_id)
elif not spreads:
r.append({'name': self.ac.account_name,
'account_id': self.ac.entity_id})
found_accounts.append(self.ac.entity_id)
self.db.rollback()
return r
def get_parent_groups(self, id, spread=None, name_prefix=None):
"""Return all groups that the group is an indirect member of. Filter
by spread and the start of the group name.
:type id: int
:param id: The entity_id.
:type spread: _SpreadCode
:param _SpreadCode: The spread code to filter by, default is None.
:type name_prefix: str
:param name_prefix: Check if the group starts with this, default None.
:rtype: list
:return: A list of appropriate groups."""
if name_prefix:
np = '%s%%' % name_prefix
else:
np = None
groups = []
for group in self.gr.search(member_id=id, indirect_members=True,
spread=spread, name=np):
groups.append(group['name'])
self.db.rollback()
return groups
####
# Other utility methods
####
def load_params(self, event):
"""Get the change params of an event.
:type event: dbrow
:param event: The db row returned by Change- or EventLog.
:rtype: dict or None
:return: The change params."""
params = event['change_params']
if isinstance(params, basestring):
return json.loads(params)
return params
def get_entity_type(self, entity_id):
"""Fetch the entity type code of an entity.
:type entity_id: int
:param entity_id: The entity id.
:rtype: long
:return: The entity type code."""
self.en.clear()
self.en.find(entity_id)
ret = self.en.entity_type
self.db.rollback()
return ret
def log_event(self, event, trigger):
"""Utility method used to create an event only in the EventLog.
:type event: dict
:param event: Dict representing an event (as returned from get_event).
:param trigger: str or list or tuple
:param trigger: The change type code we want to associate with the
event. Only the first value will be used."""
if isinstance(trigger, (list, tuple)):
trigger = trigger[0]
trigger = trigger.split(':')
ct = self.clconst.ChangeType(trigger[0], trigger[1])
params = self.load_params(event)
self.db.log_change(event['subject_entity'],
int(ct),
event['dest_entity'],
change_params=params,
skip_change=True,
skip_publish=True)
self.db.commit()
def log_event_receipt(self, event, trigger):
"""Log the "receipt" of a completed event in ChangeLog.
:type event: dict
:param event: Dict representing an event (as returned from get_event).
:param trigger: str or list or tuple
:param trigger: The change type code we want to associate with the
event. Only the first value will be used."""
# TODO: Set change_program from a sensible source to something smart
if isinstance(trigger, (list, tuple)):
trigger = trigger[0]
trigger = trigger.split(':')
ct = self.clconst.ChangeType(trigger[0], trigger[1])
parm = {'change_program': 'ExchangeIntegration',
'skip_event': True,
'skip_publish': True}
# Only log params if they actually contain something.
param = self.load_params(event)
if param:
parm['change_params'] = param
self.db.log_change(event['subject_entity'],
int(ct),
event['dest_entity'],
**parm)
self.db.commit()
####
# Email* utility methods
####
def get_email_target_info(self, target_id=None, target_entity=None):
"""Return the entity_type and entity_id of the entity the EmailTarget
points at. Look up by _either_ target_id or target_entity.
:type target_id: int
:param target_id: The EmailTargets id.
:type target_entity: int
:param target_entity: The targets target entity id.
:rtype: typle
:return: (entity_id, target_entity_id, target_entity_type, hard_quota,
soft_quota)."""
self.et.clear()
self.eq.clear()
if target_id:
self.et.find(target_id)
try:
self.eq.find(target_id)
except Errors.NotFoundError:
self.eq.clear()
elif target_entity:
self.et.find_by_target_entity(target_entity)
try:
self.eq.find(self.et.entity_id)
except Errors.NotFoundError:
self.eq.clear()
else:
self.db.rollback()
raise Errors.ProgrammingError(
'Must define either target_id og target_entity')
ret = (self.et.entity_id,
self.et.email_target_entity_id,
self.et.email_target_entity_type,
self.eq.get_quota_hard(),
self.eq.get_quota_soft())
self.db.rollback()
return ret
def get_email_domain_info(self, email_domain_id=None,
email_domain_name=None):
"""Return info about an EmailDomain.
:type email_domain_id: int
:param email_domain_id: The email domains id.
:rtype: dict
:return: {name: EmailDomain name, id: EmailDomain id}."""
self.ed.clear()
if email_domain_id:
self.ed.find(email_domain_id)
elif email_domain_name:
self.ed.find_by_domain(email_domain_name)
ret = {'name': self.ed.email_domain_name, 'id': self.ed.entity_id}
self.db.rollback()
return ret
####
# *Group utility methods
####
def get_distgroup_attributes_and_targetdata(self, subj_id):
"""Collect DistributionGroup specific information.
:type subj_id: int
:param subj_id: The entity-id of the DistributionGroup.
:rtype: dict
:return: A dict with DistributionGroup-specific data."""
# TODO: This might crash if someone where to add a spread, on a group
# which is not a DistributionGroup in Cerebrum
self.dg.clear()
self.dg.find(subj_id)
rl = True if self.dg.roomlist == 'T' else False
ret = self.dg.get_distgroup_attributes_and_targetdata(roomlist=rl)
self.db.rollback()
return ret
def get_distgroup_displayname(self, subj_id):
"""Get DisplayName for a DistributionGroup.
:type subj_id: int
:param subj_id: The DistributionGroups entity-id.
@rtype: Cerebrum.extlib.db_row.row
@return: Rows as returned by Entity.search_name_with_language()."""
self.dg.clear()
self.dg.find(subj_id)
ret = self.dg.search_name_with_language(
entity_id=subj_id,
name_variant=self.co.dl_group_displ_name,
name_language=self.co.language_nb)
self.db.rollback()
return ret
class CerebrumUtils(object):
"""Utility-class containing often used functions for Exchange."""
def __init__(self):
"""Initialize the Utils."""
self.db = Factory.get('Database')(client_encoding='UTF-8')
self.en = Factory.get('Entity')(self.db)
self.ac = Factory.get('Account')(self.db)
self.pe = Factory.get('Person')(self.db)
self.gr = Factory.get('Group')(self.db)
self.co = Factory.get('Constants')(self.db)
self.ed = EmailDomain(self.db)
self.eq = EmailQuota(self.db)
self.ef = EmailForward(self.db)
self.et = Factory.get('EmailTarget')(self.db)
self.dg = DistributionGroup(self.db)
####
# Person related methods
####
def get_person_accounts(self, person_id, spread=None):
"""Return a list of account information.
:type person_id: int
:param person_id: The person id to look up by.
:rtype: list
:return: A list of (account_id, username) tuples."""
ret = [(x['account_id'], x['name'])
for x in self.ac.search(owner_id=person_id,
owner_type=self.co.entity_person,
spread=self.co.spread_exchange_account)]
self.db.rollback()
return ret
def get_person_names(self, account_id=None, person_id=None):
"""Return a persons names.
:type account_id: int
:param account_id: The user to look up names by.
:type person_id: int
:param person_id: The person to look up names by.
:rtype: tuple
:return: A tuple consisting of first_name, last_name and the full name."""
# TODO: search_name_with_language?
if account_id:
self.ac.clear()
self.ac.find(account_id)
self.pe.clear()
self.pe.find(self.ac.owner_id)
elif person_id:
self.pe.clear()
self.pe.find(person_id)
else:
raise AssertionError('Called w/o person or account id')
ret = (self.pe.get_name(self.co.system_cached, self.co.name_first),
self.pe.get_name(self.co.system_cached, self.co.name_last),
self.pe.get_name(
self.co.system_cached,
self.co.name_full,
))
self.db.rollback()
return ret
def get_person_membership_groupnames(self, person_id):
"""List all the groups the person is a member of.
:type person_id: int
:param person_id: The persons entity_id.
:rtype: list
:return: list(string) of groupnames."""
ret = [
x['group_name']
for x in self.gr.search_members(member_id=person_id,
indirect_members=True)
]
self.db.rollback()
return ret
def is_electronic_reserved(self, person_id=None, account_id=None):
"""Check if a person has reserved themself from listing.
:type person_id: int
:param person_id: The persons entity_id.
:type account_id: int
:param account_id: The accounts entity_id.
:rtype: bool
:return: True if reserved, False otherwise."""
if person_id:
self.pe.clear()
self.pe.find(person_id)
ret = self.pe.has_e_reservation()
elif account_id:
self.ac.clear()
self.ac.find(account_id)
ret = self.ac.owner_has_ereservation()
# TODO: This is sane?
else:
ret = True
self.db.rollback()
return ret
####
# Account related methods
####
def get_account_mailaddrs(self, account_id):
"""Collect mailaddresses of an account.
:type account_id: int
:param account_id: The account_id representing the object.
:rtype: list
:return: A list of addresses."""
self.et.clear()
self.et.find_by_target_entity(account_id)
addrs = [
'%s@%s' % (x['local_part'], x['domain'])
for x in self.et.get_addresses()
]
self.db.rollback()
return addrs
def get_account_forwards(self, account_id):
"""Collect an accounts forward addresses.
:param int account_id: The account_id representing the object.
:return: A list of forward addresses."""
self.ef.clear()
self.ef.find_by_target_entity(account_id)
r = []
for fwd in self.ef.get_forward():
# Need to do keys() for now, db_row is stupid.
if 'enable' in fwd.keys() and fwd['enable'] == 'T':
r.append(fwd['forward_to'])
self.db.rollback()
return r
def get_account_local_delivery(self, account_id):
"""Check if this account should have local delivery.
:param int account_id: The account_id representing the object.
:rtype: bool
:return: Local delivery on or off."""
self.ef.clear()
self.ef.find_by_target_entity(account_id)
return self.ef.local_delivery
def get_account_name(self, account_id):
"""Return information about the account.
:type account_id: int
:param acccount_id: The accounts entity id.
:rtype: string
:return: The accounts name."""
self.ac.clear()
self.ac.find(account_id)
ret = self.ac.account_name
self.db.rollback()
return ret
def get_account_owner_info(self, account_id):
"""Return the type and id of the accounts owner.
:type account_id: int
:param acccount_id: The accounts entity id.
:rtype: tuple
:return: (entity_type, entity_id)."""
self.ac.clear()
self.ac.find(account_id)
ret = (self.ac.owner_type, self.ac.owner_id)
self.db.rollback()
return ret
def get_account_spreads(self, account_id):
"""Return the accounts spread codes.
:type account_id: int
:param acccount_id: The accounts entity id.
:rtype: list
:return: A list of the accounts spread codes."""
self.ac.clear()
self.ac.find(account_id)
ret = [x['spread'] for x in self.ac.get_spread()]
self.db.rollback()
return ret
def get_account_primary_email(self, account_id):
"""Return the accounts primary address.
:type account_id: int
:param acccount_id: The accounts entity id.
:rtype: str
:return: The accounts primary email-address."""
self.ac.clear()
self.ac.find(account_id)
ret = self.ac.get_primary_mailaddress()
self.db.rollback()
return ret
def get_account_id(self, uname):
"""Get an accounts entity id.
:type uname: str
:param uname: The users name.
:rtype: int
:return: The entity id of the account."""
self.ac.clear()
self.ac.find_by_name(uname)
ret = self.ac.entity_id
self.db.rollback()
return ret
def get_primary_account(self, person_id):
"""Get a persons primary account.
:type person_id: int
:param person_id: The persons entity id.
:rtype: int
:return: The primary accounts entity_id or None if no primary."""
self.pe.clear()
self.pe.find(person_id)
ret = self.pe.get_primary_account()
self.db.rollback()
return ret
def get_account_group_memberships(self, uname, group_spread):
"""Get a list of group names, which the user (and users owner, if
person), is a member of.
:type uname: string
:param uname: The accounts name.
:type group_spread: _SpreadCode
:param group_spread: Spread to filter groups by.
:rtype: list(tuple)
:return: A list of tuples contining group-name and -id."""
self.ac.clear()
self.ac.find_by_name(uname)
groups = []
# Fetch the groups the person is a member of
if self.ac.owner_type == self.co.entity_person:
for group in self.gr.search(member_id=self.ac.owner_id,
spread=group_spread,
indirect_members=True):
groups.append((group['name'], group['group_id']))
# Fetch the groups the account is a member of
for group in self.gr.search(member_id=self.ac.entity_id,
spread=group_spread,
indirect_members=True):
groups.append((group['name'], group['group_id']))
self.db.rollback()
return groups
####
# Group related methods
####
def construct_group_names(self, uname, gname):
"""Construct Exchange related group names.
:param str uname: The users username.
:param str gname: The owning groups groupname.
:rtype: tuple(str)
:return: A tuple consisting of FirstName, LastName and DisplayName."""
fn = uname
ln = '(owner: %s)' % gname
dn = '%s (owner: %s)' % (uname, gname)
return (fn, ln, dn)
def get_group_information(self, group_id):
"""Get a groups name and description.
:type group_id: int
:param group_id: The groups entity id.
:rtype: tuple
:return: The groups name and description."""
# This function is kinda generic. We don't care
# (yet) if it's a distribution or security group.
r = self.gr.search(group_id=group_id)[0]
ret = (r['name'], r['description'])
self.db.rollback()
return ret
def get_group_id(self, group_name):
"""Get a groups entity_id.
:type group_name: str
:param group_name: The groups name.
:rtype: int
:return: The groups entity_id."""
self.gr.clear()
self.gr.find_by_name(group_name)
ret = self.gr.entity_id
self.db.rollback()
return ret
def get_group_spreads(self, group_id):
"""Return the groupss spread codes.
:type group_id: int
:param group_id: The accounts entity id.
:rtype: list
:return: A list of the accounts spread codes."""
self.gr.clear()
self.gr.find(group_id)
ret = [x['spread'] for x in self.gr.get_spread()]
self.db.rollback()
return ret
def get_group_members(self, group_id, spread=None, filter_spread=None):
"""Collect a list of the usernames of all users in a group.
:type group_id: int
:param group_id: The groups entity_id.
:type spread: _SpreadCode
:param spread: The spread to filter by.
:type filter_spread: _SpreadCode
:param filter_spread: A spread that the user must also have.
:rtype: list
:return: A list of usernames who are members of group_id."""
found_accounts = []
r = []
for x in self.gr.search_members(group_id=group_id,
indirect_members=True,
member_spread=spread,
member_type=self.co.entity_account):
if x['member_id'] not in found_accounts:
if filter_spread in self.get_account_spreads(x['member_id']):
r.append({
'name': self.get_account_name(x['member_id']),
'account_id': x['member_id']
})
found_accounts.append(x['member_id'])
spreads = []
if spread:
spreads += [int(spread)]
if filter_spread:
spreads += [int(filter_spread)]
for x in self.gr.search_members(group_id=group_id,
indirect_members=True,
member_type=self.co.entity_person):
self.pe.clear()
self.pe.find(x['member_id'])
aid = self.pe.get_primary_account()
if aid:
self.ac.clear()
self.ac.find(aid)
if self.ac.entity_id not in found_accounts:
# If/elif used to allow usage without filter and
# filter_spread params
if spreads and \
set(spreads).issubset(
set([x['spread'] for x in self.ac.get_spread()])):
r.append({
'name': self.ac.account_name,
'account_id': self.ac.entity_id
})
found_accounts.append(self.ac.entity_id)
elif not spreads:
r.append({
'name': self.ac.account_name,
'account_id': self.ac.entity_id
})
found_accounts.append(self.ac.entity_id)
self.db.rollback()
return r
def get_parent_groups(self, id, spread=None, name_prefix=None):
"""Return all groups that the group is an indirect member of. Filter
by spread and the start of the group name.
:type id: int
:param id: The entity_id.
:type spread: _SpreadCode
:param _SpreadCode: The spread code to filter by, default is None.
:type name_prefix: str
:param name_prefix: Check if the group starts with this, default None.
:rtype: list
:return: A list of appropriate groups."""
if name_prefix:
np = '%s%%' % name_prefix
else:
np = None
groups = []
for group in self.gr.search(member_id=id,
indirect_members=True,
spread=spread,
name=np):
groups.append(group['name'])
self.db.rollback()
return groups
####
# Other utility methods
####
def unpickle_event_params(self, event):
"""Unpickle the change params of an event.
:type event: dbrow
:param event: The db row returned by Change- or EventLog.
:rtype: string
:return: The change params."""
# Hopefully, evertyhin will use UTF in the end
try:
return pickle.loads(event['change_params'])
except:
return pickle.loads(event['change_params'].encode('ISO_8859_15'))
def get_entity_type(self, entity_id):
"""Fetch the entity type code of an entity.
:type entity_id: int
:param entity_id: The entity id.
:rtype: long
:return: The entity type code."""
self.en.clear()
self.en.find(entity_id)
ret = self.en.entity_type
self.db.rollback()
return ret
def log_event(self, event, trigger):
"""Utility method used to create an event only in the EventLog.
:type event: dict
:param event: Dict representing an event (as returned from get_event).
:param trigger: str or list or tuple
:param trigger: The change type code we want to associate with the
event. Only the first value will be used."""
if isinstance(trigger, (list, tuple)):
trigger = trigger[0]
trigger = trigger.split(':')
ct = self.co.ChangeType(trigger[0], trigger[1])
param = self.unpickle_event_params(event)
self.db.log_change(event['subject_entity'],
int(ct),
event['dest_entity'],
change_params=param,
skip_change=True,
skip_publish=True)
self.db.commit()
def log_event_receipt(self, event, trigger):
"""Log the "receipt" of a completed event in ChangeLog.
:type event: dict
:param event: Dict representing an event (as returned from get_event).
:param trigger: str or list or tuple
:param trigger: The change type code we want to associate with the
event. Only the first value will be used."""
# TODO: Set change_program from a sensible source to something smart
if isinstance(trigger, (list, tuple)):
trigger = trigger[0]
trigger = trigger.split(':')
ct = self.co.ChangeType(trigger[0], trigger[1])
parm = {
'change_program': 'ExchangeIntegration',
'skip_event': True,
'skip_publish': True
}
# Only log params if they actually contain something.
param = self.unpickle_event_params(event)
if param:
parm['change_params'] = param
self.db.log_change(event['subject_entity'], int(ct),
event['dest_entity'], **parm)
self.db.commit()
####
# Email* utility methods
####
def get_email_target_info(self, target_id=None, target_entity=None):
"""Return the entity_type and entity_id of the entity the EmailTarget
points at. Look up by _either_ target_id or target_entity.
:type target_id: int
:param target_id: The EmailTargets id.
:type target_entity: int
:param target_entity: The targets target entity id.
:rtype: typle
:return: (entity_id, target_entity_id, target_entity_type, hard_quota,
soft_quota)."""
self.et.clear()
self.eq.clear()
if target_id:
self.et.find(target_id)
try:
self.eq.find(target_id)
except Errors.NotFoundError:
self.eq.clear()
elif target_entity:
self.et.find_by_target_entity(target_entity)
try:
self.eq.find(self.et.entity_id)
except Errors.NotFoundError:
self.eq.clear()
else:
self.db.rollback()
raise Errors.ProgrammingError(
'Must define either target_id og target_entity')
ret = (self.et.entity_id, self.et.email_target_entity_id,
self.et.email_target_entity_type, self.eq.get_quota_hard(),
self.eq.get_quota_soft())
self.db.rollback()
return ret
def get_email_domain_info(self,
email_domain_id=None,
email_domain_name=None):
"""Return info about an EmailDomain.
:type email_domain_id: int
:param email_domain_id: The email domains id.
:rtype: dict
:return: {name: EmailDomain name, id: EmailDomain id}."""
self.ed.clear()
if email_domain_id:
self.ed.find(email_domain_id)
elif email_domain_name:
self.ed.find_by_domain(email_domain_name)
ret = {'name': self.ed.email_domain_name, 'id': self.ed.entity_id}
self.db.rollback()
return ret
####
# *Group utility methods
####
def get_distgroup_attributes_and_targetdata(self, subj_id):
"""Collect DistributionGroup specific information.
:type subj_id: int
:param subj_id: The entity-id of the DistributionGroup.
:rtype: dict
:return: A dict with DistributionGroup-specific data."""
# TODO: This might crash if someone where to add a spread, on a group
# which is not a DistributionGroup in Cerebrum
self.dg.clear()
self.dg.find(subj_id)
rl = True if self.dg.roomlist == 'T' else False
ret = self.dg.get_distgroup_attributes_and_targetdata(roomlist=rl)
self.db.rollback()
return ret
def get_distgroup_displayname(self, subj_id):
"""Get DisplayName for a DistributionGroup.
:type subj_id: int
:param subj_id: The DistributionGroups entity-id.
@rtype: Cerebrum.extlib.db_row.row
@return: Rows as returned by Entity.search_name_with_language()."""
self.dg.clear()
self.dg.find(subj_id)
ret = self.dg.search_name_with_language(
entity_id=subj_id,
name_variant=self.co.dl_group_displ_name,
name_language=self.co.language_nb)
self.db.rollback()
return ret
class StateChecker(object):
"""Wrapper class for state-checking functions.
The StateChecker class wraps all the functions we need in order to
verify and report deviances between Cerebrum and Exchange.
"""
# Connect params
LDAP_RETRY_DELAY = 60
LDAP_RETRY_MAX = 5
# Search and result params
LDAP_COM_DELAY = 30
LDAP_COM_MAX = 3
def __init__(self, conf):
"""Initzialize a new instance of out state-checker.
:param logger logger: The logger to use.
:param dict conf: Our StateCheckers configuration.
"""
self.db = Factory.get('Database')(client_encoding='UTF-8')
self.co = Factory.get('Constants')(self.db)
self.ac = Factory.get('Account')(self.db)
self.pe = Factory.get('Person')(self.db)
self.gr = Factory.get('Group')(self.db)
self.et = Factory.get('EmailTarget')(self.db)
self.eq = EmailQuota(self.db)
self.ea = EmailAddress(self.db)
self.ef = EmailForward(self.db)
self.cu = CerebrumUtils()
self.config = conf
self._ldap_page_size = 1000
self._cache_randzone_users = self._populate_randzone_cache(
self.config['randzone_unreserve_group'])
self._cache_accounts = self._populate_account_cache(
self.co.spread_exchange_account)
self._cache_addresses = self._populate_address_cache()
self._cache_local_delivery = self._populate_local_delivery_cache()
self._cache_forwards = self._populate_forward_cache()
self._cache_quotas = self._populate_quota_cache()
self._cache_targets = self._populate_target_cache()
self._cache_names = self._populate_name_cache()
self._cache_group_names = self._populate_group_name_cache()
self._cache_no_reservation = self._populate_no_reservation_cache()
self._cache_primary_accounts = self._populate_primary_account_cache()
def init_ldap(self):
"""Initzialize LDAP connection."""
self.ldap_srv = ldap.ldapobject.ReconnectLDAPObject(
'%s://%s/' % (self.config['ldap_proto'],
self.config['ldap_server']),
retry_max=self.LDAP_RETRY_MAX,
retry_delay=self.LDAP_RETRY_DELAY)
usr = self.config['ldap_user'].split('\\')[1]
self.ldap_srv.bind_s(
self.config['ldap_user'],
read_password(usr, self.config['ldap_server']))
self.ldap_lc = ldap.controls.SimplePagedResultsControl(
True, self._ldap_page_size, '')
def _searcher(self, ou, scope, attrs, ctrls):
""" Perform ldap.search(), but retry in the event of an error.
This wraps the search with error handling, so that the search is
repeated with a delay between attempts.
"""
for attempt in itertools.count(1):
try:
return self.ldap_srv.search_ext(
ou, scope, attrlist=attrs, serverctrls=ctrls)
except ldap.LDAPError as e:
if attempt < self.LDAP_COM_MAX:
logger.debug('Caught %r in _searcher on attempt %d',
e, attempt)
time.sleep(self.LDAP_COM_DELAY)
continue
raise
def _recvr(self, msgid):
""" Perform ldap.result3(), but retry in the event of an error.
This wraps the result fetching with error handling, so that the fetch
is repeated with a delay between attempts.
It also decodes all attributes and attribute text values.
"""
for attempt in itertools.count(1):
try:
# return self.ldap_srv.result3(msgid)
rtype, rdata, rmsgid, sc = self.ldap_srv.result3(msgid)
return rtype, decode_attrs(rdata), rmsgid, sc
except ldap.LDAPError as e:
if attempt < self.LDAP_COM_MAX:
logger.debug('Caught %r in _recvr on attempt %d',
e, attempt)
time.sleep(self.LDAP_COM_DELAY)
continue
raise
def search(self, ou, attrs, scope=ldap.SCOPE_SUBTREE):
"""Wrapper for the search- and result-calls.
Implements paged searching.
:param str ou: The OU to search in.
:param list attrs: The attributes to fetch.
:param int scope: Our search scope, default is subtree.
"""
# Implementing paging, taken from
# http://www.novell.com/coolsolutions/tip/18274.html
msgid = self._searcher(ou, scope, attrs, [self.ldap_lc])
data = []
ctrltype = ldap.controls.SimplePagedResultsControl.controlType
while True:
time.sleep(1)
rtype, rdata, rmsgid, sc = self._recvr(msgid)
data.extend(rdata)
pctrls = [c for c in sc if c.controlType == ctrltype]
if pctrls:
cookie = pctrls[0].cookie
if cookie:
self.ldap_lc.cookie = cookie
time.sleep(1)
msgid = self._searcher(ou, scope, attrs, [self.ldap_lc])
else:
break
else:
logger.warn('Server ignores RFC 2696 control.')
break
return data[1:]
def close(self):
"""Close the LDAP connection."""
self.ldap_srv.unbind_s()
#
# Various cache-generating functions.
#
def _populate_randzone_cache(self, randzone):
self.gr.clear()
self.gr.find_by_name(randzone)
u = text_decoder(self.db.encoding)
return [u(x['name'])
for x in self.cu.get_group_members(self.gr.entity_id)]
def _populate_account_cache(self, spread):
u = text_decoder(self.db.encoding)
def to_dict(row):
d = dict(row)
d['name'] = u(row['name'])
d['description'] = u(row['description'])
return d
return [to_dict(r) for r in self.ac.search(spread=spread)]
def _populate_address_cache(self):
u = text_decoder(self.db.encoding)
tmp = defaultdict(list)
# TODO: Implement fetchall?
for addr in self.ea.list_email_addresses_ext():
tmp[addr['target_id']].append(
u'%s@%s' % (u(addr['local_part']), u(addr['domain'])))
return dict(tmp)
def _populate_local_delivery_cache(self):
r = {}
for ld in self.ef.list_local_delivery():
r[ld['target_id']] = ld['local_delivery']
return r
def _populate_forward_cache(self):
u = text_decoder(self.db.encoding)
tmp = defaultdict(list)
for fwd in self.ef.list_email_forwards():
if fwd['enable'] == 'T':
tmp[fwd['target_id']].append(u(fwd['forward_to']))
return dict(tmp)
def _populate_quota_cache(self):
tmp = defaultdict(dict)
# TODO: Implement fetchall?
for quota in self.eq.list_email_quota_ext():
tmp[quota['target_id']]['soft'] = quota['quota_soft']
tmp[quota['target_id']]['hard'] = quota['quota_hard']
return dict(tmp)
def _populate_target_cache(self):
u = text_decoder(self.db.encoding)
tmp = defaultdict(dict)
for targ in self.et.list_email_target_primary_addresses(
target_type=self.co.email_target_account):
tmp[targ['target_entity_id']]['target_id'] = targ['target_id']
tmp[targ['target_entity_id']]['primary'] = \
u'%s@%s' % (u(targ['local_part']), u(targ['domain']))
return dict(tmp)
def _populate_name_cache(self):
u = text_decoder(self.db.encoding)
tmp = defaultdict(dict)
for name in self.pe.search_person_names(
name_variant=[self.co.name_first,
self.co.name_last,
self.co.name_full],
source_system=self.co.system_cached):
tmp[name['person_id']][name['name_variant']] = u(name['name'])
return dict(tmp)
def _populate_group_name_cache(self):
u = text_decoder(self.db.encoding)
tmp = {}
for eid, dom, name in self.gr.list_names(self.co.group_namespace):
tmp[eid] = u(name)
return tmp
def _populate_no_reservation_cache(self):
unreserved = []
for r in self.pe.list_traits(self.co.trait_public_reservation,
fetchall=True):
if r['numval'] == 0:
unreserved.append(r['entity_id'])
return unreserved
def _populate_primary_account_cache(self):
primary = []
for acc in self.ac.list_accounts_by_type(primary_only=True):
primary.append(acc['account_id'])
return primary
###
# Mailbox related state fetching & comparison
###
def collect_cerebrum_mail_info(self):
"""Collect E-mail related information from Cerebrum.
:rtype: dict
:return: A dict of users attributes. Uname is key.
"""
res = {}
for acc in self._cache_accounts:
tmp = {}
try:
tid = self._cache_targets[acc['account_id']]['target_id']
except KeyError:
logger.warn('Could not find account with id:%d in list '
'of targets, skipping..', acc['account_id'])
continue
# Fetch addresses
tmp[u'EmailAddresses'] = sorted(self._cache_addresses[tid])
# Fetch primary address
tmp[u'PrimaryAddress'] = \
self._cache_targets[acc['account_id']]['primary']
# Fetch names
if acc['owner_type'] == self.co.entity_person:
tmp[u'FirstName'] = \
self._cache_names[acc['owner_id']][int(self.co.name_first)]
tmp[u'LastName'] = \
self._cache_names[acc['owner_id']][int(self.co.name_last)]
tmp[u'DisplayName'] = \
self._cache_names[acc['owner_id']][int(self.co.name_full)]
else:
fn, ln, dn = self.cu.construct_group_names(
acc['name'], self._cache_group_names.get(acc['owner_id'],
None))
tmp[u'FirstName'] = fn
tmp[u'LastName'] = ln
tmp[u'DisplayName'] = dn
# Fetch quotas
hard = self._cache_quotas[tid]['hard'] * 1024
soft = self._cache_quotas[tid]['soft']
tmp[u'ProhibitSendQuota'] = str(hard)
tmp[u'ProhibitSendReceiveQuota'] = str(hard)
tmp[u'IssueWarningQuota'] = str(int(hard * soft / 100.))
# Randzone users will always be shown. This overrides everything
# else.
if acc['name'] in self._cache_randzone_users:
hide = False
elif acc['owner_id'] in self._cache_no_reservation and \
acc['account_id'] in self._cache_primary_accounts:
hide = False
else:
hide = True
tmp[u'HiddenFromAddressListsEnabled'] = hide
# Collect local delivery status
tmp[u'DeliverToMailboxAndForward'] = \
self._cache_local_delivery.get(tid, False)
# Collect forwarding address
# We do this by doing a difference operation on the forwards and
# the addresses, so we only end up with "external" addresses.
s_fwds = set(self._cache_forwards.get(tid, []))
s_addrs = set(self._cache_addresses.get(tid, []))
ext_fwds = list(s_fwds - s_addrs)
if ext_fwds:
tmp[u'ForwardingSmtpAddress'] = ext_fwds[0]
else:
tmp[u'ForwardingSmtpAddress'] = None
res[acc['name']] = tmp
return res
def collect_exchange_mail_info(self, mb_ou):
"""Collect mailbox-information from Exchange, via LDAP.
:param str mb_ou: The OrganizationalUnit to search for mailboxes.
:rtype: dict
:return: A dict with the mailboxes attributes. The key is the account
name.
"""
attrs = ['proxyAddresses',
'displayName',
'givenName',
'sn',
'msExchHideFromAddressLists',
'extensionAttribute1',
'mDBUseDefaults',
'mDBOverQuotaLimit',
'mDBOverHardQuotaLimit',
'mDBStorageQuota',
'deliverAndRedirect',
'msExchGenericForwardingAddress']
r = self.search(mb_ou, attrs)
ret = {}
for cn, data in r:
if 'extensionAttribute1'in data and \
data['extensionAttribute1'] == ['not migrated'] or \
'ExchangeActiveSyncDevices' in cn:
continue
tmp = {}
name = cn[3:].split(',')[0]
for key in data:
if key == 'proxyAddresses':
addrs = []
for addr in data[key]:
if addr.startswith('SMTP:'):
tmp[u'PrimaryAddress'] = addr[5:]
addrs.append(addr[5:])
tmp[u'EmailAddresses'] = sorted(addrs)
elif key == 'displayName':
tmp[u'DisplayName'] = data[key][0]
elif key == 'givenName':
tmp[u'FirstName'] = data[key][0]
elif key == 'sn':
tmp[u'LastName'] = data[key][0]
elif key == 'mDBUseDefaults':
tmp[u'UseDatabaseQuotaDefaults'] = (
True if data[key][0] == 'TRUE' else False)
elif key == 'mDBOverQuotaLimit':
q = data[key][0]
tmp[u'ProhibitSendQuota'] = q
elif key == 'mDBOverHardQuotaLimit':
q = data[key][0]
tmp[u'ProhibitSendReceiveQuota'] = q
elif key == 'mDBStorageQuota':
q = data[key][0]
tmp[u'IssueWarningQuota'] = q
# Non-existent attribute means that the value is false. Fuckers.
# Collect status about if the mbox is hidden or not
tmp[u'HiddenFromAddressListsEnabled'] = False
if 'msExchHideFromAddressLists' in data:
val = (True if data['msExchHideFromAddressLists'][0] == 'TRUE'
else False)
tmp[u'HiddenFromAddressListsEnabled'] = val
# Collect local delivery status
tmp[u'DeliverToMailboxAndForward'] = False
if 'deliverAndRedirect' in data:
val = (True if data['deliverAndRedirect'][0] == 'TRUE'
else False)
tmp[u'DeliverToMailboxAndForward'] = val
# Collect forwarding address
tmp[u'ForwardingSmtpAddress'] = None
if 'msExchGenericForwardingAddress' in data:
val = data['msExchGenericForwardingAddress'][0]
# We split of smtp:, and store
tmp[u'ForwardingSmtpAddress'] = val.split(':')[1]
ret[name] = tmp
return ret
def compare_mailbox_state(self, ex_state, ce_state, state, config):
"""Compare the information fetched from Cerebrum and Exchange.
This method produces a dict with the state between the systems,
and a report that will be sent to the appropriate target system
administrators.
:param dict ex_state: The state in Exchange.
:param dict ce_state: The state in Cerebrum.
:param dict state: The previous state generated by this method.
:param dict config: Configuration of reporting delays for various
attributes.
:rtype: tuple
:return: A tuple consisting of the new difference-state and a
human-readable report of differences.
"""
s_ce_keys = set(ce_state.keys())
s_ex_keys = set(ex_state.keys())
diff_mb = {}
diff_stale = {}
diff_new = {}
##
# Populate some structures with information we need
# Mailboxes in Exchange, but not in Cerebrum
stale_keys = list(s_ex_keys - s_ce_keys)
for ident in stale_keys:
if state and ident in state['stale_mb']:
diff_stale[ident] = state['stale_mb'][ident]
else:
diff_stale[ident] = time.time()
# Mailboxes in Cerebrum, but not in Exchange
new_keys = list(s_ce_keys - s_ex_keys)
for ident in new_keys:
if state and ident in state['new_mb']:
diff_new[ident] = state['new_mb'][ident]
else:
diff_new[ident] = time.time()
# Check mailboxes that exists in both Cerebrum and Exchange for
# difference (& is union, in case you wondered). If an attribute is not
# in it's desired state in both this and the last run, save the
# timestamp from the last run. This is used for calculating when we nag
# to someone about stuff not beeing in sync.
for key in s_ex_keys & s_ce_keys:
for attr in ce_state[key]:
if state and key in state['mb'] and \
attr in state['mb'][key]:
t_0 = state['mb'][key][attr][u'Time']
else:
t_0 = time.time()
diff_mb.setdefault(key, {})
if attr not in ex_state[key]:
diff_mb[key][attr] = {
u'Exchange': None,
u'Cerebrum': ce_state[key][attr],
u'Time': t_0,
}
elif ce_state[key][attr] != ex_state[key][attr]:
# For quotas, we only want to report mismatches if the
# difference is between the quotas in Cerebrum and Exchange
# is greater than 1% on either side. Hope this is an
# appropriate value to use ;)
try:
if u'Quota' in attr:
exq = ex_state[key][attr]
ceq = ce_state[key][attr]
diff = abs(int(exq) - int(ceq))
avg = (int(exq) + int(ceq)) / 2
one_p = avg * 0.01
if avg + diff < avg + one_p and \
avg - diff > avg - one_p:
continue
except TypeError:
pass
diff_mb[key][attr] = {
u'Exchange': ex_state[key][attr],
u'Cerebrum': ce_state[key][attr],
u'Time': t_0,
}
ret = {'new_mb': diff_new, 'stale_mb': diff_stale, 'mb': diff_mb}
if not state:
return ret, []
now = time.time()
# By now, we have three different dicts. Loop trough them and check if
# we should report 'em
report = [u'# User Attribute Since Cerebrum_value:Exchange_value']
# Report attribute mismatches
for key in diff_mb:
for attr in diff_mb[key]:
delta = (config.get(attr) if attr in config else
config.get('UndefinedAttribute'))
if diff_mb[key][attr][u'Time'] < now - delta:
t = time.strftime(u'%d%m%Y-%H:%M', time.localtime(
diff_mb[key][attr][u'Time']))
if attr == u'EmailAddresses':
# We report the difference for email addresses, for
# redability
s_ce_addr = set(diff_mb[key][attr][u'Cerebrum'])
s_ex_addr = set(diff_mb[key][attr][u'Exchange'])
new_addr = list(s_ce_addr - s_ex_addr)
stale_addr = list(s_ex_addr - s_ce_addr)
tmp = u'%-10s %-30s %s +%s:-%s' % (key, attr, t,
str(new_addr),
str(stale_addr))
else:
tmp = (u'%-10s %-30s %s %s:%s' %
(key, attr, t,
repr(diff_mb[key][attr][u'Cerebrum']),
repr(diff_mb[key][attr][u'Exchange']))
)
report += [tmp]
# Report uncreated mailboxes
report += [u'\n# Uncreated mailboxes (uname, time)']
delta = (config.get('UncreatedMailbox') if 'UncreatedMailbox' in config
else config.get('UndefinedAttribute'))
for key in diff_new:
if diff_new[key] < now - delta:
t = time.strftime(u'%d%m%Y-%H:%M', time.localtime(
diff_new[key]))
report += [u'%-10s uncreated_mb %s' % (key, t)]
# Report stale mailboxes
report += [u'\n# Stale mailboxes (uname, time)']
delta = (config.get('StaleMailbox') if 'StaleMailbox' in config else
config.get('UndefinedAttribute'))
for key in diff_stale:
t = time.strftime(u'%d%m%Y-%H:%M', time.localtime(
diff_stale[key]))
if diff_stale[key] < now - delta:
report += [u'%-10s stale_mb %s' % (key, t)]
return ret, report