def process_hostpolicies(stream):
"""Produce a csv list of all hostpolicies."""
logger.info('process_hostpolicies started')
db = Factory.get('Database')()
component = PolicyComponent(db)
by_hosts = {}
for row in component.search_hostpolicies():
by_hosts.setdefault(row['dns_owner_name'], []).append(row)
for dns_owner_name, rows in by_hosts.iteritems():
stream.write(';'.join((dns_owner_name,
','.join(str(row['policy_name']) for row in rows))))
stream.write('\n')
logger.info('process_hostpolicies done')
def process_hostpolicies(stream):
"""Produce a csv list of all hostpolicies."""
logger.info('process_hostpolicies started')
db = Factory.get('Database')()
component = PolicyComponent(db)
by_hosts = {}
for row in component.search_hostpolicies():
by_hosts.setdefault(row['dns_owner_name'], []).append(row)
for dns_owner_name, rows in by_hosts.iteritems():
stream.write(';'.join(
(dns_owner_name, ','.join(str(row['policy_name'])
for row in rows))))
stream.write('\n')
logger.info('process_hostpolicies done')
def host_policy_list(self, operator, dns_owner_id):
"""List all roles/atoms associated to a given host."""
self.ba.assert_dns_superuser(operator.get_entity_id())
host = self._get_host(dns_owner_id)
policy = PolicyComponent(self.db)
ret = []
for row in policy.search_hostpolicies(dns_owner_id=host.entity_id):
policy.clear()
policy.find(row['policy_id'])
ret.append({
'policy_name': row['policy_name'],
'desc': policy.description
})
return sorted(ret, key=lambda r: r['policy_name'])
def host_policy_list(self, operator, dns_owner_id):
"""List all roles/atoms associated to a given host."""
self.ba.assert_dns_superuser(operator.get_entity_id())
host = self._get_host(dns_owner_id)
policy = PolicyComponent(self.db)
ret = []
for row in policy.search_hostpolicies(dns_owner_id=host.entity_id):
policy.clear()
policy.find(row['policy_id'])
ret.append({'policy_name': row['policy_name'],
'desc': policy.description})
return sorted(ret, key=lambda r: r['policy_name'])
def __init__(self, db, logger):
"""Initialize the sync with hostpolicy objects.."""
super(HostpolicySync, self).__init__(db, logger)
self.component = PolicyComponent(self.db)
self.role = Role(self.db)
self.atom = Atom(self.db)
class HostpolicySync(ADSync.GroupSync, TSDUtils):
"""Class for syncing all hostpolicy components to AD.
Note that the spread set for this sync is not used, at least for now, as
we sync *all* hostpolicy components. This could be changed in the future,
if we only need a subset of the components.
The Roles and Atoms are considered as Group objects in AD, since AD does
not have a native way of describing policy components. The group
memberships must be considered the policy relationships. Hosts added as
members of a policy group must be considered to have this policy set.
"""
def __init__(self, db, logger):
"""Initialize the sync with hostpolicy objects.."""
super(HostpolicySync, self).__init__(db, logger)
self.component = PolicyComponent(self.db)
self.role = Role(self.db)
self.atom = Atom(self.db)
def configure(self, config_args):
"""Add Hostpolicy specific configuration."""
super(HostpolicySync, self).configure(config_args)
self.rolepath = ','.join(('OU=roles', self.config['target_ou']))
self.atompath = ','.join(('OU=atoms', self.config['target_ou']))
def fetch_cerebrum_entities(self):
"""Fetch the policycomponents from Cerebrum to be compared with AD.
"""
self.logger.debug("Fetching all hostpolicies")
subset = self.config.get('subset')
for row in self.component.search():
name = row["name"].strip()
if subset and name not in subset:
continue
self.entities[name] = self.cache_entity(int(row["component_id"]),
name, row)
def cache_entity(self, entity_id, entity_name, data):
"""Wrapper method for creating a cache object for an entity.
You should call this method for new cache objects instead of creating
it directly, for easier subclassing.
@type data: dict
@param data: A row object with data about the entity to cache.
"""
# TODO: Change this to rather be using config['object_classes']:
ent = CerebrumGroup(self.logger, self.config, entity_id, entity_name,
data['description'])
# Feed the entity with the given data:
for key in ('entity_type', 'created_at', 'foundation',
'foundation_date'):
setattr(ent, key, data[key])
if data['entity_type'] == self.co.entity_hostpolicy_atom:
ent.ou = ','.join(('OU=atoms', self.config['target_ou']))
elif data['entity_type'] == self.co.entity_hostpolicy_role:
ent.ou = ','.join(('OU=roles', self.config['target_ou']))
else:
self.logger.warn("Unknown entity type for %s: %s", entity_id,
data['entity_type'])
return ent
def fetch_cerebrum_data(self):
"""Extend with fetching extra hostpolicy data, like members.
We simulate the hostpolicy relations through group memberships. The
relation from source component to target component is translated to the
source being a member of target. You would instinctively think that it
should be the other way around, but in this way it is easier in the AD
environment to fetch indirect relationships, as indirect group
memberships are easier to fetch than indirect parents.
We only sync the related hosts that are affiliated with a project. This
is by design.
"""
super(HostpolicySync, self).fetch_cerebrum_data()
# Simulate the host sync to get the correct member data (OU) for hosts:
self.logger.debug2("Simulating host sync")
host_sync = self.get_class(sync_type='hosts')(self.db, self.logger)
host_config = adconf.SYNCS['hosts'].copy()
host_config['attributes'] = {}
host_config['sync_type'] = 'hosts'
host_sync.configure(host_config)
host_sync.fetch_cerebrum_data()
host_sync.calculate_ad_values()
self.logger.debug2("Host sync simulation done, found %d hosts",
len(host_sync.id2entity))
# Fetch the memberships per component. This is a rather slow process,
# but we know that we are not getting that many different hostpolicies
# and we could live with this running for a few minutes.
i = 0
for ent in self.entities.itervalues():
# The list of members is fed with the full DN of each object, as
# registered in Cerebrum. It _should_ be the same in AD, otherwise
# the update of a given object would fail until the path is in
# sync.
members = set()
for row in self.component.search_relations(
target_id=ent.entity_id,
relationship_code=self.co.hostpolicy_contains,
indirect_relations=False):
mem = self.id2entity.get(row['source_id'])
if mem:
members.add('CN=%s,%s' % (mem.ad_id, mem.ou))
else:
self.logger.warn("Unknown member component: %s",
row['source_id'])
# Get host members of the component:
# TODO: We might want to run the hosts' AD-sync class to fetch all
# needed data, but this works for now.
for row in self.component.search_hostpolicies(
policy_id=ent.entity_id):
mem = host_sync.owner2entity.get(row['dns_owner_id'])
if mem:
members.add('CN=%s,%s' % (mem.ad_id, mem.ou))
ent.set_attribute('Member', members,
self.config['attributes'].get('Member'))
i += len(members)
self.logger.debug2("Found in total %d hostpolicy members", i)
def terminate(self):
"""Remove all of a project, except its project ID and name (acronym).
The project's entities are deleted by this method, so use with care!
For the OU object, it does almost the same as L{delete} except from
deleting the entity itself.
"""
self.write_db()
ent = EntityTrait(self._db)
ac = Factory.get('Account')(self._db)
pu = Factory.get('PosixUser')(self._db)
# Delete PosixUsers
for row in ac.list_accounts_by_type(ou_id=self.entity_id,
filter_expired=False):
try:
pu.clear()
pu.find(row['account_id'])
pu.delete_posixuser()
except Errors.NotFoundError:
# not a PosixUser
continue
# Remove all project's groups
gr = Factory.get('Group')(self._db)
for row in gr.list_traits(code=self.const.trait_project_group,
target_id=self.entity_id):
gr.clear()
gr.find(row['entity_id'])
gr.delete()
# Delete all users
for row in ac.list_accounts_by_type(ou_id=self.entity_id):
ac.clear()
ac.find(row['account_id'])
ac.delete()
# Remove every trace of person affiliations to the project:
pe = Factory.get('Person')(self._db)
for row in pe.list_affiliations(ou_id=self.entity_id,
include_deleted=True):
pe.clear()
pe.find(row['person_id'])
pe.nuke_affiliation(ou_id=row['ou_id'],
affiliation=row['affiliation'],
source=row['source_system'],
status=row['status'])
pe.write_db()
# Remove all project's DnsOwners (hosts):
dnsowner = dns.DnsOwner.DnsOwner(self._db)
policy = PolicyComponent(self._db)
update_helper = dns.IntegrityHelper.Updater(self._db)
for row in ent.list_traits(code=self.const.trait_project_host,
target_id=self.entity_id):
# TODO: Could we instead update the Subnet classes to use
# Factory.get('Entity'), and make use of EntityTrait there to
# handle this?
owner_id = row['entity_id']
ent.clear()
ent.find(owner_id)
ent.delete_trait(row['code'])
ent.write_db()
# Remove the links to policies if hostpolicy is used
for prow in policy.search_hostpolicies(dns_owner_id=owner_id):
policy.clear()
policy.find(prow['policy_id'])
policy.remove_from_host(owner_id)
# delete the DNS owner
update_helper.full_remove_dns_owner(owner_id)
# Delete all subnets
subnet = dns.Subnet.Subnet(self._db)
subnet6 = dns.IPv6Subnet.IPv6Subnet(self._db)
for row in ent.list_traits(code=(self.const.trait_project_subnet6,
self.const.trait_project_subnet),
target_id=self.entity_id):
ent.clear()
ent.find(row['entity_id'])
ent.delete_trait(row['code'])
ent.write_db()
if row['code'] == self.const.trait_project_subnet:
subnet.clear()
subnet.find(row['entity_id'])
subnet.delete()
if row['code'] == self.const.trait_project_subnet6:
subnet6.clear()
subnet6.find(row['entity_id'])
subnet6.delete()
# Remove all data from the OU except for:
# The project ID and project name
for tr in tuple(self.get_traits()):
self.delete_trait(tr)
for row in self.get_spread():
self.delete_spread(row['spread'])
for row in self.get_contact_info():
self.delete_contact_info(row['source_system'],
row['contact_type'])
for row in self.get_entity_address():
self.delete_entity_address(row['source_system'],
row['address_type'])
for row in self.search_name_with_language(entity_id=self.entity_id):
# The project name must not be removed, to avoid reuse
if row['name_variant'] == self.const.ou_name_acronym:
continue
self.delete_name_with_language(row['name_variant'])
self.write_db()
def __init__(self, db, logger):
"""Initialize the sync with hostpolicy objects.."""
super(HostpolicySync, self).__init__(db, logger)
self.component = PolicyComponent(self.db)
self.role = Role(self.db)
self.atom = Atom(self.db)
class HostpolicySync(ADSync.GroupSync, TSDUtils):
"""Class for syncing all hostpolicy components to AD.
Note that the spread set for this sync is not used, at least for now, as
we sync *all* hostpolicy components. This could be changed in the future,
if we only need a subset of the components.
The Roles and Atoms are considered as Group objects in AD, since AD does
not have a native way of describing policy components. The group
memberships must be considered the policy relationships. Hosts added as
members of a policy group must be considered to have this policy set.
"""
def __init__(self, db, logger):
"""Initialize the sync with hostpolicy objects.."""
super(HostpolicySync, self).__init__(db, logger)
self.component = PolicyComponent(self.db)
self.role = Role(self.db)
self.atom = Atom(self.db)
def configure(self, config_args):
"""Add Hostpolicy specific configuration."""
super(HostpolicySync, self).configure(config_args)
self.rolepath = ','.join(('OU=roles', self.config['target_ou']))
self.atompath = ','.join(('OU=atoms', self.config['target_ou']))
def fetch_cerebrum_entities(self):
"""Fetch the policycomponents from Cerebrum to be compared with AD.
"""
self.logger.debug("Fetching all hostpolicies")
subset = self.config.get('subset')
for row in self.component.search():
name = row["name"].strip()
if subset and name not in subset:
continue
self.entities[name] = self.cache_entity(int(row["component_id"]),
name, row)
def cache_entity(self, entity_id, entity_name, data):
"""Wrapper method for creating a cache object for an entity.
You should call this method for new cache objects instead of creating
it directly, for easier subclassing.
@type data: dict
@param data: A row object with data about the entity to cache.
"""
# TODO: Change this to rather be using config['object_classes']:
ent = CerebrumGroup(self.logger, self.config, entity_id, entity_name,
data['description'])
# Feed the entity with the given data:
for key in ('entity_type', 'created_at', 'foundation',
'foundation_date'):
setattr(ent, key, data[key])
if data['entity_type'] == self.co.entity_hostpolicy_atom:
ent.ou = ','.join(('OU=atoms', self.config['target_ou']))
elif data['entity_type'] == self.co.entity_hostpolicy_role:
ent.ou = ','.join(('OU=roles', self.config['target_ou']))
else:
self.logger.warn("Unknown entity type for %s: %s", entity_id,
data['entity_type'])
return ent
def fetch_cerebrum_data(self):
"""Extend with fetching extra hostpolicy data, like members.
We simulate the hostpolicy relations through group memberships. The
relation from source component to target component is translated to the
source being a member of target. You would instinctively think that it
should be the other way around, but in this way it is easier in the AD
environment to fetch indirect relationships, as indirect group
memberships are easier to fetch than indirect parents.
We only sync the related hosts that are affiliated with a project. This
is by design.
"""
super(HostpolicySync, self).fetch_cerebrum_data()
# Simulate the host sync to get the correct member data (OU) for hosts:
self.logger.debug2("Simulating host sync")
host_sync = self.get_class(sync_type='hosts')(self.db, self.logger)
host_config = adconf.SYNCS['hosts'].copy()
host_config['attributes'] = {}
host_config['sync_type'] = 'hosts'
host_sync.configure(host_config)
host_sync.fetch_cerebrum_data()
host_sync.calculate_ad_values()
self.logger.debug2("Host sync simulation done, found %d hosts",
len(host_sync.id2entity))
# Fetch the memberships per component. This is a rather slow process,
# but we know that we are not getting that many different hostpolicies
# and we could live with this running for a few minutes.
i = 0
for ent in self.entities.itervalues():
# The list of members is fed with the full DN of each object, as
# registered in Cerebrum. It _should_ be the same in AD, otherwise
# the update of a given object would fail until the path is in
# sync.
members = set()
for row in self.component.search_relations(
target_id=ent.entity_id,
relationship_code=self.co.hostpolicy_contains,
indirect_relations=False):
mem = self.id2entity.get(row['source_id'])
if mem:
members.add('CN=%s,%s' % (mem.ad_id, mem.ou))
else:
self.logger.warn("Unknown member component: %s",
row['source_id'])
# Get host members of the component:
# TODO: We might want to run the hosts' AD-sync class to fetch all
# needed data, but this works for now.
for row in self.component.search_hostpolicies(
policy_id=ent.entity_id):
mem = host_sync.owner2entity.get(row['dns_owner_id'])
if mem:
members.add('CN=%s,%s' % (mem.ad_id, mem.ou))
ent.set_attribute('Member', members,
self.config['attributes'].get('Member'))
i += len(members)
self.logger.debug2("Found in total %d hostpolicy members", i)
def terminate(self):
"""Remove all of a project, except its project ID and name (acronym).
The project's entities are deleted by this method, so use with care!
For the OU object, it does almost the same as L{delete} except from
deleting the entity itself.
"""
self.write_db()
ent = EntityTrait(self._db)
ac = Factory.get('Account')(self._db)
pu = Factory.get('PosixUser')(self._db)
# Delete PosixUsers
for row in ac.list_accounts_by_type(ou_id=self.entity_id,
filter_expired=False):
try:
pu.clear()
pu.find(row['account_id'])
pu.delete_posixuser()
except Errors.NotFoundError:
# not a PosixUser
continue
# Remove all project's groups
gr = Factory.get('Group')(self._db)
for row in gr.list_traits(code=self.const.trait_project_group,
target_id=self.entity_id):
gr.clear()
gr.find(row['entity_id'])
gr.delete()
# Delete all users
for row in ac.list_accounts_by_type(ou_id=self.entity_id):
ac.clear()
ac.find(row['account_id'])
ac.delete()
# Remove every trace of person affiliations to the project:
pe = Factory.get('Person')(self._db)
for row in pe.list_affiliations(ou_id=self.entity_id,
include_deleted=True):
pe.clear()
pe.find(row['person_id'])
pe.nuke_affiliation(ou_id=row['ou_id'],
affiliation=row['affiliation'],
source=row['source_system'],
status=row['status'])
pe.write_db()
# Remove all project's DnsOwners (hosts):
dnsowner = DnsOwner.DnsOwner(self._db)
policy = PolicyComponent(self._db)
update_helper = IntegrityHelper.Updater(self._db)
for row in ent.list_traits(code=self.const.trait_project_host,
target_id=self.entity_id):
# TODO: Could we instead update the Subnet classes to use
# Factory.get('Entity'), and make use of EntityTrait there to
# handle this?
owner_id = row['entity_id']
ent.clear()
ent.find(owner_id)
ent.delete_trait(row['code'])
ent.write_db()
# Remove the links to policies if hostpolicy is used
for prow in policy.search_hostpolicies(dns_owner_id=owner_id):
policy.clear()
policy.find(prow['policy_id'])
policy.remove_from_host(owner_id)
# delete the DNS owner
update_helper.full_remove_dns_owner(owner_id)
# Delete all subnets
subnet = Subnet.Subnet(self._db)
subnet6 = IPv6Subnet.IPv6Subnet(self._db)
for row in ent.list_traits(code=(self.const.trait_project_subnet6,
self.const.trait_project_subnet),
target_id=self.entity_id):
ent.clear()
ent.find(row['entity_id'])
ent.delete_trait(row['code'])
ent.write_db()
if row['code'] == self.const.trait_project_subnet:
subnet.clear()
subnet.find(row['entity_id'])
subnet.delete()
if row['code'] == self.const.trait_project_subnet6:
subnet6.clear()
subnet6.find(row['entity_id'])
subnet6.delete()
# Remove all data from the OU except for:
# The project ID and project name
for tr in tuple(self.get_traits()):
self.delete_trait(tr)
for row in self.get_spread():
self.delete_spread(row['spread'])
for row in self.get_contact_info():
self.delete_contact_info(row['source_system'],
row['contact_type'])
for row in self.get_entity_address():
self.delete_entity_address(row['source_system'],
row['address_type'])
for row in self.search_name_with_language(entity_id=self.entity_id):
# The project name must not be removed, to avoid reuse
if row['name_variant'] == self.const.ou_name_acronym:
continue
self.delete_name_with_language(row['name_variant'])
self.write_db()