def get_attribute_value(cls, attributes, attribute_name, first_only=False):
"""
The attribute value type must be decodable (str in py2, bytes in py3)
:type attributes: dict
:type attribute_name: unicode
:type first_only: bool
"""
attribute_values = attributes.get(attribute_name)
if attribute_values:
try:
if first_only or len(attribute_values) == 1:
return attribute_values[0].decode(cls.encoding)
else:
return [
val.decode(cls.encoding) for val in attribute_values
]
except UnicodeError as e:
raise AssertionException(
"Encoding error in value of attribute '%s': %s" %
(attribute_name, e))
return None
def iter_search_result(self, filter_string, attributes):
"""
type: filter_string: str
type: attributes: list(str)
"""
attr_dict = OKTAValueFormatter.get_extended_attribute_dict(attributes)
try:
self.logger.info(
"Calling okta SDK get_users with the following %s",
filter_string)
if attr_dict:
users = self.users_client.get_all_users(
query=filter_string, extended_attribute=attr_dict)
else:
users = self.users_client.get_all_users(query=filter_string)
except OktaError as e:
self.logger.warning("Unable to query users")
raise AssertionException("Okta error querying for users: %s" % e)
return users
def run(self, post_sync_data):
"""
Run the Sign sync
:param post_sync_data:
:return:
"""
if self.test_mode:
self.logger.info("Sign Sync disabled in test mode")
return
for org_name, sign_client in self.clients.items():
# create any new Sign groups
for new_group in set(self.user_groups[org_name]) - set(
sign_client.sign_groups()):
self.logger.info(
"Creating new Sign group: {}".format(new_group))
sign_client.create_group(new_group)
umapi_users = post_sync_data.umapi_data.get(org_name)
if umapi_users is None:
raise AssertionException(
"Error getting umapi users from post_sync_data")
self.update_sign_users(umapi_users, sign_client, org_name)
def get_attribute_value(cls, attributes, attribute_name, first_only=False):
"""
The attribute value type must be decodable (str in py2, bytes in py3)
:type attributes: dict
:type attribute_name: unicode
:type first_only: bool
"""
attribute_values = attributes.get(attribute_name)
if attribute_values:
try:
if isinstance(attribute_values, six.string_types):
return attribute_values
else:
if first_only:
return attribute_values[0]
return attribute_values
except UnicodeError as e:
raise AssertionException(
"Encoding error in value of attribute '%s': %s" %
(attribute_name, e))
return None
def iter_group_members(self, group, filter_string, extended_attributes):
"""
:type group: str
:type filter_string: str
:type extended_attributes: list
:rtype iterator(str, str)
"""
user_attribute_names = [
"firstName", "lastName", "login", "email", "countryCode"
]
extended_attributes = list(
set(extended_attributes) - set(user_attribute_names))
user_attribute_names.extend(extended_attributes)
res_group = self.find_group(group)
if res_group:
try:
attr_dict = OKTAValueFormatter.get_extended_attribute_dict(
user_attribute_names)
members = self.groups_client.get_group_all_users(
res_group.id, attr_dict)
except OktaError as e:
self.logger.warning("Unable to get_group_users")
raise AssertionException(
"Okta error querying for group users: %s" % e)
# Filtering users based all_users_filter query in config
for member in self.filter_users(members, filter_string):
profile = member.profile
if not profile.email:
self.logger.warning('No email attribute for login: %s',
profile.login)
continue
user = self.convert_user(member, extended_attributes)
if not user:
continue
yield (user)
else:
self.logger.warning("No group found for: %s", group)
def __init__(self, caller_options):
'''
:type caller_options: dict
'''
self.options = options = {
# these are in alphabetical order! Always add new ones that way!
'delete_strays': False,
'config_file_encoding': 'ascii',
'directory_connector_module_name': None,
'directory_connector_overridden_options': None,
'directory_group_filter': None,
'directory_group_mapped': False,
'disentitle_strays': False,
'exclude_strays': False,
'main_config_filename': DEFAULT_MAIN_CONFIG_FILENAME,
'manage_groups': False,
'remove_strays': False,
'stray_list_input_path': None,
'stray_list_output_path': None,
'test_mode': False,
'update_user_info': True,
'username_filter_regex': None,
}
options.update(caller_options)
main_config_filename = options.get('main_config_filename')
config_encoding = options['config_file_encoding']
try:
codecs.lookup(config_encoding)
except LookupError:
raise AssertionException(
"Unknown encoding '%s' specified with --config-file-encoding" %
config_encoding)
ConfigFileLoader.config_encoding = config_encoding
main_config_content = ConfigFileLoader.load_root_config(
main_config_filename)
self.logger = logger = logging.getLogger('config')
logger.info("Using main config file: %s", main_config_filename)
self.main_config = DictConfig("<%s>" % main_config_filename,
main_config_content)
def get_options(caller_config):
builder = user_sync.config.OptionsBuilder(caller_config)
builder.set_string_value('group_filter_format', six.text_type(
'(&(|(objectCategory=group)(objectClass=groupOfNames)(objectClass=posixGroup))(cn={group}))'))
builder.set_string_value('all_users_filter', six.text_type(
'(&(objectClass=user)(objectCategory=person)(!(userAccountControl:1.2.840.113556.1.4.803:=2)))'))
builder.set_string_value('group_member_filter_format', None)
builder.set_bool_value('require_tls_cert', False)
builder.set_dict_value('two_steps_lookup', None)
builder.set_string_value('string_encoding', 'utf8')
builder.set_string_value('user_identity_type_format', None)
builder.set_string_value('user_email_format', six.text_type('{mail}'))
builder.set_string_value('user_username_format', None)
builder.set_string_value('user_domain_format', None)
builder.set_string_value('user_given_name_format', six.text_type('{givenName}'))
builder.set_string_value('user_surname_format', six.text_type('{sn}'))
builder.set_string_value('user_country_code_format', six.text_type('{c}'))
builder.set_string_value('user_identity_type', None)
builder.set_int_value('search_page_size', 200)
builder.set_string_value('logger_name', LDAPDirectoryConnector.name)
builder.require_string_value('host')
builder.require_string_value('username')
builder.require_string_value('base_dn')
options = builder.get_options()
options['two_steps_enabled'] = False
if options['two_steps_lookup'] is not None:
ts_config = caller_config.get_dict_config('two_steps_lookup', True)
ts_builder = user_sync.config.OptionsBuilder(ts_config)
ts_builder.require_string_value('group_member_attribute_name')
ts_builder.set_bool_value('nested_group', False)
options['two_steps_enabled'] = True
options['two_steps_lookup'] = ts_builder.get_options()
if options['group_member_filter_format']:
raise AssertionException(
"Cannot define both 'group_member_attribute_name' and 'group_member_filter_format' in config")
else:
if not options['group_member_filter_format']:
options['group_member_filter_format'] = six.text_type('(memberOf={group_dn})')
return options
def load_users_and_groups(self, groups, extended_attributes, all_users):
"""
:type groups: list(str)
:type extended_attributes: list(str)
:type all_users: bool
:rtype (bool, iterable(dict))
"""
if all_users:
raise AssertionException(
"Okta connector has no notion of all users, please specify a --users group"
)
options = self.options
all_users_filter = options['all_users_filter']
self.logger.info('Loading users...')
self.user_by_uid = user_by_uid = {}
for group in groups:
total_group_members = 0
total_group_users = 0
for user in self.iter_group_members(group, all_users_filter,
extended_attributes):
total_group_members += 1
uid = user.get('uid')
if user and uid:
if uid not in user_by_uid:
user_by_uid[uid] = user
total_group_users += 1
user_groups = user_by_uid[uid]['groups']
if group not in user_groups:
user_groups.append(group)
self.logger.debug('Group %s members: %d users: %d', group,
total_group_members, total_group_users)
return six.itervalues(user_by_uid)
def add_user(self, attributes):
'''
:type attributes: dict
'''
if self.identity_type == user_sync.identity_type.ADOBEID_IDENTITY_TYPE:
email = self.email if self.email else self.username
if not email:
errorMessage = "ERROR: you must specify an email with an Adobe ID"
raise AssertionException(errorMessage)
params = self.convert_user_attributes_to_params({'email': email})
else:
params = self.convert_user_attributes_to_params(attributes)
onConflictValue = None
option = params.pop('option', None)
if (option == "updateIfAlreadyExists"):
onConflictValue = umapi_client.IfAlreadyExistsOptions.updateIfAlreadyExists
elif (option == "ignoreIfAlreadyExists"):
onConflictValue = umapi_client.IfAlreadyExistsOptions.ignoreIfAlreadyExists
if (onConflictValue != None):
params['on_conflict'] = onConflictValue
self.do_list.append(('create', params))
def _admin_role_mapping(sync_config):
admin_roles = sync_config.get_list('admin_roles', True)
if admin_roles is None:
return {}
mapped_admin_roles = {}
for mapping in admin_roles:
sign_role = mapping.get('sign_role')
if sign_role is None:
raise AssertionException(
"must define a Sign role in admin role mapping")
adobe_groups = mapping.get('adobe_groups')
if adobe_groups is None or not len(adobe_groups):
continue
for g in adobe_groups:
group = AdobeGroup.create(g)
group_name = group.group_name.lower()
if group.umapi_name not in mapped_admin_roles:
mapped_admin_roles[group.umapi_name] = {}
if group_name not in mapped_admin_roles[group.umapi_name]:
mapped_admin_roles[group.umapi_name][group_name] = set()
mapped_admin_roles[group.umapi_name][group_name].add(sign_role)
return mapped_admin_roles
def open_csv_file(name, mode, encoding=None):
"""
:type name: str
:type mode: str
:type encoding: str, but ignored in py2
:rtype file
"""
try:
if mode == 'r':
if is_py2():
return open(str(name), 'rb', buffering=1)
else:
kwargs = dict(buffering=1, newline='', encoding=encoding)
return open(str(name), 'r', **kwargs)
elif mode == 'w':
if is_py2():
return open(str(name), 'wb')
else:
kwargs = dict(newline='')
return open(str(name), 'w', **kwargs)
else:
raise ValueError("File mode (%s) must be 'r' or 'w'" % mode)
except IOError as e:
raise AssertionException("Can't open file '%s': %s" % (name, e))
def begin_work(config_loader):
'''
:type config_loader: user_sync.config.ConfigLoader
'''
directory_groups = config_loader.get_directory_groups()
primary_umapi_config, secondary_umapi_configs = config_loader.get_umapi_options(
)
rule_config = config_loader.get_rule_options()
# process mapped configuration after the directory groups have been loaded, as mapped setting depends on this.
if (rule_config['directory_group_mapped']):
rule_config['directory_group_filter'] = set(
directory_groups.iterkeys())
# make sure that all the adobe groups are from known umapi connector names
referenced_umapi_names = set()
for groups in directory_groups.itervalues():
for group in groups:
umapi_name = group.umapi_name
if (umapi_name != user_sync.rules.PRIMARY_UMAPI_NAME):
referenced_umapi_names.add(umapi_name)
referenced_umapi_names.difference_update(
secondary_umapi_configs.iterkeys())
if (len(referenced_umapi_names) > 0):
raise AssertionException(
'Adobe groups reference unknown umapi connectors: %s' %
referenced_umapi_names)
directory_connector = None
directory_connector_options = None
directory_connector_module_name = config_loader.get_directory_connector_module_name(
)
if (directory_connector_module_name != None):
directory_connector_module = __import__(
directory_connector_module_name, fromlist=[''])
directory_connector = user_sync.connector.directory.DirectoryConnector(
directory_connector_module)
directory_connector_options = config_loader.get_directory_connector_options(
directory_connector.name)
config_loader.check_unused_config_keys()
if (directory_connector != None and directory_connector_options != None):
# specify the default user_identity_type if it's not already specified in the options
if 'user_identity_type' not in directory_connector_options:
directory_connector_options['user_identity_type'] = rule_config[
'new_account_type']
directory_connector.initialize(directory_connector_options)
primary_name = '.primary' if secondary_umapi_configs else ''
umapi_primary_connector = user_sync.connector.umapi.UmapiConnector(
primary_name, primary_umapi_config)
umapi_other_connectors = {}
for secondary_umapi_name, secondary_config in secondary_umapi_configs.iteritems(
):
umapi_secondary_conector = user_sync.connector.umapi.UmapiConnector(
".secondary.%s" % secondary_umapi_name, secondary_config)
umapi_other_connectors[secondary_umapi_name] = umapi_secondary_conector
umapi_connectors = user_sync.rules.UmapiConnectors(umapi_primary_connector,
umapi_other_connectors)
rule_processor = user_sync.rules.RuleProcessor(rule_config)
if (len(directory_groups) == 0 and rule_processor.will_manage_groups()):
logger.warn('no groups mapped in config file')
rule_processor.run(directory_groups, directory_connector, umapi_connectors)
def get_rule_options(self):
'''
Return a dict representing options for RuleProcessor.
'''
# process directory configuration options
new_account_type = None
default_country_code = None
directory_config = self.main_config.get_dict_config(
'directory_users', True)
if directory_config:
new_account_type = directory_config.get_string(
'user_identity_type', True)
new_account_type = user_sync.identity_type.parse_identity_type(
new_account_type)
default_country_code = directory_config.get_string(
'default_country_code', True)
if not new_account_type:
new_account_type = user_sync.identity_type.ENTERPRISE_IDENTITY_TYPE
self.logger.debug("Using default for new_account_type: %s",
new_account_type)
# process exclusion configuration options
exclude_identity_types = exclude_identity_type_names = []
exclude_users = exclude_users_regexps = []
exclude_groups = exclude_group_names = []
adobe_config = self.main_config.get_dict_config('adobe_users', True)
if adobe_config:
exclude_identity_type_names = adobe_config.get_list(
'exclude_identity_types', True) or []
exclude_users_regexps = adobe_config.get_list(
'exclude_users', True) or []
exclude_group_names = adobe_config.get_list(
'exclude_adobe_groups', True) or []
for name in exclude_identity_type_names:
message_format = 'Illegal value in exclude_identity_types: %s'
identity_type = user_sync.identity_type.parse_identity_type(
name, message_format)
exclude_identity_types.append(identity_type)
for regexp in exclude_users_regexps:
try:
# add "match begin" and "match end" markers to ensure complete match
# and compile the patterns because we will use them over and over
exclude_users.append(
re.compile(r'\A' + regexp + r'\Z', re.UNICODE))
except re.error as e:
validation_message = (
'Illegal regular expression (%s) in %s: %s' %
(regexp, 'exclude_identity_types', e))
raise AssertionException(validation_message)
for name in exclude_group_names:
group = user_sync.rules.AdobeGroup.create(name)
if not group or group.get_umapi_name(
) != user_sync.rules.PRIMARY_UMAPI_NAME:
validation_message = 'Illegal value for %s in config file: %s' % (
'exclude_groups', name)
if not group:
validation_message += ' (Not a legal group name)'
else:
validation_message += ' (Can only exclude groups in primary organization)'
raise AssertionException(validation_message)
exclude_groups.append(group.get_group_name())
# get the limits
limits_config = self.main_config.get_dict_config('limits')
max_adobe_only_users = limits_config.get_int('max_adobe_only_users')
# now get the directory extension, if any
after_mapping_hook = None
extended_attributes = None
extension_config = self.get_directory_extension_options()
if extension_config:
after_mapping_hook_text = extension_config.get_string(
'after_mapping_hook')
after_mapping_hook = compile(after_mapping_hook_text,
'<per-user after-mapping-hook>',
'exec')
extended_attributes = extension_config.get_list(
'extended_attributes')
# declaration of extended adobe groups: this is needed for two reasons:
# 1. it allows validation of group names, and matching them to adobe groups
# 2. it allows removal of adobe groups not assigned by the hook
for extended_adobe_group in extension_config.get_list(
'extended_adobe_groups'):
group = user_sync.rules.AdobeGroup.create(extended_adobe_group)
if group is None:
message = 'Extension contains illegal extended_adobe_group spec: ' + str(
extended_adobe_group)
raise AssertionException(message)
options = self.options
result = {
# these are in alphabetical order! Always add new ones that way!
'after_mapping_hook': after_mapping_hook,
'default_country_code': default_country_code,
'delete_strays': options['delete_strays'],
'directory_group_filter': options['directory_group_filter'],
'directory_group_mapped': options['directory_group_mapped'],
'disentitle_strays': options['disentitle_strays'],
'exclude_groups': exclude_groups,
'exclude_identity_types': exclude_identity_types,
'exclude_strays': options['exclude_strays'],
'exclude_users': exclude_users,
'extended_attributes': extended_attributes,
'manage_groups': options['manage_groups'],
'max_adobe_only_users': max_adobe_only_users,
'new_account_type': new_account_type,
'remove_strays': options['remove_strays'],
'stray_list_input_path': options['stray_list_input_path'],
'stray_list_output_path': options['stray_list_output_path'],
'test_mode': options['test_mode'],
'update_user_info': options['update_user_info'],
'username_filter_regex': options['username_filter_regex'],
}
return result
def __init__(self, caller_options):
caller_config = user_sync.config.DictConfig(
'%s configuration' % self.name, caller_options)
options = self.get_options(caller_config)
self.options = options
self.logger = logger = user_sync.connector.helper.create_logger(
options)
logger.debug('%s initialized with options: %s', self.name, options)
LDAPValueFormatter.encoding = options['string_encoding']
self.user_identity_type = user_sync.identity_type.parse_identity_type(
options['user_identity_type'])
self.user_identity_type_formatter = LDAPValueFormatter(
options['user_identity_type_format'])
self.user_email_formatter = LDAPValueFormatter(
options['user_email_format'])
self.user_username_formatter = LDAPValueFormatter(
options['user_username_format'])
self.user_domain_formatter = LDAPValueFormatter(
options['user_domain_format'])
self.user_given_name_formatter = LDAPValueFormatter(
options['user_given_name_format'])
self.user_surname_formatter = LDAPValueFormatter(
options['user_surname_format'])
self.user_country_code_formatter = LDAPValueFormatter(
options['user_country_code_format'])
auth_method = options['authentication_method'].lower()
auth_cred_required = ['simple', 'ntlm']
if options['username'] is not None:
if auth_method in auth_cred_required:
password = caller_config.get_credential(
'password', options['username'])
else:
# Ignore specified credential if authentication method is either 'Kerberos' or 'Anonymous'
raise AssertionException(
"'username' and 'password' are not allowed when 'authentication_method' is '%s"
% auth_method)
else:
# override authentication method to anonymous if username is not specified
if auth_method != 'anonymous' and auth_method != 'kerberos':
auth_method = 'anonymous'
logger.info(
"Username not specified, overriding authentication method to 'anonymous'"
)
# this check must come after we get the password value
caller_config.report_unused_values(logger)
if auth_method != 'kerberos':
from ldap3 import Connection
if auth_method == 'anonymous':
auth = {'authentication': ldap3.ANONYMOUS}
logger.debug(
'Connecting to: %s - Authentication Method: ANONYMOUS',
options['host'])
elif auth_method == 'simple':
auth = {
'authentication': ldap3.SIMPLE,
'user': six.text_type(options['username']),
'password': six.text_type(password)
}
logger.debug(
'Connecting to: %s - Authentication Method: SIMPLE using username: %s',
options['host'], options['username'])
elif auth_method == 'ntlm':
auth = {
'authentication': ldap3.NTLM,
'user': six.text_type(options['username']),
'password': six.text_type(password)
}
logger.debug(
'Connecting to: %s - Authentication Method: NTLM using username: %s',
options['host'], options['username'])
elif auth_method == 'kerberos':
if (platform.system() == 'Windows'):
from .ldap3_extended.Connection import Connection
auth = {
'authentication': ldap3.SASL,
'sasl_mechanism': ldap3.GSSAPI
}
logger.debug(
'Connecting to: %s - Authentication Method: Kerberos',
options['host'])
else:
raise AssertionException(
'Kerberos Authentication Method is not supported on this OS. Windows Only'
)
else:
raise AssertionException(
'LDAP Authentication Method is not supported: %s' %
auth_method)
tls = None
auto_bind = ldap3.AUTO_BIND_NO_TLS
if options['require_tls_cert']:
tls = ldap3.Tls(validate=ssl.CERT_REQUIRED,
version=ssl.PROTOCOL_TLSv1_2)
auto_bind = ldap3.AUTO_BIND_TLS_BEFORE_BIND
try:
server = ldap3.Server(host=options['host'],
allowed_referral_hosts=True,
tls=tls)
connection = Connection(server,
auto_bind=auto_bind,
read_only=True,
**auth)
except Exception as e:
raise AssertionException('LDAP connection failure: %s' % e)
self.connection = connection
logger.debug('Connected as %s', connection.extend.standard.who_am_i())
self.user_by_dn = {}
self.additional_group_filters = None
def load_users_and_groups(self, groups, extended_attributes, all_users):
"""
:type groups: list(str)
:type extended_attributes: list(str)
:type all_users: bool
:rtype (bool, iterable(dict))
"""
options = self.options
user = {}
base_dn = six.text_type(options['base_dn'])
all_users_filter = six.text_type(options['all_users_filter'])
group_member_filter_format = six.text_type(
options['group_member_filter_format'])
grouped_user_records = {}
if options['two_steps_enabled']:
group_member_attribute_name = six.text_type(
options['two_steps_lookup']['group_member_attribute_name'])
# save all the users to memory for faster 2-steps lookup or all_users process
if all_users:
try:
all_users_records = dict(
self.iter_users(base_dn, all_users_filter,
extended_attributes))
except Exception as e:
raise AssertionException(
'Unexpected LDAP failure reading all users: %s' % e)
# for each group that's required, do one search for the users of that group
for group in groups:
group_users = 0
group_dn = self.find_ldap_group_dn(group)
if not group_dn:
self.logger.warning("No group found for: %s", group)
continue
group_member_subfilter = self.format_ldap_query_string(
group_member_filter_format, group_dn=group_dn)
if not group_member_subfilter.startswith('('):
group_member_subfilter = six.text_type(
'(') + group_member_subfilter + six.text_type(')')
user_subfilter = all_users_filter
if not user_subfilter.startswith('('):
user_subfilter = six.text_type(
'(') + user_subfilter + six.text_type(')')
group_user_filter = six.text_type(
'(&'
) + group_member_subfilter + user_subfilter + six.text_type(')')
group_users = 0
try:
if options['two_steps_enabled']:
for user_dn in self.iter_group_member_dns(
group_dn, group_member_attribute_name):
# check to make sure user_dn are within the base_dn scope
if self.is_dn_within_base_dn_scope(base_dn, user_dn):
# replace base_dn with user_dn and filter with all_users_filter to do user lookup based on DN
result = list(
self.iter_users(user_dn, all_users_filter,
extended_attributes))
if result:
# iter_users should only return 1 user when doing two_steps lookup.
if len(result) > 1:
raise AssertionException(
"Unexpected multiple LDAP object found in 'two_steps_lookup' mode for: %s"
% user_dn)
else:
user = result[0][1]
user['groups'].append(group)
group_users += 1
grouped_user_records[user_dn] = user
else:
for user_dn, user in self.iter_users(
base_dn, group_user_filter, extended_attributes):
user['groups'].append(group)
group_users += 1
grouped_user_records[user_dn] = user
except Exception as e:
raise AssertionException(
'Unexpected LDAP failure reading group members: %s' % e)
self.logger.debug('Count of users in group "%s": %d', group,
group_users)
# if all users are requested, do an additional search for all of them
if all_users:
ungrouped_users = 0
grouped_users = 0
try:
for user_dn, user in self.iter_users(base_dn, all_users_filter,
extended_attributes):
if not user['groups']:
ungrouped_users += 1
else:
grouped_users += 1
if groups:
self.logger.debug('Count of users in any groups: %d',
grouped_users)
self.logger.debug('Count of users not in any groups: %d',
ungrouped_users)
except Exception as e:
raise AssertionException(
'Unexpected LDAP failure reading all users: %s' % e)
self.logger.debug('Total users loaded: %d', len(self.user_by_dn))
return six.itervalues(self.user_by_dn)
def __init__(self, name, caller_options):
'''
:type name: str
:type caller_options: dict
'''
caller_config = user_sync.config.DictConfig(
'"%s umapi options"' % name, caller_options)
builder = user_sync.config.OptionsBuilder(caller_config)
builder.set_string_value('logger_name', 'umapi' + name)
builder.set_bool_value('test_mode', False)
options = builder.get_options()
server_config = caller_config.get_dict_config('server', True)
server_builder = user_sync.config.OptionsBuilder(server_config)
server_builder.set_string_value('host', 'usermanagement.adobe.io')
server_builder.set_string_value('endpoint', '/v2/usermanagement')
server_builder.set_string_value('ims_host', 'ims-na1.adobelogin.com')
server_builder.set_string_value('ims_endpoint_jwt',
'/ims/exchange/jwt')
options['server'] = server_options = server_builder.get_options()
enterprise_config = caller_config.get_dict_config('enterprise')
enterprise_builder = user_sync.config.OptionsBuilder(enterprise_config)
enterprise_builder.require_string_value('org_id')
enterprise_builder.require_string_value('api_key')
enterprise_builder.require_string_value('client_secret')
enterprise_builder.require_string_value('tech_acct')
enterprise_builder.require_string_value('priv_key_path')
options[
'enterprise'] = enterprise_options = enterprise_builder.get_options(
)
self.options = options
self.logger = logger = helper.create_logger(options)
caller_config.report_unused_values(logger)
ims_host = server_options['ims_host']
self.org_id = org_id = enterprise_options['org_id']
api_key = enterprise_options['api_key']
private_key_file_path = enterprise_options['priv_key_path']
um_endpoint = "https://" + server_options['host'] + server_options[
'endpoint']
logger.debug(
'Creating connection for org id: "%s" using private key file: "%s"',
org_id, private_key_file_path)
auth_dict = {
"org_id": org_id,
"tech_acct_id": enterprise_options['tech_acct'],
"api_key": api_key,
"client_secret": enterprise_options['client_secret'],
"private_key_file": private_key_file_path
}
try:
self.connection = connection = umapi_client.Connection(
org_id=org_id,
auth_dict=auth_dict,
ims_host=ims_host,
ims_endpoint_jwt=server_options['ims_endpoint_jwt'],
user_management_endpoint=um_endpoint,
test_mode=options['test_mode'],
user_agent="user-sync/" + APP_VERSION,
logger=self.logger,
)
except Exception as e:
raise AssertionException(
"UMAPI connection to org id '%s' failed: %s" % (org_id, e))
logger.debug('API initialized on: %s', um_endpoint)
self.action_manager = ActionManager(connection, org_id, logger)
def get_rule_options(self):
"""
Return a dict representing options for RuleProcessor.
"""
options = user_sync.rules.RuleProcessor.default_options
options.update(self.invocation_options)
# process directory configuration options
directory_config = self.main_config.get_dict_config(
'directory_users', True)
if directory_config:
# account type
new_account_type = directory_config.get_string(
'user_identity_type', True)
new_account_type = user_sync.identity_type.parse_identity_type(
new_account_type)
if new_account_type:
options['new_account_type'] = new_account_type
else:
self.logger.debug("Using default for new_account_type: %s",
options['new_account_type'])
# country code
default_country_code = directory_config.get_string(
'default_country_code', True)
if default_country_code:
options['default_country_code'] = default_country_code
# process exclusion configuration options
adobe_config = self.main_config.get_dict_config('adobe_users', True)
if adobe_config:
exclude_identity_type_names = adobe_config.get_list(
'exclude_identity_types', True)
if exclude_identity_type_names:
exclude_identity_types = []
for name in exclude_identity_type_names:
message_format = 'Illegal value in exclude_identity_types: %s'
identity_type = user_sync.identity_type.parse_identity_type(
name, message_format)
exclude_identity_types.append(identity_type)
options['exclude_identity_types'] = exclude_identity_types
exclude_users_regexps = adobe_config.get_list(
'exclude_users', True)
if exclude_users_regexps:
exclude_users = []
for regexp in exclude_users_regexps:
try:
# add "match begin" and "match end" markers to ensure complete match
# and compile the patterns because we will use them over and over
exclude_users.append(
re.compile(r'\A' + regexp + r'\Z', re.UNICODE))
except re.error as e:
validation_message = (
'Illegal regular expression (%s) in %s: %s' %
(regexp, 'exclude_identity_types', e))
raise AssertionException(validation_message)
options['exclude_users'] = exclude_users
exclude_group_names = adobe_config.get_list(
'exclude_adobe_groups', True) or []
if exclude_group_names:
exclude_groups = []
for name in exclude_group_names:
group = user_sync.rules.AdobeGroup.create(name)
if not group or group.get_umapi_name(
) != user_sync.rules.PRIMARY_UMAPI_NAME:
validation_message = 'Illegal value for %s in config file: %s' % (
'exclude_groups', name)
if not group:
validation_message += ' (Not a legal group name)'
else:
validation_message += ' (Can only exclude groups in primary organization)'
raise AssertionException(validation_message)
exclude_groups.append(group.get_group_name())
options['exclude_groups'] = exclude_groups
# get the limits
limits_config = self.main_config.get_dict_config('limits')
options['max_adobe_only_users'] = limits_config.get_int(
'max_adobe_only_users')
# now get the directory extension, if any
extension_config = self.get_directory_extension_options()
if extension_config:
after_mapping_hook_text = extension_config.get_string(
'after_mapping_hook')
options['after_mapping_hook'] = compile(
after_mapping_hook_text, '<per-user after-mapping-hook>',
'exec')
options['extended_attributes'] = extension_config.get_list(
'extended_attributes')
# declaration of extended adobe groups: this is needed for two reasons:
# 1. it allows validation of group names, and matching them to adobe groups
# 2. it allows removal of adobe groups not assigned by the hook
for extended_adobe_group in extension_config.get_list(
'extended_adobe_groups'):
group = user_sync.rules.AdobeGroup.create(extended_adobe_group)
if group is None:
message = 'Extension contains illegal extended_adobe_group spec: ' + str(
extended_adobe_group)
raise AssertionException(message)
# set the directory group filter from the mapping, if requested.
# This must come late, after any prior adds to the mapping from other parameters.
if options.get('directory_group_mapped'):
options['directory_group_filter'] = set(
six.iterkeys(self.directory_groups))
return options
def begin_work(config_loader):
"""
:type config_loader: user_sync.config.ConfigLoader
"""
directory_groups = config_loader.get_directory_groups()
rule_config = config_loader.get_rule_options()
# make sure that all the adobe groups are from known umapi connector names
primary_umapi_config, secondary_umapi_configs = config_loader.get_umapi_options(
)
referenced_umapi_names = set()
for groups in six.itervalues(directory_groups):
for group in groups:
umapi_name = group.umapi_name
if umapi_name != user_sync.rules.PRIMARY_UMAPI_NAME:
referenced_umapi_names.add(umapi_name)
referenced_umapi_names.difference_update(
six.iterkeys(secondary_umapi_configs))
if len(referenced_umapi_names) > 0:
raise AssertionException(
'Adobe groups reference unknown umapi connectors: %s' %
referenced_umapi_names)
directory_connector = None
directory_connector_options = None
directory_connector_module_name = config_loader.get_directory_connector_module_name(
)
if directory_connector_module_name is not None:
directory_connector_module = __import__(
directory_connector_module_name, fromlist=[''])
directory_connector = user_sync.connector.directory.DirectoryConnector(
directory_connector_module)
directory_connector_options = config_loader.get_directory_connector_options(
directory_connector.name)
config_loader.check_unused_config_keys()
if directory_connector is not None and directory_connector_options is not None:
# specify the default user_identity_type if it's not already specified in the options
if 'user_identity_type' not in directory_connector_options:
directory_connector_options['user_identity_type'] = rule_config[
'new_account_type']
directory_connector.initialize(directory_connector_options)
primary_name = '.primary' if secondary_umapi_configs else ''
umapi_primary_connector = user_sync.connector.umapi.UmapiConnector(
primary_name, primary_umapi_config)
umapi_other_connectors = {}
for secondary_umapi_name, secondary_config in six.iteritems(
secondary_umapi_configs):
umapi_secondary_conector = user_sync.connector.umapi.UmapiConnector(
".secondary.%s" % secondary_umapi_name, secondary_config)
umapi_other_connectors[secondary_umapi_name] = umapi_secondary_conector
umapi_connectors = user_sync.rules.UmapiConnectors(umapi_primary_connector,
umapi_other_connectors)
rule_processor = user_sync.rules.RuleProcessor(rule_config)
if len(directory_groups) == 0 and rule_processor.will_process_groups():
logger.warning(
'No group mapping specified in configuration but --process-groups requested on command line'
)
rule_processor.run(directory_groups, directory_connector, umapi_connectors)
def iter_user_groups(self):
try:
for g in umapi_client.UserGroupsQuery(self.connection):
yield g
except umapi_client.UnavailableError as e:
raise AssertionException("Error contacting UMAPI server: %s" % e)
def create_assertion_error(self, message):
return AssertionException("%s in: %s" %
(message, self.get_full_scope()))
def __init__(self, caller_options):
caller_config = user_sync.config.DictConfig(
'%s configuration' % self.name, caller_options)
builder = user_sync.config.OptionsBuilder(caller_config)
builder.set_string_value('group_filter_format', '{group}')
builder.set_string_value('all_users_filter', 'user.status == "ACTIVE"')
builder.set_string_value('string_encoding', 'utf8')
builder.set_string_value('user_identity_type_format', None)
builder.set_string_value('user_email_format', six.text_type('{email}'))
builder.set_string_value('user_username_format', None)
builder.set_string_value('user_domain_format', None)
builder.set_string_value('user_given_name_format',
six.text_type('{firstName}'))
builder.set_string_value('user_surname_format',
six.text_type('{lastName}'))
builder.set_string_value('user_country_code_format',
six.text_type('{countryCode}'))
builder.set_string_value('user_identity_type', None)
builder.set_string_value('logger_name', self.name)
host = builder.require_string_value('host')
api_token = builder.require_string_value('api_token')
options = builder.get_options()
OKTAValueFormatter.encoding = options['string_encoding']
self.user_identity_type = user_sync.identity_type.parse_identity_type(
options['user_identity_type'])
self.user_identity_type_formatter = OKTAValueFormatter(
options['user_identity_type_format'])
self.user_email_formatter = OKTAValueFormatter(
options['user_email_format'])
self.user_username_formatter = OKTAValueFormatter(
options['user_username_format'])
self.user_domain_formatter = OKTAValueFormatter(
options['user_domain_format'])
self.user_given_name_formatter = OKTAValueFormatter(
options['user_given_name_format'])
self.user_surname_formatter = OKTAValueFormatter(
options['user_surname_format'])
self.user_country_code_formatter = OKTAValueFormatter(
options['user_country_code_format'])
self.users_client = None
self.groups_client = None
self.logger = logger = user_sync.connector.helper.create_logger(
options)
self.user_identity_type = user_sync.identity_type.parse_identity_type(
options['user_identity_type'])
self.options = options
caller_config.report_unused_values(logger)
if not host.startswith('https://'):
if "://" in host:
raise AssertionException("Okta protocol must be https")
host = "https://" + host
self.user_by_uid = {}
logger.debug('%s initialized with options: %s', self.name, options)
logger.info('Connecting to: %s', host)
try:
self.users_client = okta.UsersClient(host, api_token)
self.groups_client = okta.UserGroupsClient(host, api_token)
except OktaError as e:
raise AssertionException("Error connecting to Okta: %s" % e)
logger.info('Connected')
def load_invocation_options(self):
"""Merge the invocation option defaults with overrides from the main config and the command line.
:rtype: dict
"""
options = self.invocation_defaults
# get overrides from the main config
invocation_config = self.main_config.get_dict_config(
'invocation_defaults', True)
if invocation_config:
for k, v in six.iteritems(self.invocation_defaults):
if isinstance(v, bool):
val = invocation_config.get_bool(k, True)
if val is not None:
options[k] = val
elif isinstance(v, list):
val = invocation_config.get_list(k, True)
if val:
options[k] = val
else:
val = invocation_config.get_string(k, True)
if val:
options[k] = val
# now process command line options. the order of these is important,
# because options processed later depend on the values of those processed earlier
# --connector
connector_spec = self.args['connector'] or options['connector']
connector_type = user_sync.helper.normalize_string(connector_spec[0])
if connector_type in ["ldap", "okta"]:
if len(connector_spec) > 1:
raise AssertionException(
'Must not specify a file (%s) with connector type %s' %
(connector_spec[0], connector_type))
options['directory_connector_type'] = connector_type
elif connector_type == "csv":
if len(connector_spec) != 2:
raise AssertionException(
"You must specify a single file with connector type csv")
options['directory_connector_type'] = 'csv'
options['directory_connector_overridden_options'] = {
'file_path': connector_spec[1]
}
else:
raise AssertionException('Unknown connector type: %s' %
connector_type)
# --process-groups
if self.args['process_groups'] is not None:
options['process_groups'] = self.args['process_groups']
# --strategy
if user_sync.helper.normalize_string(self.args['strategy']
or options['strategy']) == 'push':
options['strategy'] = 'push'
# --test-mode
if self.args['test_mode'] is not None:
options['test_mode'] = self.args['test_mode']
# --update-user-info
if self.args['update_user_info'] is not None:
options['update_user_info'] = self.args['update_user_info']
# --adobe-only-user-action
if options['strategy'] == 'push':
if self.args['adobe_only_user_action']:
raise AssertionException(
'You cannot specify --adobe-only-user-action when using "push" strategy'
)
self.logger.info(
"Strategy push: ignoring default adobe-only-user-action")
else:
adobe_action_spec = self.args['adobe_only_user_action'] or options[
'adobe_only_user_action']
adobe_action = user_sync.helper.normalize_string(
adobe_action_spec[0])
if adobe_action == 'preserve':
pass # no option settings needed
elif adobe_action == 'exclude':
options['exclude_strays'] = True
elif adobe_action == 'write-file':
if len(adobe_action_spec) != 2:
raise AssertionException(
'You must specify a single file for adobe-only-user-action "write-file"'
)
options['stray_list_output_path'] = adobe_action_spec[1]
elif adobe_action == 'delete':
options['delete_strays'] = True
elif adobe_action == 'remove':
options['remove_strays'] = True
elif adobe_action == 'remove-adobe-groups':
options['disentitle_strays'] = True
else:
raise AssertionException(
'Unknown option "%s" for adobe-only-user-action' %
adobe_action)
# --users and --adobe-only-user-list conflict with each other, so we need to disambiguate.
# Argument specifications override configuration options, so you must have one or the other
# either as an argument or as a configured default.
if self.args['users'] and self.args['adobe_only_user_list']:
# specifying both --users and --adobe-only-user-list is an error
raise AssertionException(
'You cannot specify both a --users arg and an --adobe-only-user-list arg'
)
elif self.args['users']:
# specifying --users overrides the configuration file default for this option
options['users'] = self.args['users']
users_spec = self.args['users']
stray_list_input_path = None
elif self.args['adobe_only_user_list']:
# specifying --adobe-only-user-list overrides the configuration file default for --users
if options['strategy'] == 'push':
raise AssertionException(
'You cannot specify --adobe-only-user-list when using "push" strategy'
)
users_spec = None
stray_list_input_path = self.args['adobe_only_user_list']
elif options['users'] and options['adobe_only_user_list']:
raise AssertionException(
'You cannot configure both a default "users" option (%s) '
'and a default "adobe-only-user-list" option (%s)' %
(' '.join(options['users']), options['adobe_only_user_list']))
elif options['users']:
users_spec = options['users']
stray_list_input_path = None
elif options['adobe_only_user_list']:
users_spec = None
stray_list_input_path = options['adobe_only_user_list']
else:
raise AssertionException(
'You must specify either a "users" option or an "adobe-only-user-list" option.'
)
# --users
if users_spec:
users_action = user_sync.helper.normalize_string(users_spec[0])
if users_action == 'all':
if options['directory_connector_type'] == 'okta':
raise AssertionException(
'Okta connector module does not support "--users all"')
elif users_action == 'file':
if options['directory_connector_type'] == 'csv':
raise AssertionException(
'You cannot specify file input with both "users" and "connector" options'
)
if len(users_spec) != 2:
raise AssertionException(
'You must specify the file to read when using the users "file" option'
)
options['directory_connector_type'] = 'csv'
options['directory_connector_overridden_options'] = {
'file_path': users_spec[1]
}
elif users_action == 'mapped':
options['directory_group_mapped'] = True
elif users_action == 'group':
if len(users_spec) != 2:
raise AssertionException(
'You must specify the groups to read when using the users "group" option'
)
options['directory_group_filter'] = users_spec[1].split(',')
else:
raise AssertionException('Unknown option "%s" for users' %
users_action)
# --adobe-only-user-list
if options['strategy'] == 'push':
self.logger.info(
"Strategy push: ignoring default adobe-only-user-list")
elif stray_list_input_path:
if options.get('stray_list_output_path'):
raise AssertionException(
'You cannot specify both an adobe-only-user-list (%s) and '
'an adobe-only-user-action of "write-file"')
# don't read the directory when processing from the stray list
self.logger.info(
'adobe-only-user-list specified, so not reading or comparing directory and Adobe users'
)
options['stray_list_input_path'] = stray_list_input_path
# --user-filter
if stray_list_input_path:
if self.args['user_filter']:
raise AssertionException(
'You cannot specify --user-filter when using an adobe-only-user-list'
)
self.logger.info(
"adobe-only-user-list specified, so ignoring default user filter specification"
)
else:
username_filter_pattern = self.args['user_filter'] or options[
'user_filter']
if username_filter_pattern:
try:
compiled_expression = re.compile(
r'\A' + username_filter_pattern + r'\Z', re.IGNORECASE)
except Exception as e:
raise AssertionException(
"Bad regular expression in user filter: %s reason: %s"
% (username_filter_pattern, e))
options['username_filter_regex'] = compiled_expression
return options
def load_from_yaml(cls, filename, path_keys):
'''
loads a yaml file, processes the resulting dict to adapt values for keys
(the path to which is defined in path_keys) to a value that represents
a file reference relative to the source file being loaded, and returns the
processed dict.
:param filename: the file to load yaml from
:param path_keys: a dict whose keys are "path_keys" such as /key1/key2/key3
and whose values are tuples: (must_exist, can_have_subdict, default_val)
which are options on the value of the key whose values
are path expanded: must the path exist, can it be a list of paths
that contains sub-dictionaries whose values are paths, and
does the key have a default value so that must be added to
the dictionary if there is not already a value found.
'''
if filename.startswith('$(') and filename.endswith(')'):
# it's a command line to execute and read standard output
dir_end = filename.index(']')
if filename.startswith('$([') and dir_end > 0:
dir = filename[3:dir_end]
cmd = filename[dir_end + 1:-1]
else:
dir = os.path.abspath(".")
cmd = filename[3:-1]
try:
bytes = subprocess.check_output(cmd, cwd=dir, shell=True)
yml = yaml.load(bytes.decode(cls.config_encoding, 'strict'))
except subprocess.CalledProcessError as e:
raise AssertionException(
"Error executing process '%s' in dir '%s': %s" %
(cmd, dir, e))
except UnicodeDecodeError as e:
raise AssertionException(
'Encoding error in process output: %s' % e)
except yaml.error.MarkedYAMLError as e:
raise AssertionException(
'Error parsing process YAML data: %s' % e)
else:
# it's a pathname to a configuration file to read
cls.filepath = os.path.abspath(filename)
if not os.path.isfile(cls.filepath):
raise AssertionException('No such configuration file: %s' %
(cls.filepath, ))
cls.filename = os.path.split(cls.filepath)[1]
cls.dirpath = os.path.dirname(cls.filepath)
try:
with open(filename, 'rb', 1) as input_file:
bytes = input_file.read()
yml = yaml.load(bytes.decode(cls.config_encoding,
'strict'))
except IOError as e:
# if a file operation error occurred while loading the
# configuration file, swallow up the exception and re-raise it
# as an configuration loader exception.
raise AssertionException(
"Error reading configuration file '%s': %s" %
(cls.filepath, e))
except UnicodeDecodeError as e:
# as above, but in case of encoding errors
raise AssertionException(
"Encoding error in configuration file '%s: %s" %
(cls.filepath, e))
except yaml.error.MarkedYAMLError as e:
# as above, but in case of parse errors
raise AssertionException(
"Error parsing configuration file '%s': %s" %
(cls.filepath, e))
# process the content of the dict
for path_key, options in path_keys.iteritems():
cls.key_path = path_key
keys = path_key.split('/')
cls.process_path_key(yml, keys, 1, *options)
return yml
def create_config_loader_options(args):
'''
This is where all the command-line arguments get set as options in the main config
so that it appears as if they were loaded as part of the main configuration file.
If you add an option that is supposed to be set from the command line here, you
had better make sure you add it to the ones read in get_rule_options.
:param args: the command-line args as parsed
:return: the configured options for the config loader.
'''
config_options = {
'delete_strays': False,
'directory_connector_module_name': None,
'directory_connector_overridden_options': None,
'directory_group_filter': None,
'directory_group_mapped': False,
'disentitle_strays': False,
'exclude_strays': False,
'manage_groups': args.manage_groups,
'remove_strays': False,
'stray_list_input_path': None,
'stray_list_output_path': None,
'test_mode': args.test_mode,
'update_user_info': args.update_user_info,
'username_filter_regex': None,
}
# --users
users_args = args.users
if users_args is not None:
users_action = None if len(
users_args) == 0 else user_sync.helper.normalize_string(
users_args.pop(0))
if (users_action == None or users_action == 'all'):
config_options[
'directory_connector_module_name'] = 'user_sync.connector.directory_ldap'
elif (users_action == 'file'):
if len(users_args) == 0:
raise AssertionException(
'Missing file path for --users %s [file_path]' %
users_action)
config_options[
'directory_connector_module_name'] = 'user_sync.connector.directory_csv'
config_options['directory_connector_overridden_options'] = {
'file_path': users_args.pop(0)
}
elif (users_action == 'mapped'):
config_options[
'directory_connector_module_name'] = 'user_sync.connector.directory_ldap'
config_options['directory_group_mapped'] = True
elif (users_action == 'group'):
if len(users_args) == 0:
raise AssertionException(
'Missing groups for --users %s [groups]' % users_action)
config_options[
'directory_connector_module_name'] = 'user_sync.connector.directory_ldap'
config_options['directory_group_filter'] = users_args.pop(0).split(
',')
else:
raise AssertionException('Unknown argument --users %s' %
users_action)
username_filter_pattern = args.username_filter_pattern
if (username_filter_pattern):
try:
compiled_expression = re.compile(
r'\A' + username_filter_pattern + r'\Z', re.IGNORECASE)
except Exception as e:
raise AssertionException(
"Bad regular expression for --user-filter: %s reason: %s" %
(username_filter_pattern, e.message))
config_options['username_filter_regex'] = compiled_expression
# --adobe-only-user-action
adobe_action_args = args.adobe_only_user_action
if adobe_action_args is not None:
adobe_action = None if not adobe_action_args else user_sync.helper.normalize_string(
adobe_action_args.pop(0))
if (adobe_action == None or adobe_action == 'preserve'):
pass # no option settings needed
elif (adobe_action == 'exclude'):
config_options['exclude_strays'] = True
elif (adobe_action == 'write-file'):
if not adobe_action_args:
raise AssertionException(
'Missing file path for --adobe-only-user-action %s [file_path]'
% adobe_action)
config_options['stray_list_output_path'] = adobe_action_args.pop(0)
elif (adobe_action == 'delete'):
config_options['delete_strays'] = True
elif (adobe_action == 'remove'):
config_options['remove_strays'] = True
elif (adobe_action == 'remove-adobe-groups'):
config_options['disentitle_strays'] = True
else:
raise AssertionException(
'Unknown argument --adobe-only-user-action %s' % adobe_action)
# --adobe-only-user-list
stray_list_input_path = args.stray_list_input_path
if stray_list_input_path:
if users_args is not None:
raise AssertionException(
'You cannot specify both --users and --adobe-only-user-list')
if config_options.get('stray_list_output_path'):
raise AssertionException(
'You cannot specify both --adobe-only-user-list and --output-adobe-users'
)
# don't read the directory when processing from the stray list
config_options['directory_connector_module_name'] = None
logger.info(
'--adobe-only-user-list specified, so not reading or comparing directory and Adobe users'
)
config_options['stray_list_input_path'] = stray_list_input_path
return config_options
def __init__(self, name, caller_options):
'''
:type name: str
:type caller_options: dict
'''
self.name = 'umapi' + name
caller_config = user_sync.config.DictConfig(
self.name + ' configuration', caller_options)
builder = user_sync.config.OptionsBuilder(caller_config)
builder.set_string_value('logger_name', self.name)
builder.set_bool_value('test_mode', False)
options = builder.get_options()
server_config = caller_config.get_dict_config('server', True)
server_builder = user_sync.config.OptionsBuilder(server_config)
server_builder.set_string_value('host', 'usermanagement.adobe.io')
server_builder.set_string_value('endpoint', '/v2/usermanagement')
server_builder.set_string_value('ims_host', 'ims-na1.adobelogin.com')
server_builder.set_string_value('ims_endpoint_jwt',
'/ims/exchange/jwt')
options['server'] = server_options = server_builder.get_options()
enterprise_config = caller_config.get_dict_config('enterprise')
enterprise_builder = user_sync.config.OptionsBuilder(enterprise_config)
enterprise_builder.require_string_value('org_id')
enterprise_builder.require_string_value('tech_acct')
options[
'enterprise'] = enterprise_options = enterprise_builder.get_options(
)
self.options = options
self.logger = logger = helper.create_logger(options)
if server_config: server_config.report_unused_values(logger)
logger.debug('UMAPI initialized with options: %s', options)
# set up the auth dict for umapi-client
ims_host = server_options['ims_host']
self.org_id = org_id = enterprise_options['org_id']
auth_dict = {
'org_id':
org_id,
'tech_acct_id':
enterprise_options['tech_acct'],
'api_key':
enterprise_config.get_credential('api_key', org_id),
'client_secret':
enterprise_config.get_credential('client_secret', org_id),
}
# get the private key
key_path = enterprise_config.get_string('priv_key_path', True)
if key_path:
data_setting = enterprise_config.has_credential('priv_key_data')
if data_setting:
raise AssertionException(
'%s: cannot specify both "priv_key_path" and "%s"' %
(enterprise_config.get_full_scope(), data_setting))
logger.debug('%s: reading private key data from file %s',
self.name, key_path)
auth_dict['private_key_file'] = key_path
else:
auth_dict['private_key_data'] = enterprise_config.get_credential(
'priv_key_data', org_id)
# this check must come after we fetch all the settings
enterprise_config.report_unused_values(logger)
# open the connection
um_endpoint = "https://" + server_options['host'] + server_options[
'endpoint']
logger.debug('%s: creating connection for org %s at endpoint %s',
self.name, org_id, um_endpoint)
try:
self.connection = connection = umapi_client.Connection(
org_id=org_id,
auth_dict=auth_dict,
ims_host=ims_host,
ims_endpoint_jwt=server_options['ims_endpoint_jwt'],
user_management_endpoint=um_endpoint,
test_mode=options['test_mode'],
user_agent="user-sync/" + APP_VERSION,
logger=self.logger,
)
except Exception as e:
raise AssertionException(
"Connection to org %s at endpoint %s failed: %s" %
(org_id, um_endpoint, e))
logger.debug('%s: connection established', self.name)
# wrap the connection in an action manager
self.action_manager = ActionManager(connection, org_id, logger)
def __init__(self, name, caller_options):
"""
:type name: str
:type caller_options: dict
"""
self.name = 'umapi' + name
caller_config = user_sync.config.DictConfig(
self.name + ' configuration', caller_options)
self.trusted = caller_config.get_bool('trusted', True)
if self.trusted is None:
self.trusted = False
builder = user_sync.config.OptionsBuilder(caller_config)
builder.set_string_value('logger_name', self.name)
builder.set_bool_value('test_mode', False)
options = builder.get_options()
server_config = caller_config.get_dict_config('server', True)
server_builder = user_sync.config.OptionsBuilder(server_config)
server_builder.set_string_value('host', 'usermanagement.adobe.io')
server_builder.set_string_value('endpoint', '/v2/usermanagement')
server_builder.set_string_value('ims_host', 'ims-na1.adobelogin.com')
server_builder.set_string_value('ims_endpoint_jwt',
'/ims/exchange/jwt')
server_builder.set_int_value('timeout', 120)
server_builder.set_int_value('retries', 3)
server_builder.set_bool_value('ssl_verify', True)
options['server'] = server_options = server_builder.get_options()
enterprise_config = caller_config.get_dict_config('enterprise')
enterprise_builder = user_sync.config.OptionsBuilder(enterprise_config)
enterprise_builder.require_string_value('org_id')
enterprise_builder.require_string_value('tech_acct')
options[
'enterprise'] = enterprise_options = enterprise_builder.get_options(
)
self.options = options
self.logger = logger = user_sync.connector.helper.create_logger(
options)
if server_config:
server_config.report_unused_values(logger)
logger.debug('UMAPI initialized with options: %s', options)
ims_host = server_options['ims_host']
self.org_id = org_id = enterprise_options['org_id']
auth_dict = make_auth_dict(self.name, enterprise_config, org_id,
enterprise_options['tech_acct'], logger)
# this check must come after we fetch all the settings
enterprise_config.report_unused_values(logger)
# open the connection
um_endpoint = "https://" + server_options['host'] + server_options[
'endpoint']
logger.debug('%s: creating connection for org %s at endpoint %s',
self.name, org_id, um_endpoint)
try:
self.connection = connection = umapi_client.Connection(
org_id=org_id,
auth_dict=auth_dict,
ims_host=ims_host,
ims_endpoint_jwt=server_options['ims_endpoint_jwt'],
user_management_endpoint=um_endpoint,
test_mode=options['test_mode'],
user_agent="user-sync/" + app_version,
logger=self.logger,
timeout_seconds=float(server_options['timeout']),
retry_max_attempts=server_options['retries'] + 1,
ssl_verify=server_options['ssl_verify'])
except Exception as e:
raise AssertionException(
"Connection to org %s at endpoint %s failed: %s" %
(org_id, um_endpoint, e))
logger.debug('%s: connection established', self.name)
# wrap the connection in an action manager
self.action_manager = ActionManager(connection, org_id, logger)
def __init__(self, name, caller_options):
"""
:type name: str
:type caller_options: dict
"""
self.name = 'umapi' + name
caller_config = user_sync.config.DictConfig(
self.name + ' configuration', caller_options)
builder = user_sync.config.OptionsBuilder(caller_config)
builder.set_string_value('logger_name', self.name)
builder.set_bool_value('test_mode', False)
options = builder.get_options()
server_config = caller_config.get_dict_config('server', True)
server_builder = user_sync.config.OptionsBuilder(server_config)
server_builder.set_string_value('host', 'usermanagement.adobe.io')
server_builder.set_string_value('endpoint', '/v2/usermanagement')
server_builder.set_string_value('ims_host', 'ims-na1.adobelogin.com')
server_builder.set_string_value('ims_endpoint_jwt',
'/ims/exchange/jwt')
server_builder.set_int_value('timeout', 120)
server_builder.set_int_value('retries', 3)
options['server'] = server_options = server_builder.get_options()
enterprise_config = caller_config.get_dict_config('enterprise')
enterprise_builder = user_sync.config.OptionsBuilder(enterprise_config)
enterprise_builder.require_string_value('org_id')
enterprise_builder.require_string_value('tech_acct')
options[
'enterprise'] = enterprise_options = enterprise_builder.get_options(
)
self.options = options
self.logger = logger = user_sync.connector.helper.create_logger(
options)
if server_config:
server_config.report_unused_values(logger)
logger.debug('UMAPI initialized with options: %s', options)
ims_host = server_options['ims_host']
self.org_id = org_id = enterprise_options['org_id']
auth_dict = {
'org_id':
org_id,
'tech_acct_id':
enterprise_options['tech_acct'],
'api_key':
enterprise_config.get_credential('api_key', org_id),
'client_secret':
enterprise_config.get_credential('client_secret', org_id),
}
# get the private key
key_path = enterprise_config.get_string('priv_key_path', True)
if key_path:
data_setting = enterprise_config.has_credential('priv_key_data')
if data_setting:
raise AssertionException(
'%s: cannot specify both "priv_key_path" and "%s"' %
(enterprise_config.get_full_scope(), data_setting))
logger.debug('%s: reading private key data from file %s',
self.name, key_path)
try:
with open(key_path, 'r') as f:
key_data = f.read()
except IOError as e:
raise AssertionException(
'%s: cannot read file "%s": %s' %
(enterprise_config.get_full_scope(), key_path, e))
else:
key_data = enterprise_config.get_credential(
'priv_key_data', org_id)
# decrypt the private key, if needed
passphrase = enterprise_config.get_credential('priv_key_pass', org_id,
True)
if passphrase:
try:
key_data = str(
RSA.importKey(
key_data,
passphrase=passphrase).exportKey().decode('ascii'))
except (ValueError, IndexError, TypeError) as e:
raise AssertionException(
'%s: Error decrypting private key, either the password is wrong or: %s'
% (enterprise_config.get_full_scope(), e))
auth_dict['private_key_data'] = key_data
# this check must come after we fetch all the settings
enterprise_config.report_unused_values(logger)
# open the connection
um_endpoint = "https://" + server_options['host'] + server_options[
'endpoint']
logger.debug('%s: creating connection for org %s at endpoint %s',
self.name, org_id, um_endpoint)
try:
self.connection = connection = umapi_client.Connection(
org_id=org_id,
auth_dict=auth_dict,
ims_host=ims_host,
ims_endpoint_jwt=server_options['ims_endpoint_jwt'],
user_management_endpoint=um_endpoint,
test_mode=options['test_mode'],
user_agent="user-sync/" + app_version,
logger=self.logger,
timeout_seconds=float(server_options['timeout']),
retry_max_attempts=server_options['retries'] + 1,
)
except Exception as e:
raise AssertionException(
"Connection to org %s at endpoint %s failed: %s" %
(org_id, um_endpoint, e))
logger.debug('%s: connection established', self.name)
# wrap the connection in an action manager
self.action_manager = ActionManager(connection, org_id, logger)
def __init__(self, caller_options):
caller_config = user_sync.config.DictConfig('<%s configuration>' % self.name, caller_options)
builder = user_sync.config.OptionsBuilder(caller_config)
# Let just ignore this
builder.set_string_value('user_identity_type', None)
builder.set_string_value('identity_type_filter', 'all')
options = builder.get_options()
if not options['identity_type_filter'] == 'all':
try:
options['identity_type_filter'] = parse_identity_type(options['identity_type_filter'])
except Exception as e:
raise AssertionException("Error parsing identity_type_filter option: %s" % e)
self.filter_by_identity_type = options['identity_type_filter']
server_config = caller_config.get_dict_config('server', True)
server_builder = user_sync.config.OptionsBuilder(server_config)
server_builder.set_string_value('host', 'usermanagement.adobe.io')
server_builder.set_string_value('endpoint', '/v2/usermanagement')
server_builder.set_string_value('ims_host', 'ims-na1.adobelogin.com')
server_builder.set_string_value('ims_endpoint_jwt', '/ims/exchange/jwt')
server_builder.set_int_value('timeout', 120)
server_builder.set_int_value('retries', 3)
options['server'] = server_options = server_builder.get_options()
enterprise_config = caller_config.get_dict_config('integration')
integration_builder = user_sync.config.OptionsBuilder(enterprise_config)
integration_builder.require_string_value('org_id')
integration_builder.require_string_value('tech_acct')
options['integration'] = integration_options = integration_builder.get_options()
self.logger = logger = user_sync.connector.helper.create_logger(options)
logger.debug('%s initialized with options: %s', self.name, options)
self.options = options
ims_host = server_options['ims_host']
self.org_id = org_id = integration_options['org_id']
auth_dict = make_auth_dict(self.name, enterprise_config, org_id, integration_options['tech_acct'], logger)
# this check must come after we fetch all the settings
caller_config.report_unused_values(logger)
# open the connection
um_endpoint = "https://" + server_options['host'] + server_options['endpoint']
logger.debug('%s: creating connection for org %s at endpoint %s', self.name, org_id, um_endpoint)
try:
self.connection = umapi_client.Connection(
org_id=org_id,
auth_dict=auth_dict,
ims_host=ims_host,
ims_endpoint_jwt=server_options['ims_endpoint_jwt'],
user_management_endpoint=um_endpoint,
test_mode=False,
user_agent="user-sync/" + app_version,
logger=self.logger,
timeout_seconds=float(server_options['timeout']),
retry_max_attempts=server_options['retries'] + 1,
)
except Exception as e:
raise AssertionException("Connection to org %s at endpoint %s failed: %s" % (org_id, um_endpoint, e))
logger.debug('%s: connection established', self.name)
self.umapi_users = []
self.user_by_usr_key = {}
def begin_work(config_loader):
"""
:type config_loader: user_sync.config.ConfigLoader
"""
directory_groups = config_loader.get_directory_groups()
rule_config = config_loader.get_rule_options()
# make sure that all the adobe groups are from known umapi connector names
primary_umapi_config, secondary_umapi_configs = config_loader.get_umapi_options(
)
referenced_umapi_names = set()
for groups in six.itervalues(directory_groups):
for group in groups:
umapi_name = group.umapi_name
if umapi_name != user_sync.rules.PRIMARY_UMAPI_NAME:
referenced_umapi_names.add(umapi_name)
referenced_umapi_names.difference_update(
six.iterkeys(secondary_umapi_configs))
if len(referenced_umapi_names) > 0:
raise AssertionException(
'Adobe groups reference unknown umapi connectors: %s' %
referenced_umapi_names)
directory_connector = None
directory_connector_options = None
directory_connector_module_name = config_loader.get_directory_connector_module_name(
)
if directory_connector_module_name is not None:
directory_connector_module = __import__(
directory_connector_module_name, fromlist=[''])
directory_connector = user_sync.connector.directory.DirectoryConnector(
directory_connector_module)
directory_connector_options = config_loader.get_directory_connector_options(
directory_connector.name)
post_sync_manager = None
# get post-sync config unconditionally so we don't get an 'unused key' error
post_sync_config = config_loader.get_post_sync_options()
if rule_config['strategy'] == 'sync':
if post_sync_config:
post_sync_manager = PostSyncManager(post_sync_config,
rule_config['test_mode'])
rule_config[
'extended_attributes'] |= post_sync_manager.get_directory_attributes(
)
else:
logger.warn('Post-Sync Connectors only support "sync" strategy')
config_loader.check_unused_config_keys()
if directory_connector is not None and directory_connector_options is not None:
# specify the default user_identity_type if it's not already specified in the options
if 'user_identity_type' not in directory_connector_options:
directory_connector_options['user_identity_type'] = rule_config[
'new_account_type']
directory_connector.initialize(directory_connector_options)
additional_group_filters = None
additional_groups = rule_config.get('additional_groups', None)
if additional_groups and isinstance(additional_groups, list):
additional_group_filters = [r['source'] for r in additional_groups]
if directory_connector is not None:
directory_connector.state.additional_group_filters = additional_group_filters
# show error dynamic mappings enabled but 'dynamic_group_member_attribute' is not defined
if additional_group_filters and directory_connector.state.options[
'dynamic_group_member_attribute'] is None:
raise AssertionException(
"Failed to enable dynamic group mappings. 'dynamic_group_member_attribute' is not defined in config"
)
primary_name = '.primary' if secondary_umapi_configs else ''
umapi_primary_connector = user_sync.connector.umapi.UmapiConnector(
primary_name, primary_umapi_config)
umapi_other_connectors = {}
for secondary_umapi_name, secondary_config in six.iteritems(
secondary_umapi_configs):
umapi_secondary_conector = user_sync.connector.umapi.UmapiConnector(
".secondary.%s" % secondary_umapi_name, secondary_config)
umapi_other_connectors[secondary_umapi_name] = umapi_secondary_conector
umapi_connectors = user_sync.rules.UmapiConnectors(umapi_primary_connector,
umapi_other_connectors)
rule_processor = user_sync.rules.RuleProcessor(rule_config)
if len(directory_groups) == 0 and rule_processor.will_process_groups():
logger.warning(
'No group mapping specified in configuration but --process-groups requested on command line'
)
rule_processor.run(directory_groups, directory_connector, umapi_connectors)
# Post sync section
if post_sync_manager:
post_sync_manager.run(rule_processor.post_sync_data)
def __init__(self, caller_options):
caller_config = user_sync.config.DictConfig(
'%s configuration' % self.name, caller_options)
builder = user_sync.config.OptionsBuilder(caller_config)
builder.set_string_value(
'group_filter_format',
six.text_type(
'(&(|(objectCategory=group)(objectClass=groupOfNames)(objectClass=posixGroup))(cn={group}))'
))
builder.set_string_value(
'all_users_filter',
six.text_type(
'(&(objectClass=user)(objectCategory=person)(!(userAccountControl:1.2.840.113556.1.4.803:=2)))'
))
builder.set_string_value('group_member_filter_format', None)
builder.set_bool_value('require_tls_cert', False)
builder.set_dict_value('two_steps_lookup', None)
builder.set_string_value('string_encoding', 'utf8')
builder.set_string_value('user_identity_type_format', None)
builder.set_string_value('user_email_format', six.text_type('{mail}'))
builder.set_string_value('user_username_format', None)
builder.set_string_value('user_domain_format', None)
builder.set_string_value('user_given_name_format',
six.text_type('{givenName}'))
builder.set_string_value('user_surname_format', six.text_type('{sn}'))
builder.set_string_value('user_country_code_format',
six.text_type('{c}'))
builder.set_string_value('user_identity_type', None)
builder.set_int_value('search_page_size', 200)
builder.set_string_value('logger_name', LDAPDirectoryConnector.name)
host = builder.require_string_value('host')
username = builder.require_string_value('username')
builder.require_string_value('base_dn')
options = builder.get_options()
if options['two_steps_lookup'] is not None:
ts_config = caller_config.get_dict_config('two_steps_lookup', True)
ts_builder = user_sync.config.OptionsBuilder(ts_config)
ts_builder.require_string_value('group_member_attribute_name')
ts_builder.set_bool_value('nested_group', False)
self.two_steps_lookup = True
options['two_steps_lookup'] = ts_builder.get_options()
if options['group_member_filter_format']:
raise AssertionException(
"Cannot define both 'group_member_attribute_name' and 'group_member_filter_format' in config"
)
else:
self.two_steps_lookup = False
if not options['group_member_filter_format']:
options['group_member_filter_format'] = six.text_type(
'(memberOf={group_dn})')
self.options = options
self.logger = logger = user_sync.connector.helper.create_logger(
options)
logger.debug('%s initialized with options: %s', self.name, options)
LDAPValueFormatter.encoding = options['string_encoding']
self.user_identity_type = user_sync.identity_type.parse_identity_type(
options['user_identity_type'])
self.user_identity_type_formatter = LDAPValueFormatter(
options['user_identity_type_format'])
self.user_email_formatter = LDAPValueFormatter(
options['user_email_format'])
self.user_username_formatter = LDAPValueFormatter(
options['user_username_format'])
self.user_domain_formatter = LDAPValueFormatter(
options['user_domain_format'])
self.user_given_name_formatter = LDAPValueFormatter(
options['user_given_name_format'])
self.user_surname_formatter = LDAPValueFormatter(
options['user_surname_format'])
self.user_country_code_formatter = LDAPValueFormatter(
options['user_country_code_format'])
password = caller_config.get_credential('password',
options['username'])
# this check must come after we get the password value
caller_config.report_unused_values(logger)
logger.debug('Connecting to: %s using username: %s', host, username)
if not options['require_tls_cert']:
ldap.set_option(ldap.OPT_X_TLS_REQUIRE_CERT, ldap.OPT_X_TLS_NEVER)
try:
# Be careful in Py2!! We are setting bytes_mode = False, so we must give all attribute names
# and other protocol-defined strings (such as username) as Unicode. But the PyYAML parser
# will always return ascii strings as str type (rather than Unicode). So we must be careful
# to upconvert all parameter strings to unicode when passing them in.
connection = ldap.initialize(host, bytes_mode=False)
connection.protocol_version = ldap.VERSION3
connection.set_option(ldap.OPT_REFERRALS, 0)
connection.simple_bind_s(six.text_type(username),
six.text_type(password))
except Exception as e:
raise AssertionException('LDAP connection failure: %s' % e)
self.connection = connection
logger.debug('Connected')
self.user_by_dn = {}
self.additional_group_filters = None
